Laboratoire ICAR » Plateforme TXM

               lib/,\

               lib/dom4j-1.6.1.jar,\

               lib/log4j-1.2.12.jar,\

               lib/TigerSearch.jar

	<classpathentry exported="true" kind="con" path="GROOVY_SUPPORT"/>

	<classpathentry exported="true" kind="con" path="GROOVY_DSL_SUPPORT"/>

 org.txm.export.ts,

Bundle-ClassPath: lib/dom4j-1.6.1.jar,

 lib/log4j-1.2.12.jar,

 lib/TigerSearch.jar,

 .

		<nature>org.eclipse.jdt.groovy.core.groovyNature</nature>

package org.txm.test;

import java.io.File;

import java.util.Arrays;

import org.txm.searchengine.ts.*;

import org.txm.utils.ExecTimer;

public class DrawTSSVG {

	String userhome = System.getProperty("user.home");

	File configdir = new File(userhome,"TXM/corpora/graal/tiger/tigersearch.logprop");

	File registrydir = new File(userhome, "TXM/corpora/graal/tiger");

	File svgfile = new File(registrydir, "result.svg");

	String id = "GRAAL";

	String query = "#n:[cat = \"Obj\"] >* #m & arity(#n, 2, 10)";

	//String query = "[]";

	public void test() throws Exception {

		TSCorpusManager manager = new TSCorpusManager(registrydir, configdir);

		if (manager.isInitialized()) {

			ExecTimer.start();

			TSCorpus corpus = manager.getCorpus(id);

			System.out.println("T features: "+corpus.getTFeatures());

			System.out.println("NT features: "+corpus.getNTFeatures());

			TSResult result = corpus.query(query);

			result.setDisplayProperties(Arrays.asList("word", "pos", "form"), "type");

			TSMatch first = result.getFirst();

			System.out.println("First: nb of sub graph: "+first.getNumberOfSubGraph());

			first.firstSubGraph();

			first.toSVGFile(new File(registrydir,"result_1_1.svg"));

			System.out.println(first.toHTML() + "<br>");

			first.nextSubGraph();

			first.toSVGFile(new File(registrydir,"result_1_2.svg"));

			System.out.println(first.toHTML() + "<br>");

			first.nextSubGraph();

			first.toSVGFile(new File(registrydir,"result_1_3.svg"));

			System.out.println(first.toHTML() + "<br>");

//			TSMatch next = result.getNext();

//			System.out.println( "Next: nb of sub graph: "+next.getNumberOfSubGraph());

//			next.firstSubGraph();

//			next.toSVGFile(new File(userhome,"TXM/Tiger/result_2_1.svg"));

//			System.out.println( next.toHTML() + "<br>");

//			next.nextSubGraph();

//			next.toSVGFile(new File(userhome,"TXM/Tiger/result_2_2.svg"));

//			System.out.println( next.toHTML() + "<br>");

//			next.nextSubGraph();

//			next.toSVGFile(new File(userhome,"TXM/Tiger/result_2_3.svg"));

//			System.out.println( next.toHTML() + "<br>");

//			next.previousSubGraph();

//			next.toSVGFile(new File(userhome,"TXM/Tiger/result_2_2bis.svg"));

//			System.out.println( next.toHTML() + "<br>");

			//	System.out.println( "make some room :)"

			//	for(File f : new File("/home/mdecorde/Bureau/tigerexports/").listFiles())

			//		f.delete()

			//

			//	System.out.println( "SAVE XML"

			//	result.toXml(new File("/home/mdecorde/Bureau/tigerexports/result.xml"), false, true)

			//

			//	System.out.println( "SAVE AS SVG"

			//	for (int i = 0 ; i < result.getNumberOfMatch() && i < 10; i++) {

			//		result.getMatch(i).toSVGFile(new File("/home/mdecorde/Bureau/tigerexports/match_"+i+".svg"))

			//	}

			//

				System.out.println("SIMPLE NO PNC");

				System.out.println(result.toConcordance(new File(registrydir, "export1.csv"), "concordance_simple", 30, Arrays.asList("cat"), Arrays.asList("pos"), false));

				System.out.println("MOT-PIVOT NO PNC");

				System.out.println(result.toConcordance(new File(registrydir, "export2.csv"), "concordance_mot-pivot", 30, Arrays.asList("cat"), Arrays.asList("pos"), false));

				System.out.println("BLOCKS NO PNC");

				System.out.println(result.toConcordance(new File(registrydir, "export3.csv"), "concordance_blocks", 30, Arrays.asList("cat"), Arrays.asList("pos"), false));

			//	System.out.println( "SIMPLE + PNC"

			//	System.out.println( result.toConcordance(new File("/home/mdecorde/Bureau/tigerexports/export12.csv"), "concordance_simple", 30, ["cat"], ["pos"], true);

			//	System.out.println( "MOT-PIVOT + PNC"

			//	System.out.println( result.toConcordance(new File("/home/mdecorde/Bureau/tigerexports/export22.csv"), "concordance_mot-pivot", 30, ["cat"], ["pos"], true);

			//	System.out.println( "BLOCKS + PNC"

			//	System.out.println( result.toConcordance(new File("/home/mdecorde/Bureau/tigerexports/export32.csv"), "concordance_blocks", 30, ["cat"], ["pos"], true);

			System.out.println(ExecTimer.stop());

		}

	}

	public static void main(String[] args) {

		DrawTSSVG d = new DrawTSSVG();

		try {

			d.test();

		} catch (Exception e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

	}

}

import org.txm.test.DrawTSSVG;

//		IStructuredSelection selection = (IStructuredSelection) HandlerUtil.getCurrentSelection(event);

//

//		Object s = selection.getFirstElement();

//		if (s instanceof Corpus) {

//			Corpus corpus = (Corpus)s;

//			openEditor(corpus);

//		} else if (s instanceof Partition) {

//			Partition partition = (Partition)s;

//			openEditor(partition);

//		}

		DrawTSSVG d = new DrawTSSVG();

		try {

			d.test();

		} catch (Exception e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

import org.txm.rcp.editors.TXMResultEditorInput;

		Composite queryPanel = this.getCommandParametersGroup();

		GridLayout qlayout = new GridLayout(11, false);

		queryAreaLayoutData.horizontalSpan = 11;

		queryAreaLayoutData.heightHint = 80;

		queryAreaLayoutData.minimumHeight = 80;

		new Label(queryPanel, SWT.NONE).setText("Sent ");

		sentSpinner = new Spinner(queryPanel, SWT.BORDER);

		sentCounterLabel = new Label(queryPanel, SWT.NONE);

		new Label(queryPanel, SWT.NONE).setText("Sub ");

		subSpinner = new Spinner(queryPanel, SWT.BORDER);

		subCounterLabel = new Label(queryPanel, SWT.NONE);

		new Label(queryPanel, SWT.NONE).setText("T ");

		TCombo = new Combo(queryPanel, SWT.READ_ONLY);

		new Label(queryPanel, SWT.NONE).setText("NT ");

		NTCombo = new Combo(queryPanel, SWT.READ_ONLY);

		okButton = new Button(queryPanel, SWT.PUSH);

		Composite mainPanel = this.getResultArea();

		mainPanel.setLayout(new GridLayout(1, false));

		svgPanel = new SVGComposite(mainPanel, SWT.EMBEDDED | SWT.NO_BACKGROUND);

		svgPanel.setLayoutData(new GridData(SWT.FILL, SWT.FILL, true, true));

		if (queryArea != null)

			queryArea.forceFocus();

		TXMResultEditorInput<TIGERSearch> ii = (TXMResultEditorInput<TIGERSearch>) input;

		ts = ii.getResult();

		corpus = ts.getCorpus();

// Script to restore punctuation etc. to results of TS query.

// Inputs:

// --- Tiger-XML Document node

// --- Java array:

// --- --- String [index][type] where:

// --- --- --- type == 0 gives the xml:id

// --- --- --- type == 1 gives the word form

// Process:

// --- Injects punctuation.

// Returns:

// --- Tiger-XML Document node.

package org.txm.export.ts;

import javax.xml.parsers.DocumentBuilderFactory

import org.txm.searchengine.cqp.CqpDataProxy;

import org.txm.searchengine.cqp.corpus.Corpus

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.QueryResult;

import org.txm.searchengine.cqp.corpus.query.Query

public class PunctInject

{

	public process (def tigerXml, def txmIdWordTable) {

		def allTs = toList(tigerXml.getElementsByTagName('t'))

		def tAttrs = getTAttrNames(allTs[0])

		def tNodeIdPrefix = allTs[0].getAttribute('id').tokenize('#').first()

		def addedWordIds = []

		int tIx = 0

		while (txmIdWordTable) {

			def word = txmIdWordTable.remove(0)

			if (tIx == allTs.size()) {

				// End of TS file, but still words left in the BFM file.

				addTNode(word, allTs.last(), tAttrs, tNodeIdPrefix, 'append')

			}

			else {

				def tNode = allTs[tIx]

				def tId = getTNodeId(tNode)

				if (tId == word[0] && tNode.getAttribute('word') == word[1]) {

					// alles gut

					tIx += 1

				}

				else if (tId == word[0]) {

					println("Mismatched Ids! ($tId)")

					tIx += 1

				}

				else if (['#', '*'].contains(tNode.getAttribute('word')) ) {

					// SRCMF duplicata; try comparing word against the next tNode next time

					// around.

					txmIdWordTable.add(0, word)

					tIx += 1

				}

				// Check that the SRCMF corpus doesn't have a bug in it...

				else if ( !(word[1] =~ /[\,\.\?\!\:\;\(\)\[\]\{\}]/)

				&& (allTs[0..tIx - 1].find{

					it.getAttribute('id') == "$tNodeIdPrefix#$tId"

				})) {

					println "Warning: word ${tNode.getAttribute('word')}, id $tId appears twice in corpus!"

					txmIdWordTable.add(0, word)

					tIx += 1

				}

				// Check that there's not an extra word in the SRCMF corpus (rare, usu. a tokenisation change)

				else if ( !(word[1] =~ /[\,\.\?\!\:\;\(\)\[\]\{\}]/)

				&& (allTs[tIx..-1].find{

					it.getAttribute('id') == "$tNodeIdPrefix#${word[0]}"

				})) {

					println "Warning: word ${tNode.getAttribute('word')}, id $tId does not appear in BFM!"

					txmIdWordTable.add(0, word)

					tIx += 1

				}

				else if (addedWordIds.contains(tId)) {

					println "Warning: word ${tNode.getAttribute('word')}, id ${tId} out of sequence in SRCMF corpus!"

					txmIdWordTable.add(0, word)

					tIx += 1

				}

				else {

					// Insert word.  In the first instance, it will have the same parent as

					// the tNode before which it's being inserted.

					addTNode(word, allTs[tIx], tAttrs, tNodeIdPrefix, 'before')

					addedWordIds.add(word[0])

				}

			}

		}

		// Second phase: move punctuation into previous sentence,

		// dependent on sequence.

		def allTerminalses = toList(tigerXml.getElementsByTagName('terminals'))

		for (def i = 1 ; i < allTerminalses.size() ; i++) {

			def ts = toList(allTerminalses[i].getElementsByTagName('t'))

			def startPunc = true

			def puncStack = []

			while (ts && startPunc) {

				if ((ts[0].getAttribute('word') =~ /[A-zÀ-ÿ0-9]/).size() == 0) {

					puncStack.add(ts.remove(0))

				}

				else {

					startPunc = false

				}

			}

			// Now, treat the punctuation stack at the beginning of the sentence

			if ( puncStack ) {

				int moveLeft = 0

				// First, identify LAST instance of sentence-final punctuation.

				def puncString = puncStack.collect{ it.getAttribute('word')[0] }.join('')

				def matches = puncString =~ /[\.\,\;\:\!\?\)\]\}»”’]/

				if (matches.size() > 0) {

					moveLeft = puncString.lastIndexOf(matches[-1]) + 1

				}

				// Second, split pairs of straight quotes

				matches = puncString =~ /(""|'')/ //"

				if (matches.size() > 0) {

					moveLeft = [moveLeft, puncString.lastIndexOf(matches[-1][0]) + 1].max()

				}

				// Now, move moveLeft punctuation nodes to the end of the prev. sentence

				ts = toList(allTerminalses[i].getElementsByTagName('t'))

				for (def j = 0 ; j < moveLeft ; j++ ) {

					allTerminalses[i - 1].appendChild(ts[j])

				}

			}

		}

		return tigerXml

	}

	private addTNode(word, tNode, tAttrs, tNodeIdPrefix, where) {

		def newTNode = tNode.getOwnerDocument().createElement('t')

		for (def anAttr : tAttrs) {

			if (anAttr == 'id') {

				newTNode.setAttribute('id', "${tNodeIdPrefix}#${word[0]}")

			}

			else if (anAttr == 'word') {

				newTNode.setAttribute('word', word[1])

			}

			else {

				newTNode.setAttribute(anAttr, '--')

			}

		}

		if (where == 'before') {

			tNode.getParentNode().insertBefore(newTNode, tNode)

		}

		else if (where == 'append') {

			tNode.getParentNode().appendChild(newTNode)

		}

		else {

			throw new IllegalArgumentException('Bad before value')

		}

	}

	public getTAttrNames(tNode) {

		def nodeMap = tNode.attributes

		def nameList = []

		for ( def i = 0 ; i < nodeMap.getLength() ; i++ ) {

			nameList.add( nodeMap.item(i).nodeName )

		}

		return nameList

	}

	public getTNodeId(tNode) {

		return tNode.getAttribute('id').tokenize('#').last()

	}

	public def toList(def iterable) {

		return iterable.findAll {true};

	}

	public static def getWords(String corpusname, String query)

	{

		CorpusManager cm = CorpusManager.getCorpusManager();

		Corpus corpus = cm.getCorpus(corpusname);

		def word_property = corpus.getProperty("word")

		def id_property = corpus.getProperty("id")

		def wordCache = cm.getCorpusProxies(corpus).get(word_property);

		def idCache = cm.getCorpusProxies(corpus).get(id_property);

		def positions = new int[corpus.getSize()];

		for(int i = 0 ; i< corpus.getSize() ; i++)

			positions[i] = i;

		def word_values = wordCache.getData(positions)

		def id_values = idCache.getData(positions)

		ArrayList<String[]> words = new ArrayList<String[]>(corpus.getSize());

		for(int p : positions)

		{

			if(id_values[p].startsWith("w"))

			{

				words.add(new String[2])

				words[p][0] = id_values[p]

				words[p][1] = word_values[p]

			}

		}

		return words;

	}

	public static void main(String[] args)

	{

		def words = [

			["w203_1", "Dominedeu"],

			["w203_2", "devemps"],

			["w203_3", "lauder"],

			["w203_4", "et"],

			["w203_5", "a"],

			["w203_6", "sus"],

			["w203_7", "sancz"],

			["w203_8", "honor"],

			["w203_9", "porter"],

			["w203_10", "»"],

			["w203_10.2", ")"],

			["w203_10.3", '.'],

			["w203_10.5", '"'],

			["w203_10.7", '"'],

			["w203_11", "in"],

			["w203_12", "su'"],

			["w203_13", "amor"],

			["w203_14", "cantomps"],

			["w203_15", "del"],

			["w203_16", "·sanz"],

			["w203_17", "quae"],

			["w203_18", "por"],

			["w203_19", "lui"],

			["w203_20", "augrent"],

			["w203_21", "granz"],

			["w203_22", "aanz"],

			["w203_23", "."],

		];

		File tigerXml = new File(args[0]);

		def factory = DocumentBuilderFactory.newInstance()

		factory.setXIncludeAware(true)

		def builder = factory.newDocumentBuilder()

		def THEDOM = builder.parse(tigerXml).documentElement

		println THEDOM.getClass()

		def NEWDOM = new PunctInject().process(THEDOM, words);

		println NEWDOM

		// File outfile = new File("outfile.xml")

		// println outfile

		//outfile.withWriter("iso-8859-1"){writer ->

		//writer.write(NEWDOM.toString())

		// }

	}

}

#! /usr/bin/groovy

package org.txm.export.ts;

import org.xml.sax.XMLReader;

import org.xml.sax.helpers.XMLReaderFactory;

import groovy.util.slurpersupport.NodeChild

/*

 * The script takes the <matches/> elements from file "Tiger_match.xml"

 * and inserts them at the end of the <s/> element bearing the same ID.

 * Inputs: three file names 

 * - TsInputName --- the TS file WITHOUT matches

 * - MatchInputName --- the TS file containing only matches.

 * - OutputFileName ---required output file.

 * To pass these arguments from within an application, call script() directly.

 */

// Filename variables

// def TsInputName = '/home/tomr/Documents/Work/lyon12/srcmf/groovy/MatchInject/ts_input.xml'

// def MatchInputName = '/home/tomr/Documents/Work/lyon12/srcmf/groovy/MatchInject/match_input.xml'

// def OutputFileName = '/home/tomr/Documents/Work/lyon12/srcmf/groovy/MatchInject/test.xml'

// Main code: checks for correct number of arguments if run from cmd line.

if (args && args.size() == 3) {

    script(args[0], args[1], args[2])

} else {

    println '''Incorrect number of arguments: three strings required.

USAGE:

******

groovy MatchInject.groovy TsInput.xml MatchInput.xml OutputFile.xml'''}

def script(String tsInputName, String matchInputName, String outputFileName) {

	script(new File(tsInputName), new File(matchInputName, new File(outputFileName)));

}

// The script.

def script(File tsInputFile, File matchInputFile, File outputFile) {

	//println "loading TsInput..."

    def TsInput = new XmlSlurper().parse(tsInputFile)

	//println "loading matchInputFile..."

	System.setProperty("org.xml.sax.driver", "com.sun.org.apache.xerces.internal.parsers.SAXParser");

	def xmlReader = XMLReaderFactory.createXMLReader();

	xmlReader.setFeature('http://xml.org/sax/features/namespaces', false)

    XmlSlurper mslurper = new XmlSlurper(xmlReader);

	def MatchInput = mslurper.parse(matchInputFile)

	//println "building OutputFile... size="+matchInputFile.length()

	def inputSentences = TsInput.'**'.findAll {it.name() == 's'};

	def matcheSentences = MatchInput.'**'.findAll { it.name() == 's' }

//	println "MATCHES"

//	for(NodeChild match : matcheSentences) {

//		//println match.getClass()

//		match.namespacePrefix = ""

//		match.namespaceMap = [:]

//		//println match

//	}

	int count = 0;

	//println("nb of input sentences: "+inputSentences.size());

    def markup = {

		mkp.xmlDeclaration()

		//mkp.declareNamespace("svg":"xmlns=\"http://www.w3.org/2000/svg\"")

        corpus(id:'TSOut') {

            body {

                inputSentences.each { sPath ->

					//if (count++%200 == 0) println((100*count/inputSentences.size()))

                    s(id:"${sPath.'@id'}") { 

                        mkp.yield(sPath.graph)

                        def sMatches = matcheSentences.find {

                            it.'@id'.toString() == sPath.'@id'.toString()

                        }

                        if (sMatches) {

                            mkp.yield(sMatches.matches)

                        }

						//matcheSentences.removeAll(sMatches)

                    }

                }

            }

        }

    }

    def processor = new groovy.xml.StreamingMarkupBuilder().bind(markup)

    outputFile.withWriter { it << groovy.xml.XmlUtil.serialize(processor)}

}

<html>

<body>

<p>Contains scripts to build the TIGERSearch concordances</p>

</body>

</html>

package org.txm.export.ts;

import javax.xml.parsers.DocumentBuilderFactory

class ConcordanceBlocks {

	def codec = 'UTF-8'

	int cx = 30;

	def ntTypes = [];

	def tTypes = [];

	File xmlfile, outfile;

	public boolean process(File xmlfile, File outfile, int cx, def ntTypes, def tTypes) {

		this.cx = cx;

		this.xmlfile = xmlfile;

		this.outfile = outfile;

		this.ntTypes = ntTypes;

		this.tTypes = tTypes;

		Writer writer = outfile.newPrintWriter(codec)

		println 'Reading XML File'

		def factory = DocumentBuilderFactory.newInstance()

		factory.setXIncludeAware(true)

		def builder = factory.newDocumentBuilder()

		def records = builder.parse(xmlfile).documentElement

		println 'done.'

		println 'done.  Pre-treating file to combine results with the same pivot...'

		records = matchCombine(records)

		println 'done.'

		println 'Calculating max. number of blocks...'

		def nBlock = 0

		def allMatches = toList(records.getElementsByTagName('match'))

		for (def match : allMatches) {

			nBlock = [

				nBlock,

				toList(match.getElementsByTagName('variable')).findAll{

					it.getAttribute('name').startsWith('#block')

				}.size()

			].max()

		}

		println "done ( $nBlock )"

		// Table headers

		println 'Writing concordance...'

		def header = ['sId', 'LeftCxOutsideSnt', 'LeftCxInsideSnt']

		for (int i = nBlock ; i > 0 ; i--) {

			header.add("${i}BlockBeforePivot")

			for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {

				header.add("${i}BlockBeforePivotType${j+1}")

			}

		}

		header.add('Pivot')

		for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {

			header.add("PivotType${j+1}")

		}

		for (int i = 1 ; i <= nBlock ; i++) {

			header.add("${i}BlockAfterPivot")

			for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {

				header.add("${i}BlockAfterPivotType${j+1}")

			}

		}

		header.addAll(['RightCxInsideSnt', 'RightCxOutsideSnt', 'Warnings'])

		writer.write(header.join("\t")+"\n");

		def allTerminals = toList(records.getElementsByTagName('t'))

		int tenPercentile = 0

		int i = 0;

		for (def aMatchNode : allMatches) {

			if ((int)(((float)++i / allMatches.size()) * 10) > tenPercentile)

				println ""+(++tenPercentile * 10)+ ' percent complete...'

			def (rowDict, inSntCxLengthLeft, inSntCxLengthRight) = match2CSVrow(aMatchNode, header);

			// Add out-of-sentence context

			def sNode = aMatchNode.parentNode.parentNode; // sentence

			def terminals = toList(sNode.getElementsByTagName('t'));

			def firstTInS = terminals[0] // get first node of the sentence

			def lastTInS = terminals[-1] // get last node of the sentence

			def firstTInSIx = allTerminals.indexOf(firstTInS) // get its position in the text

			def lastTInSIx = allTerminals.indexOf(lastTInS) // get its position in the text

			// Left context

			def lexs = []

			int start = Math.max(firstTInSIx - cx + inSntCxLengthLeft, 0);

			int end = firstTInSIx;

			if(start < end)

			for (def tNode : allTerminals.subList(start, end)){

				lexs.add(tNode.getAttribute('word'))

				if (toList(tNode.parentNode.getElementsByTagName('t'))[-1] == tNode)

					lexs.add('/')

			}

			rowDict['LeftCxOutsideSnt'] = lexs.join(" ")

			// Right context

			lexs = []

			start = lastTInSIx + 1;

			end = Math.min(allTerminals.size(), lastTInSIx + cx - inSntCxLengthRight)

			if(start < end) {

				for (def tNode : allTerminals.subList(start, end)){

					lexs.add(tNode.getAttribute('word'))

					if (toList(tNode.parentNode.getElementsByTagName('t'))[-1] == tNode)

						lexs.add('/')

				}}

			rowDict["RightCxOutsideSnt"] = lexs.join(" ")

			// Right rowDict to CSV

			//			CSVWriter.writerow(dict(zip([k for k in rowDict.iterkeys()], \

			//        [ v.encode('utf-8') for v in rowDict.itervalues() ] )))

			String line = "";

			for (int ii = 0 ; ii < header.size() ; ii++){

				String h = header.get(ii)

				String val = rowDict.getAt(h);

				if (val == null)

					line += "--"

				else

					line += val

				if (ii < header.size() -1)

					line +="\t"

			}

			writer.write(line+"\n")

			writer.flush()

		}

		writer.close();

		return true;

	}

	private def match2CSVrow(def aMatchNode, def header) {

		// Step 1: Build a LIST of DICTIONARIES to describe the variables:

		// dict(name='varname_minus_the_hash'

		//      parent='nt_nodes'

		//      terminals='t_nodes in a list')

		def sNode = aMatchNode.parentNode.parentNode // get the sentence node

		def tNodesInSentence = toList(sNode.getElementsByTagName('t')) // get all terminal nodes of the sentence

		def varDetails = []

		for ( def aVariable : aMatchNode.getElementsByTagName('variable')){

			if ( aVariable.getAttribute('name') == '#pivot' ||

			aVariable.getAttribute('name').startsWith('#block')) {

				def varParent = idKey(sNode, aVariable.getAttribute('idref'))

				varDetails.add([

							'name' : aVariable.getAttribute('name').substring(1),

							'parent': varParent,

							'terminals': toList(getTNodes(varParent)),

							'lexform': writeLexForm(varParent),

						])

				// Add types

				for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {

					def a = ''

					if (j < ntTypes.size() && varParent.getAttribute(ntTypes[0])

					) {

						a = varParent.getAttribute(ntTypes[j])

					} else if (j < tTypes.size() && varParent.getAttribute(tTypes[0])

					) {

						a = varParent.getAttribute(tTypes[j])

					}

					varDetails[-1]["type${j+1}"] = a

				}

				// write min idx in sentence of terminal nodes used

				varDetails[-1]['terminalsIx'] = varDetails[-1]['terminals'].collect{tNodesInSentence.indexOf(it)}

			}

		}

		// Sort varDetails by the start ID of the word

		varDetails =  varDetails.sort{it['terminalsIx'].min()}

		def pivotBlockPosition = 0;

		pivotBlockPosition = varDetails.findIndexOf{it['name'] == 'pivot'}

		// Write the table

		def rowDict = [:]

		rowDict['sId'] = sNode.getAttribute('id')

		// write the pivot

		rowDict['Pivot'] = varDetails[pivotBlockPosition]['lexform']

		for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {

			rowDict["PivotType${j+1}"] = varDetails[pivotBlockPosition]["type${j+1}"]

		}

		def startNextId = (varDetails[pivotBlockPosition]['terminalsIx']).min()

		def endPrevId = (varDetails[pivotBlockPosition]['terminalsIx']).max()

		// Write the Pre-pivot blocks

		def i = 0

		//println "rowDict: $rowDict"

		if ( pivotBlockPosition > 0)	{

			for ( int j = pivotBlockPosition -1 ; j >= 0 ; j--) {

				i++;

				def block = varDetails.get(j)

				//println "process block: $block"

				rowDict["${i}BlockBeforePivot"] = block['lexform']

				for ( int k = 0 ; k < [ntTypes.size(), tTypes.size()].max() ; k++ ) {

					rowDict["${i}BlockBeforePivotType${k+1}"] = block["type${k+1}"]

				}

				// Add any intervening words to the right edge.

				def rightEdge = (block['terminalsIx'].findAll{ it < startNextId }).max();

				//println ""+block['terminalsIx']+"   rightEdge: $rightEdge"

				for (int ix = rightEdge + 1 ; ix < startNextId ; ix++) // add word to reach the pivot

				{

					rowDict["${i}"+"BlockBeforePivot"] += ' {' + tNodesInSentence[ix].getAttribute('word') + '}';

				}

				startNextId = (block['terminalsIx']).min()

			}

		}

		// Write the in-sentence left context

		def lexs = tNodesInSentence.subList(0, startNextId).collect {it.getAttribute('word')}

		rowDict['LeftCxInsideSnt'] = lexs.join(" ")

		def inSntCxLengthLeft = startNextId

		// Write the Post-pivot blocks

		i = 0

		for (def block in varDetails.subList(pivotBlockPosition + 1, varDetails.size())){

			i++

			rowDict["${i}BlockAfterPivot"] = block['lexform']

			for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {

				rowDict["${i}BlockAfterPivotType${j+1}"] = block["type${j+1}"]

			}

			// Add any intervening words to the left edge.

			// Note that this isn't always possible, in particular

			// if the preceding element is discontinuous.

			def leftEdgeList = block['terminalsIx'].findAll{it > endPrevId }

			if ( leftEdgeList.size() > 0) {

				def leftEdge = leftEdgeList.min()

				for (int ix = leftEdge - 1 ; ix > endPrevId ; ix--) {

					rowDict["${i}BlockAfterPivot"] = '{' + tNodesInSentence[ix].getAttribute('word') + '} ' + rowDict["${i}BlockAfterPivot"]

				}

			}

			else {

				rowDict["${i}BlockAfterPivot"] = '{?} ' + rowDict["${i}BlockAfterPivot"]

			}

			endPrevId = block['terminalsIx'].max()

		}

		// Write the in-sentence right-context

		lexs = tNodesInSentence.subList(endPrevId + 1, tNodesInSentence.size()).collect {it.getAttribute('word')}

		rowDict['RightCxInsideSnt'] = lexs.join(" ")

		def inSntCxLengthRight = tNodesInSentence.size() - endPrevId

		return [rowDict, inSntCxLengthLeft, inSntCxLengthRight]

		return null;

	}

	public def toList(def iterable) {

		return iterable.findAll {true};

	}

	/**

	 * return the children of sNode with the id anId 

	 */

	public def idKey(def sNode, String anId) {

		return sNode.getElementsByTagName("*").find{it.getAttribute("id") == anId}

	}

	/**

	 * return a list of all terminal nodes of the node.

	 * if the node is a non-terminal, iterate over children and so on

	 */

	def getTNodes(theNode) {

		def terminals = []

		def unprocessed = [theNode]

		while( unprocessed.size() > 0) {

			def aNode = unprocessed.pop()

			def edges = toList(aNode.getElementsByTagName('edge'))

			if (edges.size() == 0)

				terminals.add(aNode)

			else

				for (def anEdge : edges)

					unprocessed.add(idKey(theNode.parentNode.parentNode, anEdge.getAttribute('idref')))

		}

		return terminals;

	}

	/**

	 * 

	 * @param theNode

	 * @return the join of the terminal nodes value in theNode a varaible node

	 */

	def writeLexForm(theNode) {

		def sNode = theNode.parentNode.parentNode // get the sentence of the variable

		def allTNodes = sNode.getElementsByTagName('t') // get all sentence children

		def tNodesInTheNode = getTNodes(theNode) // get the terminal nodes pointed by idref

		def begunNode = false

		def lexs = []

		def lexBuffer = []

		for (def aTNode : allTNodes){

			if ( aTNode in tNodesInTheNode) {

				begunNode = true

				lexs.addAll(lexBuffer)

				lexs.add(aTNode.getAttribute('word'))

				lexBuffer = []

			}

			if ( begunNode && !tNodesInTheNode.contains(aTNode))

				lexBuffer.add('[' + aTNode.getAttribute('word') + ']')

		}

		return lexs.join(" ");

	}

	def matchCombine(theDOM) {

		def matchesNodes = theDOM.getElementsByTagName('matches')

		for (def aMatchesNode : matchesNodes){

			def pivotNodes = toList(

					aMatchesNode.getElementsByTagName('variable')

					).findAll{it.getAttribute('name') == '#pivot'}

			def checkedPivots = []

			while (pivotNodes){

				def aPivotNode = pivotNodes.remove(0)

				def matchingPivotList = checkedPivots.findAll{

					it.getAttribute('idref') == aPivotNode.getAttribute('idref')

				}

				if (matchingPivotList) {

					// duplicate pivot; copy all variables

					for (def node : toList(aPivotNode.getParentNode()

					.getElementsByTagName('variable')

					)) {

						matchingPivotList[0].getParentNode().appendChild(node)

					}

					def variables = toList(

							matchingPivotList[0].getParentNode()

							.getElementsByTagName('variable')

							)

					// remove duplicates

					def checkedVariables = []

					while (variables) {

						def aVariableNode = variables.remove(0)

						def matchingVariableList = checkedVariables.findAll{

							(

									it.getAttribute('idref') == aVariableNode.getAttribute('idref')

									&& (

									it.getAttribute('name') == aVariableNode.getAttribute('name')

									|| (

									it.getAttribute('name').startsWith('#block')

									&& aVariableNode.getAttribute('name').startsWith('#block')

									)

									)

									)

						}

						if (matchingVariableList) {

							// remove variable node

							aVariableNode.getParentNode().removeChild(aVariableNode)

						}

						else {

							checkedVariables.add(aVariableNode)

						}

					}

				}

				else {

					// not the same pivot

					checkedPivots.add(aPivotNode)

				}

			}

			// Tidying up: remove empty <match /> nodes

			def matchNodes = toList(aMatchesNode.getElementsByTagName('match'))

			for (def matchNode : matchNodes) {

				if ( !toList(matchNode.getElementsByTagName('variable'))) {

					aMatchesNode.removeChild(matchNode)

				}

			}

		}

		return theDOM

	}

	public static void usage() {

		println 'concordance_blocks [OPTIONS] inputfile.xml [outputfile.csv]'

		println 'OPTIONS'

		println '-h, --help           Displays this message.'

		println '-c --context [length] Sets the number of words in context.'

	}

	static main(args) {

		if(args.length == 0)

			usage()

		int cx = 30;

		def ntTypes = ['cat'];

		def tTypes = ['pos'];

		File xmlfile = new File(args[0])

		File outfile = new File(args[1])

		ConcordanceBlocks p = new ConcordanceBlocks();

		println "START"

		p.process(xmlfile, outfile, cx, ntTypes, tTypes)

		println "END"

	}

}

	protected TSCorpusManager manager;

	protected TSCorpus tscorpus;

	protected boolean ready = false;

	protected TSResult tsresult;

	protected String T, NT;

	protected MainCorpus corpus;

	protected String query;

	public MainCorpus getCorpus() {

		return corpus;

	}

			if (q != null) {

				return q.substring(0, Math.min(10, q.length())).replaceAll("\n", "")+"...";

			} else {

				return ((TIGERSearch) o).getCorpus().getName();

			}

		return corpus != null && query != null && query.length() > 0;