Laboratoire ICAR » Plateforme TXM

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.exploit

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.*

import org.txm.macro.urs.AnalecUtils

import visuAnalec.elements.*

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.CQPSearchEngine

import org.apache.commons.lang.StringUtils;

if (!(corpusViewSelection instanceof CQPCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=REGEX", widget="String", required=true, def="CHAINE")

String schema_ursql

@Field @Option(name="minimum_schema_size", usage="minimal schema size", widget="Integer", required=true, def="3")

int minimum_schema_size

@Field @Option(name="unit_ursql", usage="TYPE@PROP=REGEX", widget="String", required=false, def="MENTION")

String unit_ursql

@Field @Option(name="word_property", usage="", widget="String", required=false, def="CATEGORIE")

String word_property

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

debug

if (!ParametersDialog.open(this)) return;

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

CQPCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus)

if (!AnalecUtils.isPropertyDefined(Schema.class, analecCorpus, schema_ursql)) {

	println "** The $schema_ursql schema URSQL cannot be computed in the corpus."

	return;

}

if (!AnalecUtils.isPropertyDefined(Unite.class, analecCorpus, unit_ursql)) {

	println "** $unit_ursql unit URSQL cannot be computed in the corpus."

	return;

}

def CQI = CQPSearchEngine.getCqiClient()

def prop = corpus.getProperty(word_property)

def schemas = AnalecUtils.selectSchemasInCorpus(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, 999999);

def freqs = [:]

def distances = 0;

def nDistances = 0

for (def schema : schemas) {

	def allUnites = schema.getUnitesSousjacentesNonTriees()

	def units = AnalecUtils.filterElements(debug, allUnites, unit_ursql)

	int nUnites = units.size();

	if (units.size() == 0) continue;

	def unit = units[0]

	String forme =  null;

	if (prop == null) { // word_property is the analec unit property to use

		forme = unit.getProp(word_property)

	} else {

		int[] pos = null;

		if (unit.getDeb() == unit.getFin()) pos = [unit.getDeb()]

		else pos = unit.getDeb()..unit.getFin()

		forme = StringUtils.join(CQI.cpos2Str(prop.getQualifiedName(), pos), " ") // ids is enough

	}

	if (!freqs.containsKey(forme)) freqs[forme] = 0;

	freqs[forme] = freqs[forme] + 1;

}

println "Index des natures de premier maillon :"

int max = 0;

def result = "";

for (def forme : freqs.keySet().sort() {it -> -freqs[it]}) {

	println "$forme\t"+freqs[forme]

	if (max < freqs[forme]) {

		max = freqs[forme]

		result = "$forme: "+freqs[forme]

	}

}

["result": result, "data": freqs]

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.exploit

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import visuAnalec.elements.*

import org.txm.searchengine.cqp.corpus.*

import org.txm.macro.urs.AnalecUtils

if (!(corpusViewSelection instanceof CQPCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=REGEX", widget="String", required=true, def="CHAINE")

String schema_ursql

@Field @Option(name="minimum_schema_size", usage="minimal schema size", widget="Integer", required=true, def="3")

int minimum_schema_size

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

debug

if (!ParametersDialog.open(this)) return;

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

CQPCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus)

// check Schema parameters

if (!AnalecUtils.isPropertyDefined(Schema.class, analecCorpus, schema_ursql)) {

	println "** The $schema_ursql schema URSQL cannot be computed in the corpus."

	return;

}

def schemas = AnalecUtils.selectSchemasInCorpus(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, 999999);

int nSchemas = schemas.size();

println "Nombre de chaînes de référence d'un texte : $nSchemas"

["result":nSchemas, "data":schemas]

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.exploit

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.*

import org.txm.macro.urs.AnalecUtils

import visuAnalec.elements.*

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.*

import org.txm.searchengine.cqp.corpus.*

import org.apache.commons.lang.StringUtils;

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=VALUE", widget="String", required=true, def="CHAINE")

String schema_ursql

@Field @Option(name="minimum_schema_size", usage="minimal schema size", widget="Integer", required=true, def="3")

int minimum_schema_size

@Field @Option(name="schema_display_property_name",usage="", widget="String", required=false, def="REF")

String schema_display_property_name

@Field @Option(name="unit_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="MENTION")

String unit_ursql

@Field @Option(name="property", usage="", widget="String", required=false, def="CATEGORIE")

String property

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

debug

if (!(corpusViewSelection instanceof CQPCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// Open the parameters input dialog box

if (!ParametersDialog.open(this)) return;

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

// END OF PARAMETERS

MainCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus)

if (!AnalecUtils.isPropertyDefined(Schema.class, analecCorpus, schema_ursql)) {

	println "** The $schema_ursql schema URSQL cannot be computed in the corpus."

	return;

}

if (!AnalecUtils.isPropertyDefined(Unite.class, analecCorpus, unit_ursql)) {

	println "** $unit_ursql unit URSQL cannot be computed in the corpus."

	return;

}

def CQI = CQPSearchEngine.getCqiClient()

def prop = corpus.getProperty(property)

if (prop == null) {

	analecCorpus.getStructure()

}

def schemas = AnalecUtils.selectSchemasInCorpus(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, 999999);

def allFreqs = [:]

def n = 0

for (def schema : schemas) {

	n++

	def freqs = [:]

	def allUnites = schema.getUnitesSousjacentesNonTriees()

	def units = AnalecUtils.filterElements(debug, allUnites, unit_ursql)

	for (def unit : units) { // no need to sort units

		String forme =  null;

		if (prop == null) { // property is the analec unit property to use

			forme = unit.getProp(property)

		} else {

			int[] pos = null;

			if (unit.getDeb() == unit.getFin()) pos = [unit.getDeb()]

			else pos = unit.getDeb()..unit.getFin()

			forme = StringUtils.join(CQI.cpos2Str(prop.getQualifiedName(), pos), " ") // ids is enough

		}

		if (!freqs.containsKey(forme)) freqs[forme] = 0;

		freqs[forme] = freqs[forme] + 1;

		if (!allFreqs.containsKey(forme)) allFreqs[forme] = 0;

		allFreqs[forme] = allFreqs[forme] + 1;

	}

	if (schema_display_property_name != null) {

		println "Index des natures de $unit_ursql de '"+schema.getProp(schema_display_property_name)+"' : "

	} else {

		println "Index des natures de $schema_ursql - $n : "

	}

	int max = 0;

	def result = "";

	for (def forme : freqs.sort() { a, b -> -a.value <=> -b.value ?: a.key <=> b.key }) {

		println forme.key+"\t"+forme.value

	}

}

int max = 0;

def result = "";

println "Index des natures de $schema_ursql : "

for (def forme : allFreqs.sort() { a, b -> -a.value <=> -b.value ?: a.key <=> b.key }) {

	println forme.key+"\t"+forme.value

	if (max < forme.value) {

		max = forme.value

		result = "$forme: "+forme.value

	}

}

return ["result":result, "data":allFreqs]

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.corpus.*

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_type",usage="", widget="String", required=true, def="CHAINE")

String schema_type

if (!ParametersDialog.open(this)) return;

MainCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus);

def unitesInSchema = new HashSet()

for (def schema : analecCorpus.getSchemas(schema_type)) {

	unitesInSchema.addAll(schema.getUnitesSousjacentes())

}

println "unites: "+analecCorpus.getToutesUnites().size()

println "unites in schema: "+unitesInSchema.size()

def set = new HashMap()

for (def u : analecCorpus.getToutesUnites()) {

	if (unitesInSchema.contains(u)) continue;

	if (!set.containsKey(u.getType())) set[u.getType()] = 0;

	set[u.getType()] = set[u.getType()] +1

}

println "unites not in schema: "+set.sort() { it -> set[it]}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.lang.StringUtils

import org.jfree.chart.JFreeChart

import org.jfree.chart.plot.XYPlot

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.Toolbox

import org.txm.progression.core.chartsengine.jfreechart.themes.highcharts.renderers.ProgressionItemSelectionRenderer

import org.txm.progression.core.functions.Progression

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.chartsengine.rcp.editors.ChartEditor

import org.txm.macro.urs.AnalecUtils

import org.txm.searchengine.cqp.AbstractCqiClient

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.searchengine.cqp.corpus.query.CQLQuery

import org.txm.rcp.Application

import org.txm.rcp.IImageKeys

import visuAnalec.donnees.Structure

import visuAnalec.elements.*

def scriptName = this.class.getSimpleName()

def parent

def selection = []

if (!(corpusViewSelection instanceof CQPCorpus)) {

	println "** $scriptName please select a Corpus to run the macro"

}

selection << corpusViewSelection

parent = corpusViewSelection

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="CHAINE")

		String schema_ursql

@Field @Option(name="minimum_schema_size", usage="Minimum size needed to consider a schema", widget="Integer", required=true, def="3")

		int minimum_schema_size

@Field @Option(name="maximum_schema_size", usage="Maximum size needed to consider a schema", widget="Integer", required=true, def="9999999")

		int maximum_schema_size

@Field @Option(name="unit_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="MENTION")

		String unit_ursql

@Field @Option(name="limit_distance_in_schema", usage="Unit distance in schema (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=true, def="0")

		int limit_distance_in_schema

@Field @Option(name="limit_cql", usage="CQL to build structure limits", widget="Query", required=true, def="<div> [] expand to div")

		limit_cql

@Field @Option(name="strict_inclusion", usage="Units must be strictly included into corpus matches", widget="Boolean", required=true, def="true")

		boolean strict_inclusion

@Field @Option(name="limit_distance", usage="Unit distance to structure limit (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=true, def="0")

		int limit_distance

@Field @Option(name="unit_property_display", usage="Unit property to count", widget="String", required=true, def="CATEGORIE")

		String unit_property_display

@Field @Option(name="struct_name", usage="Structure to display", widget="String", required=true, def="div")

		String struct_name

@Field @Option(name="struct_prop", usage="Structure property to display", widget="String", required=true, def="n")

		String struct_prop

@Field @Option(name="line_width", usage="line width", widget="Integer", required=true, def="1")

		int line_width = 2

@Field @Option(name="bande_width", usage="bande width", widget="Float", required=true, def="1.0f")

		float bande_width = 1.0f

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

		debug

if (!ParametersDialog.open(this)) return

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

	def CQI = CQPSearchEngine.getCqiClient()

def queries = []

def queryResults = []

def informations = []

for (def corpus : selection) {

	mainCorpus = corpus.getMainCorpus()

	def word = mainCorpus.getWordProperty()

	def analecCorpus = URSCorpora.getCorpus(mainCorpus.getName())

	def selectedUnits = AnalecUtils.selectUnitsInSchema(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, maximum_schema_size, 

	unit_ursql, limit_distance_in_schema, limit_cql, strict_inclusion, limit_distance);

	def query = ""

	if (limit_cql != null && !limit_cql.getQueryString().equals("\"\"")) query += limit_cql

	if (schema_ursql != null && schema_ursql.length() > 0) { if (query.length() > 0) query += " & "; query += ""+schema_ursql+ " >"}

	if (unit_ursql != null && unit_ursql.length() > 0) query += " "+unit_ursql

	query = new CQLQuery(query)

	int[] starts = new int[selectedUnits.size()];

	int[] ends = new int[selectedUnits.size()];

	def unitsinformations = []

	int n = 0;

	for (Unite unite : selectedUnits) {

		starts[n] = unite.getDeb()

		ends[n] = unite.getFin()

		unitsinformations << AnalecUtils.toString(CQI, word, unite);

		n++

	}

	def queryResult = new FakeQueryResult(corpus.getID(), corpus, query, starts, ends, null)

	queries << query

	queryResults << queryResult

	informations << unitsinformations

	if (unit_property_display != null && unit_property_display.length() > 0) {

		def propvalues = [:]

		for (def unit : selectedUnits) {

			def v = unit.getProp(unit_property_display)

			if (v == null) v = "<null>"

			else if (v.length() == 0) v = "<empty>"

			if (!propvalues.containsKey(v))propvalues[v] = []

			propvalues[v] << unit

		}

		for (def v : propvalues.keySet().sort()) {

			selectedUnits = propvalues[v]

			query = corpus.getID()+" "+limit_cql

			query = new CQLQuery(v)

			starts = new int[selectedUnits.size()];

			ends = new int[selectedUnits.size()];

			unitsinformations = []

			n = 0;

			for (Unite unite : selectedUnits) {

				starts[n] = unite.getDeb()

				ends[n] = unite.getFin()

				unitsinformations << AnalecUtils.toString(CQI, word, unite);

				n++

			}

			queryResult = new FakeQueryResult(corpus.getID(), corpus, query, starts, ends, null)

			queries << query

			queryResults << queryResult

			informations << unitsinformations

		}

	}

}

corpus = parent

try {

	def struct = corpus.getStructuralUnit(struct_name)

	def struct_p = struct.getProperty(struct_prop)

	Progression progression = new Progression(corpus, queries,

			struct, struct_p,	".*",

			true, false, false,

			line_width, false, bande_width)

	progression.stepQueries(queryResults); // new

	if (!progression.stepStructuralUnits() || monitor.isCanceled())	return

		monitor.worked(20)

	if (!progression.stepFinalize() || monitor.isCanceled()) return

		monitor.worked(20)

	monitor.syncExec(new Runnable() {

				@Override

				public void run() {

					try {

						ChartEditor charteditorpart = SWTChartsComponentProvider.openEditor(Application.swtComponentProvider.createProgressionChartEditorPart(IImageKeys.getImage(IImageKeys.ACTION_PROGRESSION), progression, progression.isMonochrome(), progression.isMonostyle(), progression.isDoCumulative()))

						JFreeChart chart = charteditorpart.getChart()

						def plot = chart.getXYPlot()

						ProgressionItemSelectionRenderer renderer = plot.getRenderer();

						renderer.setAdditionalLabelInformation(informations)

					} catch(Exception e) {e.printStackTrace()}

				}

			})

} catch(Exception e) {

	e.printStackTrace()

	return false

}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.corpus.*

import org.txm.Toolbox

import org.txm.rcp.commands.*

import org.apache.commons.lang.StringUtils

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_type",usage="", widget="String", required=true, def="CHAIE")

String schema_type

@Field @Option(name="minimum_schema_size",usage="", widget="Integer", required=true, def="3")

int minimum_schema_size

@Field @Option(name="schema_property_name",usage="", widget="String", required=false, def="")

String schema_property_name

@Field @Option(name="schema_property_value",usage="", widget="String", required=false, def=".*")

String schema_property_value

@Field @Option(name="unit_type",usage="", widget="String", required=false, def="MENTION")

String unit_type

@Field @Option(name="unit_property_name", usage="", widget="String", required=false, def="")

String unit_property_name

@Field @Option(name="unit_property_value", usage="", widget="String", required=false, def=".*")

String unit_property_value

@Field @Option(name="word_property", usage="", widget="StringArray", metaVar="word	lemma	frlemma	frolemma	#forme#	id", required=false, def="word")

String word_property

@Field @Option(name="separator", usage="", widget="String", required=true, def=", ")

String separator

@Field @Option(name="buildCQL", usage="générer la requête des unités", widget="Boolean", required=true, def='false')

def buildCQL

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpus view selection is not a Corpus"

	return;

}

if (!ParametersDialog.open(this)) return;

// END OF PARAMETERS

MainCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus)

// check Schema parameters

if (!analecCorpus.getStructure().getSchemas().contains(schema_type)) {

	println "No schema with name=$schema_type"

	return;

} else {

	if (schema_property_name.length() > 0 && schema_property_value.length() > 0) {

		// test property existance

		def props = analecCorpus.getStructure().getSchemaProperties(schema_type);

		if (!props.contains(schema_property_name)) {

			println "Schema $schema_type has no property named $schema_property_name"

			return;

		}

	}

}

// check unit parameters

if (!analecCorpus.getStructure().getUnites().contains(unit_type)) {

	println "No unit with name=$unit_type"

	return;

} else {

	if (unit_property_name.length() > 0 && unit_property_value.length() > 0) {

		// test property existance

		def props = analecCorpus.getStructure().getUniteProperties(unit_type);

		if (!props.contains(unit_property_name)) {

			println "Unit $unit_type has no property named $unit_property_name"

			return;

		}

	}

}

def CQI = CQPSearchEngine.getCqiClient()

if (buildCQL) {

	word_prop = corpus.getProperty("id")

} else {

	word_prop = corpus.getProperty(word_property)

}

def schemas = analecCorpus.getSchemas(schema_type)

schemas.sort() {it.getProps()}

def nSchemas = 0

def lens = [:]

for (def schema : schemas) {

	if (schema_property_name.length() > 0 && schema_property_value.length() > 0) {

		if (!schema.getProp(schema_property_name).matches(schema_property_value)) {

			// ignoring this schema

			continue

		}

	}

	def nUnites = 0

	for (def unit : schema.getUnitesSousjacentes()) {

		if (unit_type.length() > 0) {

			if (!unit.getType().equals(unit_type)) {

				continue

			}

		}

		if (unit_property_name.length() > 0 && unit_property_value.length() > 0) {

			if (!unit.getProp(unit_property_name).matches(unit_property_value)) {

				// ignoring this schema

				continue

			}

		}

		nUnites++

	}

	if (nUnites < minimum_schema_size) continue

	print schema.getProps().toString()+ ": "

	def first = true

	for (def unit : schema.getUnitesSousjacentes()) {

		if (unit_type.length() > 0) {

			if (!unit.getType().equals(unit_type)) {

				continue

			}

		}

		if (unit_property_name.length() > 0 && unit_property_value.length() > 0) {

			if (!unit.getProp(unit_property_name).matches(unit_property_value)) {

				// ignoring this schema

				continue

			}

		}

		String forme =  null;

		if (buildCQL) {

			int[] pos = null

			if (unit.getDeb() == unit.getFin()) pos = [unit.getDeb()]

			else pos = (unit.getDeb()..unit.getFin())

			def first2= true

			q = ""

			pos.each {

				if (first2) { first2 = false } else { q = q+" " }

				int[] pos2 = [it]

				q = q+"["+word_prop+"=\""+CQI.cpos2Str(word_prop.getQualifiedName(), pos2)[0]+"\"]"

			}

			if (first) { first = false } else { print "|" }

			print "("+q+")"

		} else {

			if (word_prop == null) { // word_property is the analec unit property to use

			forme = unit.getProp(word_property)

			} else {

			int[] pos = null

			if (unit.getDeb() == unit.getFin()) pos = [unit.getDeb()]

			else pos = (unit.getDeb()..unit.getFin())

			forme = StringUtils.join(CQI.cpos2Str(word_prop.getQualifiedName(), pos), " ") // ids is enough

			}

			if (first) { first = false } else { print separator }

			print forme

		}

	}

	println ""

	nSchemas++

}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.apache.commons.lang.StringUtils

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.Toolbox

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.macro.urs.AnalecUtils

import org.txm.searchengine.cqp.AbstractCqiClient

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.corpus.query.CQLQuery

import visuAnalec.donnees.Structure

import visuAnalec.elements.*

def scriptName = this.class.getSimpleName()

def selection = []

for (def s : corpusViewSelections) {

	if (s instanceof CQPCorpus) selection << s

	else if (s instanceof Partition) selection.addAll(s.getParts())

}

if (selection.size() == 0) {

	println "** $scriptName: please select a Corpus or a Partition in the Corpus view: "+corpusViewSelections

	return false

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=VALUE", widget="String", required=true, def="CHAINE")

		String schema_ursql

@Field @Option(name="minimum_schema_size", usage="Minimum size needed to consider a schema", widget="Integer", required=true, def="3")

		int minimum_schema_size

@Field @Option(name="unit_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="MENTION")

		String unit_ursql

@Field @Option(name="limit_cql", usage="CQL to build structure limits", widget="Query", required=true, def="<div> [] expand to div")

		limit_cql

@Field @Option(name="strict_inclusion", usage="Units must be strictly included into corpus matches", widget="Boolean", required=true, def="true")

		boolean strict_inclusion

@Field @Option(name="limit_distance", usage="Unit distance to structure limit (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=true, def="0")

		int limit_distance

@Field @Option(name="debug", usage="Show internal variable content", widget="Boolean", required=true, def="false")

		debug

if (!ParametersDialog.open(this)) return

	def CQI = CQPSearchEngine.getCqiClient()

//corpus = corpusViewSelection

for (def corpus : selection) {

	mainCorpus = corpus.getMainCorpus()

	def word = mainCorpus.getWordProperty()

	def analecCorpus = URSCorpora.getCorpus(mainCorpus.getName())

	def selectedUnits = AnalecUtils.selectUnitsInSchema(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, Integer.MAX_VALUE,

			unit_ursql, limit_cql, strict_inclusion, limit_distance);

	for (def unit : selectedUnits) {

		def props = unit.getProps();

		for (def k : props.keySet()) {

			if (props[k] == null) {

				println "$corpus\t"+unit.getDeb()+"->"+unit.getFin()+"\t"+k

			}

		}

	}

}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.corpus.*

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_type",usage="", widget="String", required=true, def="MENTION")

String schema_type

if (!ParametersDialog.open(this)) return;

MainCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus);

def map = new HashMap()

def unitesInSchema = []

def n = 0

for (def schema : analecCorpus.getSchemas(schema_type)) {

	def unites = schema.getUnitesSousjacentes()

	unitesInSchema.addAll(unites)

	n += unites.size()

}

def counts = unitesInSchema.countBy() { it };

for (def c : counts.keySet()) {

	if (counts[c] > 1) println "ERROR UNIT IN MULTIPLE SCHEMA["+c.getDeb()+", "+c.getFin()+"]="+c.getProps()+" in "+c.getSchemas().collect() {it.getProps()}

}

def set = new HashSet()

set.addAll(unitesInSchema)

for (def s : set.collect { it.getType() }) {

	if (!map.containsKey(s)) map[s] = 0;

	map[s] = map[s] +1

}

println "Unites types: "+map.sort() { it -> map[it]}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.apache.commons.lang.StringUtils

import org.txm.rcp.views.corpora.CorporaView

import groovy.transform.Field

import org.kohsuke.args4j.*

import org.txm.Toolbox

import org.txm.annotation.urs.*

import org.txm.lexicaltable.core.statsengine.r.data.LexicalTableImpl

import org.txm.macro.urs.*

import org.txm.rcp.commands.*

import org.txm.rcp.swt.widget.parameters.*

import org.txm.searchengine.cqp.CQPSearchEngine

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.corpus.query.CQLQuery;

import org.txm.statsengine.r.core.RWorkspace

import visuAnalec.donnees.*

import visuAnalec.elements.*

import cern.colt.matrix.DoubleFactory2D

import cern.colt.matrix.DoubleMatrix2D

def scriptName = this.class.getSimpleName()

def parent

def selection = []

if (!(corpusViewSelection instanceof CQPCorpus)) {

	println "** $scriptName please select a Corpus to run the macro"

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="CHAINE")

		String schema_ursql

@Field @Option(name="minimum_schema_size", usage="Minimum size needed to consider a schema", widget="Integer", required=true, def="3")

		int minimum_schema_size

@Field @Option(name="maximum_schema_size", usage="Maximum size needed to consider a schema", widget="Integer", required=true, def="9999999")

		int maximum_schema_size

@Field @Option(name="unit_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="MENTION")

		String unit_ursql

@Field @Option(name="limit_distance_in_schema", usage="Unit distance in schema (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=true, def="0")

		int limit_distance_in_schema

@Field @Option(name="limit_cql", usage="CQL to build structure limits", widget="Query", required=true, def="<div> [] expand to div")

		limit_cql

@Field @Option(name="strict_inclusion", usage="Units must be strictly included into corpus matches", widget="Boolean", required=true, def="true")

		boolean strict_inclusion

@Field @Option(name="limit_distance", usage="Unit distance to structure limit (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=true, def="0")

		int limit_distance

@Field @Option(name="unit_prop1", usage="PROP1", widget="String", required=false, def="PROP1")

		String unit_prop1

@Field @Option(name="unit_prop2", usage="PROP2", widget="String", required=false, def="PROP2")

		String unit_prop2

@Field @Option(name="corr_method", usage="try them all", widget="StringArray", metaVar="pearson	spearman	kendall", required=false, def="pearson")

		String corr_method

@Field @Option(name="corr_style", usage="try them all", widget="StringArray", metaVar="circle	square	ellipse	number	shade	color	pie", required=false, def="number")

		String corr_style

@Field @Option(name="corr_layout", usage="try them all", widget="StringArray", metaVar="full	lower	upper", required=false, def="upper")

		String corr_layout

@Field @Option(name="corr_order", usage="try them all", widget="StringArray", metaVar="AOE	FPC	hclust	alphabet", required=false, def="hclust")

		String corr_order

@Field @Option(name="output_lexicaltable", usage="create or not a lexical table with the result", widget="Boolean", required=true, def="false")

		output_lexicaltable

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

		debug

if (!ParametersDialog.open(this)) return

	if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

def CQI = CQPSearchEngine.getCqiClient()

def correlations = [:]

def values1 = new HashSet()

def values2 = new HashSet()

def corpus = corpusViewSelection

mainCorpus = corpus.getMainCorpus()

def word = mainCorpus.getWordProperty()

def analecCorpus = URSCorpora.getCorpus(mainCorpus)

def selectedUnits = AnalecUtils.selectUnitsInSchema(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, maximum_schema_size,

		unit_ursql, limit_distance_in_schema, limit_cql, strict_inclusion, limit_distance);

for (def unit : selectedUnits) {

	def value1 = unit.getProp(unit_prop1)

	if (value1 == null) value1 = "<null>"

	if (value1.length() == 0) value1 = "<empty>"

	def value2 = unit.getProp(unit_prop2)

	if (value2 == null) value2 = "<null>"

	if (value2.length() == 0) value2 = "<empty>"

	values1 << value1

	values2 << value2

	if (!correlations.containsKey(value1)) correlations[value1] = [:]

	def line = correlations[value1]

	if (!line.containsKey(value2)) line[value2] = 0

	line[value2] += 1

}

def matrix = new int[values1.size()][values2.size()];

println "\t"+values2.join("\t")

int i = 0;

for (def value1 : values1) {

	print value1

	int j = 0;

	for (def value2 : values2) {

		if (correlations[value1][value2] == null) correlations[value1][value2] = 0;

		print "\t"+correlations[value1][value2]

		matrix[i][j] = correlations[value1][value2]

		j++

	}

	println ""

	i++

}

def r = RWorkspace.getRWorkspaceInstance()

r.addVectorToWorkspace("corrlines", values1 as String[])

r.addVectorToWorkspace("corrcols", values2 as String[])

r.addMatrixToWorkspace("corrmatrix", matrix)

r.eval("rownames(corrmatrix) = corrlines")

r.eval("colnames(corrmatrix) = corrcols")

def resultsDir = new File(Toolbox.getTxmHomePath(), "results")

resultsDir.mkdirs()

file = File.createTempFile("txm_corr_pairs_", ".svg", resultsDir)

def title = "${corpus.getMainCorpus()}.${corpus}\n${unit_ursql}"

if (limit_distance > 1) title += "[${limit_distance}]."

if (limit_cql != null && !limit_cql.getQueryString().equals("\"\"")) title += "\n(${limit_cql} limits)"

title += "\t P1=$unit_prop1 P2=unit_prop2"

def plotScript = """

r1 = cor(corrmatrix, use="complete.obs", method="$corr_method");

r2 = cov(corrmatrix, use="complete.obs") ;

corrplot(r1, type="$corr_layout", order="$corr_order", method="$corr_style")

"""

// execute R script

if (!output_lexicaltable) {

	try {

		r.eval("library(corrplot)")

		try {

			r.plot(file, plotScript)

		} catch (Exception e) {

			println "** Error: "+e

		}

	} catch (Exception e) {

		println "** The 'corrplot' R package is not installed. Start R ("+RWorkspace.getExecutablePath()+") and run 'install.packages(\"corrplot\");'."

	}

}

title = "$unit_prop1 $corr_method correlations"

def lt = null;

if (output_lexicaltable) {

	mFactory = DoubleFactory2D.dense

	dmatrix = mFactory.make(values1.size(), values2.size())

	for (int ii = 0 ; ii < values1.size() ; ii++) {

		for (int jj = 0 ; jj < values2.size() ; jj++) {

			dmatrix.set(ii, jj, matrix[ii][jj])

		}

	}

	if (corpusViewSelection instanceof Partition) {

		lt = new LexicalTableImpl(dmatrix, corpusViewSelection, corpusViewSelection.getCorpus().getProperty("word"),

				values1 as String[], values2 as String[])

		lt.setCorpus(corpusViewSelection.getCorpus());

		corpusViewSelection.storeResult(lt)

	} else {

		lt = new LexicalTableImpl(dmatrix, corpus.getProperty("word"),

				values1 as String[], values2 as String[])

		lt.setCorpus(corpus);

		corpus.storeResult(lt)

	}

}

monitor.syncExec(new Runnable() {

			@Override

			public void run() { try {

					if (output_lexicaltable) {

						CorporaView.refreshObject(corpus)

						CorporaView.expand(lt)

					} else {

						OpenBrowser.openfile(file.getAbsolutePath(), "Correlations Units")

					}

				} catch (e) { e.printStackTrace() }}

		})

return correlations

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.corpus.*

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

MainCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus);

def schemas = analecCorpus.getTousSchemas()

def set = new HashMap()

for (def s : schemas.collect { it.getType() }) {

	if (!set.containsKey(s)) set[s] = 0;

	set[s] = set[s] +1

}

println "Schemas types: "+set.sort() { it -> set[it]}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.apache.commons.lang.StringUtils;

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.Toolbox;

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.AbstractCqiClient;

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.CQPSearchEngine

import visuAnalec.donnees.Structure;

import visuAnalec.elements.Unite;

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="unit_type",usage="", widget="String", required=true, def="MENTION")

String unit_type

@Field @Option(name="print_diff",usage="", widget="Boolean", required=true, def="true")

boolean print_diff

@Field @Option(name="unit_property_name1", usage="", widget="String", required=false, def="CATEGORIE")

String unit_property_name1

@Field @Option(name="unit_property_name2", usage="", widget="String", required=false, def="CATEGORIE_ORIG")

String unit_property_name2

if (!ParametersDialog.open(this)) return;

int n = 1;

int nDiff = 0;

MainCorpus corpus = corpusViewSelection

AbstractCqiClient CQI = CQPSearchEngine.getCqiClient();

def word = corpus.getWordProperty()

def analecCorpus = URSCorpora.getCorpus(corpus);

def units = analecCorpus.getUnites(unit_type)

units.sort() { a, b -> a.getDeb() <=> b.getDeb() ?: a.getFin() <=> b.getFin() }

for (Unite unit : units) {

	int[] pos = null

	if (unit.getDeb() == unit.getFin()) pos = [unit.getDeb()]

	else pos = (unit.getDeb()..unit.getFin())

	def form = StringUtils.join(CQI.cpos2Str(word.getQualifiedName(), pos), " ")

	def props = unit.getProps()

	def v1 = props.get(unit_property_name1);

	def v2 = props.get(unit_property_name2);

	if (v1 != v2) {

		if (print_diff) println "$n - ${unit.getDeb()} -> ${unit.getFin()} - $props : $form"

		nDiff++

	}

	n++

}

if (nDiff == 0) println "$unit_property_name1 and $unit_property_name2 have the same values."

else println "$unit_property_name1 and $unit_property_name2 have $nDiff/$n different values."

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.apache.commons.lang.StringUtils;

import org.apache.tools.ant.types.resources.selectors.InstanceOf;

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.Toolbox;

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.AbstractCqiClient;

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.CQPSearchEngine

import visuAnalec.donnees.Structure;

import visuAnalec.elements.Relation

import visuAnalec.elements.Unite;

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="relation_type",usage="", widget="String", required=true, def="ANAPHORE")

String relation_type

if (!ParametersDialog.open(this)) return;

MainCorpus corpus = corpusViewSelection

AbstractCqiClient CQI = CQPSearchEngine.getCqiClient();

def word = corpus.getWordProperty()

visuAnalec.donnees.Corpus analecCorpus = URSCorpora.getCorpus(corpus);

int n = 1;

def relations = null

if (relation_type.length() > 0) {

	relations = []

	for (String type : analecCorpus.getStructure().getTypes(Relation.class))

		relations.addAll(analecCorpus.getRelations(type))

} else {

	relations = analecCorpus.getToutesRelations()

}

for (Relation relation : relations) {

	def unit1 = relation.getElt1();

	def unit2 = relation.getElt2();

	def props = relation.getProps()

	if (unit1 instanceof Unite && unit2 instanceof Unite) {

		int[] pos1 = null

		if (unit1.getDeb() == unit1.getFin()) pos1 = [unit1.getDeb()]

		else pos1 = (unit1.getDeb()..unit1.getFin())

		def form1 = StringUtils.join(CQI.cpos2Str(word.getQualifiedName(), pos1), " ")

		int[] pos2 = null

		if (unit2.getDeb() == unit2.getFin()) pos2 = [unit2.getDeb()]

		else pos2 = (unit2.getDeb()..unit2.getFin())

		def form2 = StringUtils.join(CQI.cpos2Str(word.getQualifiedName(), pos2), " ")

		println "$n - $props : $form1 -> $form2"

	} else {

		println "$n - $props"

	}

	n++

}

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macroprototypes.urs.misc

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.annotation.urs.*

import org.txm.searchengine.cqp.corpus.*

if (!(corpusViewSelection instanceof MainCorpus)) {

	println "Corpora selection is not a Corpus"

	return;

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_type",usage="", widget="String", required=true, def="CHAINE")

String schema_type

if (!ParametersDialog.open(this)) return;

MainCorpus corpus = corpusViewSelection

def analecCorpus = URSCorpora.getCorpus(corpus);

def map = new HashMap()

def unitesInSchema = []

def n = 0

for (def schema : analecCorpus.getSchemas(schema_type)) {

	def unites = schema.getUnitesSousjacentes()

	unitesInSchema.addAll(unites)

	n += unites.size()

}

def counts = unitesInSchema.countBy() { it };

for (def c : counts.keySet()) {

	if (counts[c] > 1) println "ERROR UNIT IN MULTIPLE SCHEMA["+c.getDeb()+", "+c.getFin()+"]="+c.getProps()+" in "+c.getSchemas().collect() {it.getProps()}

}

def set = new HashSet()

set.addAll(unitesInSchema)

for (def s : set.collect { it.getType() }) {

	if (!map.containsKey(s)) map[s] = 0;

	map[s] = map[s] +1

}

println "Unites types: "+map.sort() { it -> map[it]}