Laboratoire ICAR » Plateforme TXM

// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// @author sheiden

// STANDARD DECLARATIONS

package org.txm.macro.urs.exploit

import groovy.transform.Field

import org.apache.commons.lang.StringUtils

import org.kohsuke.args4j.*

import org.txm.Toolbox

import org.txm.annotation.urs.*

import org.txm.lexicaltable.core.functions.LexicalTable

import org.txm.lexicaltable.core.statsengine.r.data.LexicalTableImpl

import org.txm.macro.urs.*

import org.txm.rcp.commands.*

import org.txm.rcp.swt.widget.parameters.*

import org.txm.rcp.views.corpora.CorporaView

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.CQPSearchEngine

import org.txm.statsengine.r.core.RWorkspace

import visuAnalec.donnees.*

import visuAnalec.elements.*

import cern.colt.matrix.DoubleFactory2D

import cern.colt.matrix.DoubleMatrix2D

def scriptName = this.class.getSimpleName()

def selection = []

for (def s : corpusViewSelections) {

        if (s instanceof CQPCorpus) selection << s

        else if (s instanceof Partition) selection.addAll(s.getParts())

}

if (selection.size() == 0) {

        println "** $scriptName: please select a Corpus or a Partition in the Corpus view: "+corpusViewSelections

        return false

} else {

        for (def c : selection) c.compute(false)

}

// BEGINNING OF PARAMETERS

@Field @Option(name="schema_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="CHAINE")

                String schema_ursql

@Field @Option(name="minimum_schema_size", usage="Minimum size needed to consider a schema", widget="Integer", required=false, def="3")

                int minimum_schema_size

@Field @Option(name="maximum_schema_size", usage="Maximum size needed to consider a schema", widget="Integer", required=false, def="9999999")

                int maximum_schema_size

@Field @Option(name="unit_ursql", usage="TYPE@PROP=VALUE", widget="String", required=false, def="MENTION")

                String unit_ursql

@Field @Option(name="unit_property_display", usage="Unit property to count", widget="String", required=false, def="CATEGORIE")

                String unit_property_display

@Field @Option(name="word_property_display", usage="Word property to display instead of the unit property", widget="String", required=false, def="")

                String word_property_display

@Field @Option(name="position_in_schema", usage="Unit distance in schema (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=false, def="0")

                int position_in_schema

@Field @Option(name="cql_limit", usage="CQL to build structure limits", widget="Query", required=false, def="")

                cql_limit

@Field @Option(name="strict_inclusion", usage="Units must be strictly included into corpus matches", widget="Boolean", required=false, def="true")

                strict_inclusion

@Field @Option(name="position_in_matches", usage="Unit distance to structure limit (0 = no selection, 1 = first after limit, -1 = last before limit, etc.)", widget="Integer", required=false, def="0")

                position_in_matches

//@Field @Option(name="output_2D", usage="output barplot or 3D plot", widget="Boolean", required=true, def="true")

                output_2D = true

@Field @Option(name="output_showlegend", usage="output barplot or 3D plot", widget="Boolean", required=false, def="true")

                output_showlegend

@Field @Option(name="output_fmin", usage="minimal frequency displayed", widget="Integer", required=false, def="0")

                output_fmin

@Field @Option(name="output_histogram", usage="show or not a histogram of the result", widget="Boolean", required=false, def="true")

                output_histogram

@Field @Option(name="output_lexicaltable", usage="create or not a lexical table with the result", widget="Boolean", required=false, def="false")

                output_lexicaltable

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF        ON        ALL        REALLY ALL", required=false, def="OFF")

                debug

if (!ParametersDialog.open(this)) return

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

if (word_property_display != null && word_property_display.length() > 0 && unit_property_display != null && unit_property_display.length() > 0) {

        println "Warning: both unit_property_display=$unit_property_display and word_property_display=$word_property_display are set. The index will be computed with the $word_property_display CQP property."

}

if (word_property_display.length() == 0 && unit_property_display.length() == 0) {

        println "Error: no analysis property specified in unit_property_display or word_property_display. Aborting."

        return

}

def CQI = CQPSearchEngine.getCqiClient()

def allresults = [:]

for (def corpus : selection) {

        def word = corpus.getWordProperty()

        def analecCorpus = URSCorpora.getCorpus(corpus)

        def errors = AnalecUtils.isPropertyDefined(Schema.class, analecCorpus, schema_ursql)

        if (errors.size() > 0) {

                println "** The $schema_ursql schema URSQL cannot be computed in the corpus with types: $errors."

                return;

        }

        errors=AnalecUtils.isPropertyDefined(Unite.class, analecCorpus, unit_ursql)

        if (errors.size() > 0) {

                println "** $unit_ursql unit URSQL cannot be computed in the corpus with types: $errors."

                return;

        }

        if (unit_property_display.length() > 0) {

                errors = AnalecUtils.isPropertyDefined(Unite.class, analecCorpus, AnalecUtils.getFilterParameters(unit_ursql)[0], unit_property_display).size()

                if (errors > 0) {

                        println "Error: some Unit types don't contain the $unit_property_display property: $errors"

                        return

                }

        }

        def selectedUnits = AnalecUtils.selectUnitsInSchema(debug, analecCorpus, corpus, schema_ursql, minimum_schema_size, maximum_schema_size,

                        unit_ursql, position_in_schema, cql_limit, strict_inclusion, position_in_matches);

        selectedUnits = new HashSet(selectedUnits)

        def counts = null

        if (word_property_display != null && word_property_display.length() > 0) {

                Property p = corpus.getProperty(word_property_display)

                int[] pos = null

                counts = [:]

                for (def unit : selectedUnits) {

                        if (unit.getDeb() == unit.getFin()) pos = [unit.getDeb()]

                        else pos = (unit.getDeb()..unit.getFin())

                        String s = StringUtils.join(CQI.cpos2Str(p.getQualifiedName(), pos), " ")

                        if (!counts.containsKey(s)) counts[s] = 0

                        counts[s] = counts[s] + 1

                }

        } else { // use Analec unit property

                counts = selectedUnits.countBy { if (it.getProp(unit_property_display) == null) "<null>" else it.getProp(unit_property_display) }

        }

        if (counts.containsKey("")) counts["<empty>"] = counts[""];

        counts.remove("")

        if (debug) {

                println "\n*** Statistics: "

                for (def k : counts.keySet()) {

                        if (k.size() > 0) { println "$k\t"+counts[k] }else{ println "N/A\t"+counts[k] }

                }

        }

        allresults[corpus] = counts

}

if (debug) println "RESULTS=$allresults"

def keys = new HashSet();

for (def corpus : allresults.keySet()) keys.addAll(allresults[corpus].keySet())

if (keys.size() == 0) {

        println "No values for $unit_property_display property"

        return

}

keys = keys.sort()

mFactory = DoubleFactory2D.dense

matrix = mFactory.make(keys.size(), selection.size())

if (debug) println "KEYS=$keys"

def corpus = null

if (corpusViewSelection instanceof Partition) { corpus = corpusViewSelection.getCorpus() }

else { corpus = corpusViewSelection }

def title = "${corpus.getMainCorpus()}.${corpusViewSelection}\n${unit_ursql}"

title += "[${position_in_matches}]."

if (word_property_display.length() > 0) title += "${word_property_display} frequencies"

else if (unit_property_display.length() > 0) title += "${unit_property_display} frequencies"

if (cql_limit != null && !cql_limit.getQueryString().equals("\"\"")) title += "\n(${cql_limit} limits)"

if (word_property_display.length() > 0) println "Index de la propriété $word_property_display des mots des unités $unit_ursql[$position_in_matches] de ${corpus.getMainCorpus()}.${corpusViewSelection}"

else println "Index de la propriété $unit_property_display des unités $unit_ursql[$position_in_matches] de ${corpus.getMainCorpus()}.${corpusViewSelection}"

println "$unit_property_display\t"+selection.join("\t")

keys.eachWithIndex { prop_val, i ->

        String line = ""

        if (prop_val.size() > 0) {

                line += prop_val

        } else {

                line += "N/A"

        }

        int min = 0;

        allresults.keySet().eachWithIndex { c, j ->

                def freq = allresults[c][prop_val]

                if (freq == null) freq = 0

                matrix.set(i, j, freq)

                line += "\t"+freq

                if (min < freq) min = freq

        }

        if (min >= output_fmin)

                println "$line"

}

props = keys

def r = RWorkspace.getRWorkspaceInstance()

def colnames = selection as String[]

r.addVectorToWorkspace("texts", colnames)

def rownames = props as String[]

r.addVectorToWorkspace("props", rownames)

r.addMatrixToWorkspace("textpropfreqs", matrix as DoubleMatrix2D)

r.eval("rownames(textpropfreqs) = props")

r.eval("colnames(textpropfreqs) = texts")

def resultsDir = new File(Toolbox.getTxmHomePath(), "results")

def file = File.createTempFile("txm", ".svg", resultsDir)

//if (corpusViewSelection.getClass() == org.txm.searchengine.cqp.corpus.Partition) { corpus = corpusViewSelection.getParent() } else { corpus = corpusViewSelection }

if (output_histogram) {

        def plotScript

        String legend = "rownames(textpropfreqs)"

        if (!output_showlegend) legend = "NULL"

        if (output_2D) {

                plotScript = """library(RColorBrewer);

barplot(height=textpropfreqs, main="${title}", xlab="Parties", legend=$legend, beside=TRUE, col=rainbow(length(rownames(textpropfreqs))))"""

        } else {

                plotScript = """

                   library(latticeExtra)

                   library(lattice)

                   library(RColorBrewer)

                   cloud(value~col+row, ..., panel.3d.cloud=panel.3dbars, col.facet='grey', xbase=0.4, ybase=0.4, scales=list(arrows=FALSE, col=1), par.settings = list(axis.line = list(col = "transparent")))

                   """

        }

        // execute R script

        r.plot(file, plotScript)

        //println "s1: "+r.eval("s1").asString()

        //display the SVG results graphic

}

def lt = null;

if (output_lexicaltable) {

        def parent = selection[0]

        println "parent="+parent.getClass()

        if (parent instanceof Part) {

                println "part="+parent

                parent = parent.getParent()

                println "partition="+parent

                lt = new LexicalTable(parent);

                lt.setData(new LexicalTableImpl(matrix as DoubleMatrix2D, rownames, colnames))

        } else {

                lt = new LexicalTable(parent);

                lt.setData(new LexicalTableImpl(matrix as DoubleMatrix2D, rownames, colnames))

        }

        lt.setUserName(schema_ursql+">"+unit_ursql)

}

if (file.exists() && file.getTotalSpace() > 0) {

        println "SVG file: "+file.getAbsolutePath()

        monitor.syncExec(new Runnable() {

                                @Override

                                public void run() { try {

                                                if (UnitsIndexMacro.this.output_histogram) {

                                                        //OpenSVGGraph.OpenSVGFile(file.getAbsolutePath(), selection.toString()+" Units")

                                                        OpenBrowser.openfile(file.getAbsolutePath())

                                                }

                                                if (UnitsIndexMacro.this.output_lexicaltable) {

                                                        CorporaView.refreshObject(corpus)

                                                        CorporaView.expand(lt)

                                                }

                                        } catch (e) { println "** ERROR: "+e }}

                        })

} else {

        println "Error: cannot show histogram result with $file"

}

/*

 d <- read.table(text=' x   y     z

 t1   5   high

 t1   2   low

 t1   4   med

 t2   8   high

 t2   1   low

 t2   3   med

 t3  50   high

 t3  12   med

 t3  35   low', header=TRUE)

 library(latticeExtra)

 cloud(y~x+z, d, panel.3d.cloud=panel.3dbars, col.facet='grey',

 xbase=0.4, ybase=0.4, scales=list(arrows=FALSE, col=1),

 par.settings = list(axis.line = list(col = "transparent")))

 */