Laboratoire ICAR » Plateforme TXM

/**

 * Main.

 *

 * @param args the args

 */

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-03-29 09:51:35 +0200 (mar. 29 mars 2016) $

// $LastChangedRevision: 3185 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.scripts;

/**

 * test tnt Wrapper :

 *  add NLP annotations

 *  build result matrix

 *

 */

import org.txm.utils.*;

import org.txm.scripts.importer.*;

import org.txm.scripts.importer.RGAQCJ.*;

// TODO: Auto-generated Javadoc

/* (non-Javadoc)

 * @see groovy.lang.Script#run()

 */

String rootDir = "~/xml/rgaqcj/TnT/";

String modelsDir = rootDir+"models/";

String textsDir = rootDir+"texts/";

String projsDir = rootDir+"proj/";

String srcDir = rootDir+"src/"

String xslDir = rootDir+"xsl/"

String anaDir = rootDir+"anainline/"

new File(modelsDir).mkdir();

new File(textsDir).mkdir();

new File(projsDir).mkdir();

def texts =["roland","artu","qjm","commyn1","jehpar","rgaqcj"]

//def texts =["commyn1","jehpar"]

String initiales = "RAQCJZ"

//import src

def srcfiles = ["roland.xml","artu.xml","qjm.xml","commyn1.xml","jehpar.xml"];

def anafiles = ["roland-ana.xml","artu-ana.xml","qjm-ana.xml","commyn1-ana.xml","jehpar-ana.xml"];

def xslfiles = ["bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl"];

/*

 for(int i=0 ; i < xslfiles.size() ; i++)

 {

 String xslfile = xslDir+xslfiles[i];

 String infile = srcDir+srcfiles[i];

 String outfile = anaDir+anafiles[i];

 println("create xml-txm from "+srcfiles[i]);

 ApplyXsl a = new ApplyXsl(xslfile);

 a.process(infile,outfile);

 }

 //Import to CWB

 BuildXmlRGAQCJ.process( anafiles,  rootDir)

 */

//build CWBDecode wrapper

ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/cwb-decode-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/CwbDecode.groovy"));

GroovyClassLoader gcl = new GroovyClassLoader();

gcl.addClasspath(".");

String registryPath = rootDir+"/registry";

String cwbdecodeexecDir = "~/Bureau/trunkCWB/cwb-3.0/utils/"

Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/CwbDecode.groovy"));

def cwbdecodewrapper = clazz.newInstance(cwbdecodeexecDir);

def pAttributesTrain = ["word","cat"];

def pAttributesTag = ["word"];

def sAttributes = ["s"];

cwbdecodewrapper.setC();

cwbdecodewrapper.setP(pAttributesTag);

cwbdecodewrapper.setS(sAttributes);

cwbdecodewrapper.setr(registryPath);

cwbdecodewrapper.debug(true);

int[] nbmots = [0,35306,134170,173526,198929,228149];

//create file to proj on

for(int i = 0 ; i < texts.size() ; i++)

{

        String text = texts[i];

        println("creation TTsrc "+text);

        //build src files

        cwbdecodewrapper.setP(pAttributesTag);

        if(!(initiales[i]+"").equals("Z"))

        {

                if(i > 0)

                        cwbdecodewrapper.sets(nbmots[i]+1)

                else

                        cwbdecodewrapper.sets(nbmots[i])

                cwbdecodewrapper.sete(nbmots[i+1])

        }

        else

        {

                cwbdecodewrapper.unsets();

                cwbdecodewrapper.unsete();

        }

        FileOutputStream fos = new FileOutputStream(textsDir+texts[i]+".t");

        PrintStream ps = new PrintStream(fos);

        def out = System.out;

        System.setOut(ps);

        if(System.getProperty("os.name").contains("Windows"))

                cwbdecodewrapper.cwbdecodeexe("RGAQCJ")

        else

                cwbdecodewrapper.cwbdecode("RGAQCJ")

        System.setOut(out);

        //build train files

        cwbdecodewrapper.setP(pAttributesTrain);

        fos = new FileOutputStream(modelsDir+texts[i]+".t");

        ps = new PrintStream(fos);

        System.setOut(ps);

        if(System.getProperty("os.name").contains("Windows"))

                cwbdecodewrapper.cwbdecodeexe("RGAQCJ")

        else

                cwbdecodewrapper.cwbdecode("RGAQCJ")

        System.setOut(out);

}

//need to replace <s> by nothing and </s> by \n

String encoding = "UTF-8"

for(String text : texts)

{

        //patch src files

        File f = new File(textsDir,text+".t");

        File temp = new File("tempFileCVScleaner")

        println("patch texts files "+f+": rmv <s> and replace </s>");

        Reader reader = new InputStreamReader(new FileInputStream(f),encoding);

        Writer writer = new FileWriter(temp);

        reader.eachLine

                        {

                                if(it.trim().startsWith("</s"))

                                        writer.write("\n")

                                else if(it.trim().startsWith("<s"))

                                        writer.write("")

                                else

                                        writer.write(it+"\n")

                        }

        reader.close();

        writer.close();

        if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f

}

//need to replace <s> by nothing and </s> by \n

for(String text : texts)

{

        //patch training files

        File f = new File(modelsDir,text+".t");

        File temp = new File("tempFileCVScleaner")

        println("patch models files "+f+": rmv <s> and replace </s>");

        Reader reader = new InputStreamReader(new FileInputStream(f),encoding);

        Writer writer = new FileWriter(temp);

        reader.eachLine

                        {

                                if(it.trim().startsWith("</s"))

                                        writer.write("\n")

                                else if(it.trim().startsWith("<s"))

                                        writer.write("")

                                else

                                        writer.write(it+"\n")

                        }

        reader.close();

        writer.close();

        if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f

}

//build Tnt Wrapper

//ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/tnt-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/TnT.groovy"));

gcl = new GroovyClassLoader();

gcl.addClasspath(".");

String tntexecDir = "~/Bureau/tnt/"

clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/TnT.groovy"));

def tntwrapper = clazz.newInstance(tntexecDir);

def ttrezfiles = [];

for(String text : texts)

{

        println "Apprentissage de "+text+"..."

        tntwrapper.tntpara(new File(modelsDir,text+".t"));

        //new File(text+".123").renameTo(new File(modelsDir,text+".123"));

        //new File(text+".lex").renameTo(new File(modelsDir,text+".lex"));

        for(String target : texts)

        {

                println "Projection de "+target+" sur "+target+"..."

                for(int mode=3 ; mode <=3 ; mode++)

                {

                        println("Mode "+mode+"...")

                        tntwrapper.setu(2);

                        FileOutputStream fos = new FileOutputStream(projsDir+"model_"+text+"_target_"+target+"_mode"+mode+".t");

                        ttrezfiles.add("model_"+text+"_target_"+target+"_mode"+mode+".t")

                        PrintStream ps = new PrintStream(fos);

                        def out = System.out;

                        System.setOut(ps);

                        tntwrapper.tnt(text,new File(textsDir,target+".t"));

                        System.setOut(out);

                }

        }

}

//def ttrezfiles = ["model_roland_targetroland_mode2.t", "model_roland_targetroland_mode3.t", "model_roland_targetartu_mode2.t", "model_roland_targetartu_mode3.t", "model_roland_targetqjm_mode2.t", "model_roland_targetqjm_mode3.t", "model_roland_targetcommyn1_mode2.t", "model_roland_targetcommyn1_mode3.t", "model_roland_targetjehpar_mode2.t", "model_roland_targetjehpar_mode3.t", "model_roland_targetrgaqcj_mode2.t", "model_roland_targetrgaqcj_mode3.t", "model_artu_targetroland_mode2.t","model_artu_targetroland_mode3.t", "model_artu_targetartu_mode2.t", "model_artu_targetartu_mode3.t", "model_artu_targetqjm_mode2.t", "model_artu_targetqjm_mode3.t", "model_artu_targetcommyn1_mode2.t", "model_artu_targetcommyn1_mode3.t", "model_artu_targetjehpar_mode2.t", "model_artu_targetjehpar_mode3.t", "model_artu_targetrgaqcj_mode2.t", "model_artu_targetrgaqcj_mode3.t", "model_qjm_targetroland_mode2.t, model_qjm_targetroland_mode3.t, model_qjm_targetartu_mode2.t, model_qjm_targetartu_mode3.t, model_qjm_targetqjm_mode2.t, model_qjm_targetqjm_mode3.t, model_qjm_targetcommyn1_mode2.t, model_qjm_targetcommyn1_mode3.t, model_qjm_targetjehpar_mode2.t, model_qjm_targetjehpar_mode3.t, model_qjm_targetrgaqcj_mode2.t, model_qjm_targetrgaqcj_mode3.t, model_commyn1_targetroland_mode2.t, model_commyn1_targetroland_mode3.t, model_commyn1_targetartu_mode2.t, model_commyn1_targetartu_mode3.t, model_commyn1_targetqjm_mode2.t, model_commyn1_targetqjm_mode3.t, model_commyn1_targetcommyn1_mode2.t, model_commyn1_targetcommyn1_mode3.t, model_commyn1_targetjehpar_mode2.t, model_commyn1_targetjehpar_mode3.t, model_commyn1_targetrgaqcj_mode2.t, model_commyn1_targetrgaqcj_mode3.t, model_jehpar_targetroland_mode2.t, model_jehpar_targetroland_mode3.t, model_jehpar_targetartu_mode2.t, model_jehpar_targetartu_mode3.t, model_jehpar_targetqjm_mode2.t, model_jehpar_targetqjm_mode3.t, model_jehpar_targetcommyn1_mode2.t, model_jehpar_targetcommyn1_mode3.t, model_jehpar_targetjehpar_mode2.t, model_jehpar_targetjehpar_mode3.t, model_jehpar_targetrgaqcj_mode2.t, model_jehpar_targetrgaqcj_mode3.t, model_rgaqcj_targetroland_mode2.t, model_rgaqcj_targetroland_mode3.t, model_rgaqcj_targetartu_mode2.t, model_rgaqcj_targetartu_mode3.t, model_rgaqcj_targetqjm_mode2.t, model_rgaqcj_targetqjm_mode3.t, model_rgaqcj_targetcommyn1_mode2.t, model_rgaqcj_targetcommyn1_mode3.t", "model_rgaqcj_targetjehpar_mode2.t", "model_rgaqcj_targetjehpar_mode3.t", "model_rgaqcj_targetrgaqcj_mode2.t", "model_rgaqcj_targetrgaqcj_mode3.t"];

println ttrezfiles;

//Build process infos

//remove lines which starts with %%

for(String text : ttrezfiles)

{

        //def encoding ="UTF-8"

        println "patch proj file"+text+" : remove %% lines and blank lines AND replace n\t by only one \t";

        File f = new File(projsDir,text);

        File temp = new File("tempFileCVScleaner")

        Reader reader = new InputStreamReader(new FileInputStream(f),encoding);

        Writer writer = new FileWriter(temp);

        reader.eachLine

                        {

                                if(it.trim().startsWith("%%") || it.length() == 0)

                                        writer.write("")

                                else

                                        writer.write(it.replaceAll("(\t)+","\t")+"\n")

                        }

        reader.close();

        writer.close();

        if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f

}

for(String text : texts)

{

        //def encoding ="UTF-8"

        println "patch model file"+text+" : remove blank lines";

        File f = new File(modelsDir,text+".t");

        File temp = new File("tempFileCVScleaner")

        Reader reader = new InputStreamReader(new FileInputStream(f),encoding);

        Writer writer = new FileWriter(temp);

        reader.eachLine

                        {

                                if(it.length() == 0)

                                        writer.write("")

                                else

                                        writer.write(it+"\n")

                        }

        reader.close();

        writer.close();

        if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f

}

HSQLFunctions.clearAll();

//import proj table into hsql

for(int i=0; i < ttrezfiles.size();i++)

{

        String csvfile = rootDir+"proj/"+ttrezfiles[i];

        def argsname = ["form","cat"];

        def types = ["VARCHAR(30)","VARCHAR(30)"];

        int linenumber = HSQLFunctions.ImportOrderedCSVTable(ttrezfiles[i].replace(".",""), argsname,types,new File(csvfile),"\t","UTF-8");

        println("create Table "+ttrezfiles[i].replace(".","")+" : "+linenumber+" lines");

}

for(String text : texts)

{

        File f = new File(modelsDir,text+".t");

        def argsname = ["form","cat"];

        def types = ["VARCHAR(30)","VARCHAR(30)"];

        int linenumber = HSQLFunctions.ImportOrderedCSVTable("lexbrut_"+text.replace(".",""), argsname,types,f,"\t","UTF-8");

        println("create Table lexbrut_"+text.replace(".","")+" : "+linenumber+" lines");

}

//calc richesses lexicales

LinkedHashMap<String,ArrayList<String>> richesses = new LinkedHashMap<String,ArrayList<String>>();

richesses.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);

LinkedHashMap<String,ArrayList<String>> richessesCat = new LinkedHashMap<String,ArrayList<String>>();

richessesCat.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);

LinkedHashMap<String,ArrayList<String>> richessesocc = new LinkedHashMap<String,ArrayList<String>>();

richessesocc.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);

LinkedHashMap<String,ArrayList<String>> richessesoccCat = new LinkedHashMap<String,ArrayList<String>>();

richessesoccCat.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);

for(String text : texts)

{

        richesses.put(text, new ArrayList<String>());

        richessesCat.put(text, new ArrayList<String>());

        richessesocc.put(text, new ArrayList<String>());

        richessesoccCat.put(text, new ArrayList<String>());

}

for(String text : texts)

{

        for(String model : texts)

        {

                if(text.matches(model))

                {

                        richesses.get(text).add( "0" ) ;

                        richessesCat.get(text).add( "0") ;

                        richessesocc.get(text).add( "0" ) ;

                        richessesoccCat.get(text).add( "0" ) ;

                }

                else

                {

                        String query = "SELECT count(*) FROM ((SELECT DISTINCT form FROM lexbrut_"+text+" ) MINUS (SELECT DISTINCT form FROM lexbrut_"+model+"))"

                        println(query);

                        String query2 = "SELECT count(*) FROM ((SELECT DISTINCT cat FROM lexbrut_"+text+" ) MINUS (SELECT DISTINCT cat FROM lexbrut_"+model+"))"

                        println(query2);

                        String query3 = "SELECT count(form) FROM lexbrut_"+text+" WHERE form NOT IN (SELECT form FROM lexbrut_"+model+")"

                        println(query3);

                        String query4 = "SELECT count(cat) FROM lexbrut_"+text+" WHERE cat NOT IN (SELECT cat FROM lexbrut_"+model+")"

                        println(query4);

                        HSQLFunctions.getGroovySql().eachRow(query) {

                                def rich = it.getAt(0);

                                println("rich form :"+ rich);

                                richesses.get(text).add( ""+rich ) ;

                        }

                        HSQLFunctions.getGroovySql().eachRow(query2) {

                                def rich = it.getAt(0);

                                println("rich cat :"+ rich);

                                richessesCat.get(text).add( ""+rich ) ;

                        }

                        HSQLFunctions.getGroovySql().eachRow(query3) {

                                def rich = it.getAt(0);

                                println("richocc form :"+ rich);

                                richessesocc.get(text).add( ""+rich ) ;

                        }

                        HSQLFunctions.getGroovySql().eachRow(query4) {

                                def rich = it.getAt(0);

                                println("richocc cat :"+ rich);

                                richessesoccCat.get(text).add( ""+rich ) ;

                        }

                }

        }

}

LinkedHashMap<String,ArrayList<String>> matrix = new LinkedHashMap<String,ArrayList<String>>();

matrix.put("Text", ["T    ","Vf    ","Vc    ","TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);

for(String text : texts)

{

        matrix.put(text, new ArrayList<String>());

}

ArrayList<Integer> countOccForm = new ArrayList<Integer>();

ArrayList<Integer> countOccCat = new ArrayList<Integer>();

ArrayList<Integer> countLexForm = new ArrayList<Integer>();

ArrayList<Integer> countLexCat = new ArrayList<Integer>();

for(String text : texts)

{

        println("count occ and voc : lexbrut_"+text)

        String query1 = "SELECT count(*) FROM (SELECT form from lexbrut_"+text+")";

        String query2 = "SELECT count(*) FROM (SELECT cat from lexbrut_"+text+")";

        String query3 = "SELECT count(*) FROM (SELECT DISTINCT form from lexbrut_"+text+")";

        String query4 = "SELECT count(*) FROM (SELECT DISTINCT cat from lexbrut_"+text+")";

        HSQLFunctions.getGroovySql().eachRow(query1) {

                def rich = it.getAt(0);

                println("count occ form :"+ rich);

                countOccForm.add( ""+rich ) ;

        }

        HSQLFunctions.getGroovySql().eachRow(query2) {

                def rich = it.getAt(0);

                println("coutn occ cat :"+ rich);

                countOccCat.add( ""+rich ) ;

        }

        HSQLFunctions.getGroovySql().eachRow(query3) {

                def rich = it.getAt(0);

                println("count lex form :"+ rich);

                countLexForm.add( ""+rich ) ;

        }

        HSQLFunctions.getGroovySql().eachRow(query4) {

                def rich = it.getAt(0);

                println("count lex cat :"+ rich);

                countLexCat.add( ""+rich ) ;

        }

        //println("col1 et 2 : "+nbOccurencesTxt.get("lexbrut_"+text)+" "+nbFormLexique.get("lex_"+text))

        matrix.get(text).add( "0") ;

        matrix.get(text).add( "0") ;

        matrix.get(text).add( "0") ;

}

for(String text : texts)

{

        int textindex = 0;

        for(String target : texts)

        {

                //get occurences ordonnées de la gold

                        //get occurences ordonnées de la projection

                        //compar

                String proj = "model_"+text+"_target_"+target+"_mode"+3+"t"

                println("comp "+proj+" VS lexbrut_"+target);

                String query = "SELECT count(*) FROM ((SELECT n,form,cat FROM lexbrut_"+target+" ) MINUS (SELECT n,form,cat FROM "+proj+"))"

                //HSQLFunctions.executeQuery(query);

                //println(query)

                int ttotal=0;

                int total ;

                HSQLFunctions.getGroovySql().eachRow(query) {

                        def dif = it.getAt(0);

                        /*if((""+initiales[id]) != "Z")

                         {

                         total = nbmots[id+1] - nbmots[id];

                         ttotal += total;

                         }

                         else

                         total= ttotal;

                         */

                        //total = nbOccurencesTxt.get("lexbrut_"+initiales[id])

                        println("textindex : "+textindex);

                        total = Integer.parseInt(countOccForm[textindex]);

                        println("dif "+dif+"/ tot "+total+" = "+((float)dif/(float)total));

                        Float perf = ((float)dif/(float)total)*100f;

                        matrix.get(target).add( ""+perf ) ;

                }

                textindex++;

        }

}

println("Matrice d'erreur : ");

for(String k : matrix.keySet())

{

        print(k)

        for(String f : matrix.get(k))

                print("\t"+f)

        println()

}

println("richesse lexiques en Form : ");

for(String k : richesses.keySet())

{

        print(k)

        for(String f : richesses.get(k))

                print("\t"+f)

        println()

}

println("richesse lexiques en Cat : ");

for(String k : richessesCat.keySet())

{

        print(k)

        for(String f : richessesCat.get(k))

                print("\t"+f)

        println()

}

println("richesse occurance en Form : ");

for(String k : richessesocc.keySet())

{

        print(k)

        for(String f : richessesocc.get(k))

                print("\t"+f)

        println()

}

println("richesse occurance en Cat : ");

for(String k : richessesoccCat.keySet())

{

        print(k)

        for(String f : richessesoccCat.get(k))

                print("\t"+f)

        println()

}

println("RichBrutes        roland        artu        qjm        comm        jehpar        rgaqcj");

print("Focc")

for(int i=0;i < countOccForm.size() ; i++)

        print("\t"+countOccForm[i]);

print("\nCocc")

for(int i=0;i < countOccCat.size() ; i++)

        print("\t"+countOccCat[i]);

print("\nFvoc")

for(int i=0;i < countLexForm.size() ; i++)

        print("\t"+countLexForm[i]);

print("\nCvoc")

for(int i=0;i < countLexCat.size() ; i++)

        print("\t"+countLexCat[i]);

println()