Laboratoire ICAR » Plateforme TXM

package org.txm.test

/**

 * Main.

 *

 * @param args the args

 */

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2011-11-25 11:30:11 +0100 (ven., 25 nov. 2011) $

// $LastChangedRevision: 2069 $

// $LastChangedBy: mdecorde $

//

import java.util.ArrayList;

import org.txm.utils.DeleteDir;

import org.txm.*;

import org.txm.lexicaltable.core.functions.LexicalTable

import org.txm.objects.Base;

import org.txm.ca.core.functions.CA

import org.txm.cah.core.functions.AHC

import org.txm.concordance.core.functions.Concordance

import org.txm.cooccurrence.core.functions.Cooccurrence

import org.txm.functions.*;

import org.txm.functions.diagnostic.*;

import org.txm.functions.ca.*;

import org.txm.functions.index.*;

import org.txm.functions.specificities.*;

import org.txm.functions.concordances.*;

import org.txm.functions.concordances.comparators.*;

import org.txm.referencer.core.functions.Referencer;

import org.txm.searchengine.cqp.ReferencePattern

import org.txm.searchengine.cqp.corpus.*;

import org.txm.searchengine.cqp.corpus.query.*;

testDir = new File(System.getProperty("user.home"), "TXM/testrelease");

DeleteDir.deleteDirectory testDir;

testDir.mkdir()

//get corpus

csv = [];

firstExecution = 0;

def process(String CORPUSNAME, String QUERY1, String QUERY2) {

        String ENCODING = "UTF-8"

        String COLSEPARATOR = "\t"

        String TXTSEPARATOR = ""

        int i = 0;

        def corpora = CorpusManager.getCorpusManager().getCorpora()

        MainCorpus corpus = CorpusManager.getCorpusManager().getCorpus(CORPUSNAME)

        //println "CORPUS: "+corpus

        Query query = new Query(QUERY1) //"\"..............*\"", "\"..........*\""

        Query query2 = new Query(QUERY2)

        File exporttestdir = new File(testDir, corpus.getName());

        File reportFile = new File(exporttestdir, "report.csv")

        DeleteDir.deleteDirectory exporttestdir;

        exporttestdir.mkdir()

        //println "Results are saved in dir: "+ exporttestdir

        // word properties

        def word_property = corpus.getProperty("word")

        // structure properties

        StructuralUnit text_su = corpus.getStructuralUnit("text")

        StructuralUnit s_su = corpus.getStructuralUnit("s")

        Property text_id_property = text_su.getProperty("id")

        ReferencePattern referencePattern = new ReferencePattern().addProperty(text_id_property)

        long time;

        // START START START START

        if (firstExecution == 0)

                csv << ["object", "size", "nPart", "command", "query", "query freq", "$CORPUSNAME $QUERY1 $QUERY2 mode "+Toolbox.getParam(Toolbox.CQI_NETWORK_MODE)]

        else

                csv[i++] << "$CORPUSNAME $QUERY1 $QUERY2 mode "+Toolbox.getParam(Toolbox.CQI_NETWORK_MODE)

        // INFORMATIONS

        print " INFO"

        time = System.currentTimeMillis();

        Diagnostic diag = new Diagnostic(corpus, 20)

        diag.stepGeneralInfos();

        diag.stepLexicalProperties();

        diag.stepStructuralUnits();

        diag.toHTML(new File(exporttestdir, "diag"))

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Informations", "no query", "no freq", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // LEXICON

        print " LEX"

        time = System.currentTimeMillis();

        corpus.getLexicon(word_property).toTxt(new File(exporttestdir, "lexpos"), ENCODING, COLSEPARATOR, TXTSEPARATOR);

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Lexicon", "no query", "no freq", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // INDEX

        print " INDEX"

        time = System.currentTimeMillis();

        IndexSample index = new IndexSample(corpus, query, [word_property])

        index.toTxt(new File(exporttestdir, "indexlemmafuncj"), ENCODING, COLSEPARATOR, TXTSEPARATOR)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Index", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // REFERENCER

        print " REF"

        time = System.currentTimeMillis();

        Referencer referencer = new Referencer(corpus, query, word_property, [text_id_property], true);

        referencer.getQueryMatches()

        referencer.getQueryindexes()

        referencer.groupPositionsbyId()

        referencer.toTxt(new File(exporttestdir, "referencer"), ENCODING)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Referencer", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // CONCORDANCE

        print " CONC"

        time = System.currentTimeMillis();

        Concordance concordance = new Concordance(corpus, query, word_property, [word_property, word_property], referencePattern, referencePattern, 15, 15)

        concordance.toTxt(new File(exporttestdir,"concj"), Concordance.Format.CONCORDANCE)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Concordances", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // COOCCURRENCE WORD WINDOW

        print " COOC"

        time = System.currentTimeMillis();

        Cooccurrence cooc = new Cooccurrence(corpus, query, [word_property], null, 21, 1, 1, 11, 2, 3, 1, false);

        cooc.process();

        cooc.toTxt(new File(exporttestdir, "cooc_wordwindow"), ENCODING)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences words", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // COOCCURRENCE SENTENCE WINDOW

        print " COOC"

        time = System.currentTimeMillis();

        Cooccurrence cooc2 = new Cooccurrence(corpus, query, [word_property], s_su,2, 1, 1, 1, 2, 3,1, false);

        cooc2.process();

        cooc2.toTxt(new File(exporttestdir, "cooc_swindow"), ENCODING)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences structures", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // INDEX

        print " INDEX"

        time = System.currentTimeMillis();

        index = new IndexSample(corpus, query2, [word_property])

        index.toTxt(new File(exporttestdir, "indexlemmafuncj"), ENCODING, COLSEPARATOR, TXTSEPARATOR)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Index", query2, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // REFERENCER

        print " REF"

        time = System.currentTimeMillis();

        referencer = new Referencer(corpus, query2, word_property, [text_id_property], true);

        referencer.getQueryMatches()

        referencer.getQueryindexes()

        referencer.groupPositionsbyId()

        referencer.toTxt(new File(exporttestdir, "referencer"), ENCODING)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Referencer", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // CONCORDANCE

        print " CONC"

        time = System.currentTimeMillis();

        concordance = new Concordance(corpus, query2, word_property, [word_property, word_property], referencePattern, referencePattern, 15, 15)

        concordance.toTxt(new File(exporttestdir,"concj"), Concordance.Format.CONCORDANCE)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Concordances", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // COOCCURRENCE WORD WINDOW

        print " COOC"

        time = System.currentTimeMillis();

        cooc = new Cooccurrence(corpus, query2, [word_property], null, 21, 1, 1, 11, 2, 3, 1, false);

        cooc.process();

        cooc.toTxt(new File(exporttestdir, "cooc_wordwindow"), ENCODING)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences words", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // COOCCURRENCE SENTENCE WINDOW

        print " COOC"

        time = System.currentTimeMillis();

        cooc2 = new Cooccurrence(corpus, query2, [word_property], s_su,2, 1, 1, 1, 2, 3,1, false);

        cooc2.process();

        cooc2.toTxt(new File(exporttestdir, "cooc_swindow"), ENCODING)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences structures", query, index.getT(), (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // SUBCORPORA

        print " SUBCORPUS"

        time = System.currentTimeMillis();

        Corpus DGcorpus = corpus.createSubcorpus(text_su, text_id_property, "01_DeGaulle", "dgsubcorpus")

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Subcorpus", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // PARTITIONS

        print " PARTITIONS"

        time = System.currentTimeMillis();

        Partition discours_types = corpus.createPartition(text_su, text_id_property)

        Partition discours_dates = corpus.createPartition(text_su, text_id_property)

        if (firstExecution == 0)

                csv << [corpus.getName(), corpus.getSize(), 1, "Partition 2x", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // LEXICAL TABLE

        print " LT"

        time = System.currentTimeMillis();

        LexicalTable table = discours_types.getLexicalTable(word_property, 2);

        table.exportData(new File(exporttestdir, "type_LT"), COLSEPARATOR, TXTSEPARATOR);

        if (firstExecution == 0)

                csv << [discours_types.getName(), corpus.getSize(), discours_types.getNPart(), "LT part", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // SPECIF SUBCORPUS

//        time = System.currentTimeMillis();

//        print " SPECIF"

//        SpecificitesResult specifresult2 = org.txm.functions.specificities.Specificites.specificites(DGcorpus.getParent(), DGcorpus, word_property)

//        specifresult2.toTxt(new File(exporttestdir,"dgsub_specifloc"), ENCODING, COLSEPARATOR, TXTSEPARATOR)

//        if (firstExecution == 0)

//                csv << [DGcorpus.getName(), corpus.getSize(), 1, "Specif sub", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

//        else

//                csv[i++] << (System.currentTimeMillis()-time)/1000

//

//        // SPECIF LEXICAL TABLE

//        print " SPECIF"

//        time = System.currentTimeMillis();

//        SpecificitesResult specifresult3 = org.txm.functions.specificities.Specificites.specificites(table);

//        specifresult3.toTxt(new File(exporttestdir,"speciftype"), ENCODING, COLSEPARATOR, TXTSEPARATOR)

//        if (firstExecution == 0)

//                csv << [table.getName(), corpus.getSize(), table.getNColumns(), "specif LT", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

//        else

//                csv[i++] << (System.currentTimeMillis()-time)/1000

        // AFC PARTITION

        print " AFC"

        time = System.currentTimeMillis();

        CA ca = new CA(discours_dates, word_property, 0 ,9999999)

        ca.stepLexicalTable();

        ca.stepSortTableLexical();

        ca.compute()

        ca.toSVGFactorialMap(new File(exporttestdir,"cadates"), true, true)

        ca.toSVGSingularValues(new File(exporttestdir,"cadates_singularvalues"))

        if (firstExecution == 0)

                csv << [discours_dates.getName(), corpus.getSize(), discours_dates.getNPart(), "AFC part", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // AFC LEXICAL TABLE

        print " AFC"

        time = System.currentTimeMillis();

        CA ca2 = new CA(table);

        ca2.compute()

        ca2.toSVGFactorialMap(new File(exporttestdir,"cadates"), true, true)

        ca2.toSVGSingularValues(new File(exporttestdir,"cadates_singularvalues"))

        if (firstExecution == 0)

                csv << [table.getName(), corpus.getSize(), table.getNColumns(), "specif LT", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        // CAH

        print " CAH"

        time = System.currentTimeMillis();

        AHC cah = new AHC(ca, true, AHC.getDefaultMethod(), AHC.getDefaultMetric(), 4, false)

        cah.stepCompute()

        cah.toSVG(new File(exporttestdir, "dates_cah"),RDevice.SVG);

        if (firstExecution == 0)

                csv << [discours_dates.getName(), corpus.getSize(), discours_dates.getNPart(), "CAH ca table", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]

        else

                csv[i++] << (System.currentTimeMillis()-time)/1000

        firstExecution++;

        println ""

}

println "restarting TBX..."

Toolbox.setParam(Toolbox.CQI_NETWORK_MODE, "false")

println "MEMORY MODE: "+Toolbox.restart();

process("DISCOURS", "\"..............*\"", "\"..........*\"") // first time

println "MEMORY MODE: "+Toolbox.restart();

process("DISCOURS", "\"..............*\"", "\"..........*\"")

println "restarting TBX..."

Toolbox.setParam(Toolbox.CQI_NETWORK_MODE, "true")

println "NETWORK MODE: "+Toolbox.restart();

process("DISCOURS", "\"..............*\"", "\"..........*\"")

println "NETWORK MODE: "+Toolbox.restart();

process("DISCOURS", "\"..............*\"", "\"..........*\"")

// add TOTAL line

int[] totaux = ["","","","","","",""]

for (int i = 7 ; i < csv.size() ; i++) {

        int total = 0;

        for (def line : csv) {

                total += line[i]

        }

        totaux << total

}

csv << totaux;

// WRITE ALL RESULTS

for (def line : csv) {

        for (int i = 0 ; i < firstExecution ; i++) {

                def item = line[i]

                if (i > 0) print "\t"+item

                else print item

        }

        println ""

}