Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

// This file is part of the TXM platform.

//

// The TXM platform is free software: you can redistribute it and/or modif y

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// The TXM platform is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

//

//

//

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$

//

package org.txm.scripts.importer.limsi;

import org.eclipse.swt.widgets.*;

import org.txm.scripts.importer.transcriber.compiler.*;

import org.txm.scripts.importer.transcriber.pager.*;

import org.txm.scripts.importer.transcriber.*;

import java.io.File;

import org.txm.importer.scripts.xmltxm.*;

import org.txm.stat.utils.ConsoleProgressBar

import org.txm.*;

import org.txm.core.engines.*;

import org.txm.objects.*;

import org.txm.utils.i18n.*;

import org.txm.utils.*;

import org.txm.scripts.importer.*;

import org.txm.utils.io.FileCopy;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

import org.txm.rcp.commands.*

//PARAMETERS

boolean removeInterviewer = false;//if true the transcription of speakers (en1 and enq2) defined in metadatas file will be ignored

boolean includeComments = false;

boolean ignoreTranscriberMetadata = false;

int csvHeaderNumber = 1;

int maxlines = 200;

String userDir = System.getProperty("user.home");

def MONITOR;

boolean debug = org.txm.utils.logger.Log.isPrintingErrors();

BaseParameters params;

try {params = paramsBinding;MONITOR=monitor} catch (Exception)

{        println "DEV MODE";//exception means we debug

        debug = true

        //

                params = new BaseParameters(new File(userDir, "xml/limsi/files/import.xml"))

                println "loading "+params.paramFile

                params.load()

        if (!org.txm.Toolbox.isInitialized()) {

                Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

                Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

                Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");

                Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");

                Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));

        }

}

if (params == null) { println "no parameters. Aborting"; return; }

String corpusname = params.getCorpusName();

Element corpusElem = params.corpora.get(corpusname);

String basename = params.name;

String rootDir = params.rootDir;

String lang = corpusElem.getAttribute("lang");

String model = lang

String encoding = corpusElem.getAttribute("encoding");

boolean annotate = "true" == corpusElem.getAttribute("annotate");

String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")

def xslParams = params.getXsltParams(corpusElem);

File srcDir = new File(rootDir);

File binDir = new File(Toolbox.getTxmHomePath(),"corpora/"+basename);

//binDir.deleteDir();

binDir.mkdirs();

if (!binDir.exists()) {

        println "Could not create binDir "+binDir

        return;

}

File txmDir = new File(binDir,"txm/$corpusname");

//txmDir.deleteDir();

txmDir.mkdirs();

try {

        // select only xml files

        String ext = ".xml";

        ArrayList<File> limsiFiles = srcDir.listFiles(); //find all trs files

        if (limsiFiles  == null) {

                println ("No files in "+srcDir.getAbsolutePath())

                return false;

        }

        // remove non XML files

        for (int i = 0 ; i < limsiFiles.size() ; i++) {

                File f = limsiFiles.get(i);

                if (f.getName().equals("import.xml") || !f.getName().endsWith(ext) || !f.canRead() || f.isHidden()) {

                        limsiFiles.remove(i)

                        i--;

                }

        }

        if (limsiFiles.size() == 0) {

                println ("No transcriptions in "+srcDir.getAbsolutePath())

                return false;

        } else {

                println "Number of transcriptions: "+limsiFiles.size();

        }

        if (MONITOR != null) MONITOR.worked(1, "IMPORTER")

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        println "-- IMPORTER"

        def imp = new importer(limsiFiles, binDir, txmDir) //put result in the txm folder of binDir

        if (!imp.run()) {

                println "Failed to prepare files - Aborting";

                return;

        }

        if (MONITOR != null) MONITOR.worked(20)

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")

        println "-- ANNOTATE CQP- Running NLP tools"

        boolean annotationSuccess = false;

        if (annotate && new AnnotateCQP().run(binDir, txmDir, model+".par")) {

                annotationSuccess = true;

        }

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        if (MONITOR != null) MONITOR.worked(25, "COMPILING")

        println "--COMPILING - Building Search Engine indexes"

        limsiFiles = txmDir.listFiles();

        def comp = new compiler()

        if(debug) comp.setDebug();

        comp.setAnnotationSuccess(annotationSuccess)

        if (!comp.run(binDir, txmDir, corpusname)) {

                println "Failed to compile files";

                return;

        }

        // create HTML directory but don't build an edition

        File htmlDir = new File(binDir,"HTML/$corpusname");

        htmlDir.mkdirs();

        def filelist;

        if (annotationSuccess)

                filelist = new File(binDir, "annotations").listFiles();

        else

                filelist = txmDir.listFiles();

        filelist.sort();

        def second = 0

        println "Registering texts: "+filelist.size()

        ConsoleProgressBar cpb = new ConsoleProgressBar(filelist.size());

        for (File txmFile : filelist) {

                cpb.tick()

                String txtname = txmFile.getName();

                int i = txtname.lastIndexOf(".");

                if(i > 0) txtname = txtname.substring(0, i);

                Element text = params.addText(corpusElem, txtname, txmFile);

        }

}

catch (Exception e){org.txm.utils.logger.Log.printStackTrace(e);}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

File paramFile = new File(binDir, "import.xml");

DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;

// load corpus

println "Loading corpus..."

try {LoadBinaryCorpus.loadBase(binDir)} catch(Exception e777){try {AddBases.loadBase(binDir, MONITOR)} catch(Exception e){}} // LoadBinaryCorpus does not exist if TXM version is < 0.7.7

Toolbox.restartWorkspace();

Toolbox.restartSearchEngine();

Display.getDefault().syncExec(new Runnable() {

        @Override

        public void run() {

                println "Reloading corpora view..."

                RestartTXM.reloadViews();

                println "import done."

        }

});