Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

// This file is part of the TXM platform.

//

// The TXM platform is free software: you can redistribute it and/or modif y

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// The TXM platform is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

//

//

//

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$

//

package org.txm.scripts.importer.transcriber;

import java.io.File;

import org.txm.importer.*

import org.txm.importer.scripts.xmltxm.*;

import org.txm.*;

import org.txm.core.engines.*;

import org.txm.objects.*;

import org.txm.utils.i18n.*;

import org.txm.utils.*;

import org.txm.scripts.importer.*;

import org.txm.metadatas.*;

import org.txm.utils.io.FileCopy;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

//PARAMETERS

boolean removeInterviewer = false;//if true the transcription of speakers (en1 and enq2) defined in metadatas file will be ignored

boolean includeComments = false;

boolean ignoreTranscriberMetadata = false;

int csvHeaderNumber = 1;

int maxlines = 200;

String userDir = System.getProperty("user.home");

def MONITOR;

boolean debug = org.txm.utils.logger.Log.isPrintingErrors();

BaseParameters params;

try {params = paramsBinding;MONITOR=monitor} catch (Exception)

{        println "DEV MODE";//exception means we debug

        debug = true

        params = new BaseParameters(new File(userDir, "xml/anapovoas/import.xml"))

        params.load()

        if (!org.txm.Toolbox.isInitialized()) {

                Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

                Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));

                Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

                Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, "\t");

                Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "");

                Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));

        }

}

if (params == null) { println "no parameters. Aborting"; return; }

String corpusname = params.getCorpusName();

Element corpusElem = params.corpora.get(corpusname);

String basename = params.name;

String rootDir = params.rootDir;

String lang = corpusElem.getAttribute("lang");

String model = lang

String encoding = corpusElem.getAttribute("encoding");

boolean annotate = "true" == corpusElem.getAttribute("annotate");

String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")

def xslParams = params.getXsltParams(corpusElem);

int wordsPerPage = params.getWordsPerPage("default")

String page_element = params.getPageElement("default")

boolean build_edition = params.getDoEdition("default")

File srcDir = new File(rootDir);

File binDir = new File(Toolbox.getTxmHomePath(),"corpora/"+basename);

binDir.deleteDir();

binDir.mkdirs();

if (!binDir.exists()) {

        println "Could not create binDir "+binDir

        return;

}

File txmDir = new File(binDir,"txm/$corpusname");

txmDir.deleteDir();

txmDir.mkdirs();

//get metadata values from CSV

Metadatas metadatas; // text metadata

File allMetadataFile = Metadatas.findMetadataFile(srcDir);

println "Trying to read metadatas values from: "+allMetadataFile

if (allMetadataFile.exists()) {

        File copy = new File(binDir, allMetadataFile.getName())

        if (!FileCopy.copy(allMetadataFile, copy)) {

                println "Error: could not create a copy of metadata file "+allMetadataFile.getAbsoluteFile();

                return;

        }

        metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(),

                Toolbox.getMetadataColumnSeparator(),

                Toolbox.getMetadataTextSeparator(), 1)

}

else

        println "no metadata file: "+allMetadataFile

File propertyFile = new File(srcDir, "import.properties")//default

Properties props = new Properties();

String[] metadatasToKeep;

if (propertyFile.exists() && propertyFile.canRead()) {

        FileInputStream input = new FileInputStream(propertyFile);

        props.load(input);

        input.close();

        if (props.getProperty("removeInterviewer") != null)

                removeInterviewer = Boolean.parseBoolean(props.get("removeInterviewer").toString());

        if (props.getProperty("ignoreTranscriberMetadata") != null)

                ignoreTranscriberMetadata = Boolean.parseBoolean(props.get("ignoreTranscriberMetadata").toString());

        if (props.getProperty("metadataList") != null)

                metadatasToKeep = props.get("metadataList").toString().split("|");

        if (props.getProperty("csvHeaderNumber") != null)

                csvHeaderNumber = props.get("csvHeaderNumber").toString().split("|");

        //if (props.getProperty("includeComments") != null)

        //        includeComments = props.get("includeComments").toString();

        println "import properties: "

        println " removeInterviewer: "+removeInterviewer

        println " ignoreTranscriberMetadata: "+ignoreTranscriberMetadata

        println " metadataToKeep: "+metadatasToKeep

        println " ignored csvHeaderSize: "+csvHeaderNumber

        //println " includeComments: "+includeComments

}

// Apply XSL

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(1, "XSL")

if (xsl != null && xsl.trim().length() > 0) {

        if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src")))

                srcDir = new File(binDir, "src");

        println ""

}

try {

        // select only trs files

        String ext = "trs";

        ArrayList<File> trsfiles = srcDir.listFiles(); //find all trs files

        if (trsfiles  == null) {

                println ("No files in "+srcDir.getAbsolutePath())

                return false;

        }

        for (int i = 0 ; i < trsfiles.size() ; i++) {

                File f = trsfiles.get(i);

                if (!f.getName().endsWith(ext) || !f.canRead() || f.isHidden()) {

                        trsfiles.remove(i)

                        i--;

                }

        }

        if (trsfiles.size() == 0) {

                println ("No transcription file (*.trs) found in "+srcDir.getAbsolutePath()+". Aborting.")

                return false;

        }

        if (MONITOR != null) MONITOR.worked(1, "IMPORTER")

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        println "-- IMPORTER"

        def imp = new importer(trsfiles, binDir, txmDir, metadatas, lang) //put result in the txm folder of binDir

        if (!imp.run()) {

                println "Failed to prepare files - Aborting";

                return;

        }

        if (MONITOR != null) MONITOR.worked(20)

        println "-- Xml Validation"

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        for (File infile : txmDir.listFiles()) {

                if (!ValidateXml.test(infile)) {

                        println "$infile : Validation failed";

                        infile.delete();

                }

        }

        if (MONITOR != null) MONITOR.worked(5)

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        println "-- Remove interviewer: "+removeInterviewer

        if (removeInterviewer)         {

                if (metadatas == null) {

                        println "Can't remove interviewer without a metadata.csv file defining who are the interviewers."

                } else {

                        println "Removing some speakers in "+txmDir.listFiles().length+" file(s)"

                        for (File infile : txmDir.listFiles()) {

                                String filename = infile.getName();

                                int idx = filename.indexOf(".xml");

                                if (idx > 0)

                                        filename = filename.substring(0, idx);

                                ArrayList<Pair<String, String>> metas = metadatas.get(filename)

                                //println "filename=$filename metas= $metas"

                                for (Pair p : metas) {

                                        if (p.getFirst().startsWith("enq")) {

                                                new RemoveSpeaker(infile, infile, p.getFirst())

                                        }

                                }

                        }

                }

        }

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")

        println "-- ANNOTATE - Running NLP tools"

        boolean annotationSuccess = false;

if (annotate) {

        def engine = Toolbox.getEngineManager(EngineType.ANNOTATION).getEngine("TreeTagger")

        if (engine.processDirectory(txmDir, binDir, ["lang":model])) {

                annotationSuccess = true;

        }

}

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        if (MONITOR != null) MONITOR.worked(25, "COMPILING")

        println "--COMPILING - Building Search Engine indexes"

        trsfiles = txmDir.listFiles();

        def comp = new compiler()

        if(debug) comp.setDebug();

        comp.removeInterviewers(removeInterviewer);

        comp.setIgnoreTranscriberMetadata(ignoreTranscriberMetadata);

        comp.setAnnotationSucces(annotationSuccess)

        if (!comp.run(trsfiles, corpusname, "default", binDir)) {

                println "Failed to compile files";

                return;

        }

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        File htmlDir = new File(binDir,"HTML/$corpusname");

        htmlDir.deleteDir()

        htmlDir.mkdirs();

        if (build_edition) {

                if (MONITOR != null) MONITOR.worked(20, "EDITION")

                println "-- EDITION - Building editions"

                List<File> filelist = txmDir.listFiles();

                Collections.sort(filelist);

                def second = 0

                println "Paginating texts: "

                for (File txmFile : filelist) {

                        print "."

                        String txtname = txmFile.getName();

                        int i = txtname.lastIndexOf(".");

                        if(i > 0) txtname = txtname.substring(0, i);

                        List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

                        List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

                        Element text = params.addText(corpusElem, txtname, txmFile);

                        def ed = new pager(txmFile, htmlDir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element, metadatas);

                        Element edition = params.addEdition(text, "default", htmlDir.getAbsolutePath(), "html");

                        for (i = 0 ; i < ed.getPageFiles().size();) {

                                File f = ed.getPageFiles().get(i);

                                String wordid = ed.getIdx().get(i);

                                params.addPage(edition, ""+(++i), wordid);

                        }

                        if (ed.getPageFiles().size() > 0) {

                                Element editionBD = params.addEdition(text, "onepage", htmlDir.getAbsolutePath(), "html");

                                params.addPage(editionBD, "1", ed.getIndexes().get(0));

                        }

                }

                //copy transcriber.css

                File cssfile = new File(Toolbox.getTxmHomePath(), "css/transcriber.css")

                if (cssfile.exists() && htmlDir.exists()) {

                        FileCopy.copy(cssfile, new File(htmlDir, "transcriber.css"));

                        FileCopy.copy(cssfile, new File(htmlDir, "onepage/transcriber.css"));

                        FileCopy.copy(cssfile, new File(htmlDir, "default/transcriber.css"));

                }

        }

}

catch (Exception e){org.txm.utils.logger.Log.printStackTrace(e);}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

File paramFile = new File(binDir, "import.xml");

DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;