Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

//

// This file is part of the TXM platform.

//

// The TXM platform is free software: you can redistribute it and/or modif y

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// The TXM platform is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

//

//

//

//

//

// $LastChangedDate: 2012-02-13 11:42:24 +0100 (lun., 13 févr. 2012) $

// $LastChangedRevision: 2126 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer.frantext;

import javax.xml.stream.XMLStreamReader;

import org.txm.importer.ApplyXsl2;

import org.txm.scripts.importer.bfm.importer;

import org.txm.scripts.importer.bfm.compiler;

import org.txm.scripts.importer.bfm.pager;

import org.txm.objects.*;

import org.txm.*;

import org.txm.core.engines.*;

import org.txm.utils.i18n.*;

import org.txm.importer.scripts.xmltxm.*;

import org.txm.scripts.importer.*;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

import org.txm.utils.*

import org.txm.utils.io.*;

String userDir = System.getProperty("user.home");

boolean debug = org.txm.utils.logger.Log.isPrintingErrors();

def MONITOR;

BaseParameters params;

try {params = paramsBinding;MONITOR=monitor} catch (Exception)

{        println "DEV MODE";//exception means we debug

        debug = true

        params = new BaseParameters(new File(userDir, "xml/TESTS/frantext/import.xml"))

        params.load()

        if (!org.txm.Toolbox.isInitialized()) {

                //rootDir = userDir+"/xml/TESTS/alceste";  // directory which contains the source file

                Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

                Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

                Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");

                Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");

                Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));

        }

}

if (params == null) { println "no parameters. Aborting"; return; }

String corpusname = params.getCorpusName();

Element corpusElem = params.corpora.get(corpusname);

String basename = params.name;

String rootDir = params.rootDir;

String lang = corpusElem.getAttribute("lang");

String model = lang

String encoding = corpusElem.getAttribute("encoding");

boolean annotate = "true" == corpusElem.getAttribute("annotate");

String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")

def xslParams = params.getXsltParams(corpusElem);

int wordsPerPage = params.getWordsPerPage("default")

boolean build_edition = params.getDoEdition("default")

File srcDir = new File(rootDir);

File binDir = new File(Toolbox.getTxmHomePath(),"corpora/"+basename);

binDir.deleteDir();

binDir.mkdirs();

if (!binDir.exists()) {

        println "Could not create binDir "+binDir

        return;

}

File txmDir = new File(binDir,"txm/$corpusname");

txmDir.deleteDir();

txmDir.mkdirs();

// BFM XPATH PARAMETERS

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(1, "READ XPATH PARAMETERS")

File paramDeclarationFile = new File(srcDir, "import.properties");

Properties metadataXPath = new Properties();

if (paramDeclarationFile.exists() && paramDeclarationFile.canRead()) {

        InputStreamReader input = new InputStreamReader(new FileInputStream(paramDeclarationFile) , "UTF-8");

        metadataXPath.load(input);

        input.close();

        if (!metadataXPath.containsKey("titre"))

                println "Warning: parameters property file does not contain the 'titre' metadata"

        if (!metadataXPath.containsKey("forme"))

                println "Warning: parameters property file does not contain the 'forme' metadata in param values. The default value is 'prose'"

}

else

        println "No '$paramDeclarationFile' file found"

// Apply XSL

if (MONITOR != null) MONITOR.worked(5, "APPLYING XSL")

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

File xslFile = new File(Toolbox.getTxmHomePath(), "xsl/txm-filter-teifrantext-teibfm.xsl") // force xsl

if (!ApplyXsl2.processImportSources(xslFile, srcDir, new File(binDir, "xsl"))) {

        println "Error: failed to apply xsl $xsl. Aborting import."

        return; // error durring process

}

srcDir = new File(binDir, "xsl");

// copy txm files

println "-- VALIDATION - checking XML source files well-formedness"

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

List<File> srcfiles = srcDir.listFiles();

for (File f : srcfiles) { // check XML format, and copy file into binDir

        if (f.isHidden() || f.getName().equals("import.xml") || f.getName().matches("metadata\\.....?") || f.getName().endsWith(".properties"))

                continue;

        if (ValidateXml.test(f)) {

                FileCopy.copy(f, new File(txmDir, f.getName()));

        } else {

                println "Won't process file "+f;

        }

}

if (MONITOR != null) MONITOR.worked(5)

if (txmDir.listFiles() == null) {

        println "No txm file to process"

        return;

}

if (MONITOR != null) MONITOR.worked(1, "IMPORTER")

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

println "-- IMPORTER - Reading source files"

def imp = new importer()

if (!imp.run(srcDir, binDir, txmDir, basename, metadataXPath)) {

        println "import process stopped";

        return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")

println "-- ANNOTATE - Running NLP tools - $model model"

boolean annotate_status = true;

if (annotate) {

        def engine = Toolbox.getEngineManager(EngineType.ANNOTATION).getEngine("TreeTagger")

        if (engine.processDirectory(txmDir, binDir, ["lang":model])) {

                annotate_status = true;

        }

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(25, "COMPILING")

println "-- COMPILING - Building Search Engine indexes"

def c = new compiler();

if (debug) c.setDebug();

//c.setCwbPath("~/TXM/cwb/bin");

c.setLang(lang);

c.setAnnotationDone(annotate_status)

if (!c.run(binDir, txmDir, corpusname, metadataXPath)) {

        println "import process stopped";

        return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

new File(binDir,"HTML/$corpusname").deleteDir();

new File(binDir,"HTML/$corpusname").mkdirs();

if (build_edition) {

        println "-- EDITION"

        if (MONITOR != null) MONITOR.worked(25, "EDITION")

        File outdir = new File(binDir,"/HTML/$corpusname/default/");

        outdir.mkdirs();

        files = c.getOrderedTxmFiles();

        println("Building editions: "+files.size()+" files" );

        for (File txmFile : files) {

                print "."

                String txtname = txmFile.getName();

                int idx = txtname.lastIndexOf(".");

                if(idx > 0) txtname = txtname.substring(0, idx);

                List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

                List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);//["'","(","[","{","«"];

                Element text = params.addText(corpusElem, txtname, txmFile);

                def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, corpusname, metadataXPath);

                Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");

                for (i = 0 ; i < ed.getPageFiles().size();) {

                        File f = ed.getPageFiles().get(i);

                        String wordid = ed.getIdx().get(i);

                        params.addPage(edition, ""+(++i), wordid);

                }

        }

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

File paramFile = new File(binDir, "import.xml");

DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;