Laboratoire ICAR » Plateforme TXM

package org.txm.scripts.importer.srcmf

// srcdir

// txml/master.xml

// txml/*.xml

// txm/*.xml

// 1 import xml-txm

import ims.tiger.gui.tigerregistry.TIGERRegistry;

import ims.tiger.index.writer.IndexBuilderErrorHandler;

import ims.tiger.index.writer.SimpleErrorHandler;

import ims.tiger.index.writer.XMLIndexing;

import ims.tiger.system.*;

import javax.xml.stream.*;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.io.PrintStream;

import java.net.URL;

import java.util.HashMap;

import java.util.logging.FileHandler

import java.io.File;

import org.apache.log4j.BasicConfigurator;

import org.txm.export.ts.*;

import org.txm.importer.ApplyXsl2;

import org.txm.utils.xml.DomUtils;

import org.txm.importer.ValidateXml;

import org.txm.importer.XPathResult;

import org.txm.importer.xmltxm.compiler;

import org.txm.importer.xml.pager;

import org.txm.objects.*;

import org.txm.utils.*

import org.txm.utils.io.*;

import org.txm.*;

import org.txm.importer.xmltxm.*;

import org.txm.metadatas.*;

import org.txm.utils.i18n.*;

import org.w3c.dom.Element

import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilderFactory

String userDir = System.getProperty("user.home");

boolean debug = false

def MONITOR;

BaseParameters params;

try {params = paramsBinding;MONITOR=monitor} catch (Exception)

{        println "DEV MODE";//exception means we debug

        debug = true

        params = new BaseParameters(new File(userDir, "xml/slethgier/import.xml"))

        params.load()

        if (!org.txm.Toolbox.isInitialized()) {

                Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

                Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));

                Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));

                Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

                Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");

                Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");

                Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));

        }

}

if (params == null) { println "no parameters. Aborting"; return; }

String corpusname = params.getCorpusName();

Element corpusElem = params.corpora.get(corpusname);

String basename = params.name;

String rootDir = params.rootDir;

String lang = corpusElem.getAttribute("lang");

String model = lang

String encoding = corpusElem.getAttribute("encoding");

boolean annotate = "true" == corpusElem.getAttribute("annotate");

String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")

def xslParams = params.getXsltParams(corpusElem);

File txmSrcDir = new File(rootDir, "xml-txm");

File tigerSrcDir = new File(rootDir, "tiger-xml");

File headerFile = new File(rootDir, "tiger-xml/header.xml");

File binDir = new File(Toolbox.getTxmHomePath(),"corpora/"+basename);

binDir.deleteDir();

binDir.mkdirs();

if (!binDir.exists()) {

        println "Could not create binDir "+binDir

        return;

}

if (!txmSrcDir.exists()) {

        println "Error: txm src dir does not exists: "+txmSrcDir

        return;

}

if (!tigerSrcDir.exists()) {

        println "Error: tiger src dir does not exists: "+tigerSrcDir

        return;

}

File txmDir = new File(binDir,"txm/$corpusname");

txmDir.deleteDir();

txmDir.mkdirs();

// copy txm files

List<File> srcfiles = txmSrcDir.listFiles();

for (File f : srcfiles) {// check XML format, and copy file into binDir

        if (f.getName().equals("import.xml") || f.getName().matches("metadata\\.....?") || f.getName().endsWith(".properties"))

                continue;

        if (ValidateXml.test(f)) {

                FileCopy.copy(f, new File(txmDir, f.getName()));

        } else {

                println "Won't process file "+f;

        }

}

if (txmDir.listFiles() == null) {

        println "No txm file to process"

        return;

}

List<File> filelist = txmDir.listFiles();

Collections.sort(filelist);

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(25, "COMPILING")

println "-- COMPILING - Building Search Engine indexes"

println "binDir: $binDir"

println "txmDir: $txmDir"

def c = new compiler();

if (debug) c.setDebug();

c.setLang(lang);

//c.setSortMetadata(sortMetadata)

if (!c.run(binDir, txmDir, basename, corpusname, filelist)) {

        println "import process stopped";

        return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "EDITION")

println "-- EDITION - Building edition"

new File(binDir,"HTML/$corpusname").deleteDir();

new File(binDir,"HTML/$corpusname").mkdirs();

File outdir = new File(binDir,"/HTML/$corpusname/default/");

outdir.mkdirs();

def second = 0

println "Paginating text: "

for (File srcfile : filelist) {

        String txtname = srcfile.getName();

        int i = txtname.lastIndexOf(".");

        if(i > 0) txtname = txtname.substring(0, i);

        List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

        List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

        Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);

        if (second) { print(", ") }

        if (second > 0 && (second++ % 5) == 0) println ""

        print(srcfile.getName());

        def ed = new pager(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, 500, basename, "pb");

        Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");

        for (i = 0 ; i < ed.getPageFiles().size();) {

                File f = ed.getPageFiles().get(i);

                String wordid = ed.getIdx().get(i);

                params.addPage(edition, ""+(++i), wordid);

        }

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

File paramFile = new File(binDir, "import.xml");

DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;

// TMR Modifications start here

File tigerDir = new File(binDir, "tiger");

tigerDir.mkdir();

File master = new File(tigerSrcDir, "master.xml");

File txmSrcFile = txmSrcDir.listFiles()[0]; // only one text file 

// Run SRCMF cmd line corpus import script 

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

def importer = new SrcmfImporter()

def tigerXmlAll = importer.doAllButPnc(master, txmSrcFile, headerFile, binDir)

// Create TigerXml-POS-PNC

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

println "create TIGER XML POS PNC"

PunctInjectImport injector;

if (Toolbox.isInitialized()) {

        // get idx from CQP

        // INCOMPLETE

        // def injector = new PunctInjectImport(tmpFile, words)

        println "Error: the loader is not ready to be launched from RCP"

        return;

} else {

        injector = new PunctInjectImport(tigerXmlAll, txmSrcFile)

}

injector.outputFile = new File(tigerDir, "TigerPnc.xml")

injector.process()

// Create the tigersearch.logprop file (used to launch TIGERSearch)

File logprop = new File(tigerDir, "tigersearch.logprop");

logprop.withWriter("UTF-8") { writer ->

        writer.write("""# Default log configuration of the TIGERSearch suite

log4j.rootLogger=WARN,Logfile

log4j.logger.ims.tiger.gui.tigersearch.TIGERSearch=INFO

log4j.appender.Logfile=org.apache.log4j.RollingFileAppender

log4j.appender.Logfile.File=\${user.home}/tigersearch/tigersearch.log

log4j.appender.Logfile.MaxFileSize=500KB

log4j.appender.Logfile.MaxBackupIndex=1

log4j.appender.Logfile.layout=org.apache.log4j.PatternLayout

log4j.appender.Logfile.layout.ConversionPattern=%5r %-5p [%t] %c{2} - %m%n""")        

}

// Run TigerRegistry

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

BasicConfigurator.configure();

String uri = tigerXmlAll.getAbsolutePath();

File tigerBinDir = new File(tigerDir, corpusname)

tigerBinDir.mkdir()

try {

        IndexBuilderErrorHandler handler = new SimpleErrorHandler(tigerBinDir.getAbsolutePath());

        XMLIndexing indexing = new XMLIndexing(corpusname,uri, tigerBinDir.getAbsolutePath(), handler,false);

        indexing.startIndexing();

}

catch (IOException e) { System.out.println("IO: "+e.getMessage()); }

catch (SAXException e) { System.out.println("SAX: "+e.getMessage()); }

tigerXmlAll.delete()