Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2013-05-15 14:30:48 +0200 (mer., 15 mai 2013) $

// $LastChangedRevision: 2392 $

// $LastChangedBy: mdecorde $

//

package org.txm.macroproto.importer;

import javax.xml.stream.XMLStreamReader;

import org.txm.scripts.importer.RemoveTag;

import org.txm.importer.ApplyXsl2;

import org.txm.importer.ValidateXml;

import org.txm.scripts.importer.xml.importer;

import org.txm.scripts.importer.xml.compiler;

import org.txm.scripts.importer.xml.pager;

import org.txm.objects.*;

import org.txm.tokenizer.TokenizerClasses;

import org.txm.utils.*

import org.txm.utils.io.*;

import org.txm.*;

import org.txm.importer.scripts.xmltxm.*;

import org.txm.utils.i18n.*;

import org.txm.metadatas.*;

import javax.xml.stream.*;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

import org.txm.macro.edition.EditionUpdaterMacro

import org.txm.macro.edition.AddFacsAttributeMacro

import org.txm.macro.edition.FacsEditionBuilderMacro

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import groovy.lang.Binding;

import groovy.util.GroovyScriptEngine;

import org.txm.rcp.commands.workspace.*

import org.txm.rcp.commands.*

String userDir = System.getProperty("user.home");

// BEGINNING OF PARAMETERS

@Field @Option(name="srcDirectory", usage="the directory containing the DOC/ODT/RTF files to convert", widget="Folder", required=true, def="srcDirectory")

File srcDirectory

@Field @Option(name="SpannedEdition", usage="edition xsl", widget="Boolean", required=false, def="true")

Boolean SpannedEdition // call EditionUpdater

// calls EditionUpdater :

// Defaults corpus=imported corpus, xslEdition:XTZ xsl, xslPages=XTZ xsl2, editionname=default, useTokenizedDirectory=false

@Field @Option(name="synoptic", usage="syn", widget="Boolean", required=false, def="true")

Boolean synoptic

// calls AddFacsAttribute + FacsEditionBuilder.

// Defaults AddFacsAttribute : imageDirectory=$SRC/img, outputDirectory=temp, element=pb, attribute=n, prefix=mediaPath

// Default FacsEditionBuilder : editionName=facs, element=pb, attribute=facs

@Field @Option(name="mediaPath", usage="absolute or relative path to images directory", widget="String", required=false, def="img")

String mediaPath

@Field @Option(name="facsEditionName", usage="facs edition name to create", widget="String", required=false, def="facs")

String facsEditionName

// Open the parameters input dialog box

if (!ParametersDialog.open(this)) return;

if (!mediaPath.endsWith("/")) mediaPath += "/"

println "Parameters: "

println "        srcDirectory: $srcDirectory"

println "        SpannedEdition: $SpannedEdition"

println "        synoptic: $synoptic"

println "        mediaPath: $mediaPath"

println "        facsEditionName: $facsEditionName"

def MONITOR;

boolean debug = false;

File importFile = new File(srcDirectory, "import.xml")

if (!importFile.exists()) {

        println "No import.xml file found in $srcDirectory"

        println "Aborting."

        return;

}

BaseParameters params = new BaseParameters(importFile);

params.load()

params.rootDir = srcDirectory.getAbsolutePath();

TokenizerClasses.loadFromNode(params.getTokenizerElement(params.getCorpusElement()));

if (params == null) { println "no parameters. Aborting"; return; }

String corpusname = params.getCorpusName();

Element corpusElem = params.corpora.get(corpusname);

String basename = params.name;

String rootDir = params.rootDir;

String lang = corpusElem.getAttribute("lang");

String model = lang

String encoding = corpusElem.getAttribute("encoding");

boolean annotate = "true" == corpusElem.getAttribute("annotate");

String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")

def xslParams = params.getXsltParams(corpusElem);

File srcDir = new File(rootDir);

File binDir = new File(Toolbox.getTxmHomePath(), "corpora/"+basename);

binDir.deleteDir();

binDir.mkdirs();

if (!binDir.exists()) {

        println "Could not create binDir "+binDir

        return;

}

File txmDir = new File(binDir, "txm/$corpusname");

txmDir.deleteDir();

txmDir.mkdirs();

File propertyFile = new File(rootDir, "import.properties")//default

Properties props = new Properties();

String[] metadatasToKeep;

String textSortAttribute = null;

String paginationElement = "pb";

boolean normalizeMetadata = false;

String ignoredElements = null;

boolean stopIfMalformed = false;

println "Trying to read import properties file: "+propertyFile

if (propertyFile.exists() && propertyFile.canRead()) {

        InputStreamReader input = new InputStreamReader(new FileInputStream(propertyFile) , "UTF-8");

        props.load(input);

        input.close();

        if(props.getProperty("sortmetadata") != null)

                textSortAttribute = props.get("sortmetadata").toString();

        if (props.getProperty("editionpage") != null)

                paginationElement = props.get("editionpage").toString();

        if (props.getProperty("normalizemetadata") != null)

                normalizeMetadata = Boolean.parseBoolean(props.get("normalizemetadata").toString());

        if (props.getProperty("ignoredelements") != null)

                ignoredElements = props.get("ignoredelements").toString();

        if (props.getProperty("stopifmalformed") != null)

                stopIfMalformed = Boolean.parseBoolean(props.get("stopifmalformed").toString());

        println "import properties: "

        println " sort metadata: "+textSortAttribute

        println " edition page tag: "+paginationElement

        println " normalize attributes: "+normalizeMetadata

        println " ignored elements: "+ignoredElements

        println " stop if a XML source is malformed: "+stopIfMalformed

}

File allMetadataFile = Metadatas.findMetadataFile(srcDir);

// Apply XSL

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(1, "APPLYING XSL")

if (xsl != null && xslParams != null && xsl.trim().length() > 0) {

        if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))

        // return; // error during process

        srcDir = new File(binDir, "src");

        println ""

}

String[] roots = [ System.getProperty("user.home")+"/TXM/scripts/macro/" ];

GroovyScriptEngine gse = new GroovyScriptEngine(roots, monitor.getClass().getClassLoader());

File jardir = new File(System.getProperty("user.home")+"/TXM/scripts/lib"); //$NON-NLS-1$

if (jardir.exists() && jardir.isDirectory()) {

        for (File f: jardir.listFiles(new FilenameFilter() {

                public boolean accept(File dir, String name) {

                        return name.endsWith(".jar"); //$NON-NLS-1$

                }

        })) {

                gse.getGroovyClassLoader().addURL(f.toURI().toURL());

        }

}

// Call AddFacsAttribute - must be called before starting the importer step

// Defaults AddFacsAttribute : imageDirectory=$SRC/img, outputDirectory=temp, element=pb, attribute=facs, prefix=mediaPath

if (synoptic) {

        println "-- Adding the $paginationElement@facs attributes"

        File srcDirTemp = new File(binDir, "facs_src")

        srcDirTemp.mkdir() // create a new src directory that contains the modified src files

        Binding b = new Binding();

        b.setVariable("args", ["monitor":monitor, "sourceDirectory":srcDir, "imageDirectory":new File(srcDir, "img"), "outputDirectory":srcDirTemp, "element":paginationElement, "attribute":"facs", "prefix":mediaPath]);

        gse.run "org/txm/macro/edition/AddFacsAttributeMacro.groovy", b

        println "switching srcDir to $srcDirTemp"

        srcDir = srcDirTemp;

}

// copy xml+dtd files

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

List<File> srcfiles = srcDir.listFiles();

if (srcfiles != null)

for (int i = 0 ; i < srcfiles.size() ; i++) {// check XML format, and copy file into binDir

        File f = srcfiles.get(i)

        if (f.getName().equals("import.xml") || f.getName().matches("metadata\\.....?") || f.getName().endsWith(".properties")) {

                srcfiles.remove(i);

                i--;

                continue;

        }

        if (ValidateXml.test(f)) {

                FileCopy.copy(f, new File(txmDir, f.getName()));

        } else {

                println "Won't process file "+f;

        }

}

if (txmDir.listFiles() == null) {

        println "No txm file to process"

        return;

}

//get metadatas values from CSV

Metadatas metadatas; // text metadatas

println "Trying to read metadatas from: "+allMetadataFile

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (allMetadataFile.exists()) {

        File copy = new File(binDir, allMetadataFile.getName())

        if (!FileCopy.copy(allMetadataFile, copy)) {

                println "Error: could not create a copy of metadata file "+allMetadataFile.getAbsoluteFile();

                return;

        }

        metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(), Toolbox.getMetadataColumnSeparator(), Toolbox.getMetadataTextSeparator(), 1)

} else {

        println "no metadata file: "+allMetadataFile

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(5, "IMPORTER")

println "-- IMPORTER - Reading source files"

def imp = new importer();

imp.doValidation(true) // change this to not validate xml

imp.doTokenize(true) // change this, to not tokenize xml

imp.setStopIfMalformed(stopIfMalformed);

if (!imp.run( srcDir, binDir, txmDir, basename, ignoredElements, lang)) {

        println "import process stopped";

        return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA")

if (metadatas != null) {

        println("-- INJECTING METADATA - "+metadatas.getHeadersList()+" in texts of directory "+new File(binDir,"txm"))

        for (File infile : txmDir.listFiles()) {

                print "."

                File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());

                if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {

                        outfile.delete();

                } else {

                        if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile

                        if (!infile.exists()) {

                                println "Error: could not replace $infile by $outfile"

                                return false;

                        }

                }

        }

        println ""

}

List<File> files = txmDir.listFiles()

if (files == null || files.size() == 0) {

        return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")

println "-- ANNOTATE - Running NLP tools"

boolean annotationSuccess = false;

if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {

        annotationSuccess = true;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(25, "COMPILING")

println "-- COMPILING - Building Search Engine indexes"

def c = new compiler();

if(debug) c.setDebug();

//c.setCwbPath("~/TXM/cwb/bin");

c.setOptions(textSortAttribute, normalizeMetadata);

c.setAnnotationSuccess(annotationSuccess)

c.setLang(lang);

if (!c.run(binDir, txmDir, corpusname, null, srcfiles, metadatas)) {

        println "import process stopped";

        return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(25, "EDITION")

println "-- EDITION - Building edition"

new File(binDir,"HTML/$corpusname").deleteDir();

new File(binDir,"HTML/$corpusname").mkdirs();

File outdir = new File(binDir,"/HTML/$corpusname/default/");

outdir.mkdirs();

List<File> filelist = txmDir.listFiles();

Collections.sort(filelist);

def second = 0

println "Paginating texts: "+filelist

for (File txmFile : filelist) {

        print "."

        String txtname = txmFile.getName();

        int i = txtname.lastIndexOf(".");

        if(i > 0) txtname = txtname.substring(0, i);

        List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

        List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

        Element text = params.addText(corpusElem, txtname, txmFile);

        def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, 30000, basename, paginationElement);

        Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");

        for (i = 0 ; i < ed.getPageFiles().size();) {

                File f = ed.getPageFiles().get(i);

                String wordid = ed.getIdx().get(i);

                params.addPage(edition, ""+(++i), wordid);

        }

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

File paramFile = new File(binDir, "import.xml");

DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;

// Call FacsEditionBuilder

// Default FacsEditionBuilder : editionName=facs, element=pb, attribute=facs

if (synoptic) {

        println "-- Building the '$facsEditionName' image edition..."

        b = new Binding();

        b.setVariable("args", ["monitor":monitor,"params":params, "binDirectory":binDir, "txmDirectory":txmDir,"corpus":srcDirectory, "editionName":facsEditionName, "tag":paginationElement, "attribute":"facs"]);

        gse.run "org/txm/macro/edition/FacsEditionBuilderMacro.groovy", b

        //copy images into the binary corpus

        if (mediaPath.length() > 0 && !mediaPath.startsWith("http")) {

                File binImageDirectory = new File(binDir,"HTML/$basename/$facsEditionName/$mediaPath")

                println "copying images into binary corpus: $binImageDirectory ..."

                FileCopy.copyFiles(new File(srcDirectory, "img"), binImageDirectory)

                println "copy done"

        } else {

                println "Note: the $facsEditionName edition images are not embedded in the binary corpus"

        }

}

// Call EditionUpdater

// corpus=imported corpus, xslEdition:XTZ xsl, xslPages=XTZ xsl2, editionname=default, useTokenizedDirectory=false

if (SpannedEdition) {

        println "-- Building Spanned edition..."

        Binding b = new Binding();

        b.setVariable("args", ["monitor":monitor,"params":params, "binDirectory":binDir, "txmDirectory":txmDir,"corpus":srcDirectory, "xslEdition":"txm-edition-xtz.xsl", "xslPages":"txm-edition-page-split.xsl", "editionname":"default", "useTokenizedDirectory":false]);

        gse.run "org/txm/macro/edition/EditionUpdaterMacro.groovy", b

}

// // create the ".txm" file

// try {

//         File exportzip = new File(binDir.getParent(), binDir.getName()+".txm")

//         println "creating the $exportzip binary corpus file..."

//         Zip.compress(binDir, exportzip);

//         println "done."

// } catch (IOException e) {

//         println "Error during binary file creation: $e"

// }

// load corpus

println "Loading corpus..."

try {LoadBinaryCorpus.loadBase(binDir)} catch(Exception e777){AddBases.loadBase(binDir, monitor)} // LoadBinaryCorpus does not exist if TXM version is < 0.7.7

Toolbox.restartWorkspace();

Toolbox.restartSearchEngine();

monitor.syncExec(new Runnable() {

        @Override

        public void run() {

                println "Reloading corpora view..."

                RestartTXM.reloadViews();

                println "import done."

        }

});