Laboratoire ICAR » Plateforme TXM

package org.txm.scripts.importer.xtz

import org.txm.scripts.filters.Tokeniser.SimpleTokenizerXml

import java.io.File;

import net.sf.saxon.style.XSLParam;

import org.txm.core.preferences.TBXPreferences;

import org.txm.importer.ApplyXsl2

import org.txm.scripts.importer.CleanFile

import org.txm.scripts.importer.MileStoneProjection

import org.txm.importer.ValidateXml

import org.txm.*

import javax.xml.parsers.SAXParser

import javax.xml.parsers.SAXParserFactory

import javax.xml.stream.*

import org.w3c.dom.Element

import org.xml.sax.XMLReader

import org.txm.utils.xml.DomUtils

import org.txm.metadatas.Metadatas

import org.txm.objects.*

import org.txm.importer.scripts.xmltxm.Xml2Ana

import org.txm.utils.ConsoleProgressBar

import org.txm.utils.io.*

import org.txm.importer.xtz.*

class XTZImporter extends Importer {

        public boolean cancelNow = false;

        public def filesToProcess = []

        public File frontXSLResultDirectory;

        public File tokenizedDirectory;

        public File sentencedDirectory;

        public String xslPath;

        public def xslParams;

        public String lang;

        public String wordTag;

        public boolean doTokenizeStep = false;

        public Metadatas metadata = null; // text metadata

        public XTZImporter(ImportModule module) {

                super(module);

        }

        public Metadatas getMetadata() {

                return metadata;

        }

        public void process() {

                Project project = module.getProject();

                File binDir = module.getBinaryDirectory();

                String corpusname = project.getName();

                def srcFiles = [];

                def files = inputDirectory.listFiles();

                if (files == null) {

                        reason = "No file to process in "+inputDirectory

                        return; // no file to process in the directory

                }

                for (File f : files) {

                        if (f.isHidden() || f.isDirectory())

                                continue;

                        else if (f.getName().endsWith(".properties"))

                                continue;

                        else if (f.getName().equals("import.xml"))

                                continue;

                        else if (f.getName().endsWith(".csv"))

                                continue;

                        else if (f.getName().endsWith(".dtd"))

                                continue;

                        else if (f.getName().endsWith(".xsl"))

                                continue;

                        else if (f.getName().endsWith("~"))

                                continue;

                        else if (f.getName().startsWith("."))

                                continue;

                        srcFiles << f

                }

                if (srcFiles.size() == 0) {

                        reason = "No suitable file to process in "+inputDirectory

                        return; // no file to process in the directory

                }

                // prepare front XSL if any

                xslPath = project.getFrontXSL()

                xslParams = project.getXsltParameters()

                lang = project.getLang();

                wordTag = project.getTokenizerWordElement()

                this.doTokenizeStep = project.getDoTokenizerStep()

                //prepare metadata if any

                File allMetadataFile = Metadatas.findMetadataFile(inputDirectory);

                if (allMetadataFile.exists()) {

                        File copy = new File(binDir, allMetadataFile.getName())

                        if (!FileCopy.copy(allMetadataFile, copy)) {

                                println "Error: could not create a copy of the metadata file "+allMetadataFile.getAbsoluteFile();

                                return;

                        }

                        metadata = new Metadatas(copy,

                                        Toolbox.getPreference(TBXPreferences.METADATA_ENCODING),

                                        Toolbox.getPreference(TBXPreferences.METADATA_COLSEPARATOR),

                                        Toolbox.getPreference(TBXPreferences.METADATA_TXTSEPARATOR), 1)

                }

                String cleanDirectories = project.getCleanAfterBuild();

                if (!doFixSurrogates()) return;

                if (!doSplitMergeXSLStep()) return;

                if (!doFrontXSLStep()) return;

                if (!doCheckXMLFileStep()) return;

                if (!doTokenizeStep()) return;

                if (!doPostTokenizeXSLStep()) return;

                if (!doEncodeMileStonesStep()) return;

                if (!doToXMLTXMStep()) return;

                if (!doInjectMetadataStep()) return;

                isSuccessFul = filesToProcess.size() > 0

                if ("true".equals(cleanDirectories)) {

                        new File(module.getBinaryDirectory(), "tokenized").deleteDir()

                        new File(module.getBinaryDirectory(), "src").deleteDir()

                        new File(module.getBinaryDirectory(), "split").deleteDir()

                }

        }

        /**

         * read from source directory and remove the surrogate chars not well managed

         * save the result in $bindir/src

         *

         *

         *

         */

        public boolean doFixSurrogates() {

                File srcDirectory = new File(module.getBinaryDirectory(), "src")

                def files = inputDirectory.listFiles()

                if (files == null) {

                        reason = "No file to process in $inputDirectory"

                        return false;

                }

                srcDirectory.deleteDir() // clean before copying

                srcDirectory.mkdir()

                for (File file : files) {

                        def outputFile = new File(srcDirectory, file.getName());

                        if (file.getName().toLowerCase().endsWith(".xml")) {

                                CleanFile.removeSurrogateFromXmlFile(file, outputFile)

                        } else {

                                FileCopy.copyFiles(file, outputFile)

                        }

                }

                inputDirectory = srcDirectory; // the files to process are now in the "src" directory

                return true;

        }

        /**

         * read from source directory and write the result in $bindir/txm

         *

         * only one XSL is applied

         *

         */

        public boolean doSplitMergeXSLStep() {

                File splitedDirectory = new File(module.getBinaryDirectory(), "split")

                splitedDirectory.deleteDir()

                File xslDirectory = new File(module.getSourceDirectory(), "xsl/1-split-merge")

                println "-- Split-Merge XSL Step with $xslDirectory"

                def xslFiles = xslDirectory.listFiles()

                if (xslDirectory.exists() && xslFiles != null && xslFiles.size() > 0) {

                        splitedDirectory.mkdir()

                        xslParams["output-directory"] = splitedDirectory.getAbsoluteFile().toURI().toString();

                        xslFiles.sort()

                        for (File xslFile : xslFiles) {

                                if (xslFile.isDirectory() || xslFile.isHidden() || !xslFile.getName().endsWith(".xsl")) continue;

                                //if (!xslFile.getName().matches("[1-9]-.+")) continue;

                                if (ApplyXsl2.processImportSources(xslFile, inputDirectory.listFiles(), xslParams)) {

                                        println ""

                                } else {

                                        reason = "Fail to apply split-merge XSL: $xslPath"

                                }

                                break; // THERE IS ONLY ONE XSL FILE TO APPLY

                        }

                        inputDirectory = splitedDirectory; // the files to process are now in the "src" directory

                } else {

                        //println "Nothing to do."

                }

                return true;

        }

        public boolean doEncodeMileStonesStep() {

                filesToProcess = new File(module.getBinaryDirectory(), "tokenized").listFiles()

                def milestonesString = module.getProject().getTextualPlan("MileStones")

                if (milestonesString.length() == 0) return true;

                def milestones = milestonesString.split(",")

                //def milestones = ["lb", "pb", "cb"]

                //println "milestones=$milestones type="+milestones.getClass()+" size="+milestones.size()

                if (milestones.size() == 0) return true;

                println "-- Encoding milestone $milestones into XML files..."

                ConsoleProgressBar cpb = new ConsoleProgressBar(milestones.size())

                for (def milestone : milestones) {

                        milestone = milestone.trim()// just in case

                        cpb.tick()

                        for (File inputFile : filesToProcess) {

                                File outputFile = File.createTempFile("tmp", inputFile.getName(),inputFile.getParentFile());

                                println "$inputFile, 'text', $wordTag, $milestone, $outputFile"

                                MileStoneProjection msp = new MileStoneProjection(inputFile, "text", wordTag, milestone)

                                if (!msp.process(outputFile)) {

                                        println "Fail to encode $milestone in $inputFile"

                                        return false

                                } else {

                                        if (inputFile.delete()) {

                                                FileCopy.copy(outputFile, new File(outputFile.getParent(), "copy.xml"))

                                                outputFile.renameTo(inputFile)

                                        } else {

                                                println "Fail to encode $milestone in ${inputFile}: could not replace the file."

                                                return false

                                        }

                                }

                        }

                }

                println ""

                return true;

        }

        /**

         * read from $inputDirectory and write the result in $bindir/txm

         *

         */

        public boolean doFrontXSLStep() {

                //filesToProcess = inputDirectory.listFiles();

                File frontXSLdirectory = new File(module.getSourceDirectory(), "xsl/2-front")

                println "-- Front XSL Step with the $frontXSLdirectory directory."

                def xslFiles = frontXSLdirectory.listFiles()

                xslParams["output-directory"] = outputDirectory.getAbsoluteFile().toURI().toString();

                if (frontXSLdirectory.exists() && xslFiles != null && xslFiles.size() > 0) {

                        xslFiles.sort()

                        for (File xslFile : xslFiles) {

                                if (xslFile.isDirectory() || xslFile.isHidden() || !xslFile.getName().endsWith(".xsl")) continue;

                                //if (!xslFile.getName().matches("[1-9]-.+")) continue;

                                if (ApplyXsl2.processImportSources(xslFile, inputDirectory, outputDirectory, xslParams)) {

                                        inputDirectory = outputDirectory; // the files to process are now in the "txm" directory

                                        println ""

                                } else {

                                        reason = "Fail to apply front XSL: $xslPath"

                                        return false;

                                }

                        }

                } else {

                        //println "Nothing to do."

                }

                return true;

        }

        public boolean doCheckXMLFileStep() {

                filesToProcess = []

                println "-- Checking XML files for well-formedness."

                def files = inputDirectory.listFiles()

                if (files == null || files.size() == 0) {

                        println "No XML file (*.xml) to process. Aborting"

                        return false;

                }

                files = files.sort()

                ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

                for (File f : files) {

                        cpb.tick()

                        if (f.getName().toLowerCase().endsWith(".xml"))

                                if (!f.isHidden() && f.canRead() && !f.getName().endsWith(".properties")

                                && !f.getName().startsWith("metadata") && !f.isDirectory())

                        {

                                if (!ValidateXml.test(f)) {

                                        continue;

                                } else {

                                        if (!f.getName().equals("import.xml"))

                                                filesToProcess << f

                                }

                        }

                }

                println ""

                if (filesToProcess.size() == 0) {

                        reason = "No file suitable to process in "+inputDirectory

                        return false;

                } else {

                        return true;

                }

        }

        public boolean doTokenizeStep() {

                new File(module.getBinaryDirectory(),"tokenized").mkdir()

                String outSideTextTagsRegex = "";

                String outSideTextTagsAndKeepContentRegex = "";

                String noteRegex = "";

                // get the element names to ignore

                String e1 = module.getProject().getTextualPlan("OutSideTextTags")

                def split  = e1.split(",")

                for (String s : split) {

                        outSideTextTagsRegex += "|"+s.trim()

                }

                if (outSideTextTagsRegex.trim().length() > 0) {

                        outSideTextTagsRegex = outSideTextTagsRegex.substring(1) // remove the first "|"

                }

                String e2 = module.getProject().getTextualPlan("OutSideTextTagsAndKeepContent")

                def split2  = e2.split(",")

                for (String s : split2) {

                        outSideTextTagsAndKeepContentRegex += "|"+s.trim()

                }

                if (outSideTextTagsAndKeepContentRegex.trim().length() > 0) {

                        outSideTextTagsAndKeepContentRegex = outSideTextTagsAndKeepContentRegex.substring(1) // remove the first "|"

                }

                String e3 = module.getProject().getTextualPlan("Note")

                def split3  = e3.split(",")

                for (String s : split3) {

                        noteRegex += "|"+s.trim()

                }

                if (noteRegex.trim().length() > 0) {

                        noteRegex = noteRegex.substring(1) // remove the first "|"

                }

                //if (wordTag != "w") {

                if (!doTokenizeStep) {

                        println "No tokenization do to."

                        // ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())

                        for (File f : filesToProcess) {

                                File outfile = new File(module.getBinaryDirectory(),"tokenized/"+f.getName());

                                FileCopy.copy(f, outfile);

                        }

                        return true;

                } else {

                        println "-- Tokenizing "+filesToProcess.size()+" files"

                        ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())

                        for (File f : filesToProcess) {

                                cpb.tick()

                                File infile = f;

                                File outfile = new File(module.getBinaryDirectory(),"tokenized/"+f.getName());

                                SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)

                                if (outSideTextTagsRegex != null && outSideTextTagsRegex.trim().length() > 0) {

                                        tokenizer.setOutSideTextTags(outSideTextTagsRegex)

                                }

                                if (outSideTextTagsAndKeepContentRegex != null && outSideTextTagsAndKeepContentRegex.trim().length() > 0) {

                                        tokenizer.setOutSideTextTagsAndKeepContent(outSideTextTagsAndKeepContentRegex)

                                }

                                if (noteRegex != null && noteRegex.trim().length() > 0) {

                                        tokenizer.setNote(noteRegex)

                                }

                                // tokenize !

                                if (!tokenizer.process()) {

                                        println("Failed to process "+f)

                                        outfile.delete()

                                }

                        }

                }

                filesToProcess = new File(module.getBinaryDirectory(),"tokenized").listFiles()

                println ""

                return true;

        }

        /**

         * read from $bindir/tokenized and write the result in $bindir/tokenized

         *

         */

        public boolean doPostTokenizeXSLStep() {

                filesToProcess = new File(module.getBinaryDirectory(),"tokenized").listFiles()

                File xslDirectory = new File(module.getSourceDirectory(), "xsl/3-posttok")

                def xslFiles = xslDirectory.listFiles()

                if (!xslDirectory.exists() || xslFiles == null || xslFiles.size() == 0) return true;

                println "-- Posttokenize XSL Step with $xslDirectory"

                return ApplyXsl2.processWithMultipleXSL(filesToProcess, xslDirectory, xslParams)

        }

        public boolean doInjectMetadataStep() {

                if (metadata != null) {

                        println("-- INJECTING METADATA - "+metadata.getHeadersList()+" in texts of directory "+outputDirectory)

                        ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())

                        for (File infile : filesToProcess) {

                                cpb.tick()

                                File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());

                                if (metadata.injectMetadatasInXml(infile, outfile, "text", null)) {

                                        if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile

                                        if (!infile.exists()) {

                                                println "Error: could not replace $infile by $outfile"

                                                return false;

                                        }

                                } else {

                                        outfile.delete(); // fail

                                }

                        }

                        cpb.done()

                }

                return true;

        }

        public boolean doToXMLTXMStep() {

                filesToProcess = new File(module.getBinaryDirectory(),"tokenized").listFiles()

                ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())

                println("-- Building XML-TXM ("+filesToProcess.size()+" files)")

                for (File f : filesToProcess) {

                        //ArrayList<String> milestones = new ArrayList<String>();

                        cpb.tick()

                        File file = f;

                        String txmfile = f.getName();

                        def correspType = new HashMap<String,String>()

                        def correspRef = new HashMap<String,String>()

                        //il faut lister les id de tous les respStmt

                        def respId = [];

                        //fait la correspondance entre le respId et le rapport d'execution de l'outil

                        def applications = new HashMap<String,HashMap<String,String>>();

                        //fait la correspondance entre le respId et les attributs type de la propriété ana du w txm

                        //pour construire les ref vers les taxonomies

                        def taxonomiesUtilisees = new HashMap<String,String[]>();

                        //associe un id d'item avec sa description et son URI

                        def itemsURI = new HashMap<String,HashMap<String,String>>();

                        //informations de respStmt

                        //resps (respId <voir ci-dessus>, [description, person, date])

                        def resps = new HashMap<String,String[]>();

                        //lance le traitement

                        def builder = new Xml2Ana(file);

                        builder.setWordTag(module.getProject().getTokenizerWordElement())

                        builder.setConvertAllAtrtibutes true;

                        builder.setCorrespondances(correspRef, correspType);

                        builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)

                        //builder.setAddTEIHeader();

                        if (!builder.process(new File(outputDirectory, txmfile))) {

                                println("Failed to process "+f);

                                new File(outputDirectory,txmfile).delete();

                        }

                }

                filesToProcess = outputDirectory.listFiles();

                cpb.done();

                return true;

        }

        @Override

        public void checkFiles() {

                //isSuccessFul = isSuccessFul && outputDirectory.listFiles() != null;

        }

        @Override

        public void cancel() {

                cancelNow = true;

        }

}