Laboratoire ICAR » Plateforme TXM

package org.txm.scripts.importer.xtz

import java.io.File;

import org.txm.*

import org.txm.scripts.importer.SAttributesListener

import org.txm.importer.cwb.*

import org.txm.utils.ConsoleProgressBar

import org.txm.core.preferences.TXMPreferences

import org.txm.core.preferences.TBXPreferences

import org.txm.libs.cqp.CQPLibPreferences

import javax.xml.stream.*

import org.txm.importer.xtz.*

import org.txm.objects.*

import org.txm.searchengine.cqp.corpus.*

class XTZCompiler extends Compiler {

        SAttributesListener sattrsListener; // store scanned structures

        private def anatypes = new HashSet<String>() // store scanned word attributes

        String regPath;

        String corpusname;

        String wtag;

        boolean doNormalizeAttributeValues = false;

        boolean doNormalizeAnaValues = true;

        public XTZCompiler(ImportModule module) {

                super(module);

                corpusname = module.getProject().getName();

                regPath = module.getBinaryDirectory().getAbsolutePath() + "/registry/"+corpusname.toLowerCase()

                wtag = module.getProject().getTokenizerWordElement();

                doNormalizeAttributeValues = "true".equals(module.getProject().getPreferencesScope().getNode("import").get(TBXPreferences.NORMALISEATTRIBUTEVALUES, "false"))

                doNormalizeAnaValues = "true".equals(module.getProject().getPreferencesScope().getNode("import").get(TBXPreferences.NORMALISEANAVALUES, "false"))

        }

        public void process(ArrayList<File> files) {

                super.process(files); // set member

                if (files == null) files = inputDirectory.listFiles();

                Project project = module.getProject();

                CorpusBuild corpus = project.getCorpusBuild(project.getName(), MainCorpus.class);

                if (corpus != null) {

                        if (project.getDoUpdate()) {

                                corpus.clean(); // remove old files

                        } else {

                                corpus.delete(); // remove old files and TXMResult children

                        }

                } else {

                        corpus = new MainCorpus(project);

                        corpus.setID(project.getName());

                        corpus.setName(project.getName());

                }

                corpus.setDescription("Built with the XTZ import module");

                if (!doScanStep()) return;

                if (!doCQPStep()) return;

                if (!doCWBEncodeStep()) return;

                if (!doCWBMakeAllStep()) return;

                if (module.getProject().getCleanAfterBuild()) {

                        new File(module.getBinaryDirectory(), "cqp").deleteDir()

                }

                isSuccessFul = true;

        }

        /**

         * Scan all XML-TXM files to find out structures and word properties

         */

        public boolean doScanStep() {

                // get all anatypes                

                sattrsListener = SAttributesListener.scanFiles(inputDirectory, wtag)

                println "-- Listing structures&properties to create for "+files.size()+" XML-TXM files..."

                ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

                for (File f : files) {

                        try {

                                cpb.tick();

                                getAnaTypes(f)

                        } catch (Exception e) {

                                println "Error while processing $f text: "+e

                                e.printStackTrace();

                                return false;

                        }

                }

                println ""

                return true;

        }

        private void getAnaTypes(File xmlFile) {

                def inputData = xmlFile.toURI().toURL().openStream();

                def factory = XMLInputFactory.newInstance();

                def parser = factory.createXMLStreamReader(inputData);

                boolean start = false;

                String ANA = "ana"

                String TYPE = "type"

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                        if (event == XMLStreamConstants.START_ELEMENT) { // start elem

                                if (wtag.equals(parser.getLocalName())) {

                                        start = true;

                                } else if (start && ANA.equals(parser.getLocalName())) { // ana elem

                                        for (int i = 0 ; i < parser.getAttributeCount(); i++) { // find @type

                                                if (TYPE.equals(parser.getAttributeLocalName(i))) { // @type

                                                        anatypes.add(parser.getAttributeValue(i).substring(1)); //remove the #

                                                        break;

                                                }

                                        }

                                }

                        } else if (event == XMLStreamConstants.END_ELEMENT) { // start elem

                                if (wtag.equals(parser.getLocalName())) {

                                        start = false;

                                }        

                        }

                }

                if (parser != null) parser.close();

                if (inputData != null) inputData.close();

//                for (String type : types)

//                        if (!anatypes.contains(type))

//                                anatypes << type

        }

        def cqpFiles = []

        public boolean doCQPStep() {

                println "-- Building CQP files $inputDirectory..."

                cqpDirectory.mkdir(); // if not created

                ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

                for (File xmlFile : files) {

                        cpb.tick();

                        String textname = xmlFile.getName();

                        int idx = textname.lastIndexOf(".")

                        if (idx > 0) textname = textname.substring(0, idx)

                        File cqpFile = new File(cqpDirectory, textname + ".cqp")

                        cqpFiles << cqpFile

                        // skip step if cqpFile is more recent than xmlFile

                        if (cqpFile.exists() && cqpFile.lastModified() >= xmlFile.lastModified()) continue;

                        XTZCompilerStep step = new XTZCompilerStep(xmlFile, cqpFile, textname, corpusname, "default", anatypes, wtag)

                        step.setNormalizeAnaValues(doNormalizeAnaValues)

                        step.setNormalizeAttributeValues(doNormalizeAttributeValues)

                        if (!step.process()) {

                                reason = "Fail to process $xmlFile."

                                return false;

                        }

                }

                println ""

                return true;

        }

        public boolean doCWBEncodeStep() {

                println "-- Running cwb-encode..."

                CwbEncode cwbEn = new CwbEncode()

                cwbEn.setDebug(debug)

                List<String> pargs = []

                pargs.add("id")

                for (String ana : anatypes)

                        pargs.add(ana)

                String[] pAttrs = pargs

                def structs = sattrsListener.getStructs()

                def structsProf = sattrsListener.getProfs()

                if (debug) {

                        println structs

                        println structsProf

                }

                List<String> sargs = new ArrayList<String>()

                def tmpTextAttrs = []

                for (String name : structs.keySet()) {

                        if (name == "text") {

                                for (String value : structs.get(name)) // append the attributes

                                        tmpTextAttrs << value // added after

                                continue;

                        }

                        String concat = name+":"+structsProf.get(name); // append the depth

                        for (String attributeName : structs.get(name)) // append the attributes

                                concat += "+"+attributeName.toLowerCase();

                        if (structs.get(name).size() == 0) {

                                concat += "+n";

                        } else {

                                if (!structs.get(name).contains("n"))

                                        concat += "+n"

                        }

                        if ((name == "p" || name == "body" || name == "back" || name == "front")

                                 && !concat.contains("+n+") && !concat.endsWith("+n"))

                                concat += "+n"

                        sargs.add(concat)

                }

                String textSAttributes = "text:0+id+base+project";

                for (String name : tmpTextAttrs) {

                        if (!("id".equals(name) || "base".equals(name) || "project".equals(name)))

                                textSAttributes += "+"+name.toLowerCase()

                }

                sargs.add(textSAttributes)

                sargs.add("txmcorpus:0+lang")

                sargs.sort()

                String[] sAttributes = sargs

                String[] pAttributes = pAttrs

                println " Word properties: "+pAttributes

                println " Structures: "+sargs

                File allcqpFile = new File(cqpDirectory, "all.cqp");

                allcqpFile.delete()

                try {

                        if (!CwbEncode.concat(cqpFiles, allcqpFile)) {

                                println "Fail to write the master cqp file: "+allcqpFile

                                return false;

                        }

                        if (!cwbEn.run(outputDirectory.getAbsolutePath() + "/$corpusname",

                                        allcqpFile.getAbsolutePath(),

                                        regPath, pAttributes, sAttributes, false)) {

                                println "** cwb-encode did not ends well. Activate finer logs to see details."

                                return false;

                        }

                        allcqpFile.delete(); // clean

                } catch (Exception e) {

                        println "Error while running cwb-encode: "+e

                        e.printStackTrace()

                        allcqpFile.delete(); // clean 

                        return false;

                }

                println ""

                return true;

        }

        public boolean doCWBMakeAllStep() {

                println "-- Running cwb-makeall..."

                try {

                        CwbMakeAll cwbMa = new CwbMakeAll();

                        cwbMa.setDebug(debug);

                        if (!new File(regPath).exists()) {

                                println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

                                return false;

                        }

                        if (!cwbMa.run(corpusname, new File(regPath).getParent())) {

                                println "** cwb-makeall did not ends well. Activate finer logs to see details."

                                return false;

                        }

                        // remove milestones from CWB registry and data files

                        FixMilestoneDeclarations fm = new FixMilestoneDeclarations(

                                new File(regPath), new File(outputDirectory.getAbsolutePath(), corpusname));

                        if (!fm.process()) {

                                println "Fail to verify&fix milestone declarations"

                                return false

                        }

                } catch (Exception e) {

                        println "Error while running cwb-makeall: "+e

                        return false;

                }

                return true;

        }

}