Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-05-26 17:42:36 +0200 (jeu. 26 mai 2016) $

// $LastChangedRevision: 3219 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer.discours

import org.txm.importer.cwb.BuildCwbEncodeArgsFromTEITXM;

import org.txm.importer.cwb.CwbEncode

import org.txm.importer.cwb.CwbMakeAll

import org.txm.scripts.importer.*;

import org.txm.scripts.*;

import org.txm.importer.scripts.xmltxm.*;

import org.txm.utils.treetagger.TreeTagger;

import javax.xml.stream.*;

import java.net.URL;

import java.io.File;

import java.util.HashMap;

import java.util.List;

// TODO: Auto-generated Javadoc

/**

 * The Class compiler.

 */

class compiler {

        /** The input data. */

        private def inputData;

        /** The factory. */

        private def factory;

        /** The parser. */

        private XMLStreamReader parser;

        /** The dir. */

        private def dir;

        /** The output. */

        private def output;

        /** The url. */

        private def url;

        /** The anahash. */

        private HashMap<String,String> anahash = new HashMap<String,String>() ;

        /** The text. */

        String text="";

        /** The base. */

        String base="";

        /** The project. */

        String project="";

        /** The lang. */

        private String lang ="fr";

        /** The text attributes. */

        static String textAttributes = "";

        /**

         * initialize.

         *

         */

        public compiler(){

        }

        /**

         * Instantiates a new compiler.

         *

         * @param url the url

         * @param text the text

         * @param base the base

         * @param project the project

         */

        public compiler(URL url,String text,String base, String project) {

                this.text = text

                this.base = base;

                this.project = project;

                try {

                        this.url = url;

                        inputData = url.openStream();

                        factory = XMLInputFactory.newInstance();

                        parser = factory.createXMLStreamReader(inputData);

                } catch (XMLStreamException ex) {

                        System.out.println(ex);

                }catch (IOException ex) {

                        System.err.println("IOException while parsing ");

                }

        }

        /**

         * set the language of the corpus.

         *

         * @param lang the lang

         * @return the java.lang. object

         */

        public setLang(String lang)

        {

                this.lang = lang;

        }

        /**

         * Creates the output.

         *

         * @param f the f

         * @return true, if successful

         */

        private boolean createOutput(File f){

                try {

                        output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");

                        return true;

                } catch (Exception e) {

                        System.err.println(e);

                        return false;

                }

        }

        /**

         * Go to text.

         */

        private void GoToText() {

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                        if(event == XMLStreamConstants.END_ELEMENT)

                                if(parser.getLocalName().equals("teiHeader"))

                                        return;

                }

        }

        /** The debug. */

        boolean debug = false;

        /**

         * Sets the debug.

         */

        public void setDebug()

        {

                debug = true;

        }

        /**

         * Run.

         *

         * @param files the files

         * @param basename the basename

         * @return true, if successful

         */

        public boolean run(List<File> files, File binDir, File txmDir, String corpusname) {

                String binDirPath = binDir.getAbsolutePath()+"/";

                if (!(CwbEncode.isExecutableAvailable() && CwbMakeAll.isExecutableAvailable())) {

                        println ("Error: CWB executables not well set.")

                        return false;

                }

                File cqpFile = new File(binDir,"cqp/${corpusname}.cqp");

                new File(binDir, "cqp.").deleteDir();

                new File(binDir, "cqp").mkdirs();

                new File(binDir, "data/$corpusname").deleteDir();

                new File(binDir, "data/$corpusname").mkdir();

                new File(binDir, "registry").deleteDir();

                new File(binDir, "registry").mkdir();

                //start corpus

                if (createOutput(cqpFile)) {

                        output.write("<txmcorpus lang=\""+lang+"\">\n");

                        output.close();

                }

                def second = 0

                //1- Transform into CQP file

                Collections.sort(files);

                XMLTXM2CQP cqpbuilder = null;

                for (File f : files) {

                        if (second) { print(", ") }

                        if (second > 0 && (second % 5) == 0) println ""

                        print(f.getName().replaceFirst("\\.xml", ""));

                        second++

                        cqpbuilder = new XMLTXM2CQP(f.toURI().toURL());

                        String txtname = f.getName().substring(0,f.getName().length()-4);

                        cqpbuilder.setTextInfo(txtname, corpusname, "project");

                        cqpbuilder.setBalisesToKeep(["text","p","s"]);

                        cqpbuilder.setSendToPAttributes(["s":["id"], "p":["id"]]);

                        cqpbuilder.setLang(lang);

                        if (!cqpbuilder.transformFile(cqpFile)) {

                                println("Failed to compile "+f)

                        }

                }

                //end corpus

                if (createOutput(cqpFile)) {

                        output.write("</txmcorpus>\n");

                        output.close();

                }

                if (cqpbuilder == null) {

                        println "there was no files to process: "+files

                        return false;

                }

                //2- Import into CWB

                def outDir = binDirPath;

                CwbEncode cwbEn = new CwbEncode();

                CwbMakeAll cwbMa = new CwbMakeAll();

                List<String> pAttributesList = cqpbuilder.getpAttributs();

                List<String> sAttributesList = cqpbuilder.getsAttributs();

                println "pAttrs : "+pAttributesList

                println "sAttrs : "+sAttributesList

                String[] pAttributes = pAttributesList.toArray(new String[pAttributesList.size()])

                String[] sAttributes = sAttributesList.toArray(new String[sAttributesList.size()])

                try {

                        //println "Indexing "+compiler.textAttributes

                        cwbEn.setDebug(debug);

                        String regPath = outDir + "registry/"+corpusname.toLowerCase()

                        cwbEn.run(

                                        outDir + "data/$corpusname",

                                        outDir + "/cqp/"+corpusname+".cqp",

                                        regPath,

                                        pAttributes,

                                        sAttributes);

                        if (!new File(regPath).exists()) {

                                println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

                                return false;

                        }

                        //println "Binding all indexes"

                        cwbMa.setDebug(debug);

                        cwbMa.run(corpusname, outDir + "registry");

                } catch (Exception ex) { System.err.println(ex); return false;}

                return true;

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args) {

                File dir = new File(System.getProperty("user.home"),"xml/discours/txm/");

                List<File> files = dir.listFiles();

                def c = new compiler()

                c.setCwbLoc(System.getProperty("user.home")+"/TXM/cwb/bin/")

                if (!c.run(files,"discours")) {

                        println "Compiler failed"

                        return;

                }

        }

}