Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-05-26 17:42:36 +0200 (jeu. 26 mai 2016) $

// $LastChangedRevision: 3219 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer.hyperbase;

import org.txm.importer.cwb.CwbEncode

import org.txm.importer.cwb.CwbMakeAll

import org.txm.scripts.importer.*;

import org.txm.scripts.*;

import org.txm.importer.scripts.xmltxm.*;

import org.txm.utils.treetagger.TreeTagger;

import org.txm.objects.*

import org.txm.searchengine.cqp.corpus.*

import javax.xml.stream.*;

import java.net.URL;

import java.io.File;

import java.util.HashMap;

import java.util.List;

// TODO: Auto-generated Javadoc

/**

 * The Class compiler.

 */

class compiler

{

        /** The debug. */

        private boolean debug= false;

        /** The input data. */

        private def inputData;

        /** The factory. */

        private def factory;

        /** The parser. */

        private XMLStreamReader parser;

        /** The dir. */

        private def dir;

        /** The output. */

        private def output;

        /** The url. */

        private def url;

        /** The anahash. */

        private HashMap<String,String> anahash = new HashMap<String,String>() ;

        /** The text. */

        String text="";

        /** The base. */

        String base="";

        /** The text attributes. */

        String[] textAttributes = null;

        /** The lang. */

        private String lang ="fr";

        /**

         * initialize.

         *

         */

        public compiler(){}

        /**

         * set the language of the corpus.

         *

         * @param lang the lang

         * @return the java.lang. object

         */

        public setLang(String lang)

        {

                this.lang = lang;

        }

        /** The annotation success. */

        boolean annotationSuccess = false;

        /**

         * Sets the annotation success.

         *

         * @param value the value

         * @return the java.lang. object

         */

        public setAnnotationSuccess(boolean value)

        {

                this.annotationSuccess = value

        }

        /**

         * Creates the output.

         *

         * @param f the f

         * @return true, if successful

         */

        private boolean createOutput(File f){

                try {

                        output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");

                        return true;

                } catch (Exception e) {

                        System.err.println(e);

                        return false;

                }

        }

        /**

         * Go to text.

         */

        private void GoToText()

        {

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())

                {

                        if(event == XMLStreamConstants.END_ELEMENT)

                                if(parser.getLocalName().equals("teiHeader"))

                                        return;

                }

        }

        /**

         * Run.

         *

         * @param rootDirFile the root dir file

         * @param basename the basename

         * @return true, if successful

         */

        public boolean run(Project project, File binDir, File txmDir, String corpusname)

        {

                String rootDir = binDir.getAbsolutePath();

                if (!(CwbEncode.isExecutableAvailable() && CwbMakeAll.isExecutableAvailable())) {

                        println ("Error: CWB executables not well set.")

                        return false;

                }

                CorpusBuild corpus = project.getCorpusBuild(project.getName());

                if (corpus != null) {

                        //println "CLEAN PREVIOUS CORPUS"

                        corpus.delete(); // remove old files

                }

                // make new one

                corpus = new MainCorpus(project);

                corpus.setID(project.getName());

                corpus.setName(project.getName());

                corpus.setDescription("Built with the Hyperbase import module");

                File cqpFile = new File(binDir,"cqp/"+corpusname+".cqp");

cqpFile.delete()

                new File(binDir,"cqp").mkdirs()

                new File(binDir,"data").mkdirs()

                new File(binDir,"registry").mkdirs()

                //start corpus

                if (createOutput(cqpFile)) {

                        output.write("<txmcorpus lang=\""+lang+"\">\n");

                        output.close();

                }

                String textid="";

                int counttext =0;

                List<File> files = txmDir.listFiles();

                Collections.sort(files);

                //1- Transform into CQP file

                XMLTXM2CQP cqpbuilder = null;

                println("Compiling "+files.size()+" files")

                for (File f : files) {

                        print "."

                        counttext++;

                        if (!f.exists()) {

                                println("file "+f+ " does not exists")

                        }

                        else {

                                cqpbuilder = new XMLTXM2CQP(f.toURI().toURL());

                                String txtname = f.getName().substring(0,f.getName().length()-4);

                                cqpbuilder.setTextInfo(txtname, corpusname, "project");

                                cqpbuilder.setBalisesToKeep(["text","p","s"]);

                                cqpbuilder.setSendToPAttributes(["s":["n"]]);

                                cqpbuilder.setLang(lang);

                                if (!cqpbuilder.transformFile(cqpFile)) {

                                        println("Failed to compile "+f)

                                }

                        }

                }

                println ""

                //end corpus

                if (createOutput(cqpFile)) {

                        output.write("</txmcorpus>\n");

                        output.close();

                }

                if (cqpbuilder == null) return false;

                //2- Import into CWB

                def outDir =rootDir;

                CwbEncode cwbEn = new CwbEncode();

                cwbEn.setDebug(debug);

                CwbMakeAll cwbMa = new CwbMakeAll();

                cwbMa.setDebug(debug);

                List<String> pAttributesList = cqpbuilder.getpAttributs();

                List<String> sAttributesList = cqpbuilder.getsAttributs();

                println "word properties : "+pAttributesList

                println "structures : "+sAttributesList

                String[] pAttributes = pAttributesList.toArray(new String[pAttributesList.size()])

                String[] sAttributes = sAttributesList.toArray(new String[sAttributesList.size()])

                try {

                        String regPath = outDir + "/registry/"+corpusname.toLowerCase();

                        cwbEn.run(outDir + "/data/$corpusname", outDir + "/cqp/"+corpusname+".cqp", regPath,pAttributes, sAttributes);

                        if (!new File(regPath).exists()) {

                                println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

                                return false;

                        }

                        cwbMa.run(corpusname, outDir + "/registry");

                } catch (Exception ex) {System.out.println(ex); return false;}

                return true;

        }

        /**

         * Sets the debug.

         */

        public void setDebug()

        {

                this.debug = true;

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                File dir = new File("~/xml/geo");

                def c = new compiler();

                c.setDebug();

                c.setCwbPath("~/TXM/cwb/bin");

                c.run(dir,"geo");

        }

}