Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2011-06-03 12:37:57 +0200 (Fri, 03 Jun 2011) $

// $LastChangedRevision: 1867 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer.xmltxmpara;

import java.util.ArrayList;

import org.txm.importer.cwb.BuildCwbEncodeArgs;

import org.txm.importer.cwb.CwbEncode

import org.txm.importer.cwb.CwbMakeAll

import org.txm.scripts.importer.*;

import org.txm.scripts.*;

import org.txm.importer.scripts.xmltxm.BuildTTSrc;

import org.txm.importer.scripts.xmltxm.*;

import org.txm.utils.treetagger.TreeTagger;

import javax.xml.stream.*;

import java.net.URL;

import java.io.File;

import java.util.HashMap;

import java.util.List;

// TODO: Auto-generated Javadoc

/**

 * The Class compiler.

 */

class compiler

{

        /** The debug. */

        private boolean debug= false;

        /** The input data. */

        private def inputData;

        /** The factory. */

        private def factory;

        /** The parser. */

        private XMLStreamReader parser;

        /** The dir. */

        private def dir;

        /** The output. */

        private Writer output;

        /** The url. */

        private def url;

        /** The text. */

        String text="";

        /** The base. */

        String base="";

        /** The project. */

        String project="";

        /** The text attributes. */

        String[] textAttributes = null;

        /** The lang. */

        private String lang ="fr";

        /** The anatypes. */

        private static ArrayList<String> anatypes;

        /** The s attribs. */

        private static HashMap<String, List<String>> sAttribs;

        /**

         * initialize.

         *

         */

        public compiler(){}

        /**

         * Instantiates a new compiler.

         *

         * @param url the url

         * @param text the text

         * @param base the base

         * @param project the project

         */

        public compiler(URL url,String text,String base, String project)

        {

                this.text = text

                this.base = base;

                this.project = project;

                this.textAttributes = textAttributes;

                try {

                        this.url = url;

                        inputData = url.openStream();

                        factory = XMLInputFactory.newInstance();

                        parser = factory.createXMLStreamReader(inputData);

                } catch (XMLStreamException ex) {

                        System.out.println(ex);

                }catch (IOException ex) {

                        System.err.println("IOException while parsing ");

                }

        }

        /**

         * set the language of the corpus.

         *

         * @param lang the lang

         * @return the java.lang. object

         */

        public setLang(String lang)

        {

                this.lang = lang;

        }

        /**

         * Creates the output.

         *

         * @param dirPathName the dir path name

         * @param fileName the file name

         * @return true, if successful

         */

        private boolean createOutput(File f){

                try {

                        output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");

                        return true;

                } catch (Exception e) {

                        System.err.println(e);

                        return false;

                }

        }

        /**

         * Go to text.

         */

        private void GoToText()

        {

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())

                {

                        if(event == XMLStreamConstants.END_ELEMENT)

                                if(parser.getLocalName().equals("teiHeader"))

                                        return;

                }

        }

        /**

         * Transfom file cqp.

         *

         * @param dirPathName the dir path name

         * @param fileName the file name

         * @return true, if successful

         */

        public boolean transfomFileCqp(File cqpfile)

        {

                createOutput(cqpfile);

                String headvalue=""

                String vAna = "";

                String vForm = "";

                String wordid= "";

                String vHead = "";

                int p_id = 0;

                int s_id = 0;

                boolean captureword = false;

                boolean flagForm = false;

                boolean flagAna = false;

                boolean inW = false;

                int wcounter = 1;

                GoToText();

                try

                {

                        boolean stop = false;

                        for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT && !stop; event = parser.next())

                        {

                                switch (event)

                                {

                                        case XMLStreamConstants.START_ELEMENT:

                                                switch (parser.getLocalName())

                                                {

                                                        case "w":

                                                                inW = true;

                                                                wordid = parser.getAttributeValue(null, "id")

                                                                if(wordid == null)

                                                                        wordid = "w_"+text+"_"+(wcounter++)

                                                        vAna ="";

                                                        break;

                                                        case "form":

                                                                String type = parser.getAttributeValue(null, "type");

                                                                if(type == null)

                                                                {

                                                                        flagForm = true;

                                                                }

                                                                else if(type.equals("default"))

                                                                {

                                                                        flagForm = true;

                                                                }

                                                                else

                                                                {

                                                                        flagAna = true;

                                                                        vAna += "\t";

                                                                        if(!anatypes.contains(type))

                                                                                anatypes << type;

                                                                }

                                                        vForm = "";

                                                        break;

                                                        case "ana":

                                                        flagAna = true;

                                                        vAna += "\t";

                                                        String type = parser.getAttributeValue(null, "type");

                                                                if(type != null)

                                                                {

                                                                        if(type.startsWith("#"))

                                                                                type = type.substring(1)

                                                                        if(!anatypes.contains(type))

                                                                                anatypes << type;

                                                                        break;

                                                                }

                                                        break;

                                                        default:

                                                                if(!inW)

                                                                {

                                                                        output.write("<"+parser.getLocalName().toLowerCase());

                                                                        if(!sAttribs.containsKey(parser.getLocalName()))

                                                                                sAttribs.put(parser.getLocalName().toLowerCase(), []);

                                                                        for( int i = 0 ; i < parser.getAttributeCount() ; i++)

                                                                        {

                                                                                String attrname = parser.getAttributeLocalName(i).toLowerCase();

                                                                                String attrvalue = parser.getAttributeValue(i);

                                                                                if(!(parser.getLocalName() == "text" && attrname == "id"))

                                                                                        output.write(" "+attrname+"=\""+attrvalue.replace("\"", "'")+"\"");

                                                                                if(!sAttribs.get(parser.getLocalName().toLowerCase()).contains(attrname))

                                                                                        sAttribs.get(parser.getLocalName().toLowerCase()).add(attrname)

                                                                        }

                                                                        if(parser.getLocalName() == "text")

                                                                        { // add some infos

                                                                                output.write(" id=\""+text+"\" base=\""+base+"\" project=\""+project+"\"");

                                                                        }

                                                                        output.write(">\n");

                                                                }

                                                        }

                                                break;

                                        case XMLStreamConstants.END_ELEMENT:

                                                switch (parser.getLocalName())

                                                {

                                                        case "w":

                                                        output.write( vForm.replaceAll("&", "&amp;").replaceAll("<", "&lt;") +"\t"+wordid+vAna+"\n");

                                                        vAna = "";

                                                        vForm = "";

                                                        inW = false;

                                                        break;

                                                        case "form":

                                                        flagForm = false;

                                                        flagAna = false;

                                                        break;

                                                        case "ana":

                                                        flagAna = false;

                                                        break;

                                                        default:

                                                                if(!inW)

                                                                        output.write("</"+parser.getLocalName().toLowerCase()+">\n");

                                                                if(parser.getLocalName() == "text")

                                                                        stop = true;

                                                }

                                                break;

                                        case XMLStreamConstants.CHARACTERS:

                                                if(inW)

                                                {

                                                if(flagForm)

                                                        if(flagAna)

                                                                vAna += parser.getText().trim();

                                                        else

                                                                vForm += parser.getText().trim();

                                                if(flagAna)

                                                        vAna += parser.getText().trim();

                                                }

                                                break;

                                }

                        }

                        output.close();

                        if (parser != null) parser.close();

                        if (inputData != null) inputData.close();

                }

                catch (Exception ex) {

                        System.out.println(ex);

                        if (parser != null) parser.close();

                        if (inputData != null) inputData.close();

                        return false;

                }

                return true;

        }

        /**

         * Run.

         *

         * @param rootDirFile the root dir file

         * @param basename the basename

         * @param textAttributes the text attributes

         * @return true, if successful

         */

        public boolean run(ArrayList<File> files, File binDir, String corpusname, String basename, String[] textAttributes)

        {

                anatypes = new ArrayList<String>();// init only 1 time

                sAttribs = new HashMap<String, List<String>>();// init only 1 time

                String rootDir = binDir.getAbsolutePath();

                if (!(CwbEncode.isExecutableAvailable() && CwbMakeAll.isExecutableAvailable())) {

                        println ("Error: CWB executables not well set.")

                        return false;

                }

                if(!binDir.exists())

                {

                        println ("binary directory does not exists: "+binDir)

                        return false;

                }

                if(files == null || files.size() == 0)

                {

                        println "Error: no file to process"

                        return false;

                }

                String textid = "";

                int counttext = 0;

                File cqpdir = new File(binDir,"cqp");

                File cqpfile = new File(cqpdir, corpusname.toLowerCase()+".cqp");

                //0 set Lang

                if(createOutput(cqpfile))

                {

                        output.write("<txmcorpus lang=\""+lang+"\">\n");

                        output.close();

                }

                //1- Transform into CQP file

                def builder = null;

                for (File f : files) {

                        counttext++;

                        if (!f.exists()) {

                                println("The file "+f+ " does not exists")

                        } else {

                                //println("process file "+f)

                                String txtname = f.getName().substring(0,f.getName().length()-4);

                                builder = new compiler(f.toURL(), txtname, basename, "default");

                                builder.setLang(lang);

                                if (!builder.transfomFileCqp(cqpfile))

                                        return false;

                        }

                }

                //end corpus

                if(createOutput(cqpfile))

                {

                        output.write("</txmcorpus>\n");

                        output.close();

                }

                //2- Import into CWB

                def outDir = rootDir;

                def outDirTxm = rootDir;

                CwbEncode cwbEn = new CwbEncode();

                cwbEn.setDebug(debug);

                CwbMakeAll cwbMa = new CwbMakeAll();

                cwbMa.setDebug(debug);

                List<String> pargs = ["id"];

                for(String ana : anatypes)

                        pargs.add(ana);

                List<String> sargs = [];

                //println "Found Sattributes "+this.sAttribs;

                if (sAttribs.containsKey("text")) {

                        if (!sAttribs.get("text").contains("id"))

                                sAttribs.get("text").add("id");

                        if (!sAttribs.get("text").contains("base"))

                                sAttribs.get("text").add("base");

                        if (!sAttribs.get("text").contains("project"))

                                sAttribs.get("text").add("project");

                } else {

                        sargs.add("text:0+id+base+project")

                }

                if(sAttribs.containsKey("txmcorpus"))

                {

                        if(!sAttribs.get("txmcorpus").contains("lang"))

                                sAttribs.get("txmcorpus").add("lang");

                } else {

                        sargs.add("txmcorpus:0+lang")

                }

                for(String tag : this.sAttribs.keySet())

                {

                        String sAttr = tag;

                        if(sAttribs.get(tag).size() > 0)

                                sAttr += ":";

                        for(String attr : sAttribs.get(tag))

                                sAttr +="+"+attr;

                        sargs.add(sAttr)

                }

                String[] sAttributes = sargs;

                String[] pAttributes = pargs;

                println "Corpus structural attributes: "+sAttributes;

                println "Corpus lexical attributes: "+pAttributes;

                try {

                        String regPath = outDirTxm + "/registry/"+corpusname.toLowerCase();

                        cwbEn.run(

                                outDirTxm + "/data/"+corpusname+"/",

                                cqpfile.getAbsolutePath(),

                                regPath, pAttributes, sAttributes);

                        if (!new File(regPath).exists()) {

                                println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

                                return false;

                        }

                        cwbMa.run(corpusname.toUpperCase(), outDirTxm + "/registry");

                } catch (Exception ex) {System.out.println(ex); return false;}

                System.out.println("Done.")

                return true;

        }

        /**

         * Sets the debug.

         */

        public void setDebug()

        {

                this.debug = true;

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                File dir = new File("~/xml/geo");

                def c = new compiler();

                c.setDebug();

                c.setCwbPath("~/TXM/cwb/bin");

                c.run(dir,"geo");

        }

}