Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-05-26 17:42:36 +0200 (jeu. 26 mai 2016) $

// $LastChangedRevision: 3219 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer.xml;

import java.util.ArrayList

import java.util.Collections

import org.txm.importer.cwb.BuildCwbEncodeArgs

import org.txm.importer.cwb.CwbEncode

import org.txm.importer.cwb.CwbMakeAll

import org.txm.scripts.importer.*

import org.txm.scripts.*

import org.txm.importer.scripts.xmltxm.*

import org.txm.utils.treetagger.TreeTagger

import org.txm.objects.*

import javax.xml.stream.*

import java.net.URL

import java.io.File

import java.util.HashMap

import java.util.List

import java.util.HashMap

import java.util.HashSet

import org.txm.metadatas.*

import org.txm.utils.ConsoleProgressBar

import org.txm.utils.io.FileCopy

import org.txm.searchengine.cqp.corpus.*

/**

 * The "compiler" Class of the XML/w import module.

 */

class compiler {

        /** The debug. */

        private boolean debug= false;

        /** The input data. */

        private def inputData;

        /** The factory. */

        private def factory;

        /** The parser. */

        private XMLStreamReader parser;

        /** The dir. */

        private def dir;

        /** The output. */

        private def output;

        /** The url. */

        private def url;

        /** The anatypes. */

        private static anatypes = []

        private static anavalues = [:]

        /** The anahash. */

        private HashMap<String, String> anahash = new HashMap<String, String>() ;

        private static SAttributesListener sattrsListener;

        private static HashMap<String, ArrayList<String>> structs;

        private static HashMap<String, Integer> structsProf;

        /** The text. */

        String text="";

        /** The base. */

        String base="";

        /** The text attributes. */

        String[] textAttributes = null;

        /** The lang. */

        private String lang ="fr";

        public static sortMetadata = null;

        public static normalizeMetadata = false;

        /**

         * initialize.

         *

         */

        public compiler(){}

        public void setOptions(String sortmetadata, boolean normalizemetadata)

        {

                sortMetadata = sortmetadata;

                normalizeMetadata = normalizemetadata;

        }

        /**

         * Instantiates a new compiler.

         *

         * @param url the url

         * @param text the text

         * @param base the base

         * @param project the project

         */

        public compiler(URL url, String text, String base, String projectName)

        {

                this.text = text

                this.base = base;

                this.textAttributes = textAttributes;

                try {

                        this.url = url;

                        inputData = url.openStream();

                        factory = XMLInputFactory.newInstance();

                        parser = factory.createXMLStreamReader(inputData);

                        if (sattrsListener == null)

                                sattrsListener = new SAttributesListener(parser);

                        else

                                sattrsListener.start(parser)

                } catch (XMLStreamException ex) {

                        System.out.println(ex);

                }catch (IOException ex) {

                        System.err.println("IOException while parsing ");

                }

        }

        /**

         * set the language of the corpus.

         *

         * @param lang the lang

         * @return the java.lang. object

         */

        public setLang(String lang)

        {

                this.lang = lang;

        }

        /** The annotation success. */

        boolean annotationSuccess = false;

        /**

         * Sets the annotation success.

         *

         * @param val the new annotation success

         */

        public void setAnnotationSuccess(boolean val)

        {

                this.annotationSuccess = val;

        }

        /**

         * Creates the output.

         *

         * @param dirPathName the dir path name

         * @param fileName the file name

         * @return true, if successful

         */

        private boolean createOutput(File f){

                try {

                        output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f, f.exists())) , "UTF-8");

                        return true;

                } catch (Exception e) {

                        System.err.println(e);

                        return false;

                }

        }

        /**

         * Go to text.

         */

        private void GoToText()

        {

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                        if (event == XMLStreamConstants.END_ELEMENT)

                                if (parser.getLocalName().equals("teiHeader"))

                                        return;

                }

        }

        /**

         * Transfom file cqp.

         *

         * @param dirPathName the dir path name

         * @param fileName the file name

         * @return true, if successful

         */

        public boolean transfomFileCqp(Project project, File cqpFile, HashMap<String, String> textmetadata)

        {

                if (!createOutput(cqpFile))

                        return false;

                String headvalue=""

                String vAna = "";

                String vForm = "";

                String wordid= "";

                String vHead = "";

                int p_id = 0;

                int s_id = 0;

                def divs = []

                def ncounts = [:] // contains the n values per tags with no attribute

                boolean captureword = false;

                boolean flagForm = false;

                boolean flagAna = false;

                String anatype = "";

                String anavalue = "";

                boolean stopAtFirstSort = true;

                boolean foundtei = false;

                boolean foundtext = false;

                //output.write("<txmcorpus lang=\""+lang+"\">\n");

                try {

                        String localname;

                        for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())

                        {

                                switch (event) {

                                        case XMLStreamConstants.START_ELEMENT:

                                                localname = parser.getLocalName().toLowerCase();

                                                if ("tei".equals(localname)) foundtei = true;

                                                switch (localname) {

                                                        case "text":

                                                                sattrsListener.startElement(localname);

                                                                foundtext = true;

                                                                output.write("<text id=\""+text+"\" base=\""+base+"\"" + " project=\""+project.getName()+"\"");

                                                        //                                                        for (String name : textmetadata.keySet())

                                                        //                                                                output.write(" "+name+"=\""+textmetadata.get(name)+"\"")

                                                                for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

                                                                        String attrname = parser.getAttributeLocalName(i);

                                                                        String attrvalue = parser.getAttributeValue(i).replaceAll("\"", """)

                                                                        if (normalizeMetadata)

                                                                                attrvalue = attrvalue.toLowerCase();

                                                                        if (attrname != "id")

                                                                                output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+"\"")

                                                                }

                                                                output.write(">\n");

                                                        //                                                                if (textAttributes == null) {

                                                        //                                                                        textAttributes = new String[parser.getAttributeCount()];

                                                        //

                                                        //                                                                        for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

                                                        //                                                                                textAttributes[i]=parser.getAttributeLocalName(i).toLowerCase();

                                                        //                                                                        }

                                                        //                                                                }

                                                                break;

                                                        case "w":

                                                                for (int i = 0 ; i < parser.getAttributeCount(); i++) {

                                                                        if (parser.getAttributeLocalName(i).equals("id")) {

                                                                                wordid = parser.getAttributeValue(i);

                                                                        }

                                                                }

                                                                anavalues = [:];

                                                                break;

                                                        case "form":

                                                                flagForm = true;

                                                                vForm = "";

                                                                vAna ="";

                                                                break;

                                                        case "ana":

                                                                flagAna = true;

                                                                anavalue = "";

                                                                for (int i = 0 ; i < parser.getAttributeCount(); i++)

                                                                        if ("type".equals(parser.getAttributeLocalName(i))) {

                                                                                anatype = parser.getAttributeValue(i).substring(1);//remove the #

                                                                                break;

                                                                        }

                                                                break;

                                                        default:

//                                                                if ("div" == localname ) {

//                                                                        def type = localname;

//                                                                        for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

//                                                                                String attrname = parser.getAttributeLocalName(i);

//                                                                                if ("type".equals(attrname)) {

//                                                                                        type= parser.getAttributeValue(i)

//                                                                                }

//                                                                        }

//                                                                        divs << type;

//                                                                        localname = type

//                                                                }

                                                                if (foundtei && !foundtext) break;

                                                                sattrsListener.startElement(localname);

                                                                output.write("<"+localname);

                                                                for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

                                                                        String attrname = parser.getAttributeLocalName(i);

                                                                        String attrvalue = parser.getAttributeValue(i)

                                                                        if (normalizeMetadata)

                                                                                attrvalue = attrvalue.toLowerCase();

                                                                        output.write(" "+attrname.toLowerCase()+"=\""+attrvalue.replaceAll("\"", """)+"\"")

                                                                }

                                                                if (parser.getAttributeCount() == 0) { // add the n attribute

                                                                        if (!ncounts.containsKey(localname)) ncounts.put(localname, 0);

                                                                        int ncount = ncounts.get(localname);

                                                                        ncounts.put(localname, ncount+1);

                                                                        output.write(" n=\""+ncount+"\"")

                                                                }

                                                                output.write(">\n");

                                                }

                                                break;

                                        case XMLStreamConstants.END_ELEMENT:

                                                localname = parser.getLocalName().toLowerCase();

                                                switch (localname) {

                                                        case "w":

                                                                for (String type : anatypes) {

                                                                        def v = anavalues.get(type);

                                                                        if (v != null) vAna +="\t"+v;

                                                                        else vAna +="\t";

                                                                }

                                                                vForm = vForm.replaceAll("\n", "").replaceAll("&", "&amp;").replaceAll("<", "&lt;");

                                                                if (vAna != null) {

                                                                        output.write(vForm+"\t"+wordid+vAna+"\n");

                                                                }

                                                                vAna = "";

                                                                vForm = "";

                                                                break;

                                                        case "tei":

                                                                break;

                                                        case "form":

                                                                flagForm = false;

                                                                break;

                                                        case "ana":

                                                                anavalues.put(anatype, anavalue)

                                                                flagAna = false;

                                                                break;

                                                        default:

                                                                if (foundtei && !foundtext) break;

//                                                                if ("div" == localname && divs.size() > 0) {

//                                                                        localname = divs.pop()

//                                                                }

                                                                sattrsListener.endElement(localname);

                                                                output.write("</"+localname+">\n");

                                                }

                                                break;

                                        case XMLStreamConstants.CHARACTERS:

                                                if (flagForm)

                                                        vForm += parser.getText().trim();

                                                if (flagAna) {

                                                        if (normalizeMetadata)

                                                                anavalue += parser.getText().trim().toLowerCase();

                                                        else

                                                                anavalue += parser.getText().trim();

                                                }

                                                break;

                                }

                        }

                        //output.write("</txmcorpus>");

                        output.close();

                        if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                } catch (Exception ex) {

                        System.out.println("Exception while parsing " + inputData+" of Text "+text);

                        File xmlFile = null

                        File errorDir = null

                        try {

                                xmlFile = new File(url.getFile())

                                errorDir = new File(cqpFile.getParentFile(), "compiler-error")

                                println "Warning: Moving $xmlFile to $errorDir"

                                errorDir.mkdir();

                                FileCopy.copy(xmlFile, new File(errorDir, xmlFile.getName()))

                        } catch(Exception eCopy) {

                                println "Error while moving "+url+" to "+errorDir

                        }

                        if (parser != null) parser.close();

                        if (inputData != null) inputData.close();

                        return false;

                }

                return true;

        }

        private void getAnaTypes(File xmlFile) {

                inputData = xmlFile.toURI().toURL().openStream();

                factory = XMLInputFactory.newInstance();

                parser = factory.createXMLStreamReader(inputData);

                String ana = "ana"

                HashSet<String> types = new HashSet<String>();

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                        if (event == XMLStreamConstants.START_ELEMENT) { // start elem

                                if (ana.equals(parser.getLocalName())) { // ana elem

                                        for (int i = 0 ; i < parser.getAttributeCount(); i++) { // find @type

                                                if ("type".equals(parser.getAttributeLocalName(i))) { // @type

                                                        types.add(parser.getAttributeValue(i).substring(1)); //remove the #

                                                        break;

                                                }

                                        }

                                }

                        }

                }

                if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                for (String type : types)

                        if (!anatypes.contains(type))

                                anatypes << type

        }

        /**

         * Run.

         *

         * @param rootDirFile the root dir file

         * @param basename the basename

         * @param textAttributes the text attributes

         * @param srcfiles the srcfiles

         * @return true, if successful

         */

        public boolean run(Project project, File binDir, File txmDir, String corpusname, String[] textAttributes, def srcfiles, Metadatas metadatas)

        {

                sattrsListener = null; // reset SAttribute Listener for each new import

                String rootDir = binDir.getAbsolutePath();

                anatypes = [] // reset

                anavalues = [:] // reset

                if (!(CwbEncode.isExecutableAvailable() && CwbMakeAll.isExecutableAvailable())) {

                        println ("Error: CWB executables rights are not well setted.")

                        return false;

                }

                CorpusBuild corpus = project.getCorpusBuild(project.getName());

                if (corpus != null) {

                        //println "CLEAN PREVIOUS CORPUS"

                        corpus.delete(); // remove old files

                }

                // make new one

                corpus = new MainCorpus(project);

                corpus.setID(project.getName());

                corpus.setName(project.getName());

                corpus.setDescription("Built with the XML/w import module");

                File cqpFile = new File(binDir,"cqp/"+corpusname+".cqp");

cqpFile.delete()

                new File(binDir,"cqp").mkdirs()

                new File(binDir,"data").mkdirs()

                new File(binDir,"registry").mkdirs()

                String textid = ""

                int counttext = 0

                List<File> files = txmDir.listFiles(new FileFilter() {

                        public boolean accept(File f) {

                                return !f.isDirectory() && !f.isHidden() && f.getName().endsWith(".xml");

                        }

                });

                //1- Transform into CQP file

                def builder = null

                //start corpus

                if (createOutput(cqpFile)) {

                        output.write("<txmcorpus lang=\""+lang+"\">\n")

                        output.close()

                }

                // sort files

                if (sortMetadata == null) {

                        Collections.sort(files)

                } else {

                        HashMap<File, String> sortmetadatavalues = new HashMap<File, String>()

                        for (File f : files) {

                                String value = MetadataGetter.get(f,"text", sortMetadata)

                                sortmetadatavalues.put(f, value)

                        }

                        println "sort properties value: "+sortmetadatavalues

                        Collections.sort(files, new Comparator<File>() {

                                /**

                                 * Compare.

                                 *

                                 * @param o1 the o1

                                 * @param o2 the o2

                                 * @return the int

                                 */

                                                public int compare(Object o1, Object o2) {

                                                        String v1 = sortmetadatavalues.get((File)o1)

                                                        String v2 = sortmetadatavalues.get((File)o2)

                                                        if (v1 == null || v2 == null) return 0;

                                                        return v1.compareTo(v2)

                                                }

                                        });

                }

                // get all anatypes

                for (File f : files) {

                        getAnaTypes(f)

                }

                println("Compiling "+files.size()+" files ")

                ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

                for (File f : files) {

                        cpb.tick()

                        HashMap<String, String> textmetadata;

                        if (metadatas != null)

                                textmetadata = metadatas.getTextMetadata(f)

                        else

                                textmetadata = [:]

                        counttext++;

                        if (!f.exists()) {

                                println("file "+f+ " does not exists")

                        } else {

                                String txtname = f.getName().substring(0,f.getName().length()-4)

                                builder = new compiler(f.toURI().toURL(), txtname, corpusname, "default")

                                builder.setLang(lang);

                                if (!builder.transfomFileCqp(project, cqpFile, textmetadata)) {

                                        println("Failed to compile "+f)

                                }

                        }

                }

                cpb.done()

                //end corpus

                if (createOutput(cqpFile)) {

                        output.write("</txmcorpus>\n")

                        output.close()

                }

                println ""

                //2- Import into CWB

                def outDir = rootDir

                CwbEncode cwbEn = new CwbEncode()

                cwbEn.setDebug(debug)

                CwbMakeAll cwbMa = new CwbMakeAll()

                cwbMa.setDebug(debug)

                List<String> pargs = []

                pargs.add("id")

                for (String ana : anatypes)

                        pargs.add(ana)

                String[] pAttrs = pargs

                structs = sattrsListener.getStructs()

                structsProf = sattrsListener.getProfs()

                if (debug) {

                        println structs

                        println structsProf

                }

                List<String> sargs = new ArrayList<String>()

                def tmpTextAttrs = []

                for (String name : structs.keySet()) {

                        if (name == "text") {

                                for (String value : structs.get(name)) // append the attributes

                                        tmpTextAttrs << value // added after

                                continue;

                        }

                        //if ( name == "q") continue; // added after

                        //if ( name == "foreign") continue; // added after

                        String concat = name+":"+structsProf.get(name); // append the depth

                        for (String attributeName : structs.get(name)) // append the attributes

                                concat += "+"+attributeName.toLowerCase();

                        if (structs.get(name).size() == 0) {

                                concat += "+n";

                        } else {

                                if (!structs.get(name).contains("n"))

                                        concat += "+n"

                        }

                        if ((name == "p" || name == "body" || name == "back" || name == "front")

                                 && !concat.contains("+n+") && !concat.endsWith("+n"))

                                concat += "+n"

                        sargs.add(concat)

                }

                String textSAttributes = "text:0+id+base+project";

                for (String name : tmpTextAttrs) {

                        if (!("id".equals(name) || "base".equals(name) || "project".equals(name)))

                                textSAttributes += "+"+name.toLowerCase()

                }

                //                if (metadataXPath != null) {

                //                        for (String meta : metadataXPath.keySet()) // text property declarations from metadata.csv

                //                                textSAttributes+="+"+meta;

                //                }

                sargs.add(textSAttributes)

                sargs.add("txmcorpus:0+lang")

                sargs.sort()

                String[] sAttributes = sargs

                String[] pAttributes = pAttrs

                println "P-attributes: "+pAttributes

                println "S-attributes: "+sargs

                //if(!annotationSuccess)

                //pAttributes = ["id"];

                //println "PATTRIBUTES : "+pargs;

                /*

                 ArrayList<String> wordstag = ["w"];

                 println "Getting structural attributes..."

                 BuildCwbEncodeArgs argsgetter = new BuildCwbEncodeArgs();

                 HashMap<String, HashSet<String>> allStructures = new HashMap<String, HashSet<String>>();

                 HashMap<String, Integer> allStructuresInclusion = new HashMap<String, Integer>();

                 for (File srcfile: txmDir.listFiles()) {

                 if (!(!srcfile.getName().endsWith(".csv") && srcfile.canRead() && !srcfile.isHidden() && !srcfile.isDirectory() && ValidateXml.test(srcfile)))

                 continue;

                 print "."

                 argsgetter.process(srcfile, wordstag);

                 for (String sattr : argsgetter.getSAttributes()) {

                 int idx = sattr.indexOf(":");

                 if(idx < 0 )

                 continue;

                 String name = sattr.substring(0, idx);

                 if (!allStructures.containsKey(name)) {

                 allStructures.put(name, new HashSet<String>());

                 allStructuresInclusion.put(name, 0);

                 }

                 //println "sattr: "+name

                 String attrs = sattr.substring(idx+1);

                 String[] split = attrs.split("\\+");

                 if (split.length > 0) {

                 int start = 1;

                 try {// test if first attr is a number

                 int n = Integer.parseInt(split[0]);

                 if (n > allStructuresInclusion.get(name))

                 allStructuresInclusion.put(name, n);

                 } catch(Exception e) {start = 0;}

                 for (int i = start ; i < split.length ; i++)

                 allStructures.get(name).add(split[i]);

                 }

                 }

                 }

                 // add structures+properties found in sources

                 List<String> sargs = new ArrayList<String>();

                 for (String name : allStructuresInclusion.keySet()) {

                 String concat = name+":"+allStructuresInclusion.get(name);

                 for (String value : allStructures.get(name))

                 concat += "+"+value;

                 if (name.equals("text")) {

                 concat += "+base+project"

                 if (!concat.contains("id"))

                 concat += "+id";

                 }

                 sargs.add(concat);

                 }*/

                //                for (int i = 0 ; i < sargs.size() ; i++) {

                //                        if (sargs.get(i).startsWith("text:")) {

                //                                String str = sargs.get(i);

                //                                sargs.set(i, "text:"+str.substring(6));

                //                        }

                //                }

                //                String textSAttributes = "text:0+id+base+project";

                //                if (metadatas != null) {

                //                        for (String meta : metadatas.getHeadersList()) // text property declarations from metadata.csv

                //                                textSAttributes+="+"+meta;

                //                }

                //sargs.add(textSAttributes)

                //sargs.add("txmcorpus:0+lang")

                //                String[] sAttributes = sargs;

                //                System.out.println("\nCorpus structures: "+sAttributes);

                //                System.out.println("corpus word properties: "+pAttributes);

                try {

                        String regPath = outDir + "/registry/"+corpusname.toLowerCase();

                        cwbEn.run(

                                outDir + "/data/$corpusname",

                                outDir + "/cqp/"+corpusname+".cqp",

                                regPath, pAttributes, sAttributes);

                        if (!new File(regPath).exists()) {

                                println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

                                return false;

                        }

                        cwbMa.run(corpusname, outDir + "/registry");

                } catch (Exception ex) {System.out.println(ex); return false;}

                return true;

        }

        /**

         * Sets the debug.

         */

        public void setDebug()

        {

                this.debug = true;

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                File dir = new File("~/xml/geo");

                def c = new compiler();

                c.setDebug();

                c.setCwbPath("~/TXM/cwb/bin");

                c.run(dir,"geo");

        }

}