Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2017-04-11 15:30:35 +0200 (mar. 11 avril 2017) $

// $LastChangedRevision: 3426 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer

import java.text.DateFormat;

import java.util.Date;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.LinkedHashMap;

import javax.xml.stream.*;

import java.net.URL;

import org.txm.importer.filters.*;

// TODO: Auto-generated Javadoc

/**

 * The Class XMLTXM2CQP.

 *

 * @author mdecorde

 * simple transofmration of a xml-tei-txm file into cqp file

 */

class XMLTXM2CQP

{

        /** The url. */

        private def url;

        /** The input data. */

        private def inputData;

        /** The factory. */

        private def factory;

        /** The parser. */

        private XMLStreamReader parser;

        /** The output. */

        private def output;

        /** The hashmap of txm:form and txm:ana values and the attributes hash*/

        LinkedHashMap<String, String> anahash = new LinkedHashMap<String, String>();

        LinkedHashMap<String, String> formhash = new LinkedHashMap<String, String>();

        LinkedHashMap<String, String> wordattributes = new LinkedHashMap<String, String>();

        /** The balisesfound. */

        HashMap<String, List<String>> balisesfound;// = new HashMap<String, List<String>>();

        /** The balises to keep. */

        List<String> balisesToKeep;

        /** The send to p attributes. */

        HashMap <String, List<String>> sendToPAttributes;// = new HashMap<String, List<String>>();

        /** The injected p attributes. */

        List<String> injectedPAttributes = new ArrayList<String>();

        /** The default reference : a pattern + the properties to use */

        List<String> defaultReferences = new ArrayList<String>();

        String defaultReferencePattern;

        /** The injected p attributes values. */

        HashMap <String, String> injectedPAttributesValues;// = new ArrayList<String>();

        /** The addinfos. */

        boolean addinfos = false;

        /** The txtname. */

        String txtname;

        /** The base. */

        String base;

        /** The project. */

        String project;

        /** The lang. */

        public String lang= "fr";

        public String currentForm;

        public String currentAna;

        /**

         * Sets the lang.

         *

         * @param lang the lang

         * @return the java.lang. object

         */

        public setLang(String lang)

        {

                this.lang = lang;

        }

        /**

         * Instantiates a new xMLTX m2 cqp.

         *

         * @param url the url

         */

        public XMLTXM2CQP(URL url){

                try {

                        this.url = url;

                        inputData = url.openStream();

                        factory = XMLInputFactory.newInstance();

                        parser = factory.createXMLStreamReader(inputData);

                } catch (XMLStreamException ex) {

                        System.out.println(ex);

                }catch (IOException ex) {

                        System.out.println("IOException while parsing ");

                }

        }

        /**

         * Sets the text info.

         *

         * @param name the name

         * @param base the base

         * @param project the project

         */

        public void setTextInfo(String name, String base, String project)

        {

                this.addinfos = true;

                this.txtname= name;

                this.base = base;

                this.project = project;

        }

        /**

         * Creates the output.

         *

         * @param outfile the outfile

         * @return true, if successful

         */

        private boolean createOutput(File outfile)

        {

                try {

                        output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile, outfile.exists()) , "UTF-8"));

                        return true;

                } catch (Exception e) {

                        System.err.println(e);

                        return false;

                }

        }

        /** The haspb. */

        boolean haspb = false;

        /** The haslb. */

        boolean haslb = false;

        /**

         * Transform file.

         *

         * @param outfile the outfile

         * @return true, if successful

         */

        public boolean transformFile(File outfile)

        {

                if(balisesToKeep == null)

                {

                        println "no element has been defined to be keeped"

                        return false;

                }

                haspb = false;

                haslb = false;

                boolean flagAna;

                boolean flagForm;

                boolean flagWord;

                String vWord = "";

                String vForm = "";

                String vAna = "";

                String lb_id = "";

                String pb_id = "";

                wordattributes = [:];

                balisesfound = new HashMap<String, List<String>>();

                if(!createOutput(outfile))

                        return false;

                if(sendToPAttributes != null)

                {

                        for(String tag: sendToPAttributes.keySet())

                                for(String attr : sendToPAttributes.get(tag))

                                        injectedPAttributes.add(tag+attr);

                        injectedPAttributesValues = [:];

                }

                //output.write("<txmcorpus lang=\""+lang+"\">\n");

                balisesfound.put("txmcorpus",["lang"]);

                try {

                        for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                                switch (event) {

                                        case XMLStreamConstants.START_ELEMENT:

                                                String localname = parser.getLocalName().toLowerCase();

                                        // we will only declare found tags in cwb registry

                                                if(balisesToKeep.contains(localname)) {

                                                        if(!balisesfound.containsKey(localname)) {

                                                                balisesfound.put(localname, []);

                                                        }

                                                        List<String> attrlist = balisesfound.get(localname);

                                                        for (int i= 0 ; i < parser.getAttributeCount() ;i++ )

                                                                if(!attrlist.contains(parser.getAttributeLocalName(i)))

                                                                        attrlist.add(parser.getAttributeLocalName(i));

                                                }

                                                switch (localname) {

                                                        case "w": // get word id !!

                                                                wordattributes.put("id", parser.getAttributeValue(null, "id"));

                                                                break;

                                                        case "form":

                                                                flagForm = true;

                                                                currentForm = parser.getAttributeValue(null, "type");

                                                                if(currentForm == null)

                                                                        currentForm = "default";

                                                                vForm = "";

                                                                break;

                                                        case "ana":

                                                                flagAna = true;

                                                                vAna ="";

                                                                currentAna = (parser.getAttributeValue(null,"type"));

                                                                if(currentAna != null)

                                                                        currentAna = currentAna.substring(1)// remove the #

                                                                else

                                                                        flagAna = false;

                                                                break;

                                                        default:

                                                                if (sendToPAttributes != null) {

                                                                        //println "should store $localname ? with "+sendToPAttributes.keySet()

                                                                        if (sendToPAttributes.keySet().contains(localname)) {

                                                                                //println "store attr of "+localname

                                                                                List<String> attrs = sendToPAttributes.get(localname);

                                                                                for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {

                                                                                        if (attrs.contains(parser.getAttributeLocalName(i))) {

                                                                                                injectedPAttributesValues.put(localname+parser.getAttributeLocalName(i).toLowerCase(),parser.getAttributeValue(i))

                                                                                        }

                                                                                }

                                                                        }

                                                                }

                                                                if (balisesToKeep.contains(localname)) {

                                                                        output.write("<"+localname);

                                                                        //println "write <"+localname+"..."

                                                                        //write attributes

                                                                        boolean idwritten = false;

                                                                        boolean basewritten = false;

                                                                        boolean projectwritten = false;

                                                                        for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {

                                                                                String attrname = parser.getAttributeLocalName(i).toLowerCase();

                                                                                if (attrname == "id")

                                                                                        idwritten = true;

                                                                                if (attrname == "base")

                                                                                        basewritten = true;

                                                                                if (attrname == "project")

                                                                                        projectwritten = true;

                                                                                output.write(" "+attrname+"=\""+parser.getAttributeValue(i).replace("&", "&").replace("\"", """)+"\"" );

                                                                        }

                                                                        if (localname.equals("text"))

                                                                                if (addinfos) {

                                                                                        List<String> attrlist = balisesfound.get(localname);

                                                                                        if (!idwritten) {

                                                                                                output.write(" id=\""+txtname+"\"")

                                                                                                attrlist.add("id");

                                                                                        }

                                                                                        if (!basewritten) {

                                                                                                output.write(" base=\""+base+"\"");

                                                                                                attrlist.add("base");

                                                                                        }

                                                                                        if (!projectwritten) {

                                                                                                output.write(" project=\""+project+"\"");

                                                                                                attrlist.add("project");

                                                                                        }

                                                                                }

                                                                        // finalize tag

                                                                        output.write(">\n");

                                                                }

                                                }

                                                break;

                                        case XMLStreamConstants.END_ELEMENT:

                                                String localname = parser.getLocalName().toLowerCase();

                                                switch (localname) {

                                                        case "form":

                                                                if(flagForm)

                                                                        formhash.put(currentForm, vForm);

                                                                flagForm = false;

                                                                break;

                                                        case "ana":

                                                                if(flagAna)

                                                                        anahash.put(currentAna, vAna);

                                                                flagAna = false;

                                                                break;

                                                        case "w":

                                                                vWord = "";

                                                                vWord = formhash.get("default").replaceAll("&", "&amp;").replaceAll("<", "&lt;"); // get default form

                                                                for (String form : formhash.keySet()) // and the others

                                                                        if (form != "default")

                                                                                vWord += "\t"+formhash.get(form);

                                                                for (String type : wordattributes.keySet()) // only word id ?

                                                                        vWord+="\t"+wordattributes.get(type)

                                                                if (sendToPAttributes != null) // word attributes from structure properties

                                                                {

                                                                        //println "injectedPAttributesValues: "+injectedPAttributesValues

                                                                        for(String pattr : injectedPAttributes)

                                                                                vWord+="\t"+injectedPAttributesValues.get(pattr) ;//les attributs injecter

                                                                }

                                                                for (String type : anahash.keySet()) // word annotations in txm:ana

                                                                        vWord+="\t"+anahash.get(type)

                                                                output.write(vWord+"\n");

                                                                vWord= "";

                                                                break;

                                                        default:

                                                                if (sendToPAttributes != null) // reset structure properties

                                                                {

                                                                        if (sendToPAttributes.keySet().contains(localname)) {

                                                                                for (String attr : sendToPAttributes.get(localname)) {

                                                                                        injectedPAttributesValues.put(attr, "N/A")

                                                                                }

                                                                        }

                                                                }

                                                                if (balisesToKeep.contains(localname)) {

                                                                        output.write("</"+localname+">\n");

                                                                }

                                                }

                                                break;

                                        case XMLStreamConstants.CHARACTERS:

                                                if (flagForm) {

                                                        vForm += parser.getText().trim();

                                                }

                                                if (flagAna) {

                                                        vAna += parser.getText().trim();

                                                }

                                                break;

                                }

                        }

                        //output.write("</txmcorpus>\n");

                        output.close();

                        if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                } catch (Exception ex) {

                        println "Error while parsing $url : "+ex

                        ex.printStackTrace();

                        if (parser != null) parser.close();

                        if (inputData != null) inputData.close();

                        return false;

                }

                return true;

        }

        /**

         * Gets the p attributs.

         *

         * @return the p attributs

         */

        public List<String> getpAttributs()

        {

                def pAttributs = [];

                for (String wordattr : wordattributes.keySet()) {

                        pAttributs.add(wordattr);

                }

                if (sendToPAttributes != null)

                        for (String pAttr : this.injectedPAttributes)

                                pAttributs.add(pAttr);

                for (String anakey : anahash.keySet()) {

                        pAttributs.add(anakey);

                }

                return pAttributs;

        }

        /**

         * Gets the s attributs.

         *

         * @return the s attributs

         */

        public List<String> getsAttributs()

        {

                println balisesfound

                def sAttributs = [];

                for (String balise : this.balisesfound.keySet()) {

                        List<String> sAtt = this.balisesfound.get(balise);

                        String attributes = "";

                        for (String attr : sAtt) {

                                attributes+="+"+attr;

                        }

                        if (sAtt.size() > 0)

                                sAttributs.add(balise +":"+attributes);

                        else

                                sAttributs.add(balise);

                }

                return sAttributs;

        }

        /**

         * Sets the balises to keep.

         *

         * @param balisesToKeep the new balises to keep

         */

        public void setBalisesToKeep(List<String> balisesToKeep)

        {

                if (balisesToKeep != null)

                        this.balisesToKeep = balisesToKeep;

                else

                        println("Warning: the list of elements to keep is null")

        }

        /**

         * Sets the defautl reference pattern

         * TODO: not implemented

         *

         * @param balisesToKeep the new balises to keep

         */

        public void setDefaultReference(String pattern, List<String> strucProperties)

        {

                if (defaultReferencePattern != null) {

                        this.defaultReferences = defaultReferences;

                        defaultReferencePattern = pattern;

                }

        }

        /**

         * Sets the send to p attributes.

         *

         * @param sendus the sendus

         */

        public void setSendToPAttributes(HashMap<String, List<String>> sendus)

        {

                if (sendus != null)

                        this.sendToPAttributes = sendus;

                else

                        println("Warning: the pAttributes to inject is null")

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args) {

                String rootDir = "/home/mdecorde/TXM/corpora/CORNEILLEMOLIERETER/txm/CORNEILLEMOLIERETER";

                File srcfile = new File(rootDir,"CORNEILLEP_AGESILAS_1666.xml");

                println srcfile.exists()

                File cqpfile = new File(rootDir, "out/CORNEILLEP_AGESILAS_1666.cqp");

                new File(rootDir,"out").deleteDir()

                new File(rootDir,"out").mkdir()

                System.out.println("XMLTXM2CQP : "+srcfile+" >> "+cqpfile);

                def builder = new XMLTXM2CQP(srcfile.toURL());

                def balises = ["text", "s"];

                builder.setBalisesToKeep(balises);

                builder.transformFile(cqpfile);

                println("SATTRIBUTS: "+builder.getsAttributs());

                println("PATTRIBUTS: "+builder.getpAttributs());

                return;

        }

}