Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2017-04-19 16:23:38 +0200 (mer. 19 avril 2017) $

// $LastChangedRevision: 3430 $

// $LastChangedBy: mdecorde $

//

package org.txm.importer.scripts.xmltxm

import org.txm.importer.StaxIdentityParser;

import java.text.DateFormat;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.Locale;

import javax.xml.stream.*;

import java.net.URL;

import org.txm.importer.filters.*;

import org.txm.scripts.importer.HasElement

import org.txm.scripts.importer.StaxStackWriter

import org.txm.utils.AsciiUtils;

/**

 * The Class Xml2Ana.

 *

 * @author mdecorde

 * transform : pre xml-tei file >> xml-tei-txm file

 * The pre xml-tei file must contains a minimal teiHeader with classDecl, encodingDesc and titleStmt

 *

 * you must specify the correspondance between word attributs and ana types&respStmtIDs

 * then the attributes of w tags will be transformed into interp tag

 */

class Xml2Ana extends StaxIdentityParser

{

        /** The dir. */

        private def dir;

        /** The convert all attributes. */

        private boolean convertAllAttributes = false;

        /** The corresp type. */

        HashMap<String,String> correspType;

        /** The corresp ref. */

        HashMap<String,String> correspRef;

        /** The check tags. */

        HashMap<String,Boolean> checkTags = new HashMap<String,Boolean>();

        /** The resp id. */

        def respId = [];

        /** The applications. */

        HashMap<String,File> applications;

        /** The taxonomies. */

        HashMap<String,String[]> taxonomies;

        /** The resps. */

        HashMap<String,String[]> resps;

        /** The items. */

        HashMap<String,HashMap<String,String>> items;

        /** The XML headeradded. */

        boolean XMLHeaderadded = false;

        String textname;

        String wtag = "w";

        public static final String TEXT = "text"

        public static final String ID = "id"

        /**

         * Instantiates a new xml2 ana.

         *

         * @param url the url

         * @param wordprefix the wordprefix

         */

        public Xml2Ana(File file) {

                super(file.toURI().toURL());

                //File file = new File(url.getFile()).getAbsoluteFile()

                textname = file.getName();

                int idx = textname.lastIndexOf(".");

                if (idx > 0)

                        textname = textname.substring(0, idx)

                checkTags.put("respStmt",false);

                checkTags.put("titleStmt",false);

                checkTags.put("appInfo",false);

                hasText = new HasElement(file, TEXT).process();

        }

        /**

         * Sets the convert all atrtibutes.

         *

         * @param value the value

         * @return the java.lang. object

         */

        public setConvertAllAtrtibutes(boolean value) {

                convertAllAttributes = value;

        }

        /**

         * Sets the convert all atrtibutes.

         *

         * @param value the value

         * @return the java.lang. object

         */

        public setWordTag(String wtag) {

                this.wtag = wtag

        }

        int idcount = 0;

        boolean flagWord = false;

        int firstElement = 0;

        boolean teiElementAdded = false;

        boolean teiHeaderElementAdded = false;

        boolean hasText = false;

        boolean textElementAdded = false;

        def anabalises = [];

        protected void processStartElement()

        {

//                println "checkTags=$checkTags";

//                println "parser=$parser";

                firstElement++;

                if (this.checkTags.containsKey(parser.getLocalName())) {

                        this.checkTags.put(parser.getLocalName(), true);

                }

                switch (parser.getLocalName()) {

                        case wtag:

                                if (!hasText) {

                                        writer.writeStartElement(TEXT);

                                        writer.writeAttribute(ID, textname);

                                        textElementAdded = true;

                                        hasText = true;

                                }

                                idcount++; // increment word counter

                                anabalises.clear();

                                writer.writeStartElement(parser.getLocalName()); // write w

                                for (int i = 0 ; i < parser.getNamespaceCount() ; i++) // write namespaces

                                        writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));

                                for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) { // transform attributes

                                        String type = parser.getAttributeLocalName(i);

                                        String value = parser.getAttributeValue(i);

                                        if (correspType.containsKey(type)) { // check if txm:ana

                                                String corresptype = correspType.get(type);

                                                String ref = correspRef.get(type);

                                                anabalises.add(["#"+ref, "#"+corresptype, value]);

                                        } else if (type == ID) { // keep id attribute

                                                String wordid = value

                                                if (wordid.startsWith("w")) {

                                                        if (!wordid.startsWith("w_"))

                                                                wordid = "w_"+wordid.substring(1)

                                                }

//                                                else {

//                                                        wordid = "w_"+textname+"_"+wordid;

//                                                }

                                                wordid = AsciiUtils.buildAttributeId(wordid); // remove characters not compatible with the id attribute value

                                                writer.writeAttribute(type, wordid);

                                        } else { // add attributes that was in the original <w>

                                                if (convertAllAttributes)

                                                        anabalises.add(["none","#"+type, value])

                                                else

                                                        writer.writeAttribute(type, value);

                                        }

                                }

                                flagWord = true; // start to capture the form

                                writer.writeStartElement(TXMNS, "form");

                                break;

                        case "TEI":

                                super.processStartElement();

                                boolean hasTeiNS = false;

                                boolean hasTXMNs = false;

                                for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {

                                        if (parser.getNamespaceURI(i) == TXMNS)

                                                hasTXMNs = true;

                                        else if (parser.getNamespaceURI(i) == TEINS)

                                                hasTeiNS = true;

                                }

                                if (!hasTeiNS) {

                                        writer.writeDefaultNamespace(TEINS);

                                }

                                if (!hasTXMNs)

                                        writer.writeNamespace(TXM, TXMNS);

                                break;

                        default:

                                if (TEXT.equals(localname)) {

                                        hasText = true;

                                }

                                if (firstElement == 1) { // test if first element is TEI

                                        //println "first tag: "+parser.getLocalName()

                                        if (localname != "TEI") { // "TEI" is missing

                                                teiElementAdded = true;

                                                addTEIElement();

                                        } else if (!hasText) {

                                                writer.writeStartElement(TEXT);

                                                writer.writeAttribute(ID, textname);

                                                textElementAdded = true;

                                                hasText = true;

                                        }

                                }

                                if (firstElement == 2 && teiElementAdded != true) {

                                        //println "second tag: "+parser.getLocalName()

                                        if (localname != "teiHeader") { // teiHeader is missing

                                                writeTeiHeader();

                                                hasTeiHeader = true

                                                teiHeaderElementAdded = true

                                        }

                                } else if (!hasText & (teiElementAdded | teiHeaderElementAdded)) {

                                        writer.writeStartElement(TEXT);

                                        writer.writeAttribute(ID, textname);

                                        textElementAdded = true;

                                        hasText = true;

                                }

                                super.processStartElement();

                                if (TEXT.equals(localname)) {

                                        if (!parser.getAttributeValue(null, ID)) {

                                                writer.writeAttribute(ID, textname);

                                        }

                                }

                }

        }

        protected void after()

        {

                if (textElementAdded) {

                        writer.writeEndElement(); // text

                }

                if (teiElementAdded) {

                        writer.writeEndElement(); // TEI

                }

                super.after(); // close writer, parser, etc

        }

        protected void addTEIElement()

        {

                writer.writeStartElement("TEI");

                writer.writeDefaultNamespace(TEINS);

                writer.writeNamespace(TXM, TXMNS);

                writer.writeNamespace(TEI, TEINS);

                writeTeiHeader();

        }

        protected void processCharacters()

        {

                if (flagWord) {

                        writer.writeCharacters(parser.getText().trim()); // keep form in 1 line

                } else {

                        super.processCharacters();

                }

        }

        boolean hasClassDecl = false;

        boolean hasFileDesc = false;

        boolean hasEncodingDesc = false;

        boolean hasTeiHeader = false;

        boolean hasTEI = false;

        public static String ANA = "ana"

        public static String RESP = "resp"

        public static String TYPE = "type"

        protected void processEndElement()

        {

                switch (parser.getLocalName()) {

                        case wtag:

                                writer.writeEndElement(); // txm:form

                                for (def values : anabalises)

                                {// <txm:ana resp=ref type=corresptype>value</txm:ana>

                                        writer.writeStartElement(TXMNS, ANA);

                                        writer.writeAttribute(RESP, values[0]);

                                        writer.writeAttribute(TYPE, values[1]);

                                        writer.writeCharacters(values[2]);

                                        writer.writeEndElement(); // txm:ana

                                }

                                flagWord = false;

                                break;

                        case "fileDesc":

                                hasFileDesc = true;

                                this.writeTXMResps();

                                break;

                        case "classDecl":

                                hasClassDecl=true;

                                this.writeTXMTaxonomies();

                                break;

                        case "encodingDesc":

                                hasEncodingDesc = true;

                                writeContentOfEncodingDesc();

                                break;

                        case "teiHeader":

                                hasTeiHeader = true

                                if (!hasEncodingDesc) {

                                        writer.writeStartElement("encodingDesc");

                                        writeContentOfEncodingDesc();

                                        writer.writeEndElement();

                                }

                                break;

                        case "TEI":

                                hasTEI = true;

                                if (!hasTeiHeader) {

                                        writeTeiHeader();

                                }

                                break;

                }

                super.processEndElement();

        }

        protected void writeTeiHeader()

        {

                writer.writeStartElement("teiHeader");

                writer.writeStartElement("fileDesc")

                this.writeTXMResps();

                writer.writeStartElement("titleStmt")

                writer.writeStartElement("title")

                writer.writeEndElement(); // title

                writer.writeEndElement(); // titleStmt

                writer.writeStartElement("publicationStmt")

                writer.writeEndElement(); // publicationStmt

                writer.writeStartElement("sourceDesc")

                writer.writeEndElement(); // sourceDesc

                writer.writeEndElement(); // fileDesc

                writer.writeStartElement("encodingDesc");

                writeContentOfEncodingDesc();

                writer.writeEndElement(); // encodingDesc

                writer.writeEndElement(); // teiHeader

        }

        protected void writeContentOfEncodingDesc()

        {

                writer.writeStartElement("appInfo")

                this.writeTXMApps();

                writer.writeEndElement(); // appInfo

                if (!hasClassDecl) {

                        writer.writeStartElement("classDecl");

                        this.writeTXMTaxonomies();

                        writer.writeEndElement(); // classDecl

                }

        }

        /**

         * Check resp.

         *

         * @return the string

         */

        public String checkResp()

        {

                String rez ="found tags : \n";

                for (String key : checkTags.keySet())

                        rez += "\t"+key+"\n";

                return rez;

        }

        /**

         * Sets the correspondances.

         *

         * @param correspRef the corresp ref

         * @param correspType the corresp type

         */

        public void setCorrespondances(correspRef, correspType)

        {

                this.correspRef = correspRef;

                this.correspType = correspType;

        }

        /**

         * Sets the header infos.

         *

         * @param respId the resp id

         * @param resps the resps

         * @param applications the applications

         * @param taxonomies the taxonomies

         * @param items the items

         */

        public void setHeaderInfos(respId,resps, applications, taxonomies, items)

        {

                this.respId = respId

                this.resps = resps

                this.applications = applications

                this.taxonomies = taxonomies;

                this.items = items;

        }

        /**

         * Write txm resps.

         */

        public void writeTXMResps()

        {

                for (String ref : respId) {

                        String[] infos = resps.get(ref);

                        writer.writeStartElement("respStmt");

                        writer.writeStartElement(RESP);

                        writer.writeAttribute(ID,ref);

                        writer.writeCharacters(infos[0]);

                        writer.writeStartElement("date");

                        writer.writeAttribute("when",infos[2]);

                        writer.writeCharacters(infos[3]);

                        writer.writeEndElement(); // date

                        writer.writeEndElement(); //resp

                        writer.writeStartElement("name");

                        writer.writeAttribute(TYPE, "person");

                        writer.writeCharacters(infos[1])

                        writer.writeEndElement(); // name

                        writer.writeEndElement(); //respStmt

                }

        }

        /**

         * Write txm apps.

         */

        public void writeTXMApps()

        {

                for (String ref : respId) {

                        List<String> list= applications.get(ref);

                        String ident = list.get(0);

                        String version = list.get(1);

                        File report = list.get(2);

                        writer.writeStartElement(TXMNS, "application");

                        writer.writeAttribute("ident", ident);

                        writer.writeAttribute("version", version);

                        writer.writeAttribute(RESP, ref);

                        //get txm:commandLine from GeneratedReport

                        if (report != null) {

                                writer.writeCharacters("");writer.flush();

                                Reader reader = new FileReader(report);

                                String line = reader.readLine();

                                while (line != null) {

                                        if (line.length() != 0)

                                                output.write(line+"\n");

                                        line = reader.readLine();

                                }

                                reader.close();

                        }

                        writer.writeStartElement("ab");

                        writer.writeAttribute(TYPE, "annotation");

                        for (String item : taxonomies.get(ref)) {

                                writer.writeStartElement("list");

                                writer.writeEmptyElement("ref");

                                writer.writeAttribute(TYPE, "tagset");

                                writer.writeAttribute("target", item);

                                writer.writeEndElement(); // list

                        }

                        writer.writeEndElement(); // ab

                        writer.writeEndElement(); // txm:application

                }

        }

        /**

         * Write txm taxonomies.

         */

        public void writeTXMTaxonomies()

        {

                for (String tax : items.keySet()) {

                        writer.writeStartElement("taxonomy");

                        writer.writeAttribute(ID, tax);

                        writer.writeStartElement("bibl");

                        writer.writeAttribute(TYPE, "tagset");

                        writer.writeStartElement("title");

                        writer.writeCharacters(tax);

                        writer.writeEndElement(); // title

                        for (String type : items.get(tax).keySet()) {

                                writer.writeEmptyElement("ref");

                                writer.writeAttribute(TYPE, type);

                                writer.writeAttribute("target", items.get(tax).get(type));

                        }

                        writer.writeEndElement(); // bibl

                        writer.writeEndElement(); // taxonomy

                }

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args) {

                String rootDir = "~/xml/rgaqcj/";

                new File(rootDir+"anainline/").mkdir();

                ArrayList<String> milestones = new ArrayList<String>();

                String file = "roland-p5.xml";

                String anafile = "roland-p5.xml";

                def correspType = new HashMap<String,String>()

                // correspType(attribut word wlx, attribut type de la propriété ana du w txm)

                correspType.put("p2","CATTEX2009");

                def correspRef = new HashMap<String,String>()

                // correspRef (attribut word wlx, attribut ref de la propriété ana du w txm. ref pointe vers l'identifiant du respStmt du TEIheader)

                correspRef.put("p2","ctx1");

                //il faut lister les id de tous les respStmt

                def respId = ["ctx1"];//,"TT1", "TnT1"];

                //fait la correspondance entre le respId et le rapport d'execution de l'outil

                def applications = new HashMap<String,HashMap<String,String>>();

                applications.put("ctx1",new ArrayList<String>());

                applications.get("ctx1").add("Oxygen");//app ident

                applications.get("ctx1").add("9.3");//app version

                applications.get("ctx1").add(null);//app report file path

                //fait la correspondance entre le respId et les attributs type de la propriété ana du w txm

                //pour construire les ref vers les taxonomies

                def taxonomiesUtilisees = new HashMap<String,String[]>();

                taxonomiesUtilisees.put("ctx1",["CATTEX2009"]);//,"lemma","lasla","grace"]);

                //associe un id d'item avec sa description et son URI

                def itemsURI = new HashMap<String,HashMap<String,String>>();

                itemsURI.put("CATTEX2009",new HashMap<String,String>());

                itemsURI.get("CATTEX2009").put("tagset","http://bfm.ens-lsh.fr/IMG/xml/cattex2009.xml");

                itemsURI.get("CATTEX2009").put("website","http://bfm.ens-lsh.fr/article.php3?id_article=176");

                //informations de respStmt

                //resps (respId <voir ci-dessus>, [description, person, date])

                def resps = new HashMap<String,String[]>();

                resps.put("ctx1", ["initial tagging","alavrentiev","2010-03-02","Tue Mar  2 21:02:55 Paris, Madrid 2010"])

                //lance le traitement

                def builder = new Xml2Ana(new File(rootDir+"/src/",file));

                builder.setCorrespondances(correspRef, correspType);

                builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)

                //dossier de sortie + nom fichier sortie

                builder.process(anafile);

                return

        }

}