Laboratoire ICAR » Plateforme TXM

package org.txm.scripts.importer.xtz;

import java.io.File;

import javax.xml.stream.*

import java.net.URL

import java.util.HashMap;

import org.txm.utils.*

import org.txm.utils.io.*

import org.txm.importer.xtz.*

/**

 * Compiles the CQP file of ONE text

 *

 * @author mdecorde

 *

 */

public class XTZCompilerStep extends Step {

        static String FORM = "form";

        static String ANA = "ana";

        static String ID = "id";

        static String TYPE = "type";

        static String TAB = "\t";

        static String QUOTE = "\"";

        File xmlFile

        File cqpFile

        String textname, corpusname, projectname;

        boolean normalizeAttributeValues = false;

        boolean normalizeAnaValues = true;

        boolean normalizeFormValues = true;

        def inputData;

        XMLInputFactory factory;

        XMLStreamReader parser;

        OutputStreamWriter output;

        def anavalues = [:];

        def anatypes;

        String WTAG = "w"

        public void setNormalizeAttributeValues(boolean n) {

                this.normalizeAttributeValues = n;

        }

        public void setNormalizeAnaValues(boolean n) {

                this.normalizeAnaValues = n;

        }

        public void setNormalizeFormValues(boolean n) {

                this.normalizeFormValues = n;

        }

        public XTZCompilerStep(File xmlFile, File cqpFile, String textname, String corpusname, String projectname, def anatypes, def wtag) {

                this.xmlFile = xmlFile;

                this.cqpFile = cqpFile;

                this.textname = textname

                this.corpusname = corpusname;

                this.projectname = projectname;

                this.anatypes = anatypes;

                this.WTAG = wtag

                try {

                        inputData = xmlFile.toURI().toURL().openStream();

                        factory = XMLInputFactory.newInstance();

                        parser = factory.createXMLStreamReader(inputData);

                } catch (Exception ex) {

                        System.err.println("Exception while parsing $xmlFile : "+ex);

                }

        }

        /**

         * Creates the output.

         *

         * @param dirPathName the dir path name

         * @param fileName the file name

         * @return true, if successful

         */

        private boolean createOutput(File f) {

                try {

                        output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f)) , "UTF-8");

                        return true;

                } catch (Exception e) {

                        System.err.println(e);

                        return false;

                }

        }

        /**

         * Transfom file cqp.

         *

         * @param dirPathName the dir path name

         * @param fileName the file name

         * @return true, if successful

         */

        public boolean process()

        {

                if (!createOutput(cqpFile))

                        return false;

                String headvalue=""

                String vAna = "";

                String vForm = "";

                String wordid= "";

                String vHead = "";

                int p_id = 0;

                int s_id = 0;

                def divs = []

                def ncounts = [:] // contains the n values per tags with no attribute

                boolean captureword = false;

                boolean flagWord = false;

                boolean flagForm = false;

                boolean flagAna = false;

                String anatype = "";

                String anavalue = "";

                boolean stopAtFirstSort = true;

                boolean foundtei = false;

                boolean foundtext = false;

                int nWords = 0;

                try {

                        String localname;

                        for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())

                        {

                                switch (event) {

                                        case XMLStreamConstants.START_ELEMENT:

                                                localname = parser.getLocalName().toLowerCase();

                                                if ("tei".equals(localname)) foundtei = true;

                                                switch (localname) {

                                                        case "text":

                                                                foundtext = true;

                                                                output.write("<text id=\""+textname+"\" base=\""+corpusname+QUOTE + " project=\""+projectname+QUOTE);

                                                                for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

                                                                        String attrname = parser.getAttributeLocalName(i);

                                                                        String attrvalue = parser.getAttributeValue(i)

                                                                        if (normalizeAttributeValues)

                                                                                attrvalue = attrvalue.trim();

                                                                        if (attrname != ID)

                                                                                output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+QUOTE)

                                                                }

                                                                output.write(">\n");

                                                                break;

                                                        case WTAG:

                                                                for (int i = 0 ; i < parser.getAttributeCount(); i++) {

                                                                        if (parser.getAttributeLocalName(i).equals(ID)) {

                                                                                wordid = parser.getAttributeValue(i);

                                                                        }

                                                                }

                                                                anavalues = [:];

                                                                flagWord = true

                                                                nWords++

                                                                break;

                                                        case FORM:

                                                                flagForm = true;

                                                                vForm = "";

                                                                vAna = "";

                                                                break;

                                                        case ANA:

                                                                flagAna = true;

                                                                anavalue = "";

                                                                for (int i = 0 ; i < parser.getAttributeCount(); i++) {

                                                                        //println parser.getAttributeLocalName(i)+"="+parser.getAttributeValue(i)

                                                                        if (TYPE.equals(parser.getAttributeLocalName(i))) {

                                                                                anatype = parser.getAttributeValue(i).substring(1);//remove the #

                                                                                break;

                                                                        }

                                                                }

                                                                break;

                                                        default:

                                                                if (!foundtei || !foundtext) break;

                                                                output.write("<"+localname);

                                                                for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

                                                                        String attrname = parser.getAttributeLocalName(i);

                                                                        String attrvalue = parser.getAttributeValue(i)

                                                                        if (normalizeAttributeValues)

                                                                                attrvalue = attrvalue.trim();

                                                                        output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+QUOTE)

                                                                }

                                                                if (parser.getAttributeCount() == 0) { // add the n attribute

                                                                        if (!ncounts.containsKey(localname)) ncounts.put(localname, 0);

                                                                        int ncount = ncounts.get(localname);

                                                                        ncounts.put(localname, ncount+1);

                                                                        output.write(" n=\""+ncount+QUOTE)

                                                                }

                                                                output.write(">\n");

                                                }

                                                break;

                                        case XMLStreamConstants.END_ELEMENT:

                                                localname = parser.getLocalName().toLowerCase();

                                                switch (localname) {

                                                        case WTAG:

                                                                for (String type : anatypes) {

                                                                        def v = anavalues.get(type);

                                                                        if (v != null) vAna +=TAB+v;

                                                                        else vAna +=TAB;

                                                                }

                                                                vForm = vForm.replaceAll("\n", "").replaceAll("&", "&amp;").replaceAll("<", "&lt;");

                                                                if (vAna != null) {

                                                                        output.write(vForm+TAB+wordid+vAna+"\n");

                                                                }

                                                                vAna = "";

                                                                vForm = "";

                                                                flagWord = false;

                                                                break;

                                                        case "tei":

                                                                foundtei = false;

                                                                break;

                                                        case "text":

                                                                output.write("</text>\n");

                                                                foundtext = false;

                                                                break;

                                                        case FORM:

                                                                flagForm = false;

                                                                break;

                                                        case ANA:

                                                                anavalues.put(anatype, anavalue)

                                                                flagAna = false;

                                                                break;

                                                        default:

                                                                if (!foundtei || !foundtext) break;

                                                                output.write("</"+localname+">\n");

                                                }

                                                break;

                                        case XMLStreamConstants.CHARACTERS:

                                                if (!foundtei || !foundtext) break;

                                                if (flagWord) {

                                                        if (flagForm) {

                                                                if (normalizeFormValues) {

                                                                        vForm += parser.getText().trim();

                                                                } else {

                                                                        vForm += parser.getText();

                                                                }

                                                        }

                                                        if (flagAna) {

                                                                if (normalizeAnaValues)

                                                                        anavalue += parser.getText().trim();

                                                                else

                                                                        anavalue += parser.getText();

                                                        }

                                                }

                                                break;

                                }

                        }

                        output.close();

                        if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                } catch (Exception ex) {

                        System.out.println("Exception while parsing " + inputData+" of Text "+textname+" : "+ex);

                        File errorDir = null

                        try {

                                errorDir = new File(cqpFile.getParentFile(), "compiler-error")

                                println "Warning: Moving $xmlFile to $errorDir"

                                errorDir.mkdir();

                                FileCopy.copy(xmlFile, new File(errorDir, xmlFile.getName()))

                        } catch(Exception eCopy) {

                                println "Error while moving "+xmlFile+" to "+errorDir+" : "+eCopy

                        }

                        if (parser != null) parser.close();

                        if (inputData != null) inputData.close();

                        return false;

                }

                if (nWords == 0) {

                        println "** no words written."

                }

                return nWords > 0;

        }

}