Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

// This file is part of the TXM platform.

//

// The TXM platform is free software: you can redistribute it and/or modif y

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// The TXM platform is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

//

//

//

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$

//

package org.txm.scripts.filters.Tokeniser;

import static groovy.transform.TypeCheckingMode.SKIP

import groovy.transform.CompileStatic

import java.util.regex.Matcher

import java.util.regex.Pattern

import javax.xml.stream.*

import org.txm.scripts.importer.graal.PersonalNamespaceContext

import org.txm.tokenizer.TokenizerClasses

@CompileStatic

public class SimpleTokenizerXml {

        /** The word_tags. */

        String word_tags = TokenizerClasses.word_tags;

        String word_element_to_create = TokenizerClasses.word_element_to_create;

        Pattern reg_word_tags;

        /** The intraword_tags. */

        String intraword_tags = TokenizerClasses.intraword_tags;

        /** The punct_strong. */

        String punct_strong = TokenizerClasses.punct_strong;

        /** The punct_all. */

        String punct_all = TokenizerClasses.punct_all;

        /** The word_chars. */

        String word_chars = TokenizerClasses.word_chars;

        String fclitics = null; // default behavior don't manage clitics

        String pclitics = null; // default behavior don't manage clitics

        /** The outside_text_tags_ignore_content. */

        String note_content = null;

        String outside_text_tags_keep_content = null // tag and content NOT removed but not tokenized

        String outside_text_tags = null // tag and content removed

        String startTag = null

        Pattern reg_note_content;

        Pattern reg_outside_text_tags_keep_content;

        Pattern reg_outside_text_tags;

        Pattern reg_startTag;

        Pattern reg_punct_other = Pattern.compile("\\p{P}")

        /** The DEBUG. */

        public boolean DEBUG = false;

        /** The outfile. */

        File outfile;

        /** The infile. */

        File infile;

        String lang;

        /** The buffer. */

        StringBuffer buffer;

        /** The writer. */

        XMLStreamWriter writer;

        BufferedOutputStream output;

        /** The parser. */

        XMLStreamReader parser

        /** The localname. */

        String localname;

        /** The prefix. */

        String prefix;

        String filename;

        def regElision = null;

        def reg3pts = null;

        def regPunct;

        def regFClitics = null ;

        def regPClitics = null ;

        String whitespaces;

        Pattern regWhitespaces;

        Pattern regLN;

        Pattern regCTRL;

        Pattern regSplitWhiteSpaces;

        public SimpleTokenizerXml(File infile, File outfile) {

                this(infile, outfile, "");

        }

        /**

         * Instantiates a new simple tokenizer xml.

         *

         * @param infile the infile

         * @param outfile the outfile

         */

        public SimpleTokenizerXml(File infile, File outfile, String lang) {

                this.lang = lang;

                if (lang != null)

                        if (lang.startsWith("en")) {

                                fclitics = TokenizerClasses.FClitic_en

                        } else if (lang.startsWith("fr")) {

                                fclitics = TokenizerClasses.FClitic_fr

                                pclitics = TokenizerClasses.PClitic_fr

                        } else if (lang.startsWith("gl")) {

                                fclitics = TokenizerClasses.FClitic_gl

                        } else if (lang.startsWith("it")) {

                                pclitics = TokenizerClasses.PClitic_it

                        }

                word_tags = TokenizerClasses.word_tags;

                reg_word_tags = Pattern.compile(word_tags);

                intraword_tags = TokenizerClasses.intraword_tags;

                punct_strong = TokenizerClasses.punct_strong;

                punct_all = TokenizerClasses.punct_all;

                word_chars = TokenizerClasses.word_chars;

                this.outfile = outfile;

                this.infile = infile;

                this.filename = infile.getName();

                int index = filename.lastIndexOf(".");

                if (index > 0) filename = filename.substring(0, index);

                String strRegElision = TokenizerClasses.regElision;

                if (strRegElision != null && strRegElision.length() > 0)

                        regElision = ~/^([\p{L}-]++${strRegElision}[\p{L}-]++)(.*)$/ // the test must start with the start of string  ^

                reg3pts = ~/^(.*?)(\.\.\.)(.*)$/

                String strRegPunct = TokenizerClasses.regPunct;

                if (strRegPunct != null && strRegPunct.length() > 0)

                        regPunct = ~/^(.*?)($strRegPunct)(.*)$/

                if (fclitics != null && fclitics.length() > 0)

                        regFClitics = ~/(.+)($fclitics)$/ // the test must end with the end of string $

                if (pclitics != null && pclitics.length() > 0)

                        regPClitics = /^($pclitics)(.*)/ // the test must start with the start of string  ^

                if (TokenizerClasses.whitespaces != null && TokenizerClasses.whitespaces.length() > 0)

                        regWhitespaces = ~TokenizerClasses.whitespaces;

                if (TokenizerClasses.whitespaces != null && TokenizerClasses.whitespaces.length() > 0)

                        regSplitWhiteSpaces = Pattern.compile(TokenizerClasses.whitespaces);

                regLN = Pattern.compile("/\n/");

                regCTRL = Pattern.compile("/\\p{C}/");

        }

        /**

         * Fill infos.

         *

         * @param event the event

         * @return the java.lang. object

         */

        public fillInfos(int event) {

                if (event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT) {

                        localname = parser.getLocalName();

                        prefix = parser.getPrefix();

                }

        }

        /**

         * Donothing: just write what is read

         *

         * special processing for words: prefix id attributes with "w_" and remove special characters like \n \t etc.

         *

         * @param event the event

         * @param wordid the wordid

         * @return the java.lang. object

         */

        public donothing(int event, Integer wordid) {

                if (event == XMLStreamConstants.START_ELEMENT ) {

                        localname = parser.getLocalName();

                        if (wordid != null) localname = word_element_to_create;

                        if (prefix != null && prefix.length() > 0)

                                writer.writeStartElement(prefix+":"+localname);

                        else

                                //                                if(namespace != null)

                                //                                        writer.writeStartElement(namespace, localname);

                                //                                else

                                writer.writeStartElement(localname);

                        //                        if(parser.getNamespaceCount() > 0)

                        //                                writer.writeDefaultNamespace(parser.getNamespaceURI(0))

                        //                        for(int i = 1 ; i < parser.getNamespaceCount() ; i++)

                        //                                writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));

                        String namespace_prefix;

                        for (int i = 0 ; i<   parser.getNamespaceCount() ; i++) {

                                namespace_prefix = parser.getNamespacePrefix(i);

                                if ((namespace_prefix != null)&&   (namespace_prefix.length()>   0)) {

                                        writer.writeNamespace(namespace_prefix, parser.getNamespaceURI(i));

                                } else {

                                        writer.writeDefaultNamespace(parser.getNamespaceURI(i));

                                }

                        }

                        String attrprefix, attname;

                        boolean hasId = false;

                        //boolean hasType = false

                        boolean hasN = false

                        for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

                                attname = parser.getAttributeLocalName(i);

                                attrprefix = parser.getAttributePrefix(i);

                                if ("id".equals(attname)) hasId = true;

                                //if ("type".equals(attname)) hasType = true;

                                if ("n".equals(attname)) hasN = true;

                                if (attrprefix != null && attrprefix.length() > 0)

                                        writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))

                                else

                                        writer.writeAttribute(attname, parser.getAttributeValue(i))

                        }

                        if (wordid != null && !hasId && localname == word_element_to_create)

                                writer.writeAttribute("id", "w_"+filename+"_"+wordcount);

                        if (!hasN && localname == word_element_to_create)

                                writer.writeAttribute("n", ""+wordcount);

                        writer.writeCharacters("\n");

                }

                else if(event == XMLStreamConstants.END_ELEMENT)

                {

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                }

                else if(event == XMLStreamConstants.CHARACTERS)

                {

                        //println parser.getText();

                        //writer.writeCharacters("𦟛");

                        // checks if the token starts with an high surrogate

                        //                        if(isHighSurrogate(parser.getText().charAt(0)))        {

                        //                                println "warning: invalid UTF-8 XML range, token " + parser.getText() + " has been replaced.";

                        //                                writer.writeCharacters("__invalidXMLChar__")

                        //                                //writer.writeCharacters("𦟛");

                        //                                println "high surrogate: " + Integer.toHexString((int)parser.getText().charAt(0));

                        //                                println "low surrogate: " + Integer.toHexString((int)parser.getText().charAt(1));

                        //                                int charSum = parser.getText().charAt(0) + parser.getText().charAt(1);

                        //                                println "char sum: " + charSum;

                        //                                println "test " + surrogatesPairToScalar(parser.getText().charAt(0), parser.getText().charAt(1));

                        //                                int scalar = surrogatesPairToScalar(parser.getText().charAt(0), parser.getText().charAt(1));

                        //                                //writer.writeCharacters(String.valueOf((char)112692));

                        //                                writer.writeCharacters("&#" + scalar + ";");

                        //                        }

                        //                        else

                        if (insideword) { // ensure there is not \t or \n in the word form value

                                writer.writeCharacters(parser.getText().trim().replace("\n", " ").replace("\t", " "));

                        } else {

                                writer.writeCharacters(parser.getText());

                        }

                }

        }

        /**

         * Converts the specified surrogates pair to scalar.

         * @param highSurrogate

         * @param lowSurrogate

         * @return

         */

        public int surrogatesPairToScalar(char highSurrogate, char lowSurrogate)        {

                return ((highSurrogate - 0xD800) * 0x400) + (lowSurrogate - 0xDC00) + 0x10000;

        }

        /**

         * Checks if the specified character is an high/leading surrogate.

         * @param character

         * @return

         */

        public boolean isHighSurrogate(char character)        {

                return (character >= 0xD800 && character <= 0xDBFF);

        }

        /** The wordcount. */

        int wordcount = 0;

        /** The ignorecontent. */

        boolean ignorecontent = true; // tokenization starts with element name matching $reg_startTag

        boolean insideword = false;

        /**

         * Process.

         *

         * @return true, if successful

         */

        public boolean process()

        {

                if (!infile.exists()) {

                        println "$infile does not exists"

                        return false;

                }

                XMLOutputFactory factory = XMLOutputFactory.newInstance();

                output = new BufferedOutputStream(new FileOutputStream(outfile))

                writer = factory.createXMLStreamWriter(output, "UTF-8")

                writer.setNamespaceContext(new PersonalNamespaceContext());

                def inputData = infile.toURI().toURL().openStream();

                def inputfactory = XMLInputFactory.newInstance();

                //inputfactory.setProperty("http://apache.org/xml/properties/input-buffer-size", new Integer(2048));

                //inputfactory.setExpandEntityReferences(false);

                parser = inputfactory.createXMLStreamReader(inputData, "UTF-8");

                //println "PARSER: "+parser.getClass()

                writer.writeStartDocument("UTF-8","1.0");

                writer.writeCharacters("\n");

                int previousEvent = 0;

                boolean startProcess = false;

                if (startTag == null) // if no startTag specified we process from the start

                        startProcess = true;

                ignorecontent = !startProcess;

                buffer = new StringBuffer();

                //println "process - start start tag: "+startTag+" startProcess: $startProcess"

//                println "reg_outside_text_tags_keep_content=$reg_outside_text_tags_keep_content"

//                println "reg_outside_text_tags=$reg_outside_text_tags"

//                println "reg_note_content=$reg_note_content"

                try {

                        for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                                if (!startProcess) {

                                        if (event == XMLStreamConstants.START_ELEMENT) {

                                                if (reg_startTag.matcher(parser.getLocalName()).matches()) {

                                                        startProcess = true

                                                        ignorecontent = false;

                                                }

                                        }

                                        if (!startProcess) {

                                                donothing(event, null);

                                                continue;

                                        }

                                }

                                if (previousEvent == XMLStreamConstants.CHARACTERS && previousEvent != event) {

                                        processWord(); // tokenize now!

                                        buffer.setLength(0);

                                }

                                fillInfos(event);//get localname and prefix

                                if (event == XMLStreamConstants.START_ELEMENT) {

                                        //println "Open: "+localname;

                                        localname = parser.getLocalName()

                                        if (reg_word_tags.matcher(localname).matches()) { // ignore the content of the word but keep counting

                                                //println "Found pretagged word";

                                                wordcount++;

                                                donothing(event, wordcount);

                                                //ignorecontent = true;

                                                insideword = true;

                                        } else if (reg_note_content != null && reg_note_content.matcher(localname).matches()) { // ignore the content of the tag ONLY

                                                // ignore the tag only

                                                donothing(event, null); // write the tag

                                                //println "IGNORING NOTE CONTENT OF "+localname

                                                ignorecontent = true;

                                        } else if (reg_outside_text_tags_keep_content != null && reg_outside_text_tags_keep_content.matcher(localname).matches()) { // ignore the content of the tag ONLY

                                                //         ignore the content only

                                                donothing(event, null); // write the tag

                                                //println "IGNORING CONTENT OF "+localname

                                                ignorecontent = true;

                                        } else if (reg_outside_text_tags != null && reg_outside_text_tags.matcher(localname).matches()) { // ignore the tag and its content of the tag

                                                goToEndOfElement(localname); // parse until the end of the element is passed

                                        } else {

                                                donothing(event, null);

                                        }

                                } else if(event == XMLStreamConstants.END_ELEMENT) {

                                        //println "Close: "+localname;

                                        localname = parser.getLocalName()

                                        if (reg_word_tags.matcher(localname).matches()) {

                                                //ignorecontent = false;

                                                insideword = false;

                                                writer.writeEndElement();

                                                writer.writeCharacters("\n");

                                        } else if (reg_outside_text_tags_keep_content != null && reg_outside_text_tags_keep_content.matcher(localname).matches()) { // ignore the content of the tag

                                                ignorecontent = false;

                                                donothing(event, null);

                                        } else if (reg_note_content != null && reg_note_content.matcher(localname).matches()) { // ignore the content of the tag ONLY

                                                ignorecontent = false;

                                                donothing(event, null);

                                        } else {

                                                donothing(event, null);

                                        }

                                } else if (event == XMLStreamConstants.CHARACTERS) {

                                        if (ignorecontent || insideword) {

                                                //println " dont tokenize chars: "+parser.getText().trim();

                                                donothing(event, null);

                                        } else {

                                                //println " process chars: "+parser.getText().trim();

                                                buffer.append(parser.getText());

                                                if (buffer.length() >= 128 && buffer.charAt(buffer.length()-1) == " ") {

                                                        processWord();

                                                        buffer = new StringBuffer();

                                                }

                                        }

                                } else if (event == XMLStreamConstants.COMMENT) {

                                        writer.writeComment(parser.getText())

                                } else if (event == XMLStreamConstants.DTD) {

                                        //println "DTD!";

                                } else {

                                        if (DEBUG) println "Warning in $infile: ignore XML event at location "+parser.getLocation()

                                }

                                previousEvent = event;

                        }

                        parser.close()

                        writer.close();

                        output.close();

                        inputData.close();

                } catch (Exception e) {

                        System.err.println("Error : "+infile);

                        e.printStackTrace();

                        if (writer != null) writer.close();

                        if (output != null) output.close();

                        if (parser != null) parser.close();

                        if (inputData != null) inputData.close();

                        return false;

                }

                return true;

        }

        /**

         * Set the element and content to ignore

         *

         * @param regexp

         */

        public void setOutSideTextTags(String regexp) {

                this.outside_text_tags = regexp;

                this.reg_outside_text_tags = Pattern.compile(outside_text_tags);

        }

        /**

         * Set element content to NOT tokenize

         *

         * @param regexp

         */

        public void setNote(String regexp) {

                this.note_content = regexp;

                this.reg_note_content = Pattern.compile(note_content);

        }

        /**

         * Set the element to ignore but not their content

         *

         * @param regexp

         */

        public void setOutSideTextTagsAndKeepContent(String regexp) {

                this.outside_text_tags_keep_content = regexp;

                this.reg_outside_text_tags_keep_content = Pattern.compile(outside_text_tags_keep_content);

        }

        protected void goToEndOfElement(String name) {

                //println "START ignoring tag and content of $name"

                def openedTags = []

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                        if (event == XMLStreamConstants.START_ELEMENT) {

                                openedTags << parser.getLocalName()

                                //println "append "+openedTags

                        } else if (event == XMLStreamConstants.END_ELEMENT) {

                                if (openedTags.size() == 0 && name == parser.getLocalName()) {

                                        //println "END ignoring tag and content of $name"

                                        return;

                                }

                                openedTags.pop()

                                //println "pop $openedTags"

                        }

                }

        }

        public final static String WHITESPACE = " ";

        public final static String EMPTY = "";

        /**

         * Process word.

         */

        protected void processWord() {

                String text = buffer.toString();//parser.getText().trim().replace("\t", " ");

                //if (DEBUG) println "-- chars: "+text+"--";

                text = regLN.matcher(text).replaceAll(WHITESPACE);

                text = regCTRL.matcher(text).replaceAll(EMPTY);                                                // remove ctrl characters

                if (regSplitWhiteSpaces != null) {

                        for (String s : regSplitWhiteSpaces.split(text)) {                // separate with unicode white spaces

                                //        if (DEBUG){println "process $s"}

                                        iterate(s);

                                }

                } else {

                        iterate(text);

                }

        }

        /**

         * Iterate. a String, should be called when a word is found in a String

         *

         * @param s the s

         * @return the java.lang. object

         */

        protected iterate(String s) {

                while (s != null && s.length() > 0) {

                //        if (DEBUG){println "  > $s"}

                        s = standardChecks(s);

                }

        }

        /**

         * Standard checks.

         *

         * @param s the s

         * @return the java.lang. object

         */

        //        @CompileStatic(SKIP)

        public String standardChecks(String s) {

                Matcher m;

                if (fclitics != null && (m = s =~ regFClitics) ) {

                //        if (DEBUG) println "CLITIC found: $s ->"+ m

                        iterate(m.group(1))

                        wordcount++;

                        writer.writeStartElement(word_element_to_create);

                        writeWordAttributes();// id

                        //writer.writeAttribute("type", "w");

                        writer.writeCharacters(m.group(2));

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                        return "";

                } else if (pclitics != null && (m = s =~ regPClitics) ) {

                        if (DEBUG) println "PCLITIC found: $s ->"+ m

                        wordcount++;

                        writer.writeStartElement(word_element_to_create);

                        writeWordAttributes();// id

                        //writer.writeAttribute("type", "w");

                        writer.writeCharacters(m.group(1));

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                        iterate(m.group(2))

                        return "";

                } else if (regElision != null && (m = s =~ regElision) ) {

                //        if (DEBUG) println "Elision found: $s ->"+ m.group(1)+" + "+m.group(2)+" + "+m.group(3)

                        //iterate(m.group(1))

//                        int sep = s.indexOf("'");

//                        if (sep < 0)

//                                sep = s.indexOf("’");

//                        if (sep < 0)

//                                sep = s.indexOf("‘");

                        wordcount++;

                        writer.writeStartElement(word_element_to_create);

                        writeWordAttributes();// id

                        //writer.writeAttribute("type", "w");

                        writer.writeCharacters(m.group(1));

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                        iterate(m.group(2))

                        return "";

                } else if (reg3pts != null && (m = s =~ reg3pts) )        {

                //        if(DEBUG){println "REG '...' found: $s -> "+m.group(1)+" + "+m.group(2)+" + "+m.group(3)}

                        iterate(m.group(1))

                        wordcount++;

                        writer.writeStartElement(word_element_to_create);

                        writeWordAttributes();// id

                        //writer.writeAttribute("type","pon");

                        writer.writeCharacters("...");

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                        return m.group(3);

                } else if (regPunct != null && (m = s =~ regPunct) ) {

                        if(DEBUG){println "PUNCT found: $s ->"+m.group(1)+" + "+m.group(2)+" + "+m.group(3)}

                        iterate(m.group(1));

                        wordcount++;

                        writer.writeStartElement(word_element_to_create);

                        writeWordAttributes();// id

                        //writer.writeAttribute("type","pon");

                        writer.writeCharacters(m.group(2));

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                        return m.group(3);

                } else {

        //                if(DEBUG){println "Other found: "+s}

                        wordcount++;

                        writer.writeStartElement(word_element_to_create);

                        writeWordAttributes();// id

//                        if (reg_punct_other.matcher(s).matches())

//                                writer.writeAttribute("type","pon");

//                        else

//                                writer.writeAttribute("type","w");

                        writer.writeCharacters(s);

                        writer.writeEndElement();

                        writer.writeCharacters("\n");

                        return "";

                }

        }

        /**

         * Write word attributes.

         *

         * @return the java.lang. object

         */

        protected writeWordAttributes() {

                writer.writeAttribute("id", "w_"+filename+"_"+wordcount);

                writer.writeAttribute("n",""+wordcount);

        }

        public void setStartTag(String tag)

        {

                this.startTag = tag;

                this.reg_startTag = Pattern.compile(startTag);

        }

        /**

         * Tokenize.

         *

         * @param str the str

         * @return the list

         */

        public List<String> tokenize(String str)

        {

                return str.tokenize()        // cut by whitespace

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                File SVNDIR = new File(System.getProperty("user.home"), "xml/tokenizernum");

                Runtime.getRuntime().exec("svn", "update", SVNDIR.getAbsolutePath());

                File inputFile = new File(SVNDIR, "baye1.xml")

                File outputFile = new File(SVNDIR, "baye1-t.xml")

                SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inputFile, outputFile, "fr")

                tokenizer.DEBUG = true

                tokenizer.setNote("note")

                //tokenizer.setOutSideTextTags("teiHeader")

                tokenizer.setOutSideTextTagsAndKeepContent("teiHeader")

                println outputFile.toString() + " : "+tokenizer.process();

                // FIXME: tests UTF-8 XML ranges

                //                File inputDir = new File("C:/Tools/Textometrie/___corpus/xml/test_bug_import_chinois/")

                //        File inputFile = new File(inputDir, "problem2.xml");

                /*

                File inputDir = new File("/home/mdecorde/xml/bvhepistemon2016")

                File outputDir = new File("/home/mdecorde/xml/bvhepistemon2016/out-fast")

                File outputDir2 = new File("/home/mdecorde/xml/bvhepistemon2016/out-slow")

                File outputDir3 = new File("/home/mdecorde/xml/bvhepistemon2016/out-fast2")

                File inputFile = new File(inputDir, "1542_RabelaisGrgt.xml");

                outputDir.deleteDir()

                outputDir2.deleteDir()

                outputDir3.deleteDir()

                outputDir.mkdir()

                outputDir2.mkdir()

                outputDir3.mkdir()

                long time;

                for (String lang : ["none"]) {

                        println "LANG: $lang"

                        time = System.currentTimeMillis();

                        File outputFile = new File(outputDir, "${lang}.xml")

                        SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inputFile, outputFile, lang)

                        tokenizer.process();

                        println "FAST $lang: "+(System.currentTimeMillis()-time)

//                        time = System.currentTimeMillis();

//                        File outputFile3 = new File(outputDir3, "${lang}.xml")

//                        FastSimpleTokenizerXml tokenizer3 = new FastSimpleTokenizerXml(inputFile, outputFile3, lang)

//                        tokenizer3.process();

//                        println "FAST2 $lang: "+(System.currentTimeMillis()-time)

//

//                        time = System.currentTimeMillis();

//                        File outputFile2 = new File(outputDir2, "${lang}.xml")

//                        OldSimpleTokenizerXml tokenizer2 = new OldSimpleTokenizerXml(inputFile, outputFile2, lang)

//                        tokenizer2.process();

//                        println "SLOW $lang: "+(System.currentTimeMillis()-time)

                }

*/

                println ""

                println "Done"

                //                String lang = "en"

                //                File inDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant")

                //                File outDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant-t")

                //                outDir.deleteDir()

                //                outDir.mkdir()

                //

                //                println "processing "+inDir.listFiles().size()+" files."

                //                for (def infile : inDir.listFiles()) {

                //                        if (!infile.getName().endsWith(".xml")) continue;

                //

                //                        print "."

                //                        File outfile = new File(outDir, infile.getName())

                //                        SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)

                //                        tokenizer.setDEBUG false

                //                        tokenizer.process();

                //                }

                //                println ""

                //                println "Done"

        }

}