Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$

//

package org.txm.scripts.filters.Tokeniser

import java.io.File;

import java.util.regex.Pattern;

import java.util.regex.Matcher;

// TODO: Auto-generated Javadoc

/**

########################################################################

#                                                                      #

#  tokenization script for tagger preprocessing                        #

#  Author: Helmut Schmid, IMS, University of Stuttgart                 #

#          Serge Sharoff, University of Leeds                          #

#  Groovy Translation: mdecorde                                        #

#  Description:                                                        #

#  - splits input text into tokens (one token per line)                #

#  - cuts off punctuation, parentheses etc.                            #

#  - disambiguates periods                                             #

#  - preserves SGML markup                                             #

#                                                                      #

########################################################################

*/

class TTTokenizer {

        // characters which have to be cut off at the beginning of a word

        /** The P char. */

        def PChar="\\[{(\\`\"‚„†‡‹‘’“”•–—›";

        // characters which have to be cut off at the end of a word

        /** The F char. */

        def FChar="\\]}'`\"),;:!?%‚„…†‡‰‹‘’“”•–—›";

        // character sequences which have to be cut off at the beginning of a word

        /** The P clitic. */

        def PClitic = "";

        // character sequences which have to be cut off at the end of a word

        /** The F clitic. */

        def FClitic = "";

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                File input = new File("/home/mdecorde/xml/tokenizer/test.xml");

                File output = new File("/home/mdecorde/xml/tokenizer/test-t.xml");

                def tokenizer = new TTTokenizer("", null);

                tokenizer.run(input, output)

        }

        /** The Token. */

        def Token = [:];

        /**

         * Instantiates a new tT tokenizer.

         *

         * @param lang the lang

         * @param abbr the abbr

         */

        public TTTokenizer(String lang, File abbr) {

                switch(lang) {

                case "en": // English

                          FClitic = "'(s|re|ve|d|m|em|ll)|n\'t";

                          break;

                case "it": // Italian

                          PClitic = '[dD][ae]ll\'|[nN]ell\'|[Aa]ll\'|[lLDd]\'|[Ss]ull\'|[Qq]uest\'|[Uu]n\'|[Ss]enz\'|[Tt]utt\''

                          break;

                case "fr": // French

                          PClitic = '[dcjlmnstDCJLNMST]\'|[Qq]u\'|[Jj]usqu\'|[Ll]orsqu\'';

                          FClitic = '-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mmes?|-m\'|-moi|-nous|-on|-toi|-tu|-t\'|-vous|-en|-y|-ci|-l';

                          break;

                }

                if(abbr != null && abbr.canRead()) {

                        Reader reader = new FileReader(abbr);

                        String line = reader.readLine();

                        while(line != null) {

                                line = line.replaceAll("^[ \t\r\n]+", "");

                                line = line.replaceAll("^[ \t\r\n]+\$", "");

                                if(!(line.matches("^#") || line.matches("\\s\$"))) // ignore comments

                                {

                                        Token.put(line, 1);

                                }

                                line = reader.readLine();

                        }

                        reader.close();

                }

        }

        /** The m. */

        def m;

        /**

         * Run.

         *

         * @param input the input

         * @param output the output

         * @return true, if successful

         */

        public boolean run(File input, File output) {

                if(!input.canRead()){

                        println "Can't read input file: "+input

                        return false;

                }

                if(!output.getParentFile().canWrite()){

                        println "Can't write out file: "+input

                        return false;

                }

                Reader reader = new FileReader(input);

                Writer writer = new FileWriter(output);

                String line = reader.readLine();

                while(line != null)

                {

                        println "line: "+line

                        line = line.replaceAll("\t"," "); // replace newlines and tab characters with blanks

                        line = line.replaceAll("\n"," "); // replace newlines and tab characters with blanks

                        String replace = line.replaceAll(/(<[^<> ]*) ([^<>]*>)/, '$1ÿ$2');

                        while(replace != line) // replace blanks within SGML tags

                        {

                                line = replace;

                                replace = replace.replaceAll(/(<[^<> ]*) ([^<>]*>)/, '$1ÿ$2');

                        }

                        line = replace;

                        line = line.replaceAll(/ /, "þ"); // replace whitespace with a special character

                        // tr/\377\376/ \377/; // 377=ÿ , 376=þ

                        line = line.replaceAll(/ÿ/, " "); // restore SGML tags

                        line = line.replaceAll(/þ/, " ÿ"); // restore SGML tags

                        //line = line.replaceAll(/ÿ/," ");

                        // prepare SGML-Tags for tokenization

                        line = line.replaceAll(/(<[^<>]*>)/, 'ÿ$1ÿ');

                        line = line.replaceAll(/^ÿ/,"");

                        line = line.replaceAll(/ÿ$/,"");

                        line = line.replaceAll(/ÿÿÿ*/,"ÿ");

                        String[] split1 = line.split("ÿ");

                        boolean finish = true;

                        String suffix = "";

                        for(String s : split1)

                        {

                                if(s =~ /^<.*>$/) // SGML tag

                                {

                                        writer.write("$s\n");

                                }

                                else { // add a blank at the beginning and the end of each segment

                                        s = " $s ";

                                        // insert missing blanks after punctuation

                                        s = s.replaceAll(/(\.\.\.)/, " ... ");

                                        s = s.replaceAll(/([;\!\?])([^ ])/, '$1 $2');

                                        s = s.replaceAll(/([.,:])([^ 0-9.])/, '$1 $2');

                                        for(String s2 : s.split(" "))

                                        {

                                                finish = false;

                                                suffix = "";

                                                // separate punctuation and parentheses from words

                                                while(!finish)

                                                {

                                                        //println "suffix: "+suffix

                                                        finish = true;

                                                //         cut off preceding punctuation

                                                        if((m = (s2 =~ /^([$PChar])(.*)$/)))

                                                        {

                                                                s2 = m[0][2]

                                                                writer.write(m[0][1]+"\n")

                                                                finish = false;

                                                        }

                                                //         cut off trailing punctuation

                                                        if((m = (s2 =~ /^(.*)([$FChar])$/)))

                                                        {

                                                                s2 = m[0][1];

                                                                suffix = m[0][2] + "\n$suffix";

                                                                finish = false;

                                                                if(s2.length() == 0)

                                                                        writer.write(suffix)

                                                        }

                                                        // cut off trailing periods if punctuation precedes

                                                        if((m = (s2 =~ /([$FChar])\.$/))) {

                                                                suffix = ".\n$suffix";

                                                                if(s2.length() == 0)

                                                                {

                                                                         s2 = m[0][1];

                                                                }

                                                                else

                                                                {

                                                                        suffix = m[0][1]+"\n$suffix"

                                                                        s2 = "";

                                                                }

                                                                finish = false;

                                                        }

                                                }// end while

                                                // handle explicitly listed tokens

                                                if(Token.containsKey(s2))

                                                {

                                                        writer.write("$s2\n$suffix");

                                                        continue;

                                                }

                                                // abbreviations of the form A. or U.S.A.

                                                if(s2 =~ /^([A-Za-z-]\.)+$/)

                                                {

                                                        writer.write("$s2\n$suffix");

                                                        continue;

                                                }

                                                // disambiguate periods

                                                if((m = (s2 =~ /^(..*)\.$/)) &&

                                                                (s2 !="...") &&

                                                                !( s ==~ /^[0-9]+\.$/))

                                                {

                                                        s2 = m[0][1];

                                                        suffix = ".\n$suffix";

                                                        if(Token.containsKey(s2))

                                                        {

                                                                writer.write("$s2\n$suffix");

                                                                continue;

                                                        }

                                                }

                                                // cut off clitics

                                                if(PClitic.length() > 0)

                                                {

                                                        while((m = (s2 =~ /^($PClitic)(.*)/)))

                                                        {

                                                                s2 = m[0][2];

                                                                writer.write(m[0][1] + "\n")

                                                        }

                                                }

                                                if(FClitic.length() > 0)

                                                {

                                                        while((m = (s2 =~ /(.*)($FClitic)$/)))

                                                        {

                                                                s2 = m[0][1];

                                                                writer.write(m[0][2] + "\n")

                                                        }

                                                }

                                                if(s2.length() > 0)

                                                        writer.write("$s2\n$suffix");

                                        }

                                }

                        }

                        line = reader.readLine();

                }

                writer.close()

                reader.close()

        }

}