/ - Diff - Plateforme TXM - Forge du Centre Blaise Pascal

Révision 3057

     package org.txm.tokenizer;
     import java.util.List;
     public interface StringTokenizer {
     	List<String> processText(String text);
+    }

     package org.txm.tokenizer;
     public class TTest {
     	String regex;
     	String type = "w";
     	int before=1, hit=2, after=3;
     	int before = 1, hit = 2, after = 3;
     	public TTest(String regex, String type, int before, int hit, int after) {
     		this.regex = regex;
     		this.type = type;
-...
     	public String getRegex() {
     		return regex;
+    	}
     	public TTest(String regex, int before, int hit, int after) {
     		this.regex = regex;
     		this.before = before;
     		this.hit = hit;
     		this.after = after;
+    	}
     	public TTest(String regex, String type) {
     		this.regex = regex;
     		this.type= type;
     		this.type = type;
+    	}
     	public TTest(String regex) {
     		this.regex = regex;
+    	}
     	/* (non-Javadoc)
     	/*
     	 * (non-Javadoc)
     	 * @see java.lang.Object#toString()
     	 */
     	@Override
     	public String toString() {
     		return "[regex=" + regex + ", type=" + type + ", before="+ before + ", hit=" + hit + ", after=" + after + "]";
     		return "[regex=" + regex + ", type=" + type + ", before=" + before + ", hit=" + hit + ", after=" + after + "]";
+    	}
+    }

     //
     package org.txm.tokenizer;
     import java.io.File;
     import java.io.FileWriter;
     import java.io.IOException;
     import java.lang.reflect.Field;
     import java.lang.reflect.Modifier;
     import java.util.ArrayList;
     import java.util.List;
     import java.util.prefs.Preferences;
     import java.util.regex.Pattern;
     import javax.xml.parsers.ParserConfigurationException;
     import org.eclipse.core.resources.ProjectScope;
     import org.eclipse.core.runtime.preferences.IEclipsePreferences;
     import org.osgi.service.prefs.BackingStoreException;
     import org.txm.utils.xml.DomUtils;
     import org.w3c.dom.Document;
     import org.w3c.dom.Element;
     import org.w3c.dom.Node;
     import org.w3c.dom.NodeList;
     import org.xml.sax.SAXException;
     import javax.xml.parsers.ParserConfigurationException;
     import javax.xml.stream.*;
     import java.net.URL;
     /**
      * Holder for tokenizer rules regexp
      */
     public class TokenizerClasses {
     	//// ROOTS ? ////
     	public static boolean debug = false;
     	/** The tag_all. */
     	public static String tag_all = null;
     	/** The Weblex enclitics. */
     	public static String enclitics = null;
     	public static String encliticsFR = null;
     	/** The TT enclitics. */
     	public static String FClitic_en = null;
     	public static String PClitic_fr = null;
     	public static String FClitic_fr = null;
     	public static String PClitic_it = null;
     	public static String FClitic_gl = null;
     	/** The div_tags. */
     	public static String div_tags = null;
     	/** The q_tags. */
     	public static String q_tags = null;
     	/** The extraword_tags. */
     	public static String extraword1_tags = null;
     	/** The corr_tags_no_seg. */
     	public static String corr_tags_no_seg = null;
     	/** The word_tags. */
     	public static String word_tags = null;
     	/** the element to create when a word is created */
     	public static String word_element_to_create = null;
     	/** The intraword_tags. */
     	public static String intraword_tags = null;
     	/** The punct_quotes. */
     	public static String punct_quotes = null;
     	/** The punct_strong1. */
     	public static String punct_strong1 = null;
     	/** The punct_strong2. */
     	public static String punct_strong2 = null;
     	/** The punct_paren_open1. */
     	public static String punct_paren_open1 = null;
     	/** The punct_paren_open2. */
     	public static String punct_paren_open2 = null;
     	/** The punct_paren_close1. */
     	public static String punct_paren_close1 = null;
     	/** The punct_paren_close2. */
     	public static String punct_paren_close2 = null;
     	/** The punct_weak. */
     	public static String punct_weak = null;
     	public static String entity = null;
     	public static String seg_tags = null;
     	//// COMBINAISONS ////
     	/** The corr_tags. */
     	public static String corr_tags = null;
     	/** The extraword_tags. */
     	public static String extraword_tags = null;
     	/** The punct_strong. */
     	public static String punct_strong = null;
     	/** The punct_paren_open. */
     	public static String punct_paren_open = null;
     	/** The punct_paren_close. */
     	public static String punct_paren_close = null;
     	/** The punct_paren. */
     	public static String punct_paren = null;
     	/** The punct_all. */
     	public static String punct_all = null;
     	/** The word_chars. */
     	public static String word_chars = null;
     	public static TTest[] tests = {
     			//		new TTest(/\A(.*-)(je|m[eo]i|tu|t[eo]i|lui|luy|ilz|ils?|no?u?s|vo?u?s|on|leurs|ce|ci|là|elles?)(.*)\Z/),
     			//		new TTest(/\A(.*)(-)()\Z/, "pon"),
     			//		//new TTest(/\A(.*)(Bande dessinée|eau de vie|machine à écrire|Moyen Âge|petit pois|poule d'eau|Avoir l'air|se rendre compte|faire semblant|prendre froid|s'en aller|Comme il faut|bon marché|bon enfant|en retard|en colère|à la mode|de bonne humeur|hors de propos|de travers|en vacances|de standing|à l'abandon|sans défense|pieds nus|Tout à fait|d'ores et déjà|de temps en temps|tout de suite|de gré ou de force|ne pas|ne jamais|ne plus|sans doute|on ne peut plus|Quelque chose|ce dernier|n'importe quoi|N'importe quel|beaucoup de|plus d'un|peu de|un tas de|un groupe de|je ne sais quel|De sorte que|bien que|parce que|c'est pourquoi|de même que|depuis que|Quant à|au-dessus de|par rapport à|à côté de|grâce à|hors de|le long de|Par ma barbe !|Mon dieu !|Eh bien !|Bonne nuit !|Pas de panique !)(.*)\Z/),
     			//		new TTest(/\A([Aa]ujourd'hui)(.*)/,0,1,2),
     			//
     			// new TTest(/\A(.*-)(je|m[eo]i|tu|t[eo]i|lui|luy|ilz|ils?|no?u?s|vo?u?s|on|leurs|ce|ci|là|elles?)(.*)\Z/),
     			// new TTest(/\A(.*)(-)()\Z/, "pon"),
     			// new TTest(/\A(.*)(Bande dessinée|eau de vie|machine à écrire|Moyen Âge|petit pois|poule d'eau|Avoir l'air|se rendre compte|faire semblant|prendre froid|s'en aller|Comme il faut|bon
     			// marché|bon enfant|en retard|en colère|à la mode|de bonne humeur|hors de propos|de travers|en vacances|de standing|à l'abandon|sans défense|pieds nus|Tout à fait|d'ores et déjà|de temps
     			// en temps|tout de suite|de gré ou de force|ne pas|ne jamais|ne plus|sans doute|on ne peut plus|Quelque chose|ce dernier|n'importe quoi|N'importe quel|beaucoup de|plus d'un|peu de|un tas
     			// de|un groupe de|je ne sais quel|De sorte que|bien que|parce que|c'est pourquoi|de même que|depuis que|Quant à|au-dessus de|par rapport à|à côté de|grâce à|hors de|le long de|Par ma
     			// barbe !|Mon dieu !|Eh bien !|Bonne nuit !|Pas de panique !)(.*)\Z/),
     			// new TTest(/\A([Aa]ujourd'hui)(.*)/,0,1,2),
     			// BFM ONLY
     			//		new TTest(/\A()([.·][^ .·]+[.·])(.*)\Z/, "num"),
     			// new TTest(/\A()([.·][^ .·]+[.·])(.*)\Z/, "num"),
     			// TXM REFMAN ONLY
     			//		new TTest(/\A(.*)($tag_all)(.*)\Z/, "tag"),
     			//		new TTest(/\A(.*)([0-9][0-9][\/][0-9][0-9][\/][0-9][0-9])(.*)\Z/, "date", 1,2,3),
     			//		new TTest(/\A(.*)([0-9][0-9]:[0-9][0-9]:[0-9][0-9])(.*)\Z/, "time", 1,2,3),
     			//		new TTest(/\A([^0-9]*)([0-9]+(\.[0-9]+[a-z]+))(.*)\Z/, "version", 1,2,4),
     			//		new TTest(/\A([^0-9]*)([0-9]+(\.[0-9]+)(\.[0-9]+)+)(\.[0-9]+[a-z]+)?(.*)\Z/, "version", 1,2,6),
     			//		new TTest(/\A()([a-z]++:\/\/)(.*)\Z/, "protocol", 1,2,3),
     			//		new TTest(/\A()([0-9]+([.,][0-9]+)?)()\Z/, "num", 1,2,4),
     			//		new TTest(/\A()(([A-Z]:)?([\/\\][^\/\\]+)+)()\Z/, "path", 1,2,5),
     			//		new TTest(/\A([\p{Ps}|\p{Pe}|\p{Pi}|\p{Pf}|\p{Po}|\p{S}]*)(\p{L}++['‘’])(.*)\Z/, "w", 1, 2, 3),
     			//		new TTest(/\A(.*)(\.\.\.)(.*)\Z/, "pon"),
     			//		new TTest(/\A(.*)(\p{Ps}|\p{Pe}|\p{Pi}|\p{Pf}|\p{Po}|\p{S})(.*)\Z/, "pon")
     			// new TTest(/\A(.*)($tag_all)(.*)\Z/, "tag"),
     			// new TTest(/\A(.*)([0-9][0-9][\/][0-9][0-9][\/][0-9][0-9])(.*)\Z/, "date", 1,2,3),
     			// new TTest("^(.*)([0-9][0-9]:[0-9][0-9]:[0-9][0-9])(.*)$", "time", 1, 2, 3),
     			// new TTest("^([^0-9]*)([0-9]+(\\.[0-9]+)?(\\.[0-9]+[a-z]*))(.*)$", "version", 1, 2, 5),
     			// new TTest(/\A([^0-9]*)([0-9]+(\.[0-9]+)(\.[0-9]+)+)(\.[0-9]+[a-z]+)?(.*)\Z/, "version", 1,2,6),
     			// new TTest("^(.*)(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)(.*)$", "url", 1, 2, 5),
     			// new TTest(/\A()([0-9]+([.,][0-9]+)?)()\Z/, "num", 1,2,4),
     			// new TTest(/\A()(([A-Z]:)?([\/\\][^\/\\]+)+)()\Z/, "path", 1,2,5),
     			// new TTest(/\A([\p{Ps}|\p{Pe}|\p{Pi}|\p{Pf}|\p{Po}|\p{S}]*)(\p{L}++['‘’])(.*)\Z/, "w", 1, 2, 3),
     			// new TTest(/\A(.*)(\.\.\.)(.*)\Z/, "pon"),
     			// new TTest(/\A(.*)(\p{Ps}|\p{Pe}|\p{Pi}|\p{Pf}|\p{Po}|\p{S})(.*)\Z/, "pon")
     	};
     	public static String whitespaces = null;
     	public static String regElision = null;
     	public static String regPunct = null;
     	static {
     		reset(); // Initialize static fields
+    	}
     	//punct_strong1 = ".!?";
     	//punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
     	//punct_strong = "[$punct_strong1]|$punct_strong2";
     	//punct_paren_open1 = "«";
     	//punct_paren_open2 = "<<|``|\\(|\\[|\\{";
     	//punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
     	//punct_paren_close1 = "»";
     	//punct_paren_close2 = ">>|''|‘‘|’’|\\)|\\]|\\}";
     	//punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
     	//punct_paren = "$punct_paren_open|$punct_paren_close";
     	//punct_weak = ",;∼ˆ·*:\"`'“”\\+±=/\\-≤≥<>\\—_ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
     	//punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
     	//word_chars = "[^ $punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|&[^;]+;";
     	// punct_strong1 = ".!?";
     	// punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
     	// punct_strong = "[$punct_strong1]|$punct_strong2";
     	// punct_paren_open1 = "«";
     	// punct_paren_open2 = "<<|``|\\(|\\[|\\{";
     	// punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
     	// punct_paren_close1 = "»";
     	// punct_paren_close2 = ">>|''|‘‘|’’|\\)|\\]|\\}";
     	// punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
     	// punct_paren = "$punct_paren_open|$punct_paren_close";
     	// punct_weak = ",;∼ˆ·*:\"`'“”\\+±=/\\-≤≥<>\\—_ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
     	// punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
     	// word_chars = "[^ $punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|&[^;]+;";
     	public static void reset() {
     		tag_all = "<[A-Za-z][^>]+>";
-...
     		FClitic_fr = "-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mêmes?|-m[\'‘’]|-moi|-nous|-on|-toi|-tu|-t[\'‘’]|-vous|-en|-y|-ci|-là";
     		PClitic_it = "[dD][ae]ll[\'‘’]|[nN]ell[\'‘’]|[Aa]ll[\'‘’]|[lLDd][\'‘’]|[Ss]ull[\'‘’]|[Qq]uest[\'‘’]|[Uu]n[\'‘’]|[Ss]enz[\'‘’]|[Tt]utt[\'‘’]";
     		FClitic_gl = "-la|-las|-lo|-los|-nos";
     		whitespaces = "[\\p{Z}\\p{C}]+";
     		regElision = "['‘’]";
     		regPunct = "[\\p{Ps}\\p{Pe}\\p{Pi}\\p{Pf}\\p{Po}\\p{S}]";
     		recombine();
+    	}
     	public static void recombine() {
     		corr_tags = ""+corr_tags_no_seg+"|"+seg_tags+"";
     		extraword_tags = ""+div_tags+"|"+q_tags+"|"+extraword1_tags+"";
     		punct_strong = "["+punct_strong1+"]|"+punct_strong2+"";
     		punct_paren_open = ""+punct_paren_open1+"|"+punct_paren_open2+"";
     		punct_paren_close = ""+punct_paren_close1+"|"+punct_paren_close2+"";
     		punct_paren = ""+punct_paren_open+"|"+punct_paren_close+"";
     		punct_all = ""+punct_strong+"|"+punct_paren+"|["+punct_weak+"]";
     		word_chars = "[^ "+punct_quotes+""+punct_strong1+""+punct_paren_open1+""+punct_paren_close1+""+punct_weak+"]+|"+entity+"";
     		corr_tags = "" + corr_tags_no_seg + "|" + seg_tags + "";
     		extraword_tags = "" + div_tags + "|" + q_tags + "|" + extraword1_tags + "";
     		punct_strong = "[" + punct_strong1 + "]|" + punct_strong2 + "";
     		punct_paren_open = "" + punct_paren_open1 + "|" + punct_paren_open2 + "";
     		punct_paren_close = "" + punct_paren_close1 + "|" + punct_paren_close2 + "";
     		punct_paren = "" + punct_paren_open + "|" + punct_paren_close + "";
     		punct_all = "" + punct_strong + "|" + punct_paren + "|[" + punct_weak + "]";
     		word_chars = "[^ " + punct_quotes + "" + punct_strong1 + "" + punct_paren_open1 + "" + punct_paren_close1 + "" + punct_weak + "]+|" + entity + "";
+    	}
     	/**
     	 * To dom.
+    	 *
-...
     		tokenizer.setAttribute("onlyThoseTests", "false");
     		parent.appendChild(tokenizer);
     		//String tag_all = "<[^>]+>";
     		// String tag_all = "<[^>]+>";
     		Element p = doc.createElement("param");
     		p.setAttribute("key","tag_all");
     		p.setAttribute("key", "tag_all");
     		p.setTextContent(tag_all);
     		tokenizer.appendChild(p);
     		//String enclitics = "je|m[eo]i|tu|t[eo]i|il|lui|luy|ilz|ils|no?u?s|vo?u?s|on|ce|ci|là|elles?"; // FRO
     		// String enclitics = "je|m[eo]i|tu|t[eo]i|il|lui|luy|ilz|ils|no?u?s|vo?u?s|on|ce|ci|là|elles?"; // FRO
     		p = doc.createElement("param");
     		p.setAttribute("key","enclitics");
     		p.setAttribute("key", "enclitics");
     		p.setTextContent(enclitics);
     		tokenizer.appendChild(p);
     		//String encliticsFR = "je|tu|il|elle|on|nous|vous|ils|elles|toi|moi|en|y|t|leur|lui|le|la|les"; // FR
     		// String encliticsFR = "je|tu|il|elle|on|nous|vous|ils|elles|toi|moi|en|y|t|leur|lui|le|la|les"; // FR
     		p = doc.createElement("param");
     		p.setAttribute("key","encliticsFR");
     		p.setAttribute("key", "encliticsFR");
     		p.setTextContent(encliticsFR);
     		tokenizer.appendChild(p);
     		//String div_tags = "TEI|text|front|body|div|div1|div2|div3|div4|div5|div6|back|head|trailer|p|ab|sp|speaker|list|notice|bibl";
     		// String div_tags = "TEI|text|front|body|div|div1|div2|div3|div4|div5|div6|back|head|trailer|p|ab|sp|speaker|list|notice|bibl";
     		p = doc.createElement("param");
     		p.setAttribute("key","div_tags");
     		p.setAttribute("key", "div_tags");
     		p.setTextContent(div_tags);
     		tokenizer.appendChild(p);
     		//String q_tags = "q|quote|item|stage";
     		// String q_tags = "q|quote|item|stage";
     		p = doc.createElement("param");
     		p.setAttribute("key","q_tags");
     		p.setAttribute("key", "q_tags");
     		p.setTextContent(q_tags);
     		tokenizer.appendChild(p);
     		//String extraword_tags = "$div_tags|$q_tags|expan|pb|lb|milestone|gap|note|s|locus|title|ref|hi|witDetail";
     		// String extraword_tags = "$div_tags|$q_tags|expan|pb|lb|milestone|gap|note|s|locus|title|ref|hi|witDetail";
     		p = doc.createElement("param");
     		p.setAttribute("key","extraword_tags");
     		p.setAttribute("key", "extraword_tags");
     		p.setTextContent(extraword_tags);
     		tokenizer.appendChild(p);
     		//String corr_tags_no_seg = "expan|unclear|choice|corr|sic|reg|orig|foreign|hi|title|name|supplied|subst|add|del|damage|date|idno";
     		// String corr_tags_no_seg = "expan|unclear|choice|corr|sic|reg|orig|foreign|hi|title|name|supplied|subst|add|del|damage|date|idno";
     		p = doc.createElement("param");
     		p.setAttribute("key","enclitics");
     		p.setAttribute("key", "enclitics");
     		p.setTextContent(enclitics);
     		tokenizer.appendChild(p);
     		//String corr_tags = "$corr_tags_no_seg|seg";
     		// String corr_tags = "$corr_tags_no_seg|seg";
     		p = doc.createElement("param");
     		p.setAttribute("key","corr_tags");
     		p.setAttribute("key", "corr_tags");
     		p.setTextContent(corr_tags);
     		tokenizer.appendChild(p);
     		//String word_tags = "w|abbr|num";
     		// String word_tags = "w|abbr|num";
     		p = doc.createElement("param");
     		p.setAttribute("key","word_tags");
     		p.setAttribute("key", "word_tags");
     		p.setTextContent(word_tags);
     		tokenizer.appendChild(p);
     		//String intraword_tags = "c|ex";
     		// String intraword_tags = "c|ex";
     		p = doc.createElement("param");
     		p.setAttribute("key","intraword_tags");
     		p.setAttribute("key", "intraword_tags");
     		p.setTextContent(intraword_tags);
     		tokenizer.appendChild(p);
     		//String punct_strong1 = ".!?";
     		// String punct_strong1 = ".!?";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_strong1");
     		p.setAttribute("key", "punct_strong1");
     		p.setTextContent(punct_strong1);
     		tokenizer.appendChild(p);
     		//String punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
     		// String punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_strong2");
     		p.setAttribute("key", "punct_strong2");
     		p.setTextContent(punct_strong2);
     		tokenizer.appendChild(p);
     		//String punct_strong = "[$punct_strong1]|$punct_strong2";
     		// String punct_strong = "[$punct_strong1]|$punct_strong2";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_strong");
     		p.setAttribute("key", "punct_strong");
     		p.setTextContent(punct_strong);
     		tokenizer.appendChild(p);
     		//String punct_paren_open1 = "«";
     		// String punct_paren_open1 = "«";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren_open1");
     		p.setAttribute("key", "punct_paren_open1");
     		p.setTextContent(punct_paren_open1);
     		tokenizer.appendChild(p);
     		//String punct_paren_open2 = "<<|``|\\(|\\[|\\{";
     		// String punct_paren_open2 = "<<|``|\\(|\\[|\\{";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren_open2");
     		p.setAttribute("key", "punct_paren_open2");
     		p.setTextContent(punct_paren_open2);
     		tokenizer.appendChild(p);
     		//String punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
     		// String punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren_open");
     		p.setAttribute("key", "punct_paren_open");
     		p.setTextContent(punct_paren_open);
     		tokenizer.appendChild(p);
     		//String punct_paren_close1 = "»";
     		// String punct_paren_close1 = "»";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren_close1");
     		p.setAttribute("key", "punct_paren_close1");
     		p.setTextContent(punct_paren_close1);
     		tokenizer.appendChild(p);
     		//String punct_paren_close2 = ">>|''|‘‘|’’|\\)|\\]|\\}";
     		// String punct_paren_close2 = ">>|''|‘‘|’’|\\)|\\]|\\}";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren_close2");
     		p.setAttribute("key", "punct_paren_close2");
     		p.setTextContent(punct_paren_close2);
     		tokenizer.appendChild(p);
     		//String punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
     		// String punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren_close");
     		p.setAttribute("key", "punct_paren_close");
     		p.setTextContent(punct_paren_close);
     		tokenizer.appendChild(p);
     		//String punct_paren = "$punct_paren_open|$punct_paren_close";
     		// String punct_paren = "$punct_paren_open|$punct_paren_close";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_paren");
     		p.setAttribute("key", "punct_paren");
     		p.setTextContent(punct_paren);
     		tokenizer.appendChild(p);
     		//String punct_weak = ",;∼ˆ·*:\"`'“”\\+±=/\\-≤≥<>\\—_ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
     		// String punct_weak = ",;∼ˆ·*:\"`'“”\\+±=/\\-≤≥<>\\—_ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_weak");
     		p.setAttribute("key", "punct_weak");
     		p.setTextContent(punct_weak);
     		tokenizer.appendChild(p);
     		//String punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
     		// String punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
     		p = doc.createElement("param");
     		p.setAttribute("key","punct_all");
     		p.setAttribute("key", "punct_all");
     		p.setTextContent(punct_all);
     		tokenizer.appendChild(p);
     		//String word_chars = "[^ $punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|&[^;]+;";
     		// String word_chars = "[^ $punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|&[^;]+;";
     		p = doc.createElement("param");
     		p.setAttribute("key","word_chars");
     		p.setAttribute("key", "word_chars");
     		p.setTextContent(word_chars);
     		tokenizer.appendChild(p);
     		//whitespaces = "[\\p{Z}\\p{C}]+";
     		// whitespaces = "[\\p{Z}\\p{C}]+";
     		p = doc.createElement("param");
     		p.setAttribute("key","whitespaces");
     		p.setAttribute("key", "whitespaces");
     		p.setTextContent(whitespaces);
     		tokenizer.appendChild(p);
     		//regElision = "['‘’]"
     		// regElision = "['‘’]"
     		p = doc.createElement("param");
     		p.setAttribute("key","regElision");
     		p.setAttribute("key", "regElision");
     		p.setTextContent(regElision);
     		tokenizer.appendChild(p);
     		//regPunct = "[\\p{Ps}\\p{Pe}\\p{Pi}\\p{Pf}\\p{Po}\\p{S}]"
     		// regPunct = "[\\p{Ps}\\p{Pe}\\p{Pi}\\p{Pf}\\p{Po}\\p{S}]"
     		p = doc.createElement("param");
     		p.setAttribute("key","regPunct");
     		p.setAttribute("key", "regPunct");
     		p.setTextContent(regPunct);
     		tokenizer.appendChild(p);
+    	}
     	/**
     	 * Dump.
     	 */
     	public static void dump() {
     		System.out.println("BRUT");
     		System.out.println("tag_all = "+tag_all+"");
     		System.out.println("enclitics = "+enclitics+"");
     		System.out.println("encliticsFR = "+encliticsFR+"");
     		System.out.println("div_tags = "+div_tags+"");
     		System.out.println("q_tags = "+q_tags+"");
     		System.out.println("extraword1_tags = "+extraword1_tags+"");
     		System.out.println("corr_tags_no_seg = "+corr_tags_no_seg+"");
     		System.out.println("word_tags = "+word_tags+"");
     		System.out.println("intraword_tags = "+intraword_tags+"");
     		System.out.println("punct_quotes = "+punct_quotes+"");
     		System.out.println("punct_strong1 = "+punct_strong1+"");
     		System.out.println("punct_strong2 = "+punct_strong2+"");
     		System.out.println("punct_paren_open1 = "+punct_paren_open1+"");
     		System.out.println("punct_paren_open2 = "+punct_paren_open2+"");
     		System.out.println("punct_paren_close1 = "+punct_paren_close1+"");
     		System.out.println("punct_paren_close2 = "+punct_paren_close2+"");
     		System.out.println("punct_weak = "+punct_weak+"");
     		System.out.println("entity = "+entity+"");
     		System.out.println("seg_tags = "+seg_tags+"");
     		System.out.println("tag_all = " + tag_all + "");
     		System.out.println("enclitics = " + enclitics + "");
     		System.out.println("encliticsFR = " + encliticsFR + "");
     		System.out.println("div_tags = " + div_tags + "");
     		System.out.println("q_tags = " + q_tags + "");
     		System.out.println("extraword1_tags = " + extraword1_tags + "");
     		System.out.println("corr_tags_no_seg = " + corr_tags_no_seg + "");
     		System.out.println("word_tags = " + word_tags + "");
     		System.out.println("intraword_tags = " + intraword_tags + "");
     		System.out.println("punct_quotes = " + punct_quotes + "");
     		System.out.println("punct_strong1 = " + punct_strong1 + "");
     		System.out.println("punct_strong2 = " + punct_strong2 + "");
     		System.out.println("punct_paren_open1 = " + punct_paren_open1 + "");
     		System.out.println("punct_paren_open2 = " + punct_paren_open2 + "");
     		System.out.println("punct_paren_close1 = " + punct_paren_close1 + "");
     		System.out.println("punct_paren_close2 = " + punct_paren_close2 + "");
     		System.out.println("punct_weak = " + punct_weak + "");
     		System.out.println("entity = " + entity + "");
     		System.out.println("seg_tags = " + seg_tags + "");
     		System.out.println("COMBINED");
     		System.out.println("corr_tags = "+corr_tags+"");
     		System.out.println("extraword_tags = "+extraword_tags+"");
     		System.out.println("punct_strong = "+punct_strong+"");
     		System.out.println("punct_paren_open = "+punct_paren_open+"");
     		System.out.println("punct_paren_close = "+punct_paren_close+"");
     		System.out.println("punct_paren = "+punct_paren+"");
     		System.out.println("punct_all = "+punct_all+"");
     		System.out.println("word_chars = "+word_chars+"");
     		System.out.println("whitespaces = "+whitespaces+"");
     		System.out.println("regElision = "+regElision+"");
     		System.out.println("regPunct = "+regPunct+"");
     		System.out.println("corr_tags = " + corr_tags + "");
     		System.out.println("extraword_tags = " + extraword_tags + "");
     		System.out.println("punct_strong = " + punct_strong + "");
     		System.out.println("punct_paren_open = " + punct_paren_open + "");
     		System.out.println("punct_paren_close = " + punct_paren_close + "");
     		System.out.println("punct_paren = " + punct_paren + "");
     		System.out.println("punct_all = " + punct_all + "");
     		System.out.println("word_chars = " + word_chars + "");
     		System.out.println("whitespaces = " + whitespaces + "");
     		System.out.println("regElision = " + regElision + "");
     		System.out.println("regPunct = " + regPunct + "");
     		System.out.println("TESTS");
     		for (TTest test : tests) {
     			System.out.println(" "+test+"");
     			System.out.println(" " + test + "");
+    		}
+    	}
     	public static boolean isValid() {
     		try { Pattern.compile(tag_all); }
     		catch (Exception e) { System.out.println("tag_all="+tag_all+": "+e);return false; }
     		try { Pattern.compile(enclitics); }
     		catch (Exception e) { System.out.println("enclitics="+enclitics+": "+e);return false; }
     		try { Pattern.compile(encliticsFR); }
     		catch (Exception e) { System.out.println("encliticsFR="+encliticsFR+": "+e);return false; }
     		try { Pattern.compile(div_tags); }
     		catch (Exception e) { System.out.println("div_tags="+div_tags+": "+e);return false; }
     		try { Pattern.compile(q_tags); }
     		catch (Exception e) { System.out.println("q_tags="+q_tags+": "+e);return false; }
     		try { Pattern.compile(extraword1_tags); }
     		catch (Exception e) { System.out.println("extraword1_tags="+extraword1_tags+": "+e);return false; }
     		try { Pattern.compile(corr_tags_no_seg); }
     		catch (Exception e) { System.out.println("corr_tags_no_seg="+corr_tags_no_seg+": "+e);return false; }
     		try { Pattern.compile(word_tags); }
     		catch (Exception e) { System.out.println("word_tags="+word_tags+": "+e);return false; }
     		try { Pattern.compile(intraword_tags); }
     		catch (Exception e) { System.out.println("intraword_tags="+intraword_tags+": "+e);return false; }
     		try { Pattern.compile(punct_quotes); }
     		catch (Exception e) { System.out.println("punct_quotes="+punct_quotes+": "+e);return false; }
     		try { Pattern.compile(punct_strong1); }
     		catch (Exception e) { System.out.println("punct_strong1="+punct_strong1+": "+e);return false; }
     		try { Pattern.compile(punct_strong2); }
     		catch (Exception e) { System.out.println("punct_strong2="+punct_strong2+": "+e);return false; }
     		try { Pattern.compile(punct_paren_open1); }
     		catch (Exception e) { System.out.println("punct_paren_open1="+punct_paren_open1+": "+e);return false; }
     		try { Pattern.compile(punct_paren_open2); }
     		catch (Exception e) { System.out.println("punct_paren_open2="+punct_paren_open2+": "+e);return false; }
     		try { Pattern.compile(punct_paren_close1); }
     		catch (Exception e) { System.out.println("punct_paren_close1="+punct_paren_close1+": "+e);return false; }
     		try { Pattern.compile(punct_paren_close2); }
     		catch (Exception e) { System.out.println("punct_paren_close2="+punct_paren_close2+": "+e);return false; }
     		try { Pattern.compile(punct_weak); }
     		catch (Exception e) { System.out.println("punct_weak="+punct_weak+": "+e);return false; }
     		try { Pattern.compile(entity); }
     		catch (Exception e) { System.out.println("entity="+entity+": "+e);return false; }
     		try { Pattern.compile(seg_tags); }
     		catch (Exception e) { System.out.println("seg_tags="+seg_tags+": "+e);return false; }
     		try { Pattern.compile(corr_tags); }
     		catch (Exception e) { System.out.println("corr_tags: "+e);return false; }
     		try { 	Pattern.compile(extraword_tags); }
     		catch (Exception e) { System.out.println("extraword_tags: "+e);return false; }
     		try {
     			Pattern.compile(tag_all);
+    		}
     		catch (Exception e) {
     			System.out.println("tag_all=" + tag_all + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(enclitics);
+    		}
     		catch (Exception e) {
     			System.out.println("enclitics=" + enclitics + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(encliticsFR);
+    		}
     		catch (Exception e) {
     			System.out.println("encliticsFR=" + encliticsFR + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(div_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("div_tags=" + div_tags + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(q_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("q_tags=" + q_tags + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(extraword1_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("extraword1_tags=" + extraword1_tags + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(corr_tags_no_seg);
+    		}
     		catch (Exception e) {
     			System.out.println("corr_tags_no_seg=" + corr_tags_no_seg + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(word_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("word_tags=" + word_tags + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(intraword_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("intraword_tags=" + intraword_tags + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_quotes);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_quotes=" + punct_quotes + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_strong1);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_strong1=" + punct_strong1 + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_strong2);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_strong2=" + punct_strong2 + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_paren_open1);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren_open1=" + punct_paren_open1 + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_paren_open2);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren_open2=" + punct_paren_open2 + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_paren_close1);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren_close1=" + punct_paren_close1 + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_paren_close2);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren_close2=" + punct_paren_close2 + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_weak);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_weak=" + punct_weak + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(entity);
+    		}
     		catch (Exception e) {
     			System.out.println("entity=" + entity + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(seg_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("seg_tags=" + seg_tags + ": " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(corr_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("corr_tags: " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(extraword_tags);
+    		}
     		catch (Exception e) {
     			System.out.println("extraword_tags: " + e);
     			return false;
+    		}
     		if (punct_strong != null)
     			try { 	Pattern.compile(punct_strong); }
     		catch (Exception e) { System.out.println("punct_strong: "+e);return false; }
     		try { Pattern.compile(punct_paren_open); }
     		catch (Exception e) { System.out.println("punct_paren_open: "+e);return false; }
     		try { Pattern.compile(punct_paren_close); }
     		catch (Exception e) { System.out.println("punct_paren_close: "+e);return false; }
     		try { Pattern.compile(punct_paren); }
     		catch (Exception e) { System.out.println("punct_paren: "+e);return false; }
     		try { Pattern.compile(punct_all); }
     		catch (Exception e) { System.out.println("punct_all: "+e);return false; }
     		try { Pattern.compile(word_chars); }
     		catch (Exception e) { System.out.println("word_chars: "+e);return false; }
     			try {
     			Pattern.compile(punct_strong);
+    			}
     			catch (Exception e) {
     			System.out.println("punct_strong: " + e);
     			return false;
+    			}
     		try {
     			Pattern.compile(punct_paren_open);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren_open: " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_paren_close);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren_close: " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_paren);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_paren: " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(punct_all);
+    		}
     		catch (Exception e) {
     			System.out.println("punct_all: " + e);
     			return false;
+    		}
     		try {
     			Pattern.compile(word_chars);
+    		}
     		catch (Exception e) {
     			System.out.println("word_chars: " + e);
     			return false;
+    		}
     		if (regPunct != null)
     			try { Pattern.compile(regPunct); }
     		catch (Exception e) { System.out.println("regPunct="+regPunct+": "+e);return false; }
     			try {
     			Pattern.compile(regPunct);
+    			}
     			catch (Exception e) {
     			System.out.println("regPunct=" + regPunct + ": " + e);
     			return false;
+    			}
     		if (regElision != null)
     			try { Pattern.compile(regElision); }
     		catch (Exception e) { System.out.println("regElision="+regElision+": "+e);return false; }
     			try {
     			Pattern.compile(regElision);
+    			}
     			catch (Exception e) {
     			System.out.println("regElision=" + regElision + ": " + e);
     			return false;
+    			}
     		if (whitespaces != null)
     			try { Pattern.compile(whitespaces); }
     		catch (Exception e) { System.out.println("whitespaces="+whitespaces+": "+e);return false; }
     		for(TTest test : tests) {
     			try { Pattern.compile(test.getRegex()); }
     			catch (Exception e) { System.out.println(""+test.getRegex()+": "+e);return false; }
     			try {
     			Pattern.compile(whitespaces);
+    			}
     			catch (Exception e) {
     			System.out.println("whitespaces=" + whitespaces + ": " + e);
     			return false;
+    			}
     		for (TTest test : tests) {
     			try {
     				Pattern.compile(test.getRegex());
+    			}
     			catch (Exception e) {
     				System.out.println("" + test.getRegex() + ": " + e);
     				return false;
+    			}
+    		}
     		return true;
+    	}
     	/**
     	 * old way used in TXM 0.7.9 AND in corpus 0.7.9 -> 0.8.0 restoration
+    	 *
     	 * @param tokenizerElement
     	 * @return
     	 */
     	@Deprecated
     	public static boolean loadFromNode(Element tokenizerElement) {
     		//load params
     		// load params
     		String tmp_strong_punct = null;
     		NodeList params = tokenizerElement.getElementsByTagName("param");
     		for (int i = 0 ; i < params.getLength() ; i++) {
     		for (int i = 0; i < params.getLength(); i++) {
     			Element param = (Element) params.item(i);
     			String key = param.getAttribute("name");
     			String value = param.getAttribute("value");
     			if (value == null || value.length() == 0) value = param.getTextContent();
     			if (value.length() == 0) value = null;
     			if (debug) System.out.println(" Tokenizer parametrized with "+key+"="+value+"");
     			if (debug) System.out.println(" Tokenizer parametrized with " + key + "=" + value + "");
     			if (key.equals("tag_all"))
     				tag_all = value;
     			else if (key.equals("enclitics"))
-...
     				punct_paren_close1 = value;
     			else if (key.equals("punct_paren_close2"))
     				punct_paren_close2 = value;
     			else if (key .equals("punct_weak"))
     			else if (key.equals("punct_weak"))
     				punct_weak = value;
     			else if (key.equals("entity"))
     				entity = value;
-...
     			else if (key.equals("punct_strong")) // this is temporary
     				tmp_strong_punct = value; // this is temporary
     			else
     				System.out.println("MISSING TOKENIZER KEY: "+key);
     				System.out.println("MISSING TOKENIZER KEY: " + key);
+    		}
     		//recombine
     		// recombine
     		recombine();
     		if (tmp_strong_punct != null) punct_strong = tmp_strong_punct;  // this is temporary
     		String shouldResetTests = tokenizerElement.getAttribute("onlyThoseTests");
     		if ("true".equals(shouldResetTests)) {
     			System.out.println("Warning: tokenizer only using import parameters tests");
     			//tests = new ArrayList<>();
     			// tests = new ArrayList<>();
+    		}
     		NodeList testsList = tokenizerElement.getElementsByTagName("test");
     		if (testsList.getLength() > 0) {
     			//System.out.println("Add "+testsList.getLength()+" tests to the tokenizer"
     			for (int i = 0 ; i < testsList.getLength() ; i++) {
     			// System.out.println("Add "+testsList.getLength()+" tests to the tokenizer"
     			for (int i = 0; i < testsList.getLength(); i++) {
     				Element test = (Element) testsList.item(i);
     				//				tests.add(i, new TTest(test.getTextContent(), test.getAttribute("type"),
     				//						Integer.parseInt(test.getAttribute("before")),
     				//						Integer.parseInt(test.getAttribute("hit")),
     				//						Integer.parseInt(test.getAttribute("after")))
     				//						);
     				// tests.add(i, new TTest(test.getTextContent(), test.getAttribute("type"),
     				// Integer.parseInt(test.getAttribute("before")),
     				// Integer.parseInt(test.getAttribute("hit")),
     				// Integer.parseInt(test.getAttribute("after")))
     				// );
+    			}
     			//System.out.println("Tests: "+tests);
     			// System.out.println("Tests: "+tests);
+    		}
     		if (isValid())
     			return true;
     		else {
-...
     			return false;
+    		}
+    	}
     	/**
     	 * The main method.
+    	 *
     	 * @param args the arguments
     	 * @throws IOException
     	 * @throws SAXException
     	 * @throws ParserConfigurationException
     	 * @throws IOException
     	 * @throws SAXException
     	 * @throws ParserConfigurationException
     	 */
     	public static void main(String[] args) throws ParserConfigurationException, SAXException, IOException {
     //		Document doc = DomUtils.load(new File("/home/mdecorde/xml/discours/import.xml"));
     //		Node corpora = doc.getDocumentElement().getElementsByTagName("corpora").item(0);
     //		Node corpus = ((Element)corpora).getElementsByTagName("corpus").item(0);
     //		Node tokenizer = ((Element)corpus).getElementsByTagName("tokenizer").item(0);
     //		System.out.println("OK? "+ TokenizerClasses.loadFromNode((Element) tokenizer));
     		// Document doc = DomUtils.load(new File("/home/mdecorde/xml/discours/import.xml"));
     		// Node corpora = doc.getDocumentElement().getElementsByTagName("corpora").item(0);
     		// Node corpus = ((Element)corpora).getElementsByTagName("corpus").item(0);
     		// Node tokenizer = ((Element)corpus).getElementsByTagName("tokenizer").item(0);
     		// System.out.println("OK? "+ TokenizerClasses.loadFromNode((Element) tokenizer));
+    	}
     	public static boolean loadFromProject(ProjectScope projectScope) {
     		//load params
     		// load params
     		IEclipsePreferences params = projectScope.getNode("Tokenizer");
     		try {
-...
     					String value = params.get(name, null);
     					if (value != null) {
     						try {
     							if (debug) System.out.println(" Tokenizer parametrized with "+name+"="+value);
     							if (debug) System.out.println(" Tokenizer parametrized with " + name + "=" + value);
     							field.set(field, value);
+    						}
     						catch (IllegalArgumentException e) {
-...
+    					}
+    				}
+    			}
     //			// old way, now using Reflection
     			// // old way, now using Reflection
     //			String[] keys = params.keys();
     //			for (String key : keys) {
     //				//					Element param = (Element) params.item(i);
     //				//					String key = param.getAttribute("name");
     //				String value = params.get(key, "");
     //				//					if (value == null || value.length() == 0) value = param.getTextContent();
     //				if (value.length() == 0) value = null;
     //
     //				if (debug) System.out.println(" Tokenizer parametrized with "+key+"="+value+"");
     //
     //				if (key.equals("tag_all"))
     //					tag_all = value;
     //				else if (key.equals("enclitics"))
     //					enclitics = value;
     //				else if (key.equals("encliticsFR"))
     //					encliticsFR = value;
     //				else if (key.equals("div_tags"))
     //					div_tags = value;
     //				else if (key.equals("q_tags"))
     //					q_tags = value;
     //				else if (key.equals("extraword1_tags"))
     //					extraword1_tags = value;
     //				else if (key.equals("corr_tags_no_seg"))
     //					corr_tags_no_seg = value;
     //				else if (key.equals("word_tags")) {
     //					word_tags = value;
     //					word_element_to_create = value; // FIXME for now
     //				} else if (key.equals("intraword_tags"))
     //					intraword_tags = value;
     //				else if (key.equals("punct_quotes"))
     //					punct_quotes = value;
     //				else if (key.equals("punct_strong1"))
     //					punct_strong1 = value;
     //				else if (key.equals("punct_strong2"))
     //					punct_strong2 = value;
     //				else if (key.equals("punct_paren_open1"))
     //					punct_paren_open1 = value;
     //				else if (key.equals("punct_paren_open2"))
     //					punct_paren_open2 = value;
     //				else if (key.equals("punct_paren_close1"))
     //					punct_paren_close1 = value;
     //				else if (key.equals("punct_paren_close2"))
     //					punct_paren_close2 = value;
     //				else if (key.equals("punct_weak"))
     //					punct_weak = value;
     //				else if (key.equals("entity"))
     //					entity = value;
     //				else if (key.equals("seg_tags"))
     //					seg_tags = value;
     //				else if (key.equals("regPunct"))
     //					regPunct = value;
     //				else if (key.equals("regElision"))
     //					regElision = value;
     //				else if (key.equals("whitespaces"))
     //					whitespaces = value;
     //				else if (key.equals("punct_strong")) // this is temporary
     //					tmp_strong_punct = value; // this is temporary
     ////				else
     ////					System.out.println("MISSING TOKENIZER KEY: "+key);
     //			}
     			//recombine
     			// String[] keys = params.keys();
     			// for (String key : keys) {
     			// // Element param = (Element) params.item(i);
     			// // String key = param.getAttribute("name");
     			// String value = params.get(key, "");
     			// // if (value == null || value.length() == 0) value = param.getTextContent();
     			// if (value.length() == 0) value = null;
     			//
     			// if (debug) System.out.println(" Tokenizer parametrized with "+key+"="+value+"");
     			//
     			// if (key.equals("tag_all"))
     			// tag_all = value;
     			// else if (key.equals("enclitics"))
     			// enclitics = value;
     			// else if (key.equals("encliticsFR"))
     			// encliticsFR = value;
     			// else if (key.equals("div_tags"))
     			// div_tags = value;
     			// else if (key.equals("q_tags"))
     			// q_tags = value;
     			// else if (key.equals("extraword1_tags"))
     			// extraword1_tags = value;
     			// else if (key.equals("corr_tags_no_seg"))
     			// corr_tags_no_seg = value;
     			// else if (key.equals("word_tags")) {
     			// word_tags = value;
     			// word_element_to_create = value; // FIXME for now
     			// } else if (key.equals("intraword_tags"))
     			// intraword_tags = value;
     			// else if (key.equals("punct_quotes"))
     			// punct_quotes = value;
     			// else if (key.equals("punct_strong1"))
     			// punct_strong1 = value;
     			// else if (key.equals("punct_strong2"))
     			// punct_strong2 = value;
     			// else if (key.equals("punct_paren_open1"))
     			// punct_paren_open1 = value;
     			// else if (key.equals("punct_paren_open2"))
     			// punct_paren_open2 = value;
     			// else if (key.equals("punct_paren_close1"))
     			// punct_paren_close1 = value;
     			// else if (key.equals("punct_paren_close2"))
     			// punct_paren_close2 = value;
     			// else if (key.equals("punct_weak"))
     			// punct_weak = value;
     			// else if (key.equals("entity"))
     			// entity = value;
     			// else if (key.equals("seg_tags"))
     			// seg_tags = value;
     			// else if (key.equals("regPunct"))
     			// regPunct = value;
     			// else if (key.equals("regElision"))
     			// regElision = value;
     			// else if (key.equals("whitespaces"))
     			// whitespaces = value;
     			// else if (key.equals("punct_strong")) // this is temporary
     			// tmp_strong_punct = value; // this is temporary
     			//// else
     			//// System.out.println("MISSING TOKENIZER KEY: "+key);
     			// }
     			// recombine
     			recombine();
     			if (params.getBoolean("onlyThoseTests", false)) {
     				System.out.println("Warning: tokenizer only using import parameters tests");
     				//tests = new ArrayList<>();
     				// tests = new ArrayList<>();
+    			}
     			org.osgi.service.prefs.Preferences testsList = params.node("tests");
     			String[] tests;
     			tests = testsList.childrenNames();
     			//System.out.println("Add "+testsList.getLength()+" tests to the tokenizer"
     			// System.out.println("Add "+testsList.getLength()+" tests to the tokenizer"
     			for (String testname : tests) {
     				org.osgi.service.prefs.Preferences testdef = testsList.node(testname);
     				//Element test = (Element) testsList.item(i);
     				// Element test = (Element) testsList.item(i);
     				TTest t = new TTest(testdef.get("content", null), testdef.get("type", null),
     						testdef.getInt("before",0 ),
     						testdef.getInt("before", 0),
     						testdef.getInt("hit", 0),
     						testdef.getInt("after", 0));
+    			}
     			//System.out.println("Tests: "+tests);
     		} catch (BackingStoreException e) {
     			// System.out.println("Tests: "+tests);
+    		}
     		catch (BackingStoreException e) {
     			// TODO Auto-generated catch block
     			e.printStackTrace();
+    		}
     		if (isValid()) {
     			return true;
     		} else {
+    		}
     		else {
     			reset();
     			recombine();
     			return false;
+    		}
+    	}
+    }
+    }

     // Copyright © 2010-2013 ENS de Lyon.
     // Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
     // Lyon 2, University of Franche-Comté, University of Nice
     // Sophia Antipolis, University of Paris 3.
     //
     // The TXM platform is free software: you can redistribute it
     // and/or modify it under the terms of the GNU General Public
     // License as published by the Free Software Foundation,
     // either version 2 of the License, or (at your option) any
     // later version.
     //
     // The TXM platform is distributed in the hope that it will be
     // useful, but WITHOUT ANY WARRANTY; without even the implied
     // warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     // PURPOSE. See the GNU General Public License for more
     // details.
     //
     // You should have received a copy of the GNU General
     // Public License along with the TXM platform. If not, see
     // http://www.gnu.org/licenses.
     package org.txm.tokenizer;
     import java.io.File;
     import java.util.LinkedHashMap;
     import java.util.regex.Matcher;
     import java.util.regex.Pattern;
     import org.txm.tokenizer.TokenizerClasses;
     public class SimpleStringTokenizer implements StringTokenizer {
     	/** The punct_strong. */
     	String punct_strong = TokenizerClasses.punct_strong;
     	/** The punct_all. */
     	String punct_all = TokenizerClasses.punct_all;
     	/** The word_chars. */
     	String word_chars = TokenizerClasses.word_chars;
     	String fclitics = null; // default behavior don't manage clitics
     	String pclitics = null; // default behavior don't manage clitics
     	Pattern reg_punct_other = Pattern.compile("\\p{P}");
     	/** The DEBUG. */
     	public boolean DEBUG = false;
     	String lang;
     	def regElision = null;
     	def reg3pts = null;
     	def regPunct;
     	def regFClitics = null;
     	def regPClitics = null;
     	String whitespaces;
     	Pattern regWhitespaces;
     	Pattern regSplitWhiteSpaces;
     	/**
     	 * Instantiates a new simple string tokenizer.
+    	 *
     	 * @param lang
     	 */
     	public SimpleStringTokenizer(String lang) {
     		this.lang = lang;
     		if (lang != null)
     			if (lang.startsWith("en")) {
     				fclitics = TokenizerClasses.FClitic_en;
     			} else if (lang.startsWith("fr")) {
     				fclitics = TokenizerClasses.FClitic_fr;
     				pclitics = TokenizerClasses.PClitic_fr;
     			} else if (lang.startsWith("gl")) {
     				fclitics = TokenizerClasses.FClitic_gl;
     			} else if (lang.startsWith("it")) {
     				pclitics = TokenizerClasses.PClitic_it;
+    			}
     		punct_strong = TokenizerClasses.punct_strong;
     		punct_all = TokenizerClasses.punct_all;
     		word_chars = TokenizerClasses.word_chars;
     		String strRegElision = TokenizerClasses.regElision;
     		if (strRegElision != null && strRegElision.length() > 0) {
     			regElision = ~/^([\p{L}-]++${strRegElision}[\p{L}-]++)(.*)$/ // the test must start with the start of string  ^
+    		}
     		reg3pts = ~/^(.*?)(\.\.\.)(.*)$/
     		String strRegPunct = TokenizerClasses.regPunct;
     		if (strRegPunct != null && strRegPunct.length() > 0) {
     			regPunct = ~/^(.*?)($strRegPunct)(.*)$/
+    		}
     		if (fclitics != null && fclitics.length() > 0) {
     			regFClitics = ~/(.+)($fclitics)$/ // the test must end with the end of string $
+    		}
     		if (pclitics != null && pclitics.length() > 0)
     			regPClitics = /^($pclitics)(.*)/ // the test must start with the start of string  ^
     		if (TokenizerClasses.whitespaces != null && TokenizerClasses.whitespaces.length() > 0) {
     			regWhitespaces = ~TokenizerClasses.whitespaces;
+    		}
     		if (TokenizerClasses.whitespaces != null && TokenizerClasses.whitespaces.length() > 0) {
     			regSplitWhiteSpaces = Pattern.compile(TokenizerClasses.whitespaces);
+    		}
+    	}
     	public final static String WHITESPACE = " ";
     	public final static String EMPTY = "";
     	/**
     	 * Process word.
     	 */
     	public ArrayList<String> processText(String text) {
     		ArrayList<String> result = new ArrayList<String>();
     		if (regSplitWhiteSpaces != null) {
     			for (String s : regSplitWhiteSpaces.split(text)) {		// separate with unicode white spaces
     				// if (DEBUG){println "process $s"}
     				result.addAll(iterate(s));
+    			}
+    		}
     		else {
     			result.addAll(iterate(text));
+    		}
     		return result;
+    	}
     	/**
     	 * Iterate. a String, should be called when a word is found in a String
+    	 *
     	 * @param s the s
     	 * @return the java.lang. object
     	 */
     	protected ArrayList<String> iterate(String s) {
     		ArrayList<String> result = new ArrayList<String>();
     		while (s != null && s.length() > 0) {
     			//	if (DEBUG){println "  > $s"}
     			s = standardChecks(result, s);
+    		}
     		return result;
+    	}
     	/**
     	 * Standard checks.
+    	 *
     	 * @param s the s
     	 * @return the java.lang. object
     	 */
     	//	@CompileStatic(SKIP)
     	public String standardChecks(ArrayList<String> result, String s) {
     		Matcher m;
     		for (TTest test : TokenizerClasses.tests) {
     			if ((m = s =~ test.regex)) {
     				if (DEBUG) {println "test : "+test.regex}
     				if (test.before > 0) {
     					result.addAll(iterate(m[0][test.before]))
+    				}
     				if (test.hit > 0) {
     					result.add(m[0][test.hit]);
+    				}
     				if (test.after > 0) {
     					return m[0][test.after];
+    				}
+    			}
+    		}
     		if (fclitics != null && (m = s =~ regFClitics) ) {
     			//	if (DEBUG) println "CLITIC found: $s ->"+ m
     			result.addAll(iterate(m.group(1)))
     			result.add(m.group(2));
     			return "";
     		} else if (pclitics != null && (m = s =~ regPClitics) ) {
     			if (DEBUG) println "PCLITIC found: $s ->"+ m
     			result.add(m.group(1));
     			result.addAll(iterate(m.group(2)))
     			return "";
     		} else if (regElision != null && (m = s =~ regElision) ) {
     			//	if (DEBUG) println "Elision found: $s ->"+ m.group(1)+" + "+m.group(2)+" + "+m.group(3)
     			//iterate(m.group(1))

... Ce différentiel a été tronqué car il excède la taille maximale pouvant être affichée.

Formats disponibles : Unified diff

Laboratoire ICAR » Plateforme TXM

Révision 3057