Index: tmp/org.txm.groovy.core/.classpath
===================================================================
--- tmp/org.txm.groovy.core/.classpath	(revision 0)
+++ tmp/org.txm.groovy.core/.classpath	(revision 321)
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry exported="true" kind="lib" path="bin/"/>
+	<classpathentry kind="src" path="src/groovy"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

Property changes on: tmp/org.txm.groovy.core/.classpath
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/META-INF/MANIFEST.MF
===================================================================
--- tmp/org.txm.groovy.core/META-INF/MANIFEST.MF	(revision 0)
+++ tmp/org.txm.groovy.core/META-INF/MANIFEST.MF	(revision 321)
@@ -0,0 +1,139 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: Macro
+Bundle-SymbolicName: org.txm.groovy.core
+Bundle-Version: 1.0.0.qualifier
+Bundle-RequiredExecutionEnvironment: JavaSE-1.6
+Require-Bundle: org.txm.ca.core;bundle-version="1.0.0",
+ org.txm.ca.rcp;bundle-version="1.0.0",
+ org.txm.cah.core;bundle-version="1.0.0",
+ org.txm.cah.rcp;bundle-version="1.0.0",
+ org.txm.chartsengine.core;bundle-version="1.0.0",
+ org.txm.chartsengine.jfreechart.core;bundle-version="1.0.0",
+ org.txm.chartsengine.jfreechart.rcp;bundle-version="1.0.0",
+ org.txm.chartsengine.r.core;bundle-version="1.0.0",
+ org.txm.chartsengine.r.rcp;bundle-version="1.0.0",
+ org.txm.chartsengine.raster.rcp;bundle-version="1.0.0",
+ org.txm.chartsengine.rcp;bundle-version="1.0.0",
+ org.txm.chartsengine.svgbatik.rcp;bundle-version="1.0.0",
+ org.txm.cooccurrence.core;bundle-version="1.0.0",
+ org.txm.cooccurrence.rcp;bundle-version="1.0.0",
+ org.txm.core;bundle-version="0.7.0",
+ org.txm.core.tests;bundle-version="1.0.0",
+ org.txm.index.core;bundle-version="1.0.0",
+ org.txm.index.rcp;bundle-version="1.0.0",
+ org.txm.lexicaltable.core;bundle-version="1.0.0",
+ org.txm.lexicaltable.rcp;bundle-version="1.0.0",
+ org.txm.lexicon.core;bundle-version="1.0.0",
+ org.txm.lexicon.rcp;bundle-version="1.0.0",
+ org.txm.libs.batik;bundle-version="0.0.0",
+ org.txm.libs.colt;bundle-version="1.2.0",
+ org.txm.libs.groovy-all;bundle-version="2.3.3",
+ org.txm.libs.itext;bundle-version="2.1.5",
+ org.txm.libs.jfreechart;bundle-version="1.0.17",
+ org.txm.libs.jfreesvg;bundle-version="1.7.0",
+ org.txm.libs.rserve;bundle-version="0.0.0",
+ org.txm.libs.saxon;bundle-version="9.0.0",
+ org.txm.partition.core;bundle-version="1.0.0",
+ org.txm.partition.rcp;bundle-version="1.0.0",
+ org.txm.progression.core;bundle-version="1.0.0",
+ org.txm.progression.rcp;bundle-version="1.0.0",
+ org.txm.rcp;bundle-version="0.7.8",
+ org.txm.specificities.core;bundle-version="1.0.0",
+ org.txm.specificities.rcp;bundle-version="1.0.0",
+ org.txm.statsengine.core;bundle-version="1.0.0",
+ org.txm.statsengine.r.core;bundle-version="1.0.0",
+ org.txm.statsengine.r.rcp;bundle-version="1.0.0",
+ org.txm.synopticeditor.rcp;bundle-version="1.0.0",
+ org.txm.textsbalance.core;bundle-version="1.0.0",
+ org.txm.textsbalance.rcp;bundle-version="1.0.0",
+ org.txm.utils;bundle-version="1.0.0",
+ org.txm.wordcloud.core;bundle-version="1.0.0",
+ org.txm.wordcloud.rcp;bundle-version="1.0.0",
+ org.txm.concordance.core;bundle-version="1.0.0",
+ org.txm.referencer.core;bundle-version="1.0.0"
+Export-Package: filters.BuildXmlLiturgie,
+ filters.Concatenator,
+ filters.CutHeader,
+ filters.eliminateNAttributes,
+ filters.EliminateWhiteSpaces,
+ filters.FusionHeader,
+ filters.FusionXmlHeaderBody,
+ filters.MinimalFilter,
+ filters.OneOpenTagPerLine,
+ filters.OneTagPerLine,
+ filters.ProcessEnclitics,
+ filters.ProcessQuotes,
+ filters.RegexFilter,
+ filters.ReunitBrokenTags,
+ filters.ReunitBrokenWords,
+ filters.TagSentences,
+ filters.Tokeniser,
+ filters.WordInternalElement,
+ org.txm.ant,
+ org.txm.doc,
+ org.txm.export,
+ org.txm.export.conll2009,
+ org.txm.export.ts,
+ org.txm.functions.mesures,
+ org.txm.importer,
+ org.txm.importer.alceste,
+ org.txm.importer.bfm,
+ org.txm.importer.bvh,
+ org.txm.importer.corptef,
+ org.txm.importer.discours,
+ org.txm.importer.doc,
+ org.txm.importer.factiva,
+ org.txm.importer.fleurs,
+ org.txm.importer.frantext,
+ org.txm.importer.graal,
+ org.txm.importer.hyperbase,
+ org.txm.importer.hyperprince,
+ org.txm.importer.lasla,
+ org.txm.importer.limsi,
+ org.txm.importer.perrault,
+ org.txm.importer.quick,
+ org.txm.importer.RGAQCJ,
+ org.txm.importer.teicmc,
+ org.txm.importer.tmx,
+ org.txm.importer.transcriber,
+ org.txm.importer.wtc,
+ org.txm.importer.xml,
+ org.txm.importer.xmltxm,
+ org.txm.importer.xmltxmpara,
+ org.txm.importer.xtz,
+ org.txm.macro.annotation,
+ org.txm.macro.commands,
+ org.txm.macro.cqp,
+ org.txm.macro.csv,
+ org.txm.macro.edition,
+ org.txm.macro.misc,
+ org.txm.macro.r,
+ org.txm.macro.stats,
+ org.txm.macro.text,
+ org.txm.macro.transcription,
+ org.txm.macro.txt,
+ org.txm.macro.xml,
+ org.txm.macro.xsl,
+ org.txm.macroproto,
+ org.txm.macroproto.edition,
+ org.txm.macroproto.export,
+ org.txm.macroproto.importer,
+ org.txm.macroproto.partition,
+ org.txm.macroproto.txt,
+ org.txm.scripts,
+ org.txm.scripts.clix,
+ org.txm.scripts.i18n,
+ org.txm.scripts.teitxm,
+ org.txm.scripts.tigersearch,
+ org.txm.setup,
+ org.txm.sw,
+ org.txm.tal,
+ org.txm.tal.melt,
+ org.txm.tal.stanford,
+ org.txm.test,
+ org.txm.tokenizer,
+ tests
+Bundle-ClassPath: bin/,
+ .
+Bundle-ActivationPolicy: lazy

Property changes on: tmp/org.txm.groovy.core/META-INF/MANIFEST.MF
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/.project
===================================================================
--- tmp/org.txm.groovy.core/.project	(revision 0)
+++ tmp/org.txm.groovy.core/.project	(revision 321)
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>org.txm.groovy.core</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.ManifestBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.SchemaBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.pde.PluginNature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

Property changes on: tmp/org.txm.groovy.core/.project
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/ChangeLog.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/ChangeLog.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/ChangeLog.groovy	(revision 321)
@@ -0,0 +1,80 @@
+package org.txm.setup
+
+import org.txm.importer.ApplyXsl2;
+
+// build log.xml
+// cd /home/mdecorde/workspace37/org.txm.toolbox
+// svn log --xml --verbose > /home/mdecorde/workspace37/org.txm.toolbox/log.xml
+String home = System.getProperty("user.home")
+File rootDir = new File(home, "SVN/TXMSVN")
+File xsl = new File(home, "xsl/svn2cl.xsl")
+File rawLogs = new File(home, "TEMP/portal_hist.txt")
+File log = new File(home, "TEMP/log.txt")
+File tbx = new File(home, "SVN/TXMSVN")
+
+println "** FILES:"
+println home
+println rootDir
+println xsl
+println rawLogs
+println log
+println tbx
+
+String dateLimit = "2014-01"
+String revLimit = "1"
+
+String cmd = """svn log --xml --verbose $tbx"""
+println "\n** SVN LOG: "+cmd
+def proc = cmd.execute()
+rawLogs.withWriter("UTF-8") { writer -> writer.write(proc.in.getText()) }
+proc.waitFor()
+if (proc.exitValue() != 0) {
+	println "SVN LOG error code: "+proc.exitValue()
+	println "Error: "+proc.err.getText()
+	return;
+} 
+
+println "** Filter XML : by date=$dateLimit and rev=$revLimit"
+def slurper = new XmlParser();
+def root = slurper.parse(rawLogs.toURI().toString())
+log.withWriter("UTF-8") { writer ->
+	for (def entry : root.logentry) {
+		String rev = entry.@revision
+		if (rev.compareTo(revLimit) < 0) continue
+		def msgs = entry.msg
+		
+		for (def date : entry.date) {
+			String sdate = date.text()
+			if (sdate.compareTo(dateLimit) < 0) {
+				return
+			}
+			writer.print("* "+sdate.substring(0, 10) + " - " + (sdate.substring(11, 19)))+ " REV "+rev
+		}
+		
+		boolean first = true;
+		for (def msg : msgs) {
+			String smsg = msg.text().trim()
+			if (smsg.length() == 0) continue;
+			if (first) { first = false; smsg = "\n"+smsg; }
+			smsg = smsg.replace("\nRCP", "\n** RCP")
+			smsg = smsg.replace("\nTBX", "\n** TBX")
+			smsg = smsg.replace("\nPLUGIN", "\n** PLUGIN")
+			smsg = smsg.replace("\nWEB", "\n** WEB")
+			smsg = smsg.replace("\nSETUP", "\n** SETUP")
+			smsg = smsg.replace("\nDOC", "\n** DOC")
+			smsg = smsg.replace("\nTRA", "\n** TRA")
+			smsg = smsg.replace("\nCQP", "\n** CQP")
+			smsg = smsg.replace("\nCWB", "\n** CQP")
+			smsg = smsg.replace("\nINSTALL", "\n** INSTALL")
+			smsg = smsg.replace("\nR textometry", "\n** R textometry")
+			smsg = smsg.replace(" :", ":") // hahaha
+			smsg = smsg.replace("textoemtry", "textometry") // hohoho
+			smsg = smsg.replace("# ", "*** ")
+			smsg = smsg.replace("#([0-9]+)", "[https://forge.cbp.ens-lyon.fr/redmine/issues/${1} #{1}]")
+			writer.print smsg
+		}
+		writer.println ""
+	}
+}
+
+println "RESULT SAVED IN : "+log.toURI()
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/FixFeaturePluginDeclaration.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/FixFeaturePluginDeclaration.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/FixFeaturePluginDeclaration.groovy	(revision 321)
@@ -0,0 +1,92 @@
+package org.txm.setup
+
+import org.txm.utils.io.FileCopy;
+
+import groovy.xml.*
+
+
+String home = System.getProperty("user.home")
+File rcpSrc = new File(home, "workspace441/org.txm.rcp")
+File productFile = new File(rcpSrc, "rcpapplication.product")
+
+//File dir = new File("/home/mdecorde/ECLIPSE/babel/babel-kepler/features/org.eclipse.babel.nls_eclipse_ru_4.3.0.v20131123020001")
+File dir = new File("/home/matthieu/ECLIPSE/eclipse_babel/features/org.eclipse.babel.nls_eclipse_ru_4.4.0.v20150804100819")
+
+File origFeatureFile = new File(dir, "feature_orig.xml") // will be created if not, and parsed
+File featureFile = new File(dir, "feature.xml")
+File featureResultFile = new File(dir, "result_feature.xml")
+
+if (!origFeatureFile.exists()) { // backup feature file if not already done
+	FileCopy.copy(featureFile, origFeatureFile)
+}
+
+def slurper = new XmlParser();
+
+// get used plugin IDs
+def product = slurper.parse(productFile.toURI().toString())
+def usedPlugins = new HashSet<String>()
+for (def plugin : product.plugins.plugin) {
+	usedPlugins.add(plugin.@id)
+}
+println "Number of plugins declared in product: "+usedPlugins.size()
+
+// Fix feature : remove unused plugins and fix @os @ws @arch and @nl attributes
+def feature = slurper.parse(origFeatureFile.toURI().toString())
+def plugins = feature.plugin
+def toSend = []
+for (def plugin : plugins) { // set @os @ws @arch and @nl
+	boolean remove = false;
+	String id = plugin.@id
+	
+	String hostid = id.substring(0, id.lastIndexOf(".nl_")) // remove the nl_xx.yyyyyyy part
+	println "$id >> $hostid"
+	
+	// set os
+	if (id.contains(".linux")) plugin.@os = "linux"
+	else if (id.contains(".win32")) plugin.@os = "win32"
+	else if (id.contains(".macosx")) plugin.@os = "macosx"
+	else if (id.contains(".aix")) remove = true
+	else if (id.contains(".hpux")) remove = true
+	else if (id.contains(".qnx")) remove = true
+	else if (id.contains(".solaris")) remove = true
+	
+	// set ws
+	if (id.contains(".wpf")) plugin.@ws = "wpf"
+	else if (id.contains(".win32")) plugin.@ws = "win32"
+	else if (id.contains(".photon")) remove = true
+	else if (id.contains(".motif")) remove = true
+	else if (id.contains(".gtk")) plugin.@ws = "gtk"
+	else if (id.contains(".cocoa")) plugin.@ws = "cocoa"
+	else if (id.contains(".carbon")) plugin.@ws = "carbon"
+	
+	if (id.contains(".s390")) remove = true
+	else if (id.contains(".s390x")) remove = true
+	
+	// set arch
+	if (id.contains(".x86_64")) plugin.@arch = "x86_64"
+	else if (id.contains(".x86")) plugin.@arch = "x86"
+	else if (id.contains(".ia64_32")) remove = true
+	else if (id.contains(".ia64")) remove = true
+	else if (id.contains(".spark")) remove = true
+	else if (id.contains(".ppc")) remove = true
+	else if (id.contains(".PA_RISC")) remove = true
+	
+	// set nl
+//	def m = id =~ /.+\.nl_(..)/
+//	if (m.size() > 0) {
+//		if (m[0].size() > 1) {
+//			plugin.@nl = m[0][1]
+//		}
+//	}
+	
+	if (remove) {
+		plugin.replaceNode {}
+	}
+//	if (!usedPlugins.contains(hostid)) {
+//		plugin.replaceNode {}
+//	}
+}
+
+println "WRITE RESULT FEATURE"
+//println XmlUtil.serialize(feature)
+featureResultFile.write(XmlUtil.serialize(feature))
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains Groovy scripts used to update&build TXM setups</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/FixQuotesInProperties.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/FixQuotesInProperties.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/FixQuotesInProperties.groovy	(revision 321)
@@ -0,0 +1,20 @@
+package org.txm.setup
+
+File propDirFile = new File("/home/mdecorde/workspace441/org.txm.toolbox/src/java/org/txm")
+
+for (File propFile : propDirFile.listFiles()) {
+	if (!propFile.getName().endsWith(".properties")) continue;
+
+	println "FILE: $propFile"
+	String text = propFile.getText("iso-8859-1")
+
+	propFile.withWriter("iso-8859-1") { writer->
+		for (String line : text.split("\n")) {
+			if (line.contains("{0}") || line.contains("{1}") || line.contains("{2}") || line.contains("{3}")) {
+				writer.println line.replaceAll("''", "'").replaceAll("''", "'").replaceAll("'", "''")
+			} else {
+				writer.println line
+			}
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildFeatureFromProduct.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildFeatureFromProduct.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildFeatureFromProduct.groovy	(revision 321)
@@ -0,0 +1,72 @@
+package org.txm.setup
+
+import groovy.xml.*
+
+String home = System.getProperty("user.home")
+File rcpSrc = new File(home, "workspace43/org.txm.rcp")
+File rcpFeatureSrc = new File(home, "workspace43/org.txm.rcp.feature")
+File productFile = new File(rcpSrc, "rcpapplication.product")
+File featureFile = new File(rcpFeatureSrc, "feature.xml")
+//File featureFileBackUp = new File(rcpFeatureSrc, "feature_back.xml")
+
+
+def slurper = new XmlParser();
+def product = slurper.parse(productFile.toURI().toString())
+
+def plugins = product.plugins.plugin
+def toSend = []
+for (def plugin : plugins) { // set @os @ws @arch and @nl
+	boolean remove = false;
+	String id = plugin.@id
+	if (id.contains(".linux")) plugin.@os = "linux"
+	else if (id.contains(".win32")) plugin.@os = "win32"
+	else if (id.contains(".macosx")) plugin.@os = "macosx"
+	else if (id.contains(".aix")) remove = true
+	else if (id.contains(".hpux")) remove = true
+	else if (id.contains(".qnx")) remove = true
+	else if (id.contains(".solaris")) remove = true
+	
+	if (id.contains(".wpf")) plugin.@ws = "wpf"
+	else if (id.contains(".win32")) plugin.@ws = "win32"
+	else if (id.contains(".photon")) remove = true
+	else if (id.contains(".motif")) remove = true
+	else if (id.contains(".gtk")) plugin.@ws = "gtk"
+	else if (id.contains(".cocoa")) plugin.@ws = "cocoa"
+	else if (id.contains(".carbon")) plugin.@ws = "carbon"
+	else if (id.contains(".s390")) remove = true
+	else if (id.contains(".s390x")) remove = true
+	
+	if (id.contains(".x86_64")) plugin.@arch = "x86_64"
+	else if (id.contains(".x86")) plugin.@arch = "x86"
+	else if (id.contains(".ia64_32")) remove = true
+	else if (id.contains(".ia64")) remove = true
+	else if (id.contains(".spark")) remove = true
+	else if (id.contains(".ppc")) remove = true
+	else if (id.contains(".PA_RISC")) remove = true
+	
+//	def m = id =~ /.+\.nl_(..)/
+//	if (m.size() > 0) {
+//		if (m[0].size() > 1) {
+//			plugin.@nl = m[0][1]
+//		}
+//	}
+	
+	plugin."@download-size"="0"
+	plugin."@install-size"="0"
+	plugin.@version="0.0.0"
+	plugin.@unpack="false"
+	
+	if (!remove) {
+		toSend << plugin
+	}
+}
+
+println "FEATURE"
+
+def feature = slurper.parse(featureFile.toURI().toString())
+
+for (def plugin : toSend) {
+	feature.append(plugin)
+}
+//println XmlUtil.serialize(feature)
+featureFile.write(XmlUtil.serialize(feature))
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildSplashVersion.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildSplashVersion.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildSplashVersion.groovy	(revision 321)
@@ -0,0 +1,54 @@
+package org.txm.setup;
+
+import java.awt.*;
+import java.awt.image.*;
+import java.io.File;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import javax.imageio.ImageIO;
+import org.txm.importer.XPathResult;
+
+// Update splash screen using the RCP project version set in the product file
+// The BMP image must be 32bit encoded
+
+int img_width = 316;
+
+File rcpDir = new File(System.getProperty("user.home"), "workspace442/org.txm.rcp")
+File product = new File(rcpDir, "rcpapplication.product");
+File imgFile = new File(rcpDir, "splash_sansversion.bmp");
+File imgFile2 = new File(rcpDir,"splash.bmp");
+
+String modifier = ""
+
+//String version =  org.txm.rcpapplication.Activator.getDefault().getBundle().getVersion().toString(); // "0.7.7.201412160925";
+String version = XPathResult.getXpathResponse(product, "//product/@version")
+//def date = " ("+new SimpleDateFormat("yyyy-MM-dd").format(new Date())+")"
+def date = ""
+version = "TXM "+version.replaceAll(".qualifier", date);
+
+if (modifier.length() > 0 ) version = version + " - "+modifier
+
+println "VERSION: '$version'"
+
+BufferedImage img = null;
+img = ImageIO.read(imgFile);
+Graphics g = img.getGraphics();
+g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
+
+File imgMascot = new File(System.getProperty("user.home"),"Images/splashs/"+version+".png");
+println imgMascot
+if (imgMascot.exists()) {
+	println "write mascot"
+	g.drawImage(ImageIO.read(imgMascot), 0, 0, img_width, 360, null, null);
+}
+
+Font font = new Font("Arial", Font.BOLD, 20);
+g.setColor(new Color(109,139,0)); //
+g.setFont(font);
+FontMetrics fontMetrics = g.getFontMetrics();
+int width = fontMetrics.stringWidth(version);
+g.drawString(version, img_width - width - 20, 300);
+
+ImageIO.write(img, "BMP", imgFile2);
+
+println "Done: "+imgFile2.getAbsoluteFile()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/MessagesToArchMessages.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/MessagesToArchMessages.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/MessagesToArchMessages.groovy	(revision 321)
@@ -0,0 +1,162 @@
+package org.txm.setup
+rootDir = new File("/home/mdecorde/paquets")
+linuxMessagesDir = new File(rootDir, "shared/debian/DEBIAN")
+windowsMessagesDir = new File(rootDir, "archs/win")
+macMessagesDir = new File(rootDir, "shared/mac")
+
+windowsMessages = ";START OF LANG MESSAGES\n"
+win_langs = ["en": "ENGLISH" , "fr":"FRENCH"]
+
+needed_keys = [
+	"InfosTitle", "InfosText", "WelcomeTitle", "WelcomeText",
+	"InstallRLibrariesTitle", "InstallRLibrariesText", "FinishTitle",
+	"FinishTextWindows", "FinishTextLinux", "FinishTextMac",
+	"ErrorArch32", "ErrorArch64", "UninstallWindows", "InstallTreeTagger"]
+
+if (!(writeLangFiles("en") && writeLangFiles("fr"))) {
+	println "Stoping process"
+	return;
+} 
+
+windowsMessages += ";END OF LANG MESSAGES\n"
+
+File nsis32 = new File(windowsMessagesDir, "32bit/txm.nsi")
+File nsis64 = new File(windowsMessagesDir, "64bit/txm.nsi")
+
+if (!(patchNSIS(nsis32) && patchNSIS(nsis64))) {
+	println "Error while patching nsis files"
+}
+
+def patchNSIS(File nsis) {
+	String text = nsis.getText("Windows-1252")
+	boolean startReplacing = false;
+	nsis.withWriter("Windows-1252") { writer ->
+		for (String line : text.split("\n")) {
+			if (";START OF LANG MESSAGES" == line.trim()) {
+				startReplacing = true
+				line = windowsMessages
+				writer.println line
+			}
+			if (";END OF LANG MESSAGES" == line.trim())  {
+				startReplacing = false
+				continue
+			}
+
+			if (!startReplacing) {
+				writer.println line
+			}
+		}
+	}
+	return true;
+}
+def checkProps(def props) {
+	def missing = []
+	for (String prop : needed_keys)
+		if (!props.containsKey(prop))
+			missing << prop
+	return missing
+}
+
+def writeLangFiles(String lang) {
+	println "Writing lang files for $lang"
+	File propFile = new File(rootDir, "SetupMessages_${lang}.properties")
+
+	Properties props = new Properties()
+	InputStreamReader input = new InputStreamReader(new FileInputStream(propFile) , "UTF-8")
+	props.load(input)
+	input.close()
+
+	def missingProps = checkProps(props)
+	if (missingProps.size() > 0) {
+		println "Error missings keys $lang: "+missingProps
+		return false;
+	}
+
+	String ext = ""
+	if (lang.length() > 0) ext = "."+lang
+
+	// LINUX
+	File template = new File(linuxMessagesDir, "templates"+ext)
+	template.withWriter("UTF-8") { writer ->
+		writer.println(
+"""Template: TXM/showinfos
+Type: note
+Description:"""+props.get("InfosTitle"))
+		writer.println(
+				" .\n  "+props.get("InfosText").replaceAll("\n\n", '\n .\n').replaceAll("\n([^ ])", '\n  $1').replaceAll("\n", '\n '))
+		writer.println()
+		writer.println(
+"""Template: TXM/showwelcome
+Type: note
+Description:"""+props.get("WelcomeTitle"))
+		writer.println(
+				" .\n  "+props.get("WelcomeText").replaceAll("\n\n", '\n .\n').replaceAll("\n([^ ])", '\n  $1').replaceAll("\n", '\n '))
+		writer.println()
+		writer.println(
+"""Template: TXM/askr
+Type: boolean
+Default: true
+Description:"""+props.get("InstallRLibrariesTitle"))
+		writer.println(
+				" .\n  "+props.get("InstallRLibrariesText").replaceAll("\n\n", '\n .\n').replaceAll("\n([^ ])", '\n  $1').replaceAll("\n", '\n '))
+		writer.println()
+		writer.println(
+"""Template: TXM/finish
+Type: note
+Description:"""+props.get("FinishTitle"))
+	writer.println(
+		" .\n  "+props.get("FinishTextLinux").replaceAll("\n\n", '\n .\n').replaceAll("\n([^ ])", '\n  $1').replaceAll("\n", '\n '))
+	writer.println(
+		" .\n  "+props.get("InstallTreeTagger").replaceAll("\n\n", '\n .\n').replaceAll("\n([^ ])", '\n  $1').replaceAll("\n", '\n '))
+	}
+
+	//WINDOWS NSIS
+	if (lang.length() > 0) ext = "_"+lang
+
+	StringWriter swriter = new StringWriter();
+	swriter.println(";Lang Strings "+win_langs[lang])
+	swriter.println('LangString InfosTitleMessage ${LANG_'+win_langs[lang]+'} "'+props.get("InfosTitle").replaceAll("\"", "'")+'"')
+	swriter.println('LangString InfosTextMessage ${LANG_'+win_langs[lang]+'} "'+props.get("InfosText").replaceAll("\n", '\\$\\\\n').replaceAll("\"", "'")+'"')
+	swriter.println('LangString WelcomeTitleMessage ${LANG_'+win_langs[lang]+'} "'+props.get("WelcomeTitle").replaceAll("\"", "'")+'"')
+	swriter.println('LangString WelcomeTextMessage ${LANG_'+win_langs[lang]+'} "'+props.get("WelcomeText").replaceAll("\n", '\\$\\\\n').replaceAll("\"", "'")+'"')
+	// no R libs question
+	swriter.println('LangString FinishTitleMessage ${LANG_'+win_langs[lang]+'} "'+props.get("FinishTitle".replaceAll("\"", "'"))+'"')
+	swriter.println('LangString FinishTextMessage ${LANG_'+win_langs[lang]+'} "'+(props.get("FinishTextWindows") + '$\\n' + props.get("InstallTreeTagger")).replaceAll("\n", '\\$\\\\n').replaceAll("\"", "'")+'"')
+	// Windows specific messages
+	swriter.println('LangString ErrorArch32Message ${LANG_'+win_langs[lang]+'} "'+props.get("ErrorArch32").replaceAll("\"", "'")+'"')
+	swriter.println('LangString ErrorArch64Message ${LANG_'+win_langs[lang]+'} "'+props.get("ErrorArch64").replaceAll("\"", "'")+'"')
+	swriter.println('LangString UninstallWindowsMessage ${LANG_'+win_langs[lang]+'} "'+props.get("UninstallWindows").replaceAll("\n", '\\$\\\\n').replaceAll("\"", "'")+'"')
+	swriter.close();
+
+	windowsMessages += "\n"+swriter.toString()+"\n"
+
+	//MAC 3 FILES : Intro, ReadMe and Finish
+	File intromac = new File(macMessagesDir, "intro${ext}.txt")
+	intromac.withWriter("x-MacRoman") { writer ->
+		writer.println("")
+		writer.println(props.get("InfosText"))
+	}
+	File readmemac = new File(macMessagesDir, "readme${ext}.txt")
+	readmemac.withWriter("x-MacRoman") { writer ->
+		writer.println("")
+		writer.println(props.get("WelcomeText"))
+	}
+	File rlibsmac = new File(macMessagesDir, "rlibs${ext}.txt")
+	rlibsmac.withWriter("x-MacRoman") { writer ->
+		writer.println(props.get("InstallRLibrariesTitle"))
+		writer.println(props.get("InstallRLibrariesText"))
+	}
+	File finishmac = new File(macMessagesDir, "finish${ext}.txt")
+	finishmac.withWriter("x-MacRoman") { writer ->
+		writer.println("")
+		writer.println(props.get("FinishTextMac"))
+		writer.println("")
+		writer.println(props.get("InstallTreeTagger"))
+	}
+	
+	return true;
+}
+
+
+
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildTXMHTMLDoc.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildTXMHTMLDoc.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/BuildTXMHTMLDoc.groovy	(revision 321)
@@ -0,0 +1,55 @@
+import java.io.File;
+
+import org.txm.macro.txt.SearchReplaceInDirectoryMacro;
+import org.txm.utils.io.FileCopy;
+
+String home = System.getProperty("user.home")
+String svnpath = "SVN/TXMSVN/trunk/doc/user manuals"
+File svnDir = new File(home, svnpath)
+File svnImagesDir = new File(svnDir, "ImagesTXM")
+//File odtFile = new File(home, "SVN/TXMSVN/trunk/doc/user manuals/Manuel de TXM 0.7 FR.odt")
+File htmlDir = new File(home, "TEMP/manuel")
+
+if (!htmlDir.listFiles()) {
+	println "Error: you must empty $htmlDir before. Aborting."
+	return;
+}
+
+if (!svnDir.exists()) {
+	println "Error: Cannot find SVN files $svnDir. Aborting"
+	return
+}
+
+//// Update SVN // marche pas :(
+//String cmd = "svn update \"$svnDir\""
+//println "SVN update: "+cmd
+//def proc = cmd.execute()
+//proc.in.getText()
+//proc.waitFor()
+//if (proc.exitValue() != 0) {
+//	println "SVN update error code: "+proc.exitValue()
+//	println "Error: "+proc.err.getText()
+//	return;
+//} 
+
+// Copy images
+File htmlImagesDir = new File(htmlDir, svnImagesDir.getName())
+FileCopy.copyFiles(svnImagesDir, htmlImagesDir)
+if (!htmlImagesDir.exists()) {
+	println "Failed to copy $htmlImagesDir"
+	return
+}
+
+// Fix HTML files
+// fix images source directory - remove SVN absolute path
+svnpath = svnpath.replaceAll(" ", "%20")
+new SearchReplaceInDirectoryMacro().process(htmlDir, "\\.xhtml", "UTF-8",
+	 / src="\.\.\/\.\.\/$svnpath\//, 
+	 ' src="', false)
+// fix images source directory - remove "/" of some image paths
+new SearchReplaceInDirectoryMacro().process(htmlDir, "\\.xhtml", "UTF-8",
+	/\/ImagesTXM/,
+	'ImagesTXM', false)
+new SearchReplaceInDirectoryMacro().process(htmlDir, "\\.xhtml", "UTF-8",
+	/<style media="all" type="text\/css">/, 
+	'<link rel="stylesheet" type="text/css" href="style.css"/>\n<style media="all" type="text/css">', false)
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/setup/SVNStats.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/setup/SVNStats.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/setup/SVNStats.groovy	(revision 321)
@@ -0,0 +1,142 @@
+package org.txm.setup
+
+File svnLog = new File("/home/mdecorde/SVN/ALLTXMSVN/txm/log.xml");
+File outRev = new File("/home/mdecorde/TEMP/stats_rev.tsv")
+File outPeriode = new File("/home/mdecorde/TEMP/stats_periode.tsv")
+def periodes = [1624, 1771, 1823, 2095, 2299]
+def SVNROOT = new File("/home/mdecorde/SVN/ALLTXMSVN/txm")
+def composants = ["/trunk/CWB/", "/trunk/doc/", "/trunk/groovy-all/", "/trunk/RCP/", "/trunk/textometrieR/", "/trunk/Toolbox/", "/trunk/WEB/", ""]
+
+def logs = new XmlParser().parse(svnLog);
+def revisions = new LinkedHashMap<String, Object>();
+def allPackages = new HashSet<String>();
+def allFiles = new HashSet<String>();
+
+println "Counting revisions..."
+for (def logEntry : logs.logentry) {
+	// SVN log.xml infos
+	def commitAuthor = logEntry.author.text();
+	def commitDate = logEntry.date.text();
+	def nUpdate = 0;
+	def nAdd = 0
+	def nDelete = 0;
+	def newPackages = new HashSet<String>()
+	def newFiles = new HashSet<String>()
+
+	for (def path : logEntry.paths.path) {
+		String file = path.text();
+		switch(path.'@action') {
+			case "M":
+				nUpdate++
+				break;
+			case "A":
+				nAdd++
+				if (file.matches(".+\\.[A-Za-z]{1,10}")) {
+					newFiles << file;
+				} else {
+					newPackages << file;
+				}
+				break;
+			case "D":
+				nDelete++;
+				break;
+			default:
+				break;
+		}
+	}
+
+	// SVN file explore
+	def nLines = 0;
+	def authors = [commitAuthor]
+	// count lines
+	// get @author
+
+	revisions[Integer.parseInt(logEntry.'@revision')] = [commitDate, authors, nUpdate, nAdd, nDelete, newPackages.size(), newPackages, newFiles.size(), newFiles]
+}
+
+def periodesStats = new LinkedHashMap<String, Object>()
+int currentPeriode = 0
+def nCommit = 0
+def nUpdate = 0
+def nAdd = 0
+def nDelete = 0
+def authors = new HashSet<String>()
+def newPackages = new HashSet<String>()
+def newFiles = new HashSet<String>()
+
+println "Counting periodes..."
+def revKeys = revisions.keySet().sort()
+def previousRev = revKeys[0]
+for( def rev : revKeys) {
+	if (rev >= periodes[currentPeriode]) {
+		//println "end of periode "+periodes[currentPeriode]+ " at rev "+rev
+		periodesStats[periodes[currentPeriode]] = [revisions[previousRev][0], authors, nCommit, nUpdate, nAdd, nDelete, newPackages.size(), newFiles.size(), newPackages]
+
+		currentPeriode++
+		authors = new HashSet<String>()
+		nCommit = 0;
+		nUpdate = 0;
+		nAdd = 0
+		nDelete = 0;
+		newPackages = new HashSet<String>()
+		newFiles = new HashSet<String>()
+	}
+	def infos = revisions[rev]
+	nCommit++
+	authors.addAll(infos[1])
+	nUpdate += infos[2]
+	nAdd += infos[3]
+	nDelete += infos[4]
+	newPackages.addAll(infos[6])
+	newFiles.addAll(infos[8])
+	
+	previousRev = rev
+}
+//periodesStats[2299] = [nCommit, authors, nUpdate, nAdd, nDelete, newPackages.size(), newPackages, newFiles.size(), newFiles]
+
+
+println "Counting composants of periodes..."
+for(def p : periodesStats.keySet()) {
+	def infos = periodesStats[p]
+	//println infos.size()
+	newPackages = infos[8]
+	def newPackagesComp = [:]
+
+	for (String path : newPackages) {
+	for (int i = 0 ; i < composants.size() ; i++) {
+		def comp = composants[i]
+		
+			if (path.startsWith(comp)) {
+				if (newPackagesComp[comp] == null) newPackagesComp[comp] = new HashSet<String>();
+				newPackagesComp[comp] << path.substring(comp.length())
+				break;
+			}
+		}
+	}
+	for (int i = 0 ; i < composants.size() ; i++) {
+		def comp = composants[i]
+		if (newPackagesComp[comp] != null)
+			newPackagesComp[comp] = newPackagesComp[comp] .size()
+	}
+	infos[8] = newPackagesComp;
+}
+
+println "revisions stat size: "+revisions.size()
+println "periodes stat size: "+periodesStats.size()
+
+println "writing outRev"
+outRev.withWriter("UTF-8") { writer ->
+	writer.println "rev\tdate\tauthors\tnUpdate\tnAdd\tnDelete\tnNewPackages\tnewPackages\tnNewFiles\tnewFiles"
+	for( def rev : revisions.keySet()) {
+		writer.println rev+"\t"+revisions[rev].join ("\t")
+	}
+}
+//[revisions[previousRev][0], authors, nCommit, nUpdate, nAdd, nDelete, newPackages.size(), newFiles.size(), newPackages]
+println "writing outPeriode"
+outPeriode.withWriter("UTF-8") { writer ->
+	writer.println "periode\tdate\tauthors\tnCommit\tnUpdate\tnAdd\tnDelete\tnNewPackages\tnNewFiles\tnewPackagesPerComp"
+	for(def p : periodesStats.keySet()) {
+		writer.println p+"\t"+periodesStats[p].join ("\t")
+		writer.flush()
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/Tagger.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/Tagger.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/Tagger.groovy	(revision 321)
@@ -0,0 +1,13 @@
+package org.txm.tal
+
+class Tagger {
+	File executablePath
+	File modelPath
+	String lang
+	
+	public Tagger(File executablePath, File modelPath, String lang) {
+		this.executablePath = executablePath
+		this.modelPath = modelPath
+		this.lang = lang
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLDomTokenizerAdapter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLDomTokenizerAdapter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLDomTokenizerAdapter.groovy	(revision 321)
@@ -0,0 +1,38 @@
+package org.txm.tal
+
+import org.txm.importer.StaxIdentityParser;
+import org.txm.importer.XPathResult;
+import org.w3c.dom.*;
+
+class XMLDomTokenizerAdapter {
+	String xpath;
+	boolean debug = true;
+	def nodes = []
+	
+	File infile, outfile;
+	public XMLDomTokenizerAdapter(File infile, def xpath) {
+		this.infile = infile
+		this.xpath = xpath
+	}
+	
+	public boolean process(File outfile) {
+		this.outfile = outfile
+		
+		XPathResult rez = new XPathResult(infile)
+		def nodes = rez.getNodes(xpath)
+		println nodes.size()
+//		for (Node node : rez.getNodes(xpath)) {
+//			println "Found $node"
+//			//nodes << [node, ""]
+//		}
+	}
+	
+	
+	public static void main(String[] args) {
+		File inputXMLFile = new File("/home/mdecorde/xml/comere/cmrclavardage/cmr-getalp_org-actu-tei-v1.xml");
+		File outputXMLFile = new File("/home/mdecorde/xml/comere/cmrclavardage/tmp.xml");
+		def processor = new XMLDomTokenizerAdapter(inputXMLFile, "/tei:TEI/tei:text/tei:body/tei:div/tei:post") //tei:div/tei:post/tei:p
+		processor.process(outputXMLFile)
+		//println processor.getNodes()
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLStaxTokenizerAdapter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLStaxTokenizerAdapter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLStaxTokenizerAdapter.groovy	(revision 321)
@@ -0,0 +1,87 @@
+package org.txm.tal
+
+import org.txm.importer.StaxIdentityParser;
+
+/**
+ * Experiment of Tokenizer that targets XPath results text
+ * 
+ * @author mdecorde
+ *
+ */
+class XMLStaxTokenizerAdapter extends StaxIdentityParser {
+	def xpaths;
+	boolean debug = false;
+	boolean doTokenize = false
+	String str; // string to tokenize
+	def tokenizeResults = []
+	def nsegments = 0
+	
+	public XMLStaxTokenizerAdapter(File infile, def xpaths, def tokenizeResults) {
+		super(infile.toURI().toURL());
+		this.xpaths = xpaths
+		this.tokenizeResults = tokenizeResults
+	}
+	
+	protected void processStartElement() {
+		String localname = parser.getLocalName();
+		if (debug) println "start element $currentXPath"
+		if (doTokenize) {
+			if (debug) println "Found text element $currentXPath"
+			tokenizeCurrentText();
+		} else {
+			if (xpaths.contains(currentXPath.toString())) {
+				doTokenize = true;
+				str = "";
+			}
+		}
+		super.processStartElement()
+	}
+	
+	protected void processEndElement() {
+		if (xpaths.contains(currentXPath.toString())) {
+			tokenizeCurrentText()
+			doTokenize = false;
+			if (debug) println "Found text element: stop tokenizing"
+		}
+		
+		super.processEndElement();
+	}
+	
+	protected void processCharacters() {
+		if (doTokenize) {
+			str += parser.getText();
+		} else {
+			super.processCharacters();
+		}
+	}
+	
+	protected tokenizeCurrentText() {
+		if (str.length() == 0) return;
+		def tokens = tokenizeResults[nsegments]
+		for (def token : tokens) {
+			writer.writeStartElement("w");
+			writer.writeCharacters(token);
+			writer.writeEndElement();
+			writer.writeCharacters(" ");
+		}
+
+		nsegments++;
+	}
+		
+	/**
+	 * Best tokenizer ever.
+	 * Extends this function
+	 * 
+	 * @param str
+	 * @return
+	 */
+	protected List<CharSequence> tokenize(String str) {
+		return str.tokenize();
+	}
+	
+	public static void main(String[] args) {
+		File inputXMLFile = new File("/home/mdecorde/xml/comere/cmrclavardage/cmr-getalp_org-actu-tei-v1.xml");
+		File outputXMLFile = new File("/home/mdecorde/xml/comere/cmrclavardage/tmp.xml");
+		new XMLStaxTokenizerAdapter(inputXMLFile, ["/TEI/text/body/div/post/p/"]).process(outputXMLFile)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/stanford/StanfordSegmenterMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/stanford/StanfordSegmenterMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/stanford/StanfordSegmenterMacro.groovy	(revision 321)
@@ -0,0 +1,120 @@
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import edu.stanford.nlp.international.arabic.process.*
+import edu.stanford.nlp.international.arabic.*
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import edu.stanford.nlp.ie.crf.*;
+import edu.stanford.nlp.io.*;
+import edu.stanford.nlp.ling.*;
+import edu.stanford.nlp.objectbank.*;
+import edu.stanford.nlp.process.*;
+import edu.stanford.nlp.sequences.*;
+import edu.stanford.nlp.stats.*;
+import edu.stanford.nlp.trees.*;
+import edu.stanford.nlp.util.*;
+import edu.stanford.nlp.util.concurrent.*;
+
+@Field @Option(name="inputDir", usage="input directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe")
+File inputDir
+@Field @Option(name="outputDir", usage="output directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe/out-s")
+File outputDir
+
+@Field @Option(name="model", usage="'.tagger' model file", widget="File", required=false, def="data/arabic-segmenter-atbtrain.ser.gz")
+File model
+@Field @Option(name="extension", usage="Regexp de l'extension des fichiers à modifier", widget="String", required=true, def='\\.txt')
+String extension = "\\.txt"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+outputDir.mkdir()
+if (!outputDir.exists()) {
+	println("Could not create $outputDir")
+	return;
+}
+
+// END OF PARAMETERS
+//File inputFile = new File("/home/mdecorde/xml/testarabe/test.txt")
+inputDir.eachFileMatch(~/.*$extension/) { inputFile ->
+println "Processing file: $inputFile"
+File outputFile = new File(outputDir, inputFile.getName())
+//File model = new File("/home/mdecorde/LIBRAIRIES/stanford-segmenter-2014-01-04/data/arabic-segmenter-atbtrain.ser.gz")
+Properties options = new Properties();
+//options.put("prefixMarker", "#")
+//options.put("suffixMarker", "#")
+options.put("loadClassifier", model.getAbsolutePath())
+//TODO: uncommend me
+//ArabicSegmenter segmenter = new ArabicSegmenter(options);
+segmenter.flags.inputEncoding = "UTF-8";
+segmenter.loadSegmenter(segmenter.flags.loadClassifier, options);
+
+def br = inputFile.newReader("UTF-8")
+OutputStreamWriter out = new OutputStreamWriter( new FileOutputStream(outputFile), "UTF-8");
+PrintWriter pw = new PrintWriter(out, true);
+double charsPerSec = decode(segmenter, br, pw, 5);
+IOUtils.closeIgnoringExceptions(br);
+System.err.printf("Done! Processed input text at %.2f input characters/second%n", charsPerSec);
+
+}
+
+/**
+   * Segment input and write to output stream.
+   *
+   * @param segmenter
+   * @param br
+   * @param pwOut
+   * @param nThreads
+   * @return input characters processed per second
+   */
+def decode(def segmenter, BufferedReader br,
+                               PrintWriter pwOut, int nThreads) {
+    assert nThreads > 0;
+    long nChars = 0;
+    final long startTime = System.nanoTime();
+    if (nThreads > 1) {
+		//TODO: uncommend me
+		//      MulticoreWrapper<String,String> wrapper = new MulticoreWrapper<String,String>(nThreads, segmenter);
+      try {
+        for (String line; (line = br.readLine()) != null;) {
+          nChars += line.length();
+          wrapper.put(line);
+          while (wrapper.peek()) {
+            pwOut.println(wrapper.poll());
+          }
+        }
+
+        wrapper.join();
+        while (wrapper.peek()) {
+          pwOut.println(wrapper.poll());
+        }
+
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+
+    } else {
+      nChars = segmenter.segment(br, pwOut);
+    }
+    long duration = System.nanoTime() - startTime;
+    double charsPerSec = (double) nChars / (duration / 1000000000.0);
+    return charsPerSec;
+  }
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/stanford/StanfordTaggerMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/stanford/StanfordTaggerMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/stanford/StanfordTaggerMacro.groovy	(revision 321)
@@ -0,0 +1,64 @@
+import edu.stanford.nlp.ling.*
+import edu.stanford.nlp.process.*
+import edu.stanford.nlp.tagger.maxent.*;
+
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+@Field @Option(name="inputDir", usage="input directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe/out-s")
+File inputDir
+@Field @Option(name="outputDir", usage="output directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe/out-t")
+File outputDir
+
+@Field @Option(name="model", usage="'.tagger' model file", widget="File", required=false, def="taggers/arabic-train.tagger")
+File model
+@Field @Option(name="extension", usage="Regexp de l'extension des fichiers à modifier", widget="String", required=true, def='\\.txt')
+String extension = "\\.txt"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+def modelPath = model.getAbsolutePath()
+
+outputDir.mkdir()
+if (!outputDir.exists()) {
+	println("Could not create $outputDir")
+	return;
+}
+
+inputDir.eachFileMatch(~/.*$extension/) { inputFile ->
+	String name = inputFile.getName()
+	int idx = name.indexOf(".")
+	if (idx > 0) name = name.substring(0, idx)
+	
+	new File(outputDir, name+".xml").withWriter("UTF-8") { writer ->
+		
+		writer.println('<?xml version="1.0" encoding="UTF-8"?>')
+		writer.println('<text>')
+		//TODO: uncomment me
+	//	def tagger = new MaxentTagger(modelPath);
+	//	def ptbTokenizerFactory = PTBTokenizer.factory(new CoreLabelTokenFactory(),	"untokenizable=noneKeep");
+	
+			//TODO: uncomment me
+//		def documentPreprocessor = new DocumentPreprocessor(inputFile.newReader());
+		documentPreprocessor.setTokenizerFactory(ptbTokenizerFactory);
+		int n = 0
+		
+		for (def sentence : documentPreprocessor) {
+			writer.println("<s n=\"$n\">")
+			n++;
+			def tSentence = tagger.tagSentence(sentence);
+			//println(Sentence.listToString(tSentence, false));
+			for (def taggedWord : tSentence)
+			{
+				def form = w.word();
+				def pos = w.tag();
+				writer.println("<w pos=\"$pos\">$form</w>")
+			}
+			writer.println('</s>')
+		}
+		writer.println('</text>')
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MEltTokenizer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MEltTokenizer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MEltTokenizer.groovy	(revision 321)
@@ -0,0 +1,22 @@
+package org.txm.tal.melt
+
+import org.txm.tal.XMLStaxTokenizerAdapter;
+
+class MEltTokenizer extends XMLStaxTokenizerAdapter {
+	
+	public MEltTokenizer(File infile, def xpaths) {
+		super(infile, xpaths);
+	}
+	
+	protected List<CharSequence> tokenize(String str) {
+		def ret = str.tokenize()
+		return ret;
+	}
+	
+	public static void main(String[] args) {
+		File inputXMLFile = new File("/home/mdecorde/xml/xmltest/txt1.xml");
+		File outputXMLFile = new File("/home/mdecorde/xml/xmltest/txt1-o.xml");
+		
+		new MEltTokenizer(inputXMLFile, ["/root/subtruc/submachin/"]).process(outputXMLFile)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MElt.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MElt.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MElt.groovy	(revision 321)
@@ -0,0 +1,105 @@
+
+package org.txm.tal.melt
+
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.txm.tal.Tagger
+import org.txm.tal.SlashFile2TTRez
+import org.txm.utils.StreamHog;
+
+class MElt extends Tagger {
+	
+	Process process = null;
+	InputStream meltis
+	OutputStream meltos
+	boolean debug = false
+	
+	BufferedReader reader
+	StringBuffer buffer = new StringBuffer();
+	
+	public MElt(File executablePath, File modelPath, String lang) {
+		super(executablePath, modelPath, lang);
+	}
+	
+	public boolean start() {
+		// Call Melt
+		def args = [];
+		args << "MElt"
+		//args << "-L"
+		ProcessBuilder pb = new ProcessBuilder(args);
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+			return false
+		}
+		meltis = process.getInputStream();
+		meltos = process.getOutputStream();
+		
+		new StreamHog(process.getErrorStream(), debug) // read Error Stream
+		//new StreamHog(process.getInputStream(), true) // read Error Stream
+		reader = new BufferedReader(new InputStreamReader(meltis));
+		return true;
+	}
+	
+	public boolean processText(String text) {
+		if (debug) println "Send text: $text"
+		process << text
+		process << "\n"
+		meltos.flush()
+		return true;
+	}
+	
+	public def readStream() {
+		if (debug) println "reading lines..."
+		
+		try {
+			String line = null;
+			while ((line = reader.readLine()) != null) {
+				buffer << line
+				buffer << "\n"
+			}
+		} catch (IOException e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+		}
+		
+		return "end"
+	}
+	
+	public boolean stop() {
+		meltos.close()
+		readStream()
+		
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {	}
+
+		if (e != 0) {
+			System.err .println("Process exited abnormally with code " + e )
+		}
+	}
+	
+	public def getResult() {
+		return SlashFile2TTRez.processString(buffer.toString())
+	}
+	
+	
+	public static void main(String[] args) {
+		MElt melt = new MElt(new File(""), new File(""), "fr")
+		melt.start()
+		melt.processText("""Il va.
+Un autre bout, de texte""")
+		melt.stop()
+
+		println "\nMELT DONE\n"
+		for (def r : melt.getResult()) {
+			for (def rr : r)
+				print rr+"\t"
+			println ""
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/SxPipe.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/SxPipe.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/SxPipe.groovy	(revision 321)
@@ -0,0 +1,164 @@
+
+package org.txm.tal.melt
+
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.txm.tal.Tagger
+import org.txm.tal.SlashFile2TTRez
+import org.txm.utils.StreamHog;
+
+class SxPipe {
+
+	Process process = null;
+	InputStream meltis
+	OutputStream meltos
+	boolean debug = false
+
+	BufferedReader reader
+	StringBuffer buffer = new StringBuffer();
+
+	public SxPipe() {
+	}
+
+	public boolean start() {
+		// Call Melt
+		def args = [];
+		args << "MElt"
+		args << "-t"
+		args << "-C"
+		ProcessBuilder pb = new ProcessBuilder(args);
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+			return false
+		}
+		meltis = process.getInputStream();
+		meltos = process.getOutputStream();
+
+		new StreamHog(process.getErrorStream(), debug) // read Error Stream
+		//new StreamHog(process.getInputStream(), true) // read Error Stream
+		reader = new BufferedReader(new InputStreamReader(meltis));
+		return true;
+	}
+
+	public boolean processText(String text) {
+		if (debug) println "Send text: $text"
+		process << text
+		process << "\n"
+		meltos.flush()
+		return true;
+	}
+
+	public def readStream() {
+		if (debug) println "reading lines..."
+
+		try {
+			String line = null;
+			while ((line = reader.readLine()) != null) {
+				buffer << line
+				buffer << "\n"
+			}
+		} catch (IOException e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+		}
+
+		return "end"
+	}
+
+	public boolean stop() {
+		meltos.close()
+		readStream()
+
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {	}
+
+		if (e != 0) {
+			System.err .println("Process exited abnormally with code " + e )
+		}
+	}
+
+	public def getResult() {
+		return buffer.toString()
+		//return SlashFile2TTRez.processString(buffer.toString())
+	}
+
+	public void printResult() {
+		def segs = getResult().split("\n")// \\{\\\$\\\$\\\$\\\$\\} \\\$\\\$\\\$\\\$
+
+		int i = 0;
+		for (def seg : segs) {
+			def tokens = []
+			for(def str :  seg.replaceAll("\\{", "\n{").tokenize("\n")) {
+				if (str.startsWith("{"))
+					println str.substring(1, str.indexOf("}")).replace("_ACC_O", "{").replace("_ACC_F", "}")
+			}
+		}
+	}
+
+	public def getResultTokens() {
+		def segs = getResult().split("\n")
+		def tokens = []
+
+		int i = 0, idx = -1;
+		for (def seg : segs) {
+			for(def str :  seg.replaceAll("\\{", "\n{").tokenize("\n")) {
+				if (str.startsWith("{")) {
+					idx = str.indexOf("}")
+					try{
+						tokens << [str.substring(1, idx).replace("_ACC_O", "{").replace("_ACC_F", "}"),
+							str.substring(idx+1)]
+					} catch(Exception e) {e.printStackTrace();println "STRING IS : '$str'"}
+				}
+			}
+
+		}
+		return tokens
+	}
+
+	public def getResultSegments() {
+		def segs = getResult().split("\n")
+		def segsFinal = []
+
+		int i = 0, idx = -1;
+		for (def seg : segs) {
+			def tokens = []
+			for(def str :  seg.replaceAll("\\{", "\n{").tokenize("\n")) {
+				if (str.startsWith("{")) {
+					idx = str.indexOf("}")
+					try{
+						tokens << [str.substring(1, idx).replace("_ACC_O", "{").replace("_ACC_F", "}"),
+							str.substring(idx+1)]
+					} catch(Exception e) {e.printStackTrace();println "STRING IS : '$str'"}
+				}
+			}
+			segsFinal << tokens
+		}
+		return segsFinal
+	}
+
+
+	public static void main(String[] args) {
+		SxPipe melt = new SxPipe()
+		melt.start()
+		melt.processText("""Il y a {du} pain \$\$\$\$ c'était bon. Y a-t-il assez ?
+Un autre bout, de texte : fin.""")
+		melt.stop()
+
+		melt.printResult()
+
+
+
+		//		println "\nMELT DONE\n"
+		//		for (def r : melt.getResult()) {
+		//			for (def rr : r)
+		//				print rr+"\t"
+		//			println ""
+		//		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MEltAnnotate.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MEltAnnotate.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/melt/MEltAnnotate.groovy	(revision 321)
@@ -0,0 +1,153 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2013-11-08 13:38:06 +0100 (Fri, 08 Nov 2013) $
+// $LastChangedRevision: 2569 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.tal.melt
+
+import java.text.DateFormat
+
+import org.txm.Toolbox
+import org.txm.importer.*
+import org.txm.importer.cwb.*
+import org.txm.objects.*
+import org.txm.scripts.teitxm.Annotate
+import org.txm.tal.SlashFile2TTRez
+
+// TODO: Auto-generated Javadoc
+/**
+ * Annotate and replace the TEI-TXM files of the folder $rootDirFile/txm with TreeTagger.
+ * creates $rootDirFile/interp and $rootDirFile/treetagger
+ *
+ */
+class MEltAnnotate extends Annotate {
+
+	public void initTTOutfileInfos(File rootDirFile, File modelfile)
+	{
+		reportFile = new File(rootDirFile,"NLPToolsParameters.xml");
+
+		respPerson = System.getProperty("user.name");
+		respId = "txm";
+		respDesc = "NLP annotation tool";
+		respDate = DateFormat.getDateInstance(DateFormat.SHORT, Locale.UK).format(new Date());
+		respWhen = DateFormat.getDateInstance(DateFormat.FULL, Locale.UK).format(new Date());
+
+		appIdent = "MElt";
+		appVersion = "2.0";
+
+		distributor = "";
+		publiStmt = """""";
+		sourceStmt = """""";
+
+		types = ["frpos"];
+		typesTITLE = ["frpos"];
+
+		typesDesc = ["pos tagset built from MElt"]
+		typesTAGSET = ["",""]
+		typesWEB = ["",""]
+
+		idform ="w";
+	}
+
+	/**
+	 * Apply tt.
+	 *
+	 * @param ttsrcfile the ttsrcfile
+	 * @param ttoutfile the ttoutfile
+	 * @param modelfile the modelfile
+	 * @return true, if successful
+	 */
+	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile) {
+		return applyMElt(ttsrcfile, ttoutfile)
+	}
+
+	public boolean applyMElt(File ttsrcfile, File ttoutfile) {
+		// Call Melt
+		def args = [];
+		args << "MElt"
+		args
+		ProcessBuilder pb = new ProcessBuilder(args);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		OutputStream os = process.getOutputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		ttsrcfile.eachLine ("UTF-8") { line ->
+			println "> $line"
+			os.println line
+		}
+		
+		ttoutfile.withWriter("UTF-8") { writer ->
+			String line;
+			println "getting result"
+			while ((line = br.readLine()) != null) {
+				println "< $line"
+				writer.println(line);
+			}
+			println "end of result"
+			int e = 0;
+			try {
+				e = process.waitFor();
+			} catch (Exception err) {	}
+
+			if (e != 0) {
+				System.err .println("Process exited abnormally with code " + e + " at " + DateFormat.getDateInstance(DateFormat.FULL, Locale.UK).format(new Date())); //$NON-NLS-1$ //$NON-NLS-2$
+
+				for (int c = 0; c < args.size(); c++)
+					System.out.print("" + args.get(c) + " "); //$NON-NLS-1$ //$NON-NLS-2$
+				System.out.println();
+			}
+		}
+
+		// Reformat MElt output
+		File tmpFile = new File(ttoutfile.getParentFile(),ttoutfile.getName()+".tmp")
+		new SlashFile2TTRez(ttoutfile, tmpFile, "UTF-8")
+	}
+	
+	public boolean run(File binDir, File txmDir) {
+		return run(binDir, txmDir, "fr.par")
+	}
+
+	public static void main(def args) {
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:/Program Files/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH, new File("/home/mdecorde/treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH, new File("/home/mdecorde/treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File("/home/mdecorde/TXM"));
+		
+		println "-- ANNOTATE - Running MElt tool"
+
+		File binDir = new File("/home/mdecorde/xml/melt-out")
+		File txmDir = new File("/home/mdecorde/xml/melt-in")
+		new MEltAnnotate().run(binDir, txmDir);
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLStaxTokenizerAdapter2.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLStaxTokenizerAdapter2.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/XMLStaxTokenizerAdapter2.groovy	(revision 321)
@@ -0,0 +1,160 @@
+package org.txm.tal
+
+import java.net.URL;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamReader;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.txm.importer.StaxIdentityParser;
+import org.txm.importer.graal.PersonalNamespaceContext;
+import org.txm.tal.melt.SxPipe
+
+
+/**
+ * Test of tokenizer that tokenize parts of a XML file using XPath resolved with the Stax parser
+ * 
+ * @author mdecorde
+ *
+ */
+class XMLStaxTokenizerAdapter2 {
+	def xpaths;
+	boolean debug = false;
+	boolean doTokenize = false
+	def segments = []
+	String str;
+
+	protected URL inputurl;
+	protected def inputData;
+	protected XMLInputFactory factory;
+	protected XMLStreamReader parser;
+
+	public static String TXMNS = "http://textometrie.org/1.0";
+	public static String TEINS = "http://www.tei-c.org/ns/1.0";
+	protected static PersonalNamespaceContext Nscontext = new PersonalNamespaceContext();
+
+	String currentXPath = "/"
+	String localname;
+
+	public XMLStaxTokenizerAdapter2(File infile, def xpaths) {
+		this.inputurl = infile.toURI().toURL();
+		inputData = inputurl.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		this.xpaths = xpaths
+	}
+
+	public boolean process() {
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+						currentXPath += localname+"/"
+						processStartElement();
+						break;
+					case XMLStreamConstants.CHARACTERS:
+						processCharacters();
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName();
+						processEndElement();
+						currentXPath = currentXPath.substring(0, currentXPath.length() - localname.length() -1)
+						break;
+
+				}
+			}
+		} catch(Exception e) {
+			println("Error while parsing file "+inputurl);
+			println("Location "+parser.getLocation());
+			org.txm.utils.logger.Log.printStackTrace(e);
+			parser.close();
+			return false;
+		}
+
+		if (parser !=null) parser.close();
+		return true;
+	}
+
+	protected void processStartElement() {
+		String localname = parser.getLocalName();
+		if (debug) println "start element $currentXPath"
+		if (doTokenize) {
+			if (debug) println "Found text element $currentXPath"
+			tokenizeCurrentText();
+		} else {
+			if (xpaths.contains(currentXPath)) {
+				doTokenize = true;
+				str = "";
+			}
+		}
+	}
+
+	protected void processEndElement() {
+		if (xpaths.contains(currentXPath)) {
+			tokenizeCurrentText()
+			doTokenize = false;
+			if (debug) println "Found text element: stop tokenizing"
+		}
+	}
+
+	protected void processCharacters() {
+		if (doTokenize) {
+			str += parser.getText();
+		}
+	}
+
+	int nsegments = 0;
+	protected tokenizeCurrentText() {
+		if (str.length() == 0) return;
+
+		segments << str
+		nsegments++;
+		str = "" // continue tokenizing
+	}
+
+	/**
+	 * Best tokenizer ever.
+	 * Extends this function
+	 * 
+	 * @param str
+	 * @return
+	 */
+	protected List<CharSequence> tokenize(String str) {
+		return str.tokenize();
+	}
+
+	public static void main(String[] args) {
+		File inputXMLFile = new File("/home/mdecorde/xml/comere/cmrclavardage/cmr-getalp_org-actu-tei-v1.xml");
+		def firstPass = new XMLStaxTokenizerAdapter2(inputXMLFile, ["/TEI/text/body/div/post/p/"]);
+
+		println "FIRST PASS"
+		firstPass.process()
+		println firstPass.getSegments().size()
+
+		println "TOKENIZING..."
+		StringBuffer buffer = new StringBuffer();
+		for(def segment : firstPass.getSegments()) {
+			buffer << segment.replaceAll("\n", " ") + "\n"
+		}
+		new File("/home/mdecorde/Bureau/test1.txt").withWriter {writer -> writer.println(buffer)}
+
+		SxPipe sxpipe = new SxPipe()
+		sxpipe.start()
+		sxpipe.processText(buffer.toString())
+		sxpipe.stop()
+
+		new File("/home/mdecorde/Bureau/test.txt").withWriter {writer -> writer.println(sxpipe.getResult())}
+		def tokenizedSegments = sxpipe.getResultSegments()
+		println "tokenizedSegments size "+tokenizedSegments.size()
+		//		for(def segment : firstPass.getSegments()) {
+		//			tokenizedSegments << segment.tokenize()
+		//		}
+
+		//		println "SECOND PASS"
+		//		def secondPass = new XMLStaxTokenizerAdapter(inputXMLFile, ["/TEI/text/body/div/post/p/"], tokenizedSegments);
+		//		secondPass.process(new File("/home/mdecorde/xml/comere/cmrclavardage/tmp.xml"))
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tal/SlashFile2TTRez.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tal/SlashFile2TTRez.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tal/SlashFile2TTRez.groovy	(revision 321)
@@ -0,0 +1,45 @@
+package org.txm.tal
+
+class SlashFile2TTRez {
+	public static boolean debug = true;
+
+	public SlashFile2TTRez(File inputFile, File outputFile, String encoding) {
+		outputFile.withWriter("UTF-8") { writer ->
+			int nline = 1;
+			inputFile.eachLine(encoding) { line ->
+				def rez = processString(line)
+				for (def l : rez) {
+					//if (debug) println "result: $l"
+					writer.println(l[0] + "\t" + l[1])
+				}
+				nline++;
+			}
+		}
+	}
+
+	def static empty = [null,null]
+	public static def processString(String line) {
+		def rez = [];
+		if (debug) println "process line $line"
+		if (line.length() == 0) {
+			return []
+		} else {
+			for (String str : line.tokenize()) {
+				if (debug) println " process str $str"
+				int idx = str.lastIndexOf("/")
+				if (idx > 0) {
+					rez << [str.substring(0, idx), str.substring(idx+1)]
+				} else {
+					if (debug) println "ERROR line $line"
+				}
+			}
+		}
+		return rez
+	}
+
+	public static void main(def args) {
+		File inputFile = new File("/home/mdecorde/Taggers/melt-2.0b2/txtbrut-o.txt");
+		File outputFile = new File("/home/mdecorde/Taggers/melt-2.0b2/txtbrut-tt.txt");
+		new SlashFile2TTRez(inputFile, outputFile, "UTF-8")
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ConcordancesExport_discours.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ConcordancesExport_discours.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ConcordancesExport_discours.groovy	(revision 321)
@@ -0,0 +1,183 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2012-10-15 17:35:36 +0200 (lun., 15 oct. 2012) $
+// $LastChangedRevision: 2279 $
+// $LastChangedBy: mdecorde $ 
+//
+
+// WARNING when this script is executed for the second time,
+// the window asking for the query parameter is displayed behind all the other windows,
+// so that one can have the feeling that TXM is frozen :
+// just move the other windows to get and answer the query-window hidden behind,
+// and this solves the problem.
+
+package org.txm.export
+
+// import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.concordances.*
+import org.txm.functions.concordances.comparators.*
+import org.txm.functions.ReferencePattern
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+//"ExportConcordanceBFM.csv"
+def outfile = new File("ExportConcordanceVOEUX.csv")
+
+// get the corpus
+// **PARAMETER** : name of the corpus (here DISCOURS)
+// def discours = CorpusManager.getCorpusManager().getCorpus("QUETE")
+def discours = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+
+// **PARAMETER** : selection and name of the properties - "word", "id", "text" and "text_id" are always available
+// get some properties
+// def pos = discours.getProperty("ttpos")
+// def pos = discours.getProperty("frpos")
+def pos = discours.getProperty("pos")
+println "pos: "+pos
+// def pb = discours.getProperty("pb")
+// println "pb: "+pb
+def id = discours.getProperty("id")
+println "id: "+id
+def word = discours.getProperty("word")
+println "word: "+word
+def text = discours.getStructuralUnit("text")
+println "text: "+text
+def text_id = text.getProperty("id")
+println "text_id: "+text_id
+def text_date = text.getProperty("date")
+println "text_date: "+text_date
+def text_loc = text.getProperty("loc")
+println "text_loc: "+text_loc
+
+// create a query. Edit the query here
+def querys = javax.swing.JOptionPane.showInputDialog(null, "query ")
+if (querys == null)
+	return
+def query = new Query(Query.fixQuery(querys))
+
+println "Compute concordance "
+def start = System.currentTimeMillis()
+
+// define the references pattern for each concordance line
+def referencePattern = new ReferencePattern()
+// **PARAMETER** take properties among the ones selected and defined above,
+// to compose the reference pattern (in the first column)
+//referencePattern.addProperty(text_id)
+referencePattern.addProperty(text_loc)
+referencePattern.addProperty(text_date)
+//referencePattern.addProperty(pb)
+//referencePattern.addProperty(pos)
+//referencePattern.addProperty(id)
+
+def sortReferencePattern = new ReferencePattern()
+// **PARAMETER** take properties among the ones selected and defined above,
+// to set the sorting order of the first column
+sortReferencePattern.addProperty(text_id)
+//sortReferencePattern.addProperty(pb)
+//sortReferencePattern.addProperty(pos)
+sortReferencePattern.addProperty(id)
+
+
+// compute the concordance with contexts of 15 words on each side of the keyword
+// **PARAMETER** choose the properties : here [word, pos] are the properties according to which the pivot is sorted
+// def concordance = new Concordance(discours, query, word, [word, pos], referencePattern, 10, 10)
+def concordance = new Concordance(discours, query, word, [word, pos], referencePattern, sortReferencePattern, 10, 10)
+
+// get a builtin sort function
+def comparator = new LexicographicKeywordComparator()
+comparator.initialize(discours)
+
+// sort the concordance
+//concordance.sort(comparator)
+
+println "done "+(System.currentTimeMillis()-start)
+
+// **PARAMETER** name of the output file
+//def writer = new FileWriter("ExportConcordanceBFM3TT.csv")
+def writer = new FileWriter(outfile)
+
+println "write lines"
+start = System.currentTimeMillis()
+
+// define which occurrence properties will be displayed
+concordance.setViewProperties([word])
+
+for (int i = 0 ; i < concordance.getNLines() ; i += 1000) {
+	List<Line> lines
+	if (i+1000 > concordance.getNLines())
+		lines = concordance.getLines(i, concordance.getNLines()-1)
+	else
+		lines = concordance.getLines(i, i+1000)
+		
+	// println "lines: from "+i
+	for (Line l : lines) {
+		/*
+		 A : référence (nom de texte + page)
+		 B : contexte gauche (10 mots)
+		 C : pivot (cf. la remarque sur "ce que")
+		 D : pos du pivot
+		 E : colonne vide pour les commentaires
+		 F : contexte droit (10 mots)
+		 G : identifiant de l'occurrence (pour injecter les corrections après) 
+		 */
+
+// **PARAMETERS** define the strings which will be used to print the result		
+//		String ntext = l.getViewRef().getValue(text_id)
+		String loc = l.getViewRef().getValue(text_loc)
+		String date = l.getViewRef().getValue(text_date)
+//		String page = l.getViewRef().getValue(pb)
+		String lcontext = l.leftContextToString()
+		String pivot = l.keywordToString()
+//		String rien = ""
+		String rcontext = l.rightContextToString()
+//		String ids = l.getViewRef().getValue(id)
+		
+		String poss = ""
+		for (String s : l.getKeywordsViewProperties().get(pos)) {
+			poss += s+" "
+		}
+		
+		//println l.getKeywordsViewProperties().get(pos)
+// **PARAMETERS** composition of the printed line, cf. strings defined above
+//		writer.write(ntext+"_"+page+"\t"+lcontext+"\t"+pivot+"\t"+poss+"\t"+rien+"\t"+rcontext+"\t"+ids+"\n")
+		writer.write(loc+", "+date+"\t"+lcontext+"\t"+pivot+"\t"+poss+"\t"+rcontext+"\n")
+	}
+	writer.flush()
+}
+writer.close()
+
+println "done "+(System.currentTimeMillis()-start)+" results"
+
+//BP change nom du fichier
+println("Concordance "+query.getQueryString()+" saved in file "+outfile.getAbsolutePath())
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/conll2009/ToCoNLL2009.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/conll2009/ToCoNLL2009.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/conll2009/ToCoNLL2009.groovy	(revision 321)
@@ -0,0 +1,115 @@
+package org.txm.export.conll2009
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.txm.searchengine.cqp.CqpDataProxy;
+import org.txm.searchengine.cqp.corpus.Corpus
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.Query
+
+class ToCoNLL2009 {
+
+	boolean debug = true
+	/**
+	 * To CoNLL2009.
+	 *
+	 * @param outfile the outfile, where is result is saved
+	 * @param corpus the corpus or sub-corpus to export
+	 * @param sentenceUnit the StructuralUnitProperty that identify a sentence
+	 * @param word the word property
+	 * @param lemme the lemme property
+	 * @param pos the pos property
+	 * @param encoding the encoding of the outfile
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile, Corpus corpus, StructuralUnitProperty sentenceUnit, Property word, Property lemme, Property pos, String encoding)
+	{
+		def matches = corpus.query(new Query("<s> []+ </s>"), "TMPEXPORTCONNL", false).getMatches()
+		if (debug) println matches.size()
+		int npositions = 0;
+		for (def match : matches) npositions += match.length +1
+		if (debug) println "npositions= $npositions"
+
+		int[] positions = new int[npositions+1]
+		int i = 0;
+		for (def match : matches) {
+			for (int p : match.getRange())
+				positions[i++] = p
+		}
+
+		List<String> words = CorpusManager.getCorpusManager().getCqiClient().cpos2Str(word.getQualifiedName(), positions);
+
+		List<String> lemmes = null;
+		if (lemme != null) {
+			lemmes = CorpusManager.getCorpusManager().getCqiClient().cpos2Str(lemme.getQualifiedName(), positions);
+		}
+
+		List<String> poss = null;
+		if (pos != null) {
+			poss =  CorpusManager.getCorpusManager().getCqiClient().cpos2Str(pos.getQualifiedName(), positions);
+		}
+
+		Writer writer = new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8"); //$NON-NLS-1$;
+
+		String lemmestr;
+		String posstr;
+		int id = 0;
+		int iposition = 0;
+		for (def match : matches) {
+			id = 0 // first position in sentence
+			for (i = match.start ; i < match.end ; i++) { // loop n times, n = mathc length
+
+				if (lemme != null)
+					lemmestr = lemmes.get(iposition);
+				else
+					lemmestr = "_";
+
+				if (pos != null)
+					posstr = poss.get(iposition);
+				else
+					posstr = "_";
+
+				//print(""+id+"\t"+words.get(iposition)+"\t"+lemmestr+"\t_\t"+posstr+"\t_\t_\t_\t_\t_\t_\t_\t_\t_\t_\n");
+				writer.write(""+id+"\t"+words.get(iposition)+"\t"+lemmestr+"\t_\t"+posstr+"\t_\t_\t_\t_\t_\t_\t_\t_\t_\t_\n");
+
+				id++; // next word
+				iposition++ // nextposition
+			}
+		}
+		writer.flush();
+		writer.close();
+
+		return true;
+	}
+
+	public static void main(String[] args) {
+		Corpus corpus = null
+		File outfile = new File("/home/mdecorde/Bureau/conn_export.tsv")
+		String encoding = "UTF-8"
+
+		String sentenceProperty = "s_id"
+		String posProperty = "frpos"
+		String lemmaProperty = "frlemma"
+
+
+		def split = sentenceProperty.split("_", 2)
+		StructuralUnitProperty s = null;//corpus.getStructuralUnit(split[0]).getProperty(split[1])
+		Property word = null;//corpus.getProperty("word")
+		Property lemma = null;//corpus.getProperty(lemmaProperty)
+		Property pos = null;//corpus.getProperty(posProperty)
+
+		//		if (s == null) { println "Error sentence property: $sentenceProperty"; return}
+		//		if (word == null) { println "Error no word property"; return}
+		//		if (pos == null) { println "Error pos property: $posProperty"; return}
+		//		if (lemma == null) { println "Error lemma property: $lemmaProperty"; return}
+
+
+		ToCoNLL2009 processor = new ToCoNLL2009()
+		processor.process(outfile, corpus, s, word, lemma, pos, encoding)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ExportLT.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ExportLT.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ExportLT.groovy	(revision 321)
@@ -0,0 +1,44 @@
+package org.txm.export
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.stat.data.*;
+import org.txm.functions.cooccurrences.*
+
+
+def corpus = CorpusManager.getCorpusManager().getCorpus("DISCOURS") // mettre le nom du corpus en Majuscules ici
+def partition = corpus.getPartition("types");
+
+int NTable = 1;
+for (Object o : partition.getResults()) {
+//	if (o instanceof LexicalTable) {
+		File outfile = new File("lt${NTable}.txt")
+		println "export LT of $partition in "+outfile.getAbsolutePath();
+		NTable++;
+		outfile.withWriter{writer->
+		def lt = o;
+		println "found lexical table: "+lt.getName()
+		println "n col: "+lt.getNColumns()
+		println "n line: "+lt.getNRows()
+		def cols = lt.getColNames().asStringsArray();
+		String strline = "unit\t";
+		for (int i = 0 ; i < cols.length ; i++) {
+			strline += cols[i]+"\t"
+			
+		}
+		writer.println(strline.substring(0, strline.length()-1));
+		
+		def rows = lt.getRowNames().asStringsArray();
+		for (int i = 0 ; i < rows.length ; i++) {
+			strline = "";
+			strline += rows[i]+"\t";
+			def line = lt.getRow(i).asIntArray();
+			for (int j = 0 ; j < line.length ; j++)
+				strline += line[j]+"\t";
+			writer.println(strline.substring(0, strline.length()-1));
+		}
+		
+		writer.close();
+		}
+//	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/ConcordanceBlocks.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/ConcordanceBlocks.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/ConcordanceBlocks.groovy	(revision 321)
@@ -0,0 +1,474 @@
+package org.txm.export.ts;
+
+import javax.xml.parsers.DocumentBuilderFactory
+
+class ConcordanceBlocks {
+	def codec = 'UTF-8'
+
+	int cx = 30;
+	def ntTypes = [];
+	def tTypes = [];
+	File xmlfile, outfile;
+
+	public boolean process(File xmlfile, File outfile, int cx, def ntTypes, def tTypes) {
+		this.cx = cx;
+		this.xmlfile = xmlfile;
+		this.outfile = outfile;
+		this.ntTypes = ntTypes;
+		this.tTypes = tTypes;
+
+		Writer writer = outfile.newPrintWriter(codec)
+
+		println 'Reading XML File'
+		def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+		def records = builder.parse(xmlfile).documentElement
+		println 'done.'
+
+		println 'done.  Pre-treating file to combine results with the same pivot...'
+		records = matchCombine(records)
+		println 'done.'
+
+		println 'Calculating max. number of blocks...'
+		def nBlock = 0
+
+		def allMatches = toList(records.getElementsByTagName('match'))
+
+		for (def match : allMatches) {
+			nBlock = [
+				nBlock,
+				toList(match.getElementsByTagName('variable')).findAll{
+					it.getAttribute('name').startsWith('#block')
+				}.size()
+			].max()
+		}
+
+		println "done ( $nBlock )"
+
+		// Table headers
+		println 'Writing concordance...'
+
+		def header = ['sId', 'LeftCxOutsideSnt', 'LeftCxInsideSnt']
+
+		for (int i = nBlock ; i > 0 ; i--) {
+
+			header.add("${i}BlockBeforePivot")
+
+			for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {
+
+				header.add("${i}BlockBeforePivotType${j+1}")
+
+			}
+		}
+
+		header.add('Pivot')
+
+		for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {
+
+			header.add("PivotType${j+1}")
+
+		}
+
+		for (int i = 1 ; i <= nBlock ; i++) {
+
+			header.add("${i}BlockAfterPivot")
+
+			for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {
+
+				header.add("${i}BlockAfterPivotType${j+1}")
+
+			}
+		}
+
+		header.addAll(['RightCxInsideSnt', 'RightCxOutsideSnt', 'Warnings'])
+
+		writer.write(header.join("\t")+"\n");
+
+		def allTerminals = toList(records.getElementsByTagName('t'))
+		int tenPercentile = 0
+
+		int i = 0;
+		for (def aMatchNode : allMatches) {
+			if ((int)(((float)++i / allMatches.size()) * 10) > tenPercentile)
+				println ""+(++tenPercentile * 10)+ ' percent complete...'
+
+			def (rowDict, inSntCxLengthLeft, inSntCxLengthRight) = match2CSVrow(aMatchNode, header);
+			
+			// Add out-of-sentence context
+			def sNode = aMatchNode.parentNode.parentNode; // sentence
+			def terminals = toList(sNode.getElementsByTagName('t'));
+			def firstTInS = terminals[0] // get first node of the sentence
+			def lastTInS = terminals[-1] // get last node of the sentence
+			def firstTInSIx = allTerminals.indexOf(firstTInS) // get its position in the text
+			def lastTInSIx = allTerminals.indexOf(lastTInS) // get its position in the text
+
+			// Left context
+			def lexs = []
+			int start = Math.max(firstTInSIx - cx + inSntCxLengthLeft, 0);
+			int end = firstTInSIx;
+			if(start < end)
+			for (def tNode : allTerminals.subList(start, end)){
+				lexs.add(tNode.getAttribute('word'))
+				if (toList(tNode.parentNode.getElementsByTagName('t'))[-1] == tNode)
+					lexs.add('/')
+			}
+			rowDict['LeftCxOutsideSnt'] = lexs.join(" ")
+
+			// Right context
+			lexs = []
+
+			start = lastTInSIx + 1;
+			end = Math.min(allTerminals.size(), lastTInSIx + cx - inSntCxLengthRight)
+
+			if(start < end) {
+				for (def tNode : allTerminals.subList(start, end)){
+					lexs.add(tNode.getAttribute('word'))
+					if (toList(tNode.parentNode.getElementsByTagName('t'))[-1] == tNode)
+						lexs.add('/')
+				}}
+			rowDict["RightCxOutsideSnt"] = lexs.join(" ")
+
+			// Right rowDict to CSV
+
+			//			CSVWriter.writerow(dict(zip([k for k in rowDict.iterkeys()], \
+			//        [ v.encode('utf-8') for v in rowDict.itervalues() ] )))
+
+			String line = "";
+			for (int ii = 0 ; ii < header.size() ; ii++){
+				String h = header.get(ii)
+				String val = rowDict.getAt(h);
+				if (val == null)
+					line += "--"
+				else
+					line += val
+				if (ii < header.size() -1)
+					line +="\t"
+			}
+			writer.write(line+"\n")
+			writer.flush()
+
+		}
+		writer.close();
+		return true;
+	}
+
+	private def match2CSVrow(def aMatchNode, def header) {
+		// Step 1: Build a LIST of DICTIONARIES to describe the variables:
+		// dict(name='varname_minus_the_hash'
+		//      parent='nt_nodes'
+		//      terminals='t_nodes in a list')
+		
+		def sNode = aMatchNode.parentNode.parentNode // get the sentence node
+
+		def tNodesInSentence = toList(sNode.getElementsByTagName('t')) // get all terminal nodes of the sentence
+
+		def varDetails = []
+
+		for ( def aVariable : aMatchNode.getElementsByTagName('variable')){
+			if ( aVariable.getAttribute('name') == '#pivot' ||
+			aVariable.getAttribute('name').startsWith('#block')) {
+				def varParent = idKey(sNode, aVariable.getAttribute('idref'))
+
+				varDetails.add([
+							'name' : aVariable.getAttribute('name').substring(1),
+							'parent': varParent,
+							'terminals': toList(getTNodes(varParent)),
+							'lexform': writeLexForm(varParent),
+						])
+
+				// Add types
+
+				for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {
+
+					def a = ''
+
+					if (j < ntTypes.size() && varParent.getAttribute(ntTypes[0])
+					) {
+						a = varParent.getAttribute(ntTypes[j])
+					} else if (j < tTypes.size() && varParent.getAttribute(tTypes[0])
+					) {
+						a = varParent.getAttribute(tTypes[j])
+					}
+
+					varDetails[-1]["type${j+1}"] = a
+
+				}
+
+				// write min idx in sentence of terminal nodes used
+				varDetails[-1]['terminalsIx'] = varDetails[-1]['terminals'].collect{tNodesInSentence.indexOf(it)}
+			}
+		}
+
+		// Sort varDetails by the start ID of the word
+
+		varDetails =  varDetails.sort{it['terminalsIx'].min()}
+
+		def pivotBlockPosition = 0;
+		pivotBlockPosition = varDetails.findIndexOf{it['name'] == 'pivot'}
+
+		// Write the table
+		def rowDict = [:]
+		rowDict['sId'] = sNode.getAttribute('id')
+
+		// write the pivot
+		rowDict['Pivot'] = varDetails[pivotBlockPosition]['lexform']
+
+		for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {
+
+			rowDict["PivotType${j+1}"] = varDetails[pivotBlockPosition]["type${j+1}"]
+		}
+
+		def startNextId = (varDetails[pivotBlockPosition]['terminalsIx']).min()
+		def endPrevId = (varDetails[pivotBlockPosition]['terminalsIx']).max()
+
+		// Write the Pre-pivot blocks
+
+		def i = 0
+
+		//println "rowDict: $rowDict"
+		if ( pivotBlockPosition > 0)	{
+			for ( int j = pivotBlockPosition -1 ; j >= 0 ; j--) {
+				
+				i++;
+				def block = varDetails.get(j)
+				//println "process block: $block"
+
+				rowDict["${i}BlockBeforePivot"] = block['lexform']
+
+				for ( int k = 0 ; k < [ntTypes.size(), tTypes.size()].max() ; k++ ) {
+					rowDict["${i}BlockBeforePivotType${k+1}"] = block["type${k+1}"]
+				}
+
+				// Add any intervening words to the right edge.
+
+				def rightEdge = (block['terminalsIx'].findAll{ it < startNextId }).max();
+				//println ""+block['terminalsIx']+"   rightEdge: $rightEdge"
+
+				for (int ix = rightEdge + 1 ; ix < startNextId ; ix++) // add word to reach the pivot
+				{
+					rowDict["${i}"+"BlockBeforePivot"] += ' {' + tNodesInSentence[ix].getAttribute('word') + '}';
+				}
+
+				startNextId = (block['terminalsIx']).min()
+			}
+		}
+		
+		// Write the in-sentence left context
+
+		def lexs = tNodesInSentence.subList(0, startNextId).collect {it.getAttribute('word')}
+		rowDict['LeftCxInsideSnt'] = lexs.join(" ")
+
+		def inSntCxLengthLeft = startNextId
+
+		// Write the Post-pivot blocks
+
+		i = 0
+		for (def block in varDetails.subList(pivotBlockPosition + 1, varDetails.size())){
+			i++
+			rowDict["${i}BlockAfterPivot"] = block['lexform']
+
+			for (int j = 0 ; j < [ntTypes.size(), tTypes.size()].max() ; j++) {
+				rowDict["${i}BlockAfterPivotType${j+1}"] = block["type${j+1}"]
+			}
+
+			// Add any intervening words to the left edge.
+			// Note that this isn't always possible, in particular
+			// if the preceding element is discontinuous.
+
+			def leftEdgeList = block['terminalsIx'].findAll{it > endPrevId }
+
+			if ( leftEdgeList.size() > 0) {
+				def leftEdge = leftEdgeList.min()
+				for (int ix = leftEdge - 1 ; ix > endPrevId ; ix--) {
+					rowDict["${i}BlockAfterPivot"] = '{' + tNodesInSentence[ix].getAttribute('word') + '} ' + rowDict["${i}BlockAfterPivot"]
+				}
+			}
+			else {
+				rowDict["${i}BlockAfterPivot"] = '{?} ' + rowDict["${i}BlockAfterPivot"]
+			}
+			endPrevId = block['terminalsIx'].max()
+		}
+
+		// Write the in-sentence right-context
+
+		lexs = tNodesInSentence.subList(endPrevId + 1, tNodesInSentence.size()).collect {it.getAttribute('word')}
+		rowDict['RightCxInsideSnt'] = lexs.join(" ")
+
+		def inSntCxLengthRight = tNodesInSentence.size() - endPrevId
+
+		return [rowDict, inSntCxLengthLeft, inSntCxLengthRight]
+		return null;
+	}
+
+	public def toList(def iterable) {
+		return iterable.findAll {true};
+	}
+
+	/**
+	 * return the children of sNode with the id anId 
+	 */
+	public def idKey(def sNode, String anId) {
+		return sNode.getElementsByTagName("*").find{it.getAttribute("id") == anId}
+	}
+
+	/**
+	 * return a list of all terminal nodes of the node.
+	 * if the node is a non-terminal, iterate over children and so on
+	 */
+	def getTNodes(theNode) {
+		def terminals = []
+		def unprocessed = [theNode]
+
+		while( unprocessed.size() > 0) {
+			def aNode = unprocessed.pop()
+			def edges = toList(aNode.getElementsByTagName('edge'))
+			if (edges.size() == 0)
+				terminals.add(aNode)
+			else
+				for (def anEdge : edges)
+					unprocessed.add(idKey(theNode.parentNode.parentNode, anEdge.getAttribute('idref')))
+		}
+		return terminals;
+	}
+
+	/**
+	 * 
+	 * @param theNode
+	 * @return the join of the terminal nodes value in theNode a varaible node
+	 */
+	def writeLexForm(theNode) {
+		def sNode = theNode.parentNode.parentNode // get the sentence of the variable
+		def allTNodes = sNode.getElementsByTagName('t') // get all sentence children
+		def tNodesInTheNode = getTNodes(theNode) // get the terminal nodes pointed by idref
+		def begunNode = false
+		def lexs = []
+		def lexBuffer = []
+		for (def aTNode : allTNodes){
+			if ( aTNode in tNodesInTheNode) {
+				begunNode = true
+				lexs.addAll(lexBuffer)
+				lexs.add(aTNode.getAttribute('word'))
+				lexBuffer = []
+			}
+			if ( begunNode && !tNodesInTheNode.contains(aTNode))
+				lexBuffer.add('[' + aTNode.getAttribute('word') + ']')
+		}
+		return lexs.join(" ");
+	}
+
+	def matchCombine(theDOM) {
+
+		def matchesNodes = theDOM.getElementsByTagName('matches')
+
+		for (def aMatchesNode : matchesNodes){
+
+			def pivotNodes = toList(
+					aMatchesNode.getElementsByTagName('variable')
+					).findAll{it.getAttribute('name') == '#pivot'}
+
+			def checkedPivots = []
+
+			while (pivotNodes){
+
+				def aPivotNode = pivotNodes.remove(0)
+
+				def matchingPivotList = checkedPivots.findAll{
+					it.getAttribute('idref') == aPivotNode.getAttribute('idref')
+				}
+
+				if (matchingPivotList) {
+
+					// duplicate pivot; copy all variables
+
+
+					for (def node : toList(aPivotNode.getParentNode()
+					.getElementsByTagName('variable')
+					)) {
+
+						matchingPivotList[0].getParentNode().appendChild(node)
+
+					}
+
+					def variables = toList(
+							matchingPivotList[0].getParentNode()
+							.getElementsByTagName('variable')
+							)
+
+					// remove duplicates
+
+					def checkedVariables = []
+
+					while (variables) {
+
+						def aVariableNode = variables.remove(0)
+
+						def matchingVariableList = checkedVariables.findAll{
+							(
+									it.getAttribute('idref') == aVariableNode.getAttribute('idref')
+									&& (
+									it.getAttribute('name') == aVariableNode.getAttribute('name')
+									|| (
+									it.getAttribute('name').startsWith('#block')
+									&& aVariableNode.getAttribute('name').startsWith('#block')
+									)
+									)
+									)
+						}
+
+						if (matchingVariableList) {
+
+							// remove variable node
+
+							aVariableNode.getParentNode().removeChild(aVariableNode)
+
+						}
+
+						else {
+
+							checkedVariables.add(aVariableNode)
+
+						}
+					}
+				}
+				else {
+
+					// not the same pivot
+
+					checkedPivots.add(aPivotNode)
+				}
+			}
+			// Tidying up: remove empty <match /> nodes
+			def matchNodes = toList(aMatchesNode.getElementsByTagName('match'))
+
+			for (def matchNode : matchNodes) {
+
+				if ( !toList(matchNode.getElementsByTagName('variable'))) {
+
+					aMatchesNode.removeChild(matchNode)
+				}
+			}
+		}
+
+		return theDOM
+	}
+
+	public static void usage() {
+		println 'concordance_blocks [OPTIONS] inputfile.xml [outputfile.csv]'
+		println 'OPTIONS'
+		println '-h, --help           Displays this message.'
+		println '-c --context [length] Sets the number of words in context.'
+	}
+
+	static main(args) {
+		if(args.length == 0)
+			usage()
+		int cx = 30;
+		def ntTypes = ['cat'];
+		def tTypes = ['pos'];
+		File xmlfile = new File(args[0])
+		File outfile = new File(args[1])
+		ConcordanceBlocks p = new ConcordanceBlocks();
+		println "START"
+		p.process(xmlfile, outfile, cx, ntTypes, tTypes)
+		println "END"
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/PunctInject.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/PunctInject.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/PunctInject.groovy	(revision 321)
@@ -0,0 +1,347 @@
+// Script to restore punctuation etc. to results of TS query.
+// Inputs:
+// --- Tiger-XML Document node
+// --- Java array:
+// --- --- String [index][type] where:
+// --- --- --- type == 0 gives the xml:id
+// --- --- --- type == 1 gives the word form
+// Process:
+// --- Injects punctuation.
+// Returns:
+// --- Tiger-XML Document node.
+package org.txm.export.ts;
+
+import javax.xml.parsers.DocumentBuilderFactory
+import org.txm.searchengine.cqp.CqpDataProxy;
+import org.txm.searchengine.cqp.corpus.Corpus
+import org.txm.searchengine.cqp.corpus.CorpusManager;
+import org.txm.searchengine.cqp.corpus.QueryResult;
+import org.txm.searchengine.cqp.corpus.query.Query
+
+public class PunctInject
+{
+	public process (def tigerXml, def txmIdWordTable) {
+
+		def allTs = toList(tigerXml.getElementsByTagName('t'))
+
+		def tAttrs = getTAttrNames(allTs[0])
+
+		def tNodeIdPrefix = allTs[0].getAttribute('id').tokenize('#').first()
+
+		def addedWordIds = []
+
+		int tIx = 0
+
+		while (txmIdWordTable) {
+
+			def word = txmIdWordTable.remove(0)
+
+			if (tIx == allTs.size()) {
+				// End of TS file, but still words left in the BFM file.
+				addTNode(word, allTs.last(), tAttrs, tNodeIdPrefix, 'append')
+			}
+
+			else {
+
+				def tNode = allTs[tIx]
+
+				def tId = getTNodeId(tNode)
+
+				if (tId == word[0] && tNode.getAttribute('word') == word[1]) {
+
+					// alles gut
+
+					tIx += 1
+
+				}
+
+				else if (tId == word[0]) {
+
+					println("Mismatched Ids! ($tId)")
+					tIx += 1
+				}
+
+				else if (['#', '*'].contains(tNode.getAttribute('word')) ) {
+
+					// SRCMF duplicata; try comparing word against the next tNode next time
+					// around.
+
+					txmIdWordTable.add(0, word)
+
+					tIx += 1
+
+				}
+				
+				// Check that the SRCMF corpus doesn't have a bug in it...
+
+				else if ( !(word[1] =~ /[\,\.\?\!\:\;\(\)\[\]\{\}]/)
+				&& (allTs[0..tIx - 1].find{
+					it.getAttribute('id') == "$tNodeIdPrefix#$tId"
+				})) {
+
+					println "Warning: word ${tNode.getAttribute('word')}, id $tId appears twice in corpus!"
+
+					txmIdWordTable.add(0, word)
+
+					tIx += 1
+
+				}
+				
+				// Check that there's not an extra word in the SRCMF corpus (rare, usu. a tokenisation change)
+				
+				else if ( !(word[1] =~ /[\,\.\?\!\:\;\(\)\[\]\{\}]/)
+				&& (allTs[tIx..-1].find{
+					it.getAttribute('id') == "$tNodeIdPrefix#${word[0]}"
+				})) {
+			
+					println "Warning: word ${tNode.getAttribute('word')}, id $tId does not appear in BFM!"
+
+					txmIdWordTable.add(0, word)
+					
+					tIx += 1
+				}
+
+				else if (addedWordIds.contains(tId)) {
+
+					println "Warning: word ${tNode.getAttribute('word')}, id ${tId} out of sequence in SRCMF corpus!"
+
+					txmIdWordTable.add(0, word)
+
+					tIx += 1
+
+				}
+
+				else {
+
+					// Insert word.  In the first instance, it will have the same parent as
+					// the tNode before which it's being inserted.
+
+					addTNode(word, allTs[tIx], tAttrs, tNodeIdPrefix, 'before')
+
+					addedWordIds.add(word[0])
+
+				}
+
+			}
+
+		}
+
+		// Second phase: move punctuation into previous sentence,
+		// dependent on sequence.
+
+		def allTerminalses = toList(tigerXml.getElementsByTagName('terminals'))
+
+		for (def i = 1 ; i < allTerminalses.size() ; i++) {
+
+			def ts = toList(allTerminalses[i].getElementsByTagName('t'))
+
+			def startPunc = true
+
+			def puncStack = []
+
+			while (ts && startPunc) {
+
+				if ((ts[0].getAttribute('word') =~ /[A-zÀ-ÿ0-9]/).size() == 0) {
+
+					puncStack.add(ts.remove(0))
+
+				}
+
+				else {
+
+					startPunc = false
+
+				}
+
+			}
+
+			// Now, treat the punctuation stack at the beginning of the sentence
+
+			if ( puncStack ) {
+
+				int moveLeft = 0
+
+				// First, identify LAST instance of sentence-final punctuation.
+
+				def puncString = puncStack.collect{ it.getAttribute('word')[0] }.join('')
+
+				def matches = puncString =~ /[\.\,\;\:\!\?\)\]\}»”’]/
+
+				if (matches.size() > 0) {
+
+					moveLeft = puncString.lastIndexOf(matches[-1]) + 1
+
+				}
+
+				// Second, split pairs of straight quotes
+
+				matches = puncString =~ /(""|'')/ //"
+
+				if (matches.size() > 0) {
+
+					moveLeft = [moveLeft, puncString.lastIndexOf(matches[-1][0]) + 1].max()
+				}
+
+				// Now, move moveLeft punctuation nodes to the end of the prev. sentence
+
+				ts = toList(allTerminalses[i].getElementsByTagName('t'))
+
+				for (def j = 0 ; j < moveLeft ; j++ ) {
+
+					allTerminalses[i - 1].appendChild(ts[j])
+
+				}
+
+			}
+		}
+		return tigerXml
+
+	}
+
+	private addTNode(word, tNode, tAttrs, tNodeIdPrefix, where) {
+
+		def newTNode = tNode.getOwnerDocument().createElement('t')
+
+		for (def anAttr : tAttrs) {
+
+			if (anAttr == 'id') {
+
+				newTNode.setAttribute('id', "${tNodeIdPrefix}#${word[0]}")
+
+			}
+
+			else if (anAttr == 'word') {
+
+				newTNode.setAttribute('word', word[1])
+
+			}
+
+			else {
+
+				newTNode.setAttribute(anAttr, '--')
+
+			}
+
+		}
+
+		if (where == 'before') {
+
+			tNode.getParentNode().insertBefore(newTNode, tNode)
+
+		}
+
+		else if (where == 'append') {
+
+			tNode.getParentNode().appendChild(newTNode)
+
+		}
+
+		else {
+
+			throw new IllegalArgumentException('Bad before value')
+
+		}
+
+	}
+
+	public getTAttrNames(tNode) {
+
+		def nodeMap = tNode.attributes
+
+		def nameList = []
+
+		for ( def i = 0 ; i < nodeMap.getLength() ; i++ ) {
+			nameList.add( nodeMap.item(i).nodeName )
+		}
+
+		return nameList
+
+	}
+
+	public getTNodeId(tNode) {
+
+		return tNode.getAttribute('id').tokenize('#').last()
+
+	}
+
+	public def toList(def iterable) {
+		return iterable.findAll {true};
+	}
+
+	public static def getWords(String corpusname, String query)
+	{
+		CorpusManager cm = CorpusManager.getCorpusManager();
+		Corpus corpus = cm.getCorpus(corpusname);
+		def word_property = corpus.getProperty("word")
+		def id_property = corpus.getProperty("id")
+
+		def wordCache = cm.getCorpusProxies(corpus).get(word_property);
+		def idCache = cm.getCorpusProxies(corpus).get(id_property);
+
+		def positions = new int[corpus.getSize()];
+		for(int i = 0 ; i< corpus.getSize() ; i++)
+			positions[i] = i;
+		def word_values = wordCache.getData(positions)
+		def id_values = idCache.getData(positions)
+		ArrayList<String[]> words = new ArrayList<String[]>(corpus.getSize());
+		for(int p : positions)
+		{
+			if(id_values[p].startsWith("w"))
+			{
+				words.add(new String[2])
+				words[p][0] = id_values[p]
+				words[p][1] = word_values[p]
+			}
+		}
+
+		return words;
+	}
+
+	public static void main(String[] args)
+	{
+		def words = [
+			["w203_1", "Dominedeu"],
+			["w203_2", "devemps"],
+			["w203_3", "lauder"],
+			["w203_4", "et"],
+			["w203_5", "a"],
+			["w203_6", "sus"],
+			["w203_7", "sancz"],
+			["w203_8", "honor"],
+			["w203_9", "porter"],
+			["w203_10", "»"],
+			["w203_10.2", ")"],
+			["w203_10.3", '.'],
+			["w203_10.5", '"'],
+			["w203_10.7", '"'],
+			["w203_11", "in"],
+			["w203_12", "su'"],
+			["w203_13", "amor"],
+			["w203_14", "cantomps"],
+			["w203_15", "del"],
+			["w203_16", "·sanz"],
+			["w203_17", "quae"],
+			["w203_18", "por"],
+			["w203_19", "lui"],
+			["w203_20", "augrent"],
+			["w203_21", "granz"],
+			["w203_22", "aanz"],
+			["w203_23", "."],
+		];
+		File tigerXml = new File(args[0]);
+		def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+		def THEDOM = builder.parse(tigerXml).documentElement
+
+		println THEDOM.getClass()
+
+		def NEWDOM = new PunctInject().process(THEDOM, words);
+
+		println NEWDOM
+
+		// File outfile = new File("outfile.xml")
+		// println outfile
+		//outfile.withWriter("iso-8859-1"){writer ->
+		//writer.write(NEWDOM.toString())
+		// }
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/MatchInject.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/MatchInject.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/MatchInject.groovy	(revision 321)
@@ -0,0 +1,86 @@
+#! /usr/bin/groovy
+package org.txm.export.ts;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.XMLReaderFactory;
+import groovy.util.slurpersupport.NodeChild
+
+/*
+ * The script takes the <matches/> elements from file "Tiger_match.xml"
+ * and inserts them at the end of the <s/> element bearing the same ID.
+ * Inputs: three file names 
+ * - TsInputName --- the TS file WITHOUT matches
+ * - MatchInputName --- the TS file containing only matches.
+ * - OutputFileName ---required output file.
+ * To pass these arguments from within an application, call script() directly.
+ */
+
+// Filename variables
+// def TsInputName = '/home/tomr/Documents/Work/lyon12/srcmf/groovy/MatchInject/ts_input.xml'
+// def MatchInputName = '/home/tomr/Documents/Work/lyon12/srcmf/groovy/MatchInject/match_input.xml'
+// def OutputFileName = '/home/tomr/Documents/Work/lyon12/srcmf/groovy/MatchInject/test.xml'
+
+// Main code: checks for correct number of arguments if run from cmd line.
+if (args && args.size() == 3) {
+    script(args[0], args[1], args[2])
+} else {
+    println '''Incorrect number of arguments: three strings required.
+
+USAGE:
+******
+groovy MatchInject.groovy TsInput.xml MatchInput.xml OutputFile.xml'''}
+
+def script(String tsInputName, String matchInputName, String outputFileName) {
+	script(new File(tsInputName), new File(matchInputName, new File(outputFileName)));
+}
+
+// The script.
+def script(File tsInputFile, File matchInputFile, File outputFile) {
+	//println "loading TsInput..."
+    def TsInput = new XmlSlurper().parse(tsInputFile)
+	//println "loading matchInputFile..."
+	System.setProperty("org.xml.sax.driver", "com.sun.org.apache.xerces.internal.parsers.SAXParser");
+	def xmlReader = XMLReaderFactory.createXMLReader();
+	xmlReader.setFeature('http://xml.org/sax/features/namespaces', false)
+    XmlSlurper mslurper = new XmlSlurper(xmlReader);
+	def MatchInput = mslurper.parse(matchInputFile)
+	//println "building OutputFile... size="+matchInputFile.length()
+	
+	def inputSentences = TsInput.'**'.findAll {it.name() == 's'};
+	def matcheSentences = MatchInput.'**'.findAll { it.name() == 's' }
+//	println "MATCHES"
+//	for(NodeChild match : matcheSentences) {
+//		//println match.getClass()
+//		match.namespacePrefix = ""
+//		match.namespaceMap = [:]
+//		//println match
+//	}
+	int count = 0;
+	//println("nb of input sentences: "+inputSentences.size());
+	
+    def markup = {
+		mkp.xmlDeclaration()
+		//mkp.declareNamespace("svg":"xmlns=\"http://www.w3.org/2000/svg\"")
+		
+        corpus(id:'TSOut') {
+            body {
+                inputSentences.each { sPath ->
+					//if (count++%200 == 0) println((100*count/inputSentences.size()))
+                    s(id:"${sPath.'@id'}") { 
+                        mkp.yield(sPath.graph)
+                        def sMatches = matcheSentences.find {
+                            it.'@id'.toString() == sPath.'@id'.toString()
+                        }
+						
+                        if (sMatches) {
+                            mkp.yield(sMatches.matches)
+                        }
+						//matcheSentences.removeAll(sMatches)
+                    }
+                }
+            }
+        }
+    }
+    def processor = new groovy.xml.StreamingMarkupBuilder().bind(markup)
+    outputFile.withWriter { it << groovy.xml.XmlUtil.serialize(processor)}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains scripts to build the TIGERSearch concordances</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ts/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ConcordancesExport.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ConcordancesExport.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ConcordancesExport.groovy	(revision 321)
@@ -0,0 +1,161 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.export
+
+// import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.concordances.*
+import org.txm.functions.concordances.comparators.*
+import org.txm.functions.ReferencePattern
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+def manualInit = false
+if (!org.txm.Toolbox.isInitialized())
+{
+	manualInit = true
+	org.txm.Toolbox.initialize(new File("~/TXM/install.prefs"))
+}
+
+// get the BFM corpus
+def discours = CorpusManager.getCorpusManager().getCorpus("GRAAL")
+
+// get some properties
+def pos = discours.getProperty("fropos")
+println "pos: "+pos
+def pb = discours.getProperty("pb")
+println "pb: "+pb
+def id = discours.getProperty("id")
+println "id: "+id
+def word = discours.getProperty("word")
+println "word: "+word
+def text = discours.getStructuralUnit("text")
+println "text: "+text
+def text_id = text.getProperty("id")
+println "text_id: "+text_id
+
+// create a query. Edit the query here
+def querys = javax.swing.JOptionPane.showInputDialog(null, "query ")
+if (querys == null)
+	return
+def query = new Query(Query.fixQuery(querys))
+
+println "Compute concordance "
+def start = System.currentTimeMillis()
+
+// define the references pattern for each concordance line
+def referencePattern = new ReferencePattern()
+referencePattern.addProperty(text_id)
+referencePattern.addProperty(pb)
+referencePattern.addProperty(pos)
+referencePattern.addProperty(id)
+
+def sortReferencePattern = new ReferencePattern()
+sortReferencePattern.addProperty(text_id)
+sortReferencePattern.addProperty(pb)
+sortReferencePattern.addProperty(pos)
+sortReferencePattern.addProperty(id)
+
+// compute the concordance with contexts of 15 words on each side of the keyword
+def concordance = new Concordance(discours, query, word, [word, pos], referencePattern, sortReferencePattern, 10, 10)
+
+// get a builtin sort function
+def comparator = new LexicographicKeywordComparator()
+comparator.initialize(discours)
+
+// sort the concordance
+//concordance.sort(comparator)
+
+println "done "+(System.currentTimeMillis()-start)
+
+def writer = new FileWriter("ExportConcordanceBFM3TT.csv")
+
+println "write lines"
+start = System.currentTimeMillis()
+
+// define which occurrence properties will be displayed
+concordance.setViewProperties([word])
+
+for (int i = 0 ; i < concordance.getNLines() ; i += 1000)
+{
+	List<Line> lines
+	if (i+1000 > concordance.getNLines())
+		lines = concordance.getLines(i, concordance.getNLines()-1)
+	else
+		lines = concordance.getLines(i, i+1000)
+		
+	// println "lines: from "+i
+	for (Line l : lines)
+	{
+		/*
+		 A : référence (nom de texte + page)
+		 B : contexte gauche (10 mots)
+		 C : pivot (cf. la remarque sur "ce que")
+		 D : pos du pivot
+		 E : colonne vide pour les commentaires
+		 F : contexte droit (10 mots)
+		 G : identifiant de l'occurrence (pour injecter les corrections après)
+		 */
+		
+		String ntext = l.getViewRef().getValue(text_id)
+		String page = l.getViewRef().getValue(pb)
+		String lcontext = l.leftContextToString()
+		String pivot = l.keywordToString()
+		String rien = ""
+		String rcontext = l.rightContextToString()
+		String ids = l.getViewRef().getValue(id)
+		
+		String poss = ""
+		for(String s : l.getKeywordsViewProperties().get(pos))
+		{
+			poss += s+" "
+		}
+		
+		//println l.getKeywordsViewProperties().get(pos)
+		writer.write(ntext+"_"+page+"\t"+lcontext+"\t"+pivot+"\t"+poss+"\t"+rien+"\t"+rcontext+"\t"+ids+"\n")
+	}
+	writer.flush()
+}
+writer.close()
+
+println "done "+(System.currentTimeMillis()-start)+" results"
+
+println("Concordance "+query.getQueryString()+" saved in file "+new File("ExportConcordanceBFM3TT.csv").getAbsolutePath())
+
+if(manualInit)
+	org.txm.Toolbox.shutdown()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/CooccurrencesFilter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/CooccurrencesFilter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/CooccurrencesFilter.groovy	(revision 321)
@@ -0,0 +1,57 @@
+package org.txm.export
+
+// we import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.cooccurrences.*
+
+//////////// PARAMETERS ////////////
+def corpus = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+
+def word = corpus.getProperty("word")
+def pos = corpus.getProperty("pos")
+def viewprops = [word, pos]
+
+def query = new Query(Query.fixQuery("je"))
+def PROPSVALUETOKEEP = "V.+"
+def PROPNUMBER = 1;
+
+def limit = corpus.getStructuralUnit("s") // contexts by <s>
+def includeXpivot = false; // used only if limit != null
+//def limit = null // contexts by word 
+def minleft = 1 // min = 1
+def maxleft = 20 // min = 1
+def minright = 1 // min = 1
+def maxright = 20 // min = 1
+
+def minf = 0
+def mincof = 0
+def minscore = 0
+
+def buildLTWithOnlyCooccurrents = false
+
+//////////// END OF PARAMETERS ////////////
+
+println "computing cooc lines..."
+long time = System.currentTimeMillis()
+//TODO temporary commented
+//def cooccurrence = new Cooccurrence(corpus, query, viewprops, limit, maxleft, minleft, minleft, maxright, minf,mincof, minscore, includeXpivot, buildLTWithOnlyCooccurrents)
+cooccurrence.process();
+println("Compute time : "+(System.currentTimeMillis()-time)+" ms")
+
+println "filtering lines..."
+def lines = cooccurrence.getLines()
+for(int i = 0 ; i < lines.size(); i++)
+{
+	def line = lines.get(i)
+	if( !line.props.get(PROPNUMBER).matches(PROPSVALUETOKEEP))
+	{
+		lines.remove(i--)
+	}	
+}
+
+println "writing result..."
+def file = new File("cooc.txt")
+cooccurrence.toTxt(file, "UTF-8")
+println "printed cooccurrents in "+file.getAbsolutePath()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/exportCA.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/exportCA.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/exportCA.groovy	(revision 321)
@@ -0,0 +1,28 @@
+package org.txm.export
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.stat.data.*;
+import org.txm.functions.ca.*
+
+String moncorpus = "DISCOURS"
+String mapartition = "texts"
+
+
+def corpus = CorpusManager.getCorpusManager().getCorpus(moncorpus) // mettre le nom du corpus en Majuscules ici
+def partition = corpus.getPartition(mapartition);
+
+for(Object o : partition.getResults())
+{
+//	if(o instanceof CA)	{		
+		def ca = o;
+		File colinfosfile = new File(ca.getName()+"_colinfosfile.txt");
+		File rowinfosfile = new File(ca.getName()+"_rowinfosfile.txt");
+	
+		
+		ca.exportColInfos(colinfosfile, "UTF-8", "\t", "")
+		ca.exportRowInfos(rowinfosfile, "UTF-8", "\t", "")
+		
+		println ca.getName()+" exported in : "+rowinfosfile.getAbsolutePath()+" and "+colinfosfile.getAbsolutePath()
+//	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ParaConc.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ParaConc.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ParaConc.groovy	(revision 321)
@@ -0,0 +1,134 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2011-09-27 16:44:37 +0200 (mar., 27 sept. 2011) $
+// $LastChangedRevision: 2008 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.export
+
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.concordances.*
+import org.txm.functions.concordances.comparators.*
+import org.txm.functions.ReferencePattern
+
+////// PARAMETERS //////
+
+def output = "nomfichier"
+
+def tgtCorpusName = "BLLATLAT"
+def srcCorpusName = "BLLATFRO"
+
+def tgtWords = "sum|ego" // use to highlight lat words
+def srcWords = "je|j'" // use to highlight fro words
+
+def tgtQuery = "<seg> []* \""+tgtWords+"\" []* </seg> :"+srcCorpusName+" \""+srcWords+"\"" // default lat query
+def srcQuery = "<seg> []* \""+srcWords+"\" []* </seg> :"+tgtCorpusName+" \""+tgtWords+"\"" // default fro query
+
+println tgtCorpusName+": "+tgtQuery
+println srcCorpusName+": "+srcQuery
+
+new File(output).withWriter("UTF-8"){writer->
+//// LAT CONCORDANCE ////
+def tgtCorpus = CorpusManager.getCorpusManager().getCorpus(tgtCorpusName)
+def text_id = tgtCorpus.getStructuralUnit("text").getProperty("id")
+def seg_id = tgtCorpus.getStructuralUnit("seg").getProperty("id")
+def word = tgtCorpus.getProperty("word")
+def referencePattern = new ReferencePattern()
+//referencePattern.addProperty(text_id)
+referencePattern.addProperty(seg_id)
+def tgtConcordance = new Concordance(tgtCorpus, new Query(tgtQuery), word, [word], referencePattern, referencePattern, 0, 0)
+
+//// LAT CONCORDANCE ////
+def srcCorpus = CorpusManager.getCorpusManager().getCorpus(srcCorpusName)
+text_id = srcCorpus.getStructuralUnit("text").getProperty("id")
+seg_id = srcCorpus.getStructuralUnit("seg").getProperty("id")
+word = srcCorpus.getProperty("word")
+referencePattern = new ReferencePattern()
+//referencePattern.addProperty(text_id)
+referencePattern.addProperty(seg_id)
+def srcConcordance = new Concordance(srcCorpus, new Query(srcQuery), word, [word], referencePattern, referencePattern, 0, 0)
+
+//// MERGE CONCORDANCE RESULT ////
+HashSet<String> allKeys = new HashSet<String>()
+HashMap<String, Line> srcLines = [:]
+HashMap<String, Line> tgtLines = [:]
+
+println "nb tgt result: "+tgtConcordance.getNLines()
+for(Line line : tgtConcordance.getLines(0, tgtConcordance.getNLines()))
+{
+	allKeys.add(line.getViewRef().toString())
+	tgtLines.put(line.getViewRef().toString(), line)
+}
+
+println "nb src result: "+srcConcordance.getNLines()
+for(Line line : srcConcordance.getLines(0, srcConcordance.getNLines()))
+{
+	allKeys.add(line.getViewRef().toString())
+	srcLines.put(line.getViewRef().toString(), line)
+}
+println "src seg: "+srcLines.keySet()
+println "tgt seg: "+tgtLines.keySet()
+println "nb segments: "+allKeys.size()
+
+writer.println "segment\tsource\ttarget"
+for(Line line : srcConcordance.getLines(0, srcConcordance.getNLines()))
+{
+	String key = line.getViewRef().toString();
+	def tgtLine = tgtLines.get(key)
+	String srcString = line.keywordToString();
+	String tgtString = "";
+	if(tgtLine != null)	{
+		tgtString = tgtLine.keywordToString();
+		tgtLines.remove(key)
+		}else
+	{
+		key = key+"*"
+	}
+	srcString = srcString.replaceAll("("+srcWords+")", "*\$1*");
+	tgtString = tgtString.replaceAll("("+tgtWords+")", "*\$1*");
+	writer.println key+"\t"+srcString+"\t"+tgtString
+}
+
+println "\n"
+for(String key : tgtLines.keySet()) // process remaining tgt lines
+{
+	def srcline = srcLines.get(key)
+	def tgtline = tgtLines.get(key)
+	String srcString = "";
+	String tgtString = tgtLines.keywordToString();
+	if(srcline == null){
+		srcString = srcline.keywordToString();
+	}else
+	{
+		key = key+"*"
+	}
+	srcString = srcString.replaceAll("("+srcWords+")", "*\$1*");
+	tgtString = tgtString.replaceAll("("+tgtWords+")", "*\$1*");
+	writer.println key+"\t"+srcString+"\t"+tgtString
+}
+}
+
+println "Done : written in "+new File(output).getAbsolutePath();
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>export scripts from TXM's format to xxxx's format</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/export/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/ExportCAH.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/ExportCAH.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/ExportCAH.groovy	(revision 321)
@@ -0,0 +1,35 @@
+package org.txm.export
+
+//import org.txm.rcpapplication.views.*
+import org.txm.functions.classification.*
+import org.txm.stat.engine.r.*
+// uncomment this
+//import org.eclipse.jface.viewers.IStructuredSelection
+
+// Select the CAH result in the CorporaView and run this script
+
+// PARAMETER: the output file
+def outfile = new File("/home/mdecorde/Bureau/CAH.txt")
+
+// get selected object from Corpora view
+def obj = null;
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {
+		def corporaview = CorporaView.getInstance()
+		def tree = corporaview.getTreeViewer()
+		// uncomment these 2 lines
+//		def sel = (IStructuredSelection) tree.getSelection()
+//		obj = sel.getFirstElement()
+		}
+	});
+println("Selected object: "+obj)
+
+//TODO temporary commented
+//if (obj != null && obj instanceof CAH) {
+	def rw = RWorkspace.getRWorkspaceInstance()
+	rw.eval("sink(file=\"" + outfile.getAbsolutePath().replace("\\", "\\\\")+"\")")
+	rw.eval("print("+obj.getSymbol()+")")
+	rw.eval("sink()")
+	println("CAH saved in file: "+outfile)
+//}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/export/MultipleConcordances.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/export/MultipleConcordances.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/export/MultipleConcordances.groovy	(revision 321)
@@ -0,0 +1,105 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2011-05-09 14:58:28 +0200 (lun., 09 mai 2011) $
+// $LastChangedRevision: 1839 $
+// $LastChangedBy: mdecorde $ 
+//
+
+package org.txm.export
+
+// import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.concordances.*
+import org.txm.functions.concordances.comparators.*
+import org.txm.functions.ReferencePattern
+
+//PARAMETRES
+def corpusname = "DISCOURS"
+def requetes = ["je", "tu", "vous"]
+// get the DISCOURS corpus
+// TODO: Auto-generated Javadoc
+
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+def discours = CorpusManager.getCorpusManager().getCorpus(corpusname)
+
+// get some properties
+def pos = discours.getProperty("pos")
+def word = discours.getProperty("word")
+def text = discours.getStructuralUnit("text")
+
+//get structure text
+def text_id = text.getProperty("id")
+// and define the references pattern for each concordance line
+def referencePattern = new ReferencePattern().addProperty(text_id)
+
+// get a builtin sort function
+def comparator = new LexicographicLeftContextComparator()
+comparator.initialize(discours)
+
+def noconc = 0
+for(String querystring : requetes)
+{
+	println "Computing concordance of $querystring ..."
+	// create a query. Here the concordance keyword will be the French word "Je" (the "I" pronoun in English)
+	def query = new Query(Query.fixQuery(querystring))
+	
+	// compute the concordance with contexts of 15 words on each side of the keyword
+	def concordance = new Concordance(discours, query, word, [word, pos], referencePattern, 15, 15)
+		
+	// sort the concordance
+	//concordance.sort(comparator)
+	
+	// define which occurrence properties will be displayed
+	concordance.setViewProperties([word])
+	
+	// write all the concordance in a text file
+	def file = new File("conc"+(noconc++)+".txt")
+	concordance.toTxt(file, "UTF-8")
+	println("Concordance $querystring written in file: "+file.getAbsolutePath())
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/hyperbaseLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/hyperbaseLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/hyperbaseLoader.groovy	(revision 321)
@@ -0,0 +1,160 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-06-03 15:04:53 +0200 (Wed, 03 Jun 2015) $
+// $LastChangedRevision: 2984 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.hyperbase;
+
+
+import org.txm.importer.hyperbase.importer;
+import org.txm.importer.hyperbase.compiler;
+import org.txm.importer.xml.pager;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/TESTS/hyperbase/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+
+if (!(new importer().run(srcDir, binDir, txmDir, encoding, basename, lang))) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if ((new Annotate().run(binDir, txmDir, model+".par"))) {
+	annotationSuccess = true
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+c.setAnnotationSuccess(annotationSuccess);
+if (debug) c.setDebug();
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(20, "EDITION")
+	
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+	def second = 0
+
+	println "Paginating texts: "
+
+	for (File srcfile : filelist) {
+		print "."
+		String txtname = srcfile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);
+
+		def ed = new pager(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, "pb");
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/compiler.groovy	(revision 321)
@@ -0,0 +1,316 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.hyperbase;
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler 
+{
+	
+	/** The debug. */
+	private boolean debug= false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash = new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The text attributes. */
+	String[] textAttributes = null;
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+	
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.textAttributes = textAttributes;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/** The annotation success. */
+	boolean annotationSuccess = false;
+	
+	/**
+	 * Sets the annotation success.
+	 *
+	 * @param value the value
+	 * @return the java.lang. object
+	 */
+	public setAnnotationSuccess(boolean value)
+	{
+		this.annotationSuccess = value
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param f the f
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f){
+		try {
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			
+			return false;
+		}
+	}
+	
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String corpusname) 
+	{
+		String rootDir = binDir.getAbsolutePath();
+		
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!binDir.exists())
+		{
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+		File wtcFile = new File(binDir,"wtc/"+corpusname+".wtc");
+		new File(binDir,"wtc").deleteDir();
+		new File(binDir,"wtc").mkdir();
+		new File(binDir,"data").deleteDir();
+		new File(binDir,"data").mkdir();
+		new File(binDir,"registry").mkdir();
+		
+		//start corpus
+		if (createOutput(wtcFile)) {
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+		
+		String textid="";
+		int counttext =0;
+		List<File> files = txmDir.listFiles();
+		Collections.sort(files);
+		//1- Transform into WTC file
+		XMLTXM2WTC wtcbuilder = null;
+		println("Compiling "+files.size()+" files")
+		for (File f : files) {
+			print "."
+			counttext++;
+			if (!f.exists()) {
+				println("file "+f+ " does not exists")	
+			}
+			else {	
+				wtcbuilder = new XMLTXM2WTC(f.toURI().toURL());
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				wtcbuilder.setTextInfo(txtname, corpusname, "project");
+
+				wtcbuilder.setBalisesToKeep(["text","p","s"]);
+				wtcbuilder.setSendToPAttributes(["s":["n"]]);
+				wtcbuilder.setLang(lang);
+				if (!wtcbuilder.transformFile(wtcFile)) {
+					println("Failed to compile "+f)
+				}
+			}
+		}
+		println ""
+		
+		//end corpus
+		if (createOutput(wtcFile)) {
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+		
+		if (wtcbuilder == null) return false;
+		
+		//2- Import into CWB
+		def outDir =rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		
+		List<String> pAttributesList = wtcbuilder.getpAttributs();
+		List<String> sAttributesList = wtcbuilder.getsAttributs();
+		println "word properties : "+pAttributesList
+		println "structures : "+sAttributesList
+		String[] pAttributes = pAttributesList.toArray(new String[pAttributesList.size()])
+		String[] sAttributes = sAttributesList.toArray(new String[sAttributesList.size()])
+		
+		try {
+			String regPath = outDir + "/registry/"+corpusname.toLowerCase();
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), outDir + "/data/$corpusname", outDir + "/wtc/"+corpusname+".wtc", regPath,pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname, outDir + "/registry");
+			
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		
+		return true;
+	}
+	
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Hyperbase import module</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperbase/importer.groovy	(revision 321)
@@ -0,0 +1,187 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-06-03 15:04:53 +0200 (Wed, 03 Jun 2015) $
+// $LastChangedRevision: 2984 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.hyperbase
+
+//import filters.TabulatedToXml.*;
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+//import org.txm.utils.treetagger.TreeTagger;
+import javax.xml.stream.*;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.net.URL;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.FusionHeader.*;
+import filters.TagSentences.*;
+
+import org.txm.*;
+import org.txm.utils.*;
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param encoding the encoding
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public static boolean run(File srcDir, File binDir, File txmDir, String encoding, String basename, String lang)
+	{
+		String rootDir = srcDir.getAbsolutePath()+"/"
+		//cleaning
+
+		new File(binDir,"ptokenized").deleteDir();
+		new File(binDir,"ptokenized").mkdir();
+		new File(binDir,"tokenized").deleteDir();
+		new File(binDir,"tokenized").mkdir();
+		new File(binDir,"split").deleteDir();
+		new File(binDir,"split").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+
+		//build xml files
+		if (srcDir.listFiles() == null || srcDir.listFiles().size() == 0 ) {
+			println "No corpus file in: "+srcDir;
+			return false;
+		}
+		File corpusfile = null;
+		for (File f : srcDir.listFiles()) {
+			if (!f.isHidden() && f.canRead() && f.isFile() && f.getName() != "import.xml") {
+				corpusfile = f;
+				break;
+			}
+		}
+
+		System.out.println("Corpus file: "+corpusfile);
+		if (!new Hyperbase2Xml().run(corpusfile, new File(binDir, "split"), encoding)) {
+			println("Failed to split the corpus file into text: "+corpusfile);
+			return false;
+		}
+
+		//TOKENIZE ALL FILES
+		List<File> srcfiles = new File(binDir, "split").listFiles();
+		println("Tokenizing "+srcfiles.size()+" files");
+		for (File pfile : srcfiles) {
+			print "."
+			String filename = pfile.getName().substring(0, pfile.getName().length()-4)
+			File tfile = new File(binDir, "tokenized/"+filename+".xml");
+
+			SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(pfile, tfile, lang);
+			if (!tokenizer.process()) {
+				println("Failed to tokenize file: "+pfile);
+				tfile.delete();
+			}
+		}
+		println ""
+
+		//Tag sentences
+		List<File> stokenfiles = new File(binDir,"tokenized").listFiles()
+		new File(binDir,"stokenized").mkdir();
+		println("Tag sentences of "+stokenfiles.size()+" files")
+		for (File f : stokenfiles) {
+			print "."
+			try {
+				Sequence S = new Sequence();
+				Filter F1 = new CutHeader();
+				Filter F7 = new TagSentences();
+				Filter F11 = new FusionHeader();
+				S.add(F1);
+				S.add(F7);
+				S.add(F11);
+				File infile = new File(binDir, "tokenized/"+f.getName());
+				File xmlfile = new File(binDir, "stokenized/"+f.getName());
+				File headerfile = new File(binDir, "tokenized/"+f.getName()+"header.xml");
+
+				S.SetInFileAndOutFile(infile.getPath(), xmlfile.getPath());
+				S.setEncodages("UTF-8","UTF-8");
+				Object[] arguments1 = [headerfile.getAbsolutePath()];
+				F1.SetUsedParam(arguments1);
+				Object[] arguments2 = [headerfile.getAbsolutePath(),F1];
+				F11.SetUsedParam(arguments2);
+				S.proceed();
+				S.clean();
+				headerfile.delete();//remove the prepared file to clean
+			} catch (Exception e) {println "Failed to sentence file "+f;println e;}
+		}
+		println ""
+
+		//TRANSFORM INTO XML-TEI-TXM
+		List<File> tokenfiles = new File(binDir, "stokenized").listFiles()
+		println("Building xml-txm ("+tokenfiles.size()+" files)")
+		for (File f : tokenfiles) {
+			print "."
+			File file = f;
+			String txmfile = f.getName();
+
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			//lance le traitement
+			def builder = new Xml2Ana(file);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			//builder.setAddTEIHeader();
+			if (!builder.process(new File(txmDir,txmfile))) {
+				println("Failed to process "+f);
+				new File(txmDir, txmfile).delete();
+			}
+		}
+		println ""
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/package.html	(revision 321)
@@ -0,0 +1,4 @@
+<html>
+<body>Deprecated: Graal import module
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalXpath.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalXpath.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalXpath.groovy	(revision 321)
@@ -0,0 +1,120 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.graal
+
+import org.txm.importer.*;
+import java.io.IOException;
+import org.w3c.dom.*;
+import org.xml.sax.SAXException;
+import javax.xml.parsers.*;
+import javax.xml.xpath.*;
+import javax.xml.stream.*;
+import java.net.URL;
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class GraalXpath.
+ *
+ * @authors ayepdieu, mdecorde, vchabanis
+ * 
+ * return the id of a bfm tag <milestone/>
+ */
+public class GraalXpath
+{
+	
+	/**
+	 * Gets the xpath response.
+	 *
+	 * @param fileName the xmlfile
+	 * @param N the id of the milestone
+	 * @return the id of a bfm tag <milestone/>
+	 */
+	public static String getXpathResponse(def fileName, String N) throws ParserConfigurationException, SAXException, IOException, XPathExpressionException 
+	{	
+		if (System.getProperty("os.name").contains("Windows"))
+		{
+			
+			XMLInputFactory factory = XMLInputFactory.newInstance();
+			FileInputStream inputStream = new FileInputStream(new File(fileName));
+			XMLStreamReader parser = factory.createXMLStreamReader(inputStream);
+
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				if( event== XMLStreamConstants.START_ELEMENT)
+				{
+					if(parser.getLocalName() == "milestone")
+					{
+						if(parser.getAttributeValue(null , "n")== N)
+						{
+							if(parser.getAttributeValue(null , "unit")== "column")
+							{
+								//TODO : changer l'acces par une clef String au lieu d'un index, prob avec le namespace
+								return parser.getAttributeValue(1);
+							}
+						}		
+					}
+				}
+			}
+			println( "FAILED TO FIND MILESTONE WITH N = "+N);
+			return null;
+		}
+		else
+		{
+			def testG = new XPathResult(new File(fileName));
+			String npage =testG.getXpathResponse("//tei:milestone[@unit='column' and @n='"+N+"']","xml:id")
+			return npage;
+			/*String Query = "//tei:milestone[@unit='column' and @n='"+N+"']"
+			DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		    domFactory.setNamespaceAware(true); // never forget this!
+		    DocumentBuilder builder = domFactory.newDocumentBuilder();
+		    Document doc = builder.parse(fileName);
+		   
+		    XPathFactory factory = XPathFactory.newInstance();
+		    XPath xpath = factory.newXPath();
+		    xpath.setNamespaceContext(new PersonalNamespaceContext());
+		    XPathExpression expr = xpath.compile(Query);
+
+		    Object result = expr.evaluate(doc, XPathConstants.NODESET); // don't work on windows
+		    NodeList nodes = (NodeList) result;
+		    return nodes.item(0).getAttributes().getNamedItem("xml:id").getNodeValue();
+		    */
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		def testG = new GraalXpath();
+		String npage =testG.getXpathResponse("C:\\Documents and Settings\\H\\Bureau\\resourcesTXM\\corpora\\data\\qgraal\\qgraal_cm_2009_03.xml","2")
+		println "found npage : "+npage;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/ImportInCWB.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/ImportInCWB.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/ImportInCWB.groovy	(revision 321)
@@ -0,0 +1,176 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.graal;
+
+import java.net.URL;
+import java.lang.Boolean
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ImportInCWB.
+ *
+ * @author sheiden,mdecorde
+ * 
+ * import graal wtc fiel into cwb
+ */
+public class ImportInCWB{
+
+ 	/** The cwb loc. */
+	 def cwbLoc;//chemin vers executable cqp
+	
+	/** The root dir. */
+	def rootDir;//chemin du dossier qui contien src/ et xsl/
+	
+	/** The in dir. */
+	def inDir;
+	 	
+	//dossier qui contiendra les gsp et vues dipl, cour, facs, trad et wtc
+	
+	/** The out dir. */
+	def outDir;
+	//dossiers qui contient le registre cqp et le dossier data (qui contient les corpus)
+	
+	/** The out dir txm. */
+	def outDirTxm;
+	
+ 	/** The in file. */
+	 def inFile;
+	
+	/** The in file2. */
+	def inFile2;
+	
+	/** The out file. */
+	def outFile;
+	
+	/** The out file2. */
+	def outFile2;
+	
+	/**
+	 * same args as GraalInportCqp.
+	 *
+	 * @param rootDir the root dir
+	 * @param cwbLoc the cwb loc
+	 * @param inFile the in file
+	 * @param inFile2 the in file2
+	 * @param outFile the out file
+	 * @param outFile2 the out file2
+	 */
+	public ImportInCWB(	rootDir,
+						cwbLoc,
+						inFile,
+						inFile2,
+						outFile,
+						outFile2)
+	{
+		this.rootDir = rootDir;
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		else
+			this.cwbLoc = cwbLoc;
+		this.inFile = inFile;
+		this.inFile2 = inFile2;
+		this.outFile = outFile;
+		this.outFile2 = outFile2;
+		this.inDir = this.rootDir+"src" ;
+		this.outDir = this.rootDir ;//outdir of the .gsp
+		this.outDirTxm = this.rootDir ;//outdir for data files & registry
+		this.createDirs();
+	}
+	
+	/**
+	 * create dirs to prevent errors.
+	 */
+	private void createDirs()
+	{
+		new File(outDir).mkdir();
+		new File(outDir+"/wtc").mkdir();
+		new File(outDirTxm).mkdir();
+		new File(outDirTxm+"registry").mkdir();
+		new File(outDirTxm+"data").mkdir();
+	}
+	
+	/**
+	 * call :
+	 * GraalImportCqp
+	 * cwb-encode
+	 * cwb-makeall.
+	 */
+	public void start()
+	{//transfome en wtc et compile les index
+		System.out.println "file:///"+ inDir + "/" + inFile+ ".xml";
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		
+		GraalImportCqp traitTxt = new GraalImportCqp(new URL("file:///"+ inDir + "/" + inFile+ ".xml"));
+		traitTxt.transfomFileWtc(outDir+"/wtc/", outFile + ".wtc");
+		
+		traitTxt = new GraalImportCqp(new URL("file:///"+ inDir + "/" + inFile2+ ".xml"));
+		traitTxt.transfomFileWtcBrut(outDir+"/wtc/", outFile2 + ".wtc");
+		
+		System.out.println("--Building index files ...") 
+		String[] pAttributecm = ["pos", "q", "supplied", "col", "line", "id", "dipl", "facs"];
+		String[] sAttributecm = ["p:0+n", "q:1+n", "s:0+n+id"];
+		
+		String[] pAttributes = ["pos", "id", "col"];
+		String[] sAttributes = ["p:0+n", "s:0+n"];
+		try {
+			if (System.getProperty("os.name").contains("Windows"))
+			{
+				cwbEn.run(cwbLoc + "cwb-encode.exe", outDirTxm + "data/"+outFile, outDir +"/wtc/"+outFile+".wtc", outDirTxm + "registry/"+outFile,pAttributecm, sAttributecm);
+				cwbMa.run(cwbLoc + "cwb-makeall.exe", outFile.toUpperCase(), outDirTxm + "registry");
+				
+				cwbEn.run(cwbLoc + "cwb-encode.exe", outDirTxm + "data/"+outFile2, outDir + "/wtc/"+outFile2+".wtc", outDirTxm + "registry/"+outFile2, pAttributes, sAttributes);
+				cwbMa.run(cwbLoc + "cwb-makeall.exe", outFile2.toUpperCase(), outDirTxm + "registry");
+			}
+			else
+			{
+				cwbEn.run(cwbLoc + "cwb-encode", outDirTxm + "data/"+outFile, outDir + "/wtc/"+outFile+".wtc", outDirTxm + "registry/"+outFile,pAttributecm, sAttributecm);
+				cwbMa.run(cwbLoc + "cwb-makeall", outFile.toUpperCase(), outDirTxm + "registry");
+				
+				cwbEn.run(cwbLoc + "cwb-encode", outDirTxm + "data/"+outFile2, outDir +"/wtc/"+ outFile2+".wtc", outDirTxm + "registry/"+outFile2, pAttributes, sAttributes);
+				cwbMa.run(cwbLoc + "cwb-makeall", outFile2.toUpperCase(), outDirTxm + "registry");
+			}
+		} catch (Exception ex) {System.out.println(ex);}
+				System.out.println("Done.") 
+	}
+	
+	/**
+	 * test.
+	 */
+	public void run() 
+	{
+		System.out.println "Starting script : importWTC.groovy ...";
+		System.setProperty("javax.xml.transform.TransformerFactory",  
+				 			"net.sf.saxon.TransformerFactoryImpl");  
+		this.importWTC();
+		System.out.println "GraalImport.groovy done."
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/qgraalLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/qgraalLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/qgraalLoader.groovy	(revision 321)
@@ -0,0 +1,116 @@
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-08-29 14:23:55 +0200 (jeu., 29 août 2013) $
+// $LastChangedRevision: 2519 $
+// $LastChangedBy: nilskredens $ 
+//
+package org.txm.importer.graal
+
+//
+//import org.txm.importer.graal.BuildGraalGSP;
+//import org.txm.importer.graal.ImportInCWB;
+//
+//import org.txm.importer.cwb.CwbEncode
+//import org.txm.importer.cwb.CwbMakeAll
+//import org.txm.importer.cwb.PatchCwbRegistry;
+//import org.txm.objects.*;
+//import org.txm.utils.*;
+//String userDir = System.getProperty("user.home");
+//
+//String rootDir;
+//try{rootDir = rootDirBinding;}//srcDir the selected directoy in TXM RCP via the Binding 
+//catch(Exception)
+//{	println "DEV MODE";//exception means we debug
+//	if(!org.txm.Toolbox.isInitialized())
+//	{
+//		rootDir = userDir+"/xml/qgraal/";
+//		org.txm.Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+//		org.txm.Toolbox.setParam(org.txm.Toolbox.INSTALL_DIR,new File(userDir,"TXM"));
+//	}
+//}
+//
+//String pathToCwbBinaries = org.txm.Toolbox.getParam(org.txm.Toolbox.INSTALL_DIR)+"/cwb/bin/";//chemin vers executable cqp de l'installation
+//
+//	//2- create GSP & import to WTC
+//	println "EDITION"
+//	def edition = new BuildGraalGSP(rootDir,//rootdir
+//	pathToCwbBinaries// cwb utils dir
+//	,"qgraal_cm_2009-07-d",	"qgraal_frmod_2009-07-w",//infile1 , infile2
+//	"qgraalcm","qgraalfrmod"); //outfile1 & outfile2
+//	edition.start();
+//	
+//	println "COMPILATION"
+//	def compiler = new ImportInCWB(rootDir,//rootdir
+//	pathToCwbBinaries// cwb utils dir
+//	,"qgraal_cm_2009-07-d",	"qgraal_frmod_2009-07-w",//infile1 , infile2
+//	"qgraalcm","qgraalfrmod"); //outfile1 & outfile2
+//	compiler.start();
+//
+//	//THIS IS AN OLD SCRIPT WE HAVE TO MOVE SOMES FILES
+//	new File(rootDir,"txm").deleteDir()
+//	new File(rootDir,"txm").mkdir()
+//	new File(rootDir,"HTML").deleteDir()
+//	new File(rootDir,"HTML").mkdir()
+//	//txm : copy src
+//	FileCopy.copy(new File(rootDir+"/src","qgraal_cm_2009-07-d.xml"),new File(rootDir+"/txm","qgraal.xml"))
+//	
+//	//HTML : get from/cour
+//	for(File f : new File(rootDir+"/imported/form/cour").listFiles())
+//	{
+//		String name = f.getName();
+//		String pageid= name.substring(15,name.length()-4);
+//		String newfilename = "qgraal_"+pageid+".html";
+//		println newfilename
+//		FileCopy.copy(f,new File(rootDir+"/HTML/multi",newfilename));
+//	}
+//	
+//	//data
+//	new File(rootDir,"data/qgraalcm").renameTo(new File(rootDir,"dataCM"));
+//	new File(rootDir,"data").deleteDir();
+//	new File(rootDir,"dataCM").renameTo(new File(rootDir,"data"));
+//	
+//	//registry
+//	File registryfiletopatch = new File(rootDir+"/registry",qgraalcm);
+//	File cwbindex = new File(rootDir,"data");
+//	try {
+//		PatchCwbRegistry.patch(registryfiletopatch, cwbindex);
+//	} catch (IOException e) {org.txm.utils.logger.Log.printStackTrace(e);}
+//	
+//	// copy reg
+//	try {
+//		FileCopy.copy(registryfiletopatch, new File(registrypath,"qgraalcm"));
+//	} catch (IOException e) {org.txm.utils.logger.Log.printStackTrace(e);}
+//	
+//	
+//	Workspace w = org.txm.Toolbox.workspace;
+//	Project p = w.getProject("default")
+//	p.removeBase("qgraal")
+//	Base b = p.addBase("qgraal");
+//	b.addDirectory(new File(rootDir,"txm"));
+//
+//	
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/BuildGraalGSP.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/BuildGraalGSP.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/BuildGraalGSP.groovy	(revision 321)
@@ -0,0 +1,205 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.graal;
+
+import org.txm.importer.*
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class BuildGraalGSP.
+ *
+ * @author sheiden
+ */
+
+public class BuildGraalGSP{
+ 	
+	 /** The cwb loc. */
+	 String cwbLoc;
+	
+	/** The root dir. */
+	String rootDir;
+	
+	/** The in dir. */
+	String inDir;
+	
+	/** The in dir xsl. */
+	String inDirXsl;
+	 	
+	//dossier qui contiendra les gsp et vues dipl, cour, facs, trad et wtc
+	
+	/** The out dir. */
+	String outDir;
+	//dossiers qui contient le registre cqp et le dossier data (qui contient les corpus)
+	
+	/** The out dir txm. */
+	String outDirTxm;
+	//dossier qui contient les xml sources découpés
+	
+	/** The out dir cut. */
+	String outDirCut;
+	
+	/** The in file. */
+	String inFile;
+	
+	/** The in file2. */
+	String inFile2;
+	
+	/** The out file. */
+	String outFile;
+	
+	/** The out file2. */
+	String outFile2;
+	
+	/**
+	 * Instantiates a new builds the graal gsp.
+	 *
+	 * @param rootDir the directory which contains sources and which will contains result files
+	 * @param cwbLoc the location of cwb-encode and cwb-makeall
+	 * @param inFile the path to the graalcm file
+	 * @param inFile2 the path to the graalfrmod file
+	 * @param outFile the prefix of gsp outfiles
+	 * @param outFile2 the prefix of gsp outfiles
+	 */
+	public BuildGraalGSP(String	rootDir, String cwbLoc, String inFile,
+	String inFile2,	String outFile,	String outFile2)
+	{
+		this.rootDir = rootDir;
+		this.cwbLoc = cwbLoc;
+		this.inFile = inFile;
+		this.inFile2 = inFile2;
+		this.outFile = outFile;
+		this.outFile2 = outFile2;
+		this.inDir = this.rootDir;
+		this.inDirXsl = this.rootDir + "/src/xsl";
+		this.outDir = this.rootDir + "/imported/form/";//outdir of the .gsp
+		this.outDirTxm = this.rootDir + "/imported/corpora/";//outdir for data files & registry
+		this.outDirCut = this.rootDir + "/imported/cut/";
+		this.createDirs();
+	}
+	
+	/**
+	 * create outputs dir to prevent from "dir not found" error.
+	 */
+	private void createDirs()
+	{
+		new File(outDir).mkdir();
+		new File(outDir+"/dipl").mkdir();
+		new File(outDir+"/trad").mkdir();
+		new File(outDir+"/facs").mkdir();
+		new File(outDir+"/cour").mkdir();
+		new File(outDirTxm).mkdir();
+		new File(outDirCut).mkdir();
+	}
+
+	/**
+	 * process !!!.
+	 */
+	public boolean start()
+	{
+		if(!ValidateXml.test(new File(inDir + "src/"+inFile+".xml")))
+		{
+			println "Invalid source: "+inDir + "src/"+inFile+".xml";
+			return false;
+		}
+		
+		if(!ValidateXml.test(new File(inDir + "src/"+inFile2+".xml")))
+		{
+			println "Invalid source: "+inDir + "src/"+inFile2+".xml";
+			return false;
+		}
+		
+		def timportWTCestG
+		def nPage;
+		def temp;
+		def BBM;
+		def MAN;
+		def MAF;
+		def MAD;
+		def MTR;
+		//Ouverture des Xsl
+		//Decoupe et mise en forme des page
+		println "start"
+		for (int index = 1; index < 268; index++) 
+		{
+			println "loop" + index
+			
+			//URL url = new URL("file:///" + inDirXsl + "/breakByMilestone.xsl");
+			BBM = new ApplyXsl( inDirXsl + "/breakByMilestone.xsl");
+			//println "xsl applied"
+			MAN = new GraalApplyXsl(new URL("file:///" + inDirXsl + "/qgraal_cour.xsl"));
+			//println "xsl applied 2"
+			MAF = new GraalApplyXsl(new URL("file:///" + inDirXsl + "/qgraal_facs.xsl"));
+			//println "xsl applied 3 "
+			MAD = new GraalApplyXsl(new URL("file:///" + inDirXsl + "/qgraal_dipl.xsl"));
+			//println "xsl applied 4"
+			MTR = new GraalApplyXsl(new URL("file:///" + inDirXsl + "/qgraal_frmod.xsl"));
+			//println "xsl applied 5"
+			def testG = new GraalXpath();
+			//println "xpath done"
+			
+			nPage = testG.getXpathResponse(inDir + "src/"+inFile+ ".xml",  index+"").substring(4);//recuperation du numero de page
+			//nPage=index
+			println "npage = " + nPage
+			
+			URL url = new URL("file:///" + inDir + inFile+ ".xml");
+			//System.out.println("--Converting " + url.getFile() + " to web pages.");
+			
+			//BBM.setParam("pbval1", index);
+			//println "params set 1 : "
+			//BBM.setParams("pbval2", index + 1);
+			String[] params = ["pbval1", index,"pbval2", index+1]
+			//println "params set 2"
+			BBM.process(new URL("file:///" + inDir + "/src/" + inFile+ ".xml").getFile(), outDirCut+"/"+ inFile + "_" + nPage + ".xml",params);//decoupe ancienne version
+			println "BBM Done : "+inFile + "_" + nPage + ".xml"
+			//System.out.println "L'index est : " + index + " npage " + nPage;
+				System.out.println "----Pages gsp";
+				if (index<260) {
+				BBM.process(new URL("file:///" + inDir + "/src/" + inFile2+ ".xml").getFile(), outDirCut+"/"+ inFile2 + "_" + nPage + ".xml",params);//decoupe nouvelle version
+				}
+				System.out.println "----Break By MileStone.";
+				if(index<10)
+				{					
+				MAF.ApplyXsl(new URL("file:///" + outDirCut + "/" + inFile + "_" + nPage+ ".xml"), outDir + "facs", "_" + outFile + "_facs_" + nPage + ".gsp");//mise en page facsimilaire
+				System.out.println "----Building facsimile version.";
+				}
+				MAD.ApplyXsl(new URL("file:///" + outDirCut + "/" + inFile + "_" + nPage+ ".xml"), outDir + "dipl", "_" + outFile + "_dipl_" + nPage + ".gsp");//mise en page diplomatique
+				System.out.println "----Building diplomatic version.";
+				if(index<260)
+				{					
+				MTR.ApplyXsl(new URL("file:///" + outDirCut + "/" + inFile2 + "_" + nPage+ ".xml"), outDir + "trad", "_" + outFile2 + "_trad_" + nPage + ".gsp");//mise en forme traduction
+				System.out.println "----Building traduction version.";
+				}
+
+			MAN.ApplyXsl(new URL("file:///" + outDirCut + "/" + inFile + "_" + nPage + ".xml"), outDir + "cour", "_" + outFile + "_norm_" + nPage + ".gsp");//mise en forme courante
+			System.out.println "----Building current version.";
+			System.out.println "--Page " + nPage + " OK";
+		}
+		println "end"
+		return true;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_facs.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_facs.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_facs.xsl	(revision 321)
@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- This is a "facsimilary" stylesheet for Old French XML transcriptions:
+	- most abbreviations and medieval letter variants are rendered with the 
+                                                             corresponding Unicode and MUFI characters. "Andron Scriptor Web"
+                                                             font should be installed for correct visualization;
+	- agglutinations and deglutinations are presented "as they are" in
+	  the original;
+	- direct speech is marked with darkblue color;
+	- uncertain readings are marked with grey background color;
+	- text deleted in the original is marked with rose background color;
+	- notes are rendered with [*] sign in the text body; the text of the note
+                                                             appears on "mouse-over" and is reproduced in the end.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+                <xsl:param name="form">facsimilaire</xsl:param>
+                <xsl:variable name="mode">facs</xsl:variable>
+
+                <xsl:output method="html" encoding="UTF-8" version="4.0" indent="yes"/>
+
+                <xsl:strip-space elements="*"/>
+
+                <!--                <xsl:variable name="title-formal" select="//fileDesc/titleStmt/title[@type='formal']"/>
+                                                
+                                                <xsl:variable name="title-normal" select="//fileDesc/titleStmt/title[@type='normal']"/>
+                                                
+                                                <xsl:variable name="title-ref" select="//fileDesc/titleStmt/title[@type='reference']"/>
+                                                
+                                                <xsl:variable name="author" select="//titleStmt/author"/>
+                                -->
+
+                <xsl:include href="mdv_common.xsl"/>
+
+                <xsl:template match="//tei:p|//tei:head">
+                                <p>
+                                                <xsl:if test="@n"> [ § <xsl:value-of select="@n"
+                                                  />]&#xa0; </xsl:if>
+                                                <xsl:apply-templates mode="facs"/>
+                                </p>
+                </xsl:template>
+
+                <xsl:template match="tei:pb" mode="facs">
+                                <xsl:call-template name="pb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:cb" mode="facs">
+                                <xsl:call-template name="cb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:note" mode="facs">
+                                <xsl:call-template name="note_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:q" mode="facs">
+                                <span style="background-color:khaki">
+                                                <xsl:apply-templates mode="facs"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:gap" mode="facs">
+                                <xsl:call-template name="gap_common"/>
+                </xsl:template>
+
+
+                <xsl:template match="//tei:lb" mode="facs">
+                                <xsl:choose>
+                                                <xsl:when
+                                                  test="position()=1 and not(ancestor::tei:seg)"/>
+                                                <xsl:when test="@ed='norm'"/>
+                                                <xsl:when
+                                                  test="preceding::tei:w[1][descendant::bfm:headlb]"/>
+                                                <xsl:when
+                                                  test="preceding::tei:w[1][descendant::tei:lb]"/>
+                                                <xsl:otherwise>
+                                                  <br/>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+                <xsl:template match="//bfm:headlb" mode="facs">
+                                <br/>
+                </xsl:template>
+
+                <xsl:template match="//tei:w|//bfm:punct" mode="facs">
+                                <span style="font-family:Andron Scriptor Web" class="word"
+                                                title="{@type}" id="{@xml:id}">
+                                                <xsl:apply-templates select="descendant::me:facs"
+                                                  mode="facs"/>
+                                </span>
+                                <xsl:choose>
+                                                <xsl:when test="@bfm:aggl"/>
+                                                <xsl:otherwise>
+                                                  <xsl:text> </xsl:text>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+                <xsl:template match="text()[ancestor::me:facs]" mode="facs">
+                                <xsl:value-of select="translate(., 'i', '&#x0131;')"/>
+                </xsl:template>
+
+                <xsl:template match="bfm:mdvAbbr" mode="facs">
+                                <span style="color:darkgreen">
+                                                <xsl:apply-templates mode="facs"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:sic" mode="facs">
+                                <span style="color:darkred" title="sic!">
+                                                <xsl:apply-templates mode="facs"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:subst" mode="facs"> [<xsl:choose>
+                                                <xsl:when
+                                                  test="child::tei:del[@rend='line-through']">
+                                                  <span
+                                                  style="color:red;text-decoration:line-through">
+                                                  <xsl:value-of select="child::tei:del"/>
+                                                  </span> \ <span
+                                                  style="color:blue;text-decoration:underline">
+                                                  <xsl:value-of select="child::tei:add"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:when test="child::tei:del[@rend='dotobl']">
+                                                  <span style="color:red">
+                                                  <xsl:value-of select="child::tei:del"/>
+                                                  </span> \ <span
+                                                  style="color:blue;text-decoration:underline">
+                                                  <xsl:value-of select="child::tei:add"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:when test="child::tei:del[@rend='transform']">
+                                                  <span
+                                                  style="color:red;text-decoration:line-through">
+                                                  <xsl:value-of select="child::tei:del"/>
+                                                  </span> &gt; <span
+                                                  style="color:blue;text-decoration:underline">
+                                                  <xsl:value-of select="child::tei:add"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:when
+                                                  test="child::tei:del[@rend='unmarked'] and child::tei:add[@place='overwrite']">
+                                                  <span
+                                                  style="color:red;text-decoration:line-through">
+                                                  <xsl:value-of select="child::tei:del"/>
+                                                  </span> + <span
+                                                  style="color:blue;text-decoration:underline">
+                                                  <xsl:value-of select="child::tei:add"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:when
+                                                  test="child::tei:del[@rend='unmarked'] and child::tei:add[contains(@place,'linear')]">
+                                                  <span
+                                                  style="color:red;text-decoration:line-through">
+                                                  <xsl:value-of select="child::tei:del"/>
+                                                  </span> + \ <span
+                                                  style="color:blue;text-decoration:underline">
+                                                  <xsl:value-of select="child::tei:add"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                  <span
+                                                  style="color:red;text-decoration:line-through">
+                                                  <xsl:value-of select="child::tei:del"/>
+                                                  </span> / <span
+                                                  style="color:blue;text-decoration:underline">
+                                                  <xsl:value-of select="child::tei:add"/>
+                                                  </span>
+                                                </xsl:otherwise>
+                                </xsl:choose>] </xsl:template>
+
+                <xsl:template match="tei:add" mode="facs">
+                                <span style="color:blue;text-decoration:underline">
+                                                <xsl:choose>
+                                                  <xsl:when
+                                                  test="@place='interlinear' or @place='supralinear'"
+                                                  >\<xsl:apply-templates mode="facs"/>/</xsl:when>
+                                                  <xsl:when test="starts-with(@place,'margin')"
+                                                  >\\<xsl:apply-templates mode="facs"/>//</xsl:when>
+                                                  <xsl:otherwise>
+                                                  <xsl:apply-templates mode="facs"/>
+                                                  </xsl:otherwise>
+                                                </xsl:choose>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:del" mode="facs">
+                                <xsl:choose>
+                                                <xsl:when test="@rend='dotbl'">
+                                                  <span style="color:red">
+                                                  <xsl:apply-templates mode="facs"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                  <span
+                                                  style="text-decoration:line-through;color:red">
+                                                  <xsl:apply-templates mode="facs"/>
+                                                  </span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+                <xsl:template match="tei:unclear" mode="facs">
+                                <span style="background-color:lightgrey">
+                                                <xsl:apply-templates mode="facs"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="bfm:lettrine" mode="facs">
+                                <font color="{@color}" size="{@size}00%" style="font-weight:bold">
+                                                <xsl:apply-templates mode="facs"/>
+                                </font>
+                </xsl:template>
+
+                <xsl:template match="bfm:sb" mode="facs">
+                                <xsl:text>&#xa0;</xsl:text>
+                </xsl:template>
+
+                <xsl:template match="//tei:hi[@rend='sup']" mode="facs">
+                                <sup>
+                                                <xsl:apply-templates mode="facs"/>
+                                </sup>
+                </xsl:template>
+
+
+                <!-- 
+                                                
+                                                
+                                                <xsl:template match="tei:name">
+                                                <xsl:value-of select="@reg"/>
+                                                </xsl:template>
+                                                
+                                -->
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_facs.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_facs.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_facs.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_facs.xsl	(revision 321)
@@ -0,0 +1,192 @@
+<?xml version='1.0'?>
+
+<!-- This is a "facsimilary" stylesheet for Old French XML transcriptions:
+	- most abbreviations and medieval letter variants are rendered with the 
+                                                             corresponding Unicode and MUFI characters. "Andron Scriptor Web"
+                                                             font should be installed for correct visualization;
+	- agglutinations and deglutinations are presented "as they are" in
+	  the original;
+	- direct speech is marked with darkblue color;
+	- uncertain readings are marked with grey background color;
+	- text deleted in the original is marked with rose background color;
+	- notes are rendered with [*] sign in the text body; the text of the note
+                                                             appears on "mouse-over" and is reproduced in the end.
+-->
+
+                <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+                                <xsl:import href="mdv_common.xsl"/>
+                                <xsl:param name="form">fac-similaire</xsl:param>
+                                <xsl:variable name="mode">facs</xsl:variable>
+                                
+                                <xsl:output method="html"  encoding="utf-8" version="4.0" indent="yes"/>
+                                
+                                <xsl:strip-space elements="*"/>
+                                
+                                <!--                <xsl:variable name="title-formal" select="//fileDesc/titleStmt/title[@type='formal']"/>
+                                                
+                                                <xsl:variable name="title-normal" select="//fileDesc/titleStmt/title[@type='normal']"/>
+                                                
+                                                <xsl:variable name="title-ref" select="//fileDesc/titleStmt/title[@type='reference']"/>
+                                                
+                                                <xsl:variable name="author" select="//titleStmt/author"/>
+                                -->               
+                                
+                                
+                                
+                                <xsl:template match="//tei:text//tei:p|//tei:text//tei:head">
+                                                <table width="100%" style="border:none">
+                                                                <tr style="border:none">
+                                                                                <td width="10%" valign="top" style="white-space:nowrap;padding-right:10pt;line-height:18px;font-family:Arial;font-size:14px;border:none;text-align:right">
+                                                                                                <xsl:if test="@n"><span style="color:gray;font-size:12px">[§ <xsl:value-of select="@n"/>] </span></xsl:if><xsl:apply-templates select="descendant::tei:lb|descendant::tei:milestone[@unit='column']|descendant::bfm:lettrine" mode="number"/></td>
+                                                                                <td valign="top" style="white-space:nowrap;line-height:18px;font-family:Andron Scriptor Web;font-size:14px;border:none"><xsl:apply-templates mode="facs"/></td>
+                                                                </tr>
+                                                </table>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:pb" mode="facs">
+                                                <xsl:call-template name="pb_common"/>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:cb" mode="facs">
+                                                <xsl:call-template name="cb_common"></xsl:call-template>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:note" mode="facs">
+                                                <xsl:call-template name="note_common"></xsl:call-template>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:q" mode="facs">
+                                                <span style="background-color:lightcyan"><xsl:apply-templates mode="facs"/></span>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:gap" mode="facs">
+                                                <xsl:call-template name="gap_common"></xsl:call-template>
+                                </xsl:template>
+                                
+                                <xsl:template match="//tei:lb" mode="facs">
+                                                <xsl:choose>
+                                                                <xsl:when test="@ed='facs'"><br/></xsl:when>
+                                                                <!--<xsl:when test="position()=1 and not(ancestor::tei:seg)"></xsl:when>-->
+                                                                <xsl:when test="@ed='norm' or preceding::tei:lb[1][@ed='facs']"></xsl:when>
+                                                                <xsl:when test="position() = 1 and ancestor::tei:s[not(preceding-sibling::*)]"></xsl:when>
+                                                                <xsl:when test="preceding::tei:w[1][descendant::bfm:headlb]"></xsl:when>
+                                                                <xsl:when test="preceding::tei:w[1][descendant::tei:lb]"></xsl:when>
+                                                                <xsl:otherwise><br/></xsl:otherwise>
+                                                </xsl:choose>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:lb" mode="number">
+                                                <xsl:choose>
+                                                                <xsl:when test="@ed='facs'"></xsl:when>
+                                                                <xsl:when test="position() = 1 and ancestor::tei:s[not(preceding-sibling::*)]"></xsl:when>
+                                                                <xsl:otherwise><br/>&#xa0;</xsl:otherwise>
+                                                </xsl:choose>
+                                                <xsl:if test="substring(@n, string-length(@n)) = 0 or substring(@n, string-length(@n)) = 5"><xsl:value-of select="@n"/></xsl:if>
+                                </xsl:template>
+                                
+                               
+                               
+                                <xsl:template match="//bfm:headlb" mode="facs">
+                                                <br/>
+                                </xsl:template>
+                                
+                                <xsl:template match="//tei:w" mode="facs">
+                                                <span title="{@type}" class="word" id="{@xml:id}" >
+                                                <xsl:if test="descendant::me:facs"><xsl:apply-templates select="descendant::me:facs" mode="facs"/>
+                                                <xsl:choose>
+                                                                <xsl:when test="@rend='aggl' or @rend='elision'"></xsl:when>
+                                                                <xsl:otherwise><xsl:text> </xsl:text></xsl:otherwise>
+                                                </xsl:choose></xsl:if>
+                                                                </span>
+                                </xsl:template>
+                                
+                                <xsl:template match="text()[ancestor::me:facs]" mode="facs">
+                                                <xsl:value-of select="translate(., 'i', '&#x0131;')"/>
+                                </xsl:template>
+                                
+                                <xsl:template match="bfm:mdvAbbr" mode="facs">
+                                                <xsl:apply-templates mode="facs"/>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:sic" mode="facs">
+                                                <span style="color:darkred" title="sic!"><xsl:apply-templates mode="facs"/></span>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:subst" mode="facs">
+                                                [<xsl:choose>
+                                                                <xsl:when test="child::tei:del[@rend='line-through']">
+                                                                                <span style="color:red;text-decoration:line-through"><xsl:value-of select="child::tei:del"/></span> \ <span style="color:blue;text-decoration:underline"><xsl:value-of select="child::tei:add"/></span>
+                                                                </xsl:when>
+                                                                <xsl:when test="child::tei:del[@rend='dotobl']">
+                                                                                <span style="color:red"><xsl:value-of select="child::tei:del"/></span> \ <span style="color:blue;text-decoration:underline"><xsl:value-of select="child::tei:add"/></span>
+                                                                </xsl:when>
+                                                                <xsl:when test="child::tei:del[@rend='transform']">
+                                                                                <span style="color:red;text-decoration:line-through"><xsl:value-of select="child::tei:del"/></span> &gt; <span style="color:blue;text-decoration:underline"><xsl:value-of select="child::tei:add"/></span>
+                                                                </xsl:when>
+                                                                <xsl:when test="child::tei:del[@rend='unmarked'] and child::tei:add[@place='overwrite']">
+                                                                                <span style="color:red;text-decoration:line-through"><xsl:value-of select="child::tei:del"/></span> +  <span style="color:blue;text-decoration:underline"><xsl:value-of select="child::tei:add"/></span>
+                                                                </xsl:when>
+                                                                <xsl:when test="child::tei:del[@rend='unmarked'] and child::tei:add[contains(@place,'linear')]">
+                                                                                <span style="color:red;text-decoration:line-through"><xsl:value-of select="child::tei:del"/></span> +  \ <span style="color:blue;text-decoration:underline"><xsl:value-of select="child::tei:add"/></span>
+                                                                </xsl:when>
+                                                                <xsl:otherwise>
+                                                                                <span style="color:red;text-decoration:line-through"><xsl:value-of select="child::tei:del"/></span> / <span style="color:blue;text-decoration:underline"><xsl:value-of select="child::tei:add"/></span>
+                                                                </xsl:otherwise>                                                
+                                                </xsl:choose>]
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:add" mode="facs">
+                                                <span style="color:blue;text-decoration:underline">
+                                                                <xsl:choose>
+                                                                                <xsl:when test="@place='interlinear' or @place='supralinear'">\<xsl:apply-templates mode="facs"/>/</xsl:when>
+                                                                                <xsl:when test="starts-with(@place,'margin')">\\<xsl:apply-templates mode="facs"/>//</xsl:when>
+                                                                                <xsl:otherwise><xsl:apply-templates mode="facs"/></xsl:otherwise>
+                                                                </xsl:choose>
+                                                </span>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:del" mode="facs">
+                                                <xsl:choose>
+                                                                <xsl:when test="@rend='dotbl'"><span style="color:red"><xsl:apply-templates mode="facs"/></span></xsl:when>
+                                                                <xsl:otherwise><span style="text-decoration:line-through;color:red"><xsl:apply-templates mode="facs"/></span></xsl:otherwise>
+                                                </xsl:choose>
+                                </xsl:template>
+                                
+                                <xsl:template match="tei:unclear" mode="facs">
+                                                <span style="background-color:lightgrey"><xsl:apply-templates mode="facs"/></span>
+                                </xsl:template>
+                                
+                                <xsl:template match="bfm:lettrine" mode="facs">
+                                                <font color="{@color}" size="{@size}00%" style="font-weight:bold"><xsl:apply-templates mode="facs"/></font>
+                                </xsl:template>
+                                
+                                <xsl:template match="bfm:lettrine" mode="number">
+                                                <font color="black" size="{@size}00%" style="font-weight:bold">&#xa0;</font>
+                                </xsl:template>
+
+                                <xsl:template match="bfm:sb" mode="facs">
+                                                <xsl:text>&#xa0;</xsl:text>
+                                </xsl:template>
+                                
+                                <xsl:template match="//tei:hi[@rend='sup']" mode="facs">
+                                                <sup><xsl:apply-templates mode="facs"/></sup>
+                                </xsl:template>
+                  
+                                  <xsl:template match="tei:supplied" mode="facs">
+                                <span style="color:blue">[<xsl:apply-templates mode="facs"/>]</span>
+                                <xsl:if test="not(ancestor::tei:w)"> </xsl:if>
+                </xsl:template>
+                                
+                                <xsl:template match="tei:surplus">
+                                                <span style="color:darkred" title="sic!"><xsl:apply-templates mode="facs"/></span>
+                                </xsl:template>
+                                
+                                <!-- 
+                                                
+                                                
+                                                <xsl:template match="tei:name">
+                                                <xsl:value-of select="@reg"/>
+                                                </xsl:template>
+                                                
+                                -->
+ </xsl:stylesheet>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_facs.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_cms.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_cms.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_cms.xsl	(revision 321)
@@ -0,0 +1,193 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0"
+    xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0"
+    xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+
+    <!-- Eléments communs à toutes les présentations -->
+
+    <xsl:output method="html" encoding="UTF-8" version="4.0" indent="yes"/>
+
+    <xsl:strip-space elements="*"/>
+
+    <xsl:variable name="text_id">
+        <xsl:value-of select="//tei:teiHeader//tei:title[@type='reference']"/>
+    </xsl:variable>
+
+    <xsl:template match="/">
+        <html>
+            <head> </head>
+            <body>
+                <xsl:apply-templates select="//tei:text"/>
+                <xsl:apply-templates select="//tei:note" mode="final"/>
+            </body>
+        </html>
+    </xsl:template>
+
+
+    <xsl:template match="//tei:body/tei:div">
+        <br/>
+        <span style="font-family:Times;color:indigo">[<i>
+                <xsl:value-of select="@type"/>
+            </i>&#xa0;<xsl:value-of select="@n"/>]</span>
+        <xsl:apply-templates/>
+    </xsl:template>
+
+    <xsl:template match="//tei:body/tei:div/tei:div">
+        <br/>
+        <span style="font-family:Times;color:indigo">[<i>
+                <xsl:value-of select="@type"/>
+            </i>&#xa0;<xsl:value-of select="@n"/>]</span>
+        <br/>
+        <xsl:apply-templates/>
+    </xsl:template>
+
+    <xsl:template match="tei:head">
+        <span style="font-family:Arial; color:darkred">
+            <xsl:apply-templates/>
+        </span>
+        <br/>
+    </xsl:template>
+
+    <xsl:template match="//tei:p">
+        <p>
+            <xsl:if test="@n"> [ § <xsl:value-of select="@n"/>]&#xa0; </xsl:if>
+            <xsl:apply-templates/>
+        </p>
+    </xsl:template>
+
+    <xsl:template match="tei:pb">
+        <br/>
+        <br/>
+        <span style="font-family:Times" id="{$text_id}-{@n}">[folio <xsl:value-of select="@n"/>] <a
+                href="{$text_id}-compare.html#{$text_id}-{@n}">[autres facettes]</a>
+            <xsl:if test="@facs">
+                <xsl:text> </xsl:text>
+                <a href="../{@facs}" target="_blank">[Image]</a>
+            </xsl:if>
+        </span>
+        <xsl:if test="not(following-sibling::*[1][self::cb])">
+            <br/>
+        </xsl:if>
+    </xsl:template>
+
+    <xsl:template match="//tei:cb">
+        <br/>
+        <xsl:if test="not(preceding-sibling::*[1][self::pb])">
+            <br/>
+        </xsl:if>
+        <span style="font-family:Times">[colonne <xsl:value-of select="@n"/>]</span>
+        <br/>
+    </xsl:template>
+
+    <xsl:template match="tei:note">
+        <xsl:variable name="note_num">
+            <xsl:choose>
+                <xsl:when test="@n">
+                    <xsl:value-of select="@n"/>
+                </xsl:when>
+                <xsl:otherwise>
+                    <xsl:value-of select="count(preceding::tei:note) + 1"/>
+                </xsl:otherwise>
+            </xsl:choose>
+        </xsl:variable>
+        <a style="font-size:75%;color:blue;font-family:Times" title="{.}" id="noteref_{$note_num}"
+            href="#note_{$note_num}">
+            <sup>[<xsl:value-of select="count(preceding::tei:note) + 1"/>]</sup>
+        </a>
+        <xsl:if test="following::*[1][self::tei:w]">
+            <xsl:text> </xsl:text>
+        </xsl:if>
+    </xsl:template>
+
+    <xsl:template match="tei:note" mode="final">
+        <xsl:variable name="note_num">
+            <xsl:choose>
+                <xsl:when test="@n">
+                    <xsl:value-of select="@n"/>
+                </xsl:when>
+                <xsl:otherwise>
+                    <xsl:value-of select="position()"/>
+                </xsl:otherwise>
+            </xsl:choose>
+        </xsl:variable>
+        <span id="note_{$note_num}">
+            <a href="#noteref_{$note_num}">[<xsl:value-of select="$note_num"/>]</a>
+            (fol. <xsl:value-of select="preceding::tei:pb[1]/@n"/>, col. 
+            <xsl:value-of select="preceding::tei:cb[1]/@n"/>, l. 
+            <xsl:value-of select="preceding::tei:lb[1]/@n"/>) 
+            <xsl:apply-templates/>
+        </span>
+        <br/>
+    </xsl:template>
+
+    <xsl:template match="tei:q">
+        <span style="color:darkblue">
+            <xsl:apply-templates/>
+        </span>
+    </xsl:template>
+
+
+    <xsl:template match="tei:gap">
+        <xsl:text> [...] </xsl:text>
+    </xsl:template>
+    
+    
+<!--    <xsl:template match="tei:q">
+        <span style="background-color:khaki">
+            <xsl:apply-templates/>
+        </span>
+    </xsl:template>-->
+    
+    <xsl:template match="tei:gap">
+        <xsl:apply-templates/>
+    </xsl:template>
+    
+    <xsl:template match="//tei:lb">
+        <xsl:choose>
+            <xsl:when
+                test="position()=1 and not(ancestor::tei:seg)"/>
+            <xsl:otherwise>
+                <br/>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+    
+    <xsl:template match="//tei:w">
+        <span class="word" id="{@xml:id}" title="{@type}">
+            <xsl:if
+                test="starts-with(.,':') or starts-with(.,'!') or starts-with(.,'?') or starts-with(.,';')"
+                >&#xa0;</xsl:if>
+            <xsl:value-of select="."/>
+            <xsl:choose>
+                <xsl:when
+                    test="following::*[1][starts-with(., ',') or starts-with(., '.') or starts-with(., ']') or starts-with(., ')') or starts-with(.,':') or starts-with(.,'!') or starts-with(.,'?') or starts-with(.,';')]"/>
+                <xsl:when
+                    test=".[string-length(.) = 0]"/>
+                <xsl:when test="@bfm:aggl='elision'">'</xsl:when>
+                <xsl:otherwise>
+                    <xsl:text> </xsl:text>
+                </xsl:otherwise>
+            </xsl:choose>
+        </span>
+    </xsl:template>
+    
+    <xsl:template match="tei:sic" >
+        <span style="text-decoration:line-through;color:red">
+            <xsl:apply-templates mode="norm"/>
+        </span>
+    </xsl:template>
+    
+    <xsl:template match="tei:del"> </xsl:template>
+    
+    
+    <xsl:template match="//tei:hi[@rend='sup']">
+        <sup>
+            <xsl:apply-templates/>
+        </sup>
+    </xsl:template>
+
+    <!-- fin éléments communs -->
+
+
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_cms.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_common.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_common.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_common.xsl	(revision 321)
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<xsl:stylesheet  xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+
+    <!-- Eléments communs à toutes les présentations -->
+
+<xsl:variable name="text_id"><xsl:value-of select="//tei:teiHeader//tei:title[@type='reference']"/></xsl:variable>
+
+<!--    <xsl:template match="/">
+        <html>
+            <head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
+            </head>
+            <body>
+                <div class="head"><xsl:apply-templates select="//tei:teiHeader"/></div>
+                <xsl:apply-templates select="//tei:text">
+                    <xsl:with-param name="form"><xsl:value-of select="$form"/></xsl:with-param>
+                </xsl:apply-templates>
+                <!-\-<xsl:apply-templates select="//tei:note" mode="final"></xsl:apply-templates>-\->
+            </body>
+        </html>
+    </xsl:template>
+-->    
+    <xsl:template match="/">
+                <xsl:apply-templates select="//tei:text">
+                    <xsl:with-param name="form"><xsl:value-of select="$form"/></xsl:with-param>
+                </xsl:apply-templates>
+    </xsl:template>
+
+                                                <!-- TEI-Header Processing -->
+                                
+                                                <xsl:variable name="title-source" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='source']"/>
+                
+                <xsl:variable name="title-normal" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='normal']"/>
+                
+                <xsl:variable name="title-ref" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='reference']"/>
+                
+                
+                <xsl:variable name="author" select="//tei:titleStmt/tei:author"/>
+                
+                <xsl:variable name="apostrophe" select='"&apos;"'/>
+
+                
+                
+                <xsl:template match="//tei:teiHeader">
+                                <p align="center">
+                                                
+                                                <!-- Author (only appears if not anonymous) -->
+                                                <xsl:if test="$author[not(contains(.,'anonym'))]">
+                                                                <span style="font-variant:small-caps;font-weight:bold"><xsl:value-of select="$author"/></span><br/>                
+                                                </xsl:if>
+                                                <!-- Title -->
+                                                <span style="color:darkviolet;font-family:Times;font-weight:bold;font-style:italic;font-size:200%"><xsl:value-of select="$title-normal"/></span><br/>
+                                                <!-- Scientific editor(s) -->
+                                                Edition par <xsl:apply-templates select="//tei:titleStmt/tei:editor[@role='editor']"/><br/>
+                                                Avec la collaboration de <xsl:apply-templates select="//tei:titleStmt/tei:editor[@role='associate_editor']"/><br/>
+                                                
+                                                <!-- Series Statement, Publisher, Year 	
+                                                                <xsl:apply-templates select="fileDesc/seriesStmt/title"/><br/>
+                                                                <xsl:apply-templates select="//sourceDesc//publicationStmt/pubPlace"/>,
+                                                                <xsl:apply-templates select="//sourceDesc//publicationStmt/publisher"/>,
+                                                                <xsl:apply-templates select="//sourceDesc//publicationStmt/date"/>
+                                                                
+                                                -->
+                                </p>
+                                
+                                <!-- Information on the electronic version  -->                
+                                <p align="center">_____________________________________________________<br/><br/>
+                                                <span style="font-size:75%">Edition électronique<br/>
+                                                                Dernière révision : <xsl:value-of select="//tei:revisionDesc/tei:change[1]/@when"/></span><br/>
+                                                                _____________________________________________________</p>
+                                <p align="center">Version <xsl:value-of select="$form"/>
+                                                <br/>_____________________________________________________</p>
+                </xsl:template>
+
+
+    
+    <xsl:template match="//tei:body/tei:div">
+        <br/><span style="font-family:Times;color:indigo">[<i><xsl:value-of select="@type"/></i>&#xa0;<xsl:value-of select="@n"/>]</span>
+        <xsl:apply-templates/>
+    </xsl:template>
+    
+    <xsl:template match="//tei:body/tei:div/tei:div">
+        <br/><span style="font-family:Times;color:indigo">[<i><xsl:value-of select="@type"/></i>&#xa0;<xsl:value-of select="@n"/>]</span><br/>
+        <xsl:apply-templates/>
+    </xsl:template>
+    
+    <xsl:template match="tei:head">
+        <span style="font-family:Arial; color:darkred"><xsl:apply-templates/></span><br/>
+    </xsl:template> 
+    <!-- 
+    <xsl:template match="//tei:p">
+        <p>
+            <xsl:if test="@n">
+                [ § <xsl:value-of select="@n"/>]&#xa0;
+            </xsl:if>
+            <xsl:apply-templates/>
+        </p>
+    </xsl:template>
+    -->
+    <xsl:template name="pb_common" match="tei:pb">
+<!--        <br/><br/>
+        <span style="font-family:Times" id="{$text_id}-{@n}">[folio <xsl:value-of select="@n"/>] <a href="{$text_id}-compare.html#{$text_id}-{@n}">[autres facettes]</a>
+        <xsl:if test="@facs">
+            <xsl:text> </xsl:text><a href="../{@facs}" target="_blank">[Image]</a>
+        </xsl:if>
+        </span>
+        <xsl:if test="not(following-sibling::*[1][self::cb])">
+            <br/>
+        </xsl:if>-->
+    </xsl:template>
+    
+    <xsl:template match="//tei:cb" name="cb_common">
+        <br/>
+        <xsl:if test="not(preceding-sibling::*[1][self::pb])"><br/></xsl:if>
+        <span style="font-family:Times">[colonne <xsl:value-of select="@n"/>]</span>
+        <br/>
+    </xsl:template>
+        
+    <xsl:template match="tei:note" name="note_common">
+<!--        <xsl:variable name="note_num">
+            <xsl:choose>
+                <xsl:when test="@n"><xsl:value-of select="@n"/></xsl:when>
+                <xsl:otherwise><xsl:value-of select="count(preceding::tei:note) + 1"/></xsl:otherwise>
+            </xsl:choose>            
+        </xsl:variable>
+        <a style="font-size:75%;color:blue;font-family:Times" title="{.}" id="noteref_{$note_num}" href="#note_{$note_num}"><sup>[<xsl:value-of select="count(preceding::tei:note) + 1"/>]</sup></a>
+        <xsl:if test="following::*[1][self::tei:w]"><xsl:text> </xsl:text></xsl:if>-->
+    </xsl:template>
+    
+    <xsl:template match="tei:note" mode="final">
+<!--        <xsl:variable name="note_num">
+            <xsl:choose>
+                <xsl:when test="@n"><xsl:value-of select="@n"/></xsl:when>
+                <xsl:otherwise><xsl:value-of select="position()"/></xsl:otherwise>
+            </xsl:choose>            
+        </xsl:variable>
+        <span id="note_{$note_num}"><a href="#noteref_{$note_num}">[<xsl:value-of select="$note_num"/>]</a> (fol. <xsl:value-of select="preceding::tei:pb[1]/@n"/>, col. <xsl:value-of select="preceding::tei:cb[1]/@n"/>,  l. <xsl:value-of select="preceding::tei:lb[1]/@n"/>) <xsl:apply-templates/></span><br/>
+-->    </xsl:template>
+    
+<!--    <xsl:template match="tei:q" name="q_common">
+        <span style="color:darkblue"><xsl:apply-templates/></span>
+    </xsl:template>
+-->    
+    
+    <xsl:template match="tei:gap" name="gap_common">
+        <xsl:text> [...] </xsl:text>
+    </xsl:template>
+
+<xsl:template name="supplied_common">
+    <xsl:text>[</xsl:text>
+    <xsl:apply-templates/>
+    <xsl:text>]</xsl:text>
+</xsl:template>
+
+    <!-- fin éléments communs -->
+    
+    
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_common.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_dipl.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_dipl.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_dipl.xsl	(revision 321)
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- This is a stylesheet that gives a "diplomatic" presentation of old French manuscript transcriptions
+	- abbreviations are resolved and rendered in italics;
+	- agglutinations and deglutinations are not marked (word limits 
+	  are "normalized");
+                                                          - Strong punctuation is marked with a dot followed by a capital letter;
+                                                             weak punctuation is marked with a comma.
+	- direct speech is marked with darkblue color;
+	- uncertain readings are marked with grey background color;
+	- text deleted in the original is not visualized;
+	- text corrected in the normalized version is marked with darkred
+                                                             color and followed by "(sic!)" note;
+                                                          - notes are rendered with [*] sign in the text body; the text of the note
+                                                            appears on "mouse-over" and is reproduced in the end.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+                <xsl:include href="mdv_common.xsl"/>
+
+                <xsl:param name="form">diplomatique</xsl:param>
+
+                <xsl:variable name="mode">dipl</xsl:variable>
+
+                <xsl:output method="html" encoding="UTF-8" version="4.0" indent="yes"/>
+
+                <xsl:strip-space elements="*"/>
+
+                <!--                <xsl:variable name="title-formal" select="//fileDesc/titleStmt/title[@type='formal']"/>
+                                
+                                <xsl:variable name="title-normal" select="//fileDesc/titleStmt/title[@type='normal']"/>
+                                
+                                <xsl:variable name="title-ref" select="//fileDesc/titleStmt/title[@type='reference']"/>
+                                
+                                <xsl:variable name="author" select="//titleStmt/author"/>
+                -->
+
+                <xsl:template match="//tei:p|//tei:head">
+                                <p>
+                                                <xsl:if test="@n"> [ § <xsl:value-of select="@n"
+                                                  />]&#xa0; </xsl:if>
+                                                <xsl:apply-templates mode="dipl"/>
+                                </p>
+                </xsl:template>
+
+                <xsl:template match="tei:pb" mode="dipl">
+                                <xsl:call-template name="pb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:cb" mode="dipl">
+                                <xsl:call-template name="cb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:note" mode="dipl">
+                                <xsl:call-template name="note_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:q" mode="dipl">
+                                <span style="background-color:khaki">
+                                                <xsl:apply-templates mode="dipl"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:gap" mode="dipl">
+                                <xsl:call-template name="gap_common"/>
+                </xsl:template>
+
+
+                <xsl:template match="//tei:lb[@ed='norm']" mode="dipl"> </xsl:template>
+
+                <xsl:template match="//tei:lb[not(@ed)]" mode="dipl">
+                                <xsl:choose>
+                                                <xsl:when
+                                                  test="position()=1 and not(ancestor::tei:seg)"/>
+                                                <xsl:when
+                                                  test="preceding::tei:w[1][descendant::bfm:headlb]"/>
+                                                <xsl:when
+                                                  test="preceding::tei:w[1][descendant::tei:lb]"/>
+                                                <xsl:otherwise>
+                                                  <br/>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+                <xsl:template match="//tei:lb[@ed='facs']" mode="dipl">
+                                <xsl:text>-</xsl:text>
+                                <br/>
+                </xsl:template>
+
+                <xsl:template match="//tei:w|//bfm:punct" mode="dipl">
+                                <span class="word" id="{@xml:id}" title="{@type}">
+                                                <xsl:apply-templates select="descendant::me:dipl"
+                                                  mode="dipl"/>
+                                                <xsl:if
+                                                  test="descendant::tei:sic and not(following-sibling::tei:w[1][descendant::tei:sic])">
+                                                  <i>
+                                                  <xsl:text> (sic !)</xsl:text>
+                                                  </i>
+                                                </xsl:if>
+                                                <xsl:choose>
+                                                  <xsl:when
+                                                  test="following::*[1][self::bfm:punct][contains(., ',') or contains(., '.') or contains(., ']')or contains(., ')')]"/>
+                                                  <xsl:when test="@bfm:aggl='elision'">'</xsl:when>
+                                                  <xsl:otherwise>
+                                                  <xsl:text> </xsl:text>
+                                                  </xsl:otherwise>
+                                                </xsl:choose>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:ex" mode="dipl">
+                                <xsl:choose>
+                                                <xsl:when test="@cert='high'">
+                                                  <span style="color:darkgreen;font-style:italic">
+                                                  <xsl:apply-templates mode="dipl"/>
+                                                  </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                  <span style="color:darkred;font-style:italic">
+                                                  <xsl:apply-templates mode="dipl"/>
+                                                  </span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+                <xsl:template match="tei:sic" mode="dipl">
+                                <span style="color:darkred" title="sic!">
+                                                <xsl:apply-templates mode="dipl"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:subst" mode="dipl">[<xsl:apply-templates mode="dipl"
+                                />]</xsl:template>
+
+                <xsl:template match="tei:del" mode="dipl">
+                                <span style="color:red;text-decoration:line-through">
+                                                <xsl:apply-templates mode="dipl"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:add[not(@seq)]" mode="dipl">
+                                <span style="color:blue;text-decoration:underline">
+                                                <xsl:apply-templates mode="dipl"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:add[@seq]" mode="dipl">
+                                <span style="color:blue">
+                                                <sup>&lt;<xsl:value-of select="@seq"/></sup>
+                                                <xsl:apply-templates mode="dipl"/>
+                                                <sup><xsl:value-of select="@seq"/>&gt;</sup>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:unclear" mode="dipl">
+                                <span style="background-color:lightgrey">
+                                                <xsl:apply-templates mode="dipl"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:hi[@rend='initiale']" mode="dipl">
+                                <span style="font-weight:bold;font-size:xx-large">
+                                                <xsl:apply-templates mode="dipl"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="//tei:hi[@rend='sup']" mode="dipl">
+                                <sup>
+                                                <xsl:apply-templates mode="dipl"/>
+                                </sup>
+                </xsl:template>
+
+
+
+                <!-- 
+
+
+<xsl:template match="tei:name">
+	<xsl:value-of select="@reg"/>
+</xsl:template>
+
+                -->
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_dipl.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_frmod.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_frmod.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_frmod.xsl	(revision 321)
@@ -0,0 +1,175 @@
+<?xml version='1.0'?>
+
+<!--
+                #######################################
+                Qgraal_cm BASIC stylesheet
+                
+                #######################################
+                author: Alexei Lavrentiev
+                version: 1.2
+                date: 2009-02-16
+                
+                version: 1.1
+                date: 2008-09-30
+                
+                version: 1.0
+                date: 2008-02-21
+                
+                #######################################
+                Description
+                This is a basic stylesheet for viewing BFM texts in XML-TEI
+                Modified for QGRAAL modern French translation
+                
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:tei="http://www.tei-c.org/ns/1.0"
+                version="1.0"> 
+
+                <xsl:output method="html" encoding="utf-8" version="4.0" indent="yes"/>
+                
+                <xsl:strip-space elements="*"/>
+                
+                
+                <xsl:template match="/">
+<!--                                <html>
+                                                <head>
+                                                                <title>Quête du Graal (français moderne)</title>
+                                                </head>
+                                                <body style="margin-left:10%;margin-right:10%">-->
+                                                                <xsl:apply-templates select="//tei:text"/>			
+<!--                                                </body>		
+                                </html>	-->
+                </xsl:template>
+
+                <!-- TEI-Header Processing -->
+                
+                <xsl:template match="//tei:teiHeader">
+                </xsl:template>
+                
+                <!-- End of TEI-header processing -->
+                
+                <!-- Text body processing -->
+
+                <!-- "Original" titles found sometimes immediately before text body -->
+                <xsl:template match="//tei:div[@type='titre']">
+                                <p align="center"><b><xsl:apply-templates/></b></p>
+                </xsl:template>
+                
+                <!-- Text divisions (general) -->
+                <xsl:template match="//tei:div[not(@type='titre')]">
+                                <xsl:if test="child::tei:head">
+                                                <xsl:apply-templates select="head"/><br/>                                                
+                                </xsl:if>
+                                <xsl:apply-templates select="*[not(self::tei:head)]"/>                                
+                </xsl:template>
+                
+                <!-- Paragraphs -->
+                <xsl:template match="//tei:text//tei:p">
+                                <p align="justify"><xsl:apply-templates/></p>
+                </xsl:template>
+                
+
+                
+<!-- Page breaks -->                
+              
+                <xsl:template match="//tei:pb[@ed='Pauphilet1923']">
+                                (p. <xsl:value-of select="@n"/>)
+                </xsl:template>
+
+                <xsl:template match="//tei:milestone[@unit='column']">
+                                <br/><span style="color:darkviolet;font-family:arial;font-size:80%">&lt;<xsl:value-of select="@xml:id"/>&gt;</span>
+                </xsl:template>
+                
+
+                <!-- Line breaks : note special processing for numbered lb used in BFM verse texts -->
+                <xsl:template match="tei:lb">
+                                                                <xsl:if test="not(parent::tei:p and not(preceding-sibling::*))"><br/></xsl:if>
+                </xsl:template>
+                
+              
+                <!-- Processing elements with "rend" attributes  -->
+                
+                <xsl:template match="//*[@rend='ital']">
+                                <i><xsl:apply-templates/></i>
+                </xsl:template>
+
+                <xsl:template match="//*[@rend='gras']">
+                                <b><xsl:apply-templates/></b>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='exp']">
+                                <sup><xsl:apply-templates/></sup>
+                </xsl:template>
+
+                <xsl:template match="//*[@rend='ind']">
+                                <sub><xsl:apply-templates/></sub>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='maj']">
+                                <span style="text-transform:uppercase"><xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='pmaj']">
+<span style="font-variant:small-caps"><xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='crochets']">[<xsl:apply-templates/>]</xsl:template>
+
+                <xsl:template match="//*[@rend='chevrons']">
+                                <xsl:text>&lt;</xsl:text><xsl:apply-templates/><xsl:text>&gt; </xsl:text>
+                </xsl:template>
+                                
+                <xsl:template match="//*[@rend='crochets-ital']">
+                                <xsl:text>[</xsl:text><i><xsl:apply-templates/></i><xsl:text>]</xsl:text>
+                </xsl:template>
+                                
+
+                <!-- Numbers -->
+                <xsl:template match="tei:num">
+                                <span style="color:blue"><xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <!-- Notes -->
+                <xsl:template match="tei:note">
+                                <span style="color:violet"> [<b>Note :</b> <xsl:apply-templates/>] </span>	
+                </xsl:template>
+                
+                <!-- Word formatting for morphologically tagged texts -->
+                <xsl:strip-space elements="tei:s"/>
+                <xsl:template match="//text()[starts-with(., ' ')][preceding-sibling::*[1][self::tei:w]]">
+                                <xsl:copy-of select="substring(., 2)"/>
+                </xsl:template>
+                <xsl:template match="//tei:w">
+                                <span title="{@type}" class="word" id="{@xml:id}" >
+                                <xsl:variable name="apostrophe" select='"&apos;"'/>
+                                <xsl:apply-templates/>
+                                <xsl:choose>
+                                                <xsl:when test="following::tei:w[1][starts-with(@type, 'pon')][contains(., ',') or contains(., '.') or contains(., ']')or contains(., ')')]">
+                                                </xsl:when>
+                                                <xsl:when test="following::node()[1][self::text()][starts-with(normalize-space(.), ',') or starts-with(normalize-space(.), '.') or starts-with(normalize-space(.), ']') or starts-with(normalize-space(.), ')')]">
+                                                </xsl:when>
+                                                <xsl:when test="self::tei:w[starts-with(@type, 'pon')][contains(., '[') or contains(., '(')]">
+                                                </xsl:when>
+                                                <xsl:when test="substring(., string-length(.))=$apostrophe">
+                                                </xsl:when>
+                                                <xsl:when test="following::tei:w[1][starts-with(@type, 'pon')][contains(., ':') or contains(., ';') or contains(., '!')or contains(., '?')]">
+                                                                <xsl:text>&#xa0;</xsl:text>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <xsl:text> </xsl:text>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                                                </span>
+                </xsl:template>
+                
+<xsl:template match="tei:ptr"/>
+
+                <!--  Direct speech, not currently tagged -->
+                <xsl:template match="tei:q[ancestor::tei:q]">
+                                <span style="background-color:cyan"> <xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:q[not(ancestor::tei:q)]">
+                                <span style="background-color:lightcyan"> <xsl:apply-templates/></span>
+                </xsl:template>
+                
+</xsl:stylesheet>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_frmod.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/breakByMilestone.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/breakByMilestone.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/breakByMilestone.xsl	(revision 321)
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+
+    <!-- Based on a stylesheet for creating separate XML-TEI files for
+        each page by Paul Caton (posted on TEI-L 2008-12-11)-->
+
+    <xsl:output method="xml" encoding="UTF-8"/>
+
+    <xsl:param name="pbval1">7</xsl:param>
+    <xsl:param name="pbval2">8</xsl:param>
+
+    <xsl:function name="tei:page-content" as="node()*">
+        <xsl:param name="pb1" as="node()"/>
+        <xsl:param name="pb2" as="node()"/>
+        <xsl:param name="node" as="node()"/>
+        <xsl:choose>
+            <xsl:when test="$node[self::*]">
+                <!-- $node is an element() -->
+                <xsl:choose>
+                    <!--                    <xsl:when test="$node is $pb1">
+                        <xsl:copy-of select="$node"/>
+                    </xsl:when>
+-->
+                    <xsl:when
+                        test="some $n in $node/descendant::* satisfies ($n is $pb1 or $n is $pb2)">
+                        <xsl:element name="{name($node)}" namespace="http://www.tei-c.org/ns/1.0">
+                            <!--<xsl:attribute name="testttt">http://www.menota.org/ns/1.0</xsl:attribute>-->
+                            <xsl:sequence select="for $i in ( $node/node() | $node/@* ) return tei:page-content($pb1, $pb2, $i)"/>
+                        </xsl:element>
+                    </xsl:when>
+                    <xsl:when test="($node >> $pb1) and ($node &lt;&lt; $pb2)">
+                        <xsl:copy-of select="$node"/>
+                    </xsl:when>
+                    <xsl:otherwise>
+                        <!-- do nothing -->
+                    </xsl:otherwise>
+                </xsl:choose>
+            </xsl:when>
+            <xsl:when test="$node[count(. | ../@*) = count(../@*)]">
+                <!-- $node is an attribute -->
+                <xsl:attribute name="{name($node)}">
+                    <xsl:sequence select="data($node)"/>
+                </xsl:attribute>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:choose>
+                    <xsl:when test="($node >> $pb1) and ($node &lt;&lt; $pb2)">
+                        <xsl:copy-of select="$node"/>
+                    </xsl:when>
+                    <xsl:otherwise>
+                        <!-- do nothing -->
+                    </xsl:otherwise>
+                </xsl:choose>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:function>
+
+    <xsl:template match="*|text()|@*">
+        <xsl:copy>
+            <xsl:apply-templates select="@*"/>
+            <xsl:apply-templates/>
+        </xsl:copy>
+    </xsl:template>
+
+    <xsl:template match="processing-instruction()">
+        <xsl:copy>
+            <xsl:value-of select="."/>
+        </xsl:copy>
+    </xsl:template>
+
+    <xsl:template match="tei:body">
+        <!-- <xsl:variable name="containingNode" select="//tei:text"/> -->
+        <xsl:sequence select="tei:page-content(//tei:milestone[@unit='column'][@n=$pbval1], //tei:milestone[@unit='column'][@n=$pbval2], //tei:body)"/>
+    </xsl:template>
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/breakByMilestone.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_cour.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_cour.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_cour.xsl	(revision 321)
@@ -0,0 +1,583 @@
+<?xml version='1.0'?>
+
+<!--
+                #######################################
+                Qgraal_cm BASIC stylesheet
+                
+                #######################################
+                author: Alexei Lavrentiev
+                
+                version: 3.3
+                date: 2011-06-03
+                (handling agglutinations of "en cours de figement" phrases)
+
+                version: 3.2
+                date: 2011-01-19
+                (handling inline and final notes) 
+                
+                version: 3.1
+                date: 2010-06-17
+                (utf-8 encoding; correcting spacing after "sic : " and "comma" sign)
+
+                version: 3.0
+                date: 2010-01-25
+                (adding table with line numbers and text in different columns)
+                
+                version: 2.0
+                date: 2009-08-24
+                (including multi-facet transcription)
+                
+                version: 1.2
+                date: 2009-02-16
+                
+                version: 1.1
+                date: 2008-09-30
+                
+                version: 1.0
+                date: 2008-02-21
+                
+                #######################################
+                Description
+                This is a basic stylesheet for viewing BFM texts in XML-TEI
+                
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:tei="http://www.tei-c.org/ns/1.0"  xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0"
+                version="1.0"> 
+
+                <xsl:output method = "html" encoding="utf-8" indent="yes"/>
+                
+                <xsl:strip-space elements="*"/>
+                
+                <xsl:variable name="title-source" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='source']"/>
+                
+                <xsl:variable name="title-normal" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='normal']"/>
+                
+                <xsl:variable name="title-ref" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='reference']"/>
+
+                
+                <xsl:variable name="author" select="//tei:titleStmt/tei:author"/>
+                
+                <xsl:variable name="apostrophe" select='"&apos;"'/>
+                
+<!--                <xsl:template match="/">
+                                <html>
+                                                <head>
+                                                                <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
+                                                                <title>
+                                                                                <xsl:if test="$author[not(contains(.,'anonym'))]">
+                                                                                                <xsl:value-of select="$author"/><xsl:text> : </xsl:text>
+                                                                                </xsl:if>
+                                                                                <xsl:value-of select="$title-normal"/>
+                                                                </title>
+                                                                <xsl:apply-templates select="//tei:teiHeader"/>
+                                                </head>
+                                                <body style="margin-left:10%;margin-right:10%">
+                                                                <div class="text">
+                                                                <xsl:apply-templates select="//tei:text"/>
+                                                                </div>
+                                                                <div class="notes">
+                                                                                <p align="center"><b>Notes</b></p>
+                                                                                <xsl:apply-templates select="//tei:note[not(@place='inline')]" mode="notes"/>
+                                                                                </div>
+                                                </body>		
+                                </html>	
+                </xsl:template>
+-->
+                
+                <xsl:template match="/">
+                                <xsl:apply-templates select="//tei:text//tei:p|//tei:note[@place='inline']"/>
+                </xsl:template>
+
+                <!-- TEI-Header Processing -->
+                
+                <xsl:template match="//tei:teiHeader">
+	<p align="center">
+	                
+	                <!-- Author (only appears if not anonymous) -->
+	                <xsl:if test="$author[not(contains(.,'anonym'))]">
+	                                <span style="font-variant:small-caps;font-weight:bold"><xsl:value-of select="$author"/></span><br/>                
+	                </xsl:if>
+	                <!-- Title -->
+	                <span style="color:darkviolet;font-family:Times;font-weight:bold;font-style:italic;font-size:200%"><xsl:value-of select="$title-normal"/></span><br/>
+	                <!-- Scientific editor(s) -->
+	                Edition par <xsl:apply-templates select="//tei:titleStmt/tei:editor[@role='editor']"/><br/>
+	                Avec la collaboration de <xsl:apply-templates select="//tei:titleStmt/tei:editor[@role='associate_editor']"/><br/>
+	                
+	                <!-- Series Statement, Publisher, Year 	
+	                <xsl:apply-templates select="fileDesc/seriesStmt/title"/><br/>
+	                <xsl:apply-templates select="//sourceDesc//publicationStmt/pubPlace"/>,
+	                <xsl:apply-templates select="//sourceDesc//publicationStmt/publisher"/>,
+	                <xsl:apply-templates select="//sourceDesc//publicationStmt/date"/>
+	                
+	                -->
+	</p>
+                                
+                                <!-- Information on the electronic version  -->                
+	<p align="center">_____________________________________________________<br/><br/>
+	<span style="font-size:75%">Edition électronique<br/>
+	                Dernière révision : <xsl:value-of select="//tei:revisionDesc/tei:change[1]/@when"/></span><br/>
+	                _____________________________________________________</p>
+                                <p align="center">Version courante<br/>
+                                _____________________________________________________</p>
+                                
+                </xsl:template>
+
+                <!-- Last change date formatting 
+                <xsl:template match="//revisionDesc/change/date[following-sibling::item[contains(.,'Dernière mise à jour')]]">
+                                <xsl:value-of select="substring-before(., ' 00:')"/>
+                </xsl:template> -->
+                
+                <!-- End of TEI-header processing -->
+                
+                <!-- Text body processing -->
+
+                <!-- "Original" titles found sometimes immediately before text body -->
+                <xsl:template match="//tei:div[@type='titre']">
+                                <p align="center"><b><xsl:apply-templates/></b></p>
+                </xsl:template>
+                
+                <!-- Text divisions (general) -->
+                <xsl:template match="//tei:div[not(@type='titre')]">
+                                <xsl:if test="child::tei:head">
+                                                <xsl:apply-templates select="head"/><br/>                                                
+                                </xsl:if>
+                                <xsl:apply-templates select="*[not(self::tei:head)]"/>                                
+                </xsl:template>
+                
+                <!-- Paragraphs -->
+                <xsl:template match="//tei:text//tei:p">
+                                             <table width="100%" style="border:none">
+                                                             <tr style="border:none">
+                                                                             <td width="10%" valign="top" style="white-space:nowrap;padding-right:10pt;line-height:18px;font-family:Arial;font-size:14px;border-:none;text-align:right">
+                                                                                             <xsl:if test="@n"><span style="color:gray;font-size:12px">[§ <xsl:value-of select="@n"/>] </span></xsl:if><xsl:apply-templates select="descendant::tei:lb|descendant::tei:milestone[@unit='column']" mode="number"/></td>
+                                                                             <td valign="top" style="white-space:nowrap;line-height:18px;font-family:Arial;font-size:14px;border:none">&#xa0;<xsl:apply-templates/></td>
+                                             </tr>
+                                             </table>
+                </xsl:template>
+                
+
+                
+                <!--<xsl:template match="//p[@rend]/lb[1][@n]">
+                                <br/>[<xsl:value-of select="@n"/>]
+                                </xsl:template> -->
+                
+<!-- Page breaks -->                
+              
+                <xsl:template match="//tei:pb[@ed='Pauphilet1923']">
+                                (p. <xsl:value-of select="@n"/>)
+                </xsl:template>
+
+                <xsl:template match="//tei:milestone[@unit='column'][ancestor::tei:p]">
+                                <br/>&#xa0;<br/><span style="color:darkviolet;font-family:arial;font-size:80%">&lt;<xsl:value-of select="substring-after(@xml:id,'col_')"/>&gt;</span><br/>
+                </xsl:template>
+                
+                <xsl:template match="//tei:milestone[@unit='column'][ancestor::tei:p]" mode="number">
+                                <br/>&#xa0;<br/><span style="color:darkviolet;font-family:arial;font-size:80%">&#xa0;</span><br/>
+                </xsl:template>
+                
+                <xsl:template match="//tei:milestone[@unit='column'][not(ancestor::tei:p)]">
+                                <table width="100%">
+                                                <tr>
+                                                                <td width="10%" align="right" valign="top" style="padding-right:10pt"><span style="color:darkviolet;font-family:arial;font-size:80%">&#xa0;</span></td>
+                                                                <td valign="top" style="white-space:nowrap"><span style="color:darkviolet;font-family:arial;font-size:80%">&lt;<xsl:value-of select="substring-after(@xml:id,'col_')"/>&gt;</span></td>
+                                                </tr>
+                                </table>                                
+                </xsl:template>
+                
+
+                <xsl:template match="tei:lb">
+                                <xsl:choose>
+                                                <xsl:when test="@ed='facs'"></xsl:when>
+                                                <xsl:when test="position() = 1 and ancestor::tei:s[not(preceding-sibling::*)]"></xsl:when>
+                                                <xsl:otherwise><br/></xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                
+                <xsl:template match="tei:lb" mode="number">
+                                <xsl:choose>
+                                                <xsl:when test="@ed='facs'"></xsl:when>
+                                                <xsl:when test="position() = 1 and ancestor::tei:s[not(preceding-sibling::*)]"></xsl:when>
+                                                <xsl:otherwise><br/>&#xa0;</xsl:otherwise>
+                                </xsl:choose>
+                                <xsl:if test="substring(@n, string-length(@n)) = 0 or substring(@n, string-length(@n)) = 5"><xsl:value-of select="@n"/></xsl:if>
+                </xsl:template>
+                
+              
+                <!-- Processing elements with "rend" attributes  -->
+                
+                <xsl:template match="//*[@rend='ital']">
+                                <i>
+                                                <xsl:choose>
+                                                                <xsl:when test="self::tei:hi">
+                                                                                <span style="color:blue"><xsl:apply-templates/></span>
+                                                                </xsl:when>
+                                                                <xsl:when test="self::tei:foreign">
+                                                                                <span style="color:darkblue"><xsl:apply-templates/></span>
+                                                                </xsl:when>
+                                                                <xsl:otherwise>
+                                                                                <span style="color:red"><xsl:apply-templates/></span>
+                                                                </xsl:otherwise>
+                                                </xsl:choose>
+                                </i>
+                </xsl:template>
+
+                <xsl:template match="//*[@rend='gras']">
+                                <b>
+                                                <xsl:choose>
+                                                                <xsl:when test="self::tei:hi">
+                                                                                <span style="color:blue"><xsl:apply-templates/></span>
+                                                                </xsl:when>
+                                                                <xsl:otherwise>
+                                                                                <span style="color:red"><xsl:apply-templates/></span>
+                                                                </xsl:otherwise>
+                                                </xsl:choose>
+                                </b>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='exp']">
+                                <sup>
+                                                <xsl:choose>
+                                                                <xsl:when test="self::tei:hi">
+                                                                                <span style="color:blue"><xsl:apply-templates/></span>
+                                                                </xsl:when>
+                                                                <xsl:otherwise>
+                                                                                <span style="color:red"><xsl:apply-templates/></span>
+                                                                </xsl:otherwise>
+                                                </xsl:choose>                                                
+                                </sup>
+                </xsl:template>
+
+                <xsl:template match="//*[@rend='ind']">
+                                <sub>
+                                                <xsl:choose>
+                                                                <xsl:when test="self::tei:hi">
+                                                                                <span style="color:blue"><xsl:apply-templates/></span>
+                                                                </xsl:when>
+                                                                <xsl:otherwise>
+                                                                                <span style="color:red"><xsl:apply-templates/></span>
+                                                                </xsl:otherwise>
+                                                </xsl:choose>                                                
+                                </sub>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='maj']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:hi">
+                                                                <span style="text-transform:uppercase;color:blue"><xsl:apply-templates/></span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="text-transform:uppercase;color:red"><xsl:apply-templates/></span>
+                                                </xsl:otherwise>
+                                </xsl:choose>                                
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='pmaj']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:hi">
+                                                                <span style="font-variant:small-caps;color:blue"><xsl:apply-templates/></span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="font-variant:small-caps;color:red"><xsl:apply-templates/></span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                                
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='crochets']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:corr">
+                                                                <span style="color:brown"> [<xsl:apply-templates/>] </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="color:red"> [<xsl:apply-templates/>] </span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                                
+                </xsl:template>
+
+                <xsl:template match="//*[@rend='chevrons']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:corr">
+                                                                <span style="color:brown"><xsl:text>&lt;</xsl:text><xsl:apply-templates/><xsl:text>&gt; </xsl:text></span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="color:red"><xsl:text>&lt;</xsl:text><xsl:apply-templates/><xsl:text>&gt; </xsl:text></span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                                
+                </xsl:template>
+                                
+                <xsl:template match="//*[@rend='crochets-ital']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:head">
+                                                                <span style="color:indigo">
+                                                                                <xsl:text>[</xsl:text>
+                                                                                <i>
+                                                                                                <xsl:if test="parent::div[@n]">
+                                                                                                                <xsl:value-of select="parent::div/@n"/>
+                                                                                                                <xsl:text>. </xsl:text>
+                                                                                                </xsl:if>
+                                                                                                <xsl:apply-templates/>
+                                                                                </i>
+                                                                                <xsl:text>]</xsl:text>
+                                                                </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="color:red">
+                                                                                <xsl:text> [</xsl:text>
+                                                                                <i>
+                                                                                                <xsl:if test="parent::div[@n]">
+                                                                                                                <xsl:value-of select="parent::div/@n"/>
+                                                                                                                <xsl:text>. </xsl:text>
+                                                                                                </xsl:if>
+                                                                                                <xsl:apply-templates/>
+                                                                                </i>
+                                                                                <xsl:text>] </xsl:text>
+                                                                </span>                                                               
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                                
+                <xsl:template match="//*[@rend='crochets-maj']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:head">
+                                                                <span style="font-family:Times;text-transform:uppercase;color:indigo">
+                                                                                <xsl:text>[</xsl:text>
+                                                                                <xsl:if test="parent::div[@n]">
+                                                                                                <xsl:value-of select="parent::div/@n"/>
+                                                                                                <xsl:text>. </xsl:text>
+                                                                                </xsl:if>
+                                                                                <xsl:apply-templates/>
+                                                                                <xsl:text>]</xsl:text>
+                                                                </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="text-transform:uppercase;color:red"> [<xsl:apply-templates/>] </span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='crochets-pmaj']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:head">
+                                                                <span style="font-family:Times;font-variant:small-caps;color:indigo">
+                                                                                <xsl:text>[</xsl:text>
+                                                                                <xsl:if test="parent::tei:div[@n]">
+                                                                                                <xsl:value-of select="parent::tei:div/@n"/>
+                                                                                                <xsl:text>. </xsl:text>
+                                                                                </xsl:if>
+                                                                                <xsl:apply-templates/>
+                                                                                <xsl:text>]</xsl:text>
+                                                                </span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="font-variant:small-caps;color:red"> [<xsl:apply-templates/>] </span>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='points']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:gap">
+                                                                <span style="color:brown"><xsl:text> . . . . . . . . .  </xsl:text></span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="color:red"><xsl:text> . . . . . . . . .  </xsl:text></span>
+                                                </xsl:otherwise>
+                                </xsl:choose>                                
+                </xsl:template>
+                
+                <xsl:template match="//*[@rend='susp']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:gap">
+                                                                <span style="color:brown"><xsl:text> ... </xsl:text></span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="color:red"><xsl:text> ... </xsl:text></span>
+                                                </xsl:otherwise>
+                                </xsl:choose>                                
+                </xsl:template>
+
+                <xsl:template match="//*[@rend='crochets-susp']">
+                                <xsl:choose>
+                                                <xsl:when test="self::tei:gap">
+                                                                <span style="color:brown"><xsl:text> [...] </xsl:text></span>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <span style="color:red"><xsl:text> [...] </xsl:text></span>
+                                                </xsl:otherwise>
+                                </xsl:choose>                                
+                </xsl:template>                
+                <!-- End of @rend processing -->
+
+                <!-- Gaps without normalized @rend -->
+                <xsl:template match="tei:gap[not(@rend)]">
+                                <span style="color:darkblue"><xsl:text> [...] </xsl:text></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:gap[@rend][not(@rend='susp' or @rend='points' or @rend='crochets-susp')]">
+                                <span style="color:orange"><xsl:text> [...] </xsl:text></span>
+                </xsl:template>
+                
+                <!-- Corrections without normalized @rend -->
+                
+                <xsl:template match="tei:choice">
+                                <span title="{child::tei:corr/@type}"><xsl:apply-templates select="tei:corr"/><xsl:text> </xsl:text><xsl:apply-templates select="tei:sic"/></span>
+                                <xsl:if test="not(ancestor::tei:w)">
+                                                <xsl:call-template name="spacing"/>
+                                </xsl:if>
+                </xsl:template>
+                
+                <xsl:template match="tei:corr[not(@rend)]">
+                                <span style="color:darkblue;background-color:yellow"><xsl:text> </xsl:text><xsl:apply-templates/><xsl:text> </xsl:text></span>
+                </xsl:template>
+                
+<!--                <xsl:template match="tei:corr[@rend][not(@rend='crochets' or @rend='chevrons')]">
+                                <span style="color:orange"> <i><xsl:value-of select="@rend"/></i><xsl:text> </xsl:text></span><xsl:apply-templates/>
+                </xsl:template>
+-->                
+                <xsl:template match="tei:sic">(<i>sic&#xa0;:</i><xsl:text> </xsl:text><span style="background-color:pink"><xsl:apply-templates/></span>)</xsl:template>
+                <!--  Supplied text -->
+                
+                <xsl:template match="tei:supplied">
+                                <span style="color:blue">[<xsl:apply-templates/>]</span>
+                                <xsl:if test="not(ancestor::tei:w)"><xsl:call-template name="spacing"/></xsl:if>
+                </xsl:template>
+                
+                <!-- Deletions -->
+                
+                <xsl:template match="tei:del">
+                                <span style="background-color:lightpink"><xsl:apply-templates/></span>
+                </xsl:template>
+
+                <xsl:template match="tei:add">
+                                <span style="background-color:lightgreen"><xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <!-- Redundant text -->
+                
+                <xsl:template match="tei:surplus">(<span style="background-color:lightpink"><xsl:apply-templates/></span><xsl:text> </xsl:text><i>répété</i>)<xsl:call-template name="spacing"></xsl:call-template></xsl:template>
+                
+                
+                <!-- Numbers -->
+                <xsl:template match="tei:num">
+                                <span style="color:blue"><xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <!-- Notes -->
+                <xsl:template match="tei:note[not(@place='inline')]">
+                                <xsl:variable name="note_count"><xsl:value-of select="count(preceding::tei:note[not(@place='inline')]) + 1"/></xsl:variable>
+                                <!--<a title="{.}" style="color:violet;font-size:75%" href="#note_{$note_count}" name="noteref_{$note_count}">[Note <xsl:value-of select="$note_count"/>]</a>-->
+                                <span title="{.}" style="color:violet">[<xsl:value-of select="$note_count"/>]</span>
+                                <xsl:call-template name="spacing"/>
+                </xsl:template>
+                
+                <xsl:template match="tei:note[not(@place='inline')]" mode="notes">
+                                <xsl:variable name="note_count"><xsl:value-of select="count(preceding::tei:note[not(@place='inline')]) + 1"/></xsl:variable>
+                                <p><xsl:value-of select="$note_count"/>. <a href="#noteref_{$note_count}" name="note_{$note_count}">[<xsl:value-of select="preceding::tei:milestone[@unit='column'][1]/@xml:id"/>, l. <xsl:value-of select="preceding::tei:lb[1]/@n"/>]</a><xsl:text> </xsl:text> <xsl:value-of select="."/></p>
+                </xsl:template>
+                
+                <xsl:template match="tei:note[@place='inline']">
+                                <p style="text-align:justify;text-indent:30px;padding-right:20px;padding-left:20px"><span style="color:blue;font-family:Times New Roman;font-style:italic"><xsl:value-of select="."/></span></p>
+                </xsl:template>
+                
+                <!-- Word formatting for morphologically tagged texts -->
+                <xsl:strip-space elements="tei:s"/>
+                <xsl:template match="//text()[starts-with(., ' ')][preceding-sibling::*[1][self::tei:w]]">
+                                <xsl:copy-of select="substring(., 2)"/>
+                </xsl:template>
+                
+                <xsl:template match="//tei:w"> <!-- modified 2011-06-03 -->                               
+                                <xsl:element name="span">
+                                                <xsl:attribute name="class">word</xsl:attribute>
+                                                <xsl:attribute name="title"><xsl:value-of select="@type"/></xsl:attribute>
+                                                <xsl:attribute name="id"><xsl:value-of select="@xml:id"/></xsl:attribute>
+                                                <xsl:if test="@rend='fgmt'">
+                                                                <xsl:attribute name="style">color:red</xsl:attribute>
+                                                </xsl:if>
+                                                <xsl:choose>
+                                                                <xsl:when test="descendant::me:norm">
+                                                                                <xsl:apply-templates select="descendant::me:norm"/>
+                                                                </xsl:when>
+                                                                <xsl:otherwise>
+                                                                                <xsl:apply-templates/>
+                                                                </xsl:otherwise>
+                                                </xsl:choose>
+                                                
+                                </xsl:element>
+                                <xsl:call-template name="spacing"/>
+                                <!--<xsl:choose>
+                                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][contains(., ',') or contains(., '.') or contains(., ']')or contains(., ')')]">
+                                                </xsl:when>
+                                                <xsl:when test="following::node()[1][self::text()][starts-with(normalize-space(.), ',') or starts-with(normalize-space(.), '.') or starts-with(normalize-space(.), ']') or starts-with(normalize-space(.), ')')]">
+                                                </xsl:when>
+                                                <xsl:when test="self::tei:w[starts-with(@type, 'PON')][contains(., '[') or contains(., '(')]">
+                                                </xsl:when>
+                                                <xsl:when test="substring(., string-length(.))=$apostrophe">
+                                                </xsl:when>
+                                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][contains(., ':') or contains(., ';') or contains(., '!')or contains(., '?')]">
+                                                                <xsl:text>&#xa0;</xsl:text>
+                                                </xsl:when>
+                                                <xsl:when test="@rend='elision'">&apos;</xsl:when>
+                                                <xsl:otherwise>
+                                                                <xsl:text> </xsl:text>
+                                                </xsl:otherwise>
+                                </xsl:choose>-->
+                </xsl:template>
+                
+                <xsl:template match="//tei:w[not(starts-with(@type,'PON'))]/text()">
+                                <xsl:choose>
+                                                <xsl:when test="substring(., string-length(.))=$apostrophe">
+                                                                <xsl:value-of select="translate(.,$apostrophe,'’')"/>
+                                                </xsl:when>
+                                                <xsl:otherwise>
+                                                                <xsl:value-of select="."/>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+<xsl:template name="spacing">
+                <xsl:choose>
+                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][starts-with(normalize-space(descendant::text()[1]), ',') or starts-with(normalize-space(descendant::text()[1]), '.') or starts-with(normalize-space(descendant::text()[1]), ']')or contains(., ')')]">
+                                </xsl:when>
+                                <xsl:when test="following::node()[1][self::text()][starts-with(normalize-space(.), ',') or starts-with(normalize-space(.), '.') or starts-with(normalize-space(.), ']') or starts-with(normalize-space(.), ')')]">
+                                </xsl:when>
+                                <xsl:when test="self::tei:w[starts-with(@type, 'PON')][starts-with(normalize-space(descendant::text()[1]), '[') or starts-with(normalize-space(descendant::text()[1]), '(')]">
+                                </xsl:when>
+                                <xsl:when test="substring(., string-length(.))=$apostrophe and not(@type='PONfbl' or @type='PONpdr' or self::tei:note)">
+                                </xsl:when>
+                                <xsl:when test="position()=last() and (ancestor::tei:choice or ancestor::tei:supplied[not(@rend='multi_s')] or ancestor::tei:surplus)"></xsl:when>
+                                <xsl:when test="following-sibling::*[1][self::tei:note]"></xsl:when>
+                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][contains(., ':') or contains(., ';') or contains(., '!') or contains(., '?')]">
+                                                <xsl:text>&#xa0;</xsl:text>
+                                </xsl:when>
+                                <!--
+                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][contains(descendant::me:norm, ':') or contains(descendant::me:norm, ';') or contains(descendant::me:norm, '!')or contains(descendant::me:norm, '?')]">
+                                                <xsl:text>&#xa0;</xsl:text>
+                                </xsl:when>                                
+                                -->
+                                <xsl:when test="@rend='elision'">’</xsl:when>
+                                <!-- new 201106-03 -->
+                                <xsl:when test="@rend='fgmt'"></xsl:when>
+                                <xsl:otherwise>
+                                                <xsl:text> </xsl:text>
+                                </xsl:otherwise>
+                </xsl:choose>
+                
+</xsl:template>
+
+                <xsl:template match="tei:q[ancestor::tei:q[ancestor::tei:q]]">
+                                <span style="background-color:darkcyan"> <xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:q[ancestor::tei:q][not(ancestor::tei:q[ancestor::tei:q])]">
+                                <span style="background-color:cyan"> <xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:q[not(ancestor::tei:q)]">
+                                <span style="background-color:lightcyan"> <xsl:apply-templates/></span>
+                </xsl:template>
+                
+                <xsl:template match="bfm:sb"><span style="color:violet">_</span></xsl:template>
+                                
+</xsl:stylesheet>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_cour.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_dipl.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_dipl.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_dipl.xsl	(revision 321)
@@ -0,0 +1,407 @@
+<?xml version='1.0'?>
+
+<!-- This is a stylesheet that gives a "diplomatic" presentation of old French manuscript transcriptions
+	- abbreviations are resolved and rendered in italics;
+	- agglutinations are marked with a red +
+	- deglutinations are marked with e red _
+                                                          - Strong punctuation is marked with a dot followed by a capital letter;
+                                                             weak punctuation is marked with a comma.
+	- direct speech is marked with darkblue color;
+	- uncertain readings are marked with grey background color;
+	- text deleted in the original is not visualized;
+	- text corrected in the normalized version is marked with darkred
+                                                             color and followed by "(sic!)" note;
+                                                          - notes are rendered with [*] sign in the text body; the text of the note
+                                                            appears on "mouse-over" and is reproduced in the end.
+-->
+
+
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+                <xsl:import href="mdv_common.xsl"/>
+                <xsl:param name="form">diplomatique</xsl:param>
+                <xsl:variable name="mode">dipl</xsl:variable>
+                
+                <xsl:output method="html"  encoding="utf-8" indent="yes"/>
+                
+                <xsl:strip-space elements="*"/>
+                
+                <!--                <xsl:variable name="title-formal" select="//fileDesc/titleStmt/title[@type='formal']"/>
+                                
+                                <xsl:variable name="title-normal" select="//fileDesc/titleStmt/title[@type='normal']"/>
+                                
+                                <xsl:variable name="title-ref" select="//fileDesc/titleStmt/title[@type='reference']"/>
+                                
+                                <xsl:variable name="author" select="//titleStmt/author"/>
+                -->
+                
+<!--                <xsl:template match="//tei:p|//tei:head">
+                                <p>
+                                                <xsl:if test="@n">
+                                                                [ § <xsl:value-of select="@n"/>]&#xa0;
+                                                </xsl:if>
+                                                <xsl:apply-templates mode="dipl"/>
+                                </p>
+                </xsl:template>
+-->                
+
+                <xsl:variable name="title-source" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='source']"/>
+                
+                <xsl:variable name="title-normal" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='normal']"/>
+                
+                <xsl:variable name="title-ref" select="//tei:fileDesc/tei:titleStmt/tei:title[@type='reference']"/>
+                
+                
+                <xsl:variable name="author" select="//tei:titleStmt/tei:author"/>
+                
+                <xsl:variable name="apostrophe" select='"&apos;"'/>
+                
+                
+<!--                <xsl:template match="/">
+                                <html>
+                                                <head>
+                                                                <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
+                                                                <title>
+                                                                                <xsl:if test="$author[not(contains(.,'anonym'))]">
+                                                                                                <xsl:value-of select="$author"/><xsl:text> : </xsl:text>
+                                                                                </xsl:if>
+                                                                                <xsl:value-of select="$title-normal"/>
+                                                                </title>
+                                                                <xsl:apply-templates select="//tei:teiHeader"/>
+                                                </head>
+                                                <body style="margin-left:10%;margin-right:10%">
+                                                                <div class="text">
+                                                                                <xsl:apply-templates select="//tei:text"/>
+                                                                </div>
+                                                                <div class="notes">
+                                                                                <p align="center"><b>Notes</b></p>
+                                                                                <xsl:apply-templates select="//tei:note[not(@place='inline')]" mode="notes"/>
+                                                                </div>			
+                                                </body>		
+                                </html>	
+                </xsl:template>
+-->                
+                <xsl:template match="/">
+                                <xsl:apply-templates select="//tei:note[@place='inline']" mode="dipl"/>
+                                <xsl:apply-templates select="//tei:text//tei:p|//tei:text//tei:head"/>                                
+                </xsl:template>
+
+
+                
+                <!-- TEI-Header Processing -->
+                
+                <xsl:template match="//tei:teiHeader">
+                                <p align="center">
+                                                
+                                                <!-- Author (only appears if not anonymous) -->
+                                                <xsl:if test="$author[not(contains(.,'anonym'))]">
+                                                                <span style="font-variant:small-caps;font-weight:bold"><xsl:value-of select="$author"/></span><br/>                
+                                                </xsl:if>
+                                                <!-- Title -->
+                                                <span style="color:darkviolet;font-family:Times;font-weight:bold;font-style:italic;font-size:200%"><xsl:value-of select="$title-normal"/></span><br/>
+                                                <!-- Scientific editor(s) -->
+                                                Edition par <xsl:apply-templates select="//tei:titleStmt/tei:editor[@role='editor']"/><br/>
+                                                Avec la collaboration de <xsl:apply-templates select="//tei:titleStmt/tei:editor[@role='associate_editor']"/><br/>
+                                                
+                                                <!-- Series Statement, Publisher, Year 	
+                                                                <xsl:apply-templates select="fileDesc/seriesStmt/title"/><br/>
+                                                                <xsl:apply-templates select="//sourceDesc//publicationStmt/pubPlace"/>,
+                                                                <xsl:apply-templates select="//sourceDesc//publicationStmt/publisher"/>,
+                                                                <xsl:apply-templates select="//sourceDesc//publicationStmt/date"/>
+                                                                
+                                                -->
+                                </p>
+                                
+                                <!-- Information on the electronic version  -->                
+                                <p align="center">_____________________________________________________<br/><br/>
+                                                <span style="font-size:75%">Edition électronique<br/>
+                                                                Dernière révision : <xsl:value-of select="//tei:revisionDesc/tei:change[1]/@when"/></span><br/>
+                                                                _____________________________________________________</p>
+                                <p align="center">Version diplomatique
+                                                <br/>_____________________________________________________</p>
+                </xsl:template>
+                              
+                <!-- Gaps without normalized @rend -->
+                <xsl:template match="tei:gap[not(@rend)]" mode="dipl">
+                                <span style="color:darkblue"><xsl:text> [...] </xsl:text></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:gap[@rend][not(@rend='susp' or @rend='points' or @rend='crochets-susp')]" mode="dipl">
+                                <span style="color:orange"><xsl:text> [...] </xsl:text></span>
+                </xsl:template>
+                
+                <!-- Corrections without normalized @rend -->
+                
+                <xsl:template match="tei:choice" mode="dipl">
+                                <span title="{child::tei:corr/@type}"><xsl:apply-templates select="tei:corr" mode="dipl"/><xsl:text> </xsl:text><xsl:apply-templates select="tei:sic" mode="dipl"/></span>
+                                <xsl:if test="not(ancestor::tei:w)">
+                                                <xsl:call-template name="spacing"/>
+                                </xsl:if>
+                </xsl:template>
+                
+                <xsl:template match="tei:corr[not(@rend)]" mode="dipl">
+                                <span style="color:darkblue;background-color:yellow"><xsl:text> </xsl:text><xsl:apply-templates mode="dipl"/><xsl:text> </xsl:text></span>
+                </xsl:template>
+                
+                <!--                <xsl:template match="tei:corr[@rend][not(@rend='crochets' or @rend='chevrons')]">
+                                <span style="color:orange"> <i><xsl:value-of select="@rend"/></i><xsl:text> </xsl:text></span><xsl:apply-templates/>
+                                </xsl:template>
+                -->                
+                <xsl:template match="tei:sic" mode="dipl">(<i>sic&#xa0;:</i><xsl:text> </xsl:text><span style="background-color:pink"><xsl:apply-templates mode="dipl"/></span>)</xsl:template>
+                
+                              
+                <!--  Supplied text -->
+                
+                <xsl:template match="tei:supplied" mode="dipl">
+                                <span style="color:blue">[<xsl:apply-templates mode="dipl"/>]</span>
+                                <xsl:if test="not(ancestor::tei:w)"><xsl:call-template name="spacing"/></xsl:if>
+                </xsl:template>
+                
+                <!-- Deletions -->
+                
+                <xsl:template match="tei:del" mode="dipl">
+                                <span style="background-color:lightpink"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:add" mode="dipl">
+                                <span style="background-color:lightgreen"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <!-- Redundant text -->
+                
+                <xsl:template match="tei:surplus" mode="dipl">(<span style="background-color:lightpink"><xsl:apply-templates mode="dipl"/></span><xsl:text> </xsl:text><i>répété</i>)<xsl:call-template name="spacing"></xsl:call-template></xsl:template>
+                
+                <!-- Numbers -->
+                <xsl:template match="tei:num" mode="dipl">
+                                <span style="color:blue"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                                              
+                              
+                <!-- NOTES -->
+                
+                <xsl:template match="tei:note[not(@place='inline')]" mode="dipl">
+                                <xsl:variable name="note_count"><xsl:value-of select="count(preceding::tei:note[not(@place='inline')]) + 1"/></xsl:variable>
+                                <!--<a title="{.}" style="color:violet;font-size:75%" href="#note_{$note_count}" name="noteref_{$note_count}">[Note <xsl:value-of select="$note_count"/>]</a>-->
+                                <span title="{.}" style="color:violet">[<xsl:value-of select="$note_count"/>]</span>
+                                <xsl:call-template name="spacing"/>
+                </xsl:template>
+                
+                <xsl:template match="tei:note[not(@place='inline')]" mode="notes">
+                                <xsl:variable name="note_count"><xsl:value-of select="count(preceding::tei:note[not(@place='inline')]) + 1"/></xsl:variable>
+                                <p><xsl:value-of select="$note_count"/>. <a href="#noteref_{$note_count}" name="note_{$note_count}">[<xsl:value-of select="preceding::tei:milestone[@unit='column'][1]/@xml:id"/>, l. <xsl:value-of select="preceding::tei:lb[1]/@n"/>]</a><xsl:text> </xsl:text> <xsl:value-of select="."/></p>
+                </xsl:template>
+                <xsl:template match="tei:note[@place='inline']" mode="dipl">
+                                <p style="text-align:justify;text-indent:30px;padding-right:20px;padding-left:20px"><span style="color:blue;font-family:Times;font-style:italic"><xsl:value-of select="."/></span></p>
+                </xsl:template>
+                
+                
+                
+                <xsl:template match="//tei:text//tei:p|//tei:text//tei:head">
+                                <table width="100%" style="border:none">
+                                                <tr>
+                                                                <td width="10%" valign="top" style="white-space:nowrap;padding-right:10pt;line-height:18px;font-family:Arial;font-size:14px;border:none;text-align:right">
+                                                                                <xsl:if test="@n"><span style="color:gray;font-size:12px">[§ <xsl:value-of select="@n"/>] </span></xsl:if><xsl:apply-templates select="descendant::tei:lb|descendant::tei:milestone[@unit='column']|descendant::tei:hi[@rend='initiale']" mode="number"/></td>
+                                                                <td valign="top" style="white-space:nowrap;line-height:18px;font-family:Arial;font-size:14px;border:none">&#xa0;<xsl:apply-templates mode="dipl"/></td>
+                                                </tr>
+                                </table>
+                </xsl:template>
+                
+                
+<!--                <xsl:template match="tei:pb" mode="dipl">
+                                <xsl:call-template name="pb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:cb" mode="dipl">
+                                <xsl:call-template name="cb_common"></xsl:call-template>
+                </xsl:template>
+                
+                <xsl:template match="tei:note" mode="dipl">
+                                <xsl:call-template name="note_common"></xsl:call-template>
+                </xsl:template>
+
+                <xsl:template match="tei:q" mode="dipl">
+                                <span style="background-color:lightcyan"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:gap" mode="dipl">
+                                <xsl:call-template name="gap_common"></xsl:call-template>
+                </xsl:template>
+-->                                
+      
+                <!-- Page breaks -->                
+                
+                <xsl:template match="//tei:pb[@ed='Pauphilet1923']" mode="dipl">
+                                (p. <xsl:value-of select="@n"/>)
+                </xsl:template>
+                
+                <xsl:template match="//tei:milestone[@unit='column'][ancestor::tei:p]" mode="dipl">
+                                <br/><br/><span style="color:darkviolet;font-family:arial;font-size:80%">&lt;<xsl:value-of select="substring-after(@xml:id,'col_')"/>&gt;</span><br/>
+                </xsl:template>
+                
+                <xsl:template match="//tei:milestone[@unit='column'][ancestor::tei:p]" mode="number">
+                                <br/>&#xa0;<br/><span style="color:darkviolet;font-family:arial;font-size:80%">&#xa0;</span><br/>
+                </xsl:template>
+                
+                <xsl:template match="//tei:milestone[@unit='column'][not(ancestor::tei:p)]">
+                                <table width="100%">
+                                                <tr>
+                                                                <td width="10%" align="right" valign="top" style="padding-right:10pt"><span style="color:darkviolet;font-family:arial;font-size:80%">&#xa0;</span></td>
+                                                                <td valign="top" style="white-space:nowrap"><span style="color:darkviolet;font-family:arial;font-size:80%">&lt;<xsl:value-of select="substring-after(@xml:id,'col_')"/>&gt;</span></td>
+                                                </tr>
+                                </table>                                
+                </xsl:template>
+                
+      
+                <xsl:template match="tei:lb" mode="dipl">
+                                <xsl:choose>
+                                                <xsl:when test="@ed='facs'"><xsl:text>-</xsl:text><br/></xsl:when>
+                                                <xsl:when test="@ed='norm' or preceding::tei:lb[1][@ed='facs']"></xsl:when>
+                                                <xsl:when test="position() = 1 and ancestor::tei:s[not(preceding-sibling::*)]"></xsl:when>
+                                                <xsl:otherwise><br/></xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                
+                <xsl:template match="tei:lb" mode="number">
+                                <xsl:choose>
+                                                <xsl:when test="@ed='facs'"></xsl:when>
+                                                <xsl:when test="position() = 1 and ancestor::tei:s[not(preceding-sibling::*)]"></xsl:when>
+                                                <xsl:otherwise><br/>&#xa0;</xsl:otherwise>
+                                </xsl:choose>
+                                <xsl:if test="substring(@n, string-length(@n)) = 0 or substring(@n, string-length(@n)) = 5"><xsl:value-of select="@n"/></xsl:if>
+                </xsl:template>
+                                
+                
+                <!--<xsl:template match="//tei:lb[@ed='norm']" mode="dipl">                                
+                </xsl:template>
+                
+                <xsl:template match="//tei:lb[not(@ed)]" mode="dipl">
+                                <xsl:choose>
+                                                <xsl:when test="position()=1 and not(ancestor::tei:seg)"></xsl:when>
+                                                <xsl:when test="preceding::tei:w[1][descendant::bfm:headlb]"></xsl:when>
+                                                <xsl:when test="preceding::tei:w[1][descendant::tei:lb]"></xsl:when>
+                                                <xsl:otherwise><br/></xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                
+                <xsl:template match="//tei:lb[@ed='facs']" mode="dipl">
+                                <xsl:text>-</xsl:text><br/>
+                </xsl:template>-->
+                
+                
+                
+                               
+                <xsl:template match="//tei:w" mode="dipl">
+                                <span title="{@type}" class="word" id="{@xml:id}" >
+                                                <xsl:if test="@rend='fgmt'"><xsl:attribute name="style">color:red</xsl:attribute></xsl:if>
+                                                <xsl:if test="descendant::me:dipl">
+                                                                <xsl:apply-templates select="descendant::me:dipl" mode="dipl"/>
+                                                                <xsl:if test="descendant::tei:sic and not(following-sibling::tei:w[1][descendant::tei:sic])">
+                                                                                <i><xsl:text> (sic !)</xsl:text></i>
+                                                                </xsl:if>
+                                                </xsl:if>
+                                                <xsl:call-template name="spacing"/>
+                                </span>
+                </xsl:template>
+                
+<xsl:template name="spacing">
+                <xsl:choose>
+                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][starts-with(normalize-space(descendant::text()[1]), ',') or starts-with(normalize-space(descendant::text()[1]), '.') or starts-with(normalize-space(descendant::text()[1]), ']')or contains(., ')')]">
+                                </xsl:when>
+                                <xsl:when test="following::node()[1][self::text()][starts-with(normalize-space(.), ',') or starts-with(normalize-space(.), '.') or starts-with(normalize-space(.), ']') or starts-with(normalize-space(.), ')')]">
+                                </xsl:when>
+                                <xsl:when test="self::tei:w[starts-with(@type, 'PON')][starts-with(normalize-space(descendant::text()[1]), '[') or starts-with(normalize-space(descendant::text()[1]), '(')]">
+                                </xsl:when>
+                                <xsl:when test="substring(., string-length(.))=$apostrophe and not(@type='PONfbl' or @type='PONpdr' or self::tei:note)">
+                                </xsl:when>
+                                <xsl:when test="position()=last() and (ancestor::tei:choice or ancestor::tei:supplied[not(@rend='multi_s')] or ancestor::tei:surplus)"></xsl:when>
+                                <xsl:when test="following-sibling::*[1][self::tei:note]"></xsl:when>
+                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][contains(., ':') or contains(., ';') or contains(., '!') or contains(., '?')]">
+                                                <xsl:text>&#xa0;</xsl:text>
+                                </xsl:when>
+                                <!--
+                                <xsl:when test="following::tei:w[1][starts-with(@type, 'PON')][contains(descendant::me:norm, ':') or contains(descendant::me:norm, ';') or contains(descendant::me:norm, '!')or contains(descendant::me:norm, '?')]">
+                                                <xsl:text>&#xa0;</xsl:text>
+                                </xsl:when>                                
+                                -->
+                                <xsl:when test="@rend='elision'">’</xsl:when>
+                                <!-- new 201106-03 -->
+                                <xsl:when test="@rend='fgmt'"></xsl:when>
+                                <xsl:otherwise>
+                                                <xsl:text> </xsl:text>
+                                </xsl:otherwise>
+                </xsl:choose>
+                
+</xsl:template>
+
+                
+                
+                <xsl:template match="tei:ex" mode="dipl">
+                                <xsl:choose>
+                                                <xsl:when test="@cert='high'"><span style="color:darkgreen;font-style:italic"><xsl:apply-templates mode="dipl"/></span></xsl:when>
+                                                <xsl:otherwise><span style="color:darkred;font-style:italic"><xsl:apply-templates mode="dipl"/></span></xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+                
+                <!-- added degl -->
+                <xsl:template match="bfm:sb" mode="dipl"><span style="color:red">_</span></xsl:template>
+                
+<!--                <xsl:template match="tei:sic" mode="dipl">
+                                <span style="color:darkred" title="sic!"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:subst" mode="dipl">[<xsl:apply-templates mode="dipl"/>]</xsl:template>
+                
+                <xsl:template match="tei:del" mode="dipl">
+                                <span style="color:red;text-decoration:line-through"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:add[not(@seq)]" mode="dipl">
+                                <span style="color:blue;text-decoration:underline"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+-->                
+                <xsl:template match="tei:add[@seq]" mode="dipl">
+                                <span style="color:blue"><sup>&lt;<xsl:value-of select="@seq"/></sup><xsl:apply-templates mode="dipl"/><sup><xsl:value-of select="@seq"/>&gt;</sup></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:unclear" mode="dipl">
+                                <span style="background-color:lightgrey"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                <!-- font-size:xx-large enlevé -->
+                <xsl:template match="tei:hi[@rend='initiale']" mode="dipl">
+                                <span style="font-weight:bold;font-size:large"><xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+
+                <xsl:template match="tei:hi[@rend='initiale']" mode="number">
+                                <span style="font-weight:bold;font-size:large">&#xa0;</span>
+                </xsl:template>
+                
+                <xsl:template match="//tei:hi[@rend='sup']" mode="dipl">
+                                <sup><xsl:apply-templates mode="dipl"/></sup>
+                </xsl:template>
+                
+                <!-- Discours direct -->
+                                <xsl:template match="tei:q[ancestor::tei:q[ancestor::tei:q]]" mode="dipl">
+                                <span style="background-color:darkcyan"> <xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:q[ancestor::tei:q][not(ancestor::tei:q[ancestor::tei:q])]" mode="dipl">
+                                <span style="background-color:cyan"> <xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <xsl:template match="tei:q[not(ancestor::tei:q)]" mode="dipl">
+                                <span style="background-color:lightcyan"> <xsl:apply-templates mode="dipl"/></span>
+                </xsl:template>
+                
+                <!--<xsl:template match="bfm:sb" mode="dipl"><span style="color:violet">_</span></xsl:template>-->
+
+                
+                
+                <!-- 
+
+
+<xsl:template match="tei:name">
+	<xsl:value-of select="@reg"/>
+</xsl:template>
+
+                -->
+</xsl:stylesheet>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/qgraal_dipl.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/test.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/test.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/test.xsl	(revision 321)
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0" xmlns:tei="http://www.tei-c.org/ns/1.0"> <xsl:output encoding="UTF-8" indent="yes" method="html"/>
+    <xsl:output method="html" encoding="UTF-8" version="4.0" indent="yes"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:template match="/">
+        <xsl:apply-templates select="//tei:text"/>
+    </xsl:template>
+
+
+    <xsl:template match="//tei:body/tei:div">
+        <br/>
+        <span style="font-family:Times;color:indigo">[<i>
+                <xsl:value-of select="@type"/>
+            </i>&#xa0;<xsl:value-of select="@n"/>]</span>
+        <xsl:apply-templates/>
+    </xsl:template>
+
+    <xsl:template match="//tei:body/tei:div/tei:div">
+        <br/>
+        <span style="font-family:Times;color:indigo">[<i>
+                <xsl:value-of select="@type"/>
+            </i>&#xa0;<xsl:value-of select="@n"/>]</span>
+        <br/>
+        <xsl:apply-templates/>
+    </xsl:template>
+
+    <xsl:template match="tei:head">
+        <span style="font-family:Arial; color:darkred">
+            <xsl:apply-templates/>
+        </span>
+        <br/>
+    </xsl:template>
+
+    <xsl:template match="//tei:p">
+        <p>
+            <xsl:if test="@n"> [ § <xsl:value-of select="@n"/>]&#xa0; </xsl:if>
+            <xsl:apply-templates/>
+        </p>
+    </xsl:template>
+
+    <xsl:template match="//tei:lb">
+        <xsl:choose>
+            <xsl:when test="position()=1 and not(ancestor::tei:seg)"/>
+            <xsl:otherwise>
+                <br/>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+
+    <xsl:template match="tei:q" mode="norm">
+        <span style="background-color:khaki">
+            <xsl:apply-templates/>
+        </span>
+    </xsl:template>
+    <xsl:template match="//tei:w">
+        <span class="word" id="{@xml:id}" title="{@type}">
+            <xsl:choose>
+                <xsl:when test="@type[starts-with(., 'PON')]">
+                    <xsl:apply-templates/>
+                </xsl:when>
+                <xsl:otherwise> &#xa0;<xsl:apply-templates/>
+                </xsl:otherwise>
+            </xsl:choose>
+        </span>
+    </xsl:template>
+    <xsl:template match="tei:supplied"> [<xsl:apply-templates/>] </xsl:template>
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/test.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_norm.xsl
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_norm.xsl	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_norm.xsl	(revision 321)
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- This is a stylesheet that gives a "normalized" presentation of old French manuscript transcriptions
+	- abbreviations are resolved;
+	- agglutinations and deglutinations are not marked (word limits 
+	  are "normalized");
+                                                           - punctuation is normalized as it is in the edition; 
+	- direct speech is marked with quotation marks and darkblue color;
+	- uncertain readings are not marked;
+	- text deleted in the original is not visualized;
+	- corrections are marked with darkred color;
+                                                           - notes are rendered with [*] sign in the text body; the text of the note
+                                                             appears on "mouse-over" and is reproduced in the end.
+-->
+
+
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0" xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns:me="http://www.menota.org/ns/1.0" xmlns:bfm="http://bfm.ens-lsh.fr/ns/1.0">
+                <xsl:param name="form">normalisée</xsl:param>
+
+                <xsl:variable name="mode">facs</xsl:variable>
+
+                <xsl:output method="html" encoding="UTF-8" version="4.0" indent="yes"/>
+
+                <xsl:strip-space elements="*"/>
+
+                <!--            <xsl:variable name="title-formal" select="//fileDesc/titleStmt/title[@type='formal']"/>
+                
+                <xsl:variable name="title-normal" select="//fileDesc/titleStmt/title[@type='normal']"/>
+                
+                <xsl:variable name="title-ref" select="//fileDesc/titleStmt/title[@type='reference']"/>
+                
+                <xsl:variable name="author" select="//titleStmt/author"/>
+ -->
+
+                <xsl:include href="mdv_common.xsl"/>
+
+                <xsl:template match="//tei:p|//tei:head">
+                                <p>
+                                                <xsl:if test="@n"> [ § <xsl:value-of select="@n"
+                                                  />]&#xa0; </xsl:if>
+                                                <xsl:apply-templates mode="norm"/>
+                                </p>
+                </xsl:template>
+
+                <xsl:template match="tei:pb" mode="norm">
+                                <xsl:call-template name="pb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:cb" mode="norm">
+                                <xsl:call-template name="cb_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:note" mode="norm">
+                                <xsl:call-template name="note_common"/>
+                </xsl:template>
+
+                <xsl:template match="tei:q" mode="norm">
+                                <span style="background-color:khaki">
+                                                <xsl:apply-templates mode="norm"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:gap" mode="norm">
+                                <xsl:call-template name="gap_common"/>
+                </xsl:template>
+
+                <xsl:template match="//tei:lb" mode="norm">
+                                <xsl:choose>
+                                                <xsl:when
+                                                  test="position()=1 and not(ancestor::tei:seg)"/>
+                                                <xsl:otherwise>
+                                                  <br/>
+                                                </xsl:otherwise>
+                                </xsl:choose>
+                </xsl:template>
+
+                <xsl:template match="//tei:w" mode="norm">
+                                <span class="word" id="{@xml:id}" title="{@type}">
+                                                <xsl:if
+                                                  test="starts-with(.,':') or starts-with(.,'!') or starts-with(.,'?') or starts-with(.,';')"
+                                                  >&#xa0;</xsl:if>
+                                                <xsl:apply-templates select="descendant::me:norm"
+                                                  mode="norm"/>
+                                                <xsl:choose>
+                                                  <xsl:when
+                                                  test="following::*[1][self::bfm:punct][starts-with(., ',') or starts-with(., '.') or starts-with(., ']') or starts-with(., ')') or starts-with(.,':') or starts-with(.,'!') or starts-with(.,'?') or starts-with(.,';')]"/>
+                                                  <xsl:when
+                                                  test="child::me:norm[string-length(.) = 0]"/>
+                                                  <xsl:when test="@bfm:aggl='elision'">'</xsl:when>
+                                                  <xsl:otherwise>
+                                                  <xsl:text> </xsl:text>
+                                                  </xsl:otherwise>
+                                                </xsl:choose>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:sic" mode="norm">
+                                <span style="text-decoration:line-through;color:red">
+                                                <xsl:apply-templates mode="norm"/>
+                                </span>
+                </xsl:template>
+
+                <xsl:template match="tei:del" mode="norm"> </xsl:template>
+
+
+                <xsl:template match="//tei:hi[@rend='sup']" mode="norm">
+                                <sup>
+                                                <xsl:apply-templates mode="norm"/>
+                                </sup>
+                </xsl:template>
+
+
+
+                <!-- 
+
+
+<xsl:template match="del">
+
+</xsl:template>
+
+
+
+<xsl:template match="name">
+	<xsl:value-of select="@reg"/>
+</xsl:template>
+
+                -->
+</xsl:stylesheet>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/xsl/mdv_norm.xsl
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalApplyXsl.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalApplyXsl.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalApplyXsl.groovy	(revision 321)
@@ -0,0 +1,122 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2014-09-15 10:51:15 +0200 (Mon, 15 Sep 2014) $
+// $LastChangedRevision: 2836 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.graal
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.lang.Boolean
+import javax.xml.transform.TransformerFactory
+import javax.xml.transform.stream.StreamResult
+import javax.xml.transform.stream.StreamSource
+import java.util.Map
+
+// TODO: Auto-generated Javadoc
+/** The trans data. @author ayepdieu previous version of the script ApplyXsl */
+public class GraalApplyXsl{
+	private def transData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The source. */
+	private def source;
+	
+	/** The transformer. */
+	private def transformer;
+	
+	/**
+	 * Instantiates a new graal apply xsl.
+	 *
+	 * @param urlXsl the url xsl
+	 */
+	public GraalApplyXsl(URL urlXsl){
+		try 
+		{
+			FileInputStream inputStream = new FileInputStream(new File(urlXsl.getFile()));
+			factory = new net.sf.saxon.TransformerFactoryImpl();
+			source = new StreamSource(inputStream)
+			source.setSystemId(urlXsl.getPath());
+			transformer = factory.newTransformer(source)
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * Sets the params.
+	 *
+	 * @param name the name
+	 * @param value the value
+	 */
+	public void setParams(String name, int value) {
+	    transformer.setParameter(name, value);
+	}
+	
+	/**
+	 * Sets the params.
+	 *
+	 * @param name the name
+	 * @param value the value
+	 */
+	public void setParams(String name, String value) {
+	    transformer.setParameter(name, value);
+	}
+	
+	/**
+	 * Apply xsl.
+	 *
+	 * @param urlXml the url xml
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean ApplyXsl(URL urlXml, String dirPathName, String fileName) {
+		FileInputStream inputData = new FileInputStream(new File(urlXml.getFile()));
+		def output = new File(dirPathName, fileName)
+
+		transformer.transform( new StreamSource(inputData), new StreamResult(output) )
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) 
+	{
+		String sortie = "Page-test"
+		for (int index = 0; index < 1; index++) {
+			GraalApplyXsl DecTxt = new GraalApplyXsl(new URL("file:///home/ayepdieu/srcQuete/Sources/breakByMilestone.xsl"));
+			DecTxt.setParams("pbval1", index + 1);
+			DecTxt.setParams("pbval2", index + 2);
+			DecTxt.ApplyXsl(new URL("file:///home/ayepdieu/srcQuete/Sources/qgraal_cm_2009-07-d.xml"), "/home/ayepdieu/srcQuete/Decoupe", sortie + "_" + (index+ 1) + ".xml");
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalImportCqp.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalImportCqp.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/graal/GraalImportCqp.groovy	(revision 321)
@@ -0,0 +1,397 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.graal
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.lang.Boolean
+import java.lang.System
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class GraalImportCqp.
+ *
+ * @author ayepdieu
+ * parse graal xml file to build the wtc the next step is to call importInCWB script
+ */
+public class GraalImportCqp{
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private def parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	
+	/**
+	 * initialize.
+	 *
+	 * @param url the url to the xmlfile to transform
+	 */
+	public GraalImportCqp(URL url){
+		try {
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+			//System.out.println "ouverture du document en entree reussi";
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * create the output file writer.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			dir = new File(dirPathName)
+			output = new OutputStreamWriter(new FileOutputStream(new File(dir, fileName)) , "UTF-8")
+			//System.out.println "ouverture du document en sortie reussi";
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * process !!!.
+	 *
+	 * @param dirPathName the output directory
+	 * @param fileName the outfile name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName){
+		if(findBody() && createOutput(dirPathName, fileName)){
+			String idColumn;
+			int idParagraph = 0;
+			int idSentence = 0;
+			int idLine = 0;
+			int q_id =0;
+			String idLinesuiv;
+			String idWord;
+			boolean flagVersion = false;
+			boolean flagWord = false;
+			boolean flagNorm = false;
+			boolean flagDipl = false;
+			boolean flagFacs = false;
+			//boolean flagSupplied = false;
+			int levelSupplied = 0;
+			int levelq = 0;
+			String typeWord = "";
+			String vWordDipl = "";
+			String vWordFacs = "";
+			String vWordNorm = "";
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+					switch (event) {
+						case XMLStreamConstants.START_ELEMENT:
+							switch (parser.getLocalName()) {
+								case "milestone":
+									if (parser.getAttributeValue(0) == "column") {
+										idColumn = parser.getAttributeValue(1);
+										idColumn = idColumn.substring(4);
+									}
+								break;
+								case "p":
+									idParagraph = parser.getAttributeValue(0).toInteger();
+									output.write "<p n=" + idParagraph +">\n";
+									//if(idParagraph < 13 && idParagraph > 4){
+										flagVersion = true;
+									/*}else{
+										flagVersion = false;
+									}*/
+								break;
+								case "s":
+//									String no = parser.getAttributeValue(0);
+//									if(no == null)
+//										idSentence = -1;
+//									else
+//										idSentence = no.toInteger();
+								idSentence++;
+									output.write "<s n=" + idSentence + " id=\"" + parser.getAttributeValue(1) + "\">\n";
+								break;
+								case "q":
+									output.write "<q n=\""+q_id+"\">\n";
+									q_id++;
+									//new
+									levelq = levelq + 1;
+								break;
+								
+								case "supplied":
+								levelSupplied++;
+								break;								
+								
+								case "lb":
+									idLinesuiv = parser.getAttributeValue(0);
+									if(!("facs" == idLinesuiv)){
+										idLine = idLinesuiv.toInteger();
+									}
+								break;
+								case "w":
+									typeWord = parser.getAttributeValue(0);
+									if(flagVersion  && null != parser.getAttributeValue(2)){
+										idWord = parser.getAttributeValue(2);
+									}else{
+										idWord = parser.getAttributeValue(1);
+									}
+									flagWord = true;
+									flagNorm = true;
+								break;
+								case "norm":
+									flagNorm = true;
+								break;
+								case "dipl":
+									flagDipl = true;
+								break;
+								case "facs":
+									flagFacs = true;
+								break;
+							}
+						break;
+						
+						case XMLStreamConstants.END_ELEMENT:
+							switch (parser.getLocalName()) {
+								case "p":
+									output.write "</p>\n";
+								break;
+								case "s":
+									output.write "</s>\n";
+								break;
+								case "q":
+									output.write "</q>\n";
+									levelq = levelq - 1;
+								break;
+								case "supplied":
+									levelSupplied = levelSupplied - 1;
+								break;
+								case "w":
+									output.write vWordNorm + "\t" + typeWord + "\t" + levelq + "\t" + levelSupplied.toString().substring(0,1) + "\t" + idColumn + "\t" + idLine + "\t" + idWord + "\t" + vWordDipl + "\t" + vWordFacs + "\n";
+									vWordNorm = "";
+									vWordDipl = "";
+									vWordFacs = "";
+									flagNorm = false;
+									flagDipl = false;
+									flagFacs = false;
+									flagWord = false; 
+								break;
+								case "norm":
+									flagNorm = false;
+								break;
+								case "dipl":
+									flagDipl = false;
+								break;
+								case "facs":
+									flagFacs = false;
+								break;
+							}
+						break;
+						
+						case XMLStreamConstants.CHARACTERS:
+							if(flagWord){
+								vWordNorm.trim();
+								if(flagNorm){
+									vWordNorm += parser.getText().trim();
+								}else if(flagDipl){
+									vWordDipl += parser.getText().trim();
+								}else if(flagFacs){
+									vWordFacs += parser.getText().trim();
+								}
+							}
+						break;
+					}
+				}
+				output.close();
+				parser.close();
+			}
+			catch (Exception ex) {
+				System.out.println("Xml Location: "+parser.getLocation());
+				System.out.println(ex);
+			}
+			
+		}
+	}
+	
+	/**
+	 * a simplier version of transform file to process a non-facette xml.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtcBrut(String dirPathName, String fileName){
+		if(findBody() && createOutput(dirPathName, fileName)){
+			String idColumn = "";
+			int idParagraph = 0;
+			int idSentence = 0;
+			String idWord;
+			boolean flagWord = false;
+			boolean flagCont = true;
+			boolean flagStart = true;
+			String typeWord = "";
+			String vWord = "";
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT && flagCont; event = parser.next()) {
+					switch (event) {
+						case XMLStreamConstants.START_ELEMENT:
+							//System.out.println parser.getLocalName();
+							switch (parser.getLocalName()) {
+								case "milestone":
+									if (parser.getAttributeValue(0) == "p") {
+										idParagraph = parser.getAttributeValue(1).toInteger();
+										if(flagStart){
+											output.write "<p n=" + idParagraph + ">\n";
+										}else{
+											output.write "</p>\n<p n=" + idParagraph + ">\n";
+										}
+									}else if(parser.getAttributeValue(0) == "column"){
+										idColumn = parser.getAttributeValue(1);
+										idColumn = idColumn.substring(4);
+									}else if (parser.getAttributeValue(0) != "s"){
+										idColumn = "";
+									}
+								break;
+								case "s":
+									idSentence = parser.getAttributeValue(0).toInteger();
+									output.write "<s n=" + idSentence + ">\n";
+								break;
+								case "w":
+									if(null != parser.getAttributeValue(1)){
+										typeWord = parser.getAttributeValue(0);
+										idWord = parser.getAttributeValue(1);
+									}else{
+										idWord = parser.getAttributeValue(0);
+										typeWord = "";
+									}
+									flagWord = true;
+								break;
+								case "div":
+									if (parser.getAttributeValue(0) == "note") {
+										output.write "</s>\n</p>";
+										flagCont = false;
+									}
+								break;
+							}
+						break;
+						
+						case XMLStreamConstants.END_ELEMENT:
+							switch (parser.getLocalName()) {
+								case "w":
+									output.write vWord + "\t" + typeWord + "\t" + idWord + "\t" + idColumn + "\n";
+									flagWord = false; 
+								break;
+							}
+						break;
+						case "s":
+							output.write "</s>\n";
+						break;						
+						case XMLStreamConstants.CHARACTERS:
+							if(flagWord){
+								vWord = parser.getText().trim();
+							}
+						break;
+					}
+				}
+				output.close();
+				parser.close();
+			}
+			catch (XMLStreamException ex) {
+				System.out.println(ex);
+			}
+			catch (IOException ex) {
+				System.out.println("IOException while parsing " + input);
+			}
+		}
+	}
+	
+	/**
+	 * test if the current element if the body.
+	 *
+	 * @param name the name
+	 * @return true, if is body
+	 */
+	private static boolean isBody(String name) {
+		if (name.equals("body")) return true;
+	}
+	
+	/**
+	 * bring the parser to the body element.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean findBody(){
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()){
+			if(event == XMLStreamConstants.START_ELEMENT){
+				if (isBody(parser.getLocalName())){
+					return true;
+				}
+			}
+		}
+		return false;
+	} 
+	
+	/**
+	 * no args needed.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+	 	def inDir = "/home/txm/src";
+	 	def outDir = "/home/txm/";
+	 	def inFile = "qgraal_cm_2009-07-d.xml";
+		System.out.println "file://"+ inDir + "/" + inFile;
+
+		if (args.length == 0) {
+			GraalImportCqp traitTxt = new GraalImportCqp(new URL("file://"+ inDir + "/" + inFile));
+			traitTxt.transfomFileWtc("/home/ayepdieu/srcQuete/result", "graal.wtc");
+		}else if(args.length == 3){
+			GraalImportCqp traitTxt = new GraalImportCqp(new URL(args[0]));
+			traitTxt.transfomFileWtc(args[1], args[2]);
+		}else{
+			System.err.println("Usage: java XHTMLOutliner url" );
+			return;
+		}
+		return
+	}
+	
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/compiler.groovy	(revision 321)
@@ -0,0 +1,300 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $ 
+//
+
+
+package org.txm.importer.lasla;
+
+import java.util.ArrayList;;
+
+import org.txm.importer.cwb.BuildCwbEncodeArgs;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler 
+{
+	
+	/** The debug. */
+	private boolean debug= false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The text attributes. */
+	String[] textAttributes = null;
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+	
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.textAttributes = textAttributes;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			
+			return false;
+		}
+	}
+	
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @param textAttributes the text attributes
+	 * @return true, if successful
+	 */
+	public boolean run(File rootDirFile,String basename, String[] textAttributes) 
+	{
+		String rootDir =rootDirFile.getAbsolutePath();
+		
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!new File(rootDir).exists())
+		{
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+
+		new File(rootDir,"/wtc/").deleteDir();
+		new File(rootDir,"/wtc/").mkdir();
+		new File(rootDir,"/data/").deleteDir();
+		new File(rootDir,"/data/").mkdir();
+		new File(rootDir,"registry/").mkdir();
+		
+		String textid="";
+		int counttext =0;
+		List<File> files = new File(rootDirFile,"txm").listFiles();
+		//1- Transform into WTC file
+		def builder = null;
+		for(File f : files)
+		{
+			counttext++;
+			if(!f.exists())
+			{
+				println("file "+f+ " does not exists")	
+			}
+			else
+			{	
+				wtcbuilder = new XMLTXM2WTC(f.toURL());
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				wtcbuilder.setTextInfo(txtname, basename, "project");
+
+				wtcbuilder.setBalisesToKeep(["text"]);
+				wtcbuilder.setSendToPAttributes([]);
+				wtcbuilder.setLang(lang);
+				if(!wtcbuilder.transformFile(new File(rootDir,"wtc/"+basename+".wtc")))
+				{
+					println("Failed to compile "+f)
+				}
+			}
+		}
+		
+		//2- Import into CWB
+		def outDir =rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		String[] pAttributes = ["ref","s","sent","lemme","line","pos"];
+		
+		BuildCwbEncodeArgs argsgetter = new BuildCwbEncodeArgs();
+
+		ArrayList<String> wordstag = ["w"];
+		//argsgetter.process(new File(rootDir+"/txm",basename+".wtc"), wordstag);
+		
+		String[] sAttributes = ["txmcorpus:0+lang", "text:0+id+base+project"]
+		
+		try {
+			String regPath = outDirTxm + "/registry/"+basename.toLowerCase();
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), outDirTxm + "/data", outDir + "/wtc/"+basename+".wtc", regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), basename.toUpperCase(), outDirTxm + "/registry");
+			
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		
+		System.out.println("Done.") 
+		
+		return true;
+	}
+	
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Lasla import module - not yet finished</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/pager.groovy	(revision 321)
@@ -0,0 +1,377 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.lasla
+
+;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build GEO corpus simple edition from a xml-tei. @author mdecorde */
+class pager {
+	List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+	
+	/** The pagecount. */
+	int pagecount = 0;
+	
+	/** The wordmax. */
+	int wordmax = 0;
+	
+	/** The basename. */
+	String basename = "";
+
+	/** The wordid. */
+	String wordid;
+	
+	/** The first word. */
+	boolean firstWord = true;
+	
+	/** The wordvalue. */
+	String wordvalue;
+	
+	/** The interpvalue. */
+	String interpvalue;
+	
+	/** The lastword. */
+	String lastword = " ";
+	
+	/** The wordtype. */
+	String wordtype;
+	
+	/** The flagform. */
+	boolean flagform = false;
+	
+	/** The flaginterp. */
+	boolean flaginterp = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	OutputStreamWriter writer;
+	
+	/** The multiwriter. */
+	OutputStreamWriter multiwriter = null;
+	
+	/** The infile. */
+	File infile;
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+	
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter, int max, String basename) {
+		this.basename = basename;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		try {
+			if(multiwriter != null)
+			{
+				multiwriter.write("</body>");
+				multiwriter.write("</html>");
+				multiwriter.close();
+			}
+			pagecount++;
+			File f = new File(outfile.getParent()+"/multi/",outfile.getName().substring(0,outfile.getName().length()-5)+"_"+pagecount+".html");
+			pages.add(f);
+			idxstart.add(wordid)
+			multiwriter = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("<head>");
+			multiwriter.write("<title>"+basename.toUpperCase()+" Edition - Page "+pagecount+"</title>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			this.outfile = outfile;
+
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+			createNextOutput();
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process()
+	{
+		String localname = "";
+		
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		writer.write("<html>");
+		writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.write("<head>");
+		writer.write("<title>"+basename.toUpperCase()+" edition</title>");
+		writer.write("</head>");
+		writer.write("<body>");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "head":
+							writer.write("<h2>\n")
+							multiwriter.write("<h2>\n")
+							break;
+						
+						case "lg":
+							writer.write("<p>\n")
+							multiwriter.write("<p>\n")
+							break;
+						
+						case "p":
+						case "q":
+							writer.write("<p>\n")
+							multiwriter.write("<p>\n")
+							break;
+							
+						case "lb":
+						case "br":
+						writer.write("<br/>\n")
+						multiwriter.write("<br/>\n")
+						break;
+						
+						case "w":
+							wordid=parser.getAttributeValue(null,"id");
+							if(firstWord)
+							{
+								firstWord=false;
+								this.idxstart.set(0,wordid);
+							}
+							wordcount++;
+							if(wordcount >= wordmax)
+							{
+								createNextOutput();
+								wordcount=0;
+							}
+							break;
+						
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+						
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "head":
+							writer.write("</h2>\n")
+							multiwriter.write("</h2>\n")
+							break;
+						
+						case "lg":
+							writer.write("</p>\n")
+							multiwriter.write("</p>\n")
+							break;
+						
+						case "lb":
+							writer.write("<br/>\n")
+							multiwriter.write("<br/>\n")
+							break;
+							
+						case "pb":
+							writer.write("<br/><br/>\n")
+							multiwriter.write("<br/><br/>\n")
+							break;
+						
+						case "p":
+						case "q":
+							writer.write("</p>\n")
+							multiwriter.write("</p>\n")
+							break;
+						
+						
+						case "form":
+							flagform = false
+						
+							break;
+						
+						case "ana":
+							flaginterp = false
+							break;
+						
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if(l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+							
+							if(interpvalue != null)
+								interpvalue = interpvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;");
+							
+							if(NoSpaceBefore.contains(wordvalue) || 
+									NoSpaceAfter.contains(lastword) || 
+									wordvalue.startsWith("-") || 
+									NoSpaceAfter.contains(endOfLastWord))
+							{
+								writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							else
+							{
+								writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							writer.write(wordvalue.replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+							break;
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText().replace("&","&amp;").replace("<", "&lt;"));
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText().replace("&","&amp;").replace("<", "&lt;"));
+					break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		writer.close();
+		multiwriter.write("</body>");
+		multiwriter.write("</html>");
+		multiwriter.close();
+		inputData.close();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/importer.groovy	(revision 321)
@@ -0,0 +1,168 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.lasla
+
+import javax.xml.stream.*
+
+import org.txm.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+import org.txm.objects.*
+import org.txm.scripts.*
+import org.txm.scripts.teitxm.*
+
+import filters.CutHeader.*
+import filters.FusionHeader.*
+import filters.TagSentences.*
+import filters.Tokeniser.*
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param encoding the encoding
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File rootDirFile, String encoding, String basename)
+	{
+		String filename = rootDirFile.getName();
+		String rootDir = rootDirFile.getAbsolutePath()+"/"
+		
+		//cleaning
+		File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+		
+		binDir.deleteDir();
+		binDir.mkdir();
+		new File(binDir,"src").deleteDir();
+		new File(binDir,"src").mkdir();
+		new File(binDir,"txm").deleteDir();
+		new File(binDir,"txm").mkdir();
+		
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+		
+		//separates values and build simple xml
+		List<File> srcfiles = new File(rootDir).listFiles();
+		if(srcfiles == null)
+		{
+			println "No file to process"
+			return false;
+		}
+		//println "BYEBYE FORTRAN";
+		println("Converting to xml "+srcfiles.size()+ " files")
+		for(File f : srcfiles)
+		{
+			print "."
+			File outfile = new File(binDir,"src/"+f.getName()+".xml")
+			FortranColumns2XML builder = new FortranColumns2XML(f, outfile, encoding);
+			builder.setColumnIndexes([0, 3, 4, 8, 29, 55, 67],	[3, 4, 8, 28, 54, 66, 78]);
+			builder.setColumnsLinesNames(["ref","s","sent","lemme","word","line","pos"], "w", 4);
+			builder.setAddIds(true);
+			//builder.setLineCheck(3, "l");
+			if(!builder.process())
+			{
+				println "Failed to convert file: "+f
+			}
+		}
+		println ""
+		
+		rootDir = binDir.getAbsolutePath();
+		srcfiles = new File(rootDir, "src").listFiles();
+		//TRANSFORM INTO XML-TEI-TXM
+		println("Building xml-tei-txm "+srcfiles.size()+ " files")
+		for(File f : srcfiles)
+		{
+			print "."
+			//ArrayList<String> milestones = new ArrayList<String>();
+			
+			File file = f; 
+			String txmfile = f.getName();
+			def correspType = new HashMap<String,String>()
+			//["ref","s","sent","word","lemme","line","pos"]
+			correspType.put("ref","ref");
+			correspType.put("s","s");
+			correspType.put("sent","sent");
+			correspType.put("lemme","lemme")
+			correspType.put("line","line")
+			correspType.put("pos","pos");
+			correspType.put("form","form");
+			
+			def correspRef = new HashMap<String,String>()
+			correspRef.put("ref","lasla");
+			correspRef.put("s","lasla");
+			correspRef.put("sent","lasla");
+			correspRef.put("lemme","lasla")
+			correspRef.put("line","lasla")
+			correspRef.put("pos","lasla");
+			correspRef.put("form","lasla");
+			
+			//il faut lister les id de tous les respStmt
+			def respId = ["lasla"];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String, ArrayList<String>>();
+			applications.put("lasla", new ArrayList<String>());
+			applications.get("lasla").add("txm");//app ident
+			applications.get("lasla").add("0.6");//app version
+			applications.get("lasla").add(null);//app report file path
+			
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+					//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			taxonomiesUtilisees.put("lasla",["LASLA"]);//,"lemma","lasla","grace"]);
+			
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			itemsURI.put("LASLA",new HashMap<String,String>());
+			itemsURI.get("LASLA").put("tagset","http://bfm.ens-lsh.fr/IMG/xml/cattex2009.xml");
+			itemsURI.get("LASLA").put("website","http://bfm.ens-lsh.fr/article.php3?id_article=176");
+			
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			resps.put("lasla", ["initial tagging","lasla","2010-03-02",""])
+			//lance le traitement
+			String wordprefix = "w_";
+			def builder = new Xml2Ana(file);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			//builder.setAddTEIHeader();
+			if(!builder.process(new File(new File(rootDir,"txm"),txmfile)))
+			{
+				println "Failed to build xml-txm of file: "+f;
+			}
+		}
+		println ""
+		return true;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/laslaLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/laslaLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/lasla/laslaLoader.groovy	(revision 321)
@@ -0,0 +1,167 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.lasla;
+
+import java.io.File;
+
+import org.txm.importer.lasla.importer;
+import org.txm.importer.xml.compiler;
+import org.txm.importer.xml.pager_old;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+String basename;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;basename=basenameBinding;}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		rootDir = userDir+"/xml/lasla/";
+		basename = "latin";
+		lang="fr";
+		encoding= "ISO-8859-1";
+		model="rgaqcj";//not used
+		Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,"/usr/lib/TXM");
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+//String basename = new File(rootDir).getName().toLowerCase()
+println "-- IMPORTER - Reading source files"
+def imp = new importer();
+if(!imp.run( new File(rootDir), encoding, basename))
+{
+	println "import process stopped";
+	return;
+}
+
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+File txmfiles = new File(binDir,"txm");
+files = txmfiles.listFiles()
+
+println "-- COMPILING - Building Search Engine indexes"
+
+def files = new File(binDir, "src").listFiles()
+File onesrcfile = null;
+if(files.size() > 0)
+	onesrcfile = files[0];
+if(onesrcfile == null)
+{
+	println "No XML file (extension '.xml') has been found in directory "+binDir
+	return;
+}
+
+def c = new compiler();
+c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setAnnotationSuccess(false)
+c.setLang(lang);
+if(!c.run(binDir, basename, null, files, null))
+{
+	println "import process stopped";
+	return;
+}
+
+//move registry file to cwb registry dir
+File registryfile = new File(binDir,"registry/"+basename);
+if(registryfile.exists())
+	FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "registry/"+basename))
+
+Workspace w = org.txm.Toolbox.workspace;
+Project p = w.getProject("default")
+p.removeBase(basename)
+Base b = p.addBase(basename);
+b.addDirectory(new File(binDir, "txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION - Building edition"
+new File(binDir,"HTML").deleteDir();
+new File(binDir,"HTML").mkdir();
+new File(binDir,"HTML/default").mkdir();
+List<File> filelist = new File(binDir,"txm").listFiles();
+def second = 0
+
+println "Paginating text: "
+for(String textname : b.getTextsID())
+{
+	Text text = b .getText(textname);
+	File srcfile = text.getSource();
+	File resultfile = new File(binDir, "HTML/"+srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+		
+		if (second) { print(", ") }
+		if (second > 0 && (second % 5) == 0) println ""
+		print(srcfile.getName());
+		second++
+	
+	def ed = new pager_old(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,500,basename, "br");
+	
+	Edition editionweb = text.addEdition("default","html",resultfile);
+//	println("pages "+ed.getPageFiles())
+//	println("idx "+ed.getIdx())
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+	
+//	Edition editionbp = text.addEdition("onepage","html",resultfile);
+//	editionbp.addPage(resultfile,ed.getIdx().get(0));
+}
+
+
+w.save()
+println "done"
+
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/DocBookCleaner.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/DocBookCleaner.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/DocBookCleaner.groovy	(revision 321)
@@ -0,0 +1,15 @@
+package org.txm.importer
+
+import org.txm.sw.ChangeEncoding;
+
+File origDir = new File("/home/mdecorde/xml/temoignages2/")
+File outDir = new File("/home/mdecorde/xml/temoignages3/")
+
+File f = new File(origDir, ".xml")
+println f.getText("Windows-1252")
+//for (File f : origDir.listFiles()) {
+//	if (f.isHidden() || f.isDirectory()) continue;
+//	
+//	ChangeEncoding.processFile(f, "Windows-1252", "UTF-8")
+//	println f.getText("UTF-8")
+//}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WordCounter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WordCounter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WordCounter.groovy	(revision 321)
@@ -0,0 +1,189 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2015-12-17 12:11:39 +0100 (Thu, 17 Dec 2015) $
+// $LastChangedRevision: 3087 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import java.io.BufferedReader;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory
+import org.xml.sax.helpers.DefaultHandler
+import org.xml.sax.*
+import javax.xml.stream.*;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/** Count the focused tag and add id to it you must specify  : which tag are milestones the tag to count. */
+class WordCounter extends DefaultHandler {
+    String txt ;
+	
+	/** The focus. */
+	String focus;
+    
+    /** The counter. */
+    int counter = 0;
+    
+    /** The solotags. */
+    ArrayList<String> solotags;
+    
+    /** The writer. */
+    XMLStreamWriter writer;
+	
+	/**
+	 * The initializer do the work
+	 * The file infile is replaced by the result.
+	 *
+	 * @param infile the infile
+	 * @param focus the focus
+	 * @param txt the txt
+	 */
+    WordCounter(File infile, String focus, String txt)
+    {
+    	File outfile = new File(infile.getName()+"counter");
+		this.focus=focus;
+		this.txt = txt;
+		
+		FileInputStream input = new FileInputStream(infile);
+		
+        def handler = this;
+        def reader = SAXParserFactory.newInstance().newSAXParser();
+		
+        def output = new FileOutputStream(outfile);
+        XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		writer = factory.createXMLStreamWriter(output, "UTF-8");
+		writer.writeStartDocument("UTF-8", "1.0");
+		output.write("\n")
+		
+		//Reader inputStream = new InputStreamReader(new FileInputStream(infile) , "UTF-8");
+		//BufferedReader r= new BufferedReader(inputStream);
+		
+        reader.parse(input, handler);// process !!
+		
+        input.close();
+        writer.close();
+		output.close();
+        //r.close();
+        if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+    }
+	
+	/**
+	 * process start tag.
+	 *
+	 * @param ns the ns
+	 * @param localName the localname of the current tag
+	 * @param qName the q name
+	 * @param atts the attributes of the current tag
+	 */
+    void startElement(String ns, String localName, String qName, Attributes atts) {
+    	
+    	writer.writeStartElement(qName)
+    	
+    	//writeAttrs
+        if(qName.matches(focus))
+        {
+        	counter++;
+        	writer.writeAttribute("n",""+counter)
+            writer.writeAttribute("xml:id",focus+txt+"_"+counter)
+            for(int i=0; i < atts.getLength();i++)
+            	if(atts.getQName(i) != "n" && atts.getQName(i) != "xml:id" && atts.getQName(i) != "id")
+                writer.writeAttribute(atts.getQName(i), atts.getValue(i));
+        }
+        else
+        {
+        	for(int i=0; i < atts.getLength();i++)
+                writer.writeAttribute(atts.getQName(i), atts.getValue(i));
+        }
+    }
+	
+	/**
+	 * write the CDATA, not finished.
+	 *
+	 * @param chars all the chars
+	 * @param offset where the text begin
+	 * @param length the length
+	 */
+    void characters(char[] chars, int offset, int length)
+    {
+    	writer.writeCharacters(chars, offset, length)
+    }
+	
+	/* (non-Javadoc)
+	 * @see org.xml.sax.helpers.DefaultHandler#endElement(java.lang.String, java.lang.String, java.lang.String)
+	 */
+    void endElement(String ns, String localName, String qName)
+    {
+       writer.writeEndElement();
+    }
+    
+    /**
+     * Find text id.
+     *
+     * @param infile the infile
+     * @param focus the focus
+     * @return the string
+     */
+    public static String findTextId(File infile, String focus)
+    {
+    	def inputData = infile.toURI().toURL().openStream();
+		def inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+				
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					if(parser.getLocalName() == focus)
+					{
+						String sid = ""
+						for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+							if(parser.getAttributeLocalName(i) == "id")
+							{
+								sid = parser.getAttributeValue(i); //xxx_000
+								sid = sid.substring(focus.length()); // on retire le nom du focus
+								sid = sid.split("_")[0];
+								break;
+							}
+						inputData.close();
+						parser.close();
+						return sid;
+					}
+			}
+		}
+    }
+    
+    /**
+     * The main method.
+     *
+     * @param args the arguments
+     */
+    static void main(String[] args)
+    {
+    	File infile = new File("~/xml/quote/processme-q.xml")
+        new WordCounter(infile, "s","01");
+    	new WordCounter(infile, "q","01");
+    }
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/annotate.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/annotate.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/annotate.groovy	(revision 321)
@@ -0,0 +1,146 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-11-08 13:38:06 +0100 (ven., 08 nov. 2013) $
+// $LastChangedRevision: 2569 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.corptef;
+
+import org.txm.importer.cwb.*
+import org.txm.scripts.teitxm.*;
+import org.txm.importer.*;
+import org.txm.utils.treetagger.TreeTagger;
+import org.txm.Toolbox;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class annotate.
+ */
+class annotate {
+	
+	/**
+	 * Run.
+	 *
+	 * @param dir the dir
+	 */
+	public void run(File dir)
+	{
+		String rootDir = dir.getAbsolutePath()+"/";//"~/xml/discours/src";
+		String txmDir = dir.getAbsolutePath()+"/txm/"
+
+		//cleaning
+		new File(rootDir,"annotations").deleteDir();
+		new File(rootDir,"annotations").mkdir();
+		new File(rootDir,"treetagger").deleteDir();
+		new File(rootDir,"treetagger").mkdir();
+		println txmDir
+		List<File> listfiles = new File(txmDir).listFiles();
+		for(File teifile : listfiles)
+		{
+			println("annotate "+teifile)
+			File modelfile = new File(Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH),"/rgaqcj.par");
+			if(!modelfile.exists())
+			{
+				println "Skipping ANNOTATE: Incorrect modelfile path: "+modelfile;
+				return;
+			}
+			File annotfile = new File(rootDir+"annotations",teifile.getName()+"-STDOFF.xml");
+			File ttsrcfile = new File(rootDir+"treetagger",teifile.getName()+"-src.tt");
+			File ttrezfile = new File(rootDir+"treetagger",teifile.getName()+"-out.tt");
+			
+			//prepare file to be tagged
+			def builder = new BuildTTSrc(teifile.toURL());
+			builder.process(ttsrcfile);
+			
+			//Apply TT
+			if(!new File(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/").exists() || Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH).length() == 0 )
+			{
+				println("Path to TreeTagger is wrong: "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/")
+				return;
+			}
+			TreeTagger tt = new TreeTagger(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/");
+			tt.settoken();
+			tt.setlemma();
+			tt.setquiet();
+			tt.setnounknown();
+			tt.setsgml();
+			tt.seteostag("<s>");
+			tt.treetagger( modelfile.getAbsolutePath(), ttsrcfile.getAbsolutePath(), ttrezfile.getAbsolutePath())
+			
+			//create stand-off annotation file
+			//targeted file for annotations
+			String target = teifile.getName()
+			
+			
+			//contains txm:application/txm:commandLine
+			File reportFile = new File(rootDir,"NLPToolsParameters.xml");
+			
+			String respPerson = System.getProperty("user.name");
+			String respId = "txm";
+			String respDesc = "NLP annotation tool";
+			String respDate = "Tue Mar  11 1:02:55 Paris, Madrid 2010";
+			String respWhen = ""
+			
+			String appIdent = "TreeTagger";
+			String appVersion = "3.2";
+			
+			String distributor = "";
+			String publiStmt = """""";
+			String sourceStmt = """""";
+			
+			def types = ["ttpos","ttlemma"];
+			def typesTITLE = ["ttpos","ttlemma"];
+			def typesDesc = ["fr pos","fr lemma"]
+			def typesTAGSET = ["",""]
+			def typesWEB = ["",""]
+			String idform ="w_";
+			String encoding ="UTF-8";
+			
+			def transfo = new CSV2W_ANA();
+			transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+			transfo.setResp(respId, respDesc,respDate, respPerson, respWhen);
+			transfo.setApp(appIdent, appVersion);
+			transfo.setTarget(target, reportFile);
+			transfo.setInfos(distributor,  publiStmt, sourceStmt);
+			transfo.process( ttrezfile, annotfile, encoding );
+			
+			//merge into the tei file
+			builder = new AnnotationInjection(teifile.toURL(), annotfile.toURL(), new ArrayList<String>());
+			builder.transfomFile(rootDir,"temp");
+			
+			if (!(teifile.delete() && new File(rootDir,"temp").renameTo(teifile))) println "Warning can't rename file "+new File(rootDir,"temp")+" to "+teifile
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("C:/Documents and Settings/alavrent/TXM/corpora/corptef/")
+		new annotate().run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/corptefLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/corptefLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/corptefLoader.groovy	(revision 321)
@@ -0,0 +1,134 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.corptef;
+
+import org.txm.importer.corptef.importer;
+import org.txm.importer.corptef.compiler;
+import org.txm.importer.corptef.annotate;
+import org.txm.importer.corptef.pager;
+import org.txm.objects.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		//rootDir = "C:\\Documents and Settings\\alavrent\\TXM\\sources\\corptef\\src";
+		rootDir = userDir+"/xml/corptef"
+		lang="fr";
+		encoding= "UTF-8";
+		model="rgaqcj";
+		Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File(userDir,"txminstall"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"txminstall/treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"txminstall/treetagger/models"));
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+File dir = new File(rootDir);
+
+println "-- IMPORTER - Reading source files"
+new importer().run(dir);
+
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/corptef");
+rootDir = binDir.getAbsolutePath();
+dir = new File(rootDir);
+
+println "-- ANNOTATE - Running NLP tools"
+new Annotate().run(dir,model+".par");
+
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+c.setDebug();
+//c.setCwbPath("C:/Program Files/TXM/cwb/bin");
+c.setLang(lang);
+c.run(dir);
+
+
+//move registry file to cwb registry dir
+File registryfile = new File(rootDir+"/registry","corptef");
+if(registryfile.exists())
+	org.txm.utils.io.FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"registry/corptef"))
+
+Workspace w = org.txm.Toolbox.workspace;
+Project p = w.getProject("default")
+p.removeBase("corptef")
+Base b = p.addBase("corptef");
+b.addDirectory(new File(rootDir,"txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION"
+new File(rootDir+"/HTML/").deleteDir()
+new File(rootDir+"/HTML/").mkdir();
+new File(rootDir+"/HTML/default/").mkdir();
+files = new File(rootDir,"txm").listFiles();
+
+for(String textname : b.getTextsID())
+{
+	Text text = b .getText(textname);
+	File srcfile = text.getSource();
+	File resultfile = new File(rootDir+"/HTML",srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+	println("Building edition  : "+srcfile+" to : "+resultfile );
+	
+	def ed = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,600);
+	ed.writecorrespondancesFile(new File(rootDir,"edition_correspondances.txt"))
+	Edition editionweb = text.addEdition("default","html",resultfile);
+
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+	
+//	Edition editionbp = text.addEdition("onepage","html",resultfile);
+//	editionbp.addPage(resultfile,ed.getIdx().get(0));
+}
+
+w.save();
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/compiler.groovy	(revision 321)
@@ -0,0 +1,617 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.corptef
+
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler
+{
+
+	/** The debug. */
+	private boolean debug= false;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private def output;
+
+	/** The url. */
+	private def url;
+
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+
+	/** The text. */
+	String text="";
+
+	/** The base. */
+	String base="";
+
+	/** The project. */
+	String project="";
+
+	/** The cwb loc. */
+	String cwbLoc;
+
+	/** The lang. */
+	private String lang ="fr";
+
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		createOutput(dirPathName, fileName);
+
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+
+		int p_id = 0;
+		int q_id = 0;
+		int body_id = 0;
+		int front_id = 0;
+		int back_id = 0;
+		String lb_id = 0;
+		int lb_count = 0;
+		String pb_id = 0;
+		int pb_count = 0;
+		String ab_id = 0;
+
+		boolean captureword = false;
+
+		String vExpan = "";
+		String vCorr = "";
+		String vReg = "";
+		String vOrig = "";
+		String vSic = "";
+		String vAbbr = "";
+
+		boolean flaglg = false;
+		boolean flaghead = false;
+		boolean flagAuthor = false;
+		boolean flagDate = false;
+		boolean flagWord = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+
+		boolean flagchoice = false;
+		boolean flagcorr = false;
+		boolean flagsic = false;
+		boolean flagreg = false;
+		boolean flagexpan = false;
+		boolean flagorig = false;
+		boolean flagabbr = false;
+		boolean flagfw = false;
+
+		File xpathfile = new File(url.getFile());
+		String titreId = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:title[@type='reference']/text()");
+		String auteur = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:author/text()");
+		String datecompo = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo']/@when");
+		String ssiecle = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo_sous_siecle']/@n");
+		String domaine = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textDesc/tei:domain/@type");
+		String genre = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textDesc/@n");
+		String forme = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textClass/tei:catRef/@target[contains(.,'forme')]").substring(1);
+		String dialecte = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:region[@type='dialecte_auteur']/text()");
+
+		this.GoToText()
+		output.write("<txmcorpus lang=\""+lang+"\">\n");
+		try
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event)
+				{
+					case XMLStreamConstants.START_ELEMENT:
+						switch (parser.getLocalName())
+						{
+							case "text":
+							output.write("<text id=\""+text+"\"" +
+							" titre=\""+titreId+"\"" +
+							" auteur=\""+auteur+"\"" +
+							" datecompo=\""+datecompo+"\"" +
+							" ssiecle=\""+ssiecle+"\"" +
+							" domaine=\""+domaine+"\"" +
+							" genre=\""+genre+"\"" +
+							" forme=\""+forme+"\"" +
+							" dialecte=\""+dialecte+"\"" +
+							" base=\""+base+"\"" +
+							" project=\""+project+"\">\n");
+							captureword=true;
+							break;
+
+							case "div":
+							//output.write("<div type=\""+parser.getAttributeValue(null,"type")+"\">\n");
+							break;
+
+							case "p":
+							output.write("<p n=\""+(p_id++)+"\">\n");
+							break;
+							case "ab":
+							output.write("<ab n=\""+ab_id+++"\" rend=\""+parser.getAttributeValue(null,"rend")+"\">\n");
+							break;
+							case "q":
+							output.write("<q n=\""+(q_id++)+"\">\n");
+							break;
+							case "front":
+							output.write("<front n=\""+(front_id++)+"\">\n");
+							break;
+							case "body":
+							output.write("<body n=\""+(body_id++)+"\">\n");
+							break;
+							case "back":
+							output.write("<back n=\""+(back_id++)+"\">\n");
+							break;
+							case "lb":
+							String n = parser.getAttributeValue(null,"n")
+							if(n != null)
+								lb_id = n;
+							else
+								lb_id =""+lb_count++;
+							break;
+							case "pb":
+							String n = parser.getAttributeValue(null,"n");
+							if(n != null)
+								pb_id = n;
+							else
+								pb_id = ""+(pb_count++);
+
+							break;
+
+							case "choice":
+							flagchoice = true;
+							break;
+							case "corr":
+							flagcorr = true;
+							vCorr= "";
+							break;
+							case "reg":
+							flagreg = true;
+							vReg= "";
+							break;
+							case "expan":
+							flagexpan = true;
+							vExpan= "";
+							break;
+							case "orig":
+							flagreg = true;
+							vOrig= "";
+							break;
+							case "sic":
+							flagsic = true;
+							vSic= "";
+							break;
+							case "abbr":
+							flagreg = true;
+							vAbbr= "";
+							break;
+
+							case "w":
+							for(int i = 0 ; i < parser.getAttributeCount(); i++)
+								if(parser.getAttributeLocalName(i).equals("id"))
+							{
+								wordid = parser.getAttributeValue(i);
+							}
+							vAna = "";
+							vForm = "";
+							flagWord = true;
+							break;
+							case "form":
+							flagForm = true;
+							vForm = "";
+							vAna ="";
+							break;
+
+							case "ana":
+							flagAna = true;
+							break;
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						switch (parser.getLocalName())
+						{
+							case "div":
+							break;
+
+							case "text":
+							output.write("</text>\n");
+							captureword=false;
+							break;
+							case "p":
+							output.write("</p>\n");
+							break;
+							case "ab":
+							output.write("</ab>\n");
+							break;
+							case "q":
+							output.write("</q>\n");
+							break;
+							case "front":
+							output.write("</front>\n");
+							break;
+							case "body":
+							output.write("</body>\n");
+							break;
+							case "back":
+							output.write("</back>\n");
+							break;
+
+							case "fw":
+							flagfw = false;
+							break;
+							case "choice":
+
+							if(vCorr != "")
+							{
+								//System.out.println(vCorr+" >> write corr "+vForm +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr)
+								output.write( vCorr +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+							}
+							else if(vReg != "")
+							{
+								//System.out.println("write reg "+vForm)
+								output.write( vReg +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+							}
+							else if(vExpan != "")
+							{
+								//System.out.println("write expan "+vForm)
+								output.write( vExpan +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+							}
+							flagchoice = false;
+							vCorr= "";
+							vSic= "";
+							break;
+							case "corr":
+							flagcorr = false;
+
+							break;
+							case "reg":
+							flagreg = false;
+							vReg = "";
+							break;
+							case "expan":
+							flagexpan = false;
+							vExpan= "";
+							break;
+							case "orig":
+							flagreg = false;
+							vOrig= "";
+							break;
+							case "sic":
+							flagsic = false;
+
+							break;
+							case "abbr":
+							flagreg = false;
+							vAbbr= "";
+							break;
+
+							case "w":
+							if(vAna != null)
+								if(captureword)
+							{
+								if(flagchoice)
+								{
+
+								}
+								else if(flagfw)
+								{
+
+								}
+								else
+								{
+									output.write( vForm +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+								}
+
+							}
+
+							flagWord = false;
+							break;
+
+							case "form":
+							flagForm = false;
+							break;
+
+							case "ana":
+							vAna += "\t";
+							flagAna = false;
+							break;
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						if(flagAna)
+						{
+							vAna += parser.getText().trim();
+						}
+
+						if(flagForm)
+						{
+							vForm += parser.getText().trim();
+							if(flagchoice)
+							{
+								if(flagsic)
+								{
+									vSic += parser.getText().trim();
+								}
+								if(flagorig)
+								{
+									vOrig += parser.getText().trim();
+								}
+								if(flagabbr)
+								{
+									vAbbr += parser.getText().trim();
+								}
+								if(flagcorr)
+								{
+									vCorr += parser.getText().trim();
+								}
+							}
+						}
+				}
+			}
+			output.write("</txmcorpus>");
+			output.close();
+			parser.close();
+			inputData.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+
+		return true;
+	}
+
+
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @return true, if successful
+	 */
+	public boolean run(File rootDirFile)
+	{
+		String rootDir =rootDirFile.getAbsolutePath();
+
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!new File(rootDir).exists())
+		{
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+		new File(rootDir+"/wtc/","corptef.wtc").delete();//cleaning&preparing
+		new File(rootDir,"/wtc/").deleteDir();
+		new File(rootDir,"/wtc/").mkdir();
+		new File(rootDir,"/data/").deleteDir();
+		new File(rootDir,"/data/").mkdir();
+		new File(rootDir,"registry/").mkdir();
+
+		String textid="";
+		int counttext =0;
+		List<File> files = new File(rootDirFile,"txm").listFiles();
+		Collections.sort(files);
+		//1- Transform into WTC file
+		for(File f : files)
+		{
+			counttext++;
+			if(!f.exists())
+			{
+				println("file "+f+ " does not exists")
+			}
+			else
+			{
+				println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				def builder = new compiler(f.toURL(),txtname, "corptef", "default");
+				builder.setLang lang
+				builder.transfomFileWtc(rootDir+"/wtc","corptef.wtc");
+			}
+		}
+
+		//2- Import into CWB
+		def outDir = rootDir;
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		String[] pAttributes = ["ttpos","ttlemme","id","pb","lb","orig","sic","abbr"];
+		String[] sAttributes = ["txmcorpus:0+lang", "text:0+id+titre+auteur+datecompo+ssiecle+domaine+genre+forme+dialecte+base+project","front:0+n","body:0+n","ab:0+n+rend","div:0+id+type","p:0+n","back:0+n"];
+		try {
+
+			String regPath = new File(outDir , "registry/corptef").getAbsolutePath();
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(),
+					new File(outDir, "/data").getAbsolutePath(),
+					new File(outDir, "wtc/"+"corptef.wtc").getAbsolutePath(),
+					regPath,
+					pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(),
+					"CORPTEF",
+					new File(outDir, "registry").getAbsolutePath());
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		// "C:\Documents and Settings\alavrent\TXM\cwb\bin\cwb-encode" -d "C:\Documents and Settings\alavrent\TXM\corpora\corptef\data" -f "C:\Documents and Settings\alavrent\TXM\corpora\corptef\wtc\corpte.wtc" -R "C:\Documents and Settings\alavrent\TXM\corpora\corptef\registry\corptef" -c utf8 -xsB -xsB -P ttpos -P ttlemme -P id -P pb -P lb -P orig -P sic -P abbr -S text:0+id+titre+auteur+datecompo+ssiecle+domaine+genre+forme+dialecte+base+project -S front:0+n -S body:0+n -S ab:0+n+rend -S div:0+id+type -S p:0+n -S back:0+n
+		System.out.println("Done.")
+
+		return true;
+	}
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("C:/TXM/corpora/corptef/");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("C:/Program Files/TXM/cwb/bin");
+		c.run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Corptef import module</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/pager.groovy	(revision 321)
@@ -0,0 +1,558 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.corptef;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build BFM corpus simple edition from a xml-tei. @author mdecorde */
+class pager {
+	List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+	
+	/** The pagecount. */
+	int pagecount = 0;
+	
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The wordid. */
+	String wordid;
+	
+	/** The first word. */
+	boolean firstWord = true;
+	
+	/** The wordvalue. */
+	String wordvalue;
+	
+	/** The interpvalue. */
+	String interpvalue;
+	
+	/** The lastword. */
+	String lastword = " ";
+	
+	/** The wordtype. */
+	String wordtype;
+	
+	/** The flagform. */
+	boolean flagform = false;
+	
+	/** The flaginterp. */
+	boolean flaginterp = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	OutputStreamWriter writer;
+	
+	/** The multiwriter. */
+	OutputStreamWriter multiwriter = null;
+	
+	/** The infile. */
+	File infile;
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+	
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	
+	/** The titre id. */
+	String titreId;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 */
+	pager(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter, int max) {
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		try {
+			if(multiwriter != null)
+			{
+				multiwriter.write("</body>");
+				multiwriter.write("</html>");
+				multiwriter.close();
+			}
+			pagecount++;
+			File f = new File(outfile.getParent()+"/multi/",outfile.getName().substring(0,outfile.getName().length()-5)+"_"+pagecount+".html");
+			pages.add(f);
+			idxstart.add(wordid)
+			multiwriter = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("<head>");
+			multiwriter.write("<title>BFM - "+titreId+" - "+pagecount+"</title>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			if(wordid != null)
+			{
+				String[] splited = wordid.split("_");
+				int no = Integer.parseInt(splited[1]);
+				multiwriter.write("<span id=\""+splited[0]+"_"+(no++)+"\"/>");
+			}
+						
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			this.outfile = outfile;
+
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process()
+	{
+		String localname = "";
+		
+		boolean closeTitre = false;
+		boolean closeAmen = false;
+		
+		boolean flagchoice = false;
+		boolean flagcorr = false;
+		boolean flagsic = false;
+		boolean flagreg = false;
+		boolean flagexpan = false;
+		boolean flagorig = false;
+		boolean flagabbr = false;
+		boolean flagforeign = false;
+				
+		File xpathfile = infile;
+		titreId = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:title[@type='normal']/text()");
+		String auteur = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:author");
+		String datecompo = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo']/@when");
+		String ssiecle = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo_sous_siecle']/@n");
+		String domaine = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textDesc/tei:domain/@type");
+		String genre = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textDesc/@n");
+		String forme = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textClass/tei:catRef/@target[contains(.,'forme')]");
+		String dialecte = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:region[@type='dialecte_auteur']");
+		if(titreId == null)
+			System.err.println("titreId regexp error");
+		if(auteur == null)
+			System.err.println("auteur regexp error");
+		if(datecompo == null)
+			System.err.println("datecompo regexp error");
+		if(ssiecle == null)
+			System.err.println("ssiecle regexp error");
+		if(domaine == null)
+			System.err.println("domaine regexp error");
+		if(genre == null)
+			System.err.println("genre regexp error");
+		if(forme == null)
+			System.err.println("forme regexp error");
+		
+		if(dialecte == null)
+			System.err.println("dialecte regexp error");
+		if(titreId == null || auteur == null || datecompo == null || ssiecle == null || domaine == null || genre == null || forme == null || dialecte == null)
+			return;
+		forme = forme.substring(1);
+		createNextOutput();
+		
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		writer.write("<html>");
+		writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.write("<head>");
+		writer.write("<title>BFM - "+titreId+"</title>");
+		writer.write("</head>");
+		writer.write("<body>");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+					case "text":
+						
+						writer.write("<h1 align=\"center\" style=\"color:darkviolet;font-family:Times;font-weight:bold;font-style:italic;font-size=200%\" class=\"text\">"+titreId+"</h1>")
+						writer.write("<p align=\"center\">")
+						writer.write("_____________________________________________________<br/>\n")
+						writer.write("auteur : "+auteur+"<br/>");
+						writer.write("date de composition : "+datecompo+"<br/>");
+						writer.write("sous-siècle : "+ssiecle+"<br/>");
+						writer.write("domaine : "+domaine+"<br/>");
+						writer.write("genre : "+genre+"<br/>");
+						writer.write("forme : "+forme+"<br/>");
+						writer.write("dialecte : "+dialecte+"<br/>");
+						writer.write("_____________________________________________________<br/>\n")
+						writer.write("</p>")
+						
+						
+						multiwriter.write("<h1 align=\"center\" style=\"color:darkviolet;font-family:Times;font-weight:bold;font-style:italic;font-size=200%\" class=\"text\">"+titreId+"</h1>")
+						multiwriter.write("<p align=\"center\">")
+						multiwriter.write("_____________________________________________________<br/>\n")
+						multiwriter.write("auteur : "+auteur+"<br/>");
+						multiwriter.write("date de composition : "+datecompo+"<br/>");
+						multiwriter.write("sous-siècle : "+ssiecle+"<br/>");
+						multiwriter.write("domaine : "+domaine+"<br/>");
+						multiwriter.write("genre : "+genre+"<br/>");
+						multiwriter.write("forme : "+forme+"<br/>");
+						multiwriter.write("dialecte : "+dialecte+"<br/>");
+						multiwriter.write("_____________________________________________________<br/>\n")
+						multiwriter.write("</p>")
+						
+						break;
+						break;
+						case "head":
+							writer.write("<h2>\n")
+							multiwriter.write("<h2>\n")
+							break;
+						
+						case "lg":
+							writer.write("<p>\n")
+							multiwriter.write("<p>\n")
+							break;
+						
+						case "pb":
+							if(wordcount > 0)
+								createNextOutput();
+							if(parser.getAttributeValue(null,"n") != null)
+							{
+								writer.write("<p style=\"color:red\" align=\"center\">- "+parser.getAttributeValue(null,"n")+" -</p>\n")
+								multiwriter.write("<p style=\"color:red\" align=\"center\">- "+parser.getAttributeValue(null,"n")+" -</p>\n")
+							}
+							break;
+						
+						case "ab":
+							if(parser.getAttributeValue(null,"n") != null)
+							{
+								writer.write("<p align=\"center\">"+parser.getAttributeValue(null,"n")+"</p>\n")
+								multiwriter.write("<p align=\"center\">"+parser.getAttributeValue(null,"n")+"</p>\n")
+							}
+							break;
+						case "l":
+							writer.write("<br/>\n")
+							multiwriter.write("<br/>\n")
+							break;
+						
+						case "p":
+						case "q":
+							writer.write("<p>\n")
+							multiwriter.write("<p>\n")
+							break;
+						
+						case "div":
+							if(parser.getAttributeValue(null,"type") == "titre")
+							{
+								writer.write("<h3>\n")
+								multiwriter.write("<h3>\n")
+								closeTitre = true;
+							}
+							else if(parser.getAttributeValue(null,"type") == "amen")
+							{
+								writer.write("<h3>\n")
+								multiwriter.write("<h3>\n")
+								closeAmen = true;
+							}
+							break;
+													
+						case "w":
+							wordid = parser.getAttributeValue(null,"id");
+							if(firstWord)
+							{
+								firstWord=false;
+								this.idxstart.set(0,wordid);
+							}
+							wordcount++;
+							if(wordcount >= wordmax)
+							{
+								
+								wordcount=0;
+							}
+							break;
+						
+						case "choice":
+							flagchoice = true;
+							break;
+						case "corr":
+							flagcorr = true;
+							break;
+						case "sic":
+							flagsic = true;
+							break;
+						case "reg":
+							flagreg = true;
+							break;
+						case "orig":
+							flagorig = true;
+							break;
+						case "foreign":
+							flagforeign = true;
+							break;
+						
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+						
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "head":
+							writer.write("</h2>\n")
+							multiwriter.write("</h2>\n")
+							break;
+						
+						case "div":
+							if(closeTitre)
+							{
+								writer.write("</h3>\n")
+								multiwriter.write("</h3>\n")
+								closeTitre = false;
+							}
+							else if(closeAmen)
+							{
+								writer.write("</h3>\n")
+								multiwriter.write("</h3>\n")
+								closeTitre = false;
+							}
+							break;
+						
+						case "lg":
+							writer.write("</p>\n")
+							multiwriter.write("</p>\n")
+							break;
+						
+						case "lb":
+							writer.write("<br/>\n")
+							multiwriter.write("<br/>\n")
+							break;
+						
+						case "p":
+						case "q":
+							writer.write("</p>\n")
+							multiwriter.write("</p>\n")
+							break;
+						
+						case "choice":
+							flagchoice = false;
+							break;
+						case "corr":
+							flagcorr = false;
+							break;
+						case "sic":
+							flagsic = false;
+							break;
+						case "reg":
+							flagreg = false;
+							break;
+						case "orig":
+							flagorig = false;
+							break;
+						case "foreign":
+							flagforeign = false;
+							break;
+						
+						case "form":
+							flagform = false
+							break;
+						
+						case "ana":
+							flaginterp = false
+							break;
+						
+						case "w":
+							int l = lastword.length();
+							String color = "";
+							if(flagcorr)
+								color = "style=\"color: green;\"";
+							else if(flagreg)
+								color = "style=\"color: blue;\"";
+							else if(flagforeign)
+								color = "style=\"color: blue;\"";
+							if(!flagchoice || flagcorr || flagreg)
+							{
+								String endOfLastWord = "";
+								if(l > 0)
+									endOfLastWord = lastword.subSequence(l-1, l);
+								
+								if(interpvalue != null)
+									interpvalue = interpvalue.replace("\"","&quot;");
+								
+								if(NoSpaceBefore.contains(wordvalue) || 
+										NoSpaceAfter.contains(lastword) || 
+										wordvalue.startsWith("-") || 
+										NoSpaceAfter.contains(endOfLastWord))
+								{
+									writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\" "+color+">");
+									multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\" "+color+">");
+								}
+								else
+								{
+									writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\" "+color+">");
+									multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\" "+color+">");
+								}
+								writer.write(wordvalue.replace("<", "&lt;")+"</span>");
+								multiwriter.write(wordvalue.replace("<", "&lt;")+"</span>");
+								lastword=wordvalue;
+							}
+						
+						
+							break;
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		writer.close();
+		multiwriter.write("</body>");
+		multiwriter.write("</html>");
+		multiwriter.close();
+		inputData.close();
+	}
+
+	/**
+	 * Writecorrespondances file.
+	 *
+	 * @param file the file
+	 * @return the java.lang. object
+	 */
+	public writecorrespondancesFile(File file)
+	{
+		Writer w = new OutputStreamWriter(new FileOutputStream(file,file.exists()) , "UTF-8");
+		for(int i = 0 ; i < pages.size(); i ++)
+		{
+			w.write(pages.get(i).getName()+"\t"+idxstart.get(i)+"\n")
+		}
+		w.close();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/corptef/importer.groovy	(revision 321)
@@ -0,0 +1,166 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.corptef;
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import javax.xml.stream.*;
+import java.net.URL;
+
+import org.txm.Toolbox;
+
+import filters.CutHeader.*;
+import filters.ReunitBrokenWords.*;
+import filters.Tokeniser.*;
+import filters.TagSentences.*;
+import filters.FusionHeader.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+	
+	/**
+	 * Run.
+	 *
+	 * @param dir the dir
+	 */
+	public void run(File dir)
+	{
+		//prepare
+		String rootDir = dir.getAbsolutePath()+"/";
+
+		//clean directories
+		File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/corptef");
+		binDir.deleteDir();
+		binDir.mkdir();
+		new File(binDir,"ptokenized").deleteDir();
+		new File(binDir,"ptokenized").mkdir();
+		new File(binDir,"tokenized").deleteDir();
+		new File(binDir,"tokenized").mkdir();
+		new File(binDir,"headers").deleteDir();
+		new File(binDir,"headers").mkdir();
+		new File(binDir,"txm").deleteDir();
+		new File(binDir,"txm").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();
+
+		List<File> files = new File(rootDir,"").listFiles();//scan directory split
+
+		//PREPARE EACH FILE TO BE TOKENIZED
+		for(File f : files)
+		{
+			File srcfile = f;
+			File resultfile = new File(binDir,"ptokenized/"+f.getName());
+			println("prepare tokenizer file : "+srcfile+" to : "+resultfile );
+
+			def builder2 = new OneTagPerLine(srcfile.toURL(), milestones);
+			builder2.process(resultfile);
+		}
+
+		rootDir = binDir.getAbsolutePath();
+
+		//TOKENIZE FILES
+		//Manager<Filter> filterManager = new FilterManager(ActionHome);	
+		files = new File(rootDir,"ptokenized").listFiles();//scan directory split
+		for(File infile : files)
+		{
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F2 = new ReunitBrokenWords();
+			Filter F6 = new Tokeniser(infile);
+			Filter F7 = new TagSentences();
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			S.add(F2);
+			S.add(F6);
+			//S.add(F7);
+			S.add(F11);
+
+			File xmlfile = new File(rootDir,"tokenized/"+infile.getName());
+			println("Tokenizing "+xmlfile)
+			S.SetInFileAndOutFile(infile.getAbsolutePath(), xmlfile.getAbsolutePath());
+			S.setEncodages("UTF-8","UTF-8");
+
+			Object[] arguments1 = [rootDir+"/headers/"+infile.getName()+"header.xml"];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [rootDir+"/headers/"+infile.getName()+"header.xml",F1];
+			F11.SetUsedParam(arguments2);
+
+			S.proceed();
+			S.clean();
+		}
+
+		files = new File(rootDir,"tokenized").listFiles()
+		//TRANSFORM INTO XML-TEI-TXM
+		for(File f : files)
+		{
+			//ArrayList<String> milestones = new ArrayList<String>();
+			println("Building xml-tei-txm "+f+ " >> "+f.getName())
+			File file = f; 
+			String txmfile = f.getName();
+
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();	
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+
+			//lance le traitement
+			def builder3 = new Xml2Ana(file);
+			builder3.setCorrespondances(correspRef, correspType);
+			builder3.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			builder3.transformFile(rootDir,"txm/"+txmfile);
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("C:/TXM/corpora/corptef/")
+		new importer().run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/rgaqcjLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/rgaqcjLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/rgaqcjLoader.groovy	(revision 321)
@@ -0,0 +1,139 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.RGAQCJ
+;
+
+import org.txm.importer.RGAQCJ.importer;
+import org.txm.importer.RGAQCJ.compiler;
+import org.txm.importer.bfm.pager;
+import org.txm.objects.*;
+import org.txm.utils.i18n.*;
+import org.txm.*;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		rootDir = userDir+"/xml/weblex/";
+		lang="fr";
+		encoding= "UTF-8";
+		model="rgaqcj";// not used
+		Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File(userDir,"txminstall"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"txminstall/treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"txminstall/treetagger/models"));
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+File dir = new File(rootDir);
+String basename = dir.getName();
+
+println "-- IMPORTER - Reading source files"
+File paramfile = new File(userDir, "TXM/weblexloader.prefs")
+def imp = new importer()
+if(!imp.run(dir, paramfile, basename))
+{
+	println("Stop import process")
+	return;
+}
+
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+rootDir = binDir.getAbsolutePath();
+dir = new File(rootDir);
+
+println "No annotation to do"
+
+
+println "-- COMPILER - Building Search Engine indexes"
+def c = new compiler();
+c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setLang(lang);
+c.run(dir);
+
+
+//move registry file to cwb registry dir
+File registryfile = new File(rootDir+"/registry", basename);
+if(registryfile.exists())
+	FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "registry/"+basename))
+
+Workspace w = org.txm.Toolbox.workspace;
+Project p = w.getProject("default")
+p.removeBase(basename)
+Base b = p.addBase(basename);
+b.addDirectory(new File(rootDir, "txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION"
+new File(rootDir+"/HTML/").deleteDir()
+new File(rootDir+"/HTML/").mkdir();
+new File(rootDir+"/HTML/default/").mkdir();
+files = new File(rootDir,"txm").listFiles();
+println files
+for(String textname : b.getTextsID())
+{
+	Text text = b .getText(textname);
+	File srcfile = text.getSource();
+	File resultfile = new File(rootDir+"/HTML", srcfile.getName().substring(0, srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+	println("Building edition  : "+srcfile+" to : "+resultfile );
+	
+	def ed = new pager(srcfile, resultfile, NoSpaceBefore, NoSpaceAfter, 600);
+	//ed.writecorrespondancesFile(new File(rootDir,"edition_correspondances.txt"))
+	Edition editionweb = text.addEdition("default", "html", resultfile);
+
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f, idx);
+	}
+	
+//	Edition editionbp = text.addEdition("onepage", "html", resultfile);
+//	editionbp.addPage(resultfile, ed.getIdx().get(0));
+}
+
+w.save();
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/importer.groovy	(revision 321)
@@ -0,0 +1,266 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.RGAQCJ
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import javax.xml.stream.*;
+import java.net.URL;
+import java.util.Properties;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.FusionHeader.*;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+import org.apache.commons.lang.StringUtils;
+import org.txm.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+	
+	/** The tokenize. */
+	boolean tokenize = true;
+	
+	/**
+	 * Run.
+	 *
+	 * @param dir the dir
+	 * @param paramfile the paramfile
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File dir, File paramfile, String basename)
+	{
+		if(!paramfile.exists())
+		{
+			System.err.println("Parameter file does not exists: "+paramfile.getAbsolutePath());
+			return false;
+		}
+		Properties properties = new Properties();
+		BufferedReader input = new BufferedReader(new InputStreamReader(new FileInputStream(paramfile) , "UTF-8"));
+		String line = input.readLine();
+		while(line != null)
+		{
+			String[] split = line.split("="); //$NON-NLS-1$
+			if(split.length == 2)
+			{
+				properties.put(split[0], split[1]);
+			}
+			else if(split.length > 2)
+			{
+				String[] subsplit = new String[split.length -1 ]; 
+				System.arraycopy(split, 1, subsplit, 0, split.length-1);
+				properties.put(split[0], StringUtils.join(subsplit, "=")); //$NON-NLS-1$
+			}
+			line = input.readLine();
+		}
+		
+		if(!(properties.containsKey("nbP") && properties.containsKey("nbTaxo") && properties.containsKey("nbResp")))
+		{
+			System.err.println("Missing property: nbP or nbTaxo or nbResp");
+			return false;
+		}
+		int nbP = Integer.parseInt(properties.getProperty("nbP"));
+		int nbTaxo = Integer.parseInt(properties.getProperty("nbTaxo"));
+		int nbResp = Integer.parseInt(properties.getProperty("nbResp"));
+		
+		if(nbResp == 0)
+		{
+			System.err.println("No resp ");
+			return false;
+		}
+		String rootDir = dir.getAbsolutePath()+"/";
+		ArrayList<String> milestones = new ArrayList<String>();
+		
+		//where the binaries will be created
+		File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+		binDir.deleteDir();
+		binDir.mkdir();
+		
+		new File(binDir,"txm").deleteDir();
+		new File(binDir,"txm").mkdir();
+		
+		List<File> files = new File(rootDir,"").listFiles();
+		
+		//set working directory
+		rootDir = binDir.getAbsolutePath()+"/";
+		
+		//Set import parameters
+		def correspType = new HashMap<String,String>();
+		// correspType(attribut word wlx, attribut type de la propriété ana du w txm)
+		for(int p = 1 ; p <= nbP ; p++)
+		{
+			correspType.put("p"+p, properties.get(properties.get("P"+p+"_taxo")));
+		}
+		
+		def correspRef = new HashMap<String,String>()
+		// correspRef (attribut word wlx, attribut ref de la propriété ana du w txm. ref pointe vers l'identifiant du respStmt du TEIheader)
+		for(int p = 1 ; p <= nbP ; p++)
+		{
+			String taxo=properties.get("P"+p+"_taxo")
+			String resp=properties.get(taxo+"_resp")
+			correspRef.put("p"+p,properties.get(resp));
+		}
+		
+		//il faut lister les id de tous les respStmt
+		def respId = [];
+		for(int r = 1 ; r <= nbResp ; r++)
+			respId << properties.get("R"+r);
+		
+		//fait la correspondance entre le respId et le rapport d'execution de l'outil
+		
+		def applications = new HashMap<String,HashMap<String,String>>();	
+		//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+		//pour construire les ref vers les taxonomies
+		for(int r = 1 ; r <= nbResp ; r++)
+		{
+			applications.put(properties.get("R"+r), new ArrayList<String>());
+			applications.get(properties.get("R"+r)).add(properties.get("R"+r+"_app_id"));//app ident
+			applications.get(properties.get("R"+r)).add(properties.get("R"+r+"_app_version"));//app version
+			applications.get(properties.get("R"+r)).add(properties.get("R"+r+"_app_reportfile"));//app report file path
+		}
+
+		
+		def taxonomiesUtilisees = new HashMap<String,String[]>();
+		//associe un id d'item avec sa description et son URI
+		for(int t = 1 ; t <= nbTaxo ; t++)
+		{
+			String resp_id = properties.get(properties.get("T"+t+"_resp"))
+			if(!taxonomiesUtilisees.containsKey(resp_id))
+				taxonomiesUtilisees.put(resp_id,[]);//,"lemma","lasla","grace"]);
+			taxonomiesUtilisees.get(resp_id) << properties.get("T"+t)
+		}
+		
+		def itemsURI = new HashMap<String,HashMap<String,String>>();
+		//informations de respStmt
+		//resps (respId <voir ci-dessus>, [description, person, date])
+		for(int t = 1 ; t <= nbTaxo ; t++)
+		{
+			String taxo = properties.get("T"+t);
+			itemsURI.put(taxo,new HashMap<String,String>());
+			itemsURI.get(taxo).put("tagset",properties.get("T"+t+"_tagset"));
+			itemsURI.get(taxo).put("website",properties.get("T"+t+"_web"));
+		}
+		def resps = new HashMap<String,String[]>();
+		for(int r = 1 ; r <= nbResp ; r++)
+		{
+			resps.put(properties.get("R"+r), [properties.get("R"+r+"_desc"),properties.get("R"+r+"_who"),properties.get("R"+r+"_when"),properties.get("R"+r+"_day")])
+		}
+		
+		println("Weblex import parameters : ")
+		println("resps id "+respId);
+		println("resps infos"+resps);
+		println("applications "+applications);
+		
+		println("correspType "+correspType)
+		println("correspRef "+correspRef)
+		
+		println("taxonomiesUtilisees "+taxonomiesUtilisees)
+		println("itemsURI "+itemsURI)
+				
+		//TRANSFORM INTO XML-TEI-TXM
+		for(File f : files)
+		{
+			//ArrayList<String> milestones = new ArrayList<String>();
+			File file = f; 
+			String txmfile = f.getName();
+			println("Building xml-tei-txm "+f+ " >> "+rootDir+"txm/"+txmfile)
+			
+			//lance le traitement
+			def builder3 = new Xml2Ana(file);
+			builder3.setCorrespondances(correspRef, correspType);
+			builder3.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			builder3.transformFile(rootDir+"txm/",txmfile);
+		}
+		return true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/bfm/")
+		new importer().run(dir);
+	}
+}
+
+/* PARAM FILE EXAMPLE
+nbP=3
+nbTaxo=2
+nbResp=2
+
+R1=init
+R1_desc=initial taggin
+R1_who=al
+R1_when=2010
+R1_day=Tue Mar  2 21:02:55 Paris, Madrid 2010
+R1_app_id=appR1
+R1_app_version=app1V
+R1_app_reportfile=
+
+R2=apinit
+R2_desc=second taggin
+R2_who=slh
+R2_when=2010
+R2_day=Tue Mar  2 21:02:55 Paris, Madrid 2010
+R2_app_id=appR2
+R2_app_version=app2V
+R2_app_reportfile=
+
+P1=truc1
+P1_taxo=T1
+
+P2=bidul2
+P2_taxo=T1
+
+P3=machin3
+P3_taxo=T2
+
+T1=CATTEX
+T1_web=www.google.fr
+T1_tagset=www.google.fr
+T1_resp=R1
+
+T2=TTFR
+T2_web=www.bing.fr
+T2_tagset=www.bing.fr
+T2_resp=R2 
+*/
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/importRGAQCJ.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/importRGAQCJ.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/importRGAQCJ.groovy	(revision 321)
@@ -0,0 +1,424 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.RGAQCJ
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+import org.txm.importer.filters.*;
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class BuildXmlRGAQCJ.
+ *
+ * @author mdecorde
+ * simple import in cwb of RGAQCJ
+ * structunits : initial, s, lb ...
+ * wordproperties : form, pos and n
+ */
+
+public class BuildXmlRGAQCJ
+{
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private def parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+	
+	/** The types. */
+	private ArrayList<String> types;
+	
+	/** The current type. */
+	private String currentType;
+	
+	/** The cat ref. */
+	private String catRef="";
+	
+	/** The initial. */
+	private String initial;
+	
+	/**
+	 * initialize.
+	 *
+	 * @param url the url
+	 * @param initial the initial
+	 * @param types the types
+	 */
+	public BuildXmlRGAQCJ(URL url,String initial,ArrayList<String> types){
+		try {
+			this.url = url;
+			this.types = types;
+			this.initial = initial;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * clear anaHash variable, it is used to store ana tags values then print it when the end element </ana> is found.
+	 */
+	private void fillanaHash()
+	{
+		anahash.clear();
+		for(String s : types)
+			anahash.put( s,"-" );
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName output directory
+	 * @param fileName output file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			dir = new File(dirPathName)
+			File f = new File(dir, fileName);
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * Checks if is body.
+	 *
+	 * @param name the name
+	 * @return true, if is body
+	 */
+	private static boolean isBody(String name) {
+		if (name.equals("body")) return true;
+	}
+	
+	/**
+	 * Find body.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean findBody(){
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()){
+			if(event == XMLStreamConstants.START_ELEMENT){
+				if (parser.getLocalName() == "catRef"){
+					if(parser.getAttributeValue(null,"target") == "#forme_prose")
+						catRef = "prose";
+					else if(parser.getAttributeValue(null,"target") == "#forme_vers")
+						catRef = "vers";
+					else
+						catRef = "mixte";
+					//println("catRef : "+catRef);
+				}
+				if (isBody(parser.getLocalName())){
+					return true;
+				}
+			}
+		}
+		return false;
+	} 
+	
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		if(findBody() && createOutput(dirPathName, fileName))
+		{
+			def idPb = "";
+			def idLb = "";
+			String idLinesuiv;
+			
+			boolean flagForm = false;
+			boolean flagAna = false;
+			String vAna = "";
+			String vWord = "";
+			String vForm = "";
+			
+			try 
+			{
+				File inputfile = new File(this.url.getFile());
+				output.write( "<text initiale=\""+initial+"\" forme=\""+catRef+"\">\n");
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+				{
+					switch (event) 
+					{
+						case XMLStreamConstants.START_ELEMENT:
+						switch (parser.getLocalName()) 
+						{
+							case "p":
+								output.write "<p n=" + parser.getAttributeValue(null,"n") +">\n";
+								break;
+							
+							case "s":
+								output.write "<s n=" + parser.getAttributeValue(null,"n")+ "\">\n";
+								break;
+							
+							case "pb":
+								idLinesuiv = parser.getAttributeValue(null,"n");
+								idPb = idLinesuiv;
+								break;
+								
+							case "lb":
+								idLb = parser.getAttributeValue(null,"n");
+								break;
+							
+							case "w":
+								fillanaHash();
+								vWord ="";
+							break;
+							
+							case "form":
+								flagForm =true;
+								vForm ="";
+							break;
+							case "ana":
+								flagAna = true;
+								vAna ="";
+								
+								currentType = (parser.getAttributeValue(null,"ref"));
+								if(currentType != null)
+									currentType= currentType.substring(1)
+								else
+									flagAna = false;
+							break;
+						}
+						break;
+						
+						case XMLStreamConstants.END_ELEMENT:
+						switch (parser.getLocalName()) 
+						{
+							case "p":
+								output.write( "</p>\n");
+								break;
+							
+							case "s":
+								output.write( "</s>\n");
+								break;
+
+							case "w":
+								for(String type : types)
+									vWord+="\t"+anahash.get(type)
+								if(catRef.equals("vers"))
+									output.write( vWord+ "\t" +idLb+ "\n");
+								else
+									output.write( vWord+ "\t" +idPb+ "\n");
+								vWord= "";
+								break;
+							
+							case "form":
+								flagForm = false;
+								vWord +=vForm; 
+								break;
+								
+							case "ana":
+								if(flagAna)
+									anahash.put (currentType,vAna);
+								flagAna = false;
+								break;
+						}
+						break;
+						
+						case XMLStreamConstants.CHARACTERS:
+							if(flagForm){
+								vForm += parser.getText().trim();
+							}
+							if(flagAna){
+								vAna += parser.getText().trim();
+							}
+						break;
+					}
+				}
+				output.write( "</text>\n");
+				output.close();
+				parser.close();
+			}
+			catch (XMLStreamException ex) {
+				System.out.println(ex);
+			}
+			catch (IOException ex) {
+				System.out.println("IOException while parsing " + inputData);
+			}
+		}
+		return true;
+	}
+	
+	/**
+	 * Gets the anatypes.
+	 *
+	 * @param rootDir the root dir
+	 * @param xmltxmfiles the xmltxmfiles
+	 * @return the anatypes
+	 */
+	public static def getAnatypes(String rootDir, List<String> xmltxmfiles)
+	{
+		ArrayList<String> types = new ArrayList<String>()
+		for(int i=0; i < xmltxmfiles.size();i++)
+		{
+			//println("look in "+xmltxmfiles[i]);
+			URL url = new File(rootDir+"/anainline/", xmltxmfiles[i]).toURL();
+			def inputData = url.openStream();
+			def factory = XMLInputFactory.newInstance();
+			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+			
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+				if(event == XMLStreamConstants.START_ELEMENT)
+					if(parser.getLocalName().equals("ana"))
+					{
+						String s = parser.getAttributeValue(null, "ref");
+						if(s != null)
+						{
+							s = s.substring(1, s.length());
+							if(!types.contains(s))
+							{
+								types.add(s);
+							}
+						}
+					}
+			
+			inputData=null;
+			factory=null
+			parser=null;
+		}
+		return types;		
+	}
+		
+	/**
+	 * Process.
+	 *
+	 * @param files the files
+	 * @param rootDir the root dir
+	 * @param cwbLoc the cwb loc
+	 */
+	public static void process(List<String> files, String rootDir,String cwbLoc) 
+	{
+		new File(rootDir+"wtc/").mkdir();
+		new File(rootDir+"registry/").mkdir();
+		//String[] files = ["roland-ana.xml","qgraal_cm-ana.xml","artu-ana.xml","qjm-ana.xml","commyn1-ana.xml","jehpar-ana.xml"];
+		//String rootDir = "~/xml/rgaqcj/";
+		ArrayList<String> types = BuildXmlRGAQCJ.getAnatypes(rootDir, files);
+		println(types);
+		
+		//1- Transform into WTC file
+		File f = new File(rootDir+"wtc/","RGAQCJ.wtc");
+		f.delete();
+		
+		def output = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+		output.write("<corpus name=\"RGAQCJ\">\n");
+		output.close();
+		
+		
+		def initiales = "RAQCJ";
+		for(int i=0; i < files.size();i++)
+		{
+			println("process file "+files[i])
+			String file = files[i]; 
+			def builder = new BuildXmlRGAQCJ(new File(rootDir+"/anainline/",file).toURL(),""+initiales.charAt(i),types);
+			builder.transfomFileWtc(rootDir+"wtc","RGAQCJ.wtc");
+		}
+		
+		output = new OutputStreamWriter(new FileOutputStream(f,true) , "UTF-8");
+		output.write("</corpus>\n");
+		output.close();
+
+		//2- Import into CWB
+		
+		//def cwbLoc ="~/Bureau/textometrie/CWB/cwb-3.0/utils/";//chemin vers executable cqp
+
+		def inDir = rootDir+"src/";
+		def outDir =rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		
+		types.add("line");
+		String[] pAttributes = types;
+		String[] sAttributes = ["corpus:0+name","text:0+initiale+forme", "s:0+n","p:0+n"];
+		
+		try
+		{
+			if (System.getProperty("os.name").contains("Windows"))
+			{
+				cwbEn.run(cwbLoc + "cwb-encode.exe", outDirTxm + "data/"+"RGAQCJ", outDir +"wtc/"+"RGAQCJ"+".wtc", outDirTxm + "registry/"+"rgaqcj",pAttributes, sAttributes);
+				cwbMa.run(cwbLoc + "cwb-makeall.exe", "RGAQCJ", outDirTxm + "registry");
+			}
+			else
+			{
+				cwbEn.run(cwbLoc + "cwb-encode", outDirTxm + "data/"+"RGAQCJ", outDir + "/wtc/"+"RGAQCJ.wtc", outDirTxm + "registry/"+"rgaqcj",pAttributes, sAttributes);
+				cwbMa.run(cwbLoc + "cwb-makeall", "RGAQCJ", outDirTxm + "registry");
+
+			}
+		} catch (Exception ex) {System.out.println(ex);}
+		System.out.println("Done.") 
+		
+		return
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		List<String> files = ["roland-ana.xml","qgraal_cm-ana.xml","artu-ana.xml","qjm-ana.xml","commyn1-ana.xml","jehpar-ana.xml"];
+		String rootDir = "~/xml/rgaqcj/";
+		def cwbLoc ="~/Bureau/textometrie/CWB/cwb-3.0/utils/";//chemin vers executable cqp
+		BuildXmlRGAQCJ.process( files,  rootDir, cwbLoc);
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/compiler.groovy	(revision 321)
@@ -0,0 +1,606 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.RGAQCJ
+
+
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler 
+{
+	
+	/** The debug. */
+	private boolean debug= false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+	
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+	
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		createOutput(dirPathName, fileName);
+		
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+		
+		int p_id = 0;
+		int q_id = 0;
+		int body_id = 0;
+		int front_id = 0;
+		int back_id = 0;
+		String lb_id = 0;
+		int lb_count = 0;
+		String pb_id = 0;
+		int pb_count = 0;
+		String ab_id = 0;
+		
+		boolean captureword = false;
+		
+		String vExpan = "";
+		String vCorr = "";
+		String vReg = "";
+		String vOrig = "";
+		String vSic = "";
+		String vAbbr = "";
+		
+		boolean flaglg = false;
+		boolean flaghead = false;
+		boolean flagAuthor = false;
+		boolean flagDate = false;
+		boolean flagWord = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+		
+		boolean flagchoice = false;
+		boolean flagcorr = false;
+		boolean flagsic = false;
+		boolean flagreg = false;
+		boolean flagexpan = false;
+		boolean flagorig = false;
+		boolean flagabbr = false;
+		boolean flagfw = false;
+		
+		File xpathfile = new File(url.getFile());
+		String titreId = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:title[@type='reference']/text()");
+		String auteur = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:author/text()");
+		String datecompo = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo']/@when");
+		String ssiecle = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo_sous_siecle']/@n");
+		String domaine = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textDesc/tei:domain/@type");
+		String genre = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textDesc/@n");
+		String forme = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:textClass/tei:catRef/@target[contains(.,'forme')]").substring(1);
+		String dialecte = XPathResult.getXpathResponse(xpathfile,"tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:region[@type='dialecte_auteur']/text()");
+		
+		this.GoToText()
+		output.write("<txmcorpus lang=\""+lang+"\">\n"); 
+		try 
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+						switch (parser.getLocalName()) 
+						{						
+							case "text":
+							output.write("<text id=\""+text+"\"" +
+							" titre=\""+titreId+"\"" +
+							" auteur=\""+auteur+"\"" +
+							" datecompo=\""+datecompo+"\"" +
+							" ssiecle=\""+ssiecle+"\"" +
+							" domaine=\""+domaine+"\"" +
+							" genre=\""+genre+"\"" +
+							" forme=\""+forme+"\"" +
+							" dialecte=\""+dialecte+"\"" +
+							" base=\""+base+"\"" +
+							" project=\""+project+"\">\n");
+							captureword=true;
+							break;
+							
+							case "div":
+							//output.write("<div type=\""+parser.getAttributeValue(null,"type")+"\">\n");
+							break;
+							
+							case "p":
+							output.write("<p n=\""+(p_id++)+"\">\n");
+							break;
+							case "ab":
+							output.write("<ab n=\""+ab_id+++"\" rend=\""+parser.getAttributeValue(null,"rend")+"\">\n");
+							break;
+							case "q":
+							output.write("<q n=\""+(q_id++)+"\">\n");
+							break;
+							case "front":
+							output.write("<front n=\""+(front_id++)+"\">\n");
+							break;
+							case "body":
+							output.write("<body n=\""+(body_id++)+"\">\n");
+							break;
+							case "back":
+							output.write("<back n=\""+(back_id++)+"\">\n");
+							break;
+							case "lb":
+							String n = parser.getAttributeValue(null,"n")
+							if(n != null)
+								lb_id = n;
+							else
+								lb_id =""+lb_count++;
+							break;
+							case "pb":
+							String n = parser.getAttributeValue(null,"n");
+							if(n != null)
+								pb_id = n;
+							else
+								pb_id = ""+(pb_count++);
+							
+							break;
+							
+							case "choice":
+							flagchoice = true;
+							break;
+							case "corr":
+							flagcorr = true;
+							vCorr= "";
+							break;
+							case "reg":
+							flagreg = true;
+							vReg= "";
+							break;
+							case "expan":
+							flagexpan = true;
+							vExpan= "";
+							break;
+							case "orig":
+							flagreg = true;
+							vOrig= "";
+							break;
+							case "sic":
+							flagsic = true;
+							vSic= "";
+							break;
+							case "abbr":
+							flagreg = true;
+							vAbbr= "";
+							break;
+							
+							case "w":
+							for(int i = 0 ; i < parser.getAttributeCount(); i++)
+								if(parser.getAttributeLocalName(i).equals("id"))
+								{	
+									wordid = parser.getAttributeValue(i);
+									break;
+								}
+							vAna = "";
+							vForm = "";
+							flagWord = true;
+							break;
+							case "form":
+							flagForm = true;
+							vForm = "";
+							vAna ="";
+							break;
+							
+							case "ana":
+							flagAna = true;
+							break;
+						}
+						break;
+					
+					case XMLStreamConstants.END_ELEMENT:
+						switch (parser.getLocalName()) 
+						{	
+							case "div":
+							break;
+							
+							case "text":
+							output.write("</text>\n");
+							captureword=false;
+							break;
+							case "p":
+							output.write("</p>\n");
+							break;
+							case "ab":
+							output.write("</ab>\n");
+							break;
+							case "q":
+							output.write("</q>\n");
+							break;
+							case "front":
+							output.write("</front>\n");
+							break;
+							case "body":
+							output.write("</body>\n");
+							break;
+							case "back":
+							output.write("</back>\n");
+							break;
+							
+							case "fw":
+							flagfw = false;
+							break;
+							case "choice":
+								
+								if(vCorr != "")
+								{
+									//System.out.println(vCorr+" >> write corr "+vForm +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr)
+									output.write( vCorr +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+								}
+								else if(vReg != "")
+								{
+									//System.out.println("write reg "+vForm)
+									output.write( vReg +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+								}
+								else if(vExpan != "")
+								{
+									//System.out.println("write expan "+vForm)
+									output.write( vExpan +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+								}	
+							flagchoice = false;
+							vCorr= "";
+							vSic= "";
+							break;
+							case "corr":
+							flagcorr = false;
+							
+							break;
+							case "reg":
+							flagreg = false;
+							vReg = "";
+							break;
+							case "expan":
+							flagexpan = false;
+							vExpan= "";
+							break;
+							case "orig":
+							flagreg = false;
+							vOrig= "";
+							break;
+							case "sic":
+							flagsic = false;
+							
+							break;
+							case "abbr":
+							flagreg = false;
+							vAbbr= "";
+							break;
+							
+							case "w":
+							if(vAna != null)
+								if(captureword)
+								{
+									if(flagchoice)
+									{
+										
+									}
+									else if(flagfw)
+									{
+										
+									}
+									else
+									{
+										output.write( vForm +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\n");
+									}
+									
+								}
+
+							flagWord = false;
+							break;
+							
+							case "form":
+							flagForm = false;
+							break;
+							
+							case "ana":
+							vAna += "\t";
+							flagAna = false;
+							break;
+						}
+						break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						if(flagAna)
+						{
+							vAna += parser.getText().trim();
+						}
+					
+						if(flagForm)
+						{
+							vForm += parser.getText().trim();
+							if(flagchoice)
+							{
+								if(flagsic)
+								{
+									vSic += parser.getText().trim();
+								}
+								if(flagorig)
+								{
+									vOrig += parser.getText().trim();
+								}
+								if(flagabbr)
+								{
+									vAbbr += parser.getText().trim();
+								}
+								if(flagcorr)
+								{
+									vCorr += parser.getText().trim();
+								}
+							}
+						}
+				}
+			}
+			output.write("</txmcorpus>"); 
+			output.close();
+			parser.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+		
+		return true;
+	}
+	
+	
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @return true, if successful
+	 */
+	public boolean run(File rootDirFile) 
+	{
+		String rootDir =rootDirFile.getAbsolutePath();
+		
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!new File(rootDir).exists())
+		{
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+		new File(rootDir+"/wtc/","bfm3tt.wtc").delete();//cleaning&preparing
+		new File(rootDir,"/wtc/").deleteDir();
+		new File(rootDir,"/wtc/").mkdir();
+		new File(rootDir,"/data/").deleteDir();
+		new File(rootDir,"/data/").mkdir();
+		new File(rootDir,"registry/").mkdir();
+		
+		String textid="";
+		int counttext =0;
+		List<File> files = new File(rootDirFile,"txm").listFiles();
+		//1- Transform into WTC file
+		for(File f : files)
+		{
+			counttext++;
+			if(!f.exists())
+			{
+				println("file "+f+ " does not exists")	
+			}
+			else
+			{	
+				println("process file "+f)
+				def builder = new compiler(f.toURL(),f.getName(),"bfm3tt","default");
+				builder.setLang(lang);
+				builder.transfomFileWtc(rootDir+"/wtc","bfm3tt.wtc");
+			}
+		}
+		
+		 //2- Import into CWB
+		 def outDir =rootDir;
+		 def outDirTxm = rootDir;
+		 CwbEncode cwbEn = new CwbEncode();
+		 cwbEn.setDebug(debug);
+		 CwbMakeAll cwbMa = new CwbMakeAll();
+		 cwbMa.setDebug(debug);
+		 String[] pAttributes = ["pos","id","pb","lb"];
+		 String[] sAttributes = ["txmcorpus:0+lang","text:0+id+titre+auteur+datecompo+ssiecle+domaine+genre+forme+dialecte+base+project","front:0+n","body:0+n","ab:0+n+rend","div:0+id+type","q:1+n","p:0+n","back:0+n"];
+		 try
+		 {
+		 cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), outDirTxm + "/data", outDir + "/wtc/"+"bfm3tt.wtc", outDirTxm + "/registry/"+"bfm3tt",pAttributes, sAttributes);
+		 cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), "BFM3TT", outDirTxm + "/registry");
+		 } catch (Exception ex) {System.out.println(ex); return false;}
+		 
+		System.out.println("Done.") 
+		
+		return true;
+	}
+	
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/bfm3tt");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>RGAQCJ import module - old</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RGAQCJ/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/AddAttributeInXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/AddAttributeInXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/AddAttributeInXml.groovy	(revision 321)
@@ -0,0 +1,120 @@
+package org.txm.importer
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.txm.metadatas.Entry
+import org.txm.utils.Pair;
+
+/**
+ * Add a attribute value map in a XML file
+ * Warning: if an attribute already exists its value won't be changed
+ */
+class AddAttributeInXml extends StaxIdentityParser {
+	File xmlfile;
+	String tag;
+	List<Pair<String, String>> attributes;
+	HashMap<String, String> attributesMap;
+	boolean result;
+
+	public AddAttributeInXml(File xmlfile, String tag, List<Pair<String, String>> attributes)
+	{
+		super(xmlfile.toURI().toURL());
+		this.xmlfile = xmlfile;
+		this.tag = tag;
+		this.attributes = attributes;
+	}
+
+	public AddAttributeInXml(File xmlfile, String tag, HashMap<String, String> attributesMap) {
+		super(xmlfile.toURI().toURL());
+		this.xmlfile = xmlfile;
+		this.tag = tag;
+		this.attributesMap = attributesMap;
+	}
+
+	public AddAttributeInXml(File xmlfile, String tag,	ArrayList<org.txm.metadatas.Entry> attributes) {
+		super(xmlfile.toURI().toURL());
+		this.xmlfile = xmlfile;
+		this.tag = tag;
+		this.attributes = attributes;
+	}
+
+	public boolean process(File outfile) {
+		this.result = false;
+		boolean ret = super.process(outfile)
+		return this.result & ret;
+	}
+
+	
+	/**
+	 * Rewrite the processStartElement() to update/add attributes
+	 */
+	public void processStartElement()
+	{
+		String prefix = parser.getPrefix();
+//TODO: uncomment for TXM 0.7.6 and later
+//			if (INCLUDE == localname && XI == prefix) {
+//				processingXInclude();
+//				return;
+//			}
+		
+		if (prefix.length() > 0)
+			writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+		else
+			writer.writeStartElement(localname);
+
+		for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {
+			writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+		}
+
+		HashSet<String> exists = new HashSet<String>();
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			exists.add(parser.getAttributeLocalName(i));
+			writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+		}
+		if (localname == tag && !stopInsertion) { // insert here
+			result = true
+			if (attributes != null) {
+				for (Pair<String, String> meta : attributes) {
+					if(stopAtFirst)
+						stopInsertion = true;
+					if(!exists.contains(meta.getFirst())) // I can add the new attribute
+						writer.writeAttribute(meta.getFirst(), meta.getSecond());
+				}
+			} else if (attributesMap != null) {
+				for (String name : attributesMap.keySet()) {
+					if (stopAtFirst)
+						stopInsertion = true;
+					if (!exists.contains(name)) // I can add the new attribute
+						writer.writeAttribute(name, attributesMap.get(name));
+				}
+			}
+		}
+	}
+	
+	boolean stopAtFirst = false;
+	boolean stopInsertion = false;
+	public void onlyOneElement() {
+		stopAtFirst = true;
+	}
+
+	public static void main(String[] args)
+	{
+		File xmlfile = new File("/home/mdecorde/xml/xmlçàé/essai.xml")
+		File temp = File.createTempFile("temp", ".xml", xmlfile.getParentFile());
+		String tag = "titre";
+		ArrayList<Pair<String, String>> metadatas = new ArrayList<Pair<String, String>>();
+		metadatas.add(new Pair<String, String>("name1", "newvalue1"));
+		metadatas.add(new Pair<String, String>("name2", "value2"));
+		metadatas.add(new Pair<String, String>("name3", "value3"));
+
+		AddAttributeInXml builder = new AddAttributeInXml(xmlfile, tag, metadatas);
+		builder.onlyOneElement();
+		if(builder.process(temp)) {
+			if (!(xmlfile.delete() && temp.renameTo(xmlfile))) println "Warning can't rename file "+temp+" to "+xmlfile
+		} else {
+			println "Error during process"
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HasElement.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HasElement.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HasElement.groovy	(revision 321)
@@ -0,0 +1,67 @@
+package org.txm.importer
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+/**
+ * Check if a file contains a tag
+ * @author mdecorde
+ *
+ */
+class HasElement {
+
+	HashSet<String> localnames = new HashSet<String>();
+	File xmlfile;
+	XMLStreamReader parser;
+	HashMap<String, Boolean> elements = [:];
+
+	public HasElement(File xmlfile, String localname) {
+		localnames << localname;
+		this.xmlfile = xmlfile;
+		elements[localname] = false;
+
+		init();
+	}
+
+	private void init() {
+		try {
+			
+//			println "xmlfile: "+xmlfile
+//			println "xmlfile exists: "+xmlfile.exists()
+//			println "uri: "+ xmlfile.toURI()
+//			println "url: "+ xmlfile.toURI().toURL()
+//			
+			def inputData = xmlfile.toURI().toURL().openStream();
+			def inputFactory = XMLInputFactory.newInstance();
+			parser = inputFactory.createXMLStreamReader(inputData);
+		} catch (Exception ex) {
+			System.out.println("IOException while parsing: "+ex);
+			ex.printStackTrace();
+		}
+	}
+
+	public boolean process() {
+		if (parser == null) return false;
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				String localname = parser.getLocalName();
+				if (localnames.contains(localname)) {
+					parser.close();
+					return true;
+				}
+			}
+		}
+		parser.close();
+	}
+	
+	public static void main(String[] args) {
+		File dir = new File("/home/mdecorde/xml/lamoplat")
+		println dir.listFiles()
+		for (File xmlfile : dir.listFiles()) {
+			if (!xmlfile.getName().endsWith(".xml")) continue
+			//println xmlfile
+			println "$xmlfile: "+ new HasElement(xmlfile, "text").process();
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/CommentIncluder.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/CommentIncluder.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/CommentIncluder.groovy	(revision 321)
@@ -0,0 +1,46 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer.transcriber
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class CommentIncluder.
+ */
+class CommentIncluder {
+	
+	/**
+	 * Instantiates a new comment includer.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public CommentIncluder(File infile, File outfile)
+	{
+		
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TokenizeEntities.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TokenizeEntities.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TokenizeEntities.groovy	(revision 321)
@@ -0,0 +1,56 @@
+package org.txm.importer.transcriber
+
+import org.txm.importer.StaxIdentityParser
+
+
+/**
+ * Wrap  "entities" span event in "w" tags 
+ * 
+ * @author mdecorde
+ *
+ */
+class TokenizeEntities extends StaxIdentityParser{
+	int wordcount = 1;
+	boolean writeStartWord = false;
+	boolean writeEndWord = false;
+	public TokenizeEntities(URL url) {
+		super(url);
+	}
+	
+	public void processStartElement() {
+		// close previous word
+		if (localname == "Event" && parser.getAttributeValue(null, "type") == "entities") {
+			if (parser.getAttributeValue(null, "extent") == "end") { // end of word
+				writer.writeEndElement();
+				writeEndWord = false;
+			}
+		}
+		
+		super.processStartElement();
+		
+		// start new word if necessary
+		if (localname == "Event" && parser.getAttributeValue(null, "type") == "entities") {
+			if (parser.getAttributeValue(null, "extent") == "begin") { // start of word
+				writeStartWord = true;
+			}
+		}
+	}
+	
+	public void processEndElement() {
+
+		super.processEndElement();
+		
+		if (writeStartWord) {
+			writer.writeStartElement("w");
+			//writer.writeAttribute("id", "w_entity_"+(wordcount++));
+			writeStartWord = false;
+		} 
+	}
+	
+	public static void main(String[] args) {
+		File dir = new File ("C:\\Documents and Settings\\mdecorde\\xml\\minileman2011")
+		File infile = new File(dir, "int05.trs")
+		TokenizeEntities tokenizer = new TokenizeEntities(infile.toURI().toURL());
+		println tokenizer.process(new File(dir, "out.trs")) 
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/pager.groovy	(revision 321)
@@ -0,0 +1,783 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package org.txm.importer.transcriber
+
+import java.io.File;
+import java.util.ArrayList;
+
+import javax.xml.stream.*
+
+import org.txm.importer.ApplyXsl2
+import org.txm.metadatas.Metadatas
+import org.txm.utils.io.FileCopy;
+
+
+// TODO: Auto-generated Javadoc
+/** Build Discours corpus simple edition from a xml-tei.
+ * 
+ *  @author mdecorde
+ *  
+ */
+class pager {
+	
+	boolean SIMPLE_TOOLTIP = false; // show less properties in word tooltips
+	String ENQ_HIGHLIGHT_ELEMENT = "b"
+	
+	List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The pages. */
+	def pages = [];
+	def indexes = [];
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 10;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue;
+
+	/** The interpvalue. */
+	String interpvalue;
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	boolean flagcomment = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	XMLStreamWriter writer;
+	BufferedOutputStream output;
+
+	File txmfile;
+
+	File outfile;
+
+	String corpusname ="";
+	String cuttingTag = "pb"
+	String txtname;
+	File htmlDir;
+	File defaultDir;
+	Metadatas metadatas;
+
+	def interviewers = [];
+	def eventTranslations = ["^^":"mot inconnu", "?":"orthographe incertaine",
+		"()":"rupture de syntaxe", "b":"bruit indéterminé",
+		"*":"mot corrigé",
+		"bb":"bruit de bouche", "bg":"bruit de gorge",
+		"ch":"voix chuchotée", "conv":"conversations de fond",
+		"e":"expiration", "i":"inspiration",
+		"mic":"bruits micro", "n":"reniflement",
+		"nontrant":"non transcrit", "pap":"froissement de papiers",
+		"pf":"souffle", "pi":"inintelligible",
+		"pif":"inaudible", "r":"respiration",
+		"rire":"rire du locuteur", "shh":"soufle électrique",
+		"sif":"sifflement du locuteur", "tx":"toux"];
+	String currentTime = "";
+	boolean bold = false;
+	int writenLength = 0;
+	boolean spokenTurn = false;
+	boolean firstSync = false;
+	boolean firstWho = false;
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param metadatas the metadatas
+	 */
+	pager(File txmfile, File htmlDir, String txtname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String corpusname, String cuttingTag, Metadatas metadatas) {
+		this.metadatas = metadatas
+		this.wordmax = max;
+		this.cuttingTag = cuttingTag;
+		this.corpusname = corpusname;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = txmfile.toURI().toURL();
+		this.txmfile = txmfile;
+		this.htmlDir = htmlDir;
+		this.txtname = txtname;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+
+		defaultDir = new File(htmlDir, "default")
+		defaultDir.mkdir()
+		new File(htmlDir, "onepage").mkdir()
+		outfile = new File(htmlDir, "onepage/${txtname}.html");
+		createOutput(outfile)
+
+		try {
+			process();
+		} catch(Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			if (writer != null) {
+				writer.close();
+				output.close();
+			}
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			//println "write html in : "+outfile
+			XMLOutputFactory outfactory = XMLOutputFactory.newInstance();
+			output = new BufferedOutputStream(new FileOutputStream(outfile))
+			writer = outfactory.createXMLStreamWriter(output, "UTF-8");//create a new file
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/** The events. */
+	List<String> events = [];
+	String previousEvent = "", nextEvent = "";
+	/**
+	 * Process.
+	 */
+	void process() {
+
+		String previousElem = "";
+		boolean parolesRaportees = false;
+		boolean firstWord = true;
+		boolean shouldBreak = false;
+		boolean overlapping = false;
+		int nbBreak = 0;
+		String previousSPK;
+		String localname = "";
+		ArrayList<String> whos = [];
+		HashMap<String, String> speakers = new HashMap<String, String>();
+		HashMap<String, String> topics = new HashMap<String, String>();
+
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeStartElement("html");
+		//<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		writer.writeStartElement("meta");
+		writer.writeAttribute("http-equiv", "Content-Type");
+		writer.writeAttribute("content", "text/html");
+		writer.writeAttribute("charset", "UTF-8");
+		writer.writeEndElement();
+		writer.writeStartElement("head");
+
+		//<link rel="stylesheet" type="text/css" href="class.css" />
+		writer.writeStartElement("link");
+		writer.writeAttribute("rel", "stylesheet");
+		writer.writeAttribute("type", "text/css");
+		writer.writeAttribute("href", "transcriber.css");
+		writer.writeEndElement();
+		writer.writeEndElement();
+
+		nbBreak++
+		writer.writeStartElement("body");
+		writer.writeEmptyElement("pb");
+		writer.writeAttribute("id", ""+nbBreak);
+		pages << new File(defaultDir, "${txtname}_${nbBreak}.html")
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							writer.writeStartElement("h2");
+							writer.writeAttribute("class","titre");
+							String title = parser.getAttributeValue(null, "title");
+							if (title != null) {
+								writer.writeCharacters(title);
+							} else {
+								writer.writeCharacters("Transcription "+txmfile.getName().substring(0, txmfile.getName().length() - 4));
+							}
+							writer.writeEndElement();
+							if(metadatas != null) {
+								writer.writeStartElement("table");
+								boolean grey = false;
+								for (String name : metadatas.getPropertyNames()) {
+									if ("title" == name) continue; // ignore "title" metadata
+									grey = !grey;
+									writer.writeStartElement("tr");
+									if (grey)
+										writer.writeAttribute("style","background-color:lightgrey;")
+									String value = parser.getAttributeValue(null, name);
+									if (value != null) {
+										writer.writeStartElement("td");
+										writer.writeCharacters(name);
+										writer.writeEndElement();
+										writer.writeStartElement("td");
+										writer.writeCharacters(value);
+										writer.writeEndElement();
+									}
+									//get enqueteur to style their names
+									if (name.startsWith("enq")) {
+										interviewers.add(value)
+									}
+									writer.writeEndElement();
+								}
+								writer.writeEndElement();
+							}
+							break;
+						case "Topics":
+						/*writer.writeStartElement("h2");
+					 writer.writeCharacters("Topics");
+					 writer.writeEndElement();
+					 writer.writeStartElement("ul");
+					 */
+							break;
+						case "Topic":
+							topics.put(parser.getAttributeValue(null,"id"), parser.getAttributeValue(null,"desc"))
+						/*writer.writeStartElement("li");
+					 writer.writeCharacters(parser.getAttributeValue(null,"desc"));
+					 writer.writeStartElement("ul");
+					 for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+					 {
+					 if(parser.getAttributeLocalName(i) != "desc")
+					 {
+					 writer.writeStartElement("li");
+					 writer.writeCharacters(parser.getAttributeLocalName(i)+": "+parser.getAttributeValue(i));
+					 writer.writeEndElement();
+					 }
+					 }
+					 writer.writeEndElement();
+					 writer.writeEndElement();
+					 */
+							break;
+						case "Speakers":
+						/*writer.writeStartElement("h2");
+					 writer.writeCharacters("Speakers");
+					 writer.writeEndElement();
+					 writer.writeStartElement("ul");*/
+							break;
+						case "Speaker":
+							whos.add(parser.getAttributeValue(null,"name"));
+							speakers.put(parser.getAttributeValue(null,"id"), parser.getAttributeValue(null,"name"))
+						/*writer.writeStartElement("li");
+					 writer.writeStartElement("ul");
+					 writer.writeCharacters(parser.getAttributeValue(null,"name"));
+					 for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+					 {
+					 if(parser.getAttributeLocalName(i) != "name")
+					 {
+					 writer.writeStartElement("li");
+					 writer.writeCharacters(parser.getAttributeLocalName(i)+": "+parser.getAttributeValue(i));
+					 writer.writeEndElement();
+					 }
+					 }
+					 writer.writeEndElement();
+					 writer.writeEndElement();*/
+							break;
+						case "Comment":
+							spokenTurn = true;
+							writenLength++;
+							writer.writeStartElement("span");
+							writer.writeAttribute("class", "comment");
+							writer.writeCharacters(" ["+parser.getAttributeValue(0)+"] ");
+							writer.writeEndElement();
+							flagcomment = true;
+							break;
+						case "div":
+							writer.writeStartElement("div");
+							writer.writeAttribute("class", "section");
+							String type = parser.getAttributeValue(null,"type");
+							String desc = parser.getAttributeValue(null,"topic");
+							if (type != null && desc != null) {
+								writer.writeStartElement("h3");
+								writer.writeCharacters(type+": "+desc);
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									String name = parser.getAttributeLocalName(i);
+									if (!"type".equals(name)
+									&& !"topic".equals(name)
+									&& !"startTime".equals(name)
+									&& !"endTime".equals(name)) {
+										writer.writeCharacters(", "+name+": "+parser.getAttributeValue(i));
+									}
+								}
+								writer.writeEndElement(); // h3
+							}
+							break;
+						case "sp":
+							endBoldIfNeeded()
+							firstSync = true;
+							firstWho = true;
+							spokenTurn = false;
+							overlapping = false
+
+							writer.writeStartElement("p");
+							writer.writeAttribute("class", "turn");
+
+							overlapping = ("true" == parser.getAttributeValue(null,"overlap"))
+							String spid = parser.getAttributeValue(null,"speaker");
+							whos = []
+							if (overlapping) {
+								writer.writeEmptyElement("br");
+								writeSpeaker(parser.getAttributeValue(null,"speaker"), false)
+								
+								writer.writeEmptyElement("br");
+								whos = spid.split(" ")
+							}
+
+							break;
+						case "u":
+							writer.writeCharacters("\n");
+							this.currentTime = parser.getAttributeValue(null,"time");
+						
+							if (previousElem == "u" && writenLength == 0) { // if previous u had no words, it was a silence
+								writer.writeStartElement("span");
+								writer.writeAttribute("class", "event");
+								writer.writeCharacters("[silence]");
+								writer.writeEndElement(); // span
+								writer.writeEmptyElement("br");
+							}
+							
+							String spk = parser.getAttributeValue(null, "spk")
+							if (spk != null && spk != previousSPK) {
+								endBoldIfNeeded()
+								writer.writeEmptyElement("br");
+								writeSpeaker(parser.getAttributeValue(null, "spk"), overlapping)
+								startBoldIfNeeded()
+							}
+
+							writeCurrentTime()
+							previousSPK = spk
+
+						//							writenLength = 0;
+						/*writer.writeStartElement("span");
+					 writer.writeAttribute("class", "sync");
+					 writer.writeCharacters("["+parser.getAttributeValue(null,"time")+"]");
+					 writer.writeEndElement();*/
+
+							break;
+						case "event":
+							spokenTurn = true;
+							writenLength++;
+							String desc = parser.getAttributeValue(null,"desc");
+							desc = translateEvent(desc);
+							String type = parser.getAttributeValue(null,"type");
+							if (desc.equals("paroles rapportées")) {
+								if (parser.getAttributeValue(null, "extent") == "end")
+									writer.writeCharacters("» ");
+								else if (parser.getAttributeValue(null, "extent") == "begin")
+									writer.writeCharacters(" «");
+							} else {
+								writer.writeStartElement("span");
+								writer.writeAttribute("class", "event");
+								if (parser.getAttributeValue(null, "extent") == "end") {
+									writer.writeCharacters(" <"+desc+"] ");
+									if(events.size() > 0)
+										events.remove(events.size()-1)
+								}
+								else if (parser.getAttributeValue(null, "extent") == "begin") 	{
+
+									writer.writeCharacters(" ["+desc+"> ");
+									events.add(desc)
+								}
+								else if (parser.getAttributeValue(null, "extent") == "previous") {
+									if(parser.getAttributeValue(null, "type") == "pronounce")
+										writer.writeCharacters("_["+desc+"] ");
+									else
+										writer.writeCharacters("_["+desc+"] ");
+									previousEvent = desc;
+								}
+								else if (parser.getAttributeValue(null, "extent") == "next") {
+									writer.writeCharacters(" ["+desc+"]_");
+									nextEvent = desc
+								}
+								else
+									writer.writeCharacters(" ["+desc+"] ");
+								writer.writeEndElement(); // span@class=event
+							}
+							break;
+						case "w":
+							for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+								if(parser.getAttributeLocalName(i) == "id") {
+									wordid = (parser.getAttributeValue(i));
+									break;
+								}
+							if (firstWord) {
+								indexes << wordid
+								firstWord = false;
+							}
+							wordcount++;
+							if (wordcount >= wordmax) {
+								shouldBreak = true;
+							}
+							break;
+
+						case "ana":
+							
+							String type = parser.getAttributeValue(null,"type").substring(1);
+							if (SIMPLE_TOOLTIP) {
+								if (type.contains("lemma") || type.contains("pos")) {
+									flaginterp=true;
+									interpvalue+=", ";
+								}
+							} else {
+								flaginterp=true;
+								interpvalue+=", "+type+"="
+							}
+							break;
+
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+							break;
+					}
+					previousElem = localname;
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "text":
+							break;
+						case "Topics":
+						//writer.writeEndElement();
+							break;
+						case "Topic":
+							break;
+						case "Speakers":
+						//println "Speakers: "+speakers
+						//writer.writeEndElement();
+							break;
+						case "Speaker":
+							break;
+
+						case "div":
+						//writer.writeCharacters("}");
+							
+							writer.writeEndElement(); // div
+							writer.writeCharacters("\n");
+							break;
+						case "sp":
+							//println "CLOSING: "+parser.getLocalName()
+							endBoldIfNeeded()
+							if (!spokenTurn) {
+								writer.writeStartElement("span");
+								writer.writeAttribute("class", "event");
+								writer.writeCharacters("[silence]");
+								writer.writeEndElement();
+								writer.writeEmptyElement("br");
+							}
+
+							writer.writeEndElement(); // p
+
+							if (shouldBreak) {
+								nbBreak++
+								writer.writeEmptyElement("pb");
+								writer.writeAttribute("id", ""+nbBreak);
+								writer.writeCharacters("\n");
+								
+								pages << new File(defaultDir, "${txtname}_${nbBreak}.html")
+								indexes << wordid
+
+								wordcount = 0;
+								shouldBreak = false;
+							}
+							writer.writeCharacters("\n");
+							break;
+						case "u":
+							//writer.writeEndElement() // span@class=u
+							//writer.writeEmptyElement("br");
+						//if (overlapping) writer.writeEndElement(); // b
+							break;
+						case "event":
+							break;
+						case "form":
+							flagform = false
+							break;
+						case "ana":
+							flaginterp = false
+							break;
+						case "w":
+							writenLength++;
+							spokenTurn = true;
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if(l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+
+							if(interpvalue != null)
+								interpvalue = interpvalue.replace("\"","&quot;");
+							if(events.size() > 0)
+								interpvalue = interpvalue.replace("event=", "event="+events.toString().replace("\"","&quot;")); // remove ", "
+
+							if(nextEvent.length() > 0)
+							{
+								interpvalue = interpvalue.replace("event=", "event="+nextEvent+", ")
+								nextEvent = ""
+							}
+							interpvalue = interpvalue.replace("=, ","='', "); // add '' to empty interp value
+							if (interpvalue.startsWith(", ")) interpvalue = interpvalue.substring(2)
+//							println "** SPACE TEST"
+//							println "NoSpaceBefore: "+NoSpaceBefore+" contains ? "+wordvalue
+//							println "NoSpaceAfter: "+NoSpaceAfter+" contains ? "+lastword
+//							println "wordvalue starts with '-' ? "+wordvalue
+//							println "NoSpaceAfter: "+NoSpaceAfter+" contains endOfLastWord ? "+endOfLastWord
+							if(NoSpaceBefore.contains(wordvalue) ||
+							NoSpaceAfter.contains(lastword) ||
+							wordvalue.startsWith("-") ||
+							NoSpaceAfter.contains(endOfLastWord)) {
+//								println " NO SPACE"
+							} else {
+//								println " SPACE"
+								writer.writeCharacters(" ");
+							}
+
+							if (interpvalue.contains("rapp1")) {
+								writer.writeCharacters(" «");
+							} else if (wordvalue == "\"") {
+								// don't write this char
+							} else {
+								writer.writeStartElement("span");
+								writer.writeAttribute("class", "word");
+								writer.writeAttribute("title", interpvalue);
+								writer.writeAttribute("id", wordid);
+								writer.writeCharacters(wordvalue);
+								writer.writeEndElement();
+							}
+							if (interpvalue.contains("orth")) {
+								writer.writeStartElement("span");
+								writer.writeAttribute("class", "event");
+								writer.writeCharacters("_[?]");
+								writer.writeEndElement();
+							}
+							if (interpvalue.contains("corr")) {
+								writer.writeStartElement("span");
+								writer.writeAttribute("class", "event");
+								writer.writeCharacters("_[!]");
+								writer.writeEndElement();
+							}
+
+							if (interpvalue.contains("rapp2")) {
+								writer.writeCharacters(" » ");
+							}
+							
+							lastword=wordvalue;
+							break;
+					}
+					
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText().trim());
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText().trim());
+					break;
+			}
+		}
+		writer.writeEndElement();
+
+		writer.writeEmptyElement("pb");
+		nbBreak++
+		writer.writeAttribute("id", ""+nbBreak);
+
+		writer.writeEndElement();
+		writer.close();
+		output.close();
+
+		File txmhome = new File(org.txm.Toolbox.getParam(org.txm.Toolbox.USER_TXM_HOME));
+		File xlsDir  = new File(txmhome, "xsl");
+		File xslfile = new File(xlsDir,"breakByMilestone.xsl");
+		if (!xslfile.exists()) {
+			println ""
+		}
+		//		println "xsl: "+xslfile
+		//		println "html: "+outfile
+		//		println "pages: "+pages
+		//		println "words: "+indexes
+
+
+				if (pages.size() > 1) {
+					for (int i = 1 ; i < nbBreak ; i++) {
+						ApplyXsl2 a = new ApplyXsl2(xslfile.getAbsolutePath());
+						String[] params = ["pbval1", i,"pbval2", i+1];
+		
+						File resultfile = pages[i-1]
+						//println "BBmilestones: "+i+" "+(i+1)+" in file "+resultfile
+						//println "process $outfile -> $resultfile"
+						a.process(outfile.getAbsolutePath(), resultfile.getAbsolutePath(), params);
+					}
+				} else {
+					File page = pages[0]
+					FileCopy.copy(outfile, page)
+				}
+	}
+
+	private void writeCurrentTime()
+	{
+		writer.writeStartElement("span");
+		writer.writeAttribute("class", "sync");
+		writer.writeCharacters(currentTime);
+		writer.writeEndElement();
+	}
+
+	private void writeSpeaker(String spk, boolean overlapping) {
+		
+		writer.writeStartElement("span");
+		writer.writeAttribute("class", "spk");
+		if(interviewers.contains(spk))
+			bold = true;
+		else
+			bold = false;
+		spk = spk.replaceAll('^([^0-9]*)([0-9]+)$', '$1 $2');
+		if (overlapping) writer.writeCharacters("// ")
+		writer.writeCharacters(spk+": ")
+		writer.writeEndElement(); // span@class=spk
+	}
+
+	private String translateEvent(String desc) {
+		if(eventTranslations.containsKey(desc))
+			return eventTranslations.get(desc);
+		else
+			return desc;
+	}
+	
+	boolean boldOpenned = false;
+	private void startBoldIfNeeded() {
+		if (bold) {
+			writer.writeStartElement(ENQ_HIGHLIGHT_ELEMENT);
+			boldOpenned = true;
+		}
+	}
+	
+	private endBoldIfNeeded() {
+		if (boldOpenned) {
+//			println "CLOSE BOLD"
+			writer.writeEndElement(); // b
+			boldOpenned = false;
+		}
+	}
+
+	//	private String formatTime(float time, boolean doshort)
+	//	{
+	//		String rez = " ";
+	//		//		if(time >= 3600) // >= 1h
+	//		//		{
+	//		float h = time / 3600;
+	//		time = time%3600;
+	//		float min = (time%3600) / 60;
+	//		int sec = (int)time%60;
+	//
+	//		if(min < 10)
+	//			rez = ""+(int)h+":0"+(int)min;//+":"+time%60;
+	//		else
+	//			rez = ""+(int)h+":"+(int)min;//+":"+time%60;
+	//		//if (!doshort)
+	//		if (sec > 9)
+	//			rez += ":"+(int)time%60;
+	//		else
+	//			rez += ":0"+(int)time%60;
+	//		//		}
+	//		//		else if(time >= 60) // >= 1min
+	//		//		{
+	//		//			int min = time/60;
+	//		//			if(min < 10)
+	//		//				rez = "00:0"+min;//+":"+time%60;
+	//		//			else
+	//		//				rez = "00:"+min;//+":"+time%60;
+	//		//			if(!doshort)
+	//		//				rez += ":"+(int)time%60;
+	//		//		}
+	//		//		else // < 60
+	//		//		{
+	//		//			if(time < 10)
+	//		//				return " 0:0"+time;
+	//		//			else
+	//		//				return " 0:"+time;
+	//		//		}
+	//		return rez;
+	//	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return indexes;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/FixTiming.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/FixTiming.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/FixTiming.groovy	(revision 321)
@@ -0,0 +1,227 @@
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS  FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2011-11-01 16:12:36 +0100 (mar., 01 nov. 2011) $
+// $LastChangedRevision: 2049 $
+// $LastChangedBy: sheiden $
+//
+
+package org.txm.importer.transcriber
+import java.text.DecimalFormat;
+// parameters
+
+String userdir = System.getProperty("user.home")
+File infile = new File(userdir, "xml/concattrs/Reçues/int23.trs")
+File outfile = new File(userdir, "xml/concattrs/Reçues/int23-corr.trs")
+
+public boolean checkSpk(String spk) {
+	return spk.matches(ValidateTRS.spkPattern);
+}
+
+public fixTopics(def trs) {
+	
+	def okTopics = ValidateTRS.okTopics;
+	
+	// get Topic ids
+	def ids2desc = [:];
+	def declaredTopics = trs.Topics.Topic;
+	for (def topic : declaredTopics) {
+		ids2desc[topic.@id]=topic.@desc
+	}
+	
+	// fix Sections
+	for(def section : trs.Episode.Section) {
+
+		if (section.@topic != null)
+		if (okTopics.contains(ids2desc[section.@topic])) {
+			section.@topic = ids2desc[section.@topic]
+		}
+	}
+	
+	// fix Section decl
+	for(def topic : trs.Topics.Topic) {
+		if (okTopics.contains(ids2desc[topic.@id])) {
+			topic.@id = topic.@desc
+		}
+	}
+}
+
+def fixTurn(prevTurn, turn) // end < start
+{
+	def start =  Float.parseFloat(turn.'@startTime')
+	def startS = turn.'@startTime'
+	def end =  Float.parseFloat(turn.'@endTime')
+	def endS = turn.'@endTime'
+
+	// patch previous turn
+	if (prevTurn != null) {
+		def prevstart = Float.parseFloat(prevTurn.'@startTime')
+		if (prevstart > end) {
+			println "** FixTiming: Warning: previous turn not updated ($prevstart > $end): $prevTurn"
+		} else {
+			prevTurn.'@endTime' = endS;
+		}
+	} else {
+		turn.parent().'@startTime' = endS
+	}
+
+	// patch current turn
+	turn.'@startTime' = endS;
+	turn.'@endTime' = startS;
+
+	// patch next turn
+	def i = turn.parent().children().indexOf(turn)
+	if (turn.parent().children().size() > i+1) {
+		def nextTurn = turn.parent().children().get(i+1)
+		if (nextTurn != null) {
+			def nextend = Float.parseFloat(nextTurn.'@endTime')
+			nextTurn.'@startTime' = startS; // patch next Turn even if nextTurn.end < start, loop will continue to patch next turns
+		}
+	} else {
+		turn.parent().'@endTime' = startS
+	}
+	println "** FixTiming: fixed Turn: old time[$startS, $endS] to ["+turn.@startTime+", "+turn.@endTime+"]"
+}
+
+def fixTurn2(turn, timeResolution) // start == end
+{
+	DecimalFormat formater = DecimalFormat.getInstance(Locale.ENGLISH)
+	formater.setMaximumFractionDigits(3)
+	//DecimalFormat formater = new DecimalFormat("#.000");
+	def start =  Float.parseFloat(turn.'@startTime')
+	def startS = turn.'@startTime'
+	def end =  Float.parseFloat(turn.'@endTime')+timeResolution*10
+	def endS = formater.format(end)
+	println "ENDS: "+endS
+
+	// patch current turn
+	turn.'@endTime' = endS;
+
+	// patch next turn
+	def i = turn.parent().children().indexOf(turn)
+	if (turn.parent().children().size() > i+1) {
+		def nextTurn = turn.parent().children().get(i+1)
+		if (nextTurn != null) {
+			def nextstart = Float.parseFloat(nextTurn.'@startTime')
+			nextTurn.'@startTime' = endS; // patch next Turn even if nextTurn.end < start, loop will continue to patch next turns
+		}
+	} else {
+		turn.parent().'@endTime' = endS
+	}
+
+	println "** FixTiming: fixed Turn: equal old time[$startS, $endS] to ["+turn.@startTime+", "+turn.@endTime+"]"
+}
+
+def fixSync(turn)
+{
+	def start =  Float.parseFloat(turn.'@startTime')
+	def end =  Float.parseFloat(turn.'@endTime')
+	def syncs = turn.Sync
+	def badSyncList = []
+
+	def lastValidSyncTime = start
+	def firstSync = true
+
+	for (def sync : syncs) {
+		if (firstSync) {
+			if (sync.@time != turn.'@startTime') {
+				println "** FixTiming: fixed first Sync: old time "+sync.@time+" to "+turn.'@startTime'
+			}
+			sync.@time = turn.'@startTime'
+			lastValidSyncTime = start
+			firstSync = false
+		} else {
+			def time = Float.parseFloat(sync.@time)
+			if (time > lastValidSyncTime && time < end) {
+				fixBadSyncList(badSyncList, lastValidSyncTime, time)
+				lastValidSyncTime = time
+				badSyncList = []
+			} else {
+				badSyncList << sync
+			}
+		}
+	}
+	fixBadSyncList(badSyncList, lastValidSyncTime, end)
+}
+
+def fixBadSyncList(list, start, end) {
+	def timeResolution = 0.001
+	if (start >= end) {
+		println("** FixTiming: fixBadSyncList: start($start) >= end($end)")
+		return
+	}
+	if (list.size() > 0) {
+		def dt = (end - start)/(list.size()+1)
+		if (dt < timeResolution) {
+			println("** FixTiming: fixBadSyncList: dt($dt) < $timeResolution seconds")
+			return
+		}
+		def st = start
+		list.each { sync ->
+			st += dt
+			def oldtime = sync.@time
+			sync.@time = ""+st
+			println "** FixTiming: fixed Sync: old time $oldtime to $st"
+		}
+	}
+}
+
+public def fixTRS(File infile, File outfile) {
+	def timeResolution = 0.001
+	URL u = infile.toURI().toURL()
+	InputStream ins = u.openStream()
+
+	// Open input file
+	def slurper = new XmlParser();
+	def trs = slurper.parse(infile.toURI().toString())
+
+	// First fix all <Turn>
+	def prevTurn = null
+	for (def section : trs.Episode.Section) {
+		section.Turn.each{ turn ->
+			def start = Float.parseFloat(turn.'@startTime')
+			def end =  Float.parseFloat(turn.'@endTime')
+			if (end < start) {
+				fixTurn(prevTurn, turn)
+			} else if (start == end) {
+				fixTurn2(turn, timeResolution)
+			}
+			prevTurn = turn
+			//} catch (Exception e){ org.txm.utils.logger.Log.printStackTrace(e); println "start: "+turn.'@startTime' println "end: "+turn.'@endTime'}
+		}
+	}
+
+	// Then fix all <Sync>s of Turns
+	for (def section : trs.Episode.Section) {
+		section.Turn.each{ turn ->
+			fixSync(turn)
+		}
+	}
+	
+	// Finally fix topics declarations
+	fixTopics(trs);
+
+	String xml = "";
+
+	println ""+xml
+	outfile.withWriter("UTF-8"){ writer ->
+		writer.write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE Trans SYSTEM "trans-14.dtd">\n')
+		new groovy.util.XmlNodePrinter(new PrintWriter(writer)).print(trs) }
+}
+
+/// MAIN ///
+fixTRS(infile, outfile);
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ChangeSpeaker.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ChangeSpeaker.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ChangeSpeaker.groovy	(revision 321)
@@ -0,0 +1,147 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer.transcriber;
+
+import org.txm.importer.*;
+import org.txm.utils.*;
+import org.txm.metadatas.*;
+
+import java.io.File;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.xml.parsers.*;
+import javax.xml.xpath.*;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+// TODO: Auto-generated Javadoc
+/** remove part of xml given an id. */
+public class ChangeSpeaker {
+	File outfile;
+	
+	/** The doc. */
+	Document doc;
+	
+	/**
+	 * Instantiates a new change speaker.
+	 *
+	 * @param transcriptionfile the transcriptionfile
+	 * @param outfile the outfile
+	 * @param id the id
+	 * @param newid the newid
+	 */
+	public ChangeSpeaker(File transcriptionfile, File outfile, String id, String newid)
+	{
+		this.outfile = outfile;
+		
+		String xpath = "//Turn";
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		doc = builder.parse(transcriptionfile);
+		
+		def expr = XPathFactory.newInstance().newXPath().compile(xpath);
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+		
+		
+		
+		for(def node : nodes)
+		{
+			if (node == null) continue;
+			
+			Element elem = (Element)node;
+			if(elem.getAttribute("speaker").contains(id))
+			{
+				String value = elem.getAttribute("speaker");
+				value = value.replace(id, newid);
+				elem.setAttribute("speaker", value);
+			}
+		}
+		
+		def expr2 = XPathFactory.newInstance().newXPath().compile("//Speaker[@id='"+id+"']");
+		def nodes2 = expr2.evaluate(doc, XPathConstants.NODESET);
+		
+		for(def node : nodes2)
+		{
+			if (node == null) continue;
+			
+			Element elem = (Element)node;
+			elem.setAttribute("id", newid);
+		}
+		save()
+	}
+	
+	/**
+	 * Save.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean save()
+	{
+		try {
+			// Création de la source DOM
+			Source source = new DOMSource(doc);
+			
+			// Création du fichier de sortie
+			Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8")); 
+			Result resultat = new StreamResult(writer);
+			
+			// Configuration du transformer
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			Transformer transformer = fabrique.newTransformer();
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes"); 
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); 
+			
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			return true;
+		} catch (Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			return false;
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String homedir = System.getProperty("user.home")
+		File trs1 = new File(homedir, "xml/concattrs/int18_1.trs")
+		new ChangeSpeaker(trs1, trs1, "spk2", "spk4");
+		new ChangeSpeaker(trs1, trs1, "spk3", "spk2");
+		new ChangeSpeaker(trs1, trs1, "spk4", "spk3");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ConcatTrs.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ConcatTrs.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ConcatTrs.groovy	(revision 321)
@@ -0,0 +1,436 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package org.txm.importer.transcriber
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.*;
+
+import javax.xml.stream.*;
+import java.io.File;
+import java.net.URL;
+import java.text.DecimalFormat;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ConcatTrs.
+ */
+class ConcatTrs {
+
+	/** The localname. */
+	String localname;
+
+	/** The prefix. */
+	String prefix;
+
+	/** The input data1. */
+	InputStream inputData1,/** The input data2. */
+	inputData2;
+
+	/** The parser1. */
+	XMLStreamReader parser1,
+	/** The parser2. */
+	parser2;
+
+	/** The output. */
+	BufferedOutputStream output;
+
+	/** The writer. */
+	XMLStreamWriter writer;
+
+	/** The inputfactory. */
+	XMLInputFactory inputfactory = XMLInputFactory.newInstance();
+
+	/** The factory. */
+	XMLOutputFactory factory = XMLOutputFactory.newInstance();
+
+	/** The speakers. */
+	HashMap<File, String> speakers = new HashMap<File, String>();
+
+	/**
+	 * Instantiates a new concat trs.
+	 *
+	 * @param trs1 the trs1
+	 * @param trs2 the trs2
+	 * @param result the result
+	 */
+	public ConcatTrs(File trs1, File trs2, File result)
+	{
+		// t1<-open("t1.trs")
+		// t2<-open("t2.trs")
+		// out<-open("t3.trs", "w")
+		inputData1 = trs1.toURI().toURL().openStream();
+		parser1 = inputfactory.createXMLStreamReader(inputData1);
+
+		inputData2 = trs2.toURI().toURL().openStream();
+		parser2 = inputfactory.createXMLStreamReader(inputData2);
+
+		output = new BufferedOutputStream(new FileOutputStream(result))
+		writer = factory.createXMLStreamWriter(output, "UTF-8");
+
+		// check-heads(t1, t2) // same speakers
+		if(!testSpeakers())
+		{
+			println "stop"
+			return;
+		}
+
+		//get shift time trs1
+		inputData1.close();
+		inputData1 = trs1.toURI().toURL().openStream();
+		parser1 = inputfactory.createXMLStreamReader(inputData1);
+		float shift = getLastTime(parser1);
+
+		//get last time trs2
+		inputData2.close();
+		inputData2 = trs2.toURI().toURL().openStream();
+		parser2 = inputfactory.createXMLStreamReader(inputData2);
+		float end = getLastTime(parser2);
+
+		end += shift;
+		println "shift: $shift ; end: $end"
+
+		// copy-head-body-no-tail(t1, out) // shift<-last("sp", t1).time
+		inputData1.close();
+		inputData1 = trs1.toURI().toURL().openStream();
+		parser1 = inputfactory.createXMLStreamReader(inputData1);
+		writeFirstPartAndGetLastTime(parser1, end);
+
+		// copy-body+tail-time-shifted(t2, out, shift)
+		inputData2.close();
+		inputData2 = trs2.toURI().toURL().openStream();
+		parser2 = inputfactory.createXMLStreamReader(inputData2);
+		writeSecondPartAndShiftTime(parser2, shift);
+
+		writer.flush();
+		writer.close();
+		output.close();
+
+		inputData1.close();
+		inputData2.close();
+	}
+
+	/**
+	 * Test speakers.
+	 *
+	 * @return true, if successful
+	 */
+	protected boolean testSpeakers()
+	{
+		String speakers1 = getSpeaker(parser1);
+		String speakers2 = getSpeaker(parser2);
+
+		boolean ret = speakers1 == speakers2;
+
+		if(!ret)
+		{
+			println "Speakers are differents : "
+			println "  speakers1= $speakers1"
+			println "  speakers2= $speakers2"
+		}
+		else
+			println "speakers OK $speakers1 and $speakers2"
+		return ret;
+	}
+
+	/**
+	 * Gets the speaker.
+	 *
+	 * @param parser the parser
+	 * @return the speaker
+	 */
+	protected String getSpeaker(XMLStreamReader parser)
+	{
+		//println "get speakers of $parser"
+
+		ArrayList<String> spksvalues = [];
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event)
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+				//println localname
+					if(localname == "Speaker")
+					{
+						spksvalues << "("+parser.getAttributeValue(null,"id")+"-"+parser.getAttributeValue(null,"name")+")";
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					if(localname == "Speakers")
+					{
+						Collections.sort(spksvalues);
+						return spksvalues.toString();
+					}
+					break;
+			}
+		}
+	}
+
+	/**
+	 * Gets the last time.
+	 *
+	 * @param parser the parser
+	 * @return the last time
+	 */
+	protected float getLastTime(XMLStreamReader parser)
+	{
+		float timef = 0.0f
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event)
+			{
+				case XMLStreamConstants.START_ELEMENT:
+
+					localname = parser.getLocalName();
+					prefix = parser.getPrefix();
+
+					if(localname == "Turn")
+					{
+						String time = parser.getAttributeValue(null, "endTime")
+						if(time != null)
+							timef = Float.parseFloat(time)
+					}
+			}
+		}
+		return timef;
+	}
+
+	/**
+	 * Write first part and get last time.
+	 *
+	 * @param parser the parser
+	 * @param end the end
+	 */
+	protected void writeFirstPartAndGetLastTime(XMLStreamReader parser, float end)
+	{
+		println "First part"
+
+		DecimalFormat f = new DecimalFormat("#.000"); //$NON-NLS-1$
+		writer.writeStartDocument("UTF-8", "1.0");
+		writer.writeCharacters("\n")
+		writer.writeDTD("<!DOCTYPE Trans SYSTEM \"trans-14.dtd\">")
+		writer.writeCharacters("\n")
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event)
+			{
+				case XMLStreamConstants.START_ELEMENT:
+
+					localname = parser.getLocalName();
+					prefix = parser.getPrefix();
+
+					if(prefix != null && prefix.length() > 0)
+						writer.writeStartElement(prefix+":"+localname);
+					else
+						writer.writeStartElement(localname);
+
+					for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+					{
+						String name = parser.getAttributeLocalName(i)
+						String value = parser.getAttributeValue(i);
+
+						if(parser.getAttributePrefix(i)!= "")
+							writer.writeAttribute(parser.getAttributePrefix(i)+":"+name, value);
+						else
+							writer.writeAttribute(name, value);
+					}
+
+
+					break;
+
+				case XMLStreamConstants.END_ELEMENT:
+
+					localname = parser.getLocalName()
+				// dont close the transcription here
+					if(localname != "Trans" && localname != "Episode")
+						writer.writeEndElement();
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					writer.writeCharacters(parser.getText());
+					break;
+			}
+		}
+	}
+
+	/**
+	 * Write second part and shift time.
+	 *
+	 * @param parser the parser
+	 * @param shift the shift
+	 */
+	protected void writeSecondPartAndShiftTime(XMLStreamReader parser, float shift)
+	{
+		println "Second part"
+		DecimalFormat f = new DecimalFormat("#.000"); //$NON-NLS-1$
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event)
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					prefix = parser.getPrefix();
+
+				// skip header
+					if(localname == "Trans" || localname == "Episode" || localname == "Topics" || localname == "Topic"
+					|| localname == "Speakers" || localname == "Speaker")
+						break;
+
+					if(prefix != null && prefix.length() > 0)
+						writer.writeStartElement(prefix+":"+localname);
+					else
+						writer.writeStartElement(localname);
+
+					for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+					{
+						String name = parser.getAttributeLocalName(i);
+						String value = parser.getAttributeValue(i);
+
+						if(name == "time" || name == "endTime" || name == "startTime")
+						{
+							float newf = Float.parseFloat(value)+shift;
+							value = f.format(newf)
+							value = value.replace(",",".")
+						}
+
+						if(parser.getAttributePrefix(i)!= "")
+							writer.writeAttribute(parser.getAttributePrefix(i)+":"+name, value);
+						else
+							writer.writeAttribute(name, value);
+					}
+
+
+					break;
+
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName()
+					if(localname == "Topics" || localname == "Topic"
+					|| localname == "Speakers" || localname == "Speaker" )
+						break;
+					writer.writeEndElement();
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					if (parser.getText().trim().length() > 0)
+						writer.writeCharacters(parser.getText());
+					break;
+			}
+		}
+	}
+
+	/**
+	 * Process.
+	 */
+	protected void process()
+	{
+		writer.writeStartDocument("UTF-8", "1.0");
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event)
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					prefix = parser.getPrefix();
+
+					if(prefix != null && prefix.length() > 0)
+						writer.writeStartElement(prefix+":"+localname);
+					else
+						writer.writeStartElement(localname);
+
+
+					for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+					{
+						String prefix = parser.getAttributePrefix(i)
+						String attname = parser.getAttributeLocalName(i)
+						String attvalue = parser.getAttributeValue(i);
+
+						if(prefix != "")
+							writer.writeAttribute(prefix+":"+attname, attvalue);
+						else
+							writer.writeAttribute(attname, attvalue);
+					}
+
+
+					break;
+
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName()
+					writer.writeEndElement();
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					writer.writeCharacters(parser.getText());
+					break;
+			}
+		}
+	}
+
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File homedir = new File(System.getProperty("user.home"),"xml/concattrs/new");
+		//		File trs1 = new File(homedir, "int17A.trs")
+		//		File trs2 = new File(homedir, "int17B.trs")
+		//		File trs3 = new File(homedir, "int17C.trs")
+		//		File trs4 = new File(homedir, "int17D.trs")
+		//		File tmp1 = new File(homedir, "int17tmp1.trs")
+		//		File tmp2 = new File(homedir, "int17tmp2.trs")
+		//		File trs =  new File(homedir, "int17.trs")
+
+		//		File trs1 = new File(homedir, "int24.trs")
+		//		File trs2 = new File(homedir, "int24-2.trs")
+		//		File trs3 = new File(homedir, "int24-3.trs")
+		//		File trs4 = new File(homedir, "int24-4.trs")
+		//		File trs5 = new File(homedir, "int24-5.trs")
+		//		File trs6 = new File(homedir, "int24-6.trs")
+		//		File tmp1 = new File(homedir, "int24tmp1.trs")
+		//		File tmp2 = new File(homedir, "int24tmp2.trs")
+		//		File tmp3 = new File(homedir, "int24tmp3.trs")
+		//		File tmp4 = new File(homedir, "int24tmp4.trs")
+		//		File trs =  new File(homedir, "int24-corr.trs")
+
+		File trs1 = new File(homedir, "int14.trs")
+		File trs2 = new File(homedir, "int14bis.trs")
+		File trs =  new File(homedir, "int14-corr.trs")
+
+		new ConcatTrs(trs1, trs2, trs)
+
+		ValidateTRS.checkTRS(trs)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/importer.groovy	(revision 321)
@@ -0,0 +1,312 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+
+package org.txm.importer.transcriber
+
+import java.util.ArrayList;
+import org.txm.utils.Pair;
+
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.txm.utils.logger.Log;
+import org.txm.scripts.teitxm.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import org.w3c.dom.*;
+import org.xml.sax.SAXException;
+import javax.xml.parsers.*;
+import javax.xml.xpath.*;
+
+import java.util.HashMap;
+import org.txm.importer.*;
+import org.txm.utils.*;
+import org.txm.metadatas.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/** The trans. */
+	HashMap<String, String> trans = ["trans":"//Trans"];
+
+	/** The doc. */
+	def doc;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The outdir. */
+	File txmDir;
+	File binDir;
+
+	/** The trsfiles. */
+	ArrayList<String> trsfiles;
+
+	/** The metadatas. */
+	Metadatas metadatas;
+	
+	String lang; // language used by the tokenizer
+
+	/**
+	 * Instantiates a new importer.
+	 *
+	 * @param trsfiles the trsfiles
+	 * @param outdir the outdir
+	 * @param metadatas the metadatas
+	 */
+	public importer(ArrayList<File> trsfiles, File binDir, File txmDir, Metadatas metadatas, lang) {
+		this.trsfiles = trsfiles;
+		this.txmDir = txmDir; 
+		this.binDir = binDir;
+		this.metadatas = metadatas;
+		this.lang = lang;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean run() {
+		if (trsfiles == null) {
+			println "no files to process"
+			return false;
+		}
+		txmDir.mkdir();
+		if (!txmDir.exists()) {
+			println "can't create txmDir: "+txmDir.getAbsolutePath()
+		}
+		
+		// TRS -> TEI
+		println "Converting TRS to TEI "+trsfiles.size()+" files"
+		for (File infile : trsfiles) {
+			print "."
+			String textid = infile.getName()
+			int idx = textid.indexOf(".trs")
+			if (idx > 0) textid = textid.substring(0, idx)
+			File outfile = new File(txmDir, textid+".xml")
+			TRSToTEI p = new TRSToTEI(infile);
+			if (!p.process(outfile)) {
+				println "Error while converting TRS to TEI: "+infile
+				return false;
+			}
+		}
+		println ""
+
+		if (metadatas != null) {
+			if (metadatas.getHeadersList().size() == 0) {
+				println "Malformed metadata file. Check column and text separator. Columns: "+metadatas.getHeadersList()
+				return false;
+			}
+			println "Injecting metadatas "+metadatas.getHeadersList()+" in "+trsfiles.size()+" files"
+		}
+		trsfiles = txmDir.listFiles();
+		trsfiles.sort()
+		for (File infile : trsfiles) {
+			File outfile = new File(txmDir, "tmp.xml")
+			if (metadatas != null && metadatas.isInitialized()) {
+				print "."
+				if (!metadatas.injectMetadatasInXml(infile, outfile, "text")) {
+					println("Failed to inject metadatas in "+infile)
+					outfile.delete()
+				}
+				if (!infile.delete()) {
+					println "ERROR: could not delete $infile"
+					return false
+				}
+				outfile.renameTo(infile)
+			}
+		}
+		
+		println ""
+		
+		// TOKENIZER ENTITIES
+		println "Tokenizing entities "+txmDir.listFiles().length+" files"
+		for (File pfile : txmDir.listFiles()) {
+			print "."
+			TokenizeEntities tokenizer = new TokenizeEntities(pfile.toURI().toURL());
+			File outfile = File.createTempFile("tok", ".xml", pfile.getParentFile());
+			if (tokenizer.process(outfile)) {
+				if (!(pfile.delete() && outfile.renameTo(pfile))) println "Warning can't rename file "+outfile+" to "+pfile
+			} 
+			outfile.delete();
+		}
+		println ""
+		
+		//TOKENIZE
+		println "Tokenizing "+txmDir.listFiles().length+" files from $txmDir"
+		File tokenizedDir = new File(binDir, "tokenized")
+		tokenizedDir.mkdir()
+		for (File pfile : txmDir.listFiles()) {
+			print "."
+			String filename = pfile.getName().substring(0, pfile.getName().length()-4)
+			File tfile = new File(tokenizedDir, pfile.getName())
+			try {
+				TranscriberTokenizer tokenizer = new TranscriberTokenizer(pfile, tfile, lang)
+				if (!tokenizer.process()) {
+					println("Failed to tokenize "+pfile)
+				}
+			} catch (Exception e) {
+				println "Error tokenizer: "+pfile
+				org.txm.utils.logger.Log.printStackTrace(e);
+				return false;
+			}
+		}
+		println ""
+		
+		//TRANSFORM INTO XML-TEI-TXM
+		println("Building XML-TXM ("+txmDir.listFiles().length+" files)")
+		for (File tfile : tokenizedDir.listFiles()) {
+			print "."
+			String filename = tfile.getName().substring(0, tfile.getName().length()-4)
+			File xmlfile = new File(txmDir, tfile.getName())
+			
+			def correspType = new HashMap<String,String>()
+			correspType.put("event","event");
+			correspType.put("audio","audio");
+			correspType.put("notation","notation");
+			def correspRef = new HashMap<String,String>();
+			correspRef.put("event","trs");
+			correspRef.put("audio","trs");
+			correspRef.put("notation","trs");
+			def respId = ["trs"];
+			def applications = new HashMap<String,HashMap<String,String>>();
+			applications.put("trs",new ArrayList<String>());
+			applications.get("trs").add("Transcriber");//app ident
+			applications.get("trs").add("");//app version
+			applications.get("trs").add(null);//app report file path
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			taxonomiesUtilisees.put("ctx1",["event", "audio", "notation"]);//,"lemma","lasla","grace"]);
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			itemsURI.put("event",new HashMap<String,String>());
+			itemsURI.get("event").put("tagset","orth|corr");
+			itemsURI.get("event").put("website","");
+			itemsURI.put("audio",new HashMap<String,String>());
+			itemsURI.get("audio").put("tagset","present|absent|partiel");
+			itemsURI.get("audio").put("website","");
+			itemsURI.put("notation",new HashMap<String,String>());
+			itemsURI.get("notation").put("tagset","");
+			itemsURI.get("notation").put("website","");
+			def resps = new HashMap<String,String[]>();
+			resps.put("trs", ["Transcriber annotations","TXM","",""])
+			String wordprefix = "w_";
+			
+			Xml2Ana builder = new Xml2Ana(tfile);
+			builder.setConvertAllAtrtibutes true;
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			if (!builder.process(xmlfile)) {
+				println("Failed to process "+tfile)
+				xmlfile.delete();
+			}
+		}
+		
+		println ""
+		return txmDir.listFiles() != null;
+	}
+
+	/**
+	 * Process.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param metas the metas
+	 * @return true, if successful
+	 */
+	public boolean process(File infile, File outfile, ArrayList<Pair<String, String>> metas) {
+		//inject metadatas into 	
+		this.infile = infile;
+		this.outfile = outfile;
+		def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+		doc = builder.parse(infile)
+		insert(trans.get("trans"), metas);
+		return save();
+	}
+
+	/**
+	 * Insert.
+	 *
+	 * @param xpath the xpath
+	 * @param pairs the pairs
+	 */
+	public void insert(String xpath, List<Pair<String, String>> pairs) {
+		println ("insert $pairs into $xpath")
+		def expr = XPathFactory.newInstance().newXPath().compile(xpath)
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET)
+		
+		for (Node node : nodes) {
+			Element elem = (Element)node;
+			for (Pair<String, String> p : pairs) {
+				elem.setAttribute(p.getFirst(), p.getSecond());
+			}
+		}
+	}
+
+	/**
+	 * Save.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean save() {
+		try {
+			// Création de la source DOM
+			Source source = new DOMSource(doc);
+			
+			// Création du fichier de sortie
+			Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8")); 
+			Result resultat = new StreamResult(writer);
+			
+			// Configuration du transformer
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			Transformer transformer = fabrique.newTransformer();
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes"); 
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); 
+			
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			return true;
+		} catch (Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			return false;
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRSTestWorkflow.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRSTestWorkflow.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRSTestWorkflow.groovy	(revision 321)
@@ -0,0 +1,57 @@
+package org.txm.importer.transcriber
+
+import org.txm.utils.io.FileCopy;
+
+/**
+ * Test and Fix workflow for the STDDAD master corpus
+ * @author mdecorde
+ *
+ */
+
+File rootDir = new File("/home/mdecorde/xml/leman/")
+if (!rootDir.exists()) println "Error rootDir: "+rootDir;
+
+File dtdFile = new File(rootDir, "trans-14.dtd")
+File renamedTRSDir = new File(rootDir, "final 2012-03-26")
+File corrTRSDir = new File(rootDir, "corr")
+File okTRSDir = new File(rootDir, "ready")
+
+corrTRSDir.delete();
+corrTRSDir.mkdir();
+FileCopy.copy(dtdFile, new File(corrTRSDir, dtdFile.getName()));
+//okTRSDir.delete();
+okTRSDir.mkdir();
+FileCopy.copy(dtdFile, new File(okTRSDir, dtdFile.getName()));
+
+println "1- FixTimming"
+def files = renamedTRSDir.listFiles();
+files.sort()
+for (File trsFile : files) {
+	if (!trsFile.getName().endsWith(".trs")) continue;
+	println "FIX: "+trsFile
+	File newTRSFile = new File(corrTRSDir, trsFile.getName())
+	new FixTiming().fixTRS(trsFile, newTRSFile);
+}
+
+println "2- Validate"
+files = corrTRSDir.listFiles();
+files.sort()
+for(File trsFile : files) {
+	if(!trsFile.getName().endsWith(".trs")) continue;
+	File newTRSFile = new File(okTRSDir, trsFile.getName())
+	println "VALIDATE: "+trsFile
+	if(ValidateTRS.checkTRS(trsFile))
+		FileCopy.copy(trsFile, newTRSFile);
+}
+/*
+println "3- FINALLY"
+if (renamedTRSDir.listFiles().size() != corrTRSDir.listFiles().size) {
+	println "Missing corr transcriptions"
+	return;
+}
+
+if (corrTRSDir.listFiles().size() != okTRSDir.listFiles().size) {
+	println "Missing ok transcriptions"
+	return;
+}
+*/
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/transcriberLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/transcriberLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/transcriberLoader.groovy	(revision 321)
@@ -0,0 +1,309 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package org.txm.importer.transcriber;
+
+import java.io.File;
+
+import org.txm.scripts.teitxm.*;
+import org.txm.*;
+import org.txm.objects.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.utils.*;
+import org.txm.importer.*;
+import org.txm.metadatas.*;
+import org.w3c.dom.Element
+
+//PARAMETERS
+boolean removeInterviewer = false;//if true the transcription of speakers (en1 and enq2) defined in metadatas file will be ignored
+boolean includeComments = false;
+boolean ignoreTranscriberMetadata = false;
+int csvHeaderNumber = 1;
+int maxlines = 200;
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/anapovoas/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, "\t");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+String page_element = params.getPageElement("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+//get metadata values from CSV
+Metadatas metadatas; // text metadata
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+println "Trying to read metadatas values from: "+allmetadatasfile
+if (allmetadatasfile.exists()) {
+	File copy = new File(binDir, "metadata.csv")
+	if (!FileCopy.copy(allmetadatasfile, copy)) {
+		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
+		return;
+	}
+	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+}
+else
+	println "no metadata file: "+allmetadatasfile
+
+File propertyFile = new File(srcDir, "import.properties")//default
+Properties props = new Properties();
+String[] metadatasToKeep;
+if (propertyFile.exists() && propertyFile.canRead()) {
+	FileInputStream input = new FileInputStream(propertyFile);
+	props.load(input);
+	input.close();
+
+	if (props.getProperty("removeInterviewer") != null)
+		removeInterviewer = Boolean.parseBoolean(props.get("removeInterviewer").toString());
+	if (props.getProperty("ignoreTranscriberMetadata") != null)
+		ignoreTranscriberMetadata = Boolean.parseBoolean(props.get("ignoreTranscriberMetadata").toString());
+	if (props.getProperty("metadataList") != null)
+		metadatasToKeep = props.get("metadataList").toString().split("|");
+	if (props.getProperty("csvHeaderNumber") != null)
+		csvHeaderNumber = props.get("csvHeaderNumber").toString().split("|");
+	//if (props.getProperty("includeComments") != null)
+	//	includeComments = props.get("includeComments").toString();
+
+	println "import properties: "
+	println " removeInterviewer: "+removeInterviewer
+	println " ignoreTranscriberMetadata: "+ignoreTranscriberMetadata
+	println " metadataToKeep: "+metadatasToKeep
+	println " ignored csvHeaderSize: "+csvHeaderNumber
+	//println " includeComments: "+includeComments
+}
+
+// Apply XSL
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "XSL")
+if (xsl != null && xsl.trim().length() > 0) {
+	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src")))
+		srcDir = new File(binDir, "src");
+	println ""
+}
+
+try {
+	// select only trs files
+	String ext = "trs";
+	ArrayList<File> trsfiles = srcDir.listFiles(); //find all trs files
+	if (trsfiles  == null) {
+		println ("No files in "+srcDir.getAbsolutePath())
+		return false;
+	}
+	for (int i = 0 ; i < trsfiles.size() ; i++) {
+		File f = trsfiles.get(i);
+		if (!f.getName().endsWith(ext) || !f.canRead() || f.isHidden()) {
+			trsfiles.remove(i)
+			i--;
+		}
+	}
+
+	if (trsfiles.size() == 0) {
+		println ("No transcription file (*.trs) found in "+srcDir.getAbsolutePath()+". Aborting.")
+		return false;
+	}
+
+	if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	println "-- IMPORTER"
+	def imp = new importer(trsfiles, binDir, txmDir, metadatas, lang) //put result in the txm folder of binDir
+	if (!imp.run()) {
+		println "Failed to prepare files - Aborting";
+		return;
+	}
+	if (MONITOR != null) MONITOR.worked(20)
+
+	println "-- Xml Validation"
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	for (File infile : txmDir.listFiles()) {
+		if (!ValidateXml.test(infile)) {
+			println "$infile : Validation failed";
+			infile.delete();
+		}
+	}
+
+	if (MONITOR != null) MONITOR.worked(5)
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	println "-- Remove interviewer: "+removeInterviewer
+	if (removeInterviewer) 	{
+		if (metadatas == null) {
+			println "Can't remove interviewer without a metadata.csv file defining who are the interviewers."
+		} else {
+			println "Removing some speakers in "+txmDir.listFiles().length+" file(s)"
+			for (File infile : txmDir.listFiles()) {
+				String filename = infile.getName();
+				int idx = filename.indexOf(".xml");
+				if (idx > 0)
+					filename = filename.substring(0, idx);
+
+				ArrayList<Pair<String, String>> metas = metadatas.get(filename)
+				//println "filename=$filename metas= $metas"
+				for (Pair p : metas) {
+					if (p.getFirst().startsWith("enq")) {
+						new RemoveSpeaker(infile, infile, p.getFirst())
+					}
+				}
+			}
+		}
+	}
+
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+	println "-- ANNOTATE - Running NLP tools"
+	boolean annotationSuccess = false;
+	if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+		annotationSuccess = true;
+	}
+
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+	println "--COMPILING - Building Search Engine indexes"
+	trsfiles = txmDir.listFiles();
+
+	def comp = new compiler()
+	if(debug) comp.setDebug();
+	comp.removeInterviewers(removeInterviewer);
+	comp.setIgnoreTranscriberMetadata(ignoreTranscriberMetadata);
+	comp.setAnnotationSucces(annotationSuccess)
+	if (!comp.run(trsfiles, corpusname, "default", binDir)) {
+		println "Failed to compile files";
+		return;
+	}
+	
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	
+	File htmlDir = new File(binDir,"HTML/$corpusname");
+	htmlDir.deleteDir()
+	htmlDir.mkdirs();
+	if (build_edition) {
+		
+		if (MONITOR != null) MONITOR.worked(20, "EDITION")
+		println "-- EDITION - Building editions"
+		
+		List<File> filelist = txmDir.listFiles();
+		Collections.sort(filelist);
+		def second = 0
+
+		println "Paginating texts: "
+		for (File txmFile : filelist) {
+			print "."
+			String txtname = txmFile.getName();
+			int i = txtname.lastIndexOf(".");
+			if(i > 0) txtname = txtname.substring(0, i);
+
+			List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+			List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+			Element text = params.addText(corpusElem, txtname, txmFile);
+
+			def ed = new pager(txmFile, htmlDir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element, metadatas);
+			Element edition = params.addEdition(text, "default", htmlDir.getAbsolutePath(), "html");
+
+			for (i = 0 ; i < ed.getPageFiles().size();) {
+				File f = ed.getPageFiles().get(i);
+				String wordid = ed.getIdx().get(i);
+				params.addPage(edition, ""+(++i), wordid);
+			}
+
+			if (ed.getPageFiles().size() > 0) {
+				Element editionBD = params.addEdition(text, "onepage", htmlDir.getAbsolutePath(), "html");
+				params.addPage(editionBD, "1", ed.getIndexes().get(0));
+			}
+		}
+
+		//copy transcriber.css
+		File cssfile = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "css/transcriber.css")
+		if (cssfile.exists() && htmlDir.exists()) {
+			FileCopy.copy(cssfile, new File(htmlDir, "transcriber.css"));
+			FileCopy.copy(cssfile, new File(htmlDir, "onepage/transcriber.css"));
+			FileCopy.copy(cssfile, new File(htmlDir, "default/transcriber.css"));
+		}
+	}
+}
+catch (Exception e){org.txm.utils.logger.Log.printStackTrace(e);}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/RemoveSpeaker.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/RemoveSpeaker.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/RemoveSpeaker.groovy	(revision 321)
@@ -0,0 +1,130 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer.transcriber;
+
+
+import org.txm.importer.*;
+import org.txm.importer.graal.PersonalNamespaceContext
+import org.txm.utils.*;
+import org.txm.metadatas.*;
+
+import java.io.File;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.xml.parsers.*;
+import javax.xml.xpath.*;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * Removes "u" tags of TRS file given an "u@name" value. 
+ * @author mdecorde
+ *
+ */
+public class RemoveSpeaker {
+	File outfile;
+	
+	/** The doc. */
+	Document doc;
+	
+	/**
+	 * Instantiates a new removes the speaker.
+	 *
+	 * @param transcriptionfile the transcriptionfile
+	 * @param outfile the outfile
+	 * @param id the id
+	 */
+	public RemoveSpeaker(File transcriptionfile, File outfile, String id)
+	{
+		System.setProperty("javax.xml.transform.TransformerFactory", "net.sf.saxon.TransformerFactoryImpl");
+				
+		this.outfile = outfile;
+		String xpathString = "//tei:u[@spk='"+id+"']";
+		//println "removing $xpathString in $transcriptionfile"
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		//println "domFactory: $domFactory"
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		//println "builder $builder"
+		doc = builder.parse(transcriptionfile);
+		//println "doc $doc"
+		
+		def xpath = XPathFactory.newInstance().newXPath()
+		xpath.setNamespaceContext(new PersonalNamespaceContext());
+		def expr = xpath.compile(xpathString);
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+		
+		for (def node : nodes) {
+			//println "remove node "+node
+			Element elem = (Element)node;
+			elem.getParentNode().removeChild(node);
+		}
+		save()
+	}
+	
+	/**
+	 * Save.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean save()
+	{
+		try {
+			// Création de la source DOM
+			Source source = new DOMSource(doc);
+			
+			// Création du fichier de sortie
+			Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8")); 
+			Result resultat = new StreamResult(writer);
+			
+			// Configuration du transformer
+			//TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			//println "fabrique $fabrique"
+			Transformer transformer = fabrique.newTransformer();
+			//println "transformer $transformer"
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes"); 
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); 
+			
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			return true;
+		} catch (Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			return false;
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TranscriberTokenizer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TranscriberTokenizer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TranscriberTokenizer.groovy	(revision 321)
@@ -0,0 +1,192 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer.transcriber
+
+import filters.Tokeniser.SimpleTokenizerXml;
+
+/**
+ * The Class TranscriberTokenizer : manages Transcriber notations
+ */
+class TranscriberTokenizer extends SimpleTokenizerXml{
+	
+	/** The troncature. */
+	boolean troncature = false;
+	
+	/** The audio. */
+	String audio ="F";
+	
+	/** The notation. */
+	String notation = "";
+	
+	/** paroles rapportees */
+	boolean rapp = false;
+	
+	/** The event. */
+	String event="";
+	String txtname;
+	
+	/**
+	 * Instantiates a new transcriber tokenizer.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public TranscriberTokenizer(File infile, File outfile)
+	{
+		super(infile, outfile, null)
+		txtname = infile.getName();
+		int idx = txtname.indexOf(".")
+		if (idx > 0) txtname = txtname.substring(0, idx);
+	}
+	
+	/**
+	 * Instantiates a new transcriber tokenizer.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public TranscriberTokenizer(File infile, File outfile, String lang)
+	{
+		super(infile, outfile, lang)
+		txtname = infile.getName();
+		int idx = txtname.indexOf(".")
+		if (idx > 0) txtname = txtname.substring(0, idx);
+	}
+	
+	/* (non-Javadoc)
+	 * @see filters.Tokeniser.SimpleTokenizerXml#processWord()
+	 */
+	protected void processWord()
+	{
+		//println " chars: "+parser.getText().trim();
+		String text = buffer.toString().trim().replace("\t", " ");
+		text = text.replaceAll("\\p{C}", "");
+		String previousChar = "";
+		for (String s : text.split("[\\p{Z}\\p{C}]+") )	
+		{
+			//init transcriber attributes
+			if (troncature)
+				audio = "absent";
+			else
+				audio = "present"
+			notation = s;
+			event = "";
+			
+			if (s.startsWith("\"")) {
+				rapp = true;
+				event += "#rapp1";
+			} else if(s.endsWith("\"")) {
+				rapp = false;
+				event += "#rapp2";
+			}
+
+			//test events
+			if (s.startsWith("^^")) {
+				event += "#orth";			
+				s = s.substring(2);
+			}
+			if (s.startsWith("*")) {
+				event += "#corr";
+				s = s.substring(1);
+			}
+			
+			if (event.length() > 0)
+				event = event.substring(1);//remove '|'
+			
+			// test audio
+			if (s.contains("(") || s.contains(")")) // contains ( or )
+			{
+				if (s.contains("(") ^ s.contains(")")) // ert(ert XOR ert)ert
+				{
+					audio = "partiel"
+					
+					if (s.contains("(")) // ert(ert
+					{
+						troncature = true;
+						if (s.startsWith("(")) // (ertert
+						{
+							audio = "absent";
+						}
+					} else // ert)ert
+					{
+						troncature = false;
+						if (s.endsWith(")")) // ertert)
+						{
+							audio = "absent";
+						}
+					}
+				}
+				else if (s.contains("(") && s.contains(")")) // ert(ert)ert OR ert)ert(ert
+				{
+					audio = "partiel";
+					if (s.startsWith("(") && s.endsWith(")")) // (ertert)
+					{
+						audio = "absent";
+						troncature = false;
+					} else if (s.indexOf("(") < s.indexOf(")")) // ert(ert)ert
+					{
+						troncature = false;
+					} else // ert)ert(ert
+					{
+						troncature = true;
+					}
+				}
+				s = s.replace("(","");
+				s = s.replace(")","");
+			}
+						
+			iterate(s);
+		}
+	}
+	
+	/**
+	 * Iterate.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected iterate(String s)
+	{
+		while (s != null && s.length() > 0) {
+			if (DEBUG) {println "  > $s"}
+			s = standardChecks(s);
+			event = "";
+		}
+	}
+	
+	/* (non-Javadoc)
+	 * @see filters.Tokeniser.SimpleTokenizerXml#writeWordAttributes()
+	 */
+	protected writeWordAttributes()
+	{
+		writer.writeAttribute("id","w_"+txtname+"_"+wordcount);
+		writer.writeAttribute("audio", audio);
+		writer.writeAttribute("event", event);
+		writer.writeAttribute("notation", notation);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/compiler.groovy	(revision 321)
@@ -0,0 +1,678 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package org.txm.importer.transcriber
+
+import java.io.File;
+
+import javax.xml.stream.*;
+
+import java.io.OutputStreamWriter;
+import java.util.LinkedHashMap;
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.cwb.CwbProcess;
+import org.txm.importer.cwb.PatchCwbRegistry;
+import org.txm.utils.Pair;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler {
+
+	boolean ADD_TEXTID_TO_REF = true
+	
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The output. */
+	OutputStreamWriter output;
+
+	/** The basename. */
+	String corpusname;
+
+	/** The projectname. */
+	String projectname
+
+	/** The outdir. */
+	String outdir;
+
+	/** The debug. */
+	boolean debug = false;
+
+	/** The annotation succes. */
+	boolean annotationSucces = false;
+
+	/** The removeinterviewers. */
+	boolean removeinterviewers = false;
+
+	/** The trans. */
+	HashMap<String, ArrayList<Pair<String, String>>> trans;
+
+	/** The speakers. */
+	HashMap<String, ArrayList<Pair<String, String>>> speakers;
+
+	/** The speakersname. */
+	HashMap<String, String> speakersname = new HashMap<String, String>();
+
+	/** The topics. */
+	HashMap<String, ArrayList<Pair<String, String>>> topics;
+
+
+	/** The interviewers. */
+	ArrayList<String> interviewers = [];
+	static HashSet<String> sectionAttrs;
+
+	/** The anatypes. */
+	private static anatypes = []
+	private static anavalues = [:]
+
+	/**
+	 * Removes the interviewers.
+	 *
+	 * @param value the value
+	 * @return the java.lang. object
+	 */
+	public removeInterviewers(boolean value) {
+		this.removeinterviewers = value;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param xmlfiles the xmlfiles
+	 * @param basename the basename
+	 * @param projectname the projectname
+	 * @param outdir the outdir
+	 * @return true, if successful
+	 */
+	public boolean run(List<File> xmlfiles, String corpusname, String projectname, File binDir)
+	{
+		Collections.sort(xmlfiles);
+		//println "run compiler with $xmlfiles, $basename and $outdir"
+		this.outdir = binDir;
+		this.corpusname = corpusname;
+		this.projectname = projectname;
+
+		anatypes = ["event"] // reset
+		anavalues = [:] // reset
+
+		sectionAttrs = new HashSet<String>() // reset section attributs set
+
+		File wtcDir = new File(binDir, "wtc")
+		new File(binDir, "wtc").delete();
+		wtcDir.mkdir();
+		if (!new File(binDir, "wtc").exists()) {
+			println "Can't create wtc directory"
+			return false;
+		}
+
+		File wtcFile = new File(wtcDir, corpusname.toLowerCase()+".wtc")
+
+		// get all anatypes
+		for (File f : xmlfiles) {
+			getAnaTypes(f)
+		}
+		//println "ANATYPES: "+anatypes
+		if (!createOutput(wtcFile)) return false;
+		output.write("<txmcorpus lang=\"fr\">\n")
+		output.close();
+
+		println("Compiling "+xmlfiles.size()+" files")
+		for (File f :xmlfiles) {
+			if (f.exists())
+				if (!process(f)) {
+					println("Failed to compile "+f)
+				}
+		}
+
+		if (!createOutput(wtcFile)) return false;
+		output.write("</txmcorpus>\n")
+		output.close();
+
+		//2- Import into CWB
+		File registryFile = new File(binDir, "registry/"+corpusname.toLowerCase())
+		File dataDir = new File(binDir, "data/$corpusname")
+
+		new File(binDir, "registry").mkdir();
+		if (!new File(binDir, "registry").exists()) {
+			println "Can't create registry directory"
+			return false;
+		}
+
+		String cwbLoc = CwbProcess.getCWBLocation();
+		if (cwbLoc == null) return false;
+
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbEn.setDebug(debug);
+		cwbMa.setDebug(debug);
+
+		String uAttr = "u:0+spkid+spk+scope+accent+s+time+check+dialect+type";
+		String textAttr ="text:0+base+project"
+		if (trans != null) {
+			for (String key : trans.keySet()) {
+				for (Pair p : trans.get(key)) {
+					if (ignoreTranscriberMetadata) {
+						String meta =p.getFirst();
+						if (	meta != "scribe" && meta != "audio_filename" &&
+						meta != "version" && meta != "version_date")
+							textAttr+="+"+meta
+					} else {
+						textAttr+="+"+p.getFirst()
+					}
+				}
+				break;
+			}
+		}
+
+		String sectionAttr = "div:0" // "div:0+id+topic+endtime+starttime+type"
+		for (String attr : sectionAttrs) {
+			sectionAttr += "+"+attr
+		}
+
+		List<String> pargs = ["spk", "ref", "id", "entitytype", "entityid"]
+		for (String ana : anatypes) pargs.add(ana)
+
+		String[] pAttributes = pargs
+
+		String[] sAttributes = ["txmcorpus:0+lang", uAttr , textAttr, "event:0+id+desc+type+extent", sectionAttr, "sp:0+id+speaker+endtime+starttime+overlap+time"];
+
+		println "pAttributes: $pAttributes"
+		println "sAttributes: $sAttributes"
+		//return;
+		try {
+			cwbEn.run(cwbLoc + "cwb-encode", dataDir.getAbsolutePath(),
+					wtcFile.getAbsolutePath(),
+					registryFile.getAbsolutePath(), pAttributes, sAttributes);
+			if (!registryFile.exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(cwbLoc + "cwb-makeall", corpusname, registryFile.getParent());
+			
+
+		} catch (Exception ex) {System.out.println(ex); return false;}
+
+
+		return true;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f){
+		try {
+			//File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f,f.exists())) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/** The text_id. */
+	String text_id
+
+	/** The u opened. */
+	boolean uOpened = false;
+
+	/** The idturn. */
+	int idturn = 1;
+
+	/** The idsection. */
+	int idsection = 1;
+
+	/** The idu. */
+	int idu = 1;
+
+	/** The idevent. */
+	int idevent = 1;
+
+	/** The events. */
+	List<String> events = [];
+	static int vEntityId = 0;
+	static int vEntityIdCount = 1;
+
+	/**
+	 * Process.
+	 *
+	 * @param xmlfile the xmlfile
+	 * @return true, if successful
+	 */
+	private boolean process(File xmlfile) {
+		text_id = xmlfile.getName();
+		text_id = text_id.substring(0, text_id.length() -4);
+
+		idturn = 1;
+		idsection = 1;
+		idu = 1;
+
+		boolean flagAna;
+		boolean flagForm;
+		boolean flagWord;
+		String vWord="";
+		String vForm="";
+		String vAna="";
+		String vEvents = "N/A";
+		String vEntityType = "N/A"
+		String wordid= "";
+		//def wordattributes = [:];
+		String anatype = "";
+		String anavalue = "";
+
+		String formatedTime;
+
+		LinkedHashMap<String, String> anahash = new LinkedHashMap<String, String>();
+		String currentType;
+
+		URL url = xmlfile.toURI().toURL();
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		String filename = xmlfile.getName()
+		String textid = filename.substring(0, filename.length() - 4);
+
+		createOutput(new File(outdir, "wtc/"+corpusname.toLowerCase()+".wtc"));
+		String localname;
+
+		//get all metadatas declared before Episode tag
+		speakers = new HashMap<String, ArrayList<Pair<String, String>>>();
+		trans = new HashMap<String, ArrayList<Pair<String, String>>>();
+		topics = new HashMap<String, ArrayList<Pair<String, String>>>();
+		//println "parse infos"
+		parseInfos();
+
+		//		println "Trans: $trans"
+		//		println "Topics: $topics"
+		//		println "Speakers: $speakers"
+		//		def transproperties = ""
+		//		for (String key : trans.keySet()) {
+		//			for (Pair p : trans.get(key))
+		//				transproperties+="\t"+p.getSecond();
+		//			break;
+		//		}
+		//		println "Trans properties: "+transproperties
+		List<String> localspeakers;
+
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			//print "event: "+event +" "
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+				//println localname
+					switch(localname) {
+						case "div":
+							output.write("<div");
+							for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+								String name = parser.getAttributeLocalName(i).replace("_","").toLowerCase()
+								output.write(" "+name+"=\""+parser.getAttributeValue(i)+"\"");
+								sectionAttrs << name
+							}
+							output.write ">\n"
+							break;
+						case "sp":
+							output.write("<sp");
+							writeAttributes();
+							output.write ">\n"
+							break;
+						case "u":
+							output.write("<u");
+							for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+								String name = parser.getAttributeLocalName(i).replace("_","").toLowerCase()
+								output.write(" "+name+"=\""+parser.getAttributeValue(i)+"\"");
+								if (name == "time") formatedTime = parser.getAttributeValue(i)
+								else if (name == "spk") u_name = parser.getAttributeValue(i)
+							}
+							output.write ">\n"
+							break;
+						case "event":
+							output.write("<event");
+							writeAttributes();
+							output.write ">\n"
+
+							if (parser.getAttributeValue(null, "type") == "entities") {
+								if (parser.getAttributeValue(null, "extent") == "begin") {
+									vEntityType = parser.getAttributeValue(null, "desc");
+									vEntityId = vEntityIdCount++;
+								} else {
+									vEntityType = "N/A";
+									vEntityId = 0;
+								}
+							} else if (parser.getAttributeValue(null, "type") == "pronounce") {
+								if (parser.getAttributeValue(null, "extent") == "begin")
+									events.add(parser.getAttributeValue(null, "desc"))
+								else if (parser.getAttributeValue(null, "extent") == "end")
+									events.remove(parser.getAttributeValue(null, "desc"))
+								vEvents = "";
+								for (String s : events)
+									vEvents += s+"#";
+								if (vEvents.length() > 0)
+									vEvents = vEvents.substring(0, vEvents.length()-1);
+								else
+									vEvents = ""
+							}
+							break;
+						case "w":
+							for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+								if (parser.getAttributeLocalName(i).equals("id")) {
+									wordid = parser.getAttributeValue(i);
+									break;
+								}
+							}
+							anavalues = [:];
+							break;
+						case "form":
+							flagForm = true;
+							vForm = "";
+							vAna ="";
+							break;
+						case "ana":
+							flagAna = true;
+							anavalue = "";
+							for (int i = 0 ; i < parser.getAttributeCount(); i++)
+								if (parser.getAttributeLocalName(i).equals("type")) {
+									anatype = parser.getAttributeValue(i).substring(1);//remove the #
+									break;
+								}
+							break;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+
+					switch (localname) {
+						case "text":
+							output.write("</text>\n")
+							break;
+						case "Topics":
+							break;
+						case "Topic":
+							break;
+						case "Speakers":
+							break;
+						case "Speaker":
+							break;
+						case "Episode":
+							break;
+						case "div":
+							output.write("</div>\n")
+							break;
+						case "sp":
+							output.write("</sp>\n")
+							break;
+						case "u":
+							output.write("</u>\n")
+							break;
+						case "event":
+							output.write("</event>\n")
+							break;
+						case "form":
+							flagForm = false;
+							break;
+						case "ana":
+							anavalues.put(anatype, anavalue)
+							flagAna = false;
+							break;
+						case "w":
+						// concat spk id and ref
+							String isEnq = (interviewers.contains(u_name))?"*":"";
+							String ref = (u_name+", "+formatedTime+""+isEnq)
+							if (ADD_TEXTID_TO_REF) ref = textid+", "+ref
+							vForm +="\t"+u_name+"\t"+ref
+
+						// concat entity and entity ID
+							vAna+= "\t"+vEntityType+"\t"+vEntityId;
+
+						//concat ana values
+							for (String type : anatypes) {
+								def v = anavalues.get(type);
+								if (v == null) v = "";
+
+								if ("event" == type) {
+									if (v.length() > 0)
+										vAna+="\t#"+v;
+									else
+										vAna+="\t";
+
+									//concat <Event> values
+									if (vEvents != null && vEvents.length() > 0 && vEvents != "N/A")
+										vAna += "#"+vEvents;
+								} else {
+									vAna+="\t"+v;
+								}
+							}
+
+
+							vForm = vForm.replaceAll("\n", "").replaceAll("&", "&amp;").replaceAll("<", "&lt;");
+
+							if (removeinterviewers) {
+								if (!interviewers.contains(u_name))
+									output.write(vForm+"\t"+wordid+vAna+"\n");
+							} else {
+								output.write(vForm+"\t"+wordid+vAna+"\n");
+							}
+
+							vAna = "";
+							vForm = "";
+							break;
+					}
+					break
+				case XMLStreamConstants.CHARACTERS:
+					if (flagForm)
+						vForm += parser.getText().trim();
+					if (flagAna) {
+						anavalue += parser.getText().trim();
+					}
+					break;
+			}
+		}
+
+		parser.close();
+		inputData.close();
+		output.close();
+		return true;
+	}
+
+	/** The u_name. */
+	String u_name;
+
+	/**
+	 * Write start tag.
+	 */
+	private void writeStartTag() {
+		output.write("<"+parser.getLocalName());
+		writeAttributes();
+		output.write ">\n"
+	}
+
+	/**
+	 * Write attributes.
+	 */
+	private void writeAttributes() {
+		for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+			output.write(" "+parser.getAttributeLocalName(i).replace("_","").toLowerCase()+"=\""+parser.getAttributeValue(i)+"\"");
+		}
+	}
+
+	private void getAnaTypes(File xmlFile) {
+		inputData = xmlFile.toURI().toURL().openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		String ana = "ana"
+		HashSet<String> types = new HashSet<String>();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) { // start elem
+				if (ana.equals(parser.getLocalName())) { // ana elem
+					for (int i = 0 ; i < parser.getAttributeCount(); i++) { // find @type
+						if ("type".equals(parser.getAttributeLocalName(i))) { // @type
+							types.add(parser.getAttributeValue(i).substring(1)); //remove the #
+							break;
+						}
+					}
+				}
+			}
+		}
+		parser.close()
+
+		for (String type : types)
+			if (!anatypes.contains(type))
+				anatypes << type
+	}
+
+	/**
+	 * Write start tag.
+	 *
+	 * @param id the id
+	 */
+	private void writeStartTag(int id) {
+		output.write("<"+parser.getLocalName().toLowerCase());
+		output.write(" id=\""+id+"\"");
+		writeAttributes();
+		output.write ">\n"
+	}
+
+	/**
+	 * Write end tag.
+	 */
+	private void writeEndTag() {
+		output.write("</"+parser.getLocalName().toLowerCase()+">\n");
+	}
+
+	/** The ignore transcriber metadata. */
+	boolean ignoreTranscriberMetadata = false;
+
+	/**
+	 * Sets the ignore transcriber metadata.
+	 *
+	 * @param state the new ignore transcriber metadata
+	 */
+	public void setIgnoreTranscriberMetadata(boolean state) {
+		this.ignoreTranscriberMetadata = state;
+	}
+
+	/**
+	 * Parses the infos.
+	 */
+	private void parseInfos() { //until tag Episode
+		String localname;
+		//assert(parser != null);
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				localname = parser.getLocalName();
+				switch (localname) {
+					case "text":
+						output.write("<text project=\""+projectname+"\" base=\""+corpusname+"\"")
+						for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+							if (ignoreTranscriberMetadata) {
+								if (parser.getAttributeLocalName(i) != "scribe" &&
+								parser.getAttributeLocalName(i) != "audio_filename" &&
+								parser.getAttributeLocalName(i) != "version" &&
+								parser.getAttributeLocalName(i) != "version_date")
+									output.write(" "+parser.getAttributeLocalName(i).replace("_","").toLowerCase()+"=\""+parser.getAttributeValue(i)+"\"");
+							} else {
+								output.write(" "+parser.getAttributeLocalName(i).replace("_","").toLowerCase()+"=\""+parser.getAttributeValue(i)+"\"");
+							}
+						}
+
+						output.write ">\n"
+
+						ArrayList list = new ArrayList<Pair<String, String>>()
+						trans.put("trans", list);
+
+						for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+							list.add(new Pair(parser.getAttributeLocalName(i).replace("_","").toLowerCase(), parser.getAttributeValue(i)));
+							if(parser.getAttributeLocalName(i).startsWith("enq"))
+								interviewers.add(parser.getAttributeValue(i));
+						}
+						return
+					case "Topic":
+						String id = parser.getAttributeValue(null, "id");
+						if (id != null) {
+							ArrayList list = new ArrayList<Pair<String, String>>()
+							topics.put(id, list);
+						} else {
+							println "found tag $localname with no id"
+						}
+						break;
+					case "Speaker":
+					//case "Trans":
+						String id = parser.getAttributeValue(null, "id");
+						String name = parser.getAttributeValue(null, "name");
+						if (id != null && name != null) {
+							speakersname.put(id, name);
+						} else {
+							println "found tag $localname with no id ($id)or name ($name)"
+							return;
+						}
+						if (id != null) {
+							ArrayList list = new ArrayList<Pair<String, String>>()
+							speakers.put(id, list);
+
+							for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+								list.add(new Pair(parser.getAttributeLocalName(i), parser.getAttributeValue(i)));
+							}
+						} else {
+							println "found tag $localname with no id"
+							return;
+						}
+						break;
+				}
+			}
+		}
+	}
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug() {
+		debug = true;
+	}
+
+	/**
+	 * Sets the annotation success.
+	 *
+	 * @param val the new annotation success
+	 */
+	public void setAnnotationSuccess(boolean val) {
+		annotationSucces = val
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Transcriber import module.</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRSToTEI.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRSToTEI.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRSToTEI.groovy	(revision 321)
@@ -0,0 +1,341 @@
+package org.txm.importer.transcriber
+
+import java.io.File;
+
+import javax.xml.stream.*;
+
+import java.io.OutputStreamWriter;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+
+import org.txm.importer.StaxIdentityParser;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.cwb.CwbProcess;
+import org.txm.importer.cwb.PatchCwbRegistry;
+import org.txm.utils.Pair;
+
+class TRSToTEI extends StaxIdentityParser {
+	
+	// Transcription informations
+	def informations = [:]
+	def speakers = [:];
+	HashMap<String, String> speakersname = new HashMap<String, String>();
+	def topics = [:];
+
+	List<String> localspeakers; // speakers of the current Turn
+
+	// IDs
+	int idturn = 1;
+	int idsection = 1;
+	int idu = 1;
+	int idevent = 1;
+
+	boolean uOpened = false;
+
+	// values stored
+	List<String> events = [];
+	static int vEntityId = 0;
+	static int vEntityIdCount = 1;
+	String vSpeaker="";
+	String u_name;
+	boolean flagAna;
+	boolean flagForm;
+	boolean flagWord;
+	String vWord="";
+	String vForm="";
+	String vAna="";
+	String vEvents = "N/A";
+	String vEntityType = "N/A"
+	String wordid= "";
+	String anatype = "";
+	String anavalue = "";
+
+	String formatedTime;
+	String currentType;
+	String lastTime = ""
+	String textid;
+	public TRSToTEI(File infile) {
+		super(infile.toURI().toURL());
+		textid = infile.getName()
+		int idx = textid.indexOf(".trs")
+		if (idx > 0) textid = textid.substring(0, idx)
+	}
+
+	boolean beforeBody = true;
+	protected void processStartElement() {
+		if (beforeBody) {
+			parseInfos()
+		} else {
+			processEpisodeStartElement()
+		}
+	}
+
+	protected void processEndElement() {
+		if (beforeBody) {
+			parseInfosEndElements()
+		} else {
+			processEpisodeEndElement()
+		}
+	}
+
+	/**
+	 * Parses the infos.
+	 */
+	private void parseInfos() { //until tag Episode
+		//println "parse infos: $localname"
+		switch (localname) {
+			case "Trans":
+				for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+					informations.put(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+				}
+				writer.writeStartElement("TEI");
+				writer.writeStartElement("teiHeader");
+				writer.writeStartElement("fileDesc");
+				writer.writeEndElement();
+				writer.writeStartElement("encodingDesc");
+				writer.writeEndElement();
+				writer.writeStartElement("profileDesc");
+				writer.writeEndElement();
+				writer.writeStartElement("application");
+				break;
+			case "Topics":
+				super.processStartElement();
+				break;
+			case "Topic":
+				super.processStartElement();
+				for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+					topics.put(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+				}
+				break;
+			case "Speakers":
+				super.processStartElement();
+				break;
+			case "Speaker":
+				super.processStartElement();
+				String id = parser.getAttributeValue(null, "id");
+				String name = parser.getAttributeValue(null, "name");
+				if (id != null && name != null) {
+					speakersname.put(id, name);
+				} else {
+					println "found tag $localname with no id ($id) nor name ($name)"
+					return;
+				}
+
+				if (id != null) {
+					ArrayList list = new ArrayList<Pair<String, String>>()
+					speakers.put(id, list);
+
+					for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {
+						list.add(new Pair(parser.getAttributeLocalName(i), parser.getAttributeValue(i)));
+					}
+				} else {
+					println "found tag $localname with no id"
+					return;
+				}
+				break;
+			case "Episode":
+				writer.writeEndElement() // application
+				writer.writeEndElement() // teiHeader
+				writer.writeStartElement("text")
+				writer.writeAttribute("id", textid)
+				for(def k : informations.keySet()) writer.writeAttribute(k, informations[k])
+				beforeBody = false; // end of info parsing
+				break;
+		}
+	}
+
+	private void parseInfosEndElements() { //until tag Episode
+		//println "parse infos: $localname"
+		switch (localname) {
+			case "Topics":
+				super.processEndElement();
+				break;
+			case "Topic":
+				super.processEndElement();
+				break;
+			case "Speakers":
+				super.processEndElement();
+				break;
+			case "Speaker":
+				super.processEndElement();
+				break;
+		}
+	}
+
+	
+	boolean ignoreFirstSync = false // need to skip fist Sync when multiple locutors in Turn
+	/**
+	 * Process.
+	 *
+	 * @param xmlfile the xmlfile
+	 * @return true, if successful
+	 */
+	private boolean processEpisodeStartElement() {
+		//println "parse start: $localname"
+		switch(localname) {
+			case "Section": // >> div
+				testCloseU();
+				writer.writeStartElement("div")
+				writer.writeAttribute("id", Integer.toString(idsection++))
+				writeAttributes();
+				break;
+			case "Turn": // >> sp
+				testCloseU();
+				vSpeaker = parser.getAttributeValue(null, "speaker");
+				if (vSpeaker == null) { vSpeaker="N/A"	// no spk
+				} else {
+					localspeakers = vSpeaker.split(" ")
+					if (localspeakers.size() == 0) { // only one speaker
+						//println "FOUND ONE SPEAKER"
+						if (speakersname.containsKey(vSpeaker)) {
+							vSpeaker = speakersname.get(vSpeaker);
+						}
+					}
+				}
+
+				writer.writeStartElement("sp")
+				writer.writeAttribute("id", Integer.toString(idturn++))
+				ignoreFirstSync = vSpeaker.contains(" ") // need to skip fist Sync when multiple locutors in Turn
+				writer.writeAttribute("overlap", ""+ignoreFirstSync)
+				
+				String time = parser.getAttributeValue(null, "startTime");
+				formatedTime = formatTime(time)
+				writer.writeAttribute("time", formatedTime)
+				writeAttributes();
+				break;
+			case "Sync": // >> u
+				lastTime = parser.getAttributeValue(null, "time")
+				testCloseU();
+				if (ignoreFirstSync) { // need to skip fist Sync when multiple locutors in Turn
+					ignoreFirstSync = false;
+				} else {
+					writeU()
+				}
+				break;
+			case "Who": // >> u
+				testCloseU();
+				int n = Integer.parseInt(parser.getAttributeValue(null, "nb")) -1;
+				if (localspeakers.size() <= n || n < 0) {
+					println "\nWarning: Mismatch speaker number declaration between <Who> and <Turn> tags at line "+parser.getLocation().getLineNumber()+" of the '"+inputurl.getFile()+"' transcription file.";
+					vSpeaker = "#"+(n+1)+"?";
+				} else {
+					vSpeaker = localspeakers.get(n);
+				}
+				writeU()
+				break;
+			case "Event": // >> event
+				writer.writeStartElement("event")
+				writer.writeAttribute("id", Integer.toString(idevent++))
+				writeAttributes();
+				break;
+			default:
+				super.processStartElement();
+		}
+	}
+
+	void processEpisodeEndElement() {
+		switch (localname) {
+			case "Section":
+				if (uOpened) {
+					super.processEndElement(); // u
+					uOpened = false;
+				}
+				super.processEndElement(); //div
+				break;
+			case "Turn":
+				if (uOpened) {
+					super.processEndElement(); // u
+					uOpened = false;
+				}
+				super.processEndElement(); // sp
+				break;
+			case "Sync":
+			case "Who":
+			case "Episode":
+				break
+			case "Trans":
+				writer.writeEndElement(); // text
+				writer.writeEndElement(); //TEI
+				break
+			default:
+				super.processEndElement();
+		}
+	}
+
+	/**
+	 * If a 'u' tag is opened, close it.
+	 */
+	private testCloseU() {
+		if (uOpened) {
+			super.processEndElement(); // u
+			uOpened = false;
+		}
+	}
+
+	/**
+	 * Write u.
+	 */
+	private void writeU() {
+		writer.writeStartElement("u")
+		writer.writeAttribute("s", lastTime);
+		writer.writeAttribute("time", formatTime(lastTime));
+//		println "write u for vSpeaker=$vSpeaker"
+//		println "getting spk name? ="+speakers.get(vSpeaker)
+//		println "speakers: $speakers"
+		def attributes = speakers.get(vSpeaker)
+		if (attributes == null) { // in case of Who@n wrong number
+			if (vSpeaker.startsWith("#") && vSpeaker.endsWith("?")) { // don't show "N/A" vSpeaker
+				writer.writeAttribute("spk", vSpeaker)
+				writer.writeAttribute("spkid", vSpeaker)
+			}
+		} else {
+			for (Pair p : attributes) {
+//				println " write attribute "+p.getFirst()+" "+p.getSecond()
+				String attrn = p.getFirst().toString();
+				if (attrn == "name") { // rename @name to @spk
+					attrn = "spk"
+					u_name = p.getSecond();
+				} else if (attrn == "id") { // rename @id to @spkid
+					attrn = "spkid"
+				}
+
+				writer.writeAttribute(attrn,p.getSecond())
+			}
+		}
+		uOpened = true;
+	}
+	private String formatTime(String time) {
+		try {
+			return formatTime(Float.parseFloat(time))
+		} catch(Exception e) {return formatTime(0.0f)}
+	}
+
+	private String formatTime(float time) {
+		String rez = " ";
+		float h = time / 3600;
+		time = time%3600;
+		float min = (time%3600) / 60;
+		int sec = (int)time%60
+		if (min < 10)
+			rez = ""+(int)h+":0"+(int)min;//+":"+time%60;
+		else
+			rez = ""+(int)h+":"+(int)min;//+":"+time%60;
+
+		if (sec >= 10)
+			rez += ":"+sec;
+		else
+			rez += ":0"+sec;
+
+		return rez;
+	}
+
+	public static void main(String[] args) {
+		File infile = new File("/home/mdecorde/xml/trs/int01.trs")
+		File outfile = new File("/home/mdecorde/xml/trs/test-tei.xml")
+
+		TRSToTEI p = new TRSToTEI(infile);
+		p.process(outfile)
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/BuildTemplates.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/BuildTemplates.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/BuildTemplates.groovy	(revision 321)
@@ -0,0 +1,89 @@
+package org.txm.importer.transcriber
+import java.nio.charset.Charset
+import org.txm.utils.CsvReader
+
+/// START PARAMETERS ///
+File dir = new File("C:/") // output and input directory
+File csvfile = new File(dir, "test.csv") // 3 cols: int enq ext
+String spkSeparator = "\\|" // regular expression
+char colSeparator = '\t' // csv column separator
+String csvencoding = "ISO-8859-1" // csv encoding
+
+//transcriptions header
+String date = "111128" // date of transcription edition
+String scribes= "studentN" // scribe name
+String encoding = "ISO-8859-1" // transcription encoding
+
+/// END PARAMETERS ///
+String canvas="""<?xml version="1.0" encoding="@ENCODING"?>
+<!DOCTYPE Trans SYSTEM "trans-14.dtd">
+<Trans scribe="@SCRIBE" audio_filename="@AUDIOFILE" version="1" version_date="@DATE">
+<Speakers>
+@SPEAKERS</Speakers>
+<Episode>
+<Section type="report" startTime="0" endTime="2.0">
+<Turn startTime="0" endTime="1.0" speaker="@FIRSTSPK">
+<Sync time="0"/>
+
+</Turn>
+</Section></Episode></Trans>
+""";
+String spkCanvas="<Speaker id=\"@ID\" name=\"@ID\" check=\"no\" dialect=\"native\" accent=\"\" scope=\"local\"/>"
+
+//load csv
+CsvReader csvreader = new CsvReader(csvfile.getAbsolutePath(), colSeparator, Charset.forName(csvencoding))
+
+//check header
+csvreader.readHeaders();
+def titles = [];
+for(String title : csvreader.getHeaders())
+	titles << title
+assert(titles.contains("int"))
+assert(titles.contains("enq"))
+assert(titles.contains("ext"))
+
+// one transcription per csv line
+while (csvreader.readRecord()){
+
+	String ints = csvreader.get("int")
+	String enqs = csvreader.get("enq")
+	String exts = csvreader.get("ext")
+	
+	def intsS = ints.split(spkSeparator)
+	def enqsS = enqs.split(spkSeparator)
+	def extsS = exts.split(spkSeparator)
+	assert(intsS.size() > 0)
+	assert(enqsS.size() > 0)
+	
+	// build speaker declarations
+	String speakers = ""
+	for(String s : intsS)
+		if(s.length() > 0)
+			speakers += spkCanvas.replace("@ID", s)+"\n"
+	for(String s : enqsS)
+		if(s.length() > 0)
+			speakers += spkCanvas.replace("@ID", s)+"\n"
+	for(String s : extsS)
+		if(s.length() > 0)
+			speakers += spkCanvas.replace("@ID", s)+"\n"
+
+	// some infos
+	String transfile = intsS[0]
+	String firstSpeaker = enqsS[0]
+			
+	// fill infos
+	String content = canvas
+	content = content.replace("@ENCODING", encoding)
+	content = content.replace("@SCRIBE", scribes)
+	content = content.replace("@AUDIOFILE", dir.getAbsolutePath()+transfile+".mp3")
+	content = content.replace("@DATE", date)
+	content = content.replace("@SPEAKERS", speakers)
+	content = content.replace("@FIRSTSPK", firstSpeaker)
+	
+	//write transcription
+	File outfile = new File(dir, transfile+".trs")
+	println "create: "+outfile
+	outfile.withWriter(encoding) { out ->
+      out.println content
+    }
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/CheckWhos.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/CheckWhos.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/CheckWhos.groovy	(revision 321)
@@ -0,0 +1,115 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer.transcriber
+
+import java.io.File;
+import javax.xml.stream.*;
+import java.io.OutputStreamWriter;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class CheckWhos.
+ */
+class CheckWhos {
+	
+	/**
+	 * Test.
+	 *
+	 * @param xmlfile the xmlfile
+	 * @return true, if successful
+	 */
+	public static boolean test(File xmlfile)
+	{
+		boolean ret = true;
+		URL url = xmlfile.toURI().toURL();
+		def inputData = url.openStream();
+		XMLInputFactory factory = XMLInputFactory.newInstance();
+		def parser = factory.createXMLStreamReader(inputData);
+		
+		String localname;
+		String vSpeaker;
+		
+		List<String> localspeakers;
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "Turn":
+						
+							vSpeaker = parser.getAttributeValue(null, "speaker");			
+							if(vSpeaker == null)
+							{
+								localspeakers = [];
+								vSpeaker = "N/A"
+							}
+							else
+								localspeakers = vSpeaker.split(" ")
+							break;
+						
+						case "Who":
+						
+							int n = Integer.parseInt(parser.getAttributeValue(null, "nb")) -1;
+							if(localspeakers.size() <= n)
+							{
+								println "$xmlfile: Who error : line "+parser.getLocation().lineNumber+" col "+parser.getLocation().columnNumber+" in file "+xmlfile;
+								vSpeaker = "N/A";
+								ret = false;
+							}
+						
+							break;
+							
+					}
+			}
+		}
+		return ret;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String userdir = System.getProperty("user.home")
+		for(File trsfile : new File(userdir, "xml/cedre").listFiles())
+		{
+			if(trsfile.getName().endsWith(".trs"))
+			{
+				if(!CheckWhos.test(trsfile))
+					println trsfile.getName()+" not ok"
+			}
+		}
+	}
+	
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ValidateTRS.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ValidateTRS.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/ValidateTRS.groovy	(revision 321)
@@ -0,0 +1,509 @@
+package org.txm.importer.transcriber
+
+import java.text.DecimalFormat
+
+import org.txm.utils.ExecTimer
+import org.w3c.dom.*
+
+/**
+ * TRS file time validation for the STDDAD master corpus
+ * @author mdecorde
+ *
+ */
+class ValidateTRS {
+	def doc;
+	def root;
+	def episodeElem;
+	def topicsElem;
+	def speakersElem;
+
+	def declaredTopics = [];
+	def topics = [];
+
+	def mandatoryTopics = ["parcours", "ressources", "débouchés", "acteurs", "métier", "images", "autres"]
+	def declaredSpeakers = [];
+	def speakers = [];
+	def spkDict = [:];
+	def topicDict = [:];
+
+	def sections = [];
+	def turns = new LinkedHashMap();
+	def syncs = new LinkedHashMap();
+	def entities = [];
+
+	def messages = [];
+
+	float sectionsTime = 0;
+	int numberOfSection = 0;
+	HashMap<String, Float> sectionsTimesByTopic = new HashMap<String, Float>();
+	HashMap<String, Integer> numberOfSectionByTopic = new HashMap<String, Integer>();
+
+	float turnsTime = 0;
+	int numberOfTurn = 0;
+	HashMap<String, Float> turnsTimesBySpk = new HashMap<String, Float>();
+	HashMap<String, Integer> numberOfTurnsBySpk = new HashMap<String, Integer>();
+
+	int totalWords = 0;
+	HashMap<String, Integer> totalWordsByTopic = new HashMap<String, Integer>();
+	HashMap<String, Integer> totalWordsBySpk = new HashMap<String, Integer>();
+
+	def static okTopics = ["débouchés", "parcours","ressources",
+		"métier", "acteurs", "autres", "images"];
+
+	def static spkPattern = "((int|enq)[0-9][0-9])|(ext[0-9][0-9][0-9])"
+
+	public static boolean checkSpk(String spk)
+	{
+		return spk.matches("((int|enq)[0-9][0-9])|(ext[0-9][0-9][0-9])");
+	}
+
+	public ValidateTRS(File trsFile)
+	{
+		def slurper = new XmlParser();
+		root = slurper.parse(trsFile.toURI().toString())
+		episodeElem = root.Episode;
+		speakersElem = root.Speakers;
+		topicsElem = root.Topics
+		init();
+	}
+
+	public boolean test(){
+		//ckeckTimes();
+	}
+
+	public void init() {
+		initSpeakersDecl();
+		initTopicsDecl();
+
+		initSections();
+		initTurns();
+		initSyncs();
+
+		initSpeakers();
+		initTopics();
+		initEntities();
+	}
+
+	public void initEntities() {
+		for(def section : sections)
+			for(def turn : turns.get(section))
+				for(def entity : turn.Event)
+					if (entity.@type == "entities")
+						entities.add(entity.@desc)
+	}
+
+	public void initTopicsDecl(){
+		declaredTopics = topicsElem.Topic;
+		for (def topic:declaredTopics) {
+			topicDict[topic.@id]=topic.@desc
+		}
+	}
+
+	public void initSpeakersDecl(){
+		declaredSpeakers = speakersElem.Speaker;
+		for (def spk:declaredSpeakers) {
+			spkDict[spk.@id]=spk.@name
+		}
+	}
+
+	public void initSections() {
+		sections = episodeElem.Section;
+		numberOfSection = sections.size()
+	}
+
+	public void initTurns() {
+		for(def section : sections) {
+			turns[section] = section.Turn
+			numberOfTurn += turns[section].size()
+			//numberOfTurn
+		}
+	}
+
+	public void initSyncs() {
+		for(def section : sections)
+			for(def turn : turns.get(section))
+				syncs[turn] = turn.Sync
+	}
+
+	public void initTopics(){
+		for(def section : sections) {
+			String topic = section.@topic;
+			if (section.@type != "report")
+				topic = section.@type
+			if (topic == null) topic = "notopic"	
+			topics.add(topic);
+		}
+
+		topics.sort()
+	}
+
+	public void initSpeakers(){
+		for(def section : sections)
+			for(def turn : turns.get(section)) {
+				String spk = turn.@speaker;
+				if (spk == null) spk = "nospk"
+				speakers.add(spk);
+			}
+		speakers.sort();
+	}
+
+	public boolean topicsCheck() {
+		messages = []
+		boolean ret = false;
+		def topicsIds = declaredTopics.collect { topic -> topic.@id }
+		for(def topic : topics)
+			if( topic != null && !topicsIds.contains(topic) && topic != "nontrans" && topic != "notopic") {
+				messages << "missing topic declaration: "+topic
+				ret = true;
+			}
+		return ret;
+	}
+
+	public def speakersCheck() {
+		messages = []
+		boolean ret = false;
+		def spksIds = declaredSpeakers.collect { spk -> spk.@id }
+		for(def spk : speakers)  {
+			if (spk == null) continue; // ignore null spk
+			def split = spk.split(" ")
+			for(def sspk : split) {
+				if(!spksIds.contains(sspk) && sspk != "nospk") {
+					println "missing spk declaration: "+sspk
+					ret = true;
+				}
+			}
+		}
+		return ret;
+	}
+
+	String NULL = "null"
+	public void printSortedList(def list, def dict) {
+		def all = [:]
+		for(def value : list) {
+			if (value == null) {
+				if(!all.containsKey(NULL))
+					all[NULL] = 0
+				all[NULL] = all[NULL] + 1
+			} else {
+				def split = value.split(" ")
+				if (dict != null) {
+					for(def sspk : split) {
+						if (!(all.containsKey(sspk)))
+							all[sspk] = 0;
+						all[sspk] = all[sspk] + 1
+					}
+				} else {
+					if (!(all.containsKey(value)))
+						all[value] = 0;
+					all[value] = all[value] + 1
+				}
+			}
+		}
+
+		def toprint = [];
+		for(def speaker : all.keySet())
+			toprint.add([speaker, all[speaker]]);
+
+		toprint = toprint.sort{it -> it.get(1)}
+
+		for(def line : toprint)
+			if (dict != null) {
+				if (dict[line.get(0)] != null)
+					println line.get(0)+"\t"+dict[line.get(0)]+"\t"+line.get(1)
+				else
+				println line.get(0)+"\t"+line.get(0)+"\t"+line.get(1)
+			} else {
+				println line.get(0)+"\t"+line.get(1)
+			}
+	}
+
+	public def timeCheck() {
+		messages = [];
+		boolean warning = false;
+		boolean error = false
+		def previousSectionStart = -1.0f;
+		def previousSectionEnd = -1.0f;
+
+		def previousTurnStart = -1.0f;
+		def previousTurnEnd = -1.0f;
+
+		for(def section : sections) {
+			def sectionStart = Float.parseFloat(section.'@startTime')
+			def sectionEnd =  Float.parseFloat(section.'@endTime')
+			def topic = section.@topic;
+			if (section.@type != "report")
+				topic = section.@type;
+			if (topic == null) topic = "notopic"
+			
+			def time = sectionEnd - sectionStart;
+			if (!sectionsTimesByTopic.containsKey(topic)) {
+				sectionsTimesByTopic.put(topic, 0);
+				numberOfSectionByTopic.put(topic, 0);
+				totalWordsByTopic.put(topic, 0);
+			}
+			if (topic != null && topic != "null" && topic.length() > 0) {
+				sectionsTime += time;
+			}
+			sectionsTimesByTopic.put(topic, sectionsTimesByTopic.get(topic) + time);
+			numberOfSectionByTopic.put(topic, numberOfSectionByTopic.get(topic) + 1);
+
+			boolean sectionBug = false;
+
+			if ( sectionEnd <= sectionStart) { sectionBug = true;
+				messages << "sectionEnd <= sectionStart : $sectionEnd <= $sectionStart"
+			}
+
+			if ( sectionStart < previousSectionStart) { sectionBug = true;
+				messages << "sectionStart < previousSectionStart : $sectionStart < $previousSectionStart"
+			}
+			if ( sectionStart < previousSectionEnd) { sectionBug = true;
+				messages << "sectionStart < previousSectionEnd : $sectionStart < $previousSectionEnd"
+			}
+			if ( sectionEnd < previousSectionStart) { sectionBug = true;
+				messages << "sectionEnd < previousSectionStart : $sectionEnd < $previousSectionStart"
+			}
+			if ( sectionEnd < previousSectionEnd) { sectionBug = true;
+				messages << "sectionEnd < previousSectionEnd : $sectionEnd < $previousSectionEnd"
+			}
+
+			for(def turn : turns.get(section)) {
+				def turnStart = Float.parseFloat(turn.'@startTime')
+				def turnEnd =  Float.parseFloat(turn.'@endTime')
+				def spk = ""+turn.@speaker;
+				if (turn.@speaker == "null") spk = "nospk"
+				time = turnEnd - turnStart;
+				int nbwords = turn.text().split().length
+
+				for(String subspk : spk.split(" ")) {
+					String spkname = spkDict[subspk];
+					if (spkname == null) spkname = "nospk"
+					if (!totalWordsBySpk.containsKey(spkname)) {
+						totalWordsBySpk.put(spkname, 0);
+						turnsTimesBySpk.put(spkname, 0);
+						numberOfTurnsBySpk.put(spkname, 0);
+					}
+				}
+
+				if (spk != "null") turnsTime += time;
+				totalWords += nbwords
+				for(String subspk : spk.split(" ")) {
+					String spkname = spkDict[subspk];
+					if (spkname == null) spkname = "nospk"
+					totalWordsBySpk.put(spkname, totalWordsBySpk.get(spkname) + nbwords)
+					numberOfTurnsBySpk.put(spkname, numberOfTurnsBySpk.get(spkname) + 1);
+					turnsTimesBySpk.put(spkname, turnsTimesBySpk.get(spkname) + time);
+				}
+				totalWordsByTopic.put(topic,totalWordsByTopic.get(topic) +nbwords);
+
+				boolean bug = false;
+
+				if ( turnEnd <= turnStart) { bug = true;
+					messages << " turnEnd <= turnStart : $turnEnd <= $turnStart"
+				}
+
+				//				if ( turnStart < previousSectionStart) { bug = true;
+				//					println " turnStart < previousSectionStart : $turnStart < $previousSectionStart"
+				//				}
+				//				if ( turnStart < previousSectionEnd) { bug = true;
+				//					println " turnStart < previousSectionEnd : $turnStart < $previousSectionEnd"
+				//				}
+				//				if ( turnEnd < previousSectionStart) { bug = true;
+				//					println " turnEnd < previousSectionStart : $turnEnd < $previousSectionStart"
+				//				}
+				//				if ( turnEnd < previousSectionEnd) { bug = true;
+				//					println " turnEnd < previousSectionEnd : $turnEnd < $previousSectionEnd"
+				//				}
+
+				if ( turnStart < previousTurnStart) { bug = true;
+					messages << " turnStart < previousTurnStart : $turnStart < $previousTurnStart"
+				}
+				if ( turnStart < previousTurnEnd) { bug = true;
+					messages << " turnStart < previousTurnEnd : $turnStart < $previousTurnEnd"
+				}
+				if ( turnEnd < previousSectionStart) { bug = true;
+					messages << " turnEnd < previousTurnStart : $turnEnd < $previousTurnStart"
+				}
+				if ( turnEnd < previousTurnEnd) { bug = true;
+					messages << " turnEnd < previousTurnEnd : $turnEnd < $previousTurnEnd"
+				}
+				if (turnStart < sectionStart) { bug = true;
+					messages << " turnStart < sectionStart : $turnStart < $sectionStart"
+				}
+				if (turnStart > sectionEnd) { bug = true;
+					messages << " turnStart > sectionEnd : $turnStart > $sectionEnd"
+				}
+				if ( turnEnd < sectionStart) { bug = true;
+					messages << " turnEnd < sectionStart : $turnEnd < $sectionStart"
+				}
+				if (turnEnd > sectionEnd) { bug = true;
+					messages << " turnEnd > sectionEnd : $turnEnd > $sectionEnd"
+				}
+
+				for(def sync : syncs.get(turn)) {
+					boolean syncBug = false;
+					def syncTime = Float.parseFloat(sync.'@time')
+					if ( syncTime < turnStart) { syncBug = true;
+						messages << "  syncTime < turnStart : $syncTime < $turnStart"
+					}
+					if ( syncTime > turnEnd) { syncBug = true;
+						messages << "  syncTime > turnEnd : $syncTime > $turnEnd"
+					}
+
+					if (syncBug) {
+						warning = true;
+					}
+				}
+
+				previousTurnStart = turnStart;
+				previousTurnEnd = turnEnd;
+				if (bug) {
+					error = true;
+				}
+			}
+
+			previousSectionStart = sectionStart;
+			previousSectionEnd = sectionEnd;
+			if ( sectionBug) {
+				error = true;
+			}
+		}
+
+		return [error, warning];
+	}
+
+	public void printErrors()
+	{
+		for(String mess : messages)
+			println mess;
+	}
+
+	public void printAll()
+	{
+		println "Declared topics: $declaredTopics"
+		println "Topics: $topics"
+
+		println "Declared speakers: $declaredSpeakers"
+		println "Speakers: $speakers"
+
+		println "Sections: "+sections.size()
+		println "Turns: "+syncs.size()
+		println "Entities: $entities"
+	}
+
+	public static HashSet<String> allTopicDesc = new HashSet<String>();
+	public static HashSet<String> allSpkName = new HashSet<String>();
+	public static void checkDirectory(File dir) {
+		allTopicDesc = new HashSet<String>();
+		allSpkName = new HashSet<String>();
+		//println "trs dir: $dir"
+		for (File trsFile : dir.listFiles().sort()) {
+			if (!trsFile.getName().endsWith(".trs")) continue; // ignore
+			checkTRS(trsFile)
+
+		}
+
+		//println allTopicDesc;
+		//println allSpkName;
+
+	}
+
+	public static boolean checkTRS(File trsFile) {
+		boolean ret = true;
+		println "\n***** $trsFile.name *****"
+		def tester = new ValidateTRS(trsFile);
+		//tester.printAll();
+
+		def (error, warning) = tester.timeCheck();
+		if( error ) {
+			print "\nTIMES: "
+			//if( error || warning) {
+			println "errors: "+error+" warnings: "+warning;
+			tester.printErrors();
+			ret = false;
+		} //else println "all declared"
+
+		println "\nSPEAKERS: id, name, frequency"
+		if (tester.speakersCheck()) {
+			println "errors:"
+			tester.printErrors();
+			ret = false;
+		} //else println "all declared"
+		tester.printSortedList(tester.speakers, tester.spkDict);
+
+		println "\nSECTIONS: id, title, frequency"//+ tester.declaredTopics
+		if (tester.topicsCheck()) {
+			println "errors:"
+			tester.printErrors();
+			ret = false;
+		} //else println "all declared"
+		tester.printSortedList(tester.topics, tester.topicDict);
+
+		println "\nENTITIES: value, frequency"
+		tester.printSortedList(tester.entities, null);
+
+		println ""
+		for(def topic : tester.declaredTopics) {
+			allTopicDesc.add(topic.@desc)
+			if (!okTopics.contains(topic.@desc)) {
+				if(tester.topics.contains(topic.@id))
+					println "topic error: "+topic.@desc
+
+			}
+		}
+		//println ""
+		for(def spk : tester.declaredSpeakers) {
+			allSpkName.add(spk.@name)
+			if(!checkSpk(spk.@name)) {
+				if(tester.speakers.contains(spk.@id))
+					println "spk error: "+spk.@name
+			}
+		}
+		DecimalFormat formater = new DecimalFormat("0.00");
+
+		println "Statistics:"
+		println " Approx nb words: "+tester.totalWords;
+		println " Total time: "+ExecTimer.formatSecs(tester.turnsTime)
+		println " Approx word rate (word/sec): "+formater.format(tester.totalWords/tester.turnsTime)
+		println " Approx word rate (word/min): "+formater.format(60*(tester.totalWords/tester.turnsTime))
+		println ""
+
+		def meanSections = [:];
+		for(String key : tester.sectionsTimesByTopic.keySet())
+			meanSections.put(key, ExecTimer.formatSecs(tester.sectionsTimesByTopic.get(key) / tester.numberOfSectionByTopic.get(key)))
+		def meanTurns = [:];
+		for(String key : tester.turnsTimesBySpk.keySet())
+			meanTurns.put(key, ExecTimer.formatSecs(tester.turnsTimesBySpk.get(key) / tester.numberOfTurnsBySpk.get(key)))
+
+		for(String key : tester.sectionsTimesByTopic.keySet())
+			tester.sectionsTimesByTopic.put(key, ExecTimer.formatSecs(tester.sectionsTimesByTopic.get(key)))
+		for(String key : tester.turnsTimesBySpk.keySet())
+			tester.turnsTimesBySpk.put(key, ExecTimer.formatSecs(tester.turnsTimesBySpk.get(key)))
+
+		println " Number of sections: "+tester.numberOfSection
+		println " Sections time: "+ExecTimer.formatSecs(tester.sectionsTime);
+		println " Mean time of sections: "+ExecTimer.formatSecs(tester.sectionsTime/tester.numberOfSection)
+
+		println " Sections time by topic: "+tester.sectionsTimesByTopic
+		println " Number of sections by topic: "+tester.numberOfSectionByTopic
+		println " Mean time of sections by topic: "+meanSections
+		println ""
+
+		println " Number of turns: "+tester.numberOfTurn
+		println " Turns time: "+ExecTimer.formatSecs(tester.turnsTime);
+		println " Mean time of turns: "+ExecTimer.formatSecs(tester.turnsTime/tester.numberOfTurn)
+
+		println " Turns time by spk: "+tester.turnsTimesBySpk
+		println " Number of turns by spk: "+tester.numberOfTurnsBySpk
+		println " Mean time of turns by spk: "+meanTurns
+		println ""
+
+		return ret;
+	}
+
+	public static void main(String[] args) {
+		//File dir = new File("/home/sheiden/enslyon_projets/Textométrie/SpUV/EVS/master 1 STDDAD 2011/enquête 2011/Reçues")
+		File dir = new File("/home/mdecorde/xml/concattrs/ready")
+		ValidateTRS.checkDirectory(dir);
+		//ValidateTRS.checkTRS(new File("/home/mdecorde/xml/concattrs/int40-21.trs"))
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRS2SRT.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRS2SRT.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/TRS2SRT.groovy	(revision 321)
@@ -0,0 +1,225 @@
+package org.txm.importer.transcriber
+/**
+ * Converts a file from Transcriber XML to STR subtitle file
+ * 
+ * @author mdecorde
+ *
+ */
+
+
+import java.util.ArrayList
+
+import javax.xml.parsers.*
+import javax.xml.stream.*
+
+import java.net.URL
+
+class TRS2SRT {
+
+	private def url
+	private def inputData
+	private def factory
+	private XMLStreamReader parser
+	def writer
+
+	def colors = ["white", "red", "blue", "green", "yellow", "grey", "violet", "pink", "orange", "brown"];
+	def icolor = 0;
+
+	File trsFile;
+	boolean debug = false
+
+	public TRS2SRT(File trsFile) {
+		inputData = trsFile.toURI().toURL().openStream()
+		factory = XMLInputFactory.newInstance()
+		parser = factory.createXMLStreamReader(inputData)
+
+		this.trsFile = trsFile
+	}
+
+	/**
+	 * Transform the trsFile to STR format
+	 * 
+	 * @param srtFile: result file
+	 */
+	public void process(File srtFile) {
+		writer = srtFile.newWriter("UTF-8")
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					processStartElement()
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					processEndElement()
+					break;
+				case XMLStreamConstants.CHARACTERS:
+					processText()
+					break;
+			}
+		}
+		writer.close()
+	}
+
+	def speakers = [:]
+	def currentSpeakers = []
+	def currentSpeaker = "";
+	def currentSpeach = ""
+	def currentStartTime = ""
+	def currentEndTime = ""
+	def inTurn = false
+	def nTurn = 1
+	def syncCounter = 0;
+	def whoCounter = 0;
+
+	protected void processStartElement() {
+		String localname = parser.getLocalName()
+		switch (localname) {
+			case "Speaker":
+				def info = [:]
+				speakers[parser.getAttributeValue(null, "id")] = info
+				info["name"] = parser.getAttributeValue(null, "name")
+				info["color"] = colors[icolor]
+				info["count"] = 0;
+				icolor++;
+				break;
+			case "Episode":
+				if (debug) println "speakers $speakers"
+				break;
+			case "Turn":
+				if (debug) println "Turn!"
+				currentSpeakers = parser.getAttributeValue(null, "speaker").split(" ")
+				currentSpeaker = currentSpeakers[0]
+				currentSpeach = ""
+				String s = parser.getAttributeValue(null, "startTime")
+				String e = parser.getAttributeValue(null, "endTime")
+				currentStartTime = Float.parseFloat(s)
+				if (e != null) currentEndTime = Float.parseFloat(e)
+				else currentEndTime = currentStartTime+3.0; // temporary fix
+				inTurn = true
+				syncCounter = 0;
+				whoCounter = 0;
+				break;
+			case "Event":
+			if (debug) println "Event!"
+				currentSpeach += " ["+parser.getAttributeValue(null, "desc")+"] "
+				break;
+			case "Sync": // cut a Turn, the Turn@endTime must be replace with Sync@time
+				if (debug) println "Sync!"
+				syncCounter++; 
+				if (syncCounter > 1) { // ignore first 'Sync', there is no speech to write
+					def end = currentEndTime
+					currentEndTime = Float.parseFloat(parser.getAttributeValue(null, "time"))
+					writeSRTTurn() // write previous Turn, so currentEndTime is the Sync@time
+					currentEndTime = end; // restore Turn@endTime
+				}
+				break;
+			case "Who":
+				if (debug) println "Who!"
+				whoCounter++;
+				if (whoCounter > 1) { // ignore first 'Who', there is no speach to write
+					int n = Integer.parseInt(parser.getAttributeValue(null, "nb")) - 1
+					if (currentSpeakers.size() <= n) {
+						println "'Who@nb' Error at "+parser.getLocation()
+						break;
+					}
+					
+					writeSRTTurn() // write previous speach		
+					// switch current speaker
+					currentSpeaker = currentSpeakers[Integer.parseInt(parser.getAttributeValue(null, "nb")) - 1]
+				}
+				break;
+		}
+	}
+
+	protected void processEndElement() {
+		String localname = parser.getLocalName()
+		switch (localname) {
+			case "Speaker":
+				break;
+			case "Turn":
+				inTurn = false
+				writeSRTTurn()
+				break;
+			case "Sync":
+				break;
+			case "Who":
+				break;
+		}
+	}
+
+	protected writeSRTTurn() {
+		currentSpeach = currentSpeach.trim()
+
+		if (currentSpeach.length() == 0) return; // nothing to write
+
+		//println "Writing Turn of '$currentSpeaker': "+speakers[currentSpeaker]
+		def color = speakers[currentSpeaker]["color"]
+
+		if (speakers[currentSpeaker]["count"] < 2) {
+			speakers[currentSpeaker]["count"] = speakers[currentSpeaker]["count"] + 1;
+			currentSpeach = speakers[currentSpeaker]["name"]+"\t"+currentSpeach
+		}
+
+		currentSpeach = "<font color=\"$color\">$currentSpeach</font>".replaceAll("\n", " ")
+
+		def s = formatTime(currentStartTime)
+		def e = formatTime(currentEndTime)
+		currentStartTime = currentEndTime
+		writer.println """
+$nTurn
+$s --> $e
+$currentSpeach"""
+		nTurn++
+		currentSpeach = "" // reset speach
+	}
+
+	protected void processText() {
+		String txt = parser.getText();
+		if (inTurn) {
+			currentSpeach += txt
+		}
+	}
+
+	private String formatTime(float time) {
+		String rez = " ";
+
+		int ms = (time - (int)time) * 1000
+
+		float h = time / 3600;
+		time = time%3600;
+
+		float min = (time%3600) / 60;
+		int sec = (int)time%60
+
+		if (min < 10)
+			rez = ""+(int)h+":0"+(int)min;//+":"+time%60;
+		else
+			rez = ""+(int)h+":"+(int)min;//+":"+time%60;
+
+		if (sec >= 10)
+			rez += ":"+sec;
+		else
+			rez += ":0"+sec;
+
+		rez += "."+ms
+		return rez;
+	}
+
+	public static void main(String[] args) {
+		if (args.length == 0) {
+			println "Usage:"
+			println "java -jar TRS2SRT.jar file1 file2 file3"
+			println "\nResult files are saved in the save directory as fileN files."
+		}
+	//	args = ["/home/mdecorde/CCC/RECETTE/TRS2SRT/victor/pazinterior_ok.trs"]
+		for (String path : args) {
+			println "Processing path=$path ..."
+			
+			File trsFile = new File(path)
+			String name = trsFile.getName()
+			name = name.substring(0, name.indexOf("."))
+			File srtFile = new File(trsFile.getParentFile(), name+".srt")
+			TRS2SRT t = new TRS2SRT(trsFile)
+			t.process(srtFile)
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/compiler.groovy	(revision 321)
@@ -0,0 +1,306 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.alceste;
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * build wtc
+ * build CWB indexes
+ * create registry file.
+ *
+ * @author mdecorde
+ */
+class compiler 
+{
+	/** The debug. */
+	private boolean debug = false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String, String> anahash = new HashMap<String, String>() ;
+	
+	/** The text. */
+	String text = "";
+	
+	/** The base. */
+	String base = "";
+	
+	/** The project. */
+	String project = "";
+	
+	/** The text attributes. */
+	String[] textAttributes = null;
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	String lang = "fr";
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+	
+	
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url, String text, String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.textAttributes = textAttributes;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (Exception ex) {
+			System.err.println("IOException while parsing: "+ex);
+		}
+	}
+	
+	/** The annotation success. */
+	boolean annotationSuccess = false;
+	
+	/**
+	 * Sets the annotation success.
+	 *
+	 * @param val the new annotation success
+	 */
+	public void setAnnotationSuccess(boolean val)
+	{
+		this.annotationSuccess = val;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if (!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param f the f
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f){
+		try {
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			return false;
+		}
+	}
+	
+	/**
+	 * Sets the lang.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @param textAttributes the text attributes
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String corpusname, String[] textAttributes) 
+	{
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if (!binDir.exists()) {
+			println ("binary directory does not exists: "+binDir)
+			return false;
+		}
+		File wtcFile = new File(binDir,"wtc/"+corpusname+".wtc");
+		new File(binDir, "wtc").deleteDir();
+		new File(binDir, "wtc").mkdir();
+		new File(binDir, "data/$corpusname").deleteDir();
+		new File(binDir, "data/$corpusname").mkdir();
+		new File(binDir, "registry").mkdir();
+		
+		String textid = "";
+		int counttext = 0;
+		List<File> files = txmDir.listFiles();
+		if (files == null || files.size() == 0) {
+			println "no file to compile"
+			return false;
+		}
+		
+		//start corpus
+		if (createOutput(wtcFile)) {
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+		
+		Collections.sort(files);
+		//1- Transform into WTC file
+		XMLTXM2WTC wtcbuilder = null;
+		println("process "+files.size()+" files")
+		for (File f : files) {
+			print "."
+			counttext++;
+			if (!f.exists()) {
+				println("file "+f+ " does not exists")	
+			} else {			
+				wtcbuilder = new XMLTXM2WTC(f.toURI().toURL());
+				String txtname = f.getName().substring(0, f.getName().length()-4);
+				wtcbuilder.setTextInfo(txtname, corpusname, "default");
+
+				wtcbuilder.setBalisesToKeep(["text", "s", "p", "lb"]);
+				wtcbuilder.setSendToPAttributes(["lb":["n"], "p":["n"], "s":["n"]])
+				wtcbuilder.setLang(lang);
+				if (!wtcbuilder.transformFile(wtcFile)) {
+					println("Failed to compile "+f)
+				}
+			}
+		}
+		println ""
+		
+		//end corpus
+		if (createOutput(wtcFile)) {
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+		
+		if (wtcbuilder == null) {
+			println "there was no files in bin dir : "+txmDir
+			return false;
+		}
+		
+		//2- Import into CWB
+		String rootDir = binDir.getAbsolutePath();
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		
+		List<String> pAttributesList = wtcbuilder.getpAttributs();
+		List<String> sAttributesList = wtcbuilder.getsAttributs();
+		println "pAttrs : "+pAttributesList
+		println "sAttrs : "+sAttributesList
+		String[] pAttributes = pAttributesList.toArray(new String[pAttributesList.size()])
+		String[] sAttributes = sAttributesList.toArray(new String[sAttributesList.size()])
+		
+		try {
+			String regPath = rootDir + "/registry/"+corpusname.toLowerCase(); // CQP wants lowercase registry name
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), rootDir + "/data/${corpusname}", rootDir + "/wtc/"+corpusname+".wtc", regPath,pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname.toUpperCase(), rootDir + "/registry");
+		} catch (Exception ex) {System.out.println(ex); return false;}
+
+		return true;
+	}
+	
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Alceste import module</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/importer.groovy	(revision 321)
@@ -0,0 +1,185 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-12-16 17:28:35 +0100 (Fri, 16 Dec 2016) $
+// $LastChangedRevision: 3377 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.alceste
+
+
+//import filters.TabulatedToXml.*;
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.utils.io.FileCopy;
+import org.txm.scripts.teitxm.*;
+
+import javax.xml.stream.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.net.URL;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.FusionHeader.*;
+import filters.TagSentences.*;
+
+import org.txm.Toolbox;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param encoding the encoding
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public static boolean run(File rootDirFile, File binDir, File txmDir, String encoding, String basename, String lang)
+	{
+		if (rootDirFile.listFiles() == null || rootDirFile.listFiles().size() == 0) {
+			println "Error: no file to process in "+rootDirFile;
+			return false;
+		}
+		File srcfile;
+		for (File f : rootDirFile.listFiles()) {
+			if (!f.isHidden() && f.canRead() && f.isFile() && f.getName() != "import.xml" && !f.getName().endsWith("~")) {
+				srcfile = f
+				break;
+			}
+		}
+
+		File splitDir = new File(binDir, "split");
+		File tokenizeDir = new File(binDir, "tokenized");
+		File stokenizeDir = new File(binDir,"stokenized")
+		stokenizeDir.deleteDir();
+		stokenizeDir.mkdir();
+		tokenizeDir.deleteDir();
+		tokenizeDir.mkdir();
+		splitDir.deleteDir();
+		splitDir.mkdir();
+
+		// Build xml files from the alceste file
+		Alceste2Xml transformer = new Alceste2Xml()
+		if (!transformer.run(srcfile, splitDir, encoding)) {
+			return false;
+		}
+
+		// Tokenize xml files
+		List<File> srcfiles = splitDir.listFiles();
+		println (srcfiles.size()+ " texts found in "+srcfile)
+
+		println("Tokenizing files ("+srcfiles.size()+")")
+		for (File f : srcfiles) {
+			print "."
+			File resultfile = new File(tokenizeDir, f.getName());
+			try {
+				def builder = new SimpleTokenizerXml(f, resultfile, lang);
+				builder.setStartTag("text")
+				if (!builder.process()) {
+					println "Failed to tokenize: "+f;
+					resultfile.delete();
+				}
+			}
+			catch(Exception e){org.txm.utils.logger.Log.printStackTrace(e); println "Failed to tokenize: "+f; return false;}
+		}
+		println ""
+		
+		List<File> stokenfiles = tokenizeDir.listFiles();	
+		println("Tagging sentences of "+stokenfiles.size()+" files")
+		for (File f : stokenfiles) {
+			print "."
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F7 = new TagSentences();
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			S.add(F7);
+			S.add(F11);
+			File infile = f;
+			File xmlfile = new File(stokenizeDir, f.getName());
+			File headerfile = new File(f.getParentFile(), f.getName()+"header.xml");
+
+			S.SetInFileAndOutFile(infile.getPath(), xmlfile.getPath());
+			S.setEncodages("UTF-8","UTF-8");
+			Object[] arguments1 = [headerfile.getAbsolutePath()];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [headerfile.getAbsolutePath(),F1];
+			F11.SetUsedParam(arguments2);
+			if (!S.proceed()) {
+				println "Failed to tag file: "+f
+			}
+			S.clean();
+			headerfile.delete();//remove the prepared file to clean
+		}
+		println ""
+
+		//TRANSFORM INTO XML-TEI-TXM
+		List<File> tokenfiles = stokenizeDir.listFiles()
+		println("Building xml-tei-txm ("+tokenfiles.size()+" files)")
+		for (File f : tokenfiles) {
+			print "."
+			File file = f;
+			File txmfile = new File(txmDir, f.getName());
+
+			def correspType = new HashMap<String, String>()
+			def correspRef = new HashMap<String, String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			//lance le traitement
+			def builder = new Xml2Ana(file);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			if(!builder.process(txmfile))
+			{
+				println "Failed to build xml-txm of file: "+file;
+				txmfile.delete();
+			}
+		}
+		println ""
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/alcesteLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/alcesteLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/alceste/alcesteLoader.groovy	(revision 321)
@@ -0,0 +1,160 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-06-03 15:04:53 +0200 (Wed, 03 Jun 2015) $
+// $LastChangedRevision: 2984 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.alceste;
+
+import org.txm.importer.alceste.importer;
+import org.txm.importer.alceste.compiler;
+import org.txm.importer.xml.pager_old;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/TESTS/alceste2/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+
+if (!(new importer().run(srcDir, binDir, txmDir, encoding, basename, lang))) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = true;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+c.setAnnotationSuccess(annotationSuccess)
+if (debug) c.setDebug();
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+	
+	if (MONITOR != null) MONITOR.worked(20, "EDITION")
+
+	println "-- EDITION - Building edition"
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+	def second = 0
+
+	println "Paginating texts: "
+
+	for (File srcfile : filelist) {
+		print "."
+		String txtname = srcfile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);
+
+		def ed = new pager_old(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, "pb");
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XMLText2TXTCSV.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XMLText2TXTCSV.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XMLText2TXTCSV.groovy	(revision 321)
@@ -0,0 +1,32 @@
+package org.txm.importer
+
+File dir = new File("/home/mdecorde/xml/voeux/split_xml")
+File outdir = new File("/home/mdecorde/xml/voeux/split_txtcsv")
+println "1) xml -> txt + write metadata.csv"
+outdir.deleteDir()
+outdir.mkdir()
+File metadatafile = new File(outdir, "metadata.csv")
+String csvString = ""
+
+def files = dir.listFiles()
+files.sort()
+for(File f : files)
+{
+	File outfile = new File(outdir, f.getName()+".txt");
+	String text = f.getText("UTF-8");
+	String texttag = text.find("<text id.*>")
+//	println texttag
+//	texttag = texttag.replaceAll('<text id="([^"]+)"', '<text id="$1.txt"')
+//	println "> "+texttag
+	text = text.replaceAll("<text.*>", "").replace("</text>", "");
+	outfile.withWriter("UTF-8"){writer -> writer.write(text) }
+	csvString += texttag.replace("<text id=","").replace(" loc=", ",").replace(" annee=", ",").replace("\">", "\"")+"\n"
+}
+
+println "2) write metadata.csv"
+metadatafile.withWriter("UTF-8"){csvwriter -> 
+	csvwriter.write("\"id\",\"loc\",\"annee\"\n");
+	csvwriter.write(csvString)}
+
+println "3) rename Voeux_*"
+outdir.eachFileMatch(~/Voeux_.*/) {file-> file.renameTo(new File(outdir, file.getName().substring(6, 10)+".txt")) }
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/AddAttributeValuesInXML.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/AddAttributeValuesInXML.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/AddAttributeValuesInXML.groovy	(revision 321)
@@ -0,0 +1,99 @@
+package org.txm.importer
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.txm.metadatas.Entry
+import org.txm.utils.Pair;
+
+/**
+ * Add a attribute value map in a XML file
+ * Warning: if an attribute already exists its value won't be changed
+ */
+class AddAttributeValuesInXML extends StaxIdentityParser {
+	File xmlFile;
+	String tag, attribute;
+	List<String> values;
+	HashMap<String, String> attributesMap;
+	boolean result;
+
+	public AddAttributeValuesInXML(File xmlFile, String tag, String attribute, List<String> values)
+	{
+		super(xmlFile.toURI().toURL());
+		this.xmlFile = xmlFile;
+		this.tag = tag;
+		this.attribute = attribute;
+		this.values = values;
+	}
+
+	public boolean process(File outfile) {
+		this.result = false;
+		boolean ret = super.process(outfile)
+		return this.result & ret;
+	}
+
+	/**
+	 * Rewrite the processStartElement() to update/add attributes
+	 */
+	int n = 0;
+	public void processStartElement()
+	{
+		if (localname != tag) {
+			super.processStartElement()
+		} else {
+			String prefix = parser.getPrefix();
+			if (INCLUDE == localname && XI == prefix) {
+				processingXInclude();
+				return;
+			}
+
+			if (prefix.length() > 0)
+				writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+			else
+				writer.writeStartElement(localname);
+
+			for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {
+				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+			}
+
+			// get attributes
+			HashMap<String, String> attributes = new HashMap<String, String>();
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				attributes[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i);
+			}
+			// add/update the value
+			if (n < values.size()) {
+				attributes[attribute] = values[n]; 
+			} else {
+				println "ERROR: not enough values to insert for file $xmlFile, at XML parser location: l="+parser.getLocation().getLineNumber()+",c="+parser.getLocation().getColumnNumber()+")."
+			}
+			n++
+			
+			// write attributes
+			for (def k : attributes.keySet()) {
+				writer.writeAttribute(k, attributes[k])
+			}
+		}
+	}
+	
+	@Override
+	public void after() {
+		super.after();
+		
+		if (n != values.size()) {
+			println "ERROR: number of $tag ("+n+") missmatch the number of values to insert: "+values.size()
+		}
+	}
+
+	public static void main(String[] args)
+	{
+		File xmlfile = new File("/home/mdecorde/xml/macroaddfacsattribute/src/test1.xml")
+		File temp = File.createTempFile("temp", ".xml", xmlfile.getParentFile());
+
+		AddAttributeValuesInXML builder = new AddAttributeValuesInXML(xmlfile, "pb", "facs", ["V1","V1.2", "V2", "V3"]);
+		builder.process(temp);
+		
+		println "Done"
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Frantext import module (uses the BFM import module)</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/frantextLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/frantextLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/frantext/frantextLoader.groovy	(revision 321)
@@ -0,0 +1,213 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+//
+//
+// $LastChangedDate: 2012-02-13 11:42:24 +0100 (lun., 13 févr. 2012) $
+// $LastChangedRevision: 2126 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.frantext;
+
+import javax.xml.stream.XMLStreamReader;
+
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.bfm.importer;
+import org.txm.importer.bfm.compiler;
+import org.txm.importer.bfm.pager;
+import org.txm.objects.*;
+import org.txm.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.scripts.teitxm.*;
+import org.txm.importer.*;
+import org.w3c.dom.Element
+import org.txm.utils.*;
+
+String userDir = System.getProperty("user.home");
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+def MONITOR;
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/TESTS/frantext/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		//rootDir = userDir+"/xml/TESTS/alceste";  // directory which contains the source file
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+// BFM XPATH PARAMETERS
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "READ XPATH PARAMETERS")
+File paramDeclarationFile = new File(srcDir, "import.properties");
+Properties metadataXPath = new Properties();
+if (paramDeclarationFile.exists() && paramDeclarationFile.canRead()) {
+	InputStreamReader input = new InputStreamReader(new FileInputStream(paramDeclarationFile) , "UTF-8");
+	metadataXPath.load(input);
+	input.close();
+
+	if (!metadataXPath.containsKey("titre"))
+		println "Warning: parameters property file does not contain the 'titre' metadata"
+	if (!metadataXPath.containsKey("forme"))
+		println "Warning: parameters property file does not contain the 'forme' metadata in param values. The default value is 'prose'"
+}
+else
+	println "No '$paramDeclarationFile' file found"
+
+// Apply XSL
+if (MONITOR != null) MONITOR.worked(5, "APPLYING XSL")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+File xslFile = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "xsl/txm-filter-teifrantext-teibfm.xsl") // force xsl
+
+if (!ApplyXsl2.processImportSources(xslFile, srcDir, new File(binDir, "xsl"))) {
+	println "Error: failed to apply xsl $xsl. Aborting import."
+	return; // error durring process
+}
+srcDir = new File(binDir, "xsl");
+
+// copy txm files
+println "-- VALIDATION - checking XML source files well-formedness"
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+List<File> srcfiles = srcDir.listFiles();
+for (File f : srcfiles) { // check XML format, and copy file into binDir
+	if (f.isHidden() || f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties"))
+		continue;
+	if (ValidateXml.test(f)) {
+		FileCopy.copy(f, new File(txmDir, f.getName()));
+	} else {
+		println "Won't process file "+f;
+	}
+}
+if (MONITOR != null) MONITOR.worked(5)
+
+if (txmDir.listFiles() == null) {
+	println "No txm file to process"
+	return;
+}
+
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+def imp = new importer()
+if (!imp.run(srcDir, binDir, txmDir, basename, metadataXPath)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools - $model model"
+boolean annotate_status = true;
+if (annotate && !(new Annotate().run(binDir, txmDir, model+".par"))) {
+	println "Skipping Annotate process";
+	annotate_status = false;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if (debug) c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setLang(lang);
+c.setAnnotationDone(annotate_status)
+if (!c.run(binDir, txmDir, corpusname, metadataXPath)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION"
+	if (MONITOR != null) MONITOR.worked(25, "EDITION")
+	
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	files = c.getOrderedTxmFiles();
+	println("Building editions: "+files.size()+" files" );
+
+	for (File txmFile : files) {
+		print "."
+		String txtname = txmFile.getName();
+		int idx = txtname.lastIndexOf(".");
+		if(idx > 0) txtname = txtname.substring(0, idx);
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);//["'","(","[","{","«"];
+
+		Element text = params.addText(corpusElem, txtname, txmFile);
+
+		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, corpusname, metadataXPath);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Factiva2XmlTxm.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Factiva2XmlTxm.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Factiva2XmlTxm.groovy	(revision 321)
@@ -0,0 +1,261 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// script de recherche de contenus entre balises dans un fichier xml
+
+/**
+ *  @author Daniel Marin, Florent Bédécarrats
+ *  @author Matthieu Decorde
+ *  @copyright 2011 Daniel Marin &  Florent Bédécarrats, for original Perl version
+ *  @copyright 2011 ENS de Lyon, for translation in Groovy
+ *  This software is licensed under the GNU GPL version 3.0 or later. 
+ */
+package org.txm.importer;
+
+public run(File rootDir, String rep_final ) {
+
+	// A RENSEIGNER #########################################
+	
+	def fich_xml = rootDir.listFiles().grep(~/.*\.xml/)
+	println "$rootDir : $fich_xml"
+
+	def ext = ".xml"
+	//def rep_final = rootDir.getAbsolutePath()+"/out" // le nom du répertoire dans lequel on va créer les fichiers txm
+
+	List<String> Balises = ["<baseLanguage>","<date>","<sourceName>","<name>","<name>"] // balises XML à détecter
+	List<String> FinBalises = ["</baseLanguage>","</date>","</sourceName>","</name>","</name>"]
+
+	List<String> Sections = ["<ppsarticle>","<industry ","<region ","<headline>","<tailParagraphs>"] // Balises indiquant qu'à 'intérieur de la section se trouvent les infos (sur plusieurs lignes contrairement à @Balises)
+	List<String> FinSections = ["</ppsarticle>","</industry>","</region>","</headline>","</tailParagraphs>"]
+
+	def Headline = [] // le tableau dans lequel on va stocker le contenu de <headline>
+	def TailPara = [] // le tableau dans lequel on va stocker le contenu de <tailParagraphs>
+
+	def NomsMeta = ["language=","date=","source=","subject=","country="] // les noms qui interviennent dans les méta-données du fichier final
+	// FIN A RENSEIGNER #####################################
+
+	def TestMeta = [0,0,0,0,0] // Tableau qui indique si les métadonnées sont disponibles dans le fichier
+	def MetaData = [] //le tableau des metadonnées
+	def metaData = "" // la chaine dans laquelle on va stocker les métadonnées
+	String temp = ""
+	def fich_out = ">"
+	def nom_in = ""
+
+	def new_fich=0 // variable booleenne qui indique si on doit créer un nouveau txm
+	def industry=0 // variable booleenne qui indique qu'on est dans la section <industry> pour renseigner "subject"
+	def premIndustry=0 // variable booleenne qui indique qu'on a passé le 1er <industry> pour renseigner "subject"
+	def region=0 // variable booleenne qui indique qu'on est dans la section <region> pour renseigner "country"
+	def premRegion=0 // variable booleenne qui indique qu'on a passé le 1er <region> pour renseigner "country"
+	def copie=0 // variable booleenne qui indique si l'on doit copier le contenu de la ligne dans le fichier txm
+	def copieSec3=0 // variable qui indique si on est à l'intérieur de la section 3
+	def copieSec4=0 // variable qui indique si on est à l'intérieur de la section 4
+
+	def num_fich=0 // le numéro du fichier traité
+	def char_num_fich="" // pour numéroter les fichiers fich_001 pour garder l'ordre alphabétique
+
+	def i = 1
+
+	println "Création du répertoire dans lequel on va mettre les fichiers finaux txm"
+	if (new File(rep_final).exists()) {
+		println "Le répertoire $rep_final existe"
+	}
+	else {
+		if (!new File(rep_final).mkdir()) {
+			println "Echec de la création du répertoire: $rep_final"
+			return
+		}
+		if (!new File(rep_final).canWrite()) {
+			println "Le répertoire '$rep_final' final est read-only"
+			return
+		}
+	}
+
+	FileWriter ECRITURE
+	for (File fich_in : fich_xml) {
+		if (fich_in.getName() == "import.xml") continue;
+		println "Validate $fich_in...";
+		if (!ValidateXml.test(fich_in)) {
+			println "skip $fich_in"
+			continue;
+		}
+		// open(LECTURE, fich_in) || die("erreur d'ouverture de $fich_in")
+		println "on parcourt l'ensemble de $fich_in"
+		def reader = new FileReader(fich_in)
+		String line = reader.readLine()
+		while (line != null)
+		{
+			// on stocke le contenu de la ligne (et on efface le retour chariot à la fin)
+			temp = line
+			temp = temp.trim()
+			temp = (temp =~ /^\s+/).replaceAll("")
+			temp = (temp =~ /^\t/).replaceAll("")
+			temp = temp.replaceAll("</paragraph>", "</paragraph><br/>")
+
+			// On indique si on est dans la 1ère section de industry ou region pour prélever <name>
+			if ((temp =~ Sections[1]) && (premIndustry==0)){
+				industry=1
+				premIndustry=1
+			}
+			else if (temp =~ FinSections[1]){
+				industry=0
+			}
+
+			if ((temp =~ Sections[2]) && (premRegion==0)){
+				region = 1
+				premRegion = 1
+			}
+			else if (temp =~ FinSections[2]){
+				region = 0
+			}
+
+			// on regarde si la balise de création de nouveau fichier est présente
+			if (temp =~ Sections[0]){
+				new_fich=1
+				num_fich = num_fich+1
+				nom_in = fich_in.getName()
+				nom_in = (nom_in =~ /$ext/).replaceAll("")
+				// Petit test pour numéroter les fichiers en 001, 002 plutôt que 1, 2 (ordre alphabétique)
+				if (num_fich < 10){
+					char_num_fich = "00"+num_fich
+				} else if (num_fich<100) {
+					char_num_fich = "0"+num_fich
+				} else {
+					char_num_fich = num_fich
+				}
+				fich_out = new File(rep_final, nom_in + "_" + char_num_fich + ext)
+				//open(ECRITURE,"> $fich_out") || die("erreur d'ouverture de $fich_out ! $! \n")
+				println("new file : "+fich_out)
+				if (ECRITURE != null) ECRITURE.close()
+				ECRITURE = new FileWriter(fich_out)
+			}
+
+			// Remplissage des métadonnées : ####################################################
+			// Language
+			if (temp =~ Balises[0]){
+				temp = (temp =~ Balises[0]).replaceAll("")
+				temp = (temp =~ FinBalises[0]).replaceAll("")
+
+				MetaData[0]=temp
+				print " language trouve : "+MetaData[0]+" \n"
+				TestMeta[0]=1
+
+			}
+			// date
+			if (temp =~ Balises[1]) {
+				temp = (temp =~ Balises[1]).replaceAll("")
+				temp = (temp =~ FinBalises[1]).replaceAll("")
+				MetaData[1]=temp
+				print " date trouvee : "+MetaData[1]+" \n"
+				TestMeta[1]=1
+			}
+			// source
+			if (temp =~ Balises[2]){
+				temp = (temp =~ Balises[2]).replaceAll("")
+				temp = (temp =~ FinBalises[2]).replaceAll("")
+				MetaData[2]=temp
+				print " source trouvee : "+MetaData[2]+" \n"
+				TestMeta[2]=1
+			}
+			// subject (on doit être dans la 1ère section <industry>)
+			if ((temp =~ Balises[3]) && (industry==1)){
+				temp = (temp =~ Balises[3]).replaceAll("")
+				temp = (temp =~ FinBalises[3]).replaceAll("")
+				MetaData[3]=temp
+				industry=0
+				print " Sujet trouve : "+MetaData[3]+" \n"
+				TestMeta[3]=1
+			}
+			// country
+			if ((temp =~ Balises[4]) && (region==1)){
+				temp = (temp =~ Balises[4]).replaceAll("")
+				temp = (temp =~ FinBalises[4]).replaceAll("")
+				MetaData[4]=temp
+				region=0
+				print " Pays trouve : "+MetaData[4]+" \n"
+				TestMeta[4]=1
+
+				// Toutes les méta-données ont été trouvées, on assemble la chaîne
+				metaData = "<article "
+				for (i=0;i<(NomsMeta.size()+1);i++){
+					if (TestMeta[i]==1){
+						metaData = metaData+NomsMeta[i]+"\""+MetaData[i]+"\" "
+					}
+					else if (TestMeta[i]==0){
+						metaData = metaData +NomsMeta[i]+"\"\" "
+					}
+				}
+				metaData = "$metaData>\n"
+				print " metaData : $metaData \n"
+			}
+			// FIN Remplissage des métadonnées : #################################################
+
+			// On copie ou pas si on est ou pas dans <headline> ou <tailParagraphs>
+			if (temp =~ Sections[3]){
+				copieSec3=1
+			}
+			if (temp =~ FinSections[3]){
+				Headline.add(temp)
+				copieSec3=0
+			}
+			if (temp =~ Sections[4]){
+				copieSec4=1
+			}
+			if (temp =~ FinSections[4]){
+				TailPara.add(temp)
+				copieSec4=0
+			}
+
+			if (copieSec3 == 1){
+				Headline.add(temp)
+			}
+			if (copieSec4 == 1){
+				TailPara.add(temp)
+			}
+
+			if (temp =~ FinSections[0]){
+				ECRITURE.write("$metaData\n")
+				for (i=0;i<(Headline.size());i++){
+					if (i==0 || i==Headline){
+						ECRITURE.write(Headline[i].toString()+"\n")
+					}
+					else{
+						ECRITURE.write("\t"+Headline[i]+"\n")
+					}
+				}
+				for (i=0;i<(TailPara.size());i++){
+					if (i==0 || i==TailPara){
+						ECRITURE.write(TailPara[i].toString()+"\n")
+					}
+					else{
+						ECRITURE.write("\t"+TailPara[i]+"\n")
+					}
+				}
+				ECRITURE.write("</article>")
+
+				new_fich=0
+				industry=0
+				region=0
+				premIndustry=0
+				premRegion=0
+				MetaData=[]
+				TestMeta=[0,0,0,0,0]
+				Headline=[]
+				TailPara=[]
+
+				ECRITURE.close()
+				print "fichier $char_num_fich traite.\n"
+			}
+			line = reader.readLine()
+		}
+		ECRITURE.close()
+		reader.close()
+		num_fich = 0
+	}
+}
+
+String userDir = System.getProperty("user.home");
+File rootDir = new File(userDir, "xml/factiva")
+File outdir = new File(rootDir, "out2")
+run(rootDir, outdir.getAbsolutePath());
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/pager.groovy	(revision 321)
@@ -0,0 +1,366 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.perrault;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build Discours corpus simple edition from a xml-tei. @author mdecorde */
+class pager {
+	List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+	
+	/** The pagecount. */
+	int pagecount = 0;
+	
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The wordid. */
+	String wordid;
+	
+	/** The first word. */
+	boolean firstWord = true;
+	
+	/** The wordvalue. */
+	String wordvalue;
+	
+	/** The interpvalue. */
+	String interpvalue;
+	
+	/** The lastword. */
+	String lastword = " ";
+	
+	/** The wordtype. */
+	String wordtype;
+	
+	/** The flagform. */
+	boolean flagform = false;
+	
+	/** The flaginterp. */
+	boolean flaginterp = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	def writer;
+	
+	/** The multiwriter. */
+	def multiwriter = null;
+	
+	/** The infile. */
+	File infile;
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+	
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 */
+	pager(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter, int max) {
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		try {
+			if(multiwriter != null)
+			{
+				multiwriter.write("</body>");
+				multiwriter.write("</html>");
+				multiwriter.close();
+			}
+			pagecount++;
+			File f = new File(outfile.getParent()+"/multi/",outfile.getName().substring(0,outfile.getName().length()-6)+"_"+pagecount+".html");
+			pages.add(f);
+			idxstart.add(wordid)
+			multiwriter = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<head>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("<title>Perrault Edition - Page "+pagecount+"</title>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			this.outfile = outfile;
+
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+			createNextOutput();
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process()
+	{
+		String localname = "";
+		
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+		writer.write("<html>");
+		writer.write("<head>");
+		writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.write("<title>Perrault Edition</title>");
+		writer.write("</head>");
+		writer.write("<body>");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{				
+						case "text":
+						XPathResult testG = new XPathResult(infile);
+						String title =testG.getXpathResponse("//tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:title[type='sub']","")
+						writer.write("<h1>"+title+"</h1>\n")
+						multiwriter.write("<h1>"+title+"</h1>\n")
+						break;
+						
+						case "head":
+						writer.write("<h2>\n")
+						multiwriter.write("<h2>\n")
+						break;
+						
+						case "lg":
+						writer.write("<p>\n")
+						multiwriter.write("<p>\n")
+						break;
+						
+						case "p":
+						case "q":
+						writer.write("<p>\n")
+						multiwriter.write("<p>\n")
+						break;
+						
+						case "w":
+						wordid=parser.getAttributeValue(null,"id");
+						if(firstWord)
+						{
+							firstWord=false;
+							this.idxstart.set(0,wordid);
+						}
+						wordcount++;
+						if(wordcount >= wordmax)
+						{
+							createNextOutput();
+							wordcount=0;
+						}
+						break;
+						
+						case "ana":
+						flaginterp=true;
+						interpvalue+=" "+parser.getAttributeValue(null,"type")+":"
+						break;
+						
+						case "form":
+						wordvalue="";
+						interpvalue ="";
+						flagform=true;
+						break;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{				
+						case "head":
+						writer.write("</h2>\n")
+						multiwriter.write("</h2>\n")
+						break;
+						
+						case "lg":
+						writer.write("</p>\n")
+						multiwriter.write("</p>\n")
+						break;
+						
+						case "l":
+							writer.write("<br/>\n")
+							multiwriter.write("<br/>\n")
+							break;
+						
+						case "p":
+						case "q":
+						writer.write("</p>\n")
+						multiwriter.write("</p>\n")
+						break;
+						
+						case "form":
+						flagform = false
+						
+						break;
+						
+						case "ana":
+						flaginterp = false
+						break;
+						
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if(l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+							
+							if(interpvalue != null)
+								interpvalue = interpvalue.replace("\"","&quot;");
+							
+							if(NoSpaceBefore.contains(wordvalue) || 
+									NoSpaceAfter.contains(lastword) || 
+									wordvalue.startsWith("-") || 
+									NoSpaceAfter.contains(endOfLastWord))
+							{
+								writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							else
+							{
+								writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							writer.write(wordvalue.replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+						break;
+						
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		writer.close();
+		multiwriter.write("</body>");
+		multiwriter.write("</html>");
+		multiwriter.close();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/importer.groovy	(revision 321)
@@ -0,0 +1,243 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+
+package org.txm.importer.perrault
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import javax.xml.stream.*;
+import java.net.URL;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.FusionHeader.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+	
+	/**
+	 * Split.
+	 *
+	 * @param file the file
+	 * @return the list
+	 */
+	public static List<File> split(File file)
+	{
+		File f  = file;
+		
+		println "split file "+f;
+		String rootDir = f.getParent()+"/";
+		String xslfile = rootDir+"/xsl/splitcorpus.xsl";
+		String outfile = rootDir+"split_temp.xml";
+		
+		//get the splited file name
+		//String outfilename = new XPathResult(f).getXpathResponse("//TEI/text/body/div/head","");
+		
+		ApplyXsl a = new ApplyXsl(xslfile);
+		//a.SetParam("xpathtag", "//TEI");//coupe //text
+		//a.SetParam("xpathfilename", "/body/div/head");//cherche a partir de //xpathtag
+		a.process(f.getPath(),outfile);
+		
+		new File(outfile).delete();
+		
+		List<File> files = new File(f.getParent(),"split").listFiles();
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param srcfiles the srcfiles
+	 */
+	public static void run(File[] srcfiles)
+	{
+		List<File> files = null;
+		File fullfile;
+		String rootDir ="";
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		
+		for(File f : srcfiles)
+		{
+			files = importer.split(f);
+			rootDir = f.getParent()+"/"
+			fullfile = f;
+			new File(rootDir+"ptokenized").deleteDir();
+			new File(rootDir+"ptokenized").mkdir();
+			new File(rootDir+"tokenized").deleteDir();
+			new File(rootDir+"tokenized").mkdir();
+			new File(rootDir+"split").deleteDir();
+			new File(rootDir+"split").mkdir();
+			new File(rootDir+"txm").deleteDir();
+			new File(rootDir+"txm").mkdir();
+			
+			String xslfile = rootDir+"xsl/splitcorpus.xsl";
+			String infile = f.getPath();
+			String outfile = rootDir+"split_temp.xml";
+			
+			ApplyXsl a = new ApplyXsl(xslfile);
+			a.process(infile,outfile);
+			
+			new File(outfile).delete();
+		}
+		
+		//get header
+		String header =""
+		Reader reader = new FileReader(fullfile);
+		String cline = reader.readLine();
+		while(!cline.trim().contains("<text>"))
+		{
+			header += cline+"\n";
+			cline = reader.readLine();
+		}
+		
+		//put splited into tei file
+		for(File f : files)
+		{
+			File temp = new File(f.getParent(),"temp");
+			Writer writer = new OutputStreamWriter(new FileOutputStream(temp) , "UTF-8");
+			writer.write(header);
+			writer.write("""<text>\n<body>\n""")
+			f.eachLine{String line->
+				if(!line.startsWith("<?xml"))
+				writer.write(line+"\n");
+			}
+			
+			writer.write("""</body>\n</text>\n</TEI>""")
+			writer.close();
+			
+			if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+		}
+		
+		//PREPARE EACH SPLITED FILE TO BE TOKENIZED
+		println files
+		for(File f : files)
+		{
+			File srcfile = f;
+			File resultfile = new File(rootDir+"ptokenized",f.getName()+"-src.xml");
+			println("prepare tokenizer file : "+srcfile+" to : "+resultfile );
+			def builder = new OneTagPerLine(srcfile.toURL(), milestones);
+			builder.process(resultfile);
+		}
+		
+		//TOKENIZE FILES
+		files = new File(rootDir,"ptokenized").listFiles()	
+		for(File f : files)
+		{
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F6 = new Tokeniser(f);
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			S.add(F6);
+			S.add(F11);
+			File infile = new File(rootDir+"ptokenized",f.getName());
+			File xmlfile = new File(rootDir+"tokenized",f.getName()+"-out.xml");
+			File headerfile = new File(rootDir+"/ptokenized/",f.getName()+"header.xml");
+			println("Tokenizing "+xmlfile)
+			S.SetInFileAndOutFile(infile.getPath(), xmlfile.getPath());
+			S.setEncodages("UTF-8","UTF-8");
+			Object[] arguments1 = [headerfile.getAbsolutePath()];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [headerfile.getAbsolutePath(),F1];
+			F11.SetUsedParam(arguments2);
+			S.proceed();
+
+			System.out.println("exits src tokenized : "+infile.exists());
+			S= null;
+			F1=null;
+			F6=null;
+			F11=null;
+			System.out.println("delete src tokenized : "+infile.delete());//remove the prepared file to clean
+			headerfile.delete();//remove the prepared file to clean
+		}
+
+		//TRANSFORM INTO XML-TEI-TXM
+		files = new File(rootDir,"tokenized").listFiles()	
+		for(File f : files)
+		{
+			//ArrayList<String> milestones = new ArrayList<String>();
+			println("Building xml-tei-txm "+f+ " >> "+f.getName()+"-TXM.xml")
+			File file = f; 
+			String txmfile = f.getName()+"-TXM.xml";
+			
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();	
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+					//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			//lance le traitement
+			String wordprefix = "w_c_";
+			def builder = new Xml2Ana(file);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			builder.transformFile(rootDir+"txm/",txmfile);
+		}
+		
+		//rename files correctly
+		files = new File(rootDir,"txm").listFiles();
+		for(File file : files)
+		{
+			String txmfile = file.getName();
+			txmfile = txmfile.tokenize(".").get(0);
+			txmfile = txmfile.substring(0, txmfile.length()-1);
+			txmfile += ".xml"
+			file.renameTo(new File(file.getParent(),txmfile));
+		}
+		
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File[] files = [new File("~/xml/perrault/perrault.xml")];
+		new importer().run(files);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/allimport.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/allimport.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/allimport.groovy	(revision 321)
@@ -0,0 +1,659 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.perrault;
+
+import javax.xml.stream.*
+
+import org.txm.importer.*
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.filters.*
+import org.txm.scripts.*
+import org.txm.scripts.teitxm.*
+import org.txm.utils.treetagger.TreeTagger
+
+import filters.CutHeader.*
+import filters.FusionHeader.*
+import filters.Tokeniser.*
+
+// TODO: Auto-generated Javadoc
+/**
+ * Split.
+ *
+ * @param file the file
+ * @return the list
+ */
+public static List<File> split(File file)
+{
+	File f  = file;
+	
+	println "split file "+f;
+	String rootDir = f.getParent()+"/";
+	String xslfile = rootDir+"splitcorpus.xsl";
+	String outfile = rootDir+"split_temp.xml";
+	
+	//get the splited file name
+	//String outfilename = new XPathResult(f).getXpathResponse("//TEI/text/body/div/head","");
+	
+	ApplyXsl a = new ApplyXsl(xslfile);
+	//a.SetParam("xpathtag", "//TEI");//coupe //text
+	//a.SetParam("xpathfilename", "/body/div/head");//cherche a partir de //xpathtag
+	a.process(f.getPath(),outfile);
+	
+	new File(outfile).delete();
+	
+	List<File> files = new File(f.getParent(),"split").listFiles();
+}
+
+/**
+ * Run1.
+ *
+ * @param srcfiles the srcfiles
+ */
+public void run1(File[] srcfiles)
+{
+	List<File> files = null;
+	File fullfile;
+	String rootDir ="";
+	ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+	milestones.add("tagUsage");
+	milestones.add("pb");
+	milestones.add("lb");
+	
+	for(File f : srcfiles)
+	{
+		files = split(f);
+		rootDir = f.getParent()+"/"
+		fullfile = f;
+		new File(rootDir+"tokenized").deleteDir();
+		new File(rootDir+"tokenized").mkdir();
+		new File(rootDir+"split").deleteDir();
+		new File(rootDir+"split").mkdir();
+		new File(rootDir+"txm").deleteDir();
+		new File(rootDir+"txm").mkdir();
+		
+		String xslfile = rootDir+"splitcorpus.xsl";
+		String infile = f.getPath();
+		String outfile = rootDir+"split_temp.xml";
+		
+		ApplyXsl a = new ApplyXsl(xslfile);
+		a.process(infile,outfile);
+		
+		new File(outfile).delete();
+	}
+	
+	//get header
+	String header =""
+	Reader reader = new FileReader(fullfile);
+	String cline = reader.readLine();
+	while(!cline.trim().contains("<text>"))
+	{
+		header += cline+"\n";
+		cline = reader.readLine();
+	}
+	
+	//put splited into tei file
+	for(File f : files)
+	{
+		File temp = new File(f.getParent(),"temp");
+		Writer writer = new FileWriter(temp)
+		writer.write(header);
+		writer.write("""<text>\n<body>\n""")
+		f.eachLine{String line->
+			if(!line.startsWith("<?xml"))
+				writer.write(line+"\n");
+		}
+		
+		writer.write("""</body>\n</text>\n</TEI>""")
+		writer.close();
+		
+		if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+	}
+	
+	//PREPARE EACH SPLITED FILE TO BE TOKENIZED
+	println files
+	for(File f : files)
+	{
+		File srcfile = f;
+		File resultfile = new File(rootDir+"tokenized",f.getName()+"-src.xml");
+		println("prepare tokenizer file : "+srcfile+" to : "+resultfile );
+		def builder = new OneTagPerLine(srcfile.toURL(), milestones);
+		builder.process(resultfile);
+	}
+	
+	//TOKENIZE FILES
+	//Manager<Filter> filterManager = new FilterManager(ActionHome);		
+	for(File f : files)
+	{
+		Sequence S = new Sequence();
+		Filter F1 = new CutHeader();
+		Filter F6 = new Tokeniser(f);
+		Filter F11 = new FusionHeader();
+		S.add(F1);
+		S.add(F6);
+		S.add(F11);
+		File infile = new File(rootDir+"tokenized",f.getName()+"-src.xml");
+		File xmlfile = new File(rootDir+"tokenized",f.getName()+"-out.xml");
+		File headerfile = new File(rootDir+"/tokenized/",f.getName()+"header.xml");
+		println("Tokenize "+xmlfile)
+		S.SetInFileAndOutFile(infile.getPath(), xmlfile.getPath());
+		S.setEncodages("UTF-8","UTF-8");
+		Object[] arguments1 = [headerfile.getAbsolutePath()];
+		F1.SetUsedParam(arguments1);
+		Object[] arguments2 = [headerfile.getAbsolutePath(),F1];
+		F11.SetUsedParam(arguments2);
+		S.proceed();
+		S.clean();
+		infile.delete();//remove the prepared file to clean
+		headerfile.delete();//remove the prepared file to clean
+	}
+	files = new File(rootDir,"tokenized").listFiles()	
+	
+	
+	//TRANSFORM INTO XML-TEI-TXM
+	for(File f : files)
+	{
+		//ArrayList<String> milestones = new ArrayList<String>();
+		println("build xml-tei-txm "+f+ " >> "+f.getName()+"-TXM.xml")
+		File file = f; 
+		String txmfile = f.getName()+"-TXM.xml";
+		
+		def correspType = new HashMap<String,String>()
+		def correspRef = new HashMap<String,String>()
+		//il faut lister les id de tous les respStmt
+		def respId = [];
+		//fait la correspondance entre le respId et le rapport d'execution de l'outil
+		def applications = new HashMap<String,HashMap<String,String>>();	
+		//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+				//pour construire les ref vers les taxonomies
+		def taxonomiesUtilisees = new HashMap<String,String[]>();
+		//associe un id d'item avec sa description et son URI
+		def itemsURI = new HashMap<String,HashMap<String,String>>();
+		//informations de respStmt
+		//resps (respId <voir ci-dessus>, [description, person, date])
+		def resps = new HashMap<String,String[]>();
+		//lance le traitement
+		String wordprefix = "w_c_";
+		def builder = new Xml2Ana(file);
+		builder.setCorrespondances(correspRef, correspType);
+		builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+		builder.transformFile(rootDir+"txm/",txmfile);
+	}
+	
+	//rename files correctly
+	files = new File(rootDir,"txm").listFiles();
+	for(File file : files)
+	{
+		String txmfile = file.getName();
+		txmfile = txmfile.tokenize(".").get(0)+".xml"
+		file.renameTo(new File(file.getParent(),txmfile));
+	}
+	
+}
+
+/**
+ * Run2.
+ *
+ * @param rootDirFile the root dir file
+ */
+public void run2(File rootDirFile)
+{
+	String rootDir = rootDirFile.getAbsolutePath()+"/";
+	if(!new File(rootDir,"models").exists() || !new File(rootDir,"txm").exists())
+	{
+		println "no models dir or no txm dir, check rootDir "+rootDirFile;
+		return;
+	}
+	//cleaning
+	new File(rootDir,"annotations").deleteDir();
+	new File(rootDir,"annotations").mkdir();
+	new File(rootDir,"treetagger").deleteDir();
+	new File(rootDir,"treetagger").mkdir();
+	
+	ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+	milestones.add("tagUsage");
+	milestones.add("pb");
+	milestones.add("lb");
+	
+	List<File> files = new File(rootDir,"txm").listFiles()	
+	//BUILD TT FILE READY TO BE TAGGED
+	for(File f : files)
+	{
+		File srcfile = f;
+		File resultfile = new File(rootDir+"treetagger/",f.getName()+".tt");
+		new BuildTTSrc(srcfile.toURL()).process(resultfile)
+	}
+	
+	//APPLY TREETAGGER
+	files = new File(rootDir,"treetagger").listFiles()	
+	for(File f : files)
+	{
+		File modelfile = new File(rootDir+"models/","fr.par");
+		File infile = f
+		File outfile = new File(f.getParent(),f.getName()+"-out.tt");
+		println("3- APPLY TT on : "+infile+" with : "+modelfile +" >>  "+outfile);
+		
+		TreeTagger tt = new TreeTagger(System.getProperty("user.home")+"/TXM/treetagger/bin/");
+		tt.settoken();
+		tt.setlemma();
+		tt.setquiet();
+		tt.setsgml();
+		tt.setnounknown();
+		tt.seteostag("<s>");
+		tt.treetagger( modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())
+		infile.delete();
+	}
+	
+	//BUILD STAND-OFF FILES
+	//contains txm:application/txm:commandLine
+	File reportFile = new File(rootDir,"NLPToolsParameters.xml");
+	
+	String respPerson = System.getProperty("user.name");
+	String respId = "txm";
+	String respDesc = "NLP annotation tool";
+	String respDate = "";
+	String respWhen = ""
+	
+	String appIdent = "TreeTagger";
+	String appVersion = "3.2";
+	
+	String distributor = "";
+	String publiStmt = """""";
+	String sourceStmt = """""";
+	
+	def types = ["pos","lemme"];
+	def typesTITLE = ["",""];
+	def typesDesc = ["",""];
+	def typesTAGSET = ["",""];
+	def typesWEB = ["",""];
+	String idform ="w_c_";
+	
+	files = new File(rootDir,"treetagger").listFiles()	
+	for(File f : files)
+	{
+		String target = f.getAbsolutePath();
+		File ttfile = f
+		File posfile = new File(rootDir+"annotations/",f.getName()+"-STOFF.xml");
+		
+		def encoding ="UTF-8";
+		def transfo = new CSV2W_ANA();
+		println("build w-interp "+ttfile.getName()+ ">>"+posfile.getName())
+		transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+		transfo.setResp(respId, respDesc,respDate, respPerson, respWhen);
+		transfo.setApp(appIdent, appVersion);
+		transfo.setTarget(target, reportFile);
+		transfo.setInfos(distributor,  publiStmt, sourceStmt);
+		transfo.process( ttfile, posfile, encoding );
+	}
+	
+	files = new File(rootDir,"annotations").listFiles();
+	List<File> txmfiles = new File(rootDir,"txm").listFiles();
+	for(int i = 0 ; i< files.size();i++)
+	{
+		File srcfile = txmfiles.get(i);
+		File pos1file = files.get(i);
+		File temp = new File(rootDir,"temp"); 
+		
+		println("5- inject annotation in file : "+srcfile+" with : "+pos1file );
+		
+		def builder = new org.txm.scripts.teitxm.AnnotationInjection(srcfile.toURL(), pos1file.toURL(), milestones);
+		builder.transfomFile(temp.getParent(),temp.getName());
+		
+		if (!(srcfile.delete() && temp.renameTo(srcfile))) println "Warning can't rename file "+temp+" to "+srcfile
+	}
+	
+}
+
+def inputData;
+def factory;
+XMLStreamReader parser;
+def dir;
+def output;
+def url;
+HashMap<String,String> anahash =new HashMap<String,String>() ;
+String text="";
+String base="";
+String project="";
+
+/**
+ * initialize.
+ *
+ * @param url the url
+ * @param text the text
+ * @param base the base
+ * @param project the project
+ */
+public void compil(URL url,String text,String base, String project)
+{
+	this.text = text
+	this.base = base;
+	this.project = project;
+	try {
+		this.url = url;
+		inputData = url.openStream();
+		
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+	} catch (XMLStreamException ex) {
+		System.out.println(ex);
+	}catch (IOException ex) {
+		System.out.println("IOException while parsing ");
+	}
+}
+
+/**
+ * Creates the output.
+ *
+ * @param dirPathName the dir path name
+ * @param fileName the file name
+ * @return true, if successful
+ */
+private boolean createOutput(String dirPathName, String fileName){
+	try {
+		File f = new File(dirPathName, fileName)
+		output = new java.io.FileWriter(f,f.exists())
+		return true;
+	} catch (Exception e) {
+		System.out.println(e.getLocalizedMessage());
+		return false;
+	}
+}
+
+/**
+ * clear anaHash variable, it is used to store ana tags values then print it when the end element </ana> is found.
+ */
+private void fillanaHash()
+{
+	anahash.clear();
+	for(String s : types)
+		anahash.put( s,"-" );
+}
+
+
+/**
+ * Go to text.
+ */
+private void GoToText()
+{
+	for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+	{
+		if(event == XMLStreamConstants.END_ELEMENT)
+			if(parser.getLocalName().equals("teiHeader"))
+				return;
+	}
+}
+
+/**
+ * Transfom file wtc.
+ *
+ * @param dirPathName the dir path name
+ * @param fileName the file name
+ * @return true, if successful
+ */
+public boolean transfomFileWtc(String dirPathName, String fileName)
+{
+	createOutput(dirPathName, fileName);
+	
+	String headvalue=""
+	String vAna = "";
+	String vForm = "";
+	String wordid= "";
+	String vHead = "";
+	
+	int p_id = 0;
+	int q_id = 0;
+	int lg_id = 0;
+	int l_id = 0;
+	
+	boolean flaglg = false;
+	boolean flaghead = false;
+	boolean flagAuthor = false;
+	boolean flagDate = false;
+	boolean flagForm = false;
+	boolean flagAna = false;
+	
+	this.GoToText()
+	
+	try 
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+						
+						case "head"://get attr lang
+						flaghead =true;
+						vHead="";
+						break;
+						
+						case "text":
+						output.write("<text id=\""+text+"\" base=\""+base+"\" project=\""+project+"\">\n");
+						break;
+						
+						case "p":
+						output.write("<"+parser.getLocalName()+" id=\""+text+"_p_"+(p_id++)+"\">\n");
+						break;
+						case "q":
+						output.write("<"+parser.getLocalName()+" id=\""+text+"_q_"+(q_id++)+"\">\n");
+						break;
+						case "l":
+						output.write("<"+parser.getLocalName()+" id=\""+text+"_l_"+(l_id++)+"\">\n");
+						break;
+						
+						case "lg":
+						flaglg = true;
+						break;
+						
+						case "s":
+						output.write( "<s>\n");
+						break;
+						
+						case "w":
+						for(int i = 0 ; i < parser.getAttributeCount(); i++)
+							if(parser.getAttributeLocalName(i).equals("id"))
+							{	
+								wordid = parser.getAttributeValue(i);
+							}
+						break;
+						case "form":
+						flagForm = true;
+						vForm = "";
+						vAna ="";
+						break;
+						
+						case "ana":
+						flagAna = true;
+						break;
+					}
+					break;
+				
+				case XMLStreamConstants.END_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+						case "head"://get attr lang
+						flaghead =false;
+						if(flaglg)
+							output.write("<moral id=\""+text+"_moral_"+(lg_id++)+"\" head=\""+vHead+"\">\n");
+						break;
+						
+						case "text":
+						output.write("</text>\n");
+						break;
+						
+						case "p":
+						case "q":
+						case "l":
+						output.write("</"+parser.getLocalName()+">\n");
+						break;
+						
+						case "lg":
+						output.write("</moral>\n");
+						flaglg = false;
+						break;
+						
+						case "s":
+						output.write( "</s>\n");
+						break;
+						
+						case "w":
+						if(!(flaghead && flaglg))
+							if(vAna != null)
+								output.write( vForm +vAna+"\t"+wordid+"\n");
+						vAna = "";
+						vForm = "";
+						break;
+						
+						case "form":
+						flagForm = false;
+						break;
+						
+						case "ana":
+						flagAna = false;
+						break;
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagForm)
+						vForm += parser.getText().trim();
+					if(flagAna)
+						vAna += "\t" +parser.getText().trim();
+					if(flaghead && flaglg)
+						vHead += parser.getText().trim();
+					break;
+			}
+		}
+		output.close();
+		parser.close();
+	}
+	catch (XMLStreamException ex) {
+		System.out.println(ex);
+	}
+	catch (IOException ex) {
+		System.out.println("IOException while parsing " + inputData);
+	}
+	
+	return true;
+}
+
+/**
+ * Run3.
+ *
+ * @param files the files
+ * @return true, if successful
+ */
+public boolean run3(List<File> files) 
+{
+	String rootDir ="";
+	if(files.size() > 0)
+		rootDir = files.get(0).getParentFile().getParentFile().getAbsolutePath()+"/";//"~/xml/perrault/";
+	def cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+	
+	if(!new File(cwbLoc).exists())
+	{
+		println ("CWB path error: "+cwbLoc)
+		return false;
+	}
+	if(!new File(rootDir).exists())
+	{
+		println ("binary directory does not exists: "+rootDir)
+		return false;
+	}
+	new File(rootDir+"wtc/","perrault.wtc").delete();//cleaning&preparing
+	new File(rootDir+"wtc/").deleteDir();
+	new File(rootDir+"wtc/").mkdir();
+	new File(rootDir+"registry/").mkdir();
+	
+	String textid="";
+	int counttext =0;
+	//1- Transform into WTC file
+	for(File f : files)
+	{
+		counttext++;
+		if(!f.exists())
+		{
+			println("file "+f+ " does not exists")	
+		}
+		else
+		{	
+			println("process file "+f)
+			compil(f.toURL(),"text"+counttext,"perrault","default");
+			transfomFileWtc(rootDir+"wtc","perrault.wtc");
+		}
+	}
+	
+	//2- Import into CWB
+	def outDir =rootDir;
+	def outDirTxm = rootDir;
+	
+	CwbEncode cwbEn = new CwbEncode();
+	CwbMakeAll cwbMa = new CwbMakeAll();
+	
+	String[] pAttributes = ["id","pos","lemme"];
+	//String[] pAttributes = ["id"];
+	String[] sAttributes = ["text:0+id+base+project","p:0+id","q:0+id","moral:0+head+id","l:0+id"];
+	
+	try
+	{
+		cwbEn.run(cwbLoc + "cwb-encode", outDirTxm + "data/"+"PERRAULT", outDir + "/wtc/"+"perrault.wtc", outDirTxm + "registry/"+"perrault",pAttributes, sAttributes);
+		cwbMa.run(cwbLoc + "cwb-makeall", "PERRAULT", outDirTxm + "registry");
+		
+	} catch (Exception ex) {System.out.println(ex); return false;}
+	
+	System.out.println("Done.") 
+	
+	return true;
+}
+
+////FIN
+println "IMPORTER"
+File[] files = [new File("~/xml/perrault/perrault.xml")];
+run1(files);
+
+println "ANNOTATE"
+File rootDir = new File("~/xml/perrault/");
+run2(rootDir);
+
+println "COMPIL"
+File directory = new File("~/xml/perrault/txm/");
+files = directory.listFiles();
+ArrayList<File> Lfiles = new ArrayList<File>();
+for(File f : files)
+	Lfiles.add f
+run3(Lfiles);
+
+//move registry file to cwb registry dir
+File registryfile = new File("~/xml/perrault/txm/registry/perrault");
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/perraultLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/perraultLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/perraultLoader.groovy	(revision 321)
@@ -0,0 +1,133 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.perrault;
+
+import java.io.File;
+import org.txm.importer.perrault.importer;
+import org.txm.importer.perrault.compiler;
+import org.txm.importer.perrault.annotate;
+import org.txm.importer.perrault.pager;
+import org.txm.objects.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		rootDir = userDir+"/xml/perrault/";
+		lang="fr";
+		encoding= "UTF-8"; // not used
+		model="fr";
+		Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File(userDir,"txminstall"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"txminstall/treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"txminstall/treetagger/models"));
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+String corpusSrc = "perrault.xml";
+
+println "-- IMPORTER - Reading source files"
+File[] files = [new File(rootDir,corpusSrc)];
+new importer().run(files);
+
+println "-- ANNOTATE - Running NLP tools"
+new Annotate().run(new File(rootDir), model+".par");//the files in ./txm
+
+println "-- COMPILING - Building Search Engine indexes"
+files = new File(rootDir,"txm/").listFiles();
+ArrayList<File> Lfiles = new ArrayList<File>();
+for(File f : files)
+	Lfiles.add f
+def c = new compiler()
+//c.setCwbPath("")// for developers
+c.setLang(lang);
+c.run(Lfiles);
+
+//move registry file to cwb registry dir
+File registryfile = new File(rootDir+"/registry","perrault");
+if(registryfile.exists())
+	org.txm.utils.io.FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"registry/perrault"))
+	
+
+Workspace w = org.txm.Toolbox.workspace;
+Project p = w.getProject("default")
+p.removeBase("perrault")
+Base b = p.addBase("perrault");
+b.addDirectory(new File(rootDir,"txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION"
+new File(rootDir+"/HTML/").deleteDir()
+new File(rootDir+"/HTML/").mkdir();
+new File(rootDir+"/HTML/default/").mkdir();
+files = new File(rootDir,"txm").listFiles();
+
+for(String textname : b.getTextsID())
+{
+	Text text = b .getText(textname);
+	File srcfile = text.getSource();
+	File resultfile = new File(rootDir+"/HTML",srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+	println("Building edition  : "+srcfile+" to : "+resultfile );
+	
+	def ed = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,600);
+	Edition editionweb = text.addEdition("default","web",resultfile);
+	//println("pages "+ed.getPageFiles())
+	//println("idx "+ed.getIdx())
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+	
+//	Edition editionbp = text.addEdition("bp","integral",resultfile);
+//	editionbp.addPage(resultfile,ed.getIdx().get(0));
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/annotate.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/annotate.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/annotate.groovy	(revision 321)
@@ -0,0 +1,176 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-11-08 13:38:06 +0100 (ven., 08 nov. 2013) $
+// $LastChangedRevision: 2569 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.perrault
+
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import org.txm.Toolbox;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class annotate.
+ */
+class annotate {
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 */
+	public static void run(File rootDirFile)
+	{
+		String rootDir = rootDirFile.getAbsolutePath()+"/";
+
+		//cleaning
+		new File(rootDir,"annotations").deleteDir();
+		new File(rootDir,"annotations").mkdir();
+		new File(rootDir,"treetagger").deleteDir();
+		new File(rootDir,"treetagger").mkdir();
+		
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		
+		List<File> files = new File(rootDir,"txm").listFiles()	
+		//BUILD TT FILE READY TO BE TAGGED
+		for(File f : files)
+		{
+			File srcfile = f;
+			File resultfile = new File(rootDir+"treetagger/",f.getName()+".tt");
+			new BuildTTSrc(srcfile.toURL()).process(resultfile)
+		}
+		
+		//APPLY TREETAGGER
+		files = new File(rootDir,"treetagger").listFiles()	
+		for(File f : files)
+		{
+			File modelfile = new File(Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH),"/fr.par");
+			if(!modelfile.exists())
+			{
+				println "Skipping ANNOTATE: Incorrect modelfile path: "+modelfile;
+				return;
+			}
+			File infile = f
+			File outfile = new File(f.getParent(),f.getName()+"-out.tt");
+			println("3- APPLY TT on : "+infile+" with : "+modelfile +" >>  "+outfile);
+			
+			if(!new File(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/").exists())
+			{
+				println("Path to TreeTagger is wrong "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/")
+				return;
+			}
+			TreeTagger tt = new TreeTagger(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/");
+			tt.settoken();
+			tt.setlemma();
+			tt.setquiet();
+			tt.setsgml();
+			tt.setnounknown();
+			tt.seteostag("<s>");
+			tt.treetagger( modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())
+			infile.delete();
+		}
+		
+		//BUILD STAND-OFF FILES
+		//contains txm:application/txm:commandLine
+		File reportFile = new File(rootDir,"NLPToolsParameters.xml");
+		
+		String respPerson = System.getProperty("user.name");
+		String respId = "txm";
+		String respDesc = "NLP annotation tool";
+		String respDate = "";
+		String respWhen = ""
+		
+		String appIdent = "TreeTagger";
+		String appVersion = "3.2";
+		
+		String distributor = "";
+		String publiStmt = """""";
+		String sourceStmt = """""";
+		
+		def types = ["pos","lemme"];
+		def typesTITLE = ["",""];
+		def typesDesc = ["",""];
+		def typesTAGSET = ["",""];
+		def typesWEB = ["",""];
+		String idform ="w_c_";
+		
+		files = new File(rootDir,"treetagger").listFiles()	
+		for(File f : files)
+		{
+			String target = f.getAbsolutePath();
+			File ttfile = f
+			File posfile = new File(rootDir+"annotations/",f.getName()+"-STOFF.xml");
+			
+			def encoding ="UTF-8";
+			def transfo = new CSV2W_ANA();
+			println("build w-interp "+ttfile.getName()+ ">>"+posfile.getName())
+			transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+			transfo.setResp(respId, respDesc,respDate, respPerson, respWhen);
+			transfo.setApp(appIdent, appVersion);
+			transfo.setTarget(target, reportFile);
+			transfo.setInfos(distributor,  publiStmt, sourceStmt);
+			transfo.process( ttfile, posfile, encoding );
+		}
+		
+		files = new File(rootDir,"annotations").listFiles();
+		List<File> txmfiles = new File(rootDir,"txm").listFiles();
+		files.sort();
+		txmfiles.sort();
+		for(int i = 0 ; i< files.size();i++)
+		{
+			File srcfile = txmfiles.get(i);
+			
+			File pos1file = files.get(i);
+			File temp = new File(rootDir,"temp"); 
+
+			println("5- inject annotation in file : "+srcfile+" with : "+pos1file );
+			
+			def builder = new org.txm.scripts.teitxm.AnnotationInjection(srcfile.toURL(), pos1file.toURL(), milestones);
+			builder.transfomFile(temp.getParent(),temp.getName());
+			
+			if (!(srcfile.delete() && temp.renameTo(srcfile))) println "Warning can't rename file "+temp+" to "+srcfile
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File rootDir = new File("~/xml/perrault/");
+		new annotate().run(rootDir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/compiler.groovy	(revision 321)
@@ -0,0 +1,413 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.perrault
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler 
+{
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+	
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+	
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		createOutput(dirPathName, fileName);
+		
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+		
+		int p_id = 0;
+		int q_id = 0;
+		int lg_id = 0;
+		int l_id = 0;
+		
+		boolean flaglg = false;
+		boolean flaghead = false;
+		boolean flagAuthor = false;
+		boolean flagDate = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+		
+		this.GoToText()
+		output.write("<txmcorpus lang=\""+lang+"\">\n"); 
+		try 
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+						
+						case "head"://get attr lang
+							flaghead =true;
+							vHead="";
+						break;
+												
+						case "text":
+						output.write("<text id=\""+text+"\" base=\""+base+"\" project=\""+project+"\">\n");
+						break;
+						
+						case "p":
+						output.write("<"+parser.getLocalName()+" id=\""+text+"_p_"+(p_id++)+"\">\n");
+						break;
+						case "q":
+						output.write("<"+parser.getLocalName()+" id=\""+text+"_q_"+(q_id++)+"\">\n");
+						break;
+						case "l":
+							output.write("<"+parser.getLocalName()+" id=\""+text+"_l_"+(l_id++)+"\">\n");
+						break;
+						
+						case "lg":
+							flaglg = true;
+						break;
+						
+						case "s":
+						output.write( "<s>\n");
+						break;
+						
+						case "w":
+							for(int i = 0 ; i < parser.getAttributeCount(); i++)
+								if(parser.getAttributeLocalName(i).equals("id"))
+								{	
+									wordid = parser.getAttributeValue(i);
+								}
+						break;
+						case "form":
+						flagForm = true;
+						vForm = "";
+						vAna ="";
+						break;
+						
+						case "ana":
+						flagAna = true;
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.END_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+						case "head"://get attr lang
+						flaghead =false;
+						if(flaglg)
+							output.write("<moral id=\""+text+"_moral_"+(lg_id++)+"\" head=\""+vHead+"\">\n");
+						break;
+						
+						case "text":
+						output.write("</text>\n");
+						break;
+						
+						case "p":
+						case "q":
+						case "l":
+						output.write("</"+parser.getLocalName()+">\n");
+						break;
+						
+						case "lg":
+						output.write("</moral>\n");
+						flaglg = false;
+						break;
+						
+						case "s":
+						output.write( "</s>\n");
+						break;
+						
+						case "w":
+						if(!(flaghead && flaglg))
+							if(vAna != null)
+								output.write( vForm +vAna+"\t"+wordid+"\n");
+						vAna = "";
+						vForm = "";
+						break;
+						
+						case "form":
+						flagForm = false;
+						break;
+						
+						case "ana":
+						flagAna = false;
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.CHARACTERS:
+					if(flagForm)
+					vForm += parser.getText().trim();
+					if(flagAna)
+					vAna += "\t" +parser.getText().trim();
+					if(flaghead && flaglg)
+						vHead += parser.getText().trim();
+					break;
+				}
+			}
+			output.write("</txmcorpus>"); 
+			output.close();
+			parser.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+		
+		return true;
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param files the files
+	 * @return true, if successful
+	 */
+	public boolean run(List<File> files) 
+	{
+		String rootDir ="";
+		if(files.size() > 0)
+			rootDir = files.get(0).getParentFile().getParentFile().getAbsolutePath()+"/";//"~/xml/perrault/";
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!new File(rootDir).exists())
+		{
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+		new File(rootDir+"wtc/","perrault.wtc").delete();//cleaning&preparing
+		new File(rootDir+"wtc/").deleteDir();
+		new File(rootDir+"wtc/").mkdir();
+		new File(rootDir+"registry/").mkdir();
+		
+		String textid="";
+		int counttext =0;
+		//1- Transform into WTC file
+		for(File f : files)
+		{
+			counttext++;
+			if(!f.exists())
+			{
+				println("file "+f+ " does not exists")	
+			}
+			else
+			{	
+				println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				def builder = new compiler(f.toURL(), txtname, "perrault", "default");
+				builder.setLang(lang);
+				builder.transfomFileWtc(rootDir+"wtc","perrault.wtc");
+			}
+		}
+		
+		//2- Import into CWB
+		def outDir =rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		
+		String[] pAttributes = ["pos","lemme","id"];
+		//String[] pAttributes = ["id"];
+		String[] sAttributes = ["txmcorpus:0+lang","text:0+id+base+project","p:0+id","q:0+id","moral:0+head+id","l:0+id"];
+		
+		try
+		{
+			cwbEn.run(cwbLoc + "cwb-encode", outDirTxm + "data", outDir + "/wtc/"+"perrault.wtc", outDirTxm + "registry/"+"perrault",pAttributes, sAttributes);
+			cwbMa.run(cwbLoc + "cwb-makeall", "PERRAULT", outDirTxm + "registry");
+			
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		
+		System.out.println("Done.") 
+		
+		return true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/perrault/txm/");
+		List<File> files = dir.listFiles();
+		new compiler().run(files);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Perault import module : teiCorpus</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/perrault/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CsvToXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CsvToXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CsvToXml.groovy	(revision 321)
@@ -0,0 +1,185 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-12-17 12:11:39 +0100 (Thu, 17 Dec 2015) $
+// $LastChangedRevision: 3087 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer
+
+import java.io.FileInputStream
+import java.io.InputStreamReader
+import java.nio.charset.Charset
+import org.txm.utils.*
+
+import javax.xml.stream.*
+import java.net.URL
+
+/**
+ * The Class CsvToXml: allow to convert CSV files to XML files
+ *
+ * @author mdecorde
+ */
+class CsvToXml {
+
+	/** The titles. */
+	File xmlfile
+	List<String> titles = []
+
+	private CsvReader getCSVReader(File csvfile, String fieldSeparator, String encoding) {
+		CsvReader csvreader = new CsvReader(csvfile.getAbsolutePath(), fieldSeparator.charAt(0), Charset.forName(encoding))
+
+		//get titles
+		csvreader.readHeaders()
+		titles = []
+		for (String title : csvreader.getHeaders()) {
+			titles << title
+		}
+
+		if (titles.size() == 0) {
+			println "the CSV file as no column"
+			return null
+		}
+
+		if (titles.contains("text") == 0) {
+			println "the CSV file as no 'text' column"
+			return null
+		}
+
+		if (titles.contains("id") == 0) {
+			println "the CSV file as no 'id' column"
+			return null
+		}
+		
+		return csvreader
+	}
+	
+	/**
+	 * Convert a CSV file to ONE XML file using the first line to declare metadata
+	 *
+	 * @param csvfile the csvfile
+	 * @param outfile the outfile
+	 * @param fieldSeparator the field separator
+	 * @param textSeparator the text separator
+	 * @param encoding the encoding
+	 * @return true, if successful
+	 */
+	public boolean toOneXMLFile(File csvfile, File xmlfile, String fieldSeparator, String textSeparator, String encoding) {
+		CsvReader csvreader = getCSVReader(csvfile, fieldSeparator, encoding)
+		if (csvreader == null) return false
+		
+		println "Metadata properties declared: "+titles
+
+		XMLOutputFactory factory = XMLOutputFactory.newInstance()
+		FileOutputStream output = new FileOutputStream(xmlfile)
+		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8")
+
+		writer.writeStartDocument("UTF-8","1.0")
+		writer.writeStartElement("text")
+		while (csvreader.readRecord()) {
+			
+			writer.writeStartElement("div")
+
+			for (int i = 0 ; i < titles.size() ; i++) {
+				if (titles[i] == "text") continue
+				writer.writeAttribute(AsciiUtils.buildId(titles.get(i)), csvreader.get(i))
+			}
+
+			writer.writeCharacters(csvreader.get("text")) // get text content
+			writer.writeEndElement() // div
+		}
+
+		writer.writeEndElement() // text
+		writer.close()
+		output.close()
+		csvreader.close()
+
+		return true;
+	}
+	
+	/**
+	* Convert a CSV file to SEVERAL XML file using the first line to declare metadata
+	*
+	* @param csvfile the csvfile
+	* @param outfile the outfile
+	* @param fieldSeparator the field separator
+	* @param textSeparator the text separator
+	* @param encoding the encoding
+	* @return true, if successful
+	*/
+	public boolean toMultipleXMLFiles(File csvfile, File outDir, String fieldSeparator, String textSeparator, String encoding) {
+		CsvReader csvreader = getCSVReader(csvfile, fieldSeparator, encoding)
+		if (csvreader == null) return false
+		
+		if (!outDir.exists()) outDir.mkdirs()
+		if (!outDir.exists()) {
+			println "Out directory does not exist and could not create it"
+			return false;
+		}
+		
+		while (csvreader.readRecord()) {
+			
+			File xmlfile = new File(outDir, csvreader.get("id")+".xml")
+			
+			XMLOutputFactory factory = XMLOutputFactory.newInstance()
+			FileOutputStream output = new FileOutputStream(xmlfile)
+			XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8")
+	
+			writer.writeStartDocument("UTF-8","1.0")
+			writer.writeStartElement("text")
+			
+			for (int i = 0 ; i < titles.size() ; i++) {
+				if (titles[i] == "text") continue
+				writer.writeAttribute(AsciiUtils.buildId(titles.get(i)), csvreader.get(i))
+			}
+
+			writer.writeCharacters(csvreader.get("text")) // get text content
+			
+			writer.writeEndElement() //text
+			writer.close()
+			output.close()
+		}
+		return true
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	static public void main(String[] args)
+	{
+		String home = System.getProperty("user.home")
+		File csvfile = new File(home, "xml/csv/test.csv")
+		File outfile = new File(home, "xml/csv/test.xml")
+		File outdir = new File(home, "xml/csv/out")
+
+		String fieldSeparator = "\t"
+		String textSeparator = "" //or "'"
+		CsvToXml builder = new CsvToXml()
+		
+		builder.toMultipleXMLFiles(csvfile, outdir, fieldSeparator, textSeparator, "UTF-8")
+		
+		builder.toOneXMLFile(csvfile, outfile, fieldSeparator, textSeparator, "UTF-8")
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/PartialDomIdentityParser.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/PartialDomIdentityParser.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/PartialDomIdentityParser.groovy	(revision 321)
@@ -0,0 +1,179 @@
+package org.txm.importer
+
+import java.util.ArrayList;
+
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamWriter;
+
+import java.io.File;
+import java.io.Writer;
+import java.net.URL;
+
+import javax.xml.stream.*;
+
+class PartialDomIdentityParser extends StaxIdentityParser {
+	
+	boolean domMode = false;
+	def partialDom = [];
+	def currentNode;
+	
+	public PartialDomIdentityParser(File infile) {
+		super(infile)
+	}
+	
+	protected void processNamespace() {
+		super.processNamespace()
+	}
+	
+	protected void processStartElement()
+	{
+		if (parser.getLocalName() == "test") {
+			println "Build dom..."
+			Tag testTag = new Tag(parser);
+			println testTag.toString()
+			testTag.writeDom(writer)
+		} else {
+			super.processStartElement()
+		}
+	}
+	
+	protected void writeAttributes() {
+		super.writeAttributes();
+	}
+	
+	protected void processCharacters()
+	{
+		super.processCharacters()
+	}
+	
+	protected void processProcessingInstruction()
+	{
+		super.processProcessingInstruction()
+	}
+	
+	protected void processDTD()
+	{
+		super.processDTD()
+	}
+	
+	protected void processCDATA()
+	{
+		super.processCDATA();
+	}
+	
+	protected void processComment()
+	{
+		super.processComment()
+	}
+	
+	protected void processEndElement()
+	{
+		super.processEndElement();
+	}
+	
+	protected void processEndDocument() {
+		super.processEndDocument()
+	}
+	
+	protected void processEntityReference() {
+		super.processEntityReference()
+	}
+	
+	public class Tag {
+		public String[] attnames, attvalues, attprefix;
+		public String localname, prefix
+		int count;
+		public def children = [];
+
+		public Tag(def parser) {
+			prefix = parser.getPrefix()
+			localname = parser.getLocalName();
+			count = parser.getAttributeCount()
+			attnames =  new String[count]
+			attvalues = new String[count]
+			attprefix = new String[count]
+			for (int i = 0 ; i < count ; i++) {
+				attnames[i] = parser.getAttributeLocalName(i)
+				attprefix[i] = parser.getAttributePrefix(i).toString()
+				attvalues[i] = parser.getAttributeValue(i).toString()
+			}
+			buildNodeContent(parser)
+		}
+		
+		private void buildNodeContent(def parser) {
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+					//println "event "+event
+					switch (event) {
+						case XMLStreamConstants.START_ELEMENT:
+							children << new Tag(parser);
+							break;
+						case XMLStreamConstants.CHARACTERS:
+							children << parser.getText()
+							break;
+						case XMLStreamConstants.END_ELEMENT:
+							if (localname == parser.getLocalName()) {
+								return; // end of stax parsing, the Tag is built
+							}
+							break;
+						case XMLStreamConstants.END_DOCUMENT:
+							println "ERROR: END OF DOCUMENT REACHED"
+							break;
+					}
+				}
+			} catch(Exception e) {
+				println("Error while parsing partial content of file "+inputurl);
+				println("Location "+parser.getLocation());
+				org.txm.utils.logger.Log.printStackTrace(e);
+				return;
+			}
+		}
+
+		public String toString() {
+			def out = new StringWriter()
+			XMLStreamWriter strwriter = outfactory.createXMLStreamWriter(out);//create a new file
+			println ""+(strwriter != null)
+			writeDom(strwriter);
+			return out.toString()
+		}
+		
+		public writeDom(def writer) {
+			// Open the Tag
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname)
+			else
+				writer.writeStartElement(localname)
+
+			// write Attributes
+			for (int i = 0 ; i < count ; i++) {
+				if (attprefix[i] != null && attprefix[i].length() > 0) {
+					writer.writeAttribute(attprefix[i]+":"+attnames[i], attvalues[i])
+				} else {
+					writer.writeAttribute(attnames[i], attvalues[i])
+				}
+			}
+			
+			// write children recursivelly
+			for (def child : children) {
+				if (child instanceof String) {
+					writer.writeCharacters(child)
+				} else if (child instanceof Tag) {
+					child.writeDom(writer)
+				} else {
+					println "Error: can't write $child"
+				}
+			}
+			
+			// close the Tag
+			writer.writeEndElement() 
+		}
+	}
+	
+	public static void main(String[] args) {
+		File infile = new File("/home/mdecorde/xml/partialdom/test.xml")
+		File outfile = new File("/home/mdecorde/xml/partialdom/test-o.xml")
+		PartialDomIdentityParser p = new PartialDomIdentityParser(infile);
+		println p.process(outfile)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WriteIdAndNAttributesUltraGeneric.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WriteIdAndNAttributesUltraGeneric.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WriteIdAndNAttributesUltraGeneric.groovy	(revision 321)
@@ -0,0 +1,141 @@
+package org.txm.importer
+
+class WriteIdAndNAttributesUltraGeneric extends StaxIdentityParser {
+
+	String baseID = ""
+	def rules = [];
+
+	def idCounters = [:]
+	def nCounters = [:]
+	def parentIds = [:]
+	
+	def elements;
+	def elementShortNames
+	def elementParents
+	def elementResets
+	def elementSelection
+	public WriteIdAndNAttributesUltraGeneric(File xmlFile, def elements, def elementSelection, def elementShortNames, def elementParents, def elementResets) {
+		super(xmlFile)
+		
+		this.elements = elements;
+		this.elementSelection = elementSelection
+		this.elementShortNames = elementShortNames;
+		this.elementParents = elementParents
+		this.elementResets = elementResets
+		
+		this.baseID = xmlFile.getName();
+		if (baseID.indexOf(".") > 0) baseID = baseID.substring(0, baseID.indexOf("."))
+		
+		for (String element : elements) {
+			idCounters[element] = 1;
+			nCounters[element] = 1;
+			parentIds[element] = null;
+		}
+	}
+
+	/**
+	 * Special rule to add @corresp=#ref to its reference element to element which @attribute=@value
+	 */
+	public setReferenceRule(String element, String attribute, String value) {
+		this.rules << [element, attribute, value]
+	}
+	
+	protected void writeAttributes() {
+		if (!idCounters.containsKey(localname) ) {
+			super.writeAttributes();
+			return;
+		} 
+		println "write attributes $localname"
+		boolean idFound = false
+		boolean nFound = false
+		
+		
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			String name = parser.getAttributeLocalName(i);
+			String value = parser.getAttributeValue(i);
+			
+			println " at $name=$value"
+			
+			//String element_sign = [localname, name, value]
+			
+			if ("id" == name) {
+				if (value == null) {
+					value = getNextID()
+				} 
+				
+				parentIds[localname] = value
+			} else if ("n" == name) {
+				if (value == null) {
+					value = getNextN()
+				} else {
+					nCounters[localname] = Integer.parseInt(value)
+				}
+			}
+			
+			String attrPrefix = parser.getAttributePrefix(i);
+			if (attrPrefix != null && attrPrefix.length() > 0)
+				writer.writeAttribute(attrPrefix+":"+name, value);
+			else
+				writer.writeAttribute(name, value);
+		}
+		
+		if (!idFound) {
+			String value = getNextID()
+			writer.writeAttribute("id", value);
+			parentIds[localname] = value
+		}
+		if (!nFound) {
+			String value = getNextN()
+			writer.writeAttribute("n", value);
+		}
+		
+		
+		idCounters[localname]++
+	}
+	
+	protected void processStartElement() {
+		
+		if (idCounters.containsKey(localname)) {
+			println "Element: "+localname
+			for (String resetElement : elementResets[localname]) {
+				println "reset N: "+resetElement
+				nCounters[resetElement] = 1
+			}
+		}
+		
+		super.processStartElement();
+	}
+	
+	protected String getNextID() {
+		String value = elementShortNames[localname]
+		
+		String parent = elementParents[localname]
+		if (parent != null)
+			value += "_"+parentIds[parent]
+		else
+			value += "_"+baseID
+			
+		value += "-"+idCounters[localname]
+		
+		return value
+	}
+	
+	protected String getNextN() {
+		String value = Integer.toString(nCounters[localname])
+		nCounters[localname]++
+		return value
+	}
+	
+	public static void main(String[] args) {
+		File xmlFile = new File("/home/mdecorde/TEMP/idnmissing.xml")
+		File outFile = new File("/home/mdecorde/TEMP/idnmissing-fix.xml")
+		
+		WriteIdAndNAttributesUltraGeneric wiana = new WriteIdAndNAttributesUltraGeneric(xmlFile, 
+			["milestone", "pb", "cb", "lb"],
+			["milestone":"unit=surface", "pb":null, "cb":null, "lb":null],
+			["milestone":"surf", "pb":"page", "cb":"col", "lb":"line"], 
+			["milestone":null, "pb":null, "cb":"pb", "lb":"cb"],
+			["milestone":[], "pb":["cb", "lb"], "cb":[ "lb"], "lb":[]])
+		println wiana.process(outFile)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SplitBy.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SplitBy.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SplitBy.groovy	(revision 321)
@@ -0,0 +1,204 @@
+package org.txm.importer
+
+import java.io.Writer;
+import java.io.File;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+
+class SplitBy 
+{
+	
+	XMLOutputFactory factory = XMLOutputFactory.newInstance();
+	FileOutputStream output;
+	XMLStreamWriter writer;
+	def parser;
+	ArrayList<String> newfiles = [];
+	
+	public SplitBy(File xmlfile)
+	{
+		try {
+			def inputData = xmlfile.toURI().toURL().openStream();
+			def inputFactory = XMLInputFactory.newInstance();
+			parser = inputFactory.createXMLStreamReader(inputData);
+			
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	boolean inby = false;
+	public boolean process(File outdir, String by, String idAttribute)
+	{
+		outdir.mkdirs();
+		try
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				
+				switch(event)
+				{
+					case XMLStreamConstants.START_ELEMENT:
+						if(parser.getLocalName() == by)
+						{
+							
+							inby = true;
+							if(writer != null)
+							{
+								writer.close();
+								output.close();
+							}
+							
+							String filename = "";
+							for(int i = 0 ; i < parser.getAttributeCount(); i++)
+								if(parser.getAttributeLocalName(i) == idAttribute)
+								{
+									filename = parser.getAttributeValue(i);
+									break;
+								}
+							File newfile = new File(outdir,filename+".xml")
+							if(files.contains(newfile))
+							{
+								println "Item declared twice "+filename+" at location "+parser.getLocation();
+								return false;
+							}
+							files.add(newfile);
+							output = new FileOutputStream(newfile)
+							writer = factory.createXMLStreamWriter(output, "UTF-8");//create a new file
+							
+						}
+						if(writer != null)
+						{
+							writeEvent(event);
+							if(parser.getLocalName() == "TEI")
+							{ // write namespaces
+								writer.writeNamespace("tei", "http://www.tei-c.org/ns/1.0")
+								writer.writeNamespace("txm", "http://textometrie.org/1.0")
+							}
+						}
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						if(inby)
+						{
+							if(writer != null)
+							{
+								writer.writeEndElement();
+							}
+						}
+						if(parser.getLocalName() == by)
+						{
+							inby = false; // write the end element $by before
+						}
+						break;
+					case XMLStreamConstants.CHARACTERS:
+						if(writer != null)
+						{
+							writer.writeCharacters(parser.getText());
+						}
+						break;
+				}
+				
+			}
+			if(writer != null)
+				writer.close();
+			if(output != null)
+				output.close();
+			return true;
+		}
+		catch(Exception e){
+			println "XML Error at location: "+parser.getLocation()
+			e.printStackTrace()
+			if(writer != null)
+				writer.close()
+				output.close()
+				return false
+			}
+	}
+	
+	/**
+	 * write the current event.
+	 *
+	 * @param event the stax event
+	 */
+	private void writeEvent(int event)
+	{
+		def prefix = parser.getPrefix();
+		if (event == XMLStreamConstants.START_ELEMENT ) 
+		{
+			def localname = parser.getLocalName(); // write element
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname);
+			else
+				writer.writeStartElement(localname);
+			
+			for(int i = 0 ; i < parser.getNamespaceCount() ; i++) //write namespaces
+			{
+				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+			}
+			
+			String attrprefix; // write attributes
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++)
+			{
+				attrprefix = parser.getAttributePrefix(i);
+				if (attrprefix != null & attrprefix.length() > 0)
+					writer.writeAttribute(attrprefix+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+				else
+					writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+			}
+			//writer.writeCharacters("\n");
+		}
+		else if (event == XMLStreamConstants.END_ELEMENT)
+		{
+			writer.writeEndElement();
+			//writer.writeCharacters("\n");
+		}
+		else if (event == XMLStreamConstants.CHARACTERS)
+		{
+			writer.writeCharacters(parser.getText());
+		}
+		else if (event == XMLStreamConstants.COMMENT)
+		{
+			writer.writeComment(parser.getText());
+		}
+	}
+	
+	public ArrayList<File> getFiles()
+	{
+		return newfiles;
+	}
+	
+	public static void main(String[] args)
+	{
+		File xmlfile = new File("C:\\Documents and Settings\\mdecorde\\Bureau\\MISAT2011\\corpus\\tiempo\\ELTIEMPO.2002-2010.MIN.xml");
+		File okfile = new File("C:\\Documents and Settings\\mdecorde\\Bureau\\MISAT2011\\corpus\\tiempo\\");
+		File outdir = new File("C:\\Documents and Settings\\mdecorde\\Bureau\\MISAT2011\\corpus\\tiempo\\tiempo");
+		outdir.deleteDir();
+		outdir.mkdir();
+		
+		//		Reader reader = new InputStreamReader(new FileInputStream(xmlfile) , "ISO-8859-1");
+		//		Writer writer = new OutputStreamWriter(new FileOutputStream(okfile) , "UTF-8");
+		//		String line = reader.readLine();
+		//		while(line != null)
+		//		{
+		//			line = line.replaceAll(' ([a-zA-Z0-9]+)="(.*)""', ' $1="$2"')
+		//			writer.write(line+"\n");
+		//			line = reader.readLine();
+		//		}
+		//		reader.close();
+		//		writer.close();
+		//		
+		//		
+		////		if(!ValidateXml.test(okfile))
+		////		{
+		////			println "XML not valid";
+		////			return;
+		////		}
+		
+		def splitter = new SplitBy(xmlfile);
+		if(splitter.process(outdir, "texte", "id"))
+			println "success";
+		else
+			println "failed !"
+		
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RemoveTag.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RemoveTag.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RemoveTag.groovy	(revision 321)
@@ -0,0 +1,119 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer;
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.transform.dom.DOMSource
+import javax.xml.transform.stream.StreamResult
+import javax.xml.xpath.*
+
+import org.txm.metadatas.*
+import org.txm.utils.*
+import org.w3c.dom.Document
+import org.w3c.dom.Element
+
+/**
+ * Removes tags of XML file given a XPath. 
+ * @author mdecorde
+ *
+ */
+public class RemoveTag {
+	File outfile;
+	
+	/** The doc. */
+	Document doc;
+	
+	/**
+	 *
+	 * @param xmlfile the xmlfile
+	 * @param outfile the outfile
+	 * @param xpath the XPath
+	 */
+	public RemoveTag(File xmlfile, File outfile, String xpath)
+	{
+		this.outfile = outfile;
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		doc = builder.parse(xmlfile);
+		
+		def expr = XPathFactory.newInstance().newXPath().compile(xpath);
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+		
+		if (nodes != null)
+		for(def node : nodes)
+		{
+			//println "Remove node "+node
+			Element elem = (Element)node;
+			elem.getParentNode().removeChild(node);
+		}
+		save()
+		doc = null
+	}
+	
+	/**
+	 * Save.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean save()
+	{
+		try {
+			// Création de la source DOM
+			Source source = new DOMSource(doc);
+			
+			// Création du fichier de sortie
+				Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8")); 
+			Result resultat = new StreamResult(writer);
+			
+			// Configuration du transformer
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			Transformer transformer = fabrique.newTransformer();
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes"); 
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); 
+			
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			return true;
+		} catch (Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			return false;
+		}
+	}
+	
+	public static void main(String[] args) {
+		RemoveTag rt = new RemoveTag(
+			new File("/home/mdecorde/TXM/corpora/graal/import.xml"),
+			new File("/home/mdecorde/TXM/corpora/graal/import-o.xml"),
+			"//edition[@name='courante']"
+			)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Pager.groovy	(revision 321)
@@ -0,0 +1,152 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer
+import java.util.regex.Pattern
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class Pager.
+ */
+class Pager {
+	
+	//Prototype of generic pager
+	
+	//cut infos
+	
+	/** The cuttag. */
+	String cuttag = "w";// tag to cut, default <w>
+	
+	/** The cutattr. */
+	String cutattr;//attr of cuttag to test
+	
+	/** The cutvaluereg. */
+	Pattern cutvaluereg;// regex value
+	
+	/** The cutnumber. */
+	int cutnumber = 500;// number of cuttag before cutting, default 500 words
+	
+	//xpaths
+	
+	/** The xpaths. */
+	HashMap<String, String> xpaths = new HashMap<String, String>();
+	
+	//tag classes
+	
+	/** The ptags. */
+	public List<String> ptags = ["p", "lg", ];
+	
+	/** The linetags. */
+	public List<String> linetags = ["br", "lb", "l"];
+	
+	/** The dbllinetags. */
+	public List<String> dbllinetags = [];
+	
+	/** The commenttags. */
+	public List<String> commenttags = ["note"];
+	
+	/** The corrtags. */
+	public List<String> corrtags = ["corr"];
+	
+	/** The regtags. */
+	public List<String> regtags = ["reg"];
+	
+	/** The foreigntags. */
+	public List<String> foreigntags = ["foreign"];
+	
+	/**
+	 * Sets the cut infos.
+	 *
+	 * @param tag the tag
+	 * @param attr the attr
+	 * @param valuereg the valuereg
+	 * @param number the number
+	 */
+	public void setCutInfos(String tag, String attr, String valuereg, int number)
+	{
+		if(tag != null && tag.length() > 0)
+			cuttag = tag;
+		
+		try
+		{
+			Pattern p = Pattern.compile(valuereg);
+			cutvaluereg = p;
+		}
+		catch(Exception e){System.err.println("Pager: setCutInfos: Wrong regex syntax");}
+		
+		if(cutattr != null && cutattr.length() > 0)
+			cutattr = attr;
+		
+		if(cutnumber > 0)	
+			cutnumber = number;
+	}
+	
+	/**
+	 * Sets the x paths.
+	 *
+	 * @param xpaths the xpaths
+	 */
+	public void setXPaths(HashMap<String, String> xpaths)
+	{
+		if(xpaths != null)
+			this.xpaths = xpaths
+	}
+	
+	/**
+	 * Adds the x path.
+	 *
+	 * @param name the name
+	 * @param xpath the xpath
+	 */
+	public void addXPath(String name, String xpath)
+	{
+		if(xpath != null && name != null)
+			xpaths.put(name, xpath)
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param infile the infile
+	 * @return true, if successful
+	 */
+	public boolean run(File infile)
+	{
+		
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		Pager p = new Pager()
+		p.setCutInfos("w","type", "{[|]@^[|`[{`]@^pon.*", 600);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/EncodeTEIQuotes.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/EncodeTEIQuotes.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/EncodeTEIQuotes.groovy	(revision 321)
@@ -0,0 +1,457 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class EncodeTEIQuotes.
+ */
+public class EncodeTEIQuotes
+{
+	
+	/** The name. */
+	def name;
+	
+	/** The initsize. */
+	def initsize;
+	
+	/** The quote. */
+	String quote ="\"";
+	
+	/** The quote_type. */
+	String quote_type= "pon"
+	
+		/**
+		 * Gets the name.
+		 *
+		 * @param node the xml node
+		 * @return the name of the node
+		 */
+	String getName(def node)
+	{
+		def name = node.name();
+		try{name = name.getLocalPart()}catch(Exception e){return name;} // might throw an exception depending on the node class
+		return name;
+	}
+	
+	/**
+	 * method to know if the Node is an element.
+	 *
+	 * @param n the n
+	 * @return true, if is elem
+	 */
+	boolean isElem(def n)
+	{
+		try{ n.name(); return true; }
+		catch(MissingMethodException e){ return false;}
+	}
+	
+	/**
+	 * Contains open quote.
+	 *
+	 * @param selem the node to process
+	 * @return true if the node contains a tag <q> at the beginning
+	 */
+	boolean containsOpenQuote(Node selem)
+	{
+		def children = selem.children()
+		//println "OTS: "+selem.attributes()
+		for(int i = 0 ; i < children.size()-1 ; i++)//on part de la fin jusqu'au 2e w
+		{
+			def child = children.get(i)
+			if(isElem(child))
+			{
+				name = getName(child)
+				if(name == "w" && child.@type != quote_type)
+				{
+					//println "not opening Q"
+					return false;
+				}
+				else if(name == "w" && child.@type == quote_type && child.text().equals(quote))
+				{
+					//println "opening Q"
+					return true;
+				}
+			}
+		}
+		//println "not opening Q"
+		return false;
+	}
+	
+	/**
+	 * Contains close quote.
+	 *
+	 * @param selem the node to process
+	 * @return true if the node contains a tag <q> at the end
+	 */
+	boolean containsCloseQuote(Node selem)
+	{
+		if(selem.w.size() == 0)
+			return false
+		//println "CTS: "+selem.attributes()
+		def children = selem.children()
+		for(int i = children.size()-1 ; i > 0 ; i--)//on part de la fin jusqu'au 2e w
+		{
+			def child = children.get(i)
+			if(isElem(child))
+			{
+				name = getName(child)
+				if(name == "w" && child.@type != quote_type)
+				{
+					//println "not closing Q"
+					return false;
+				}
+				else if(name == "w" && child.@type == quote_type && child.text().equals(quote))
+				{
+					//println "closing Q"
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+	
+	/**
+	 * Next children are words.
+	 *
+	 * @param children the children
+	 * @param index the index
+	 * @return true, if successful
+	 */
+	def boolean nextChildrenAreWords(def children, def index)
+	{
+		//println "at "+children.get(index).attributes()+" test("+children.size()+") "+children
+		for(int i = index ; i < children.size() ; i++)
+		{
+			def nextC = children.get(i)
+			if(isElem(nextC) && getName(nextC) == "w" && nextC.@type != quote_type)
+				return true;
+		}
+		//println "no more words"
+		return false;
+	}
+	
+	/**
+	 * Process containers.
+	 *
+	 * @param root the root
+	 * @param containers the containers
+	 * @return the java.lang. object
+	 */
+	def processContainers(Node root, containers)
+	{
+		int countcontainer = 0;
+		int countinits = 0;
+		int countnews = 0;
+		int countinitquote = 0
+		int quotecount = 0;
+		boolean openq = false;
+		// cut sentences whit a wpon not first nor last
+		int initsizecontainer = root.children().size()
+		//println "process "+initsizecontainer+" elements"
+		for(int k = 0 ; k < initsizecontainer ; k++)
+		{
+			def node = root.children().get(0);
+			root.children().remove(0);
+			//println getName(node)
+			if(isElem(node) && containers.contains(getName(node)))
+			{
+				countcontainer++;
+				Node pelem = node;
+				openq = false;
+				//println "PAB================="
+				int initsize = pelem.children().size()
+				for(int j = 0 ; j < initsize ; j++) // pour chq fils de p
+				{
+					def subnode = pelem.children().get(0);
+					pelem.children().remove(0)
+					//println getName(subnode)
+					if(isElem(subnode) && getName(subnode) == "s") // si c un S
+					{
+						//println "\nS: "+subnode.attributes()
+						def selem = subnode;
+						def newS = pelem.appendNode(selem.name(), selem.attributes())
+						boolean isOpenningQuote = false;
+						if(containsOpenQuote(selem))
+						{
+							isOpenningQuote = true
+							if(openq)
+							{
+								//println "ERROR: unclosed quote before "+selem.@id
+							}
+							openq = false;
+							//println "force openq false"
+						}
+						int tempwponcount = 0
+						for(int i = 0 ; i < selem.children().size() ; i++) // append chacun de ses fils au S courant
+						{
+							def child = selem.children().get(i) // test si == wpon"
+							if(isElem(child) && getName(child) == "w" && child.@type == quote_type && child.text().equals(quote))
+							{
+								countinitquote++
+								//println "Q: "+child.attributes()
+								//println "openq "+openq
+								if(openq)
+								{
+									newS.children().add(child)
+									if(nextChildrenAreWords(selem.children(), i))
+									{                                                                             
+										//println "NEW S"
+										newS = pelem.appendNode(selem.name(), selem.attributes())
+									}
+									//println "set openq false"
+									openq = false;
+								}
+								else
+								{
+									if(tempwponcount != 0)// premier wpon"
+									{
+										//println "pas premier wpon''"
+										if(nextChildrenAreWords(selem.children(), i))// si y'a aut' chose deriere, new S
+										{
+											newS = pelem.appendNode(selem.name(), selem.attributes())
+											//println "NEW S"
+											//println "set openq true"
+											openq = true
+										}
+									}
+									else // tempwponcount == 0
+									{
+										if(!isOpenningQuote)
+										{
+											if(nextChildrenAreWords(selem.children(), i))// si y'a aut' chose deriere, new S
+											{
+												newS = pelem.appendNode(selem.name(), selem.attributes())
+											}
+										}
+										//println "set openq true"
+										openq = true
+									}
+									newS.children().add(child)
+								}
+								tempwponcount++;
+							}
+							else
+							{
+								newS.children().add(child)
+							}
+						}
+					}
+					else
+						pelem.children().add(subnode)
+				}
+			}
+			root.children().add(node)
+		}
+		// A partir d'ici on a que des phrases de la forme :
+		// s q wwww s
+		// s q wwww q s
+		// s wwww q s
+		//println "****"
+		//wrap sentences in q tags
+		initsize = root.children().size()
+		for(int j = 0 ; j < initsize ; j++)
+		{
+			def node = root.children().get(0);
+			root.children().remove(0)
+			if(isElem(node) && containers.contains(getName(node)))
+			{
+				Node pelem = node
+				//println "PPPPPPPP"
+				def newp = root.appendNode(getName(pelem), pelem.attributes());
+				//newp.attribute("test")
+				openq = false;
+				def children = pelem.children()
+				//println children
+				Node qelem;
+				for(Node subnode : children)
+				{
+					if(isElem(subnode) && (getName(subnode) == "s"))
+					{
+						countnews++;
+						//println "ELEM s"
+						Node selem = subnode;
+						if(containsOpenQuote(selem))
+						{
+							// println "openning wpon"
+							if(openq)
+							{
+								// println "ERROR: unclosed quote (found swpon)"
+								//on ferme, et ouvre un new
+								qelem.appendNode("note",[type:"auto"]).setValue("unclosed quote")
+								Map attrs = [:];
+								attrs.put("xml:id",""+(quotecount+1))
+								qelem = newp.appendNode("q", attrs)
+								quotecount++;
+								qelem.appendNode(getName(pelem), selem.attributes(), selem.value());
+								if(containsCloseQuote(selem)) // la phrase étaient bien balisée
+									openq = false;
+							}
+							else
+							{
+								// println "open q + append current s"
+								openq = true;
+								Map attrs = [:];
+								attrs.put("xml:id",""+(quotecount+1))
+								qelem = newp.appendNode("q", attrs)
+								quotecount++;
+								qelem.appendNode(getName(selem), selem.attributes(), selem.value());
+								if(containsCloseQuote(selem)) // la phrase étaient bien balisée
+									openq = false;
+							}
+						}
+						else if(containsCloseQuote(selem))
+						{
+							//println "closing wpon"
+							if(openq)
+							{
+								// println "close q + add dernier s du quote"
+								qelem.appendNode(getName(selem), selem.attributes(), selem.value());
+								openq = false;
+							}
+							else
+							{
+								//println "ERROR: unopened quote"
+								Map attrs = [:];
+								attrs.put("xml:id",""+(quotecount+1))
+								qelem = newp.appendNode("q", attrs)
+								qelem.appendNode(getName(selem), selem.attributes(), selem.value());
+								quotecount++;
+								qelem.appendNode("note",[type:"auto"]).setValue("unopened quote")
+							}
+						}
+						else // no wpon
+						{
+							// println "normal s"
+							if(openq)
+							{
+								// println "append to q"
+								qelem.appendNode(getName(selem), selem.attributes(), selem.value());
+							}
+							else
+							{
+								// println "append to p"
+								newp.appendNode(getName(selem), selem.attributes(), selem.value());
+							}
+						}
+					}
+					else
+					{
+						if(openq)
+						{
+							qelem.children().add(subnode)
+						}
+						else
+						{
+							newp.children().add(subnode)
+						}
+					}
+				}
+				if(openq)// un quote fermé a la fin du P !!
+				{
+					// println "ERROR: unmatched quote"
+					qelem.appendNode("note",[type:"auto"]).setValue("unmatched quote")
+				}
+				//root.children().add(newp)
+			}
+			else
+				root.children().add(node)
+		}
+		//count sentences
+		println "processed "+countcontainer+" "+containers
+		println "initial number of &quot; "+countinitquote;
+		println "created "+(countnews )+" init "+ countinits+" s"
+		println "create "+(quotecount)+" quotes elements"
+	}
+	
+	/**
+	 * Instantiates a new encode tei quotes.
+	 *
+	 * @param nodesToInspect the nodes to inspect
+	 * @param containers the containers
+	 * @param quote_value the quote_value
+	 * @param quote_type the quote_type
+	 */
+	public EncodeTEIQuotes(List<Node> nodesToInspect, containers, String quote_value, String quote_type)
+	{
+		this.quote= quote_value;
+		this.quote_type= quote_type;
+		println "process "+nodesToInspect.size()+" elements with containers "+containers
+		for(groovy.util.slurpersupport.Node root : nodesToInspect)//.body)
+		{
+			println(" root : "+root.children().size())
+			processContainers(root, containers)
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		List<String> containers = ["p","ab"]; // elements qui contiennent des <s>
+		
+		File infile = new File(System.getProperty("user.home"), "xml/quote/gormont.xml")
+		File outfile = new File(System.getProperty("user.home"), "xml/quote/gormont-q.xml")
+		
+		def doc = new XmlParser().parse (infile)
+		List<Node> nodesToInspect = doc.text.body
+		// nodesToInspect << doc.text.body.div // on en rajoute
+		
+		/*************************/
+		new org.txm.importer.EncodeTEIQuotes(nodesToInspect, containers, "\"", "pon");
+		
+		//copy the doc in "outfile" File  
+		String encoding = "UTF-8"
+		OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(outfile) , encoding);
+		writer.write("<?xml version=\"1.0\" encoding=\""+encoding+"\"?>\n")
+		def pwriter = new PrintWriter(writer, true)
+		XmlNodePrinter xmlwriter = new XmlNodePrinter(pwriter)
+		xmlwriter.setPreserveWhitespace(false)
+		xmlwriter.print(doc)
+		pwriter.close()
+		writer.close()
+		xmlwriter = null
+		//println "write output file "+outfile
+		
+		//update counts
+		if(outfile.exists())
+		{
+			String txtid = org.txm.importer.WordCounter.findTextId(infile, "s"); // retrouve l'id du text qui a été concaténé aux id des S originels (ex : s19_12 >> 19)
+			new org.txm.importer.WordCounter(outfile, "s", txtid);
+			new org.txm.importer.WordCounter(outfile, "q", txtid);
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/GetAttributeValue.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/GetAttributeValue.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/GetAttributeValue.groovy	(revision 321)
@@ -0,0 +1,47 @@
+package org.txm.importer
+
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.io.File;
+import javax.xml.stream.*;
+import java.net.URL;
+
+class GetAttributeValue {
+	public static String process(File xmlFile, String path, String attribute) {
+		String value = "N/A";
+		String currentpath = "";
+		boolean done = false;
+		
+		try {
+			URL url = xmlFile.toURI().toURL();
+			def inputData = url.openStream();
+			def factory = XMLInputFactory.newInstance();
+			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+			for (int event = parser.next(); !done && event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						currentpath += "/"+parser.getLocalName();// append the current tag to the current path
+						if (path == currentpath) {
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								if (parser.getAttributeName(i) == attribute) {
+									value = parser.getAttributeValue(i)
+									done = true;
+									break;
+								}
+							}
+						}
+						
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						currentpath = currentpath.substring(0,currentpath.length() -1 - parser.getLocalName().length()) // remove tag from the path
+						break;
+					
+				}
+			}
+			
+		}
+		catch(Exception e){println("File "+xmlFile+"\n"+e); return "N/A";}
+		return value;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java	(revision 321)
@@ -0,0 +1,257 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.logging.Level;
+
+import org.txm.Toolbox;
+import org.txm.core.preferences.TBXPreferences;
+import org.txm.functions.ProgressWatcher;
+import org.txm.importer.ImportKeys;
+import org.txm.objects.BaseParameters;
+import org.txm.utils.DeleteDir;
+import org.txm.utils.xml.DomUtils;
+import org.txm.utils.logger.Log;
+
+public class ImportModule {
+
+	protected BaseParameters importParameters;
+	
+	protected String corpusVersionProduced;
+
+	protected File sourceDirectory;
+	protected File binaryDirectory;
+
+	protected Importer importer;
+	protected Annotater annotater;
+	protected Compiler compiler;
+	protected Pager pager;
+
+	protected boolean isSuccessful = true;
+	protected String reason = "none";
+	protected boolean debug = false;
+	protected boolean multithread = false;
+	protected boolean updateCorpus = false;
+	protected String corpusName;
+
+	ProgressWatcher monitor;
+	
+	public void setMonitor(ProgressWatcher monitor) {
+		this.monitor = monitor;
+	}
+
+	
+	public boolean isMultiThread() {
+		return multithread;
+	}
+
+	public boolean isDebugging() {
+		return debug;
+	}
+
+	public ImportModule(File importParametersFile) {
+		try {
+			BaseParameters b = new BaseParameters(importParametersFile);
+			init(b);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	public ImportModule(BaseParameters p) {
+		init(p);
+	}
+	
+	public boolean isUpdatingCorpus() {
+		return updateCorpus;
+	}
+
+	protected void init(BaseParameters p) {
+		this.importParameters = p;
+		this.importParameters.load();
+		corpusName = importParameters.name;
+		//this.debug = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.DEBUG));
+		
+		if (Log.getLevel().intValue() < Level.WARNING.intValue()) {
+			debug = true;
+		}
+		this.multithread = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.MULTITHREAD));
+		this.updateCorpus = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.UPDATECORPUS));
+
+		
+		this.sourceDirectory = importParameters.paramFile.getParentFile();
+		this.binaryDirectory = new File(Toolbox.getParam(TBXPreferences.USER_TXM_HOME), "corpora/"+corpusName.toUpperCase());
+
+		if (!updateCorpus) { // clean directories only if it's a new import
+			DeleteDir.deleteDirectory(binaryDirectory);
+			binaryDirectory.mkdir();
+			
+			File txmDir = new File(binaryDirectory, "txm");
+			txmDir.mkdir();
+		}
+	}
+
+	public void start() throws InterruptedException {
+
+		binaryDirectory.mkdirs(); // ensure output exists
+		//System.out.println("ImportModule.start");
+		if (!updateCorpus) { // create XML-TXM files and annotate
+			//System.out.println("ImportModule.start: not updating");
+			if (importer != null) {
+				//System.out.println("ImportModule.start: importer: "+importer);
+				if (monitor != null) System.out.println("-- IMPORTER - Reading source files");
+				importer.process();
+				//importer.checkFiles();
+				isSuccessful = isSuccessful & importer.isSuccessFul();
+				if (!isSuccessful) {
+					System.out.println("Error while importing corpus during 'importer' step, reason="+importer.getReason());
+					return;
+				}
+			} else {
+				System.out.println("XML-TXM files already produced in "+new File(binaryDirectory, "txm/"+corpusName));
+			}
+
+			boolean annotate = "true".equals(importParameters.getCorpusElement().getAttribute("annotate"));
+			if (annotate && annotater != null) {
+				if (monitor != null) System.out.println("-- ANNOTATE - Running NLP tools");
+				annotater.process();
+				isSuccessful = isSuccessful & annotater.isSuccessFul();
+				if (!isSuccessful) {
+					System.out.println("Error while importing corpus during 'annotate' step, reason="+annotater.getReason());
+					return;
+				}
+			} else {
+				//System.out.println("XML-TXM files already annotated.");
+			}
+		} else {
+			System.out.println("Updating corpus...");
+		}
+		
+		//System.out.println("GET FILES ORDER");
+		final ArrayList<File> files = getTXMFilesOrder();
+		
+		Thread Tcompiler = new Thread() {
+			public void run() {
+				if (compiler != null) {
+					if (monitor != null) System.out.println("-- COMPILING - Building Search Engine indexes");
+					compiler.process(files);
+					isSuccessful = isSuccessful & compiler.isSuccessFul();
+					if (!isSuccessful) {
+						System.out.println("Error while importing corpus during 'compiler' step, reason="+compiler.getReason());
+						return;
+					}
+				} else {
+					System.out.println("No CQP index created.");
+				}
+			}
+		};
+
+		Thread Tpager = new Thread() {
+			public void run() {
+
+				if (pager != null) {
+					if (monitor != null) System.out.println("-- EDITION - Building edition");
+					pager.process(files);
+					isSuccessful = isSuccessful & pager.isSuccessFul();
+					if (!isSuccessful) {
+						System.out.println("Error while importing corpus during 'pager' step, reason="+pager.getReason());
+						return;
+					}
+				} else {
+					System.out.println("No edition produced.");
+				}
+			}
+		};
+
+		Tcompiler.start();
+		if (!multithread) {
+			Tcompiler.join(); // wait for the end if not multithreaded
+			if (!isSuccessful) { // don't call pager is compiler step failed
+				return;
+			}
+		}
+		
+		Tpager.start();
+		if (multithread) Tcompiler.join(); // wait for both thread to end
+		Tpager.join();		
+	}
+	
+	protected ArrayList<File> getTXMFilesOrder() {
+		//System.out.println("DEFAULT FILES ORDER");
+		File txmDirectory = new File(binaryDirectory, "txm/"+corpusName);
+		ArrayList<File> files = new ArrayList<File>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {
+			@Override
+			public boolean accept(File file) {
+				return file.isFile() && file.getName().endsWith(".xml");
+			}
+		})));
+		
+		Collections.sort(files);
+		
+		return files;
+	}
+
+
+	public void end() {
+		File paramFile = new File(binaryDirectory, "import.xml");
+		try {
+			DomUtils.save(importParameters.root.getOwnerDocument(), paramFile);
+			isSuccessful = true;
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			isSuccessful = false;
+		}
+	}
+	
+	public String getCorpusName() {
+		return corpusName;
+	}
+
+	public String getReason() {
+		return reason;
+	}
+
+	public boolean isSuccessFul() {
+		return isSuccessful;
+	}
+
+	public BaseParameters getParameters() {
+		return importParameters;
+	}
+
+	public File getSourceDirectory() {
+		return sourceDirectory;
+	}
+
+	public File getBinaryDirectory() {
+		return binaryDirectory;
+	}
+	
+	public void process() throws InterruptedException {
+		start();
+		if (isSuccessful)
+			end();
+	}
+
+	public static void main(String[] args) {
+		File importParametersFile = new File("/home/mdecorde/xml/brown/import.xml");
+
+		ImportModule module = new ImportModule(importParametersFile);
+		System.out.println("Parameters: "+module.getParameters());
+		try {
+			module.start();
+
+			if (module.isSuccessful) {
+				System.out.println("Import sucessful. reloading corpora...");
+			} else {
+				System.out.println("Import failed, reason = "+module.getReason());
+			}
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZCompilerStep.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZCompilerStep.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZCompilerStep.groovy	(revision 321)
@@ -0,0 +1,267 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import javax.xml.stream.*
+import java.net.URL
+import java.util.HashMap;
+import org.txm.utils.*
+
+public class XTZCompilerStep {
+
+	static String FORM = "form";
+	static String ANA = "ana";
+	static String ID = "id";
+	static String TYPE = "type";
+	static String TAB = "\t";
+	static String QUOTE = "\"";
+
+	File xmlFile
+	File wtcFile
+	String textname, corpusname, projectname;
+	boolean normalizeAttributeValues = false;
+	boolean normalizeAnaValues = true;
+	boolean normalizeFormValues = true;
+
+	def inputData;
+	XMLInputFactory factory;
+	XMLStreamReader parser;
+	OutputStreamWriter output;
+
+	def anavalues = [:];
+	def anatypes;
+
+	String WTAG = "w"
+
+	public void setNormalizeAttributeValues(boolean n) {
+		this.normalizeAttributeValues = n;
+	}
+
+	public void setNormalizeAnaValues(boolean n) {
+		this.normalizeAnaValues = n;
+	}
+
+	public void setNormalizeFormValues(boolean n) {
+		this.normalizeFormValues = n;
+	}
+
+	public XTZCompilerStep(File xmlFile, File wtcFile, String textname, String corpusname, String projectname, def anatypes, def wtag) {
+		this.xmlFile = xmlFile;
+		this.wtcFile = wtcFile;
+		this.textname = textname
+		this.corpusname = corpusname;
+		this.projectname = projectname;
+		this.anatypes = anatypes;
+		this.WTAG = wtag
+
+		try {
+			inputData = xmlFile.toURI().toURL().openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+
+		} catch (Exception ex) {
+			System.err.println("Exception while parsing $xmlFile : "+ex);
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f) {
+		try {
+			output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f)) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			return false;
+		}
+	}
+
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean process()
+	{
+		if (!createOutput(wtcFile))
+			return false;
+
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+
+
+		int p_id = 0;
+		int s_id = 0;
+
+		def divs = []
+		def ncounts = [:] // contains the n values per tags with no attribute
+
+		boolean captureword = false;
+		boolean flagWord = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+
+		String anatype = "";
+		String anavalue = "";
+		boolean stopAtFirstSort = true;
+		boolean foundtei = false;
+		boolean foundtext = false;
+
+		try {
+			String localname;
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName().toLowerCase();
+						if ("tei".equals(localname)) foundtei = true;
+						switch (localname) {
+							case "text":
+								foundtext = true;
+								output.write("<text id=\""+textname+"\" base=\""+corpusname+QUOTE + " project=\""+projectname+QUOTE);
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									String attrname = parser.getAttributeLocalName(i);
+									String attrvalue = parser.getAttributeValue(i)
+
+									if (normalizeAttributeValues)
+										attrvalue = attrvalue.trim();
+
+									if (attrname != ID)
+										output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+QUOTE)
+								}
+								output.write(">\n");
+
+								break;
+
+							case WTAG:
+								for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+									if (parser.getAttributeLocalName(i).equals(ID)) {
+										wordid = parser.getAttributeValue(i);
+									}
+								}
+								anavalues = [:];
+								flagWord = true
+								break;
+							case FORM:
+								flagForm = true;
+								vForm = "";
+								vAna = "";
+								break;
+
+							case ANA:
+								flagAna = true;
+								anavalue = "";
+								for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+									//println parser.getAttributeLocalName(i)+"="+parser.getAttributeValue(i)
+									if (TYPE.equals(parser.getAttributeLocalName(i))) {
+										anatype = parser.getAttributeValue(i).substring(1);//remove the #
+										break;
+									}
+								}
+								break;
+
+							default:
+								if (foundtei && !foundtext) break;
+
+								output.write("<"+localname);
+
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									String attrname = parser.getAttributeLocalName(i);
+
+									String attrvalue = parser.getAttributeValue(i)
+									if (normalizeAttributeValues)
+										attrvalue = attrvalue.trim();
+
+									output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+QUOTE)
+								}
+								if (parser.getAttributeCount() == 0) { // add the n attribute
+									if (!ncounts.containsKey(localname)) ncounts.put(localname, 0);
+									int ncount = ncounts.get(localname);
+									ncounts.put(localname, ncount+1);
+									output.write(" n=\""+ncount+QUOTE)
+								}
+								output.write(">\n");
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName().toLowerCase();
+						switch (localname) {
+							case WTAG:
+								for (String type : anatypes) {
+									def v = anavalues.get(type);
+									if (v != null) vAna +=TAB+v;
+									else vAna +=TAB;
+								}
+								vForm = vForm.replaceAll("\n", "").replaceAll("&", "&amp;").replaceAll("<", "&lt;");
+								if (vAna != null) {
+									output.write(vForm+TAB+wordid+vAna+"\n");
+								}
+								vAna = "";
+								vForm = "";
+								flagWord = false;
+								break;
+
+							case "tei":
+								break;
+							case FORM:
+								flagForm = false;
+								break;
+							case ANA:
+								anavalues.put(anatype, anavalue)
+								flagAna = false;
+								break;
+							default:
+								if (foundtei && !foundtext) break;
+
+								output.write("</"+localname+">\n");
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						if (flagWord) {
+							if (flagForm) {
+								if (normalizeFormValues) {
+									vForm += parser.getText().trim();
+								} else {
+									vForm += parser.getText();
+								}
+							}
+							if (flagAna) {
+								if (normalizeAnaValues)
+									anavalue += parser.getText().trim();
+								else
+									anavalue += parser.getText();
+							}
+						}
+						break;
+				}
+			}
+
+			output.close();
+			parser.close();
+		} catch (Exception ex) {
+			System.out.println("Exception while parsing " + inputData+" of Text "+textname+" : "+ex);
+			File errorDir = null
+			try {
+				errorDir = new File(wtcFile.getParentFile(), "compiler-error")
+				println "Warning: Moving $xmlFile to $errorDir"
+				errorDir.mkdir();
+				FileCopy.copy(xmlFile, new File(errorDir, xmlFile.getName()))
+			} catch(Exception eCopy) {
+				println "Error while moving "+xmlFile+" to "+errorDir+" : "+eCopy
+			}
+			return false;
+		}
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/xtzLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/xtzLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/xtzLoader.groovy	(revision 321)
@@ -0,0 +1,75 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-06-03 15:04:53 +0200 (mer., 03 juin 2015) $
+// $LastChangedRevision: 2984 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.xtz;
+
+import javax.xml.stream.*
+
+import org.txm.*
+import org.txm.metadatas.*
+import org.txm.objects.*
+import org.txm.scripts.teitxm.*
+import org.txm.tokenizer.TokenizerClasses
+import org.txm.utils.*
+import org.txm.utils.i18n.*
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/srcmfmadridxmltxm/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		TokenizerClasses.loadFromNode(params.getTokenizerElement(params.getCorpusElement()));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+
+if (params == null) { println "no parameters. Aborting"; return; }
+
+//params.getKeyValueParameters().put(ImportKeys.CLEAN, "false")
+//params.getKeyValueParameters().put(ImportKeys.MULTITHREAD, "false")
+//params.getKeyValueParameters().put(ImportKeys.DEBUG, "false")
+//params.getKeyValueParameters().put(ImportKeys.UPDATECORPUS, "false")
+
+XTZImport i = new XTZImport(params);
+i.process();
+readyToLoad = i.isSuccessful
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZCompiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZCompiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZCompiler.groovy	(revision 321)
@@ -0,0 +1,249 @@
+package org.txm.importer.xtz
+
+import java.io.File;
+
+import org.txm.*
+import org.txm.importer.SAttributesListener
+import org.txm.importer.cwb.*
+import org.txm.stat.utils.ConsoleProgressBar
+
+import javax.xml.stream.*
+
+
+class XTZCompiler extends Compiler {
+
+	SAttributesListener sattrsListener; // store scanned structures
+	private def anatypes = new HashSet<String>() // store scanned word attributes
+	
+	String regPath;
+	String cwbLoc;
+	String corpusname;
+	String wtag;
+	
+	boolean doNormalizeAttributeValues = false;
+	boolean doNormalizeAnaValues = true;
+	
+	public XTZCompiler(ImportModule module) {
+		super(module);
+
+		corpusname = module.getParameters().getCorpusName();
+		regPath = module.getBinaryDirectory().getAbsolutePath() + "/registry/"+corpusname.toLowerCase()
+		cwbLoc = Toolbox.getParam(Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		wtag = module.getParameters().getWordElement().getTextContent();
+		
+		doNormalizeAttributeValues = "true".equals(module.getParameters().getKeyValueParameters().get(ImportKeys.NORMALISEATTRIBUTEVALUES))
+		doNormalizeAnaValues = "true".equals(module.getParameters().getKeyValueParameters().get(ImportKeys.NORMALISEANAVALUES))
+	}
+
+	public void process(ArrayList<File> files) {
+		super.process(files); // set member
+		
+		if (files == null) files = inputDirectory.listFiles();
+		
+		if (!doScanStep()) return;
+		if (!doWTCStep()) return;
+		if (!doCWBEncodeStep()) return;
+		if (!doCWBMakeAllStep()) return;
+		
+		String cleanDirectories = module.getParameters().getKeyValueParameters().get(ImportKeys.CLEAN);
+		if ("true".equals(cleanDirectories)) {
+			new File(module.getBinaryDirectory(), "wtc").deleteDir()
+		}
+		
+		isSuccessFul = true;
+	}
+
+	/**
+	 * Scan all XML-TXM files to find out structures and word properties
+	 */
+	public boolean doScanStep() {
+		// get all anatypes		
+		sattrsListener = SAttributesListener.scanFiles(inputDirectory, wtag)
+		println "-- Listing structures&properties to create for "+files.size()+" XML-TXM files..."
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File f : files) {
+			try {
+				cpb.tick();
+				getAnaTypes(f)
+			} catch (Exception e) {
+				println "Error while processing $f text: "+e
+				e.printStackTrace();
+				return false;
+			}
+		}
+		println ""
+		return true;
+	}
+	
+	private void getAnaTypes(File xmlFile) {
+		def inputData = xmlFile.toURI().toURL().openStream();
+		def factory = XMLInputFactory.newInstance();
+		def parser = factory.createXMLStreamReader(inputData);
+		boolean start = false;
+		String ANA = "ana"
+		String TYPE = "type"
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) { // start elem
+				if (wtag.equals(parser.getLocalName())) {
+					start = true;
+				} else if (start && ANA.equals(parser.getLocalName())) { // ana elem
+					for (int i = 0 ; i < parser.getAttributeCount(); i++) { // find @type
+						if (TYPE.equals(parser.getAttributeLocalName(i))) { // @type
+							anatypes.add(parser.getAttributeValue(i).substring(1)); //remove the #
+							break;
+						}
+					}
+				}
+			} else if (event == XMLStreamConstants.END_ELEMENT) { // start elem
+				if (wtag.equals(parser.getLocalName())) {
+					start = false;
+				}	
+			}
+		}
+		parser.close()
+
+//		for (String type : types)
+//			if (!anatypes.contains(type))
+//				anatypes << type
+	}
+
+	def wtcFiles = []
+	public boolean doWTCStep() {
+		println "-- Building WTC files $inputDirectory..."
+		wtcDirectory.mkdir(); // if not created
+		
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File xmlFile : files) {
+			cpb.tick();
+			String textname = xmlFile.getName();
+			int idx = textname.indexOf(".")
+			if (idx > 0) textname = textname.substring(0, idx)
+			
+			File wtcFile = new File(wtcDirectory, textname + ".wtc")
+			wtcFiles << wtcFile
+			// skip step if wtcFile is more recent than xmlFile
+			if (wtcFile.exists() && wtcFile.lastModified() >= xmlFile.lastModified()) continue;
+			
+			XTZCompilerStep step = new XTZCompilerStep(xmlFile, wtcFile, textname, corpusname, "default", anatypes, wtag)
+			step.setNormalizeAnaValues(doNormalizeAnaValues)
+			step.setNormalizeAttributeValues(doNormalizeAttributeValues)
+			if (!step.process()) {
+				reason = "Fail to process $xmlFile."
+				return false;
+			}
+		}
+		println ""
+		return true;
+	}
+
+	public boolean doCWBEncodeStep() {
+		println "-- Running cwb-encode..."
+		CwbEncode cwbEn = new CwbEncode()
+		cwbEn.setDebug(debug)
+		
+		List<String> pargs = []
+		pargs.add("id")
+		for (String ana : anatypes)
+			pargs.add(ana)
+
+		String[] pAttrs = pargs
+
+		def structs = sattrsListener.getStructs()
+		def structsProf = sattrsListener.getProfs()
+
+		if (debug) {
+			println structs
+			println structsProf
+		}
+		
+		List<String> sargs = new ArrayList<String>()
+		def tmpTextAttrs = []
+		for (String name : structs.keySet()) {
+			if (name == "text") {
+				for (String value : structs.get(name)) // append the attributes
+					tmpTextAttrs << value // added after
+				continue;
+			}
+
+			String concat = name+":"+structsProf.get(name); // append the depth
+			for (String attributeName : structs.get(name)) // append the attributes
+				concat += "+"+attributeName.toLowerCase();
+				
+			if (structs.get(name).size() == 0) {
+				concat += "+n";
+			} else {
+				if (!structs.get(name).contains("n"))
+					concat += "+n"
+			}
+				
+			if ((name == "p" || name == "body" || name == "back" || name == "front")
+				 && !concat.contains("+n+") && !concat.endsWith("+n"))
+				concat += "+n"
+				
+			sargs.add(concat)
+		}
+
+		String textSAttributes = "text:0+id+base+project";
+		for (String name : tmpTextAttrs) {
+			if (!("id".equals(name) || "base".equals(name) || "project".equals(name)))
+				textSAttributes += "+"+name.toLowerCase()
+		}
+
+		sargs.add(textSAttributes)
+		sargs.add("txmcorpus:0+lang")
+
+		sargs.sort()
+
+		String[] sAttributes = sargs
+		String[] pAttributes = pAttrs
+		println " Word properties: "+pAttributes
+		println " Structures: "+sargs
+		File allwtcFile = new File(wtcDirectory, "all.wtc");
+		try {		
+			if (!CwbEncode.concat(wtcFiles, allwtcFile)) {
+				println "Fail to write the master wtc file: "+allwtcFile
+				return false;
+			}
+						
+			cwbEn.run(cwbLoc + "cwb-encode", outputDirectory.getAbsolutePath() + "/$corpusname",
+					allwtcFile.getAbsolutePath(),
+					regPath, pAttributes, sAttributes, false);
+				
+			allwtcFile.delete(); // clean
+		} catch (Exception e) {
+			println "Error while running cwb-encode: "+e
+			e.printStackTrace()
+			allwtcFile.delete(); // clean 
+			return false;
+		}
+		println ""
+		return true;
+	}
+
+	public boolean doCWBMakeAllStep() {
+		println "-- Running cwb-makeall..."
+		try {
+			CwbMakeAll cwbMa = new CwbMakeAll();
+			cwbMa.setDebug(debug);
+
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname, module.getBinaryDirectory().getAbsolutePath() + "/registry");
+			
+			// remove milestones from CWB registry and data files
+			FixMilestoneDeclarations fm = new FixMilestoneDeclarations(
+				new File(regPath), new File(outputDirectory.getAbsolutePath(), corpusname));			
+			if (!fm.process()) {
+				println "Fail to verify&fix milestone declarations"
+				return false
+			}
+		} catch (Exception e) {
+			println "Error while running cwb-makeall: "+e
+			return false;
+		}
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZImport.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZImport.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZImport.groovy	(revision 321)
@@ -0,0 +1,146 @@
+package org.txm.importer.xtz;
+
+import org.txm.*
+import org.txm.metadatas.Metadatas
+import org.txm.objects.BaseParameters
+import org.txm.utils.io.FileCopy;
+
+public class XTZImport extends ImportModule {
+
+	public XTZImport(File importParametersFile) {
+		super(importParametersFile);
+	}
+
+	public XTZImport(BaseParameters importParameters) {
+		super(importParameters);
+	}
+
+	@Override
+	public void init(BaseParameters p) {
+		super.init(p);
+		
+		importer = new XTZImporter(this)
+		compiler = new XTZCompiler(this)
+		annotater = new TTAnnotater(this);
+		pager = new XTZPager(this)
+	}
+	
+	@Override
+	protected ArrayList<File> getTXMFilesOrder() {
+		//System.out.println("XTZ FILES ORDER");
+		if (importer == null) {
+			println "no importer step, using default text order"
+			return super.getTXMFilesOrder();
+		}
+		Metadatas metadata = importer.getMetadata();
+		if (metadata == null) {
+			println "no metadata, using default text order"
+			return super.getTXMFilesOrder();
+		}
+		File txmDirectory = new File(binaryDirectory, "txm/"+corpusName);
+		ArrayList<File> files = new ArrayList<File>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {
+			@Override
+			public boolean accept(File file) {
+				return file.isFile() && file.getName().endsWith(".xml");
+			}
+		})));
+	
+		final HashMap<File, String> textorder = new HashMap<File, String>();
+		for (File f : files) {
+			HashMap<String, String> m = metadata.getTextMetadata(f);
+			if (m != null && m.containsKey("textorder")) textorder[f] = m["textorder"];
+		}
+		println "Sorting texts using 'textorder' metadata values: "+textorder
+		Collections.sort(files, new Comparator<File>() {
+			public int compare(File f1, File f2) {
+				String o1 = textorder[f1];
+				String o2 = textorder[f2];
+				if (o1 == null && o2 == null) {
+					return f1.getName().compareTo(f2.getName());
+				} else if (o1 == null) {
+					return 1
+				} else if (o2 == null) {
+					return -1
+				} else {
+					int c = o1.compareTo(o2);
+					if (c == 0) return f1.getName().compareTo(f2.getName());
+					else return c;
+				}
+			}
+		});
+		//println files
+		return files;
+	}
+	
+	public void start() throws InterruptedException {
+		super.start();
+
+		if (isSuccessful) {
+			
+			//declare a local KR
+			List<String> krnames = importParameters.getKnowledgeRepositoryNames();
+			if (krnames.size() == 0) {
+				importParameters.createKnowledgeRepositoryElement("DEFAULT"); // set a default KR shared by all XTZ corpus
+			} else if (krnames.size() == 1 && krnames.get(0).equals("DEFAULT")) {
+				// nothing to do
+			} else {
+				println("Corpus is using custom Knowledge repositories: "+importParameters.getKnowledgeRepositoryNames());
+			}
+			
+			//copy sub directories
+			if (isUpdatingCorpus()) {
+
+			} else {
+				def dirToCopy = ["xsl", "css", "dtd"]
+				println "--- Copying subdirectories $dirToCopy"
+				for (String dir : dirToCopy) {
+					File origDirectory = new File(this.sourceDirectory, dir)
+					if (origDirectory.exists()) {
+						print "."
+						File copyDirectory = new File(this.binaryDirectory, dir)
+						FileCopy.copyFiles(origDirectory, copyDirectory)
+					}
+				}
+				println ""
+			}
+		}
+	}
+
+	public static void main(String[] args) {
+
+		long start = System.currentTimeMillis()
+
+		String userDir = System.getProperty("user.home")
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(userDir, "TXM"));
+
+		//File importParametersFile = new File("/home/mdecorde/xml/brown-for-xtz/import.xml");
+		//		File importParametersFile = new File("/home/mdecorde/xml/qgraalc/qgraal_cw/import.xml"); // new import
+		//File importParametersFile = new File("/home/mdecorde/TXM/corpora/QGRAALXTZ/import.xml"); // corpus update
+		//		File importParametersFile = new File("/home/mdecorde/xml/annotation/import.xml"); // new import
+		//File importParametersFile = new File("/home/mdecorde/xml/baiptest/import.xml"); // new import
+		//File importParametersFile = new File("/home/mdecorde/TXM/corpora/BAIP/import.xml"); // corpus update
+		File importParametersFile = new File("/home/mdecorde/xml/xtzsmall/import.xml");
+		BaseParameters b = new BaseParameters(importParametersFile);
+		b.getKeyValueParameters().put(ImportKeys.CLEAN, "false")
+		b.getKeyValueParameters().put(ImportKeys.MULTITHREAD, "false")
+		b.getKeyValueParameters().put(ImportKeys.DEBUG, "true")
+		b.getKeyValueParameters().put(ImportKeys.UPDATECORPUS, "false")
+
+		XTZImport i = new XTZImport(b);
+		i.start();
+		
+		println "Done: "+i.isSuccessful
+
+		long end = System.currentTimeMillis()
+		println "TIME: "+(end-start)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java	(revision 321)
@@ -0,0 +1,21 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+
+
+/**
+ * 
+ * Takes the XML-TXM files and wrap a TAL Tool to update the XML-TXM files
+ * 
+ * @author mdecorde
+ *
+ */
+public abstract class Annotater extends ImportStep {
+
+	public Annotater(ImportModule module) {
+		super(module);
+		
+		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.corpusName);
+		outputDirectory = new File(module.getBinaryDirectory(), "txm");
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java	(revision 321)
@@ -0,0 +1,49 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.txm.utils.DeleteDir;
+
+/**
+ * Takes the XML-TXM files and build an edition
+ * 
+ * @author mdecorde
+ *
+ */
+public class Pager extends ImportStep {
+
+	protected File htmlDirectory;
+	protected String corpusname;
+	protected ArrayList<File> files;
+
+	public Pager(ImportModule module, String editionName) {
+		super(module);
+
+		corpusname = module.getCorpusName();
+
+		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
+		htmlDirectory = new File(module.getBinaryDirectory(), "HTML/"+corpusname);
+		outputDirectory  = new File(htmlDirectory, editionName);
+
+		if (!module.isUpdatingCorpus()) {
+			DeleteDir.deleteDirectory(outputDirectory);
+			outputDirectory.mkdirs();
+		}
+	}
+
+	@Override
+	public void cancel() {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public void process() {
+		process(null); // no default files order set
+	}
+	
+	public void process(ArrayList<File> files) {
+		this.files = files;
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java	(revision 321)
@@ -0,0 +1,24 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+
+/**
+ * Takes any form of source files
+ * 
+ * After this step, the XML-TXM files are created.
+ * 
+ * they are validated before continuing 
+ * @author mdecorde
+ *
+ */
+public abstract class Importer extends ImportStep {
+
+	public Importer(ImportModule module) {
+		super(module);
+		inputDirectory = module.getSourceDirectory();
+		outputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
+		outputDirectory.mkdirs();
+	}
+
+	public abstract void checkFiles();
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZFacsPagerStep.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZFacsPagerStep.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZFacsPagerStep.groovy	(revision 321)
@@ -0,0 +1,351 @@
+package org.txm.importer.xtz
+
+import org.txm.importer.*
+import org.xml.sax.Attributes
+import org.txm.importer.filters.*
+import java.util.ArrayList
+import javax.xml.parsers.*
+import javax.xml.stream.*
+import java.net.URL
+import org.xml.sax.InputSource
+import org.xml.sax.helpers.DefaultHandler
+import java.io.FileFilter
+
+class XTZFacsPagerStep extends Step {
+
+	private def url
+	private def inputData
+	private def factory
+	private XMLStreamReader parser
+	OutputStreamWriter writer
+	StaxStackWriter pagedWriter = null
+
+	File editionDir
+	File imageDirectory // contains all the images sub-diretories
+	int iImages
+	File textImagesDirectory // contains the images for the txtname
+	File xmlFile
+	File htmlFile
+
+	def pages = []
+	def imageAttributes = [:];
+	
+	def tag, attribute, txtname, corpusname
+	boolean firstWord
+	boolean cutBefore = true;
+	String wtag;
+	boolean debug = false;
+
+	public XTZFacsPagerStep(File xmlFile, File editionDir, File imageDirectory, String txtname, String corpusname, String tag, String attribute, String wtag, boolean debug) {
+		inputData = xmlFile.toURI().toURL().openStream()
+		factory = XMLInputFactory.newInstance()
+		parser = factory.createXMLStreamReader(inputData)
+
+		this.xmlFile = xmlFile
+		this.editionDir = editionDir
+		editionDir.mkdirs()
+		this.imageDirectory = imageDirectory
+		this.tag = tag
+		this.attribute = attribute
+		this.txtname = txtname
+		this.corpusname= corpusname
+		this.wtag= wtag;
+		this.debug = debug
+
+		imageAttributes = ["src":"", 
+			"id":"txm_image", 
+			"alt":"Le fac-similé de cette page n'est pas accessible", 
+			//"style":"display:block;width:100%;", 
+			"onLoad":"viewer.toolbarImages='images/icons';viewer.onload=viewer.toolbar;new viewer({image: this, frame: ['100%','100%']});"
+			]
+		
+		if (imageDirectory != null)
+			textImagesDirectory = new File(imageDirectory, txtname)
+		iImages = 0;
+	}
+
+	int n = 1;
+	private boolean createNextOutput() {
+		try {
+			def tags = closeMultiWriter();
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+
+			if (wordid != null) {//wordid = "w_0";
+				//println " add page $n $wordid, page=$pages"
+				pages << [htmlFile, wordid] // store the previous page
+				n++
+			}
+
+			// Page suivante
+			htmlFile = new File(editionDir, "${txtname}_${n}.html")
+
+			firstWord = true
+			//println "SET FIRST WORD=true"
+			pagedWriter = new StaxStackWriter(htmlFile, "UTF-8");
+			if (debug) println "Create file $htmlFile"
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html");
+			pagedWriter.writeStartElement("head");
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(corpusname+" Edition - Page "+n)
+			pagedWriter.writeEndElement(); // </title>
+			pagedWriter.writeStartElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"]);
+			pagedWriter.writeEndElement()
+			pagedWriter.writeStartElement("link", ["rel":"stylesheet", "media":"all", "type":"text/css","href":"txm.css"]);
+			pagedWriter.writeEndElement()
+			pagedWriter.writeStartElement("link", ["rel":"stylesheet", "media":"all", "type":"text/css","href":"${corpusname}.css"]);
+			pagedWriter.writeEndElement()
+			pagedWriter.writeStartElement("script", ["type":"text/javascript", "src":"js/viewer/Simple_Viewer_beta_1.1-min.js"]);
+			pagedWriter.writeEndElement()
+			pagedWriter.writeStartElement("script", ["type":"text/javascript", "src":"js/viewer/toolbar-ext.js"]);
+			pagedWriter.writeEndElement()
+			pagedWriter.writeStartElement("link", ["rel":"stylesheet", "media":"all", "type":"text/css","href":"js/viewer/toolbar-ext.css"]);
+			pagedWriter.writeEndElement()
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+			pagedWriter.writeStartElement("div", ["id":"txmeditionpage"]);
+			pagedWriter.writeStartElements(tags);
+
+			wordid = "${wtag}_0"; // default value if no word is found
+			return true;
+		} catch (Exception e) {
+			System.out.println("Error while creating HTML file: "+e);
+			return false;
+		}
+	}
+
+	private def closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+
+			if (firstWord) { // there was no words
+				pagedWriter.writeCharacters("");
+				pagedWriter.write("<span id=\"${wtag}_0\"/>");
+			}
+			pagedWriter.writeEndElements();
+			pagedWriter.close();
+			return tags;
+		} else {
+			return [];
+		}
+	}
+
+	private writeImg(String src) {
+		
+		
+		if (src == null || src.length() == 0) {
+			pagedWriter.writeStartElement("p", ["class":"no-img"]);
+			pagedWriter.writeCharacters("Cette page n'est pas disponible en mode fac-similé.");
+			pagedWriter.writeEndElement(); // </div>
+		} else {
+			imageAttributes["src"] = src
+			pagedWriter.writeEmptyElement("img", imageAttributes);
+		}
+		
+	}
+
+	/**
+	 * Process the XML-TXM file. if imageDirectory is not null, then it will be use to fetch the images, if not then the pb@facs attribute contains the image path
+	 */
+	public boolean process() {
+		if (textImagesDirectory != null) {
+			return processWithImages();
+		} else {
+			return processWithoutImages();
+		}
+	}
+	
+	String wordid = null;
+	def imagesFiles;
+	public boolean processWithImages() {
+
+		if (!textImagesDirectory.exists()) {
+			println ("No images directory found: "+textImagesDirectory)
+			return false;
+		}
+
+		// Scan textImagesDirectory for images
+		imagesFiles = textImagesDirectory.listFiles(new FileFilter() {
+					public boolean accept(File f) {
+						return !f.isHidden() && !f.isDirectory()
+					}
+				});
+		if (imagesFiles == null || imagesFiles.size() == 0) {
+			println ("No facs images found in "+textImagesDirectory)
+			return false;
+		}
+		imagesFiles.sort(); // sort by name
+		iImages = 0;
+		//println "For text "+xmlFile.getName()
+		//println " using images : $imagesFiles"
+
+		// parse XML-TXM files
+		boolean start = false
+		String localname
+
+		createNextOutput(); // empty page
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							start = true
+							printStartPage();
+							break;
+							break;
+						case wtag:
+							if (firstWord) {
+								wordid = parser.getAttributeValue(null, "id");
+								//println "found word: $wordid"
+								firstWord = false;
+							}
+							break;
+						case tag:
+							if (debug) println "** TAG $tag $attribute : "+parser.getAttributeValue(null, attribute);
+
+							if (iImages >= imagesFiles.size() ) {
+								println " ERROR Cound not find the $iImages th image for file $xmlFile at location "+parser.getLocation().getLineNumber()
+							} else {
+								String imgPath = "res/images/$corpusname/${editionDir.getName()}/$txtname/"+imagesFiles[iImages].getName()
+								if (imgPath == null) {
+									println " ERROR in $xmlFile no value found for tag=$tag attribute=@$attribute iImages=@iImages at location "+parser.getLocation().getLineNumber()
+								} else {
+									if (cutBefore) {
+										if (debug) println " cut before"
+										createNextOutput()
+										if (debug) println " write img $imgPath"
+										writeImg(imgPath)
+									} else {
+										if (debug) println " write img $imgPath"
+										writeImg(imgPath)
+										if (debug) println " cut after"
+										createNextOutput()
+									}
+								}
+							}
+							iImages++ // next image file
+							break;
+					}
+					break;
+			}
+		}
+		closeMultiWriter()
+		pages << [htmlFile, wordid] // add the last page (no pb encountered
+
+		return pages.size() > 1
+	}
+
+	public boolean processWithoutImages() {
+
+		// parse XML-TXM files
+		boolean start = false
+		String localname
+
+		createNextOutput(); // empty page
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							start = true
+							printStartPage()
+							break;
+							
+						case wtag:
+							if (firstWord) {
+								wordid = parser.getAttributeValue(null, "id");
+								//println "found word: $wordid"
+								firstWord = false;
+							}
+							break;
+							
+						case tag:
+							String url = "" // get image URL 
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								if (parser.getAttributeLocalName(i).equals(attribute)) {
+									url = parser.getAttributeValue(i);
+									continue;
+								}
+							}
+							if (debug) println "** TAG $tag $attribute : $url"
+
+							if (cutBefore) {
+								if (debug) println " cut before"
+								createNextOutput()
+								if (debug) println " write img $url"
+								writeImg(url)
+							} else {
+								if (debug) println " write img $url"
+								writeImg(url)
+								if (debug) println " cut after"
+								url()
+							}
+
+							break;
+					}
+					break;
+			}
+		}
+		closeMultiWriter()
+		pages << [htmlFile, wordid] // add the last page (no pb encountered
+
+		return pages.size() > 1
+	}
+
+	public void printStartPage() {
+		LinkedHashMap attributes = new LinkedHashMap();
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			attributes[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i).toString()
+		}
+		pagedWriter.write("\n");
+		pagedWriter.writeStartElement("p")
+		if (attributes.containsKey("id")) {
+			pagedWriter.writeElement("h3", attributes["id"])
+		}
+		
+		pagedWriter.writeStartElement("table");
+		for (String k : attributes.keySet()) {
+			if (k == "id") continue;
+			pagedWriter.writeStartElement("tr");
+			pagedWriter.writeElement("td", k);
+			pagedWriter.writeElement("td", attributes[k]);
+			pagedWriter.writeEndElement();
+		}
+		pagedWriter.writeEndElement() // table
+		pagedWriter.writeEndElement() // p
+		
+		pagedWriter.writeCharacters("");
+	}
+	
+	public def getPageFiles() {
+		return pages
+	}
+	
+	public static void main(String[] args) {
+		File txmFile = new File("/home/mdecorde/TXM/corpora/BVHEPISTEMON2016/txm/BVHEPISTEMON2016/1546_RabelaisTL.xml")
+		String txtname = "1532_RabelaisPrnstctn"
+		String corpusname = "BVH"
+		File newEditionDirectory = new File("/tmp/xtzpagertest/")
+		newEditionDirectory.deleteDir()
+		newEditionDirectory.mkdir()
+		File imageDirectory = null;
+		
+		def ed = new XTZFacsPagerStep(txmFile, newEditionDirectory, imageDirectory, txtname, corpusname, "pb", "facs", "w", true);
+		if (!ed.process()) {
+			println "Fail to build edition for text: $txmFile"
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java	(revision 321)
@@ -0,0 +1,54 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.util.HashMap;
+
+/**
+ * One of the step of an import module
+ * 
+ * @author mdecorde
+ *
+ */
+public abstract class ImportStep {
+
+	protected File inputDirectory, outputDirectory;
+	protected ImportModule module;
+	
+	protected HashMap<String, Object> stepProperties = new HashMap<String, Object>();
+	protected boolean isSuccessFul = false;
+	protected String reason = "not set.";
+	protected boolean stopAtFirstError = true;
+	protected boolean debug = true;
+	
+	public ImportStep(ImportModule module) {
+		this.module = module;
+		debug = module.debug;
+	}
+	
+	public File getInputDirectory() {
+		return inputDirectory;
+	}
+	
+	public File getOutputDirectory() {
+		return outputDirectory;
+	}
+	
+	public ImportModule getImportModule() {
+		return module;
+	}
+	
+	public boolean isSuccessFul() {
+		return isSuccessFul;
+	}
+	
+	public String getReason() {
+		return reason;
+	}
+	
+	/**
+	 * Called when a step is interrupted to clean streams and stuff
+	 */
+	public abstract void cancel();
+	
+	public abstract void process();
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/TTAnnotater.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/TTAnnotater.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/TTAnnotater.groovy	(revision 321)
@@ -0,0 +1,56 @@
+package org.txm.importer.xtz
+
+import java.io.File;
+
+import org.txm.scripts.teitxm.Annotate
+
+
+/**
+ * Wraps current Annotate class into the import workflow
+ * 
+ * @author mdecorde
+ *
+ */
+class TTAnnotater extends Annotater {
+
+	Annotate a;
+	
+	public TTAnnotater(ImportModule module) {
+		super(module);
+	}
+
+	@Override
+	public void process() {
+
+		//String model = module.getParameters().get(ImportKeys.TTMODEL);
+		String cleanDirectories = module.getParameters().getKeyValueParameters().get(ImportKeys.CLEAN);
+		
+		String corpusname = module.getParameters().getCorpusName();
+		def corpusElem = module.getParameters().getCorpora().get(corpusname);
+		String lang = corpusElem.getAttribute("lang");
+
+		def hash = [:]
+		for (File txmFile : inputDirectory.listFiles()) {
+			hash[txmFile.getName()] = lang;
+		}
+		
+		a = new Annotate()
+		if (module.isMultiThread()) {
+			isSuccessFul = a.run(module.getBinaryDirectory(), inputDirectory, hash);
+		} else {
+			isSuccessFul = a.run(module.getBinaryDirectory(), inputDirectory, lang+".par");
+		}
+		
+		if ("true".equals(cleanDirectories)) {
+			new File(module.getBinaryDirectory(), "treetagger").deleteDir()
+			new File(module.getBinaryDirectory(), "annotations").deleteDir()
+		}
+		
+		if (!isSuccessFul) reason = "TreeTagger annotation failed."
+	}
+
+	@Override
+	public void cancel() {
+		a.setCancelNow();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZPager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZPager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZPager.groovy	(revision 321)
@@ -0,0 +1,461 @@
+package org.txm.importer.xtz
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.txm.objects.BaseParameters
+import org.w3c.dom.Element
+import org.txm.importer.*
+import org.txm.stat.utils.ConsoleProgressBar
+import org.txm.utils.BundleUtils;
+import org.txm.utils.i18n.*
+import org.txm.utils.io.FileCopy;
+
+import javax.xml.stream.*
+
+class XTZPager extends Pager {
+
+	BaseParameters params;
+
+	Element corpusElem;
+	String lang;
+	String page_element;
+	String wordTag;
+	int wordsPerPage;
+
+	File cssDirectory, jsDirectory, imagesDirectory;
+
+	public XTZPager(ImportModule module) {
+		super(module, "default");
+
+		params = module.getParameters()
+
+		corpusElem = params.corpora.get(corpusname);
+		lang = corpusElem.getAttribute("lang");
+		wordsPerPage = params.getWordsPerPage("default")
+		page_element = params.getPageElement("default")
+		wordTag = module.getParameters().getWordElement().getTextContent()
+
+		cssDirectory = new File(module.getSourceDirectory(), "css")
+		jsDirectory = new File(module.getSourceDirectory(), "js")
+		imagesDirectory = new File(module.getSourceDirectory(), "images")
+	}
+
+	public void process(ArrayList<File> files) {
+		super.process(files);
+		
+		if (files == null) {
+			files = inputDirectory.listFiles();
+			if (files != null) Collections.sort(files);
+		}
+		
+		if (!doDefaultEditionStep()) return;
+		if (!doFacsEditionStep()) return;
+		if (!doPostEditionXSLStep()) return;
+
+		isSuccessFul = true;
+		println ""
+	}
+
+	public boolean doDefaultEditionStep() {
+		
+		boolean build_edition = module.getParameters().getDoEdition("default")
+		if (!build_edition) {
+			return true;
+		}
+		
+		def second = 0
+
+		println "-- Building 'default' edition of  ${files.size()} texts..."
+
+		def css = ["css/txm.css", "css/${corpusname}.css"] // default CSS inclusion
+
+		// scan existing css files that must be declared in each HTML page
+		if (cssDirectory.exists()) {
+			def cssFiles = cssDirectory.listFiles();
+			if (cssFiles != null)
+				for (File cssFile : cssFiles) {
+					if (cssFile.isFile() && !cssFile.isHidden() && cssFile.getName().endsWith(".css"))
+						css << "css/"+cssFile.getName();
+				}
+		}
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File txmFile : files) {
+			cpb.tick()
+			String textname = txmFile.getName();
+			int i = textname.lastIndexOf(".");
+			if (i > 0) textname = textname.substring(0, i);
+
+			File firstHTMLPageFile = new File(outputDirectory, textname+"_1.html");
+			if (firstHTMLPageFile.exists() && firstHTMLPageFile.lastModified() >= txmFile.lastModified()) continue;
+
+			List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+			List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+			Element texts = params.getTextsElement(corpusElem);
+			Element text = params.getTextElement(texts, textname);
+			if (text == null) { // just in case
+				text = params.addText(corpusElem, textname, txmFile)
+			}
+			def ed = new XTZDefaultPagerStep(this, txmFile, textname, NoSpaceBefore, NoSpaceAfter, css);
+			Element edition = params.getEditionElement(text, "default");
+			if (edition == null) {
+				edition = params.addEdition(text, "default", outputDirectory.getAbsolutePath(), "html");
+			} else { // remove existing pages if any
+				def children = edition.getChildNodes();
+				while (children.getLength() > 0 ) {
+					edition.removeChild(children.item(0))
+				}
+			}
+
+			for (i = 0 ; i < ed.getPageFiles().size();) {
+				File f = ed.getPageFiles().get(i);
+				String wordid = ed.getIdx().get(i);
+				params.addPage(edition, ""+(++i), wordid);
+			}
+		}
+		
+		// copy default TXM css file in the "facs" edition directory
+		File csshtmlDirectory = new File(outputDirectory, "css")
+		csshtmlDirectory.mkdirs()
+		BundleUtils.copyFiles("org.txm.core", "res", "org/txm/css", "txm.css", csshtmlDirectory);
+		
+		// copy CSS files in the "default" edition directory
+		if (cssDirectory.exists()) {
+			FileCopy.copyFiles(cssDirectory, csshtmlDirectory)
+		}
+		if (jsDirectory.exists()) {
+			File jshtmlDirectory = new File(outputDirectory, "js")
+			FileCopy.copyFiles(jsDirectory, jshtmlDirectory)
+		}
+		if (imagesDirectory.exists()) {
+			File imageshtmlDirectory = new File(outputDirectory, "images")
+			FileCopy.copyFiles(imagesDirectory, imageshtmlDirectory)
+		}
+
+		// save changes
+		println ""
+		return true;
+	}
+
+	public boolean doFacsEditionStep() {
+
+		boolean mustBuildFacsEdition = module.getParameters().getDoFacsEdition();
+		if (!mustBuildFacsEdition) return true;
+		
+		String imageDirectoryPath = module.getParameters().getFacsEditionImageDirectory().trim();
+		File imageDirectory = new File(imageDirectoryPath);
+		if (!imageDirectoryPath.startsWith("http") && imageDirectoryPath.length()== 0 && !imageDirectory.exists() && !imageDirectory.isDirectory()) {
+			imageDirectory = null;
+		}
+
+		def second = 0
+
+		println "-- Building 'facs' edition of ${files.size()} texts..."
+		File newEditionDirectory = new File(htmlDirectory, "facs");
+		newEditionDirectory.mkdir();
+
+		// declare the "facs" edition
+		Element editionDefinitionElement = params.getEditionDefinitionElement(corpusElem, "facs");
+		if (editionDefinitionElement == null) {
+			editionDefinitionElement = params.addEditionDefinition(corpusElem, "facs", "groovy", "XTZPager.groovy")
+		}
+		Element textsElement = params.getTextsElement(corpusElem)
+
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File txmFile : files) {
+			cpb.tick()
+			String txtname = txmFile.getName();
+			int i = txtname.lastIndexOf(".");
+			if (i > 0) txtname = txtname.substring(0, i);
+
+			File firstHTMLPageFile = new File(newEditionDirectory, txtname+"_1.html");
+			if (firstHTMLPageFile.exists() && firstHTMLPageFile.lastModified() >= txmFile.lastModified()) continue;
+
+			List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+			List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+			// the text should already exists
+			Element text = params.getTextElement(textsElement, txtname)
+			if (text == null) { // just in case
+				text = params.addText(corpusElem, txtname, txmFile)
+			}
+			Element edition = params.getEditionElement(text, "facs");
+			if (edition == null) {
+				edition = params.addEdition(text, "facs", newEditionDirectory.getAbsolutePath(), "html");
+			} else {
+				def children = edition.getChildNodes();
+				while (children.getLength() > 0 ) {
+					edition.removeChild(children.item(0))
+				}
+			}
+
+			try {
+				def ed = new XTZFacsPagerStep(txmFile, newEditionDirectory, imageDirectory, txtname, corpusname, "pb", "facs", wordTag, debug);
+				if (!ed.process()) {
+					println "Fail to build edition for text: $txmFile"
+					continue;
+				}
+
+				def pages = ed.getPageFiles()
+				for (i = 0 ; i < pages.size();) {
+					File f = pages[i][0];
+					String wordid = pages[i][1]
+					params.addPage(edition, ""+(++i), wordid);
+					//println "add facs page: $f $wordid"
+				}
+			} catch (Exception e) {
+				println "Error while processing $txmFile text: "+e
+				e.printStackTrace();
+				return false;
+			}
+		}
+
+		if (!imageDirectoryPath.startsWith("http") && imageDirectory != null) { // copy files only if local
+			File editionImagesDirectory = new File(newEditionDirectory, "res/images/"+corpusname+"/facs");
+			editionImagesDirectory.mkdirs();
+			FileCopy.copyFiles(imageDirectory, editionImagesDirectory);
+		}
+		
+		// copy SimpleViewer files in the "facs" edition directory
+		File jshtmlDirectory = new File(newEditionDirectory, "js")
+		jshtmlDirectory.mkdirs()
+		BundleUtils.copyFiles("org.txm.toolbox", "res", "org/txm/js", "viewer", jshtmlDirectory);
+		BundleUtils.copyFiles("org.txm.toolbox", "res", "org/txm/", "images", newEditionDirectory);
+		
+		// copy default TXM css file in the "facs" edition directory
+		File csshtmlDirectory = new File(newEditionDirectory, "css")
+		csshtmlDirectory.mkdirs()
+		BundleUtils.copyFiles("org.txm.toolbox", "res", "org/txm/css", "txm.css", csshtmlDirectory);
+		
+		// copy CSS/JS/Images sources files in the "facs" edition directory
+		if (cssDirectory.exists()) {
+			FileCopy.copyFiles(cssDirectory, csshtmlDirectory)
+		}
+		if (jsDirectory.exists()) {
+			FileCopy.copyFiles(jsDirectory, jshtmlDirectory)
+		}
+		if (imagesDirectory.exists()) {
+			File imageshtmlDirectory = new File(newEditionDirectory, "images")
+			FileCopy.copyFiles(imagesDirectory, imageshtmlDirectory)
+		}
+		
+		Element editionsElement = params.getEditionsElement(corpusElem);
+		editionsElement.setAttribute("default","default,facs");
+
+		println ""
+		return true;
+	}
+
+	/**
+	 * read from $bindir/txm and write the result in $bindir/txm
+	 *
+	 */
+	public boolean doPostEditionXSLStep() {
+
+		File xslDirectory = new File(module.getSourceDirectory(), "xsl/4-edition")
+		if (xslDirectory.exists()) {
+
+			// prepare XSL parameters
+			def xslParams = module.getParameters().getXsltParams(corpusElem);
+			String s = module.getParameters().getWordsPerPage("default");
+			if (s != null && s.length() > 0)
+
+			// shared XSL parameters
+			xslParams["number-words-per-page"] = Integer.parseInt(s);
+			xslParams["pagination-element"] = module.getParameters().getPageElement("default")
+			xslParams["import-xml-path"] = module.getParameters().paramFile.toURI().toString()
+			//println "XSL PARAMS: "+xslParams
+
+			def xslFiles = xslDirectory.listFiles()
+			xslFiles.sort()
+			def editionsCreated = [:]
+			for (File xslFile : xslFiles) {
+				if (xslFile.isDirectory() || xslFile.isHidden() || !xslFile.getName().endsWith(".xsl")) continue;
+				if (!xslFile.getName().matches("[1-9]-.+")) continue;
+				
+				String xslName = xslFile.getName().substring(2); // remove the "1-", "2-", etc.
+				int idx2 = xslName.indexOf(".")
+				if (idx2 > 0) xslName = xslName.substring(0, idx2)
+				else {
+					println "$xslFile is not a '.xsl' file"
+					continue;
+				}
+				int idx3 = xslName.indexOf("-")
+				if (idx3 < 0) {
+					println "$xslFile file does not follow the '{Number}-{editionName}-{step}.xsl' name pattern"
+					continue;
+				}
+				String pagerStep = xslName.substring(idx3 + 1);
+				String editionName = xslName.substring(0, idx3);
+
+				int idx = editionName.indexOf(".")
+				if (idx > 0) editionName = editionName.substring(0, idx);
+				println "-- Building '$editionName' XSL edition with step '$pagerStep'..."
+
+				File newEditionDirectory = new File(htmlDirectory, editionName);
+				xslParams["output-directory"] = newEditionDirectory.toURI().toString()
+
+				if (editionsCreated[editionName] == null) { // first XSL, replace an edition
+					editionsCreated[editionName] = xslFile
+					//if (!importModule.isUpdatingCorpus()) {
+					//TODO: optimisation if update is enable,
+					newEditionDirectory.deleteDir(); // delete previous edition if any
+					//}
+					newEditionDirectory.mkdir()
+
+					boolean deleteOutputFiles = "pager" == pagerStep;
+					if (ApplyXsl2.processImportSources(xslFile, inputDirectory, newEditionDirectory, xslParams, deleteOutputFiles)) {
+						println ""
+					} else {
+						reason = "Fail to apply edition XSL: $xslFile"
+						return false;
+					}
+
+					// copy CSS files in the newEditionDirector edition directory
+					if (cssDirectory.exists()) {
+						File csshtmlDirectory = new File(newEditionDirectory, "css")
+						FileCopy.copyFiles(cssDirectory, csshtmlDirectory)
+					}
+					if (jsDirectory.exists()) {
+						File jshtmlDirectory = new File(newEditionDirectory, "js")
+						FileCopy.copyFiles(jsDirectory, jshtmlDirectory)
+					}
+					if (imagesDirectory.exists()) {
+						File imageshtmlDirectory = new File(newEditionDirectory, "images")
+						FileCopy.copyFiles(imagesDirectory, imageshtmlDirectory)
+					}
+				} else { // N+1 XSL working with HTML files
+					def htmlFiles = newEditionDirectory.listFiles()
+					htmlFiles.sort()
+
+					if (ApplyXsl2.processImportSources(xslFile, htmlFiles, xslParams)) {
+						if ("pager".equals(pagerStep)) {
+							// delete the one page HTML files only if the XSL step is "pager"
+							for (File f : htmlFiles) f.delete();
+						}
+						//	println ""
+					} else {
+						reason = "Fail to apply edition XSL: $xslFile"
+						return false;
+					}
+				}
+			}
+
+			// UPDATE import.xml: for each XML-TXM file, we must retrieve the first word ID from the XSL output files
+			//println "retrieve word ids from $inputDirectory"
+			println "-- Fetching page word IDs..."
+			ConsoleProgressBar cpb = new ConsoleProgressBar(editionsCreated.keySet().size())
+			for (String editionName : editionsCreated.keySet()) {
+				cpb.tick()
+				//ensure edition definition is declared
+				def editionsElement = params.getEditionsElement(corpusElem)
+				def e = params.getEditionDefinitionElement(editionsElement, editionName);
+				if (e == null) {
+					params.addEditionDefinition(corpusElem, editionName, "xsl", editionsCreated[editionName].toString());
+				}
+
+				File newEditionDirectory = new File(htmlDirectory, editionName);
+				File xslFile = editionsCreated[editionName]
+				for (File txmFile : inputDirectory.listFiles()) {
+					if (txmFile.isDirectory()) continue;
+					String textName = txmFile.getName()
+					int idx4 = textName.indexOf(".")
+					if (idx4 > 0) textName = textName.substring(0, idx4);
+
+					getFirstWordIDs(textName, editionName, newEditionDirectory, xslFile);
+				}
+			}
+			println ""
+		}
+		return true;
+	}
+
+	private void getFirstWordIDs(String textName, String editionName, File newEditionDirectory, File xslFile) {
+		//		println "call getFirstWordIDs textName=$textName editionName=$editionName dir=$newEditionDirectory xsl=$xslFile"
+		Element textsElement = params.getTextsElement(corpusElem)
+		Element textElement = params.getTextElement(textsElement, textName)
+		if (textElement == null) { // just in case
+			textElement = params.addText(corpusElem, textName, new File(inputDirectory, textName+".xml"))
+		}
+
+		Element editionElement = params.getEditionElement(textElement, editionName);
+		if (editionElement == null) { // the edition does not exist, declare it
+			editionElement = params.addEdition(textElement, editionName, newEditionDirectory.getAbsolutePath(), "html");
+		} else { // the edition already exists, remove pages from the existing edition
+			def children = editionElement.getChildNodes();
+			while (children.getLength() > 0 ) {
+				editionElement.removeChild(children.item(0))
+			}
+		}
+		editionElement.setAttribute("mode", "xsl");
+		editionElement.setAttribute("script", xslFile.getName());
+
+		LinkedHashMap<File, String> words = new LinkedHashMap<File, String>()
+		def files = []
+		newEditionDirectory.eachFile() {it -> if (it.isFile()) files << it}
+
+		files.sort() { f1, f2 ->
+			String s1 = f1.getName()
+			String s2 = f2.getName()
+			int n1 = Integer.parseInt(s1.substring(s1.lastIndexOf("_")+1, s1.lastIndexOf(".")))
+			int n2 = Integer.parseInt(s2.substring(s2.lastIndexOf("_")+1, s2.lastIndexOf(".")))
+			return n1 - n2;
+		}
+
+		for (File f : files) {
+			String pagename = f.getName();
+			if (pagename.startsWith(textName+"_")) { // this is a page
+				String firstWordID = getMetaContent(f);
+				pagename = pagename.substring((textName+"_").length(), pagename.indexOf(".html")) // !!!!
+				params.addPage(editionElement, pagename, firstWordID)
+			}
+		}
+	}
+
+	public static String getMetaContent(File f) {
+		def inputData = f.toURI().toURL().openStream();
+		def factory = XMLInputFactory.newInstance();
+		factory.setProperty("javax.xml.stream.supportDTD", false); // ignore the DTD declared in doctype
+
+		def parser = factory.createXMLStreamReader(inputData);
+		String META = "meta"
+		String BODY = "body"
+		String NAME = "name"
+		String DESCRIPTION = "txm:first-word-id"
+		String CONTENT = "content"
+
+		String content = "";
+		String desc = "";
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) { // start elem
+
+				if (META.equals(parser.getLocalName())) { // ana elem
+					desc = "";
+					// fetch attribute values
+					for (int i = 0 ; i < parser.getAttributeCount(); i++) { // scan attributes
+						if (NAME.equals(parser.getAttributeLocalName(i))) { // found @name
+							desc = parser.getAttributeValue(i)
+						} else if (CONTENT.equals(parser.getAttributeLocalName(i))) { // found @content
+							content = parser.getAttributeValue(i)
+						}
+					}
+					if (DESCRIPTION.equals(desc)) { // stop now
+						break;
+					}
+				} else if (BODY.equals(parser.getLocalName())) { // no need to go further, meta@name="description" not found :(
+					content = "";
+					break;
+				}
+			}
+		}
+		parser.close()
+
+		return content;
+	}
+
+	public static void main(def args) {
+		println "RESULT: "+getMetaContent(new File("/home/mdecorde/TXM/corpora/QGRAALXTZ/HTML/QGRAALXTZ/default", "qgraal_cm_test201510_page_160_2.html"))
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZDefaultPagerStep.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZDefaultPagerStep.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZDefaultPagerStep.groovy	(revision 321)
@@ -0,0 +1,519 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.stream.*;
+
+import org.txm.importer.StaxStackWriter;
+
+public class XTZDefaultPagerStep {
+
+	List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename = "";
+	String txtname = "";
+	File outdir;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue = "";
+
+	/** The interpvalue. */
+	String interpvalue = "";
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	OutputStreamWriter writer;
+
+	/** The pagedWriter. */
+	StaxStackWriter pagedWriter = null;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	String paginationElement;
+	def cssList;
+	def wordTag = "w";
+	def noteElements = new HashSet<String>();
+	def outOfTextElements = new HashSet<String>();
+	XTZPager pager;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	public XTZDefaultPagerStep(XTZPager pager, File infile, String txtname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, def cssList) {
+		this.pager = pager;
+		this.paginationElement = pager.page_element;
+		this.cssList = cssList;
+		this.basename = pager.corpusname;
+		this.txtname = txtname;
+		this.outdir = pager.outputDirectory;
+		this.wordmax = pager.wordsPerPage;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+		this.wordTag= pager.wordTag;
+		outdir.mkdirs()
+
+		inputData = new BufferedInputStream(url.openStream());
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		
+		String notesListString = pager.getImportModule().getParameters().getNoteElement().getTextContent()
+		if (notesListString != null) for (def s : notesListString.split(",")) noteElements << s;
+		
+		String elems = pager.getImportModule().getParameters().getOutSideTextTagsAndKeepContentElement().getTextContent()
+		if (elems != null) for (def s : elems.split(",")) outOfTextElements << s;
+		
+		process();
+	}
+
+	private def closeMultiWriter() {
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+
+			if (firstWord) { // there was no words
+				pagedWriter.writeCharacters("");
+				this.idxstart.add("${wordTag}_0")
+				pagedWriter.write("<span id=\"${wordTag}_0\"/>");
+				//				}
+			}
+			pagedWriter.writeEndElements();
+
+			if (notes.size() > 0) {
+				pagedWriter.writeEmptyElement("hr", ["id":"notes", "width":"20%", "align":"left"]);
+				//pagedWriter.writeStartElement("ol");
+				int i = 1;
+				for (String note : notes) {
+					//pagedWriter.writeStartElement("li");
+					pagedWriter.writeStartElement("a", ["href":"#noteref_"+i, "name":"note_"+i]);
+					pagedWriter.writeStartElement("sup")
+					pagedWriter.writeCharacters(""+i)
+					pagedWriter.writeEndElement() // </sub>
+					pagedWriter.writeEndElement() // </a>
+					pagedWriter.writeCharacters(note)
+					pagedWriter.writeEmptyElement("br")
+					//pagedWriter.writeEndElement() // </li>
+					i++;
+				}
+				//pagedWriter.writeEndElement() // </ol>
+				notes.clear()
+			}
+
+			pagedWriter.close();
+			return tags;
+		} else {
+			return [];
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput() {
+		wordcount = 0;
+		try {
+
+			def tags = closeMultiWriter();
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")
+			pages.add(outfile);
+			firstWord = true; // waiting for next word
+
+			pagedWriter = new StaxStackWriter(outfile, "UTF-8");
+
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html");
+			pagedWriter.writeEmptyElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"]);
+			for (String css : cssList) {
+				pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"$css"]);
+			}
+			pagedWriter.writeStartElement("head");
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(basename.toUpperCase()+" Edition - Page "+pagecount)
+			pagedWriter.writeEndElement(); // </title>
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+			pagedWriter.writeStartElement("div", ["class": "txmeditionpage"]) //<div>
+			pagedWriter.writeStartElements(tags);
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput() {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private void goToText() {
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.END_ELEMENT)
+				if (parser.getLocalName().matches("teiHeader"))
+					return;
+		}
+	}
+
+	def notes = []
+	def currentOutOfTextElements = [] // stack of element with out of text to edit opened element
+	def writeOutOfTextToEditText = false
+	/**
+	 * Process.
+	 */
+	public boolean process() {
+		boolean flagNote = false;
+		String noteContent = "";
+		String rend = ""
+		goToText();
+
+		String localname = "";
+		createNextOutput();
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			rend = "";
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					if (outOfTextElements.contains(localname)) {
+						currentOutOfTextElements << localname
+						writeOutOfTextToEditText = true;
+					} else if (currentOutOfTextElements.size() > 0) {
+						currentOutOfTextElements << localname
+					}			
+					
+					rend = parser.getAttributeValue(null, "rend")
+					if (rend == null) rend = "";
+					switch (localname) {
+						case "text":
+							LinkedHashMap attributes = new LinkedHashMap();
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								attributes[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i).toString()
+							}
+							pagedWriter.write("\n");
+							pagedWriter.writeStartElement("p")
+							pagedWriter.writeAttribute("class", rend);
+							if (attributes.containsKey("id")) {
+								pagedWriter.writeElement("h3", attributes["id"])
+							}
+
+							pagedWriter.writeStartElement("table");
+							for (String k : attributes.keySet()) {
+								if (k == "id") continue;
+								if (k == "rend") continue;
+
+								pagedWriter.writeStartElement("tr");
+								pagedWriter.writeElement("td", k);
+								pagedWriter.writeElement("td", attributes[k]);
+								pagedWriter.writeEndElement();
+							}
+							pagedWriter.writeEndElement() // table
+							pagedWriter.writeEndElement() // p
+
+							pagedWriter.writeCharacters("");
+							break;
+						case "ref":
+							pagedWriter.writeStartElement("a")
+							pagedWriter.writeAttribute("href", parser.getAttributeValue(null, "target"));
+
+							pagedWriter.writeAttribute("target", "_blank");
+							pagedWriter.writeAttribute("class", rend);
+							break;
+						case "head":
+							pagedWriter.write("\n");
+							pagedWriter.writeStartElement("h2", ["class":rend])
+							break;
+						case "graphic":
+							pagedWriter.write("\n");
+							String url = parser.getAttributeValue(null, "url")
+							if (url != null) {
+								pagedWriter.writeStartElement("center", ["class":rend])
+								pagedWriter.writeEmptyElement("img", ["src":url, "align":"middle"])
+								pagedWriter.writeEndElement() // div
+							}
+							break;
+						case "table":
+							pagedWriter.writeStartElement("table", ["class":rend])
+							pagedWriter.write("\n");
+							break;
+						case "row":
+							pagedWriter.writeStartElement("tr", ["class":rend])
+							break;
+						case "cell":
+							pagedWriter.writeStartElement("td", ["class":rend])
+							break;
+						case "list":
+							String type = parser.getAttributeValue(null,"type");
+							if ("unordered" == type) {
+								pagedWriter.writeStartElement("ul", ["class":rend])
+							} else {
+								pagedWriter.writeStartElement("ol", ["class":rend])
+							}
+							break
+						case "item":
+							pagedWriter.writeStartElement("li", ["class":rend])
+							break;
+						case "hi":
+						case "emph":
+							if ("i".equals(rend) || "italic".equals(rend)) {
+								pagedWriter.writeStartElement("i", ["class":rend])
+							} else if ("b".equals(rend) || "bold".equals(rend)) {
+								pagedWriter.writeStartElement("b", ["class":rend])
+							} else {
+								if ("emph".equals(localname)) {
+									pagedWriter.writeStartElement("i", ["class":rend])
+								} else { // hi
+									pagedWriter.writeStartElement("b", ["class":rend])
+								}
+							}
+
+							break;
+						case "p":
+							pagedWriter.write("\n");
+							pagedWriter.writeStartElement("p", ["class":rend])
+							break;
+						case paginationElement:
+							createNextOutput();
+							wordcount=0;
+							pagedWriter.write("\n");
+							if (parser.getAttributeValue(null,"n") != null) {
+								pagedWriter.writeElement("p", ["class":"txmeditionpb", "align":"center"], parser.getAttributeValue(null,"n"))
+							}
+							break;
+						case "lb":
+							pagedWriter.writeEmptyElement("br")
+							break;
+						case wordTag:
+							wordid = parser.getAttributeValue(null,"id");
+							if (firstWord) {
+								firstWord = false;
+								this.idxstart.add(wordid);
+							}
+							wordcount++;
+							if (wordcount >= wordmax) {
+								createNextOutput();
+							}
+							break;
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+							break;
+						default:
+							if (noteElements.contains(localname)) {
+								flagNote = true;
+								noteContent = ""
+							}
+							break;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					if (currentOutOfTextElements.size() > 0) currentOutOfTextElements.pop()
+					writeOutOfTextToEditText = currentOutOfTextElements.size() > 0
+					
+					switch (localname) {
+						case "p":
+							pagedWriter.writeEndElement() // </p>
+							pagedWriter.write("\n");
+							break;
+						case "head":
+							pagedWriter.writeEndElement() // </h2>
+							pagedWriter.write("\n");
+							break;
+						case "list":
+							pagedWriter.writeEndElement(); // ul or ol
+							pagedWriter.write("\n");
+							break
+						case "item":
+							pagedWriter.writeEndElement(); // li
+							pagedWriter.write("\n");
+							break;
+						case "hi":
+							pagedWriter.writeEndElement(); // b
+							break;
+						case "emph":
+							pagedWriter.writeEndElement(); // i
+							break;
+						case "table":
+							pagedWriter.writeEndElement(); // table
+							pagedWriter.write("\n");
+							break;
+						case "row":
+							pagedWriter.writeEndElement(); // tr
+							break;
+						case "cell":
+							pagedWriter.writeEndElement(); // td
+							break;
+						case "ref":
+							pagedWriter.writeEndElement() // </a>
+							break;
+						case "form":
+							flagform = false
+							break;
+						case "ana":
+							flaginterp = false
+							break;
+						case wordTag:
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if (l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+
+							if (interpvalue != null)
+								interpvalue = interpvalue;
+
+							if (NoSpaceBefore.contains(wordvalue) ||
+							NoSpaceAfter.contains(lastword) ||
+							wordvalue.startsWith("-") ||
+							NoSpaceAfter.contains(endOfLastWord)) {
+								pagedWriter.writeStartElement("span", ["title":interpvalue, "id":wordid]);
+							} else {
+								pagedWriter.writeCharacters(" ");
+								pagedWriter.writeStartElement("span", ["title":interpvalue, "id":wordid]);
+							}
+
+							pagedWriter.writeCharacters(wordvalue);
+							pagedWriter.writeEndElement();
+							lastword=wordvalue;
+							break;
+						default:
+							if (noteElements.contains(localname)) {
+								flagNote = false;
+								if (noteContent.length() > 0) {
+									notes << noteContent;
+									pagedWriter.writeStartElement("a", ["href":"#note_"+notes.size(), "name":"noteref_"+notes.size(), "title":noteContent]);
+									pagedWriter.writeStartElement("sup")
+									pagedWriter.writeCharacters(""+notes.size())
+									pagedWriter.writeEndElement() // </sub>
+									pagedWriter.writeEndElement() // </span>
+								}
+							}
+							break;
+					}
+					break;
+				case XMLStreamConstants.CHARACTERS:
+					if (flagform && parser.getText().length() > 0) {
+						wordvalue+=(parser.getText());
+						if (flagNote == parser.getText().length() > 0)
+							noteContent += parser.getText().replace("\n", " ");
+					} else	if (flaginterp && parser.getText().length() > 0) {
+						interpvalue+=(parser.getText());
+					} else if (flagNote == parser.getText().length() > 0) {
+						noteContent += parser.getText().replace("\n", " ");
+					} else if (writeOutOfTextToEditText) {
+						pagedWriter.writeCharacters(parser.getText())
+					}
+					break;
+			}
+		}
+		closeMultiWriter();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java	(revision 321)
@@ -0,0 +1,61 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.txm.utils.DeleteDir;
+
+/**
+ * Takes XML-TXM files, build the WTC files and call cwb utils
+ * 
+ * @author mdecorde
+ *
+ */
+public class Compiler extends ImportStep {
+
+	protected File wtcDirectory, registryDirectory, dataDirectory;
+	protected ArrayList<File> files;
+	
+	/**
+	 * Creates the output directories
+	 * 
+	 * @param module
+	 */
+	public Compiler(ImportModule module) {
+		super(module);
+
+		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
+		wtcDirectory = new File(module.getBinaryDirectory(), "wtc");		
+		outputDirectory = new File(module.getBinaryDirectory(), "data");
+		registryDirectory = new File(module.getBinaryDirectory(), "registry");
+		dataDirectory = new File(outputDirectory, module.getCorpusName());
+
+		DeleteDir.deleteDirectory(outputDirectory);
+		outputDirectory.mkdirs();
+		
+		DeleteDir.deleteDirectory(dataDirectory);
+		dataDirectory.mkdirs();
+		
+		DeleteDir.deleteDirectory(registryDirectory);
+		registryDirectory.mkdirs();
+		
+		if (!module.isUpdatingCorpus()) {
+			DeleteDir.deleteDirectory(wtcDirectory);
+			wtcDirectory.mkdir();
+		}
+	}
+
+	@Override
+	public void cancel() {
+		// TODO Auto-generated method stub
+	}
+
+	@Override
+	public void process() {
+		process(null); // no default files order set
+	}
+	
+	public void process(ArrayList<File> files) {
+		this.files = files;
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java	(revision 321)
@@ -0,0 +1,12 @@
+package org.txm.importer.xtz;
+
+public class Step {
+
+	public Step() {
+		// TODO Auto-generated constructor stub
+	}
+
+	public boolean process() {
+		return true;
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZImporter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZImporter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/XTZImporter.groovy	(revision 321)
@@ -0,0 +1,444 @@
+package org.txm.importer.xtz
+
+import filters.Tokeniser.SimpleTokenizerXml
+
+import java.io.File;
+
+import net.sf.saxon.style.XSLParam;
+
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.MileStoneProjection;
+import org.txm.importer.ValidateXml;
+import org.txm.*;
+
+import javax.xml.stream.*;
+
+import org.w3c.dom.Element
+import org.txm.metadatas.Metadatas
+import org.txm.objects.BaseParameters
+import org.txm.scripts.teitxm.Xml2Ana
+import org.txm.stat.utils.ConsoleProgressBar
+import org.txm.utils.*;
+import org.txm.utils.io.FileCopy;
+
+class XTZImporter extends Importer {
+
+	boolean cancelNow = false;
+	def filesToProcess = []
+
+	File frontXSLResultDirectory;
+	File tokenizedDirectory;
+	File sentencedDirectory;
+
+	String xslPath;
+	def xslParams;
+	String lang;
+	String wordTag;
+	boolean skipDoTokenizeStep = false;
+	
+	Metadatas metadata = null; // text metadata
+
+	public XTZImporter(ImportModule module) {
+		super(module);
+	}
+	
+	public Metadatas getMetadata() {
+		return metadata;
+	}
+
+	public void process() {
+
+		BaseParameters params = module.getParameters();
+		File binDir = module.getBinaryDirectory();
+
+		String corpusname = params.getCorpusName();
+		Element corpusElem = params.corpora.get(corpusname);
+
+		def srcFiles = [];
+		def files = inputDirectory.listFiles();
+		if (files == null) {
+			reason = "No file to process in "+inputDirectory
+			return; // no file to process in the directory
+		}
+
+		for (File f : files) {
+			if (f.isHidden() || f.isDirectory())
+				continue;
+			else if (f.getName().endsWith(".properties"))
+				continue;
+			else if (f.getName().equals("import.xml"))
+				continue;
+			else if (f.getName().endsWith(".csv"))
+				continue;
+			else if (f.getName().endsWith(".dtd"))
+				continue;
+			else if (f.getName().endsWith(".xsl"))
+				continue;
+			else if (f.getName().endsWith("~"))
+				continue;
+			else if (f.getName().startsWith("."))
+				continue;
+
+			srcFiles << f
+		}
+
+		if (srcFiles.size() == 0) {
+			reason = "No suitable file to process in "+inputDirectory
+			return; // no file to process in the directory
+		}
+
+		// prepare front XSL if any
+		xslPath = params.getXsltElement(corpusElem).getAttribute("xsl")
+		xslParams = params.getXsltParams(corpusElem);
+
+		lang = corpusElem.getAttribute("lang");
+		
+		wordTag = module.getParameters().getWordElement().getTextContent()
+		this.skipDoTokenizeStep = module.getParameters().getSkipTokenization()
+
+		//prepare metadata if any
+		File allmetadatafile = new File(inputDirectory, "metadata.csv");
+		if (allmetadatafile.exists()) {
+			File copy = new File(binDir, "metadata.csv")
+			if (!FileCopy.copy(allmetadatafile, copy)) {
+				println "Error: could not create a copy of the metadata file "+allmetadatafile.getAbsoluteFile();
+				return;
+			}
+			metadata = new Metadatas(copy,
+					Toolbox.getParam(Toolbox.METADATA_ENCODING),
+					Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR),
+					Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+		}
+
+		String cleanDirectories = module.getParameters().getKeyValueParameters().get(ImportKeys.CLEAN);
+
+		if (!doSplitMergeXSLStep()) return;
+		if (!doFrontXSLStep()) return;
+		if (!doCheckXMLFileStep()) return;
+		if (!doTokenizeStep()) return;
+		if (!doPostTokenizeXSLStep()) return;
+		if (!doEncodeMileStonesStep()) return;
+		if (!doToXMLTXMStep()) return;
+		if (!doInjectMetadataStep()) return;
+
+		isSuccessFul = filesToProcess.size() > 0
+
+		if ("true".equals(cleanDirectories)) {
+			new File(module.getBinaryDirectory(), "tokenized").deleteDir()
+			new File(module.getBinaryDirectory(), "src").deleteDir()
+		}
+	}
+
+	/**
+	 * read from source directory and write the result in $bindir/txm
+	 * 
+	 * only one XSL is applied
+	 *
+	 */
+	public boolean doSplitMergeXSLStep() {
+		File srcDirectory = new File(module.getBinaryDirectory(), "src")
+		srcDirectory.mkdir()
+		xslParams["output-directory"] = srcDirectory.getAbsoluteFile().toURI().toString();
+
+		File xslDirectory = new File(module.getSourceDirectory(), "xsl/1-split-merge")
+		println "-- Split-Merge XSL Step with $xslDirectory"
+		def xslFiles = xslDirectory.listFiles()
+		if (xslDirectory.exists() && xslFiles != null && xslFiles.size() > 0) {
+			
+			xslFiles.sort()
+			for (File xslFile : xslFiles) {
+				if (xslFile.isDirectory() || xslFile.isHidden() || !xslFile.getName().endsWith(".xsl")) continue;
+				//if (!xslFile.getName().matches("[1-9]-.+")) continue;
+				
+				if (ApplyXsl2.processImportSources(xslFile, inputDirectory, srcDirectory, xslParams, true)) {
+					println ""
+				} else {
+					reason = "Fail to apply split-merge XSL: $xslPath"
+				}
+				break; // THERE IS ONLY ONE XSL FILE TO APPLY
+			}
+			inputDirectory = srcDirectory; // the files to process are now in the "src" directory
+		} else {
+			//println "Nothing to do."
+		}
+		return true;
+	}
+	
+	public boolean doEncodeMileStonesStep() {
+		filesToProcess = new File(module.getBinaryDirectory(), "tokenized").listFiles()
+		def milestonesString = module.getParameters().getMilestonesElement().getTextContent().trim()
+		if (milestonesString.length() == 0) return true;
+		
+		def milestones = milestonesString.split(",")
+		//def milestones = ["lb", "pb", "cb"]
+		//println "milestones=$milestones type="+milestones.getClass()+" size="+milestones.size()
+		if (milestones.size() == 0) return true;
+		
+		println "-- Encoding milestone $milestones into XML files..."
+		ConsoleProgressBar cpb = new ConsoleProgressBar(milestones.size())
+		for (def milestone : milestones) {
+			milestone = milestone.trim()// just in case
+			cpb.tick()
+			
+			for (File inputFile : filesToProcess) {
+				File outputFile = File.createTempFile("tmp", inputFile.getName(),inputFile.getParentFile());
+				MileStoneProjection msp = new MileStoneProjection(inputFile, "body", wordTag, milestone)
+				if (!msp.process(outputFile)) {
+					println "Fail to encode $milestone in $inputFile"
+					return false
+				} else {
+					if (inputFile.delete()) {
+						outputFile.renameTo(inputFile)
+					} else {
+						println "Fail to encode $milestone in ${inputFile}: could not replace the file."
+						return false
+					}
+				}
+			}
+		}
+		println ""
+		return true;
+	}
+
+	/**
+	 * read from $inputDirectory and write the result in $bindir/txm 
+	 *
+	 */
+	public boolean doFrontXSLStep() {
+
+		//filesToProcess = inputDirectory.listFiles();
+
+		File frontXSLdirectory = new File(module.getSourceDirectory(), "xsl/2-front")
+		println "-- Front XSL Step with $frontXSLdirectory"
+		def xslFiles = frontXSLdirectory.listFiles()
+		if (frontXSLdirectory.exists() && xslFiles != null && xslFiles.size() > 0) {
+			
+			xslFiles.sort()
+			for (File xslFile : xslFiles) {
+				if (xslFile.isDirectory() || xslFile.isHidden() || !xslFile.getName().endsWith(".xsl")) continue;
+				//if (!xslFile.getName().matches("[1-9]-.+")) continue;
+
+				if (ApplyXsl2.processImportSources(xslFile, inputDirectory, outputDirectory, xslParams)) {
+					inputDirectory = outputDirectory; // the files to process are now in the "txm" directory
+					println ""
+				} else {
+					reason = "Fail to apply front XSL: $xslPath"
+					return false;
+				}
+			}
+		} else {
+			//println "Nothing to do."
+		}
+		return true;
+	}
+
+	public boolean doCheckXMLFileStep() {
+		filesToProcess = []
+		println "-- Check XML files for well-formedness."
+		def files = inputDirectory.listFiles()
+
+		if (files == null || files.size() == 0) {
+			println "No XML file (*.xml) to process. Aborting"
+			return false;
+		}
+		files = files.sort()
+
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File f : files) {
+			cpb.tick()
+			if (f.getName().toLowerCase().endsWith(".xml"))
+				if (!f.isHidden() && f.canRead() && !f.getName().endsWith(".properties")
+				&& !f.getName().startsWith("metadata") && !f.isDirectory())
+			{
+				
+				if (!ValidateXml.test(f)) {
+					continue;
+				} else {
+					if (!f.getName().equals("import.xml"))
+						filesToProcess << f
+				}
+			}
+		}
+		println ""
+		if (filesToProcess.size() == 0) {
+			reason = "No file suitable to process in "+inputDirectory
+			return false;
+		} else {
+			return true;
+		}
+	}
+
+
+	public boolean doTokenizeStep() {
+
+		println "-- Tokenizing "+filesToProcess.size()+" files"
+		new File(module.getBinaryDirectory(),"tokenized").mkdir()
+
+		String outSideTextTagsRegex = "";
+		String outSideTextTagsAndKeepContentRegex = "";
+		String noteRegex = "";
+		// get the element names to ignore
+		Element textualPlanEelement = module.getParameters().getTextualPlans();
+
+		if (textualPlanEelement != null) {
+			Element e1 = module.getParameters().getOutSideTextTagsElement()
+			def split  = e1.getTextContent().split(",")
+			for (String s : split) {
+				outSideTextTagsRegex += "|"+s.trim()
+			}
+			if (outSideTextTagsRegex.trim().length() > 0) {
+				outSideTextTagsRegex = outSideTextTagsRegex.substring(1) // remove the first "|"
+			}
+			
+			Element e2 = module.getParameters().getOutSideTextTagsAndKeepContentElement()
+			def split2  = e2.getTextContent().split(",")
+			for (String s : split2) {
+				outSideTextTagsAndKeepContentRegex += "|"+s.trim()
+			}
+			if (outSideTextTagsAndKeepContentRegex.trim().length() > 0) {
+				outSideTextTagsAndKeepContentRegex = outSideTextTagsAndKeepContentRegex.substring(1) // remove the first "|"
+			}
+			
+			Element e3 = module.getParameters().getNoteElement()
+			def split3  = e3.getTextContent().split(",")
+			for (String s : split3) {
+				noteRegex += "|"+s.trim()
+			}
+			if (noteRegex.trim().length() > 0) {
+				noteRegex = noteRegex.substring(1) // remove the first "|"
+			}
+		}
+
+		//if (wordTag != "w") {
+		if (skipDoTokenizeStep ) {
+			println "No tokenization do to."
+			// ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())
+			for (File f : filesToProcess) {
+				File outfile = new File(module.getBinaryDirectory(),"tokenized/"+f.getName());
+				FileCopy.copy(f, outfile);
+			}
+			return true;
+		} else {
+			ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())
+			for (File f : filesToProcess) {
+				cpb.tick()
+				File infile = f;
+				File outfile = new File(module.getBinaryDirectory(),"tokenized/"+f.getName());
+				SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)
+				if (wordTag.length() > 0)
+					tokenizer.setWord_tags(wordTag);
+
+				if (outSideTextTagsRegex != null && outSideTextTagsRegex.trim().length() > 0) {
+					tokenizer.setOutSideTextTags(outSideTextTagsRegex)
+				}
+				
+				if (outSideTextTagsAndKeepContentRegex != null && outSideTextTagsAndKeepContentRegex.trim().length() > 0) {
+					tokenizer.setOutSideTextTagsAndKeepContent(outSideTextTagsAndKeepContentRegex)
+				}
+				
+				if (noteRegex != null && noteRegex.trim().length() > 0) {
+					tokenizer.setNote(noteRegex)
+				}
+
+				// tokenize ! \o/
+				if (!tokenizer.process()) {
+					println("Failed to process "+f)
+					outfile.delete()
+				}
+			}
+		}
+		
+		filesToProcess = new File(module.getBinaryDirectory(),"tokenized").listFiles()
+		println ""
+
+		return true;
+	}
+
+	/**
+	 * read from $bindir/tokenized and write the result in $bindir/tokenized
+	 *
+	 */
+	public boolean doPostTokenizeXSLStep() {
+
+		filesToProcess = new File(module.getBinaryDirectory(),"tokenized").listFiles()
+		File xslDirectory = new File(module.getSourceDirectory(), "xsl/3-posttok")
+		def xslFiles = xslDirectory.listFiles()
+		if (!xslDirectory.exists() || xslFiles == null || xslFiles.size() == 0) return true;
+
+		println "-- Posttokenize XSL Step with $xslDirectory"
+		return ApplyXsl2.processWithMultipleXSL(filesToProcess, xslDirectory, xslParams)
+	}
+
+	public boolean doInjectMetadataStep() {
+		if (metadata != null) {
+			println("-- INJECTING METADATA - "+metadata.getHeadersList()+" in texts of directory "+outputDirectory)
+			ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())
+			for (File infile : filesToProcess) {
+				cpb.tick()
+				File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+				if (metadata.injectMetadatasInXml(infile, outfile, "text", null)) {
+					if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+					if (!infile.exists()) {
+						println "Error: could not replace $infile by $outfile"
+						return false;
+					}
+				} else {
+					outfile.delete(); // fail
+				}
+			}
+			println ""
+		}
+		return true;
+	}
+
+	public boolean doToXMLTXMStep() {
+		filesToProcess = new File(module.getBinaryDirectory(),"tokenized").listFiles()
+		ConsoleProgressBar cpb = new ConsoleProgressBar(filesToProcess.size())
+		println("-- Building XML-TXM ("+filesToProcess.size()+" files)")
+		for (File f : filesToProcess) {
+			//ArrayList<String> milestones = new ArrayList<String>();
+			cpb.tick()
+			File file = f;
+			String txmfile = f.getName();
+
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			//lance le traitement
+			def builder = new Xml2Ana(file);
+			builder.setWordTag(module.getParameters().getWordElement().getTextContent())
+			builder.setConvertAllAtrtibutes true;
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			//builder.setAddTEIHeader();
+			if (!builder.process(new File(outputDirectory, txmfile))) {
+				println("Failed to process "+f);
+				new File(outputDirectory,txmfile).delete();
+			}
+		}
+		filesToProcess = outputDirectory.listFiles();
+		println ""
+		return true;
+	}
+
+	@Override
+	public void checkFiles() {
+		//isSuccessFul = isSuccessFul && outputDirectory.listFiles() != null;
+	}
+
+	@Override
+	public void cancel() {
+		cancelNow = true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/NiceToXML.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/NiceToXML.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/NiceToXML.groovy	(revision 321)
@@ -0,0 +1,202 @@
+package org.txm.importer
+
+import javax.xml.stream.XMLStreamException
+import org.apache.tools.ant.types.resources.selectors.InstanceOf;
+import groovy.xml.*
+
+def root = new File("/home/mdecorde/xml/temoignagesnice/corpus Matrice - fichiers xmlisés/")
+File srcdir = new File(root, "orig");
+File outdir = new File(root, "tmp");
+File okdir = new File(root, "ok");
+File ok2dir = new File(root, "ok2");
+File temoignagedir = new File(root, "temoignages");
+ok2dir.deleteDir()
+ok2dir.mkdir()
+
+//rename title -> head
+for (def file : outdir.listFiles()) {
+	if (!file.getName().endsWith(".xml")) continue;
+	def doc = new XmlParser().parse(file);
+	
+	for (def note : doc.body.chapter.title) {
+		println note
+		note.name = "head"
+	}
+	
+	new File(ok2dir, file.getName()).withWriter("UTF-8") { writer ->
+		new XmlNodePrinter(new PrintWriter(writer)).print(doc)
+	}
+}
+
+/*
+// ADD chapter@title
+for (def file : outdir.listFiles()) {
+	if (!file.getName().endsWith(".xml")) continue;
+	def doc = new XmlParser().parse(file);
+	
+	for (def chapter : doc.body.chapter) {
+		for (def title : chapter.title) {
+			chapter.@title = title.text()
+			break;
+		}
+	}
+	
+	new File(ok2dir, file.getName()).withWriter("UTF-8") { writer ->
+		new XmlNodePrinter(new PrintWriter(writer)).print(doc)
+	}
+}
+*/
+//FIX figure and caption inclusions
+/*
+for (def file : outdir.listFiles()) {
+	if (!file.getName().endsWith(".xml")) continue;
+	def doc = new XmlParser().parse(file);
+	for (def note : doc.body."**".figure) {
+		note.name = "note"
+		//println "fig : $note"
+		//if ("Image :" == note.text()) {
+			def children = note.parent().children()
+			int i = children.indexOf(note)
+			//println i + " < "+children.size()
+			def nextChild = children[i+1]
+			if (nextChild != null && nextChild.name().toString() == "caption") {
+				println nextChild
+
+				note.value = "Images : "+note.text()// + " "+nextChild.text()
+				//println note
+
+				children.remove(i+1)
+				note.append(nextChild)
+			}
+		//}
+	}
+
+	new File(ok2dir, file.getName()).withWriter("UTF-8") { writer ->
+		new XmlNodePrinter(new PrintWriter(writer)).print(doc)
+	}
+}
+*/
+/*
+// DOCBOOK -> DOCBOOK TEXT ONLY
+for (def file : outdir.listFiles()) {
+	def doc = new XmlParser().parse(file);
+
+	def body = null
+	def bookinfo = null
+	def preface = null
+	for (def e : doc.body) body = e
+
+	for (def e : doc.bookinfo) {
+		doc.remove(e)
+	}
+	for (def e : doc.preface) {
+		doc.remove(e)
+	}
+	for (def e : doc.appendix) {
+		doc.remove(e)
+	}
+	for (def e : doc.chapter) {
+		doc.remove(e)
+	}
+
+	if (body == null) {
+		println "error text: "+file
+		continue
+	}
+
+	new File(ok2dir, file.getName()).withWriter("UTF-8") { writer ->
+		new XmlNodePrinter(new PrintWriter(writer)).print(doc)
+	}
+	//		writer.print XmlUtil.serialize(new StreamingMarkupBuilder().bind {
+	//			mkp.yield body
+	//		  })
+}
+*/
+//DOCBOOK to TEI
+/*
+ for (def file : outdir.listFiles()) {
+ def doc = new XmlParser().parse(file);
+ def body = null
+ //def bookinfo = null
+ for (def e : doc.body) body = e
+ //for (def e : doc.bookinfo) bookinfo = e
+ //println body.getClass()
+ if (body == null) {
+ println "error text: "+file
+ continue
+ }
+ //	bookinfo.name = "teiHeader"
+ body.name = "text"
+ def teins = new groovy.xml.Namespace("http://www.tei-c.org/ns/1.0",'tei')
+ //	for (def node : body."**") {
+ //		if (node instanceof String) continue
+ //		def name = node.name()
+ //		if (name instanceof String)
+ //			node.name = teins.get(name)
+ //		else 
+ //			node.name = teins.get(name.getLocalPart())
+ //	}
+ for (def figure : body."**".figure) {
+ figure.name = "note"
+ figure.value = "Image : " + figure.caption.text()
+ }
+ for (def chapter : body."**".chapter) {
+ chapter.name = "div"
+ chapter.@type = "chapter"
+ }
+ for (def caption : body."**".title) {
+ caption.name = "head"
+ }
+ for (def para : body."**".para) {
+ para.name = "p"
+ }
+ def newdoc = new Node(null, "TEI");
+ newdoc.@xmlns="http://www.tei-c.org/ns/1.0";
+ newdoc.append(new Node(null, "teiHeader"))
+ newdoc.append(body)
+ new File(okdir, file.getName()).withWriter("UTF-8") { writer ->
+ new XmlNodePrinter(new PrintWriter(writer)).print(newdoc)
+ }
+ //		writer.print XmlUtil.serialize(new StreamingMarkupBuilder().bind {
+ //			mkp.yield body
+ //		  })
+ }
+ */
+
+// remove TEI
+/*outdir.deleteDir()
+ outdir.mkdir()
+ def errors = []
+ for (def file : srcdir.listFiles()) {
+ if (file.isDirectory()) continue;
+ //new EncodingConverter(file, "Windows-1252", "UTF-8")
+ File outfile = new File(outdir, file.getName());
+ outfile.withWriter("UTF-8") { writer ->
+ file.eachLine("UTF-8") { line ->
+ if (line.trim() == "<TEI>") {
+ } else if (line.trim() == "</TEI>") {
+ writer.println("</book>")
+ } else if (line.trim() == "<book lang=\"fr\"/>") {
+ writer.println("<book lang=\"fr\">")
+ } else {
+ writer.println(line)
+ }
+ }		
+ }
+ try {
+ ValidateXml.testAndThrow(outfile);
+ } catch (XMLStreamException e) {
+ println file.getName() + " : "+ e.getMessage()
+ errors << file
+ if (e.getMessage().contains('Message: The element type "TEI" must be terminated by the matching end-tag "</TEI>"')) {
+ println "Delete line : "+e.location.lineNumber
+ }
+ println ""
+ }
+ }
+ */
+println "done"
+//if (errors.size() > 0)
+//	println ""+errors.size()+" errors : $errors"
+//String content = file.getText("Windows-1252")
+//println content
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/compiler.groovy	(revision 321)
@@ -0,0 +1,387 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.hyperprince
+;
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler 
+{
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private def parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * Instantiates a new compiler.
+	 */
+	public compiler(){}
+	
+	/**
+	 * initialize.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8")
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * clear anaHash variable, it is used to store ana tags values then print it when the end element </ana> is found.
+	 */
+	private void fillanaHash()
+	{
+		anahash.clear();
+		for(String s : types)
+		anahash.put( s,"-" );
+	}
+	
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @param idtext the idtext
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName,idtext)
+	{
+		createOutput(dirPathName, fileName);
+		
+		String author = "";
+		String date = "";
+		String lang = "";
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		
+		boolean flagAuthor = false;
+		boolean flagDate = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+		output.write("<txmcorpus lang=\""+lang+"\">\n"); 
+		try 
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+						
+						case "monogr"://get attr lang
+						lang = parser.getAttributeValue(null,"lang");
+						//idtext = parser.getAttributeValue(null,"id");
+						break;
+						
+						case "author"://get text
+						flagAuthor = true;
+						author ="";
+						break;
+						
+						case "date"://get text
+						flagDate = true;
+						date = "";
+						break;
+						
+						case "text"://write tag+ lang + date + author
+						output.write("<text lang=\""+lang+"\" auteur=\""+author+"\" date=\""+date+"\" id=\""+idtext+"\" base=\""+base+"\" project=\""+project+"\">\n");
+						break;
+						
+						case "front"://get type,id, write tag
+						case "back"://get type,id, write tag
+						case "head"://get type,id, write tag
+						case "seg"://get type,id,n write tag
+						output.write("<"+parser.getLocalName()+" id=\""+parser.getAttributeValue(null,"id")+"\" type=\""+parser.getAttributeValue(null,"type")+"\">\n");
+						break;
+						
+						case "body"://get type,id, write tag BODY
+						//	for(int i = 0 ; i < parser.getAttributeCount(); i++)
+								
+						output.write("<body id=\""+parser.getAttributeValue(null,"id")+"\" type=\""+parser.getAttributeValue(null,"type")+"\">\n");
+						break;
+						
+						case "s":
+						output.write( "<s>\n");
+						break;
+						
+						case "w":
+						wordid = parser.getAttributeValue(null,"id")
+						break;
+						case "form":
+						flagForm = true;
+						vForm = "";
+						vAna ="";
+						break;
+						
+						case "ana":
+						flagAna = true;
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.END_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+						case "author"://get text
+						flagAuthor = false;
+						break;
+						
+						case "date"://get text
+						flagDate = false;
+						break;
+						
+						case "text"://write tag+ lang + date + author
+						output.write("</text>\n");
+						break;
+						
+						case "front"://get type,id, write tag
+						case "back"://get type,id, write tag
+						case "head"://get type,id, write tag
+						case "seg"://get type,id,n write tag
+						output.write("</"+parser.getLocalName()+">\n");
+						break;
+						
+						case "div0"://get type,id, write tag BODY
+						output.write("</body>\n");
+						break;
+						
+						case "s":
+						output.write( "</s>\n");
+						break;
+						
+						case "w":
+						output.write( vForm +vAna+"\t"+wordid+"\n");
+						vAna = "";
+						vForm = "";
+						break;
+						
+						case "form":
+						flagForm = false;
+						break;
+						
+						case "ana":
+						flagAna = false;
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.CHARACTERS:
+					if(flagForm)
+					vForm += parser.getText().trim();
+					if(flagAna)
+					vAna += "\t" +parser.getText().trim();
+					if(flagDate)
+					date += parser.getText().trim();
+					if(flagAuthor)
+					author += parser.getText().trim();
+					break;
+				}
+			}
+			output.write("</txmcorpus>"); 
+			output.close();
+			parser.close();
+			inputData.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+		
+		return true;
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDir the root dir
+	 */
+	public boolean run(String rootDir) 
+	{
+		//String rootDir = "~/xml/Hyperprince/";
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		new File(rootDir,"wtc/hyperprince.wtc").delete();//cleaning&preparing
+		new File(rootDir,"wtc/").deleteDir();
+		new File(rootDir,"wtc/").mkdir();
+		new File(rootDir,"registry/").mkdir();
+		
+		//1- Transform into WTC file
+		List<File> files = new File(rootDir,"txm").listFiles();
+		for(File f : files) {
+			if(!f.exists()) {
+				println("file "+f+ " does not exists")	
+			}
+			else
+			{	
+				println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				def builder = new compiler(f.toURL(),txtname, "hyperprince", "default");
+				builder.setLang lang
+				builder.transfomFileWtc(rootDir,"wtc/hyperprince.wtc",f.getName());
+			}
+		}
+		
+		//2- Import into CWB
+		def outDir =rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug true;
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug true;
+		
+		String[] pAttributes = ["pos","lemme","id"];
+		String[] sAttributes = ["txmcorpus:0+lang", "text:0+auteur+date+id+lang+base+project","front:0+type+id","body:0+type+id","back:0+type+id","head:0+type+id", "seg:0+type+n+id"];
+		
+		try {
+			String regPath =outDirTxm + "/registry/"+"hyperprince"
+			cwbEn.run(cwbLoc + "/cwb-encode", outDirTxm + "data", outDir + "/wtc/"+"hyperprince.wtc", regPath,pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(cwbLoc + "/cwb-makeall", "HYPERPRINCE", outDirTxm + "/registry");
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		
+		System.out.println("Done.") 
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Hyperprince import module : multi-ling corpus</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/pager.groovy	(revision 321)
@@ -0,0 +1,459 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.hyperprince;
+
+import org.txm.objects.*;
+import org.txm.importer.XPathResult;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+import org.txm.utils.i18n.*;
+
+// TODO: Auto-generated Javadoc
+/** Build Discours corpus simple edition from a xml-tei. @author mdecorde */
+class pager {
+	List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+	
+	/** The pagecount. */
+	int pagecount = 0;
+	
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The wordid. */
+	String wordid;
+	
+	/** The first word. */
+	boolean firstWord = true;
+	
+	/** The wordvalue. */
+	String wordvalue;
+	
+	/** The interpvalue. */
+	String interpvalue;
+	
+	/** The lastword. */
+	String lastword = " ";
+	
+	/** The wordtype. */
+	String wordtype;
+	
+	/** The flagform. */
+	boolean flagform = false;
+	
+	/** The flaginterp. */
+	boolean flaginterp = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	def writer;
+	
+	/** The multiwriter. */
+	def multiwriter = null;
+	
+	/** The infile. */
+	File infile;
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+	
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	
+	/** The segid. */
+	String segid;
+	
+	/** The title. */
+	String title;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 */
+	pager(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter, int max) {
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		try {
+			
+			if(multiwriter != null)
+			{
+				multiwriter.write("</body>");
+				multiwriter.write("</html>");
+				multiwriter.close();
+			}
+			pagecount++;
+			File f = new File(outfile.getParent()+"/multi/",outfile.getName().substring(0,2)+"_"+"page_"+pagecount+".html");
+			pages.add(f);
+			idxstart.add(wordid)
+			multiwriter = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<head>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			
+			if(segid != null)
+			{
+				multiwriter.write("<title>Hyperprince "+segid+"</title>");
+			}else {
+				multiwriter.write("<title>Hyperprince "+title+"</title>");
+			}
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			this.outfile = outfile;
+
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+			createNextOutput();
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process()
+	{
+		String localname = "";
+		String query = "//tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:author/text()";
+		title = XPathResult.getXpathResponse(infile, query);
+		
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+		writer.write("<html>");
+		writer.write("<head>");
+		writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.write("<title>Hyperprince "+title+"</title>");
+		writer.write("</head>");
+		writer.write("<body>");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{	
+						case "text":
+							//XPathResult testG = new XPathResult(infile);
+							
+							//String title = testG.getXpathResponse("//tei:TEI/tei:teiHeader/tei:fileDesc/tei:titleStmt/tei:title","")
+							//testG = null;
+							writer.write("<h1>"+title+"</h1>\n")
+							multiwriter.write("<h1>"+title+"</h1>\n")
+							break;
+						
+						case "front":
+							writer.write("<h2>"+parser.getAttributeValue(null, "type")+"</h2>\n")
+							multiwriter.write("<h2>"+parser.getAttributeValue(null, "type")+"</h2>\n")
+							break;
+						
+						case "div0":
+							writer.write("<h2>Book</h2>\n")
+							multiwriter.write("<h2>Book</h2>\n")
+							break;
+						
+						case "back":
+							writer.write("<h2>"+parser.getAttributeValue(null, "type")+"</h2>\n")
+							multiwriter.write("<h2>"+parser.getAttributeValue(null, "type")+"</h2>\n")
+							break;
+						
+						case "seg":
+							segid=parser.getAttributeValue(null,"id");
+							createNextOutput();
+							
+							writer.write("<p>\n");
+							multiwriter.write("<p>\n");
+							break;
+						
+						case "w":
+							wordid=parser.getAttributeValue(null,"id");
+							if(firstWord)
+							{
+								firstWord=false;
+								this.idxstart.set(0,wordid);
+							}
+							/*wordcount++;
+							if(wordcount >= wordmax)
+							{
+								createNextOutput();
+								wordcount=0;
+							}*/
+							break;
+						
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type")+":"
+							break;
+						
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+							break;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "teiHeader":
+						
+							break;
+						
+						case "seg":
+							writer.write("</p>\n");
+							break;
+						
+						case "form":
+							flagform = false
+						
+							break;
+						
+						case "ana":
+							flaginterp = false
+							break;
+						
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if(l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+							
+							if(interpvalue != null)
+								interpvalue = interpvalue.replace("\"","&quot;");
+							
+							if(NoSpaceBefore.contains(wordvalue) || 
+									NoSpaceAfter.contains(lastword) || 
+									wordvalue.startsWith("-") || 
+									NoSpaceAfter.contains(endOfLastWord))
+							{
+								writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							else
+							{
+								writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							writer.write(wordvalue.replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+							break;
+						
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		writer.close();
+		multiwriter.write("</body>");
+		multiwriter.write("</html>");
+		multiwriter.close();
+		
+		parser.close();
+		inputData.close();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		/*String rootDir = System.getProperty("user.home")+"/xml/hyperprince/";
+		new File(rootDir,"HTML/").deleteDir();
+		new File(rootDir+"HTML/").mkdir();
+		new File(rootDir+"HTML/multi/").mkdir();
+		List<File> filelist = new File(rootDir,"txm").listFiles();
+		
+		for(File srcfile : filelist)
+		{
+			File resultfile = new File(rootDir+"HTML",srcfile.getName().substring(0,2)+".html");
+			List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+			List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+			println("build discours xml-tei file : "+srcfile+" to : "+resultfile );
+			
+			def builder = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,1500);
+		}
+		
+		return;*/
+		
+		
+		String userDir = System.getProperty("user.home");
+		String lang= "fr"
+		String rootDir;
+		println "DEV MODE";//exception means we debug
+		if(!org.txm.Toolbox.isInitialized()){
+			rootDir = "D:\\Travail_Sev\\Projets\\Corpus\\TXM\\hyperprince"; //userDir+"/xml/hyperprince/";//"D:\\Travail_Sev\\Projets\\Corpus\\TXM\\hyperprince";
+			org.txm.Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+			org.txm.Toolbox.setParam(org.txm.Toolbox.INSTALL_DIR,new File(userDir,"TXM"));
+		}
+		
+		Workspace w = org.txm.Toolbox.workspace;
+		Project p = w.getProject("default")
+		p.removeBase("hyperprince")
+		Base b = p.addBase("hyperprince");
+		b.addDirectory(new File(rootDir,"txm"));
+		
+		
+		println "-- EDITION"
+		new File(rootDir+"/HTML/").deleteDir();
+		new File(rootDir+"/HTML/").mkdir();
+		new File(rootDir,"/HTML/multi").mkdir();
+		List<File> filelist = new File(rootDir,"txm").listFiles();
+
+		for(String textname : b.getTextsID())
+		{
+			Text text = b .getText(textname);
+			File srcfile = text.getSource();
+			File resultfile = new File(rootDir+"/HTML",srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+			List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+			List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+			println("build hyperprince xml-tei file : "+srcfile+" to : "+resultfile );
+			
+			def ed = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,500);
+			
+			Edition editionweb = text.addEdition("default","html",resultfile);
+			//println("pages "+ed.getPageFiles())
+			//println("idx "+ed.getIdx())
+			for(int i = 0 ; i < ed.getPageFiles().size();i++)
+			{
+				File f = ed.getPageFiles().get(i);
+				String idx = ed.getIdx().get(i);
+				editionweb.addPage(f,idx);
+			}
+			
+			//Edition (version Pincemin)
+//			Edition editionbp = text.addEdition("onepage","html",resultfile);
+//			editionbp.addPage(resultfile,ed.getIdx().get(0));
+		}
+		w.save();
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/hyperprinceLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/hyperprinceLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/hyperprinceLoader.groovy	(revision 321)
@@ -0,0 +1,137 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.hyperprince;
+
+import org.txm.importer.hyperprince.importer;
+import org.txm.importer.hyperprince.compiler;
+import org.txm.objects.*;
+import org.txm.Toolbox;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		rootDir = userDir+"/xml/hyperprince/";
+		lang="fr";
+		encoding= "UTF-8";
+		model="rgaqcj";
+		Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File(userDir,"TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"TXM/treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"TXM/treetagger/models"));
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+File homedir = new File(rootDir);
+String corpusSrc = "Corpus-Hyperprince_2009-06-10.xml";//"Corpus-Hyperprince_2010-04-06.xml";
+
+//IMPORT TXT
+if(!new File(rootDir,corpusSrc).exists())
+{
+	println("source file : "+new File(rootDir,corpusSrc)+" does not exists")
+	return false;
+}
+
+println "-- IMPORTER - Reading source files"
+List<File> srcfiles = [new File(rootDir,corpusSrc)];
+new importer().run(rootDir, corpusSrc);
+
+//ANNOTATE
+println "-- ANNOTATE - Running NLP tools"
+new Annotate().run(homedir,model+".par");
+
+//COMPILATION
+println "COMPILING"
+def c = new compiler()
+//c.setCwbPath("D:\\Travail_Sev\\Logiciels\\TXM\\cwb\\bin"); // for developers
+c.setLang(lang);
+c.run(rootDir);
+
+//move registry file to cwb registry dir
+File registryfile = new File(rootDir+"/registry","hyperprince");
+if(registryfile.exists())
+	org.txm.utils.io.FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"registry/hyperprince"))
+
+Workspace w = org.txm.Toolbox.workspace;
+Project p = w.getProject("default")
+p.removeBase("hyperprince")
+Base b = p.addBase("hyperprince");
+b.addDirectory(new File(rootDir,"txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION"
+new File(rootDir+"/HTML/").deleteDir();
+new File(rootDir+"/HTML/").mkdir();
+new File(rootDir,"/HTML/default").mkdir();
+List<File> filelist = new File(rootDir,"txm").listFiles();
+
+for(String textname : b.getTextsID())
+{
+	Text text = b .getText(textname);
+	File srcfile = text.getSource();
+	File resultfile = new File(rootDir+"/HTML",srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+	println("build hyperprince xml-tei file : "+srcfile+" to : "+resultfile );
+	
+	def ed = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,500);
+	
+	Edition editionweb = text.addEdition("default","html",resultfile);
+	//println("pages "+ed.getPageFiles())
+	//println("idx "+ed.getIdx())
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+	
+//	Edition editionbp = text.addEdition("onepage","html",resultfile);
+//	editionbp.addPage(resultfile,ed.getIdx().get(0));
+}
+
+w.save()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/importer.groovy	(revision 321)
@@ -0,0 +1,186 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2015-06-03 15:04:53 +0200 (Wed, 03 Jun 2015) $
+// $LastChangedRevision: 2984 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.hyperprince
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import javax.xml.stream.*;
+import java.net.URL;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.TagSentences.*;
+import filters.FusionHeader.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ *
+ * @author mdecorde
+ * simple import in cwb of RGAQCJ
+ * structunits : initial, s, lb ...
+ * wordproperties : form, pos and n
+ */
+
+class importer
+{		
+	
+	/** The align list. */
+	HashSet<String>  alignList;
+	
+	/** The aa. */
+	AnalyzeAlignement aa ;
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDir the root dir
+	 * @param corpusSrc the corpus src
+	 */
+	public void run(String rootDir, String corpusSrc)//List<File> srcfiles = [new File(rootDir,"Corpus-Hyperprince_2009-06-10.xml")];
+	{
+		//String ActionHome = "~/Bureau/trunkToolbox/0.4.6/org.txm.toolbox/GroovyImportScripts/filters";
+		
+		//-0 split, parceque c'est un corpus et ca a pls header partout partout
+		ArrayList<String> milestones = new ArrayList<String>();//the tags which stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		
+		File corpussrc = new File(rootDir,corpusSrc);
+		rootDir = corpussrc.getParent()+"/";
+		
+		new File(rootDir+"ptokenized").deleteDir();
+		new File(rootDir+"ptokenized").mkdir();
+		new File(rootDir+"tokenized").deleteDir();
+		new File(rootDir+"tokenized").mkdir();
+		new File(rootDir+"split").deleteDir();
+		new File(rootDir+"split").mkdir();
+		new File(rootDir+"txm").deleteDir();
+		new File(rootDir+"txm").mkdir();
+		
+		String xslfile = rootDir+"/xsl/splitcorpus.xsl";
+		String outfile = rootDir+"split_temp.xml";
+		
+		aa = new AnalyzeAlignement(corpussrc);
+		alignList = aa.process()
+		ApplyXsl a = new ApplyXsl(xslfile);
+		a.process(corpussrc.getAbsolutePath(),outfile);
+		new File(outfile).delete()
+
+		
+		List<File> files = new File(rootDir,"split").listFiles();//scan directory split
+		//PREPARE EACH FILE TO BE TOKENIZED
+		for(File f : files)
+		{
+			File srcfile = f;
+			//new EncodingConverter(srcfile, "iso-8859-1");
+			File resultfile = new File(rootDir+"ptokenized",f.getName());
+			println("prepare tokenizer file : "+srcfile+" to : "+resultfile );
+			
+			def builder = new OneTagPerLine(srcfile.toURL(), milestones);
+			builder.process(resultfile);
+		}
+		
+		//TOKENIZE FILES
+		//Manager<Filter> filterManager = new FilterManager(ActionHome);
+		files = new File(rootDir,"ptokenized").listFiles();//scan directory split
+		for(File infile : files)
+		{
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F6 = new Tokeniser(infile);
+			Filter F7 = new TagSentences();
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			S.add(F6);
+			//S.add(F7);
+			S.add(F11);
+			
+			//File infile = new File(rootDir+"tokenized",f.getName()+"-src.xml");
+			File xmlfile = new File(rootDir+"tokenized",infile.getName());
+			println("Tokenizing "+xmlfile)
+			S.setEncodages("UTF-8","UTF-8");
+			S.SetInFileAndOutFile(infile.getAbsolutePath(), xmlfile.getAbsolutePath());
+			
+			Object[] arguments1 = [infile.getParent()+"/"+infile.getName()+"header.xml"];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [infile.getParent()+"/"+infile.getName()+"header.xml",F1];
+			F11.SetUsedParam(arguments2);
+			
+			S.proceed();
+		}
+		
+		files = new File(rootDir,"tokenized").listFiles()			
+		//TRANSFORM INTO XML-TEI-TXM
+		for(File f : files)
+		{
+			//ArrayList<String> milestones = new ArrayList<String>();
+			println("Building xml-tei-txm "+f+ " >> "+f.getName()+".xml")
+			File file = f; 
+			String txmfile = f.getName().subSequence(0, 2)+".xml";
+			
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();	
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+					//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			
+			//lance le traitement
+			def builder = new Xml2Ana(file);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			builder.transformFile(rootDir+"txm/",txmfile);
+		}
+		
+		//rename files correctly
+		files = new File(rootDir,"txm").listFiles();
+		for(File file : files)
+		{
+			String txmfile = file.getName();
+			file = aa.addAlignement(file, alignList);
+			txmfile = txmfile.tokenize(".").get(0)+".xml"
+			file.renameTo(new File(file.getParent(),txmfile));
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/AnalyzeAlignement.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/AnalyzeAlignement.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/hyperprince/AnalyzeAlignement.groovy	(revision 321)
@@ -0,0 +1,281 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2015-12-17 12:11:39 +0100 (Thu, 17 Dec 2015) $
+// $LastChangedRevision: 3087 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.hyperprince
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import javax.xml.stream.*;
+
+import org.txm.functions.concordances.comparators.NullComparator;
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class AnalyzeAlignement.
+ */
+class AnalyzeAlignement {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/**
+	 * Instantiates a new analyze alignement.
+	 *
+	 * @param file the file
+	 */
+	public AnalyzeAlignement(File file){
+		try {
+			this.url = file.toURI().toURL();
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			factory.setProperty(XMLInputFactory.IS_VALIDATING,false)
+			parser = factory.createXMLStreamReader(inputData);
+			
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * Process.
+	 *
+	 * @return the list
+	 */
+	public List <Set <String>> process(){
+		List <Set <String>> list = new ArrayList <Set <String>>();
+		
+		String localname = "";
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			String prefix = parser.getPrefix();
+			if(prefix == null || prefix == "")
+				prefix = "";
+			else
+				prefix +=":";
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+				
+				// <link targets="#i1_Pr-Seg0 #f1_Pr-Seg0 #f3_Pr-Seg0 #f5_Pr-Seg0 #f4_Pr-Seg0 "/>
+					if(localname == "link"){
+						for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+						{
+							String att = parser.getAttributeLocalName(i)
+							if(att == "targets"){
+								List<String> vals = parser.getAttributeValue(i).split(" ");
+								HashSet<String> newVals = new HashSet<String>();
+								//vals*.trim(); dans le cas du #
+								vals.each{ it->
+									newVals << it.substring(1);
+								}
+								list.add(newVals);
+							}
+						}
+					}
+			}
+		}
+		parser.close()
+		return list;
+	}
+	
+	/**
+	 * Adds the alignement.
+	 *
+	 * @param file the file
+	 * @param alignIds the align ids
+	 * @return the file
+	 */
+	public File addAlignement(File file, HashSet<String> alignIds){
+		try {
+			this.url = file.toURI().toURL();
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			factory.setProperty(XMLInputFactory.IS_VALIDATING,false)
+			parser = factory.createXMLStreamReader(inputData);
+			
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+		
+		/*
+		 File outfile = new File(outdir,textname+language+".xml");
+		 Writer output = new OutputStreamWriter(new FileOutputStream(outfile) , "UTF-8");
+		 def writer = factory.createXMLStreamWriter(output, "UTF-8")*/
+		String textname = file.getName().substring(0,file.getName().length()-4)+"_";
+		
+		// create XML writer
+		File tempFile = new File(file.getParent(),textname+"temp.xml");
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		FileOutputStream output = new FileOutputStream(tempFile)
+		def writer = factory.createXMLStreamWriter(output, "UTF-8")
+		
+		
+		String localname = "";
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeStartElement ("TEI");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			String prefix = parser.getPrefix();
+			if(prefix == null || prefix == "")
+				prefix = "";
+			else
+				prefix +=":";
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					if(prefix == null){
+						writer.writeStartElement(prefix, localname);
+					}else {
+						writer.writeStartElement(localname);
+					}
+					for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+					{
+						writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+						if("id" == parser.getAttributeLocalName(i)){
+							if((parser.getAttributeValue(i)).startsWith("w")){
+							}else {
+								String idAlign = searchForAlignId(parser.getAttributeValue(i), alignIds);
+								writer.writeAttribute("pid", idAlign);
+								if(idAlign == "-1"){
+									System.out.println(parser.getAttributeValue(i)+" - ALIGN ID is ? "+idAlign);
+								}
+							}
+						}
+					}
+				
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					writer.writeEndElement();
+				
+					break;
+				case XMLStreamConstants.CHARACTERS:
+					writer.writeCharacters(parser.getText());
+					break;
+			}
+		}
+		writer.writeEndElement(); // close TEI
+		parser.close();
+		writer.close();
+		output.close()
+		inputData.close();
+		return copyfile(tempFile.getAbsolutePath(), file.getAbsolutePath());
+
+	}
+	
+	/**
+	 * Copyfile.
+	 *
+	 * @param srFile the sr file
+	 * @param dtFile the dt file
+	 * @return the file
+	 */
+	private static File copyfile(String srFile, String dtFile){
+		File f2 ;
+		try{
+			File f1 = new File(srFile);
+			f2 = new File(dtFile);
+			InputStream inStr = new FileInputStream(f1);
+			
+			//For Append the file.
+			//OutputStream out = new FileOutputStream(f2,true);
+			
+			//For Overwrite the file.
+			OutputStream out = new FileOutputStream(f2);
+			
+			byte[] buf = new byte[1024];
+			int len;
+			while ((len = inStr.read(buf)) > 0){
+				out.write(buf, 0, len);
+			}
+			inStr.close();
+			out.close();
+			System.out.println("File copied.");
+		}
+		catch(FileNotFoundException ex){
+			System.out.println(ex.getMessage() + " in the specified directory.");
+			return null;
+		}
+		catch(IOException e){
+			System.out.println(e.getMessage());      
+		}
+		return f2;
+	}
+	
+	/**
+	 * Search for align id.
+	 *
+	 * @param id the id
+	 * @param alignIds the align ids
+	 * @return the string
+	 */
+	public String searchForAlignId(String id, HashSet<String> alignIds){
+		int i = 0;
+		for(Set<String> set: alignIds){
+			i++;
+			if(set.contains(id)){
+				return ""+i;
+			}
+		}
+		return "-1";
+	}
+	
+	
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/DisplayXmlTags.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/DisplayXmlTags.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/DisplayXmlTags.groovy	(revision 321)
@@ -0,0 +1,183 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.io.File;
+import javax.xml.stream.*;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/** count and display the tags of an xml file. @author mdecorde */
+class DisplayXmlTags {
+	ArrayList<String> paths = new ArrayList<String>(); // contains the xpath of
+	// the tags
+	
+	/** The counts. */
+	HashMap<String, Integer> counts = new HashMap<String, Integer>(); // contains the counts per tag
+	
+	/** The chars. */
+	HashMap<String, Integer> chars = new HashMap<String, Integer>(); // contains the char counts per tag
+
+	/** The currentpath. */
+	String currentpath = "";
+	
+	/** The sum. */
+	public int sum = 0;
+
+	/**
+	 * Instantiates a new display xml tags.
+	 *
+	 * @param infile : the file to parse
+	 */
+	public DisplayXmlTags(File infile) {
+		if (infile.isDirectory()) {
+			for (File f : infile.listFiles()) {
+				processxmlFile(f);
+			}
+		} else
+			processxmlFile(infile);
+	}
+
+	/**
+	 * run the script.
+	 *
+	 * @param xmlfile the xmlfile
+	 * @return true, if successful
+	 */
+	private boolean processxmlFile(File xmlfile)
+	{
+		try
+		{
+			URL url = xmlfile.toURI().toURL();
+			def inputData = url.openStream();
+			def factory = XMLInputFactory.newInstance();
+			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+					
+						currentpath += "/"+parser.getLocalName();// append the current tag to the current path
+						if(!paths.contains(currentpath))
+						{
+							paths.add(currentpath);
+							counts.put(currentpath, 0);
+							chars.put(currentpath, 0);
+						}
+						counts.put(currentpath, counts.get(currentpath)+1); // increment path count
+					
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						currentpath = currentpath.substring(0,currentpath.length() -1 - parser.getLocalName().length()) // remove tag from the path
+						break;
+					case XMLStreamConstants.CHARACTERS:
+						
+						chars.put(currentpath, chars.get(currentpath)+parser.getText().trim().length());
+						sum += parser.getText().trim().length();
+				}
+			}
+			
+		}
+		catch(Exception e){println("File "+xmlfile+"\n"+e); return false;}
+		return true;
+	}
+
+	/**
+	 * Gets the tag hierarchy.
+	 *
+	 * @return the hierarchy of the tags
+	 */
+	public ArrayList<String> getTagHierarchy() {
+		return paths;
+	}
+
+	/**
+	 * return the counts of a tag.
+	 *
+	 * @param path : the tag path (ex : /TEI/text/p")
+	 * @return the count
+	 */
+	public int getCount(String path) {
+		return counts.get(path);
+	}
+
+	/**
+	 * Gets the counts.
+	 *
+	 * @return all the tags counts
+	 */
+	public int getCounts() {
+		return counts;
+	}
+	
+	/**
+	 * return the counts of chars of  a tag.
+	 *
+	 * @param path : the tag path (ex : /TEI/text/p")
+	 * @return the char
+	 */
+	public int getChar(String path) {
+		return chars.get(path);
+	}
+
+	/**
+	 * Gets the chars.
+	 *
+	 * @return all the tags counts
+	 */
+	public int getChars() {
+		return chars;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String userhome = System.getProperty("user.home"); 
+		DisplayXmlTags diag = new DisplayXmlTags(new File(userhome, "xml/manuelTXM/Manuel_TEI_FR_0_5.xml"));
+		ArrayList<String> paths = diag.getPaths();
+		Collections.sort(paths);
+		if(paths != null)
+			for(String s : paths)
+			{
+				print s+" : "+diag.getCount(s);
+				if(diag.getChar(s) > 0)
+					print " (chars "+diag.getChar(s)+")";
+					
+				println ""
+			}
+		
+		println "total chars : "+diag.sum;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/compiler.groovy	(revision 321)
@@ -0,0 +1,533 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $
+//
+
+
+package org.txm.importer.xmltxm;
+
+import java.util.ArrayList;;
+
+import org.txm.*;
+import org.txm.importer.cwb.BuildCwbEncodeArgs;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler
+{
+	String sortMetadata;
+	/** The debug. */
+	private boolean debug= false;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private Writer output;
+
+	/** The url. */
+	private def url;
+
+	/** The text. */
+	String text="";
+
+	/** The base. */
+	String base="";
+
+	/** The project. */
+	String project="";
+
+	/** The cwb loc. */
+	String cwbLoc;
+
+	/** The lang. */
+	private String lang ="fr";
+
+	/** The s attribs. */
+	private static HashMap<String, List<String>> sAttribs;
+
+	/** The anatypes. */
+	private static anatypes = []
+	private static anavalues = [:]
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (Exception ex) {
+			System.out.println("Error while creating indexes: $ex");
+			ex.printStackTrace();
+		}
+	}
+
+	public void setSortMetadata(String sortMetadata)
+	{
+		this.sortMetadata = sortMetadata;
+	}
+
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f) {
+		try {
+			//File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f,f.exists())) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+		println "Error while create WTC otput file: "+e
+			e.printStackTrace();
+
+			return false;
+		}
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private boolean GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.END_ELEMENT)
+				if (parser.getLocalName().equals("teiHeader") || parser.getLocalName().equals("teiheader")) {
+					return true;
+				}
+		}
+		return false;
+	}
+
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(File wtcFile)
+	{
+		createOutput(wtcFile);
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+		String anatype = null;
+		String anavalue = null;
+		int p_id = 0;
+		int s_id = 0;
+
+		boolean captureword = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+		boolean inW = false;
+		int wcounter = 1;
+		if (!GoToText()) {
+			println "Error: no teiHeader tag found in text '"+this.text+"' (please check file format or content)"
+			return false;
+		}
+		int wc = 0; // TEMP FOR TEST ONLY
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						//println "start: "+parser.getLocalName()
+						switch (parser.getLocalName()) {
+							case "w":
+							
+							inW = true;
+							anavalues = [:]
+							wordid = parser.getAttributeValue(null, "id")
+							if (wordid == null)
+								wordid = "w_"+text+"_"+(wcounter++)
+
+							vAna ="";
+							break;
+
+							case "form":
+							String type2 = parser.getAttributeValue(null, "type");
+							if(type2 == null || type2.equals("default")) {
+								flagForm = true;
+								vForm = "";
+							} else {
+								flagAna = true;
+								vAna += "\t";
+								if(!anatypes.contains(type2))
+									anatypes << type2;
+							}
+							
+							break;
+
+							case "ana":
+							flagAna = true;
+							anavalue = "";
+							anatype = parser.getAttributeValue(null, "type");
+							if (anatype != null) {
+								if(anatype.startsWith("#"))
+									anatype = anatype.substring(1)
+								break;
+							}
+							break;
+
+							default:
+							if (!inW) {
+								output.write("<"+parser.getLocalName().toLowerCase());
+								if (!sAttribs.containsKey(parser.getLocalName()))
+									sAttribs.put(parser.getLocalName().toLowerCase(), []);
+
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									String attrname = parser.getAttributeLocalName(i).toLowerCase();
+									String attrvalue = parser.getAttributeValue(i);
+									if (!(parser.getLocalName() == "text" && attrname == "id"))
+										output.write(" "+attrname+"=\""+attrvalue.replace("\"", "'")+"\"");
+
+									if (!sAttribs.get(parser.getLocalName().toLowerCase()).contains(attrname))
+										sAttribs.get(parser.getLocalName().toLowerCase()).add(attrname)
+								}
+
+								if (parser.getLocalName() == "text") {
+									output.write(" id=\""+text+"\" base=\""+base+"\" project=\""+project+"\"");
+								}
+								output.write(">\n");
+							}
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						switch (parser.getLocalName()) {
+							case "TEI":
+								break;
+							case "w":
+							for (String t : anatypes) {
+								def v = anavalues.get(t);
+								if (v != null) vAna +="\t"+v;
+								else vAna +="\t";
+							}
+							
+							output.write( vForm.replaceAll("&", "&amp;").replaceAll("<", "&lt;") +"\t"+wordid+vAna+"\n");
+							vAna = "";
+							vForm = "";
+							inW = false;
+							break;
+
+							case "form":
+							flagForm = false;
+							flagAna = false;
+							break;
+
+							case "ana":
+							anavalues.put(anatype, anavalue)
+							flagAna = false;
+							break;
+
+							default:
+							if(!inW)
+								output.write("</"+parser.getLocalName().toLowerCase()+">\n");
+
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						if(inW)
+						{	
+							if(flagForm) {	
+								vForm += parser.getText().trim();
+							}
+							else if (flagAna) {
+								anavalue += parser.getText().trim();
+							}
+						}
+						break;
+				}
+			}
+
+			output.close();
+			parser.close();
+		}
+		catch (Exception ex) {
+			System.out.println("Error while writing WTC file $ex");
+			ex.printStackTrace();
+			return false;
+		}
+		return true;
+	}
+
+
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String basename, String corpusname, List<File> files)
+	{
+		anatypes = new ArrayList<String>();// init only 1 time
+		anavalues = [:]
+		sAttribs = new HashMap<String, List<String>>();// init only 1 time
+		String rootDir = binDir.getAbsolutePath();
+
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if(!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!new File(rootDir).exists()) {
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+
+		File wtcFile = new File(binDir,"wtc/"+corpusname.toLowerCase()+".wtc");
+		new File(rootDir,"/wtc/").deleteDir();
+		new File(rootDir,"/wtc/").mkdir();
+		new File(rootDir,"data/"+corpusname).deleteDir();
+		new File(rootDir,"data/"+corpusname).mkdir();
+		new File(rootDir,"registry/").mkdir();
+
+		String textid = "";
+		int counttext = 0;
+		//List<File> files = txmDir.listFiles();
+		
+		// get all anatypes
+		for (File f : files) {
+			getAnaTypes(f)
+		}
+
+		//0 set Lang
+		if (createOutput(wtcFile)) {
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+		//1- Transform into WTC file
+		def builder = null;
+		for (File f : files) {
+			counttext++;
+			if (!f.exists()) {
+				println("file "+f+ " does not exists")
+			} else {
+				//println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				builder = new compiler(f.toURI().toURL(), txtname, corpusname.toLowerCase(), "default");
+				builder.setLang(lang);
+				if(!builder.transfomFileWtc(wtcFile))
+					return false;
+			}
+		}
+
+		//end corpus
+		if (createOutput(wtcFile)) {
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+
+		//2- Import into CWB
+
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		List<String> pargs = ["id"];
+		for(String ana : anatypes)
+			pargs.add(ana);
+
+		List<String> sargs = [];
+		//println "Found Sattributes "+this.sAttribs;
+		if(sAttribs.containsKey("text")) {
+			if(!sAttribs.get("text").contains("id"))
+				sAttribs.get("text").add("id");
+			if(!sAttribs.get("text").contains("base"))
+				sAttribs.get("text").add("base");
+			if(!sAttribs.get("text").contains("project"))
+				sAttribs.get("text").add("project");
+		} else {
+			sargs.add("text:0+id+base+project")
+		}
+
+		if (sAttribs.containsKey("txmcorpus")) {
+			if(!sAttribs.get("txmcorpus").contains("lang"))
+				sAttribs.get("txmcorpus").add("lang");
+		} else {
+			sargs.add("txmcorpus:0+lang")
+		}
+
+		for (String tag : this.sAttribs.keySet()) {
+			String sAttr = tag;
+			if(sAttribs.get(tag).size() > 0)
+				sAttr += ":";
+			for(String attr : sAttribs.get(tag))
+				sAttr +="+"+attr;
+			sargs.add(sAttr)
+		}
+
+
+
+		String[] sAttributes = sargs;
+		String[] pAttributes = pargs;
+		println "sAttributes : "+sAttributes;
+		println "pAttributes : "+pAttributes;
+		try {
+			println cwbLoc
+			String regPath = rootDir + "/registry/"+corpusname.toLowerCase() 
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), 
+				rootDir + "/data/$corpusname", 
+				wtcFile.getAbsolutePath(), 
+				regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname, rootDir + "/registry");
+
+		} catch (Exception ex) {
+			System.out.println("Error while creating indexes with CQP tools: $ex");
+			ex.printStackTrace();
+			return false;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+	
+	private void getAnaTypes(File xmlFile) {
+		inputData = xmlFile.toURI().toURL().openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		String ana = "ana"
+		HashSet<String> types = new HashSet<String>();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) { // start elem
+				if (ana.equals(parser.getLocalName())) { // ana elem
+					for (int i = 0 ; i < parser.getAttributeCount(); i++) { // find @type
+						if ("type".equals(parser.getAttributeLocalName(i))) { // @type
+							types.add(parser.getAttributeValue(i).substring(1)); //remove the #
+							break;
+						}
+					}
+				}
+			}
+		}
+		parser.close()
+
+		for (String type : types)
+			if (!anatypes.contains(type))
+				anatypes << type
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>XML-TXM import module</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/xmltxmLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/xmltxmLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxm/xmltxmLoader.groovy	(revision 321)
@@ -0,0 +1,241 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+// $LastChangedDate: 2016-05-11 08:38:47 +0200 (Wed, 11 May 2016) $
+// $LastChangedRevision: 3211 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.xmltxm
+
+import javax.xml.stream.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.net.URL;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.io.File;
+
+import org.txm.importer.*;
+import org.txm.importer.xmltxm.compiler;
+import org.txm.importer.xml.pager;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.metadatas.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+import org.w3c.dom.Element;
+
+String userDir = System.getProperty("user.home");
+boolean debug = false
+def MONITOR;
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/anapovoas/FOLLOWING TEST OF MATTHIEU/OUTPUT 2/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+String page_element = params.getPageElement("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+
+// Apply XSL
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "APPLYING XSL")
+if (xsl != null && xslParams != null && xsl.trim().length() > 0) {
+	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))
+	// return; // error during process
+	srcDir = new File(binDir, "src");
+	println ""
+}
+
+// copy txm files
+println "Copying XML-TXM files..."
+List<File> srcfiles = srcDir.listFiles();
+for (File f : srcfiles) {// check XML format, and copy file into binDir
+	if (f.isHidden() || f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties"))
+		continue;
+	if (ValidateXml.test(f)) {
+		FileCopy.copy(f, new File(txmDir, f.getName()));
+	} else {
+		println "Won't process file "+f;
+	}
+}
+
+def files = txmDir.listFiles();
+if (files == null || files.size() == 0) {
+	println "No txm file to process"
+	return;
+}
+
+////get metadata values from CSV
+//Metadatas metadatas; // text metadata
+//if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+//if (allmetadatasfile.exists()) {
+//	println "Trying to read metadata from: "+allmetadatasfile
+//	File copy = new File(binDir, "metadata.csv")
+//	if (!FileCopy.copy(allmetadatasfile, copy)) {
+//		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
+//		return;
+//	}
+//	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+//} else {
+//	println "no metadata file: "+allmetadatasfile
+//}
+//
+//if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+//if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA - from csv file: "+allmetadatasfile)
+//if (metadatas != null) {
+//	println("Injecting metadata: "+metadatas.getHeadersList()+" in texts of directory "+txmDir)
+//	for (File infile : files) {
+//		print "."
+//		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+//
+//		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {
+//			outfile.delete();
+//		} else {
+//			if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+//			if (!infile.exists()) {
+//				println "Error: could not replace $infile by $outfile"
+//				return false;
+//			}
+//		}
+//	}
+//	println ""
+//}
+
+//sorting files per metadata "date" with format
+def dates = [:]
+def useDates = false
+SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd.MM.yyyy");
+for (File infile : files) { // get dates
+	String str = GetAttributeValue.process(infile, "/TEI/text", "date")
+
+	if (str != "N/A") {
+		dates[infile] = simpleDateFormat.parse(str)
+		useDates = true
+	}
+}
+
+//sort
+if (useDates) {
+	files.sort() { f-> dates[f] }
+} else {
+	files.sort()
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if (debug) c.setDebug();
+c.setLang(lang);
+if (!c.run(binDir, txmDir, basename, corpusname, Arrays.asList(files))) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	if (MONITOR != null) MONITOR.worked(20, "EDITION")
+	println "-- EDITION - Building edition"
+	
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+
+	def second = 0
+
+	println "Paginating text: "
+	for (File srcfile : files) {
+
+		String txtname = srcfile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);
+
+		if (second) { print(", ") }
+		if (second > 0 && (second++ % 5) == 0) println ""
+		print(".");
+
+		def ed = new pager(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/compiler.groovy	(revision 321)
@@ -0,0 +1,302 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.discours
+
+import org.txm.importer.cwb.BuildCwbEncodeArgsFromTEITXM;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler {
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private def output;
+
+	/** The url. */
+	private def url;
+
+	/** The anahash. */
+	private HashMap<String,String> anahash = new HashMap<String,String>() ;
+
+	/** The text. */
+	String text="";
+
+	/** The base. */
+	String base="";
+
+	/** The project. */
+	String project="";
+
+	/** The cwb loc. */
+	String cwbLoc;
+
+	/** The lang. */
+	private String lang ="fr";
+
+	/** The text attributes. */
+	static String textAttributes = "";
+
+	/**
+	 * initialize.
+	 *
+	 */
+
+	public compiler(){
+	}
+
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project) {
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path) {
+		if(!new File(path).exists())
+			System.err.println("Wrong CWB directory path: "+path)
+		cwbLoc = path;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param f the f
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f){
+		try {
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			return false;
+		}
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private void GoToText() {
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+
+	/** The debug. */
+	boolean debug = false;
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		debug = true;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param files the files
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(List<File> files, File binDir, File txmDir, String corpusname) {
+		String binDirPath = binDir.getAbsolutePath()+"/";
+
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if(!new File(cwbLoc).exists() || !binDir.exists())
+		{
+			println ("Wrong CWB directory path: "+cwbLoc+" , or Wrong corpus source path:"+binDirPath)
+			return false;
+		}
+		File wtcFile = new File(binDir,"wtc/${corpusname}.wtc");
+		new File(binDir, "wtc.").deleteDir();
+		new File(binDir, "wtc").mkdirs();
+		new File(binDir, "data/$corpusname").deleteDir();
+		new File(binDir, "data/$corpusname").mkdir();
+		new File(binDir, "registry").deleteDir();
+		new File(binDir, "registry").mkdir();
+
+		//start corpus
+		if (createOutput(wtcFile)) {
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+
+		def second = 0
+		//1- Transform into WTC file
+
+		Collections.sort(files);
+		XMLTXM2WTC wtcbuilder = null;
+		for (File f : files) {
+			if (second) { print(", ") }
+			if (second > 0 && (second % 5) == 0) println ""
+			print(f.getName().replaceFirst("\\.xml", ""));
+			second++
+
+			wtcbuilder = new XMLTXM2WTC(f.toURI().toURL());
+			String txtname = f.getName().substring(0,f.getName().length()-4);
+			wtcbuilder.setTextInfo(txtname, corpusname, "project");
+
+			wtcbuilder.setBalisesToKeep(["text","p","s"]);
+			wtcbuilder.setSendToPAttributes(["s":["id"], "p":["id"]]);
+			wtcbuilder.setLang(lang);
+			if (!wtcbuilder.transformFile(wtcFile)) {
+				println("Failed to compile "+f)
+			}
+
+		}
+
+		//end corpus
+		if (createOutput(wtcFile)) {
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+
+		if (wtcbuilder == null) {
+			println "there was no files to process: "+files
+			return false;
+		}
+
+		//2- Import into CWB
+		def outDir = binDirPath;
+
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+
+		List<String> pAttributesList = wtcbuilder.getpAttributs();
+		List<String> sAttributesList = wtcbuilder.getsAttributs();
+		println "pAttrs : "+pAttributesList
+		println "sAttrs : "+sAttributesList
+		String[] pAttributes = pAttributesList.toArray(new String[pAttributesList.size()])
+		String[] sAttributes = sAttributesList.toArray(new String[sAttributesList.size()])
+
+		try {
+			//println "Indexing "+compiler.textAttributes
+			cwbEn.setDebug(debug);
+			String regPath = outDir + "registry/"+corpusname.toLowerCase()
+			cwbEn.run(
+					cwbLoc + "cwb-encode",
+					outDir + "data/$corpusname",
+					outDir + "/wtc/"+corpusname+".wtc",
+					regPath,
+					pAttributes,
+					sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			//println "Binding all indexes"
+			cwbMa.setDebug(debug);
+			cwbMa.run(cwbLoc + "cwb-makeall", corpusname, outDir + "registry");
+
+		} catch (Exception ex) { System.err.println(ex); return false;}
+
+		return true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		File dir = new File(System.getProperty("user.home"),"xml/discours/txm/");
+		List<File> files = dir.listFiles();
+		def c = new compiler()
+		c.setCwbLoc(System.getProperty("user.home")+"/TXM/cwb/bin/")
+		if (!c.run(files,"discours")) {
+			println "Compiler failed"
+			return;
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Discours import module : CNR + CSV</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/importer.groovy	(revision 321)
@@ -0,0 +1,282 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-12-16 17:28:35 +0100 (Fri, 16 Dec 2016) $
+// $LastChangedRevision: 3377 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.discours
+
+//import filters.TabulatedToXml.*;
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.metadatas.Metadatas
+import org.txm.scripts.teitxm.*;
+
+import javax.xml.stream.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.txm.*;
+import org.txm.utils.*;
+import org.txm.utils.i18n.DetectBOM;
+import org.txm.utils.io.FileCopy;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/**
+	 * Run.
+	 *
+	 * @param dir the dir
+	 * @param encoding the encoding
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File srcDir, File binDir, File txmDir, String encoding, String basename)
+	{
+		String rootDir = srcDir.getAbsolutePath()+"/"
+		// scanning directory brut/*.cnr
+		String extension = "cnr";
+		File[] list = srcDir.listFiles();
+		
+		if (list == null || list.length == 0) return false;
+
+		LinkedList filelist = new LinkedList();
+		for (File cnr : list) {
+			if (cnr.getName().endsWith("."+extension)) {
+				filelist.add(cnr.getName());
+			}
+		}
+		Collections.sort(filelist);
+
+		// get the headers
+		def csvfile = new File(srcDir, "metadata.csv");
+		Metadatas metadatas; // text metadata
+		
+		println "Trying to read metadata from: "+csvfile
+		if (csvfile.exists()) {
+			File copy = new File(binDir, "metadata.csv")
+			if (!FileCopy.copy(csvfile, copy)) {
+				println "Error: could not create a copy of metadata file "+csvfile.getAbsoluteFile();
+				return;
+			}
+			metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+		} else {
+			println "No metadata file: "+csvfile
+			println "Aborting"
+			return false;
+		}
+		
+		// build each xml file from cnr
+		def attrs = ["para", "sent", "form", "lem", "pos", "func"];
+		int formid = 2;
+		String separator = "\t";
+
+		String encodingAll = null;
+		if (encoding == "??") {
+			encodingAll = new CharsetDetector(srcDir).getEncoding();
+			println "Guessed encoding: $encodingAll"
+		}
+
+		print "Converting CNR to XML:"
+		def second = 0
+		for (String id : metadatas.keySet()) // loop on text nodes
+		{
+			int sentence=-1, paragraph=-1, idword=0;
+			String lastopenlocalname= "", localname = "";
+			int para = -1, sent = -1;
+			
+			// = textnode.attribute("id")
+			ArrayList<org.txm.metadatas.Entry> metas = metadatas.get(id);
+			
+			String filename = id +".cnr";
+			File srcfile = new File(srcDir, filename);
+			if (!srcfile.exists()) {
+				println "Can't find CNR file : "+filename
+				continue
+			}
+			File teifile = new File(txmDir, id+".xml");
+			//println("||"+srcfile+">>"+teifile.getAbsolutePath());
+			OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(teifile), "UTF-8");
+
+//			if (second) { print(", ") }
+//			if ((second % 5) == 0) println ""
+//			print(filename.replaceFirst("\\.cnr", ""));
+//			second++
+
+			// HEADER
+			writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			writer.write("<TEI xmlns=\"http://www.tei-c.org/ns/1.0\" xmlns:txm=\"http://textometrie.org/1.0\">\n");
+			writer.write("<teiHeader type=\"text\">\n");
+			writer.write("<fileDesc>\n");
+			writer.write("<titleStmt>\n");
+			writer.write("<title>"+id+"</title>\n");
+			writer.write("<respStmt>\n");
+			writer.write("<resp id=\"cordial\">initial tagging</resp>")
+			writer.write("</respStmt>\n");
+			writer.write("</titleStmt>\n");
+			writer.write("</fileDesc>\n");
+			writer.write("<encodingDesc>\n");
+			writer.write("<classDecl>\n");
+			writer.write("<taxonomy id=\"pos\"><bibl type=\"tagset\"/></taxonomy>\n")
+			writer.write("<taxonomy id=\"func\"><bibl type=\"tagset\"/></taxonomy>\n")
+			writer.write("<taxonomy id=\"lemma\"><bibl type=\"tagset\"/></taxonomy>\n")
+			writer.write("</classDecl>\n");
+			writer.write("</encodingDesc>\n");
+			writer.write("</teiHeader>\n");
+			writer.write("<text ")
+			for (org.txm.metadatas.Entry metadata : metas) {
+				String name = metadata.getId()
+				String value = metadata.getValue()
+				if (name != "id")
+					writer.write(" "+name+"=\""+value.replace("&", "&amp;").replace("<", "&lt;").replace("\"", "&quot;")+"\"");	
+			}
+			writer.write(" id=\""+id.replace("&", "&amp;").replace("<", "&lt;").replace("\"", "&quot;")+"\"");
+			writer.write(">\n");
+
+			//def content = srcfile.getText(encoding)
+			String tmpEncoding = encoding;
+			if (encodingAll != null) {
+				tmpEncoding = encodingAll
+				if (srcfile.length() > CharsetDetector.MINIMALSIZE) {
+					tmpEncoding = new CharsetDetector(srcfile).getEncoding();
+					//println "file encoding: $tmpEncoding"	
+				}
+			}
+			def input = new FileInputStream(srcfile)
+			def reader = new BufferedReader(new InputStreamReader(input , tmpEncoding));
+			DetectBOM bomdetector = new DetectBOM(srcfile);
+			for (int ibom = 0 ; ibom < bomdetector.getBOMSize() ; ibom++) input.read()
+
+			// BODY
+			String line = reader.readLine();
+			int nline = 0;
+			def errors = [];
+			while (line != null) {
+				nline++;
+				String[] fields = line.split(separator);
+				if (fields.size() == attrs.size()) {
+					idword++;
+					
+					String paraV = fields[0].replaceAll("\\p{C}", "")
+					String sentV = fields[1].replaceAll("\\p{C}", "")
+					String formV = fields[2].replaceAll("\\p{C}", "").replace("&","&amp;").replace("<","&lt;");
+					String lemV = fields[3].replaceAll("\\p{C}", "").replace("&","&amp;").replace("<","&lt;");
+					String posV = fields[4].replaceAll("\\p{C}", "").replace("&","&amp;").replace("<","&lt;");
+					String funcV = fields[5].replaceAll("\\p{C}", "").replace("&","&amp;").replace("<","&lt;");
+					
+					try {
+						para = Integer.parseInt(paraV);
+					} catch(Exception e){}
+					if (para != paragraph) {
+						if (paragraph != -1) {
+							if (sentence != -1) {
+								writer.write("</s>\n")
+								sentence = -1;
+							}
+							writer.write("</p>\n")
+						}
+						writer.write("<p id=\"p_${para}\">\n");
+						paragraph = para;
+					}
+					
+					try {
+						sent = Integer.parseInt(sentV);
+					} catch(Exception e){}
+					if (sent != sentence) {
+						if (sentence != -1)
+							writer.write("</s>\n")
+						writer.write("<s id=\"s_${sent}\">\n");
+						sentence = sent;
+					}
+					
+					writer.write("<w id=\"w_${id}_${idword}\">\n");
+					writer.write(" <txm:form>$formV</txm:form>\n");
+					writer.write(" <txm:ana resp=\"#cordial\" type=\"#pos\">$posV</txm:ana>\n");
+					writer.write(" <txm:ana resp=\"#cordial\" type=\"#func\">$funcV</txm:ana>\n");
+					writer.write(" <txm:ana resp=\"#cordial\" type=\"#lemma\">$lemV</txm:ana>\n");
+					writer.write(" <txm:ana resp=\"#cordial\" type=\"#sent\">$sentV</txm:ana>\n");
+					writer.write(" <txm:ana resp=\"#cordial\" type=\"#para\">$paraV</txm:ana>\n");
+					writer.write(" <txm:ana resp=\"#txm\" type=\"#ref\">${id}, §${paraV}</txm:ana>\n");
+					writer.write("</w>\n");
+				} else {
+					//println "error : $nline : '$line'"
+					errors << nline
+				}
+				line = reader.readLine();
+			}
+
+			// CLOSE
+			if(sentence != -1) { writer.write("</s>\n") }
+			if(paragraph != -1) { writer.write("</p>\n") }
+			paragraph=-1;
+			sentence=-1;
+			writer.write("</text>\n");
+			writer.write("</TEI>\n");
+			writer.close();
+			
+			if (errors.size() > 0) {
+				println "Errors in file $filename. Wrong number of columns at lines: "+errors
+			} else print "."
+		}
+		println ""
+		return true;
+	}
+
+	/**
+	 * Gets the metadata.
+	 *
+	 * @return the metadata
+	 */
+	public List<String> getMetadatas()
+	{
+		return metaDatas;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File(System.getProperty("user.home"), "xml/discours/");
+		new importer().run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/discoursLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/discoursLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/discoursLoader.groovy	(revision 321)
@@ -0,0 +1,152 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2014-11-20 11:30:43 +0100 (Thu, 20 Nov 2014) $
+// $LastChangedRevision: 2904 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.discours;
+
+import org.txm.importer.discours.compiler;
+import org.txm.importer.xml.pager;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.utils.i18n.*;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true;
+	params = new BaseParameters(new File(userDir, "xml/discours/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+def imp = new importer();
+if (!imp.run(srcDir, binDir, txmDir, encoding, basename)) {
+	println "import process stopped";
+	return;
+}
+
+println "-- ANNOTATE - Running NLP tools"
+println "No annotation to do"
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println("-- COMPILING - Building Search Engine indexes")
+List<File> files = txmDir.listFiles();
+def c = new compiler()
+if (debug) c.setDebug();
+// no annotation
+c.setLang(lang);
+if (!c.run(files, binDir, txmDir, corpusname)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(20, "EDITION")
+	
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+
+	println "Paginating texts: "
+
+	for (File srcfile : filelist) {
+		print "."
+		String txtname = srcfile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);
+
+		def ed = new pager(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, "pb");
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Shared tools for import modules</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/compiler.groovy	(revision 321)
@@ -0,0 +1,735 @@
+
+
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-05-26 17:42:36 +0200 (Thu, 26 May 2016) $
+// $LastChangedRevision: 3219 $
+// $LastChangedBy: mdecorde $
+//
+
+
+package org.txm.importer.xml;
+
+import java.util.ArrayList
+import java.util.Collections
+
+import org.txm.importer.cwb.BuildCwbEncodeArgs
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*
+import org.txm.scripts.*
+import org.txm.scripts.teitxm.*
+import org.txm.utils.io.FileCopy;
+import org.txm.utils.treetagger.TreeTagger
+
+import javax.xml.stream.*
+
+import java.net.URL
+import java.io.File
+import java.util.HashMap
+import java.util.List
+import java.util.HashMap
+import java.util.HashSet
+
+import org.txm.metadatas.*
+
+/**
+ * The "compiler" Class of the XML/w import module.
+ */
+class compiler {
+	
+	/** The debug. */
+	private boolean debug= false;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private def output;
+
+	/** The url. */
+	private def url;
+
+	/** The anatypes. */
+	private static anatypes = []
+	private static anavalues = [:]
+
+	/** The anahash. */
+	private HashMap<String, String> anahash = new HashMap<String, String>() ;
+
+	private static SAttributesListener sattrsListener;
+	private static HashMap<String, ArrayList<String>> structs;
+	private static HashMap<String, Integer> structsProf;
+
+	/** The text. */
+	String text="";
+
+	/** The base. */
+	String base="";
+
+	/** The project. */
+	String project="";
+
+	/** The text attributes. */
+	String[] textAttributes = null;
+
+	/** The cwb loc. */
+	String cwbLoc;
+
+	/** The lang. */
+	private String lang ="fr";
+
+	public static sortMetadata = null;
+	public static normalizeMetadata = false;
+
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+
+	public void setOptions(String sortmetadata, boolean normalizemetadata)
+	{
+		sortMetadata = sortmetadata;
+		normalizeMetadata = normalizemetadata;
+	}
+
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url, String text, String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.textAttributes = textAttributes;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+
+			if (sattrsListener == null)
+				sattrsListener = new SAttributesListener(parser);
+			else
+				sattrsListener.start(parser)
+
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+
+	/** The annotation success. */
+	boolean annotationSuccess = false;
+
+	/**
+	 * Sets the annotation success.
+	 *
+	 * @param val the new annotation success
+	 */
+	public void setAnnotationSuccess(boolean val)
+	{
+		this.annotationSuccess = val;
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if (!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f){
+		try {
+			output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f,f.exists())) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			return false;
+		}
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.END_ELEMENT)
+				if (parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(File wtcFile, HashMap<String, String> textmetadata)
+	{
+		if (!createOutput(wtcFile))
+			return false;
+
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+
+		int p_id = 0;
+		int s_id = 0;
+
+		def divs = []
+		def ncounts = [:] // contains the n values per tags with no attribute
+
+		boolean captureword = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+
+		String anatype = "";
+		String anavalue = "";
+		boolean stopAtFirstSort = true;
+		boolean foundtei = false;
+		boolean foundtext = false;
+		//output.write("<txmcorpus lang=\""+lang+"\">\n");
+		try {
+			String localname;
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName().toLowerCase();
+						if ("tei".equals(localname)) foundtei = true;
+						switch (localname) {
+							case "text":
+								sattrsListener.startElement(localname);
+								foundtext = true;
+								output.write("<text id=\""+text+"\" base=\""+base+"\"" + " project=\""+project+"\"");
+							//							for (String name : textmetadata.keySet())
+							//								output.write(" "+name+"=\""+textmetadata.get(name)+"\"")
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									String attrname = parser.getAttributeLocalName(i);
+									String attrvalue = parser.getAttributeValue(i)
+									if (normalizeMetadata)
+										attrvalue = attrvalue.toLowerCase();
+									if (attrname != "id")
+										output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+"\"")
+								}
+								output.write(">\n");
+
+							//								if (textAttributes == null) {
+							//									textAttributes = new String[parser.getAttributeCount()];
+							//
+							//									for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+							//										textAttributes[i]=parser.getAttributeLocalName(i).toLowerCase();
+							//									}
+							//								}
+
+								break;
+
+
+							case "w":
+								for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+									if (parser.getAttributeLocalName(i).equals("id")) {
+										wordid = parser.getAttributeValue(i);
+									}
+								}
+								anavalues = [:];
+								break;
+							case "form":
+								flagForm = true;
+								vForm = "";
+								vAna ="";
+								break;
+
+							case "ana":
+								flagAna = true;
+								anavalue = "";
+								for (int i = 0 ; i < parser.getAttributeCount(); i++)
+									if ("type".equals(parser.getAttributeLocalName(i))) {
+										anatype = parser.getAttributeValue(i).substring(1);//remove the #
+										break;
+									}
+								break;
+
+							default:
+//								if ("div" == localname ) {
+//									def type = localname;
+//									for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+//										String attrname = parser.getAttributeLocalName(i);
+//										if ("type".equals(attrname)) {
+//											type= parser.getAttributeValue(i)
+//										}
+//									}
+//									divs << type;
+//									localname = type
+//								}
+
+								if (foundtei && !foundtext) break;
+
+								sattrsListener.startElement(localname);
+								output.write("<"+localname);
+
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									String attrname = parser.getAttributeLocalName(i);
+									String attrvalue = parser.getAttributeValue(i)
+									if (normalizeMetadata)
+										attrvalue = attrvalue.toLowerCase();
+									output.write(" "+attrname.toLowerCase()+"=\""+attrvalue+"\"")
+								}
+								if (parser.getAttributeCount() == 0) { // add the n attribute
+									if (!ncounts.containsKey(localname)) ncounts.put(localname, 0);
+									int ncount = ncounts.get(localname);
+									ncounts.put(localname, ncount+1);
+									output.write(" n=\""+ncount+"\"")
+								}
+								output.write(">\n");
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName().toLowerCase();
+						switch (localname) {
+							case "w":
+								for (String type : anatypes) {
+									def v = anavalues.get(type);
+									if (v != null) vAna +="\t"+v;
+									else vAna +="\t";
+								}
+								vForm = vForm.replaceAll("\n", "").replaceAll("&", "&amp;").replaceAll("<", "&lt;");
+								if (vAna != null) {
+									output.write(vForm+"\t"+wordid+vAna+"\n");
+								}
+								vAna = "";
+								vForm = "";
+								break;
+
+							case "tei":
+								break;
+							case "form":
+								flagForm = false;
+								break;
+							case "ana":
+								anavalues.put(anatype, anavalue)
+								flagAna = false;
+								break;
+							default:
+								if (foundtei && !foundtext) break;
+
+//								if ("div" == localname && divs.size() > 0) {
+//									localname = divs.pop()
+//								}
+
+								sattrsListener.endElement(localname);
+								output.write("</"+localname+">\n");
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						if (flagForm)
+							vForm += parser.getText().trim();
+						if (flagAna) {
+							if (normalizeMetadata)
+								anavalue += parser.getText().trim().toLowerCase();
+							else
+								anavalue += parser.getText().trim();
+						}
+						break;
+				}
+			}
+			//output.write("</txmcorpus>");
+			output.close();
+			parser.close();
+		} catch (Exception ex) {
+			System.out.println("Exception while parsing " + inputData+" of Text "+text);
+			File xmlFile = null
+			File errorDir = null		
+			try {
+				xmlFile = new File(url.getFile())
+				errorDir = new File(wtcFile.getParentFile(), "compiler-error")
+				println "Warning: Moving $xmlFile to $errorDir"
+				errorDir.mkdir();
+				FileCopy.copy(xmlFile, new File(errorDir, xmlFile.getName()))
+			} catch(Exception eCopy) {
+				println "Error while moving "+url+" to "+errorDir
+			}
+			return false;
+		}
+		return true;
+	}
+
+	private void getAnaTypes(File xmlFile) {
+		inputData = xmlFile.toURI().toURL().openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		String ana = "ana"
+		HashSet<String> types = new HashSet<String>();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) { // start elem
+				if (ana.equals(parser.getLocalName())) { // ana elem
+					for (int i = 0 ; i < parser.getAttributeCount(); i++) { // find @type
+						if ("type".equals(parser.getAttributeLocalName(i))) { // @type
+							types.add(parser.getAttributeValue(i).substring(1)); //remove the #
+							break;
+						}
+					}
+				}
+			}
+		}
+		parser.close()
+
+		for (String type : types)
+			if (!anatypes.contains(type))
+				anatypes << type
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @param textAttributes the text attributes
+	 * @param srcfiles the srcfiles
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String corpusname, String[] textAttributes, def srcfiles, Metadatas metadatas)
+	{
+		sattrsListener = null; // reset SAttribute Listener for each new import
+		String rootDir = binDir.getAbsolutePath();
+		anatypes = [] // reset
+		anavalues = [:] // reset
+		if (cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if (!binDir.exists()) {
+			println ("binary directory does not exists: "+binDir)
+			return false;
+		}
+
+		File wtcFile = new File(binDir, "wtc/"+corpusname+".wtc")
+		new File(binDir, "wtc").deleteDir()
+		new File(binDir, "wtc").mkdir()
+		new File(binDir, "data").deleteDir()
+		new File(binDir, "data").mkdir()
+		new File(binDir, "registry").mkdir()
+
+		String textid = ""
+		int counttext = 0
+		List<File> files = txmDir.listFiles()
+		//1- Transform into WTC file
+		def builder = null
+
+		//start corpus
+		if (createOutput(wtcFile)) {
+			output.write("<txmcorpus lang=\""+lang+"\">\n")
+			output.close()
+		}
+
+		// sort files
+		if (sortMetadata == null) {
+			Collections.sort(files)
+		} else {
+			HashMap<File, String> sortmetadatavalues = new HashMap<File, String>()
+			for (File f : files) {
+				String value = MetadataGetter.get(f,"text", sortMetadata)
+				sortmetadatavalues.put(f, value)
+			}
+			println "sort properties value: "+sortmetadatavalues
+			Collections.sort(files, new Comparator<File>() {
+				/**
+				 * Compare.
+				 *
+				 * @param o1 the o1
+				 * @param o2 the o2
+				 * @return the int
+				 */
+						public int compare(Object o1, Object o2) {
+							String v1 = sortmetadatavalues.get((File)o1)
+							String v2 = sortmetadatavalues.get((File)o2)
+							if (v1 == null || v2 == null) return 0;
+							return v1.compareTo(v2)
+						}
+					});
+		}
+
+		// get all anatypes
+		for (File f : files) {
+			getAnaTypes(f)
+		}
+
+		println("Compiling "+files.size()+" $files ")
+		for (File f : files) {
+			print "."
+			HashMap<String, String> textmetadata;
+			if (metadatas != null)
+				textmetadata = metadatas.getTextMetadata(f)
+			else
+				textmetadata = [:]
+
+			counttext++;
+			if (!f.exists()) {
+				println("file "+f+ " does not exists")
+			} else {
+				String txtname = f.getName().substring(0,f.getName().length()-4)
+				builder = new compiler(f.toURI().toURL(), txtname, corpusname, "default")
+				builder.setLang(lang);
+				if (!builder.transfomFileWtc(wtcFile, textmetadata)) {
+					println("Failed to compile "+f)
+				}
+			}
+		}
+
+		//end corpus
+		if (createOutput(wtcFile)) {
+			output.write("</txmcorpus>\n")
+			output.close()
+		}
+		println ""
+		//2- Import into CWB
+		def outDir = rootDir
+
+		CwbEncode cwbEn = new CwbEncode()
+		cwbEn.setDebug(debug)
+		CwbMakeAll cwbMa = new CwbMakeAll()
+		cwbMa.setDebug(debug)
+
+		List<String> pargs = []
+		pargs.add("id")
+		for (String ana : anatypes)
+			pargs.add(ana)
+
+		String[] pAttrs = pargs
+
+		structs = sattrsListener.getStructs()
+		structsProf = sattrsListener.getProfs()
+
+		if (debug) {
+			println structs
+			println structsProf
+		}
+		
+		List<String> sargs = new ArrayList<String>()
+		def tmpTextAttrs = []
+		for (String name : structs.keySet()) {
+			if (name == "text") {
+				for (String value : structs.get(name)) // append the attributes
+					tmpTextAttrs << value // added after
+				continue;
+			}
+			//if ( name == "q") continue; // added after
+			//if ( name == "foreign") continue; // added after
+			String concat = name+":"+structsProf.get(name); // append the depth
+			for (String attributeName : structs.get(name)) // append the attributes
+				concat += "+"+attributeName.toLowerCase();
+				
+			if (structs.get(name).size() == 0) {
+				concat += "+n";
+			} else {
+				if (!structs.get(name).contains("n"))
+					concat += "+n"
+			}
+				
+			if ((name == "p" || name == "body" || name == "back" || name == "front")
+				 && !concat.contains("+n+") && !concat.endsWith("+n"))
+				concat += "+n"
+				
+			sargs.add(concat)
+		}
+
+		String textSAttributes = "text:0+id+base+project";
+		for (String name : tmpTextAttrs) {
+			if (!("id".equals(name) || "base".equals(name) || "project".equals(name)))
+				textSAttributes += "+"+name.toLowerCase()
+		}
+		//		if (metadataXPath != null) {
+		//			for (String meta : metadataXPath.keySet()) // text property declarations from metadata.csv
+		//				textSAttributes+="+"+meta;
+		//		}
+		sargs.add(textSAttributes)
+		sargs.add("txmcorpus:0+lang")
+
+		sargs.sort()
+
+		String[] sAttributes = sargs
+		String[] pAttributes = pAttrs
+		println "P-attributes: "+pAttributes
+		println "S-attributes: "+sargs
+
+		//if(!annotationSuccess)
+		//pAttributes = ["id"];
+
+		//println "PATTRIBUTES : "+pargs;
+		/*
+		 ArrayList<String> wordstag = ["w"];
+		 println "Getting structural attributes..."
+		 BuildCwbEncodeArgs argsgetter = new BuildCwbEncodeArgs();
+		 HashMap<String, HashSet<String>> allStructures = new HashMap<String, HashSet<String>>();
+		 HashMap<String, Integer> allStructuresInclusion = new HashMap<String, Integer>();
+		 for (File srcfile: txmDir.listFiles()) {
+		 if (!(!srcfile.getName().endsWith(".csv") && srcfile.canRead() && !srcfile.isHidden() && !srcfile.isDirectory() && ValidateXml.test(srcfile)))
+		 continue;
+		 print "."
+		 argsgetter.process(srcfile, wordstag);
+		 for (String sattr : argsgetter.getSAttributes()) {
+		 int idx = sattr.indexOf(":");
+		 if(idx < 0 )
+		 continue;
+		 String name = sattr.substring(0, idx);
+		 if (!allStructures.containsKey(name)) {
+		 allStructures.put(name, new HashSet<String>());
+		 allStructuresInclusion.put(name, 0);
+		 }
+		 //println "sattr: "+name
+		 String attrs = sattr.substring(idx+1);
+		 String[] split = attrs.split("\\+");
+		 if (split.length > 0) {
+		 int start = 1;
+		 try {// test if first attr is a number
+		 int n = Integer.parseInt(split[0]);
+		 if (n > allStructuresInclusion.get(name))
+		 allStructuresInclusion.put(name, n);
+		 } catch(Exception e) {start = 0;}
+		 for (int i = start ; i < split.length ; i++)
+		 allStructures.get(name).add(split[i]);
+		 }
+		 }
+		 }
+		 // add structures+properties found in sources
+		 List<String> sargs = new ArrayList<String>();
+		 for (String name : allStructuresInclusion.keySet()) {
+		 String concat = name+":"+allStructuresInclusion.get(name);
+		 for (String value : allStructures.get(name))
+		 concat += "+"+value;
+		 if (name.equals("text")) {
+		 concat += "+base+project"
+		 if (!concat.contains("id"))
+		 concat += "+id";
+		 }
+		 sargs.add(concat);
+		 }*/
+
+		//		for (int i = 0 ; i < sargs.size() ; i++) {
+		//			if (sargs.get(i).startsWith("text:")) {
+		//				String str = sargs.get(i);
+		//				sargs.set(i, "text:"+str.substring(6));
+		//			}
+		//		}
+
+		//		String textSAttributes = "text:0+id+base+project";
+		//		if (metadatas != null) {
+		//			for (String meta : metadatas.getHeadersList()) // text property declarations from metadata.csv
+		//				textSAttributes+="+"+meta;
+		//		}
+		//sargs.add(textSAttributes)
+		//sargs.add("txmcorpus:0+lang")
+
+		//		String[] sAttributes = sargs;
+		//		System.out.println("\nCorpus structures: "+sAttributes);
+		//		System.out.println("corpus word properties: "+pAttributes);
+
+		try {
+			String regPath = outDir + "/registry/"+corpusname.toLowerCase();
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), 
+				outDir + "/data/$corpusname", 
+				outDir + "/wtc/"+corpusname+".wtc", 
+				regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname, outDir + "/registry");
+		} catch (Exception ex) {System.out.println(ex); return false;}
+
+		return true;
+	}
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>XML import module</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/pager_old.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/pager_old.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/pager_old.groovy	(revision 321)
@@ -0,0 +1,370 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.xml;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build a simple edition from a xml-tei. 
+ * @author mdecorde */
+class pager_old {
+	List<String> NoSpaceBefore;
+	//StaxStack staxStack;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename = "";
+	String txtname = "";
+	File outdir;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue;
+
+	/** The interpvalue. */
+	String interpvalue;
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	OutputStreamWriter writer;
+
+	/** The multiwriter. */
+	OutputStreamWriter multiwriter = null;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	String editionPage;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager_old(File infile, File outdir, String txtname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String basename, String editionPage) {
+		this.editionPage = editionPage;
+		this.basename = basename;
+		this.txtname = txtname;
+		this.outdir = outdir;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		//staxStack = new StaxStack(parser)
+		process();
+	}
+
+	private void closeMultiWriter()
+	{
+		if (multiwriter != null) {
+			//staxStack.closeStack(multiwriter);
+			if (firstWord) { // there was no words
+				this.idxstart.add("w_0")
+				multiwriter.write("<span id=\"w_0\"/>");
+			}
+			multiwriter.write("</body>");
+			multiwriter.write("</html>"); 
+			multiwriter.close();
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		wordcount = 0;
+		try {
+			closeMultiWriter();
+			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")
+			pages.add(outfile);
+			firstWord = true; // waiting for next word
+
+			multiwriter = new OutputStreamWriter(new FileOutputStream(outfile) , "UTF-8");
+
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("<head>");
+			multiwriter.write("<title>"+basename.toUpperCase()+" Edition - Page "+pagecount+"</title>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+
+			//staxStack.openStack(multiwriter);
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput() {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process() {
+		String localname = "";
+		createNextOutput();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							if (parser.getAttributeValue(null,"id") != null)
+								multiwriter.write("<h3>"+parser.getAttributeValue(null,"id")+"</h3>\n")
+							multiwriter.write("<table>\n");
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								multiwriter.write("<tr>");
+								multiwriter.write("<td>"+parser.getAttributeLocalName(i)+": </td>");
+								multiwriter.write("<td>"+parser.getAttributeValue(i)+"</td>");
+								multiwriter.write("</tr>\n");
+							}
+							multiwriter.write("</table><br/>\n");
+							break;
+						case "head":
+							multiwriter.write("<h2>\n")
+							//staxStack.onStartElement("h2", null);
+							break;
+						case "graphic":
+							String url = parser.getAttributeValue(null, "url")
+						//println "URL: "+url
+							if (url != null) multiwriter.write("<div><img src=\"$url\"/></div>");
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							String rend = parser.getAttributeValue(null, "rend")
+							if (rend == null) rend = "normal"
+							multiwriter.write("<p class=\"$rend\">\n")
+							//staxStack.onStartElement("p", null);
+							break;
+						//case "pb":
+						case editionPage:
+							createNextOutput();
+							wordcount=0;
+							if (parser.getAttributeValue(null,"n") != null) {
+								multiwriter.write("<p style=\"color:red\" align=\"center\">- "+parser.getAttributeValue(null,"n")+" -</p>\n")
+							}
+							break;
+						case "lb":
+						case "br":
+							multiwriter.write("<br/>\n")
+							break;
+						case "w":
+							wordid = parser.getAttributeValue(null,"id");
+
+							if (firstWord) {
+								firstWord = false;
+								this.idxstart.add(wordid);
+							}
+							wordcount++;
+							if (wordcount >= wordmax) {
+								createNextOutput();
+							}
+							break;
+
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "head":
+							multiwriter.write("</h2>\n")
+							
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							multiwriter.write("</p>\n")
+							
+							break;
+
+
+						case "form":
+							flagform = false
+							break;
+
+						case "ana":
+							flaginterp = false
+							break;
+
+						case "w":
+
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if (l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+
+							if (interpvalue != null)
+								interpvalue = interpvalue.replace("&", "&amp;").replace("<", "&lt;").replace("\"","&quot;");
+
+							if (NoSpaceBefore.contains(wordvalue) ||
+							NoSpaceAfter.contains(lastword) ||
+							wordvalue.startsWith("-") ||
+							NoSpaceAfter.contains(endOfLastWord)) {
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							} else {
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+						//writer.write(wordvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+							break;
+					}
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					if (flagform)
+						if (parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if (flaginterp)
+						if (parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					break;
+			}
+		}
+		//writer.write("</body>");
+		//writer.write("</html>");
+		//writer.close();
+		closeMultiWriter();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/pager.groovy	(revision 321)
@@ -0,0 +1,417 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.xml;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.*
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+/** Build a simple edition from a xml-tei. 
+ * @author mdecorde */
+class pager {
+
+	List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename = "";
+	String txtname = "";
+	File outdir;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue = "";
+
+	/** The interpvalue. */
+	String interpvalue = "";
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	OutputStreamWriter writer;
+
+	/** The pagedWriter. */
+	StaxStackWriter pagedWriter = null;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	String paginationElement;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager(File infile, File outdir, String txtname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String basename, String paginationElement) {
+		this.paginationElement = paginationElement;
+		this.basename = basename;
+		this.txtname = txtname;
+		this.outdir = outdir;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+
+		process();
+	}
+
+	private def closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+
+			if (firstWord) { // there was no words
+				pagedWriter.writeCharacters("");
+				//				if (this.idxstart.size() == 0) {
+				//					this.idxstart.add("w_0")
+				//					pagedWriter.write("<span id=\"w_0\"/>");
+				//				} else {
+				this.idxstart.add("w_0")
+				pagedWriter.write("<span id=\"w_0\"/>");
+				//				}
+			}
+			pagedWriter.writeEndElements();
+			pagedWriter.close();
+			return tags;
+		} else {
+			return [];
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput() {
+		wordcount = 0;
+		try {
+			def tags = closeMultiWriter();
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")
+			pages.add(outfile);
+			firstWord = true; // waiting for next word
+
+			pagedWriter = new StaxStackWriter(outfile, "UTF-8");
+
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html");
+			pagedWriter.writeEmptyElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"]);
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"txm.css"]);
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"${basename}.css"]);
+			pagedWriter.writeStartElement("head");
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(basename.toUpperCase()+" Edition - Page "+pagecount)
+			pagedWriter.writeEndElement(); // </title>
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+
+			pagedWriter.writeStartElements(tags);
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput() {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process() {
+		boolean flagNote = false;
+		String noteContent = "";
+
+		String localname = "";
+		createNextOutput();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							pagedWriter.write("\n");
+							if (parser.getAttributeValue(null,"id") != null)
+								pagedWriter.writeElement("h3", parser.getAttributeValue(null,"id"))
+							pagedWriter.writeStartElement("table");
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								pagedWriter.writeStartElement("tr");
+								pagedWriter.writeElement("td", parser.getAttributeLocalName(i));
+								pagedWriter.writeElement("td", parser.getAttributeValue(i).toString());
+								pagedWriter.writeEndElement();
+							}
+							pagedWriter.writeEndElement() // table
+							pagedWriter.writeEmptyElement("br")
+							pagedWriter.writeCharacters("");
+							break;
+						case "note":
+							flagNote = true;
+							noteContent = ""
+							break;
+						case "a":
+							pagedWriter.writeStartElement("a")
+							pagedWriter.writeAttribute("href", parser.getAttributeValue(null,"href"));
+							String target = parser.getAttributeValue(null,"target");
+							if (target == null) target = "_blank";
+							pagedWriter.writeAttribute("target", target);
+							break;
+						case "head":
+							pagedWriter.write("\n");
+							pagedWriter.writeStartElement("h2")
+							break;
+						case "graphic":
+							pagedWriter.write("\n");
+							String url = parser.getAttributeValue(null, "url")
+							if (url != null) {
+								pagedWriter.writeStartElement("div")
+								pagedWriter.writeEmptyElement("img", ["src":url])
+								pagedWriter.writeEndElement() // div
+							}
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							pagedWriter.write("\n");
+							String rend = parser.getAttributeValue(null, "rend")
+							if (rend == null) rend = "normal"
+							pagedWriter.writeStartElement("p",["class":rend])
+							break;
+						//case "pb":
+						case paginationElement:
+							createNextOutput();
+							wordcount=0;
+							pagedWriter.write("\n");
+							if (parser.getAttributeValue(null,"n") != null) {
+								pagedWriter.writeElement("p", ["style":"color:red", "align":"center"], "- "+parser.getAttributeValue(null,"n")+" -")
+							}
+							break;
+						case "lb":
+						case "br":
+							pagedWriter.writeEmptyElement("br")
+							break;
+						case "w":
+							wordid = parser.getAttributeValue(null,"id");
+							if (firstWord) {
+								firstWord = false;
+								this.idxstart.add(wordid);
+							}
+							wordcount++;
+							if (wordcount >= wordmax) {
+								createNextOutput();
+							}
+							break;
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+							break;
+						//						default:
+						//							pagedWriter.writeStartElement(localname)
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "head":
+							pagedWriter.writeEndElement() // </h2>
+							break;
+						case "note":
+							flagNote = false;
+							if (noteContent.length() > 0) {
+								pagedWriter.writeStartElement("span", ["style":"color:red;", "title":noteContent]);
+								pagedWriter.writeCharacters("[*]");
+								pagedWriter.writeEndElement() // </span>
+							}
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							pagedWriter.writeEndElement() // </p>
+							break;
+						case "a":
+							pagedWriter.writeEndElement() // </a>
+							break;
+						case "form":
+							flagform = false
+							break;
+						case "ana":
+							flaginterp = false
+							break;
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if (l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+
+							if (interpvalue != null)
+								interpvalue = interpvalue;
+
+							if (NoSpaceBefore.contains(wordvalue) ||
+							NoSpaceAfter.contains(lastword) ||
+							wordvalue.startsWith("-") ||
+							NoSpaceAfter.contains(endOfLastWord)) {
+								pagedWriter.writeStartElement("span", ["title":interpvalue, "id":wordid]);
+							} else {
+								pagedWriter.writeCharacters(" ");
+								pagedWriter.writeStartElement("span", ["title":interpvalue, "id":wordid]);
+							}
+
+							pagedWriter.writeCharacters(wordvalue);
+							pagedWriter.writeEndElement();
+							lastword=wordvalue;
+							break;
+						//						default:
+						//							pagedWriter.writeEndElement()
+					}
+					break;
+				case XMLStreamConstants.CHARACTERS:
+					if (flagform && parser.getText().length() > 0) {
+						wordvalue+=(parser.getText());
+						if (flagNote == parser.getText().length() > 0)
+							noteContent += parser.getText().replace("\n", " ");
+					} else	if (flaginterp && parser.getText().length() > 0) {
+						interpvalue+=(parser.getText());
+					} else if (flagNote == parser.getText().length() > 0) {
+						noteContent += parser.getText().replace("\n", " ");
+					}
+					break;
+			}
+		}
+		closeMultiWriter();
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/importer.groovy	(revision 321)
@@ -0,0 +1,209 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-12-01 17:48:31 +0100 (Thu, 01 Dec 2016) $
+// $LastChangedRevision: 3351 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.xml
+
+//import filters.TabulatedToXml.*;
+import java.io.File
+
+import javax.xml.stream.*
+
+import org.txm.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+import org.txm.objects.*
+import org.txm.scripts.*
+import org.txm.scripts.teitxm.*
+
+import filters.CutHeader.*
+import filters.FusionHeader.*
+import filters.TagSentences.*
+import filters.Tokeniser.*
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/** The validation. */
+	boolean validation;
+
+	/** The tokenize. */
+	boolean tokenize;
+
+	/** The sentence. */
+	boolean sentence = false;
+
+	/**
+	 * Do tokenize.
+	 *
+	 * @param b the b
+	 */
+	public void doTokenize(boolean b) { tokenize = b; }
+
+	/**
+	 * Do validation.
+	 *
+	 * @param b the b
+	 */
+	public void doValidation(boolean b) { validation = b; }
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File srcDir, File binDir, File txmDir, String basename, String ignoredElements, String lang)
+	{
+		new File(binDir,"tokenized").deleteDir();
+		new File(binDir,"tokenized").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+
+		//TEST EACH XML FILE IF VALID
+		List<File> srcfiles = txmDir.listFiles();
+		List<File> okfiles = []
+
+		println "Sources cleaning & validation";
+				def files = txmDir.listFiles()
+		for (File f : txmDir.listFiles()) { // clean directory
+			String name = f.getName()
+			if (f.isHidden() || !name.toLowerCase().endsWith(".xml")) {
+				if (!f.delete()) {
+					println "WARNING: could not clean $txmDir directory: TXM could not delete $f"
+					return false;
+				}
+			}
+		}
+		
+		files = txmDir.listFiles()
+		if (files == null || files.size() == 0) {
+			println "No XML file (*.xml) to process. Aborting"
+			return false;
+		}
+		files = files.sort()
+		for (File f : txmDir.listFiles()) {
+			if (!f.getName().toLowerCase().endsWith(".xml")) continue;
+			
+			if (!f.isHidden() && f.canRead() && !f.getName().endsWith(".properties") && !f.getName().startsWith("metadata") && !f.isDirectory()) {
+				print "."
+				if (!ValidateXml.test(f)) {
+					if (stopIfMalformed)
+						return;
+					else
+						continue;
+				} else {
+					if (!f.getName().equals("import.xml"))
+						okfiles.add(f);
+				}
+			}
+		}
+		println ""
+
+		okfiles.sort()
+		println "Files processed: "+okfiles
+		if (okfiles.size() == 0) {
+			println "No file. Check if the file extensions are '.xml'"
+			return false;
+		}
+		
+		//println "ptokenfiles "+ptokenfiles
+		//TOKENIZE FILES
+		List<File> tokenfiles;
+		if (tokenize || sentence) {
+			println "Tokenizing "+okfiles.size()+" files"
+			for (File f : okfiles) {
+				print "."
+				File infile = f;
+				File outfile = new File(binDir,"tokenized/"+f.getName());
+				SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)
+				if (ignoredElements != null && ignoredElements.trim().length() > 0)
+					tokenizer.setOutSideTextTagsAndKeepContent(ignoredElements)
+				if (!tokenizer.process()) {
+					println("Failed to process "+f)
+					outfile.delete()
+				}
+			}
+			tokenfiles = new File(binDir,"tokenized").listFiles()
+			println ""
+		} else {
+			tokenfiles = okfiles;
+		}
+
+//		Change XML tag "text" to "textunit"
+//		for (File f : tokenfiles) {
+//			if (!RenameTag.rename(f, "text", "textunit")) {
+//				println "Failed to rename <text> tag to <textunit> in file "+f
+//				return false;
+//			}
+//		}
+
+		//TRANSFORM INTO XML-TEI-TXM
+		println("Building XML-TXM ("+tokenfiles.size()+" files)")
+		for (File f : tokenfiles) {
+			//ArrayList<String> milestones = new ArrayList<String>();
+			print "."
+			File file = f;
+			String txmfile = f.getName();
+
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			//lance le traitement
+			def builder = new Xml2Ana(file);
+			builder.setConvertAllAtrtibutes true;
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			//builder.setAddTEIHeader();
+			if (!builder.process(new File(txmDir,txmfile))) {
+				println("Failed to process "+f);
+				new File(txmDir,txmfile).delete();
+			}
+		}
+		println ""
+		return true;
+	}
+
+	boolean stopIfMalformed = false;
+	public void setStopIfMalformed(boolean b) {
+		stopIfMalformed = b;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/xmlLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/xmlLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xml/xmlLoader.groovy	(revision 321)
@@ -0,0 +1,301 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-11-29 16:47:07 +0100 (Tue, 29 Nov 2016) $
+// $LastChangedRevision: 3349 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.xml;
+
+import javax.xml.stream.XMLStreamReader;
+
+import org.txm.sw.RemoveTag;
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.ValidateXml;
+import org.txm.importer.xml.importer;
+import org.txm.importer.xml.compiler;
+import org.txm.importer.xml.pager_old;
+import org.txm.objects.*;
+import org.txm.tokenizer.TokenizerClasses;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.txm.metadatas.*;
+
+import javax.xml.stream.*;
+
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/bvh/bugnames/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		TokenizerClasses.loadFromNode(params.getTokenizerElement(params.getCorpusElement()));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+String page_element = params.getPageElement("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir, "txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+File propertyFile = new File(rootDir, "import.properties")//default
+Properties props = new Properties();
+String[] metadatasToKeep;
+
+String textSortAttribute = null;
+boolean normalizeMetadata = false;
+String ignoredElements = null;
+boolean stopIfMalformed = false;
+
+println "Trying to read import properties file: "+propertyFile
+if (propertyFile.exists() && propertyFile.canRead()) {
+	InputStreamReader input = new InputStreamReader(new FileInputStream(propertyFile) , "UTF-8");
+	props.load(input);
+	input.close();
+	if(props.getProperty("sortmetadata") != null)
+		textSortAttribute = props.get("sortmetadata").toString();
+	if (props.getProperty("normalizemetadata") != null)
+		normalizeMetadata = Boolean.parseBoolean(props.get("normalizemetadata").toString());
+	if (props.getProperty("ignoredelements") != null)
+		ignoredElements = props.get("ignoredelements").toString();
+	if (props.getProperty("stopifmalformed") != null)
+		stopIfMalformed = Boolean.parseBoolean(props.get("stopifmalformed").toString());
+
+	println "import properties: "
+	println " sort metadata: "+textSortAttribute
+	println " normalize attributes: "+normalizeMetadata
+	println " ignored elements: "+ignoredElements
+	println " stop if a XML source is malformed: "+stopIfMalformed
+}
+
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+
+// Apply XSL
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "APPLYING XSL")
+if (xsl != null && xslParams != null && xsl.trim().length() > 0) {
+	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))
+	// return; // error during process
+	srcDir = new File(binDir, "src");
+	println ""
+}
+
+// copy xml+dtd files
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+List<File> srcfiles = srcDir.listFiles();
+if (srcfiles != null)
+	for (int i = 0 ; i < srcfiles.size() ; i++) {// check XML format, and copy file into binDir
+		File f = srcfiles.get(i)
+		if (f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties")) {
+			srcfiles.remove(i);
+			i--;
+			continue;
+		}
+		if (ValidateXml.test(f)) {
+			FileCopy.copy(f, new File(txmDir, f.getName()));
+		} else {
+			println "Won't process file "+f;
+		}
+	}
+
+if (txmDir.listFiles() == null) {
+	println "No txm file to process"
+	return;
+}
+
+// filtering
+/*def xpaths = params.getExcludeXpaths()
+ if (xpaths != null) {
+ println "Filtering XML files with xpaths: $xpaths"
+ for (File infile : txmDir.listFiles()) {
+ print "."
+ if (!RemoveTag.xpath(infile, xpaths)) {
+ println "Failed to filter $infile"
+ return
+ }
+ }
+ println ""
+ }*/
+
+//get metadata values from CSV
+Metadatas metadatas; // text metadata
+
+println "Trying to read metadata from: "+allmetadatasfile
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (allmetadatasfile.exists()) {
+	println " Metadata file found: "+allmetadatasfile
+	File copy = new File(binDir, "metadata.csv")
+	if (!FileCopy.copy(allmetadatasfile, copy)) {
+		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
+		return;
+	}
+	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+} else {
+	println " No metadata file: "+allmetadatasfile
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(5, "IMPORTER")
+println "-- IMPORTER - Reading source files"
+def imp = new importer();
+imp.doValidation(true) // change this to not validate xml
+boolean skipDoTokenizeStep = params.getSkipTokenization()
+imp.doTokenize(!skipDoTokenizeStep) // change this, to not tokenize xml
+imp.setStopIfMalformed(stopIfMalformed);
+if (!imp.run( srcDir, binDir, txmDir, basename, ignoredElements, lang)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA")
+if (metadatas != null) {
+	println("-- INJECTING METADATA - "+metadatas.getHeadersList()+" in texts of directory "+new File(binDir,"txm"))
+	for (File infile : txmDir.listFiles()) {
+		print "."
+		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {
+			outfile.delete();
+		} else {
+			if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+			if (!infile.exists()) {
+				println "Error: could not replace $infile by $outfile"
+				return false;
+			}
+		}
+	}
+	println ""
+}
+List<File> files = txmDir.listFiles()
+if (files == null || files.size() == 0) {
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = true;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if(debug) c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setOptions(textSortAttribute, normalizeMetadata);
+c.setAnnotationSuccess(annotationSuccess)
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname, null, srcfiles, metadatas)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(25, "EDITION")
+	
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+	def second = 0
+
+	println "Paginating texts: "+filelist
+
+	for (File txmFile : filelist) {
+		print "."
+		String txtname = txmFile.getName();
+		int i = txtname.lastIndexOf(".");
+		if (i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(corpusElem, txtname, txmFile);
+
+		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/fleursLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/fleursLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/fleursLoader.groovy	(revision 321)
@@ -0,0 +1,147 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.fleurs
+;
+
+import java.io.File;
+
+import org.txm.importer.fleurs.compiler;
+import org.txm.importer.fleurs.pager;
+import org.txm.importer.fleurs.importer;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		rootDir = userDir+"/xml/fleurs/";
+		lang="fr";
+		encoding= "UTF-8";// not used
+		model="rgaqcj"; // not used
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File(userDir,"TXMinstall"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+String basename = "fleurs";
+System.out.println("basename : "+basename);
+println "-- IMPORTER - Reading source files"
+File homedir = new File(rootDir);
+def imp = new importer();
+imp.run(homedir, basename);
+
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+rootDir = binDir.getAbsolutePath()+"/";
+
+println "-- ANNOTATE - Running NLP tools"
+println "No annotation to do"
+//new Annotate().run(new File(rootDir),model+".par");
+
+println "-- COMPILING - Building Search Engine indexes"
+List<File> files = new File(rootDir,"txm").listFiles();
+def c = new compiler()
+//c.setCwbPath("")// for developers
+c.setLang(lang);
+c.run(rootDir);
+
+//move registry file to cwb registry dir
+File registryfile = new File(rootDir,"registry/"+basename);
+if(registryfile.exists())
+	FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"registry/"+basename.toLowerCase()))
+
+Workspace w = org.txm.Toolbox.workspace;
+
+Project p = w.getProject("default")
+p.removeBase(basename)
+Base b = p.addBase(basename);
+b.addDirectory(new File(rootDir,"txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION - Building edition"
+new File(rootDir,"HTML").deleteDir();
+new File(rootDir,"HTML").mkdir();
+new File(rootDir,"HTML/default").mkdir();
+List<File> filelist = new File(rootDir,"txm").listFiles();
+def second = 0
+
+println "Paginating text: "
+for(Text text : b.getTexts())
+{
+	File srcfile = text.getSource();
+	File resultfile = new File(rootDir,"HTML/"+srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+		
+		if (second) { print(", ") }
+		if (second > 0 && (second % 5) == 0) println ""
+		print(srcfile.getName());
+		second++
+	
+	def ed = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,500,basename);
+	
+	Edition editionweb = text.addEdition("default","html",resultfile);
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+	
+//	Edition editionbp = text.addEdition("onepage","html",resultfile);
+//	editionbp.addPage(resultfile,ed.getIdx().get(0));
+}
+
+w.save();
+println ""
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/compiler.groovy	(revision 321)
@@ -0,0 +1,305 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.fleurs;
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler{
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private def parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * Instantiates a new compiler.
+	 */
+	public compiler(){}
+	
+	/**
+	 * initialize.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8")
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+		
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		createOutput(dirPathName, fileName);
+		
+		String vForm = "";
+		String wordid= "";
+		String w_l= "";
+		String w_lemme= "";
+		String w_pos= "";
+		String w_x= "";
+		String w_z= "";
+		
+		boolean flagForm = false;
+		output.write("<txmcorpus lang=\""+lang+"\">\n"); 
+		try 
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+					case "recueil":
+						output.write("<text id=\""+text+"\" base=\"fleurs\" project=\"default\">\n");
+						break;
+					case "w":
+						flagForm= true;
+						w_l = parser.getAttributeValue(null,"l");
+						w_lemme = parser.getAttributeValue(null,"lemme");
+						w_pos = parser.getAttributeValue(null,"pos");
+						w_x= parser.getAttributeValue(null,"x");
+						w_z= parser.getAttributeValue(null,"z");
+						wordid= parser.getAttributeValue(null,"id");
+						vForm ="";
+						break;
+					default:
+						output.write("<"+parser.getLocalName())
+						for(int i =0; i < parser.getAttributeCount() ; i++)
+								output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"")
+							output.write(">\n");
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.END_ELEMENT:
+					switch (parser.getLocalName()) 
+					{
+					case "recueil":
+						output.write("</text>\n");
+						break;
+					case "w":
+						flagForm= false;
+						output.write(vForm+"\t"+wordid+"\t"+w_l+"\t"+w_lemme+"\t"+w_pos+"\t"+w_x+"\t"+w_z+"\n")
+						break;
+					default:
+						output.write("</"+parser.getLocalName()+">\n");
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						if(flagForm)
+							vForm += parser.getText().trim();
+					
+					break;
+				}
+			}
+			output.write("</txmcorpus>"); 
+			output.close();
+			parser.close();
+			inputData.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+		
+		return true;
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDir the root dir
+	 */
+	public boolean run(String rootDir) 
+	{
+		//String rootDir = "~/xml/Hyperprince/";
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		new File(rootDir,"wtc/fleurs.wtc").delete();//cleaning&preparing
+		new File(rootDir,"wtc/").deleteDir();
+		new File(rootDir,"wtc/").mkdir();
+		new File(rootDir,"registry/").mkdir();
+		
+		//1- Transform into WTC file
+		List<File> files = new File(rootDir,"txm").listFiles();
+		for (File f : files) {
+			if (!f.exists()) {
+				println("file "+f+ " does not exists")	
+			}
+			else
+			{	
+				println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				def builder = new compiler(f.toURL(), txtname, "fleurs", "default");
+				builder.setLang lang
+				builder.transfomFileWtc(rootDir,"wtc/fleurs.wtc");
+			}
+		}
+		
+		//2- Import into CWB
+		def outDir =rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug true;
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug true;
+		
+		String[] pAttributes = ["id","l","lemme","pos","x","z"];
+		String[] sAttributes = ["txmcorpus:0+lang", "text:0+id+base+project","poeme:0+titre+genre+type+section+tranche_chrono+date+annee+titre_abr+preorig+schema1+schema2+n","strophe:0+ordre+nb_vers+structure","div:0+type+structure","vers:0+numvers+tphon+stru+n_syl+n_pied+cesure+rime"];
+		
+		try {
+			String regPath = outDirTxm + "/registry/"+"fleurs"
+			cwbEn.run(cwbLoc + "/cwb-encode", outDirTxm + "data", outDir + "/wtc/"+"fleurs.wtc", regPath,pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(cwbLoc + "/cwb-makeall", "FLEURS", outDirTxm + "/registry");
+			
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		
+		System.out.println("Done.") 
+		return true;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Fleur import module : Astartex</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/BuildEdition.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/BuildEdition.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/BuildEdition.groovy	(revision 321)
@@ -0,0 +1,264 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.fleurs;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build Discours corpus simple edition from a xml-tei. @author mdecorde */
+class BuildEdition {
+	int wordcount;
+	
+	/** The No space before. */
+	List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+	
+	/** The wordid. */
+	String wordid;
+	
+	/** The wordvalue. */
+	String wordvalue;
+	
+	/** The interpvalue. */
+	String interpvalue;
+	
+	/** The flaginterp. */
+	boolean flaginterp = false;
+	
+	/** The lastword. */
+	String lastword = " ";
+	
+	/** The wordtype. */
+	String wordtype;
+	
+	/** The flagform. */
+	boolean flagform = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	def writer;
+
+	/**
+	 * Instantiates a new builds the edition.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 */
+	BuildEdition(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter) {
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Process.
+	 */
+	void process()
+	{
+		String localname = "";
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "teiHeader":
+						if(parser.getAttributeValue(null,"type").equals("text"))
+						{
+							writer.write("<h1 class=\"text\">Discours</h1>")
+						}
+						else if(parser.getAttributeValue(null,"type").equals("corpus"))
+						{
+							writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+							writer.write("<html>");
+							writer.write("<head>");
+							writer.write("<title>Discours edition</title>");
+							writer.write("</head>");
+							writer.write("<body>");
+						}
+						break;
+						
+						case "text":
+							writer.write("<ul>")
+							writer.write("<li>type : "+parser.getAttributeValue(null,"type")+"</li>")
+							writer.write("<li>locuteur : "+parser.getAttributeValue(null,"loc")+"</li>")
+							writer.write("<li>date : "+parser.getAttributeValue(null,"date")+"</li>")
+							writer.write("</ul>")
+						
+						break;
+						case "p":
+							writer.write("<p>");
+						break;
+						
+						case "w":
+						wordid=parser.getAttributeValue(null,"id");
+						wordcount++;
+						break;
+						
+						case "ana":
+						flaginterp=true;
+						interpvalue+=" "+parser.getAttributeValue(null,"type")+":"
+						break;
+						
+						case "form":
+						wordvalue="";
+						interpvalue ="";
+						flagform=true;
+					}
+				break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "teiHeader":
+
+						break;
+						
+						case "p":
+							writer.write("</p>");
+						break;
+						
+						case "form":
+						flagform = false
+						
+						break;
+						
+						case "ana":
+						flaginterp = false
+						break;
+						
+						case "w":
+						int l = lastword.length();
+						interpvalue = interpvalue.replace("\"","&quot;");
+						if(NoSpaceBefore.contains(wordvalue))
+						writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+						else if(NoSpaceAfter.contains(lastword.substring(l-1, l)))
+						writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+						else if(wordvalue.startsWith("-") && wordvalue.length() > 0)
+						writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+						else
+						writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+						writer.write(wordvalue);
+						writer.write("</span>");
+						lastword=wordvalue;
+						break;
+					}
+				break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+				break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		inputData.close();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String rootDir = "~/.txm/cwb/data/discours/src/";
+		new File(rootDir+"/identity/").mkdir();
+		
+		File srcfile = new File(rootDir,"discours-p5.xml");
+		File resultfile = new File(rootDir,"discours-p5.html");
+		List<String> NoSpaceBefore = [",",".",")","]","}"];
+		List<String> NoSpaceAfter = ["'","(","[","{"];
+		println("build discours xml-tei file : "+srcfile+" to : "+resultfile );
+		
+		def builder = new BuildEdition(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter);
+		
+		return;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/pager.groovy	(revision 321)
@@ -0,0 +1,335 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.fleurs;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build Discours corpus simple edition from a xml-tei. @author mdecorde */
+class pager {
+	List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+	
+	/** The pagecount. */
+	int pagecount = 0;
+	
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename;
+	
+	/** The wordid. */
+	String wordid;
+	
+	/** The first word. */
+	boolean firstWord = true;
+	
+	/** The wordvalue. */
+	String wordvalue;
+	
+	/** The interpvalue. */
+	String interpvalue;
+	
+	/** The lastword. */
+	String lastword = " ";
+	
+	/** The wordtype. */
+	String wordtype;
+	
+	/** The flagform. */
+	boolean flagform = false;
+	
+	/** The flaginterp. */
+	boolean flaginterp = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	OutputStreamWriter writer;
+	
+	/** The multiwriter. */
+	OutputStreamWriter multiwriter = null;
+	
+	/** The infile. */
+	File infile;
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+	
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter, int max, String basename) {
+		this.basename = basename;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		try {
+			if(multiwriter != null)
+			{
+				multiwriter.write("</body>");
+				multiwriter.write("</html>");
+				multiwriter.close();
+			}
+			pagecount++;
+			File f = new File(outfile.getParent()+"/multi/",outfile.getName().substring(0,outfile.getName().length()-5)+"_"+pagecount+".html");
+			pages.add(f);
+			idxstart.add(wordid)
+			multiwriter = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("<head>");
+			multiwriter.write("<title>"+basename.toUpperCase()+" Edition - Page "+pagecount+"</title>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			this.outfile = outfile;
+
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+			// createNextOutput();
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process()
+	{
+		String localname = "";
+		
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		writer.write("<html>");
+		writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.write("<head>");
+		writer.write("<title>"+basename.toUpperCase()+" edition</title>");
+		writer.write("</head>");
+		writer.write("<body>");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "poeme":
+							createNextOutput();
+						writer.write("<h1 class=\"text\">"+parser.getAttributeValue(null,"titre")+"</h1>")
+						writer.write("<ul>")
+						for(int i =0 ; i < parser.getAttributeCount() ; i++)
+							writer.write("<li>"+parser.getAttributeLocalName(i)+" : "+parser.getAttributeValue(i)+"</li>");
+						writer.write("</ul><br/>")
+						
+						multiwriter.write("<h1 class=\"text\">"+parser.getAttributeValue(null,"titre")+"</h1>")
+						multiwriter.write("<ul>")
+						for(int i =0 ; i < parser.getAttributeCount() ; i++)
+							multiwriter.write("<li>"+parser.getAttributeLocalName(i)+" : "+parser.getAttributeValue(i)+"</li>");
+						multiwriter.write("</ul><br/>")
+						break;
+						
+						case "vers":
+							multiwriter.write("<br/>")
+							writer.write("<br/>")
+							break;
+							
+						case "strophe":
+							multiwriter.write("<br/>")
+							writer.write("<br/>")
+							break;
+							
+						case "w":
+						wordid=parser.getAttributeValue(null,"id");
+						if(firstWord)
+						{
+							firstWord=false;
+							this.idxstart.set(0,wordid);
+						}
+						interpvalue = "";
+						for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+						{
+							if(parser.getAttributeLocalName(i) != "id")
+								interpvalue += " "+parser.getAttributeLocalName(i)+":"+parser.getAttributeValue(i);
+						}
+
+						flagform = true;
+						wordvalue = "";
+						break;
+
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if(l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+							
+							if(interpvalue != null)
+								interpvalue = interpvalue.replace("\"","&quot;");
+							
+							if(NoSpaceBefore.contains(wordvalue) || 
+									NoSpaceAfter.contains(lastword) || 
+									wordvalue.startsWith("-") || 
+									NoSpaceAfter.contains(endOfLastWord))
+							{
+								writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							else
+							{
+								writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							writer.write(wordvalue.replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+						flagform = false;
+						break;
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		writer.close();
+		multiwriter.write("</body>");
+		multiwriter.write("</html>");
+		multiwriter.close();
+		inputData.close();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/BuildTEI.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/BuildTEI.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/BuildTEI.groovy	(revision 321)
@@ -0,0 +1,245 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.fleurs;
+
+import javax.xml.stream.*;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class BuildTEI.
+ */
+class BuildTEI {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The output. */
+	private OutputStreamWriter output;
+
+	/**
+	 * Instantiates a new builds the tei.
+	 *
+	 * @param discoursxml the discoursxml
+	 */
+	public BuildTEI(File discoursxml) {
+		this.url = discoursxml.toURI().toURL();
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			output = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Write header.
+	 */
+	public void writeHeader() {
+		output.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+		output
+				.write("<teiCorpus xmlns=\"http://www.tei-c.org/ns/1.0\" xmlns:txm=\"http://textometrie.org/1.0\">\n");
+		output.write("<teiHeader type=\"corpus\">\n");
+		output.write("</teiHeader>\n");
+	}
+
+	/**
+	 * Write footer.
+	 */
+	public void writeFooter() {
+		output.write("</teiCorpus>\n");
+	}
+
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+	 */
+	public void process(File outfile)
+	{
+		createOutput(outfile);
+		int sentence=0;
+		int paragraph=0;
+		int idword=0;
+		
+		String lastopenlocalname= "";
+		String localname = "";
+				
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+					lastopenlocalname = localname;
+					switch (localname) {
+						case "corpus":
+							this.writeHeader();
+						break;
+						case "w":
+							idword++;	
+							
+							int para = Integer.parseInt(parser.getAttributeValue(null,"para"));
+							int sent = Integer.parseInt(parser.getAttributeValue(null,"sent"));
+							if(sent != sentence)
+							{
+								output.write("</s>\n")
+								
+								if(para != paragraph)
+								{
+									output.write("</p>\n")
+									output.write("<p id=\"p_"+para+"\">\n");
+									paragraph = para;
+								}
+								
+								output.write("<s id=\"s_"+sent+"\">\n");
+								sentence = sent;
+							}
+
+							output.write("<w id=\"w_"+idword+"\">\n");
+							output.write(" <txm:form>"+parser.getAttributeValue(null,"form")+"</txm:form>\n");
+							output.write(" <txm:ana resp=\"#cordial\" type=\"#pos\">"+parser.getAttributeValue(null,"pos")+"</interp>\n");
+							output.write(" <txm:ana resp=\"#cordial\" type=\"#func\">"+parser.getAttributeValue(null,"func")+"</interp>\n");
+							output.write(" <txm:ana resp=\"#cordial\" type=\"#lemma\">"+parser.getAttributeValue(null,"lem")+"</interp>\n");
+							output.write("</w>\n");
+							
+						break;
+						case "text":
+							output.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+							output.write("<TEI xmlns=\"http://www.tei-c.org/ns/1.0\" xmlns:txm=\"http://textometrie.org/1.0\">\n");
+							output.write("<teiHeader type=\"text\">\n");
+							output.write("<fileDesc>\n");
+							output.write("<titleStmt>\n");
+							output.write("<title>"+parser.getAttributeValue(null,"file")+"</title>\n");
+							output.write("<respStmt>\n");
+							output.write("<resp id=\"cordial\">initial tagging</resp>")
+							output.write("</respStmt>\n");
+							output.write("</titleStmt>\n");
+							output.write("</fileDesc>\n");
+							output.write("<encodingDesc>\n");
+							output.write("<classDecl>\n");
+							output.write("<taxonomy id=\"pos\"><bibl type=\"tagset\"/></taxonomy>\n")
+							output.write("<taxonomy id=\"func\"><bibl type=\"tagset\"/></taxonomy>\n")
+							output.write("<taxonomy id=\"lemma\"><bibl type=\"tagset\"/></taxonomy>\n")
+							output.write("</classDecl>\n");
+							output.write("</encodingDesc>\n");
+							output.write("</teiHeader>\n");
+							output.write("<text id=\""+parser.getAttributeValue(null,"id")+"\" type=\""+parser.getAttributeValue(null,"type")+"\" loc=\""+parser.getAttributeValue(null,"loc")+"\" date=\""+parser.getAttributeValue(null,"date")+"\" file=\""+parser.getAttributeValue(null,"file")+"\">\n");
+							output.write("<p id=\"p_1\">\n");
+							output.write("<s id=\"s_1\">\n");
+							paragraph=1;
+							sentence=1;
+						break;
+					}
+					break;	
+					
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName();
+					switch (localname) {				
+						
+						case "w":
+						break;
+						
+						case "text":
+							output.write("</s>\n");
+							output.write("</p>\n");
+							paragraph=1;
+							sentence=1;
+							output.write("</text>\n");
+							output.write("</TEI>\n");
+						break;
+						
+						case "corpus":
+							this.writeFooter();
+						break;
+					}
+					break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						//output.write(parser.getText().trim());				
+					break;
+				}
+			}
+			output.close();
+			parser.close();
+			inputData.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/discours/src";
+
+		File srcfile = new File(rootDir, "discours.xml");
+		File resultfile = new File(rootDir, "discours-tei.xml");
+		println("build discours xml-tei file : " + srcfile + " to : "
+				+ resultfile);
+
+		def builder = new BuildTEI(srcfile);
+		builder.process(resultfile);
+
+		return;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/fleurs/importer.groovy	(revision 321)
@@ -0,0 +1,239 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.fleurs
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import javax.xml.stream.*;
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.List;
+
+import org.txm.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer 
+{
+	
+	/**
+	 * Run.
+	 *
+	 * @param dir the dir
+	 * @param basename the basename
+	 */
+	public void run(File dir, String basename)
+	{
+		String rootDir = dir.getAbsolutePath()+"/"
+		//cleaning
+		File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+		binDir.deleteDir();
+		binDir.mkdir();
+		new File(binDir,"txm").deleteDir();
+		new File(binDir,"txm").mkdir();
+		
+		File srcfile = new File(rootDir).listFiles()[0];
+		File resultfile = new File(binDir,"txm/"+srcfile.getName());
+		
+		this.process(srcfile, resultfile);		
+	}
+	
+	/**
+	 * Process.
+	 *
+	 * @param srcfile the srcfile
+	 * @param resultfile the resultfile
+	 */
+	public void process(File srcfile, File resultfile)
+	{
+		
+		String localname= "";
+		URL url = srcfile.toURI().toURL();
+		def inputData = url.openStream();
+		def factory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+		OutputStreamWriter output = new OutputStreamWriter(new FileOutputStream(resultfile) , "UTF-8");
+		
+		println resultfile.getAbsolutePath()
+		
+		String poeme_titre ="";
+		String poeme_type ="";
+		String poeme_genre ="";
+		String poeme_section ="";
+		String poeme_tranchechrono ="";
+		String poeme_date ="";
+		String poeme_annee ="";
+		String poeme_titreabr ="";
+		String poeme_preorig ="";
+		String poeme_schema1 ="";
+		String poeme_schema2 ="";
+		
+		int wordid = 1;
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "M"://f, l, x test si null
+						case "MC":
+						case "PC":
+							String x = parser.getAttributeValue(null,"x");
+							String z = parser.getAttributeValue(null,"z");
+							String l = parser.getAttributeValue(null,"l");
+							String f = parser.getAttributeValue(null,"f");
+							String pos = "none";
+							String lemme = "none";
+							if(l != null)
+							{
+								String[] split = l.split("_");
+								if(split.length == 2)
+								{
+									pos = split[1];
+									lemme = split[0];
+								}
+							}
+						
+							if(x == null)
+								x = "n/a";
+							if(z == null)
+								z = "1";
+							output.write("<w id=\"w_"+(wordid++)+"\" l=\""+l+"\" lemme=\""+lemme+"\" pos=\""+pos+"\" x=\""+x+"\" z=\""+z+"\">"+f+"</w>\n")
+							break;
+						
+						case "RECUEIL":
+							output.write("<recueil>\n");
+						
+							break;
+						case "POEME":
+							poeme_titre = parser.getAttributeValue(null,"TITRE");
+							poeme_type = parser.getAttributeValue(null,"TYPE");
+							poeme_genre = parser.getAttributeValue(null,"GENRE");
+							poeme_section = parser.getAttributeValue(null,"SECTION");
+							poeme_tranchechrono = parser.getAttributeValue(null,"TRANCHE_CHRONO");
+							poeme_date = parser.getAttributeValue(null,"DATE");
+							poeme_annee = parser.getAttributeValue(null,"ANNEE");
+							poeme_titreabr = parser.getAttributeValue(null,"TITRE_ABR");
+							poeme_preorig = parser.getAttributeValue(null,"PREORIG");
+							poeme_schema1 = parser.getAttributeValue(null,"SCHEMA1");
+							poeme_schema2 = parser.getAttributeValue(null,"SCHEMA2");
+							break;	
+						case "ASTX_ATTR":
+							output.write("<poeme");
+							output.write(" titre=\""+poeme_titre+"\"");
+							if(poeme_type == "")
+								output.write(" type=\"v\"");
+							else
+								output.write(" type=\""+poeme_type+"\"");
+							if(poeme_genre == "")
+								output.write(" genre=\"v\"");
+							else
+								output.write(" genre=\""+poeme_genre+"\"");
+							output.write(" section=\""+poeme_section+"\"");
+							output.write(" tranche_chrono=\""+poeme_tranchechrono+"\"");
+							output.write(" date=\""+poeme_date+"\"");
+							output.write(" annee=\""+poeme_annee+"\"");
+							output.write(" titre_abr=\""+poeme_titreabr+"\"");
+							output.write(" preorig=\""+poeme_preorig+"\"");
+							output.write(" schema1=\""+poeme_schema1+"\"");
+							output.write(" schema2=\""+poeme_schema2+"\"");
+							output.write(" n=\""+poeme_schema2+"\"");
+							output.write(">\n");
+							break;
+						
+						case "HUITAIN":
+						case "SIZAIN":
+							output.write("<div type=\""+localname.toLowerCase()+"\"");
+							for(int i =0; i < parser.getAttributeCount() ; i++)
+								output.write(" "+parser.getAttributeLocalName(i).toLowerCase()+"=\""+parser.getAttributeValue(i)+"\"")
+							output.write(">\n");
+							break;
+						case "STROPHE":
+						case "VERS":
+							output.write("<"+localname.toLowerCase());
+							for(int i =0; i < parser.getAttributeCount() ; i++)
+								output.write(" "+parser.getAttributeLocalName(i).toLowerCase()+"=\""+parser.getAttributeValue(i)+"\"")
+							output.write(">\n");
+							break;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						
+						case "RECUEIL":
+							output.write("</recueil>\n");
+							break;
+						
+						case "ASTX_ATTR":
+							output.write("</poeme>\n");
+							break;
+						
+						case "HUITAIN":
+						case "SIZAIN":
+							output.write("</div>\n");
+							break;
+						case "STROPHE":
+						case "VERS":
+							output.write("</"+localname.toLowerCase()+">\n");
+							break;
+					}
+					break;
+				case XMLStreamConstants.CHARACTERS:
+				//output.write(parser.getText().trim());				
+					break;
+			}
+		}
+		output.close();
+		parser.close();
+		inputData.close();
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File(System.getProperty("user.home"),"xml/fleurs/");
+		String rootDir = dir.getAbsolutePath();
+		File homedir = new File(rootDir);
+		def imp = new importer();
+		imp.run(homedir);
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SAttributesListener.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SAttributesListener.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SAttributesListener.groovy	(revision 321)
@@ -0,0 +1,217 @@
+package org.txm.importer
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.stream.*;
+
+/**
+ * Read an XML file and find out : XML elements, their attributes and recursivity level
+ * names are lowercases
+ * 
+ * @author mdecorde
+ *
+ */
+class SAttributesListener {
+
+	private static HashMap<String,ArrayList<String>> structs = new HashMap<String, ArrayList<String>>();
+	private static HashMap<String, Integer> structsCountProf = new HashMap<String, Integer>();
+	private static HashMap<String, Integer> structsMaxProf = new HashMap<String, Integer>();
+	private static String structPath = "/";
+	private XMLStreamReader parser;
+
+	SAttributesListener() {
+		structs = new HashMap<String, ArrayList<String>>();
+		structsCountProf = new HashMap<String, Integer>();
+		structsMaxProf = new HashMap<String, Integer>();
+		structPath = "/";
+	}
+	
+	SAttributesListener(XMLStreamReader parser) {
+		this();
+		this.parser = parser;
+	}
+	
+	public void appendResultsTo(SAttributesListener another) {
+		structs = another.structs;
+		structsCountProf = another.structsCountProf;
+		structsMaxProf = another.structsMaxProf;
+	}
+
+	public void start(def parser) {
+		this.parser = parser;
+	}
+
+	def W = "w";
+	def ANA = "ana";
+	def FORM = "form";
+	/**
+	 * Call this method for each START_ELEMENT stax event
+	 * @param localname the element localname
+	 */
+	public void startElement(String localname) {
+		localname = localname.toLowerCase();
+
+		//String localname = parser.getLocalName();
+		if(localname.equals(W)) return;
+		if(localname.equals(ANA)) return;
+		if(localname.equals(FORM)) return;
+
+		structPath += localname+"/"
+		//println "add: "+structPath
+		def attrs = structs.get(localname)
+		if (!structs.containsKey(localname)) {
+			attrs = new HashSet();
+			structs.put(localname, attrs);
+			structsCountProf.put(localname, 0)
+			structsMaxProf.put(localname, 0)
+		} //else {
+		
+		// get structure recursion
+		int prof = structsCountProf.get(localname)+1
+		structsCountProf.put(localname, prof)
+		if (structsMaxProf.get(localname) < prof )
+			structsMaxProf.put(localname, prof)
+		
+		// get the structure attributes
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			attrs << parser.getAttributeLocalName(i).toLowerCase();
+		}
+	}
+
+	/**
+	 * Call this method for each END_ELEMENT stax event
+	 * @param localname the element localname
+	 */
+	public void endElement(String localname) {
+		localname = localname.toLowerCase();
+		//String localname = parser.getLocalName();
+		if(localname.equals(W)) return;
+		if(localname.equals(ANA)) return;
+		if(localname.equals(FORM)) return;
+
+		if (structPath.length() > 1) {
+			int idx = structPath.lastIndexOf("/");
+			if (idx > 0) {
+				structPath = structPath.substring(0, idx)
+				//println "end of $localname "+(structsCountProf.get(localname))
+				//if (structsCountProf.get(localname) != null)
+				structsCountProf.put(localname, structsCountProf.get(localname)-1)
+			}
+			//println "pop: "+structPath
+		}
+	}
+
+	boolean firstGetStructs = true;
+	public HashMap<String,ArrayList<String>> getStructs() {
+		if (structsCountProf.get("div") > 0)
+			structs.remove("div1")
+		if (structsCountProf.get("div") > 1)
+			structs.remove("div2")
+		if (structsCountProf.get("div") > 2)
+			structs.remove("div3")
+		if (structsCountProf.get("div") > 3)
+			structs.remove("div4")
+		if (structsCountProf.get("div") > 4)
+			structs.remove("div5")
+		if (structsCountProf.get("div") > 5)
+			structs.remove("div6")
+
+		if (firstGetStructs) {
+			firstGetStructs = false;
+			def keys = []
+			keys.addAll(structs.keySet());
+			for( String key : keys) {
+				def value = structs.get(key);
+				structs.remove(key)
+				structs.put(key.toLowerCase(), value);
+			}
+		}
+
+		return structs;
+	}
+
+	boolean firstGetstructsCountProf = true;
+	public HashMap<String, Integer> getProfs() {
+
+		//		if (firstGetstructsCountProf) {
+		//			firstGetstructsCountProf = false;
+		//			def keys = []
+		//			keys.addAll(structsCountProf.keySet());
+		//			for( String key : keys) {
+		//				def value = structsCountProf.get(key);
+		//				structsCountProf.remove(key)
+		//				structsCountProf.put(key.toLowerCase(), value);
+		//			}
+		//		}
+		HashMap<String, Integer> clone = new HashMap<String, Integer>();
+		for (String key : structsMaxProf.keySet()) {
+			if (structsMaxProf.get(key) > 0)
+				clone.put(key, structsMaxProf.get(key)-1)
+			else
+				clone.put(key, 0)
+		}
+		return clone;
+	}
+	
+	public static SAttributesListener scanFile(File xmlFile) {
+		return scanFile(xmlFile, null)
+	}
+	
+	public void setParser(def parser) {
+		this.parser = parser;
+	}
+	
+	/**
+	 * Merge results in the parentListener
+	 * 
+	 * @param xmlFile
+	 * @param parentListener results are appended to the parentListener if any
+	 * @return
+	 */
+	public static SAttributesListener scanFile(File xmlFile, SAttributesListener parentListener) {
+		
+		def start = false;
+		def inputData = xmlFile.toURI().toURL().openStream();
+		def factory = XMLInputFactory.newInstance();
+		def parser = factory.createXMLStreamReader(inputData);
+		
+		SAttributesListener listener;
+		if (parentListener != null) {
+			listener = parentListener;
+			listener.setParser(parser)
+		} else {
+			listener = new SAttributesListener(parser);
+		}
+		String TEXT = "text";
+		//HashSet<String> types = new HashSet<String>();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) { // start elem
+				if (TEXT.equals(parser.getLocalName())) start = true;
+				if (start) listener.startElement(parser.getLocalName())
+			} else if (event == XMLStreamConstants.END_ELEMENT) { // end elem
+				if (start) listener.endElement(parser.getLocalName())
+				if (TEXT.equals(parser.getLocalName())) start = false;
+			}
+		}
+		parser.close()
+		
+		return listener
+	}
+	
+	public static SAttributesListener scanFiles(File xmlDirectory, String wordTag) {
+		SAttributesListener listener = new SAttributesListener()
+		listener.W = wordTag
+		for (File xmlFile : xmlDirectory.listFiles()) {
+			if (xmlFile.isFile() && !xmlFile.isHidden() && xmlFile.getName().toLowerCase().endsWith(".xml")) {
+				scanFile(xmlFile, listener); // results saved in 'listener' data
+//				println "LISTENER RESULT with ${xmlFile.getName()}: "+listener
+//				println " prof: "+listener.getStructs()
+//				println " prof: "+listener.getProfs()
+//				println " path: "+listener.structPath
+			}
+		}
+		
+		return listener;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/BuildBiblio.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/BuildBiblio.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/BuildBiblio.groovy	(revision 321)
@@ -0,0 +1,193 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer.bfm;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import org.txm.utils.*;
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class BuildBiblio.
+ */
+class BuildBiblio {
+	
+	/**
+	 * Process.
+	 *
+	 * @param csvfile the csvfile
+	 * @param separator the separator
+	 * @param encoding the encoding
+	 * @param outputfile the outputfile
+	 * @param buildIndex the build index
+	 * @return true, if successful
+	 */
+	public static boolean process(File csvfile, String separator, String encoding, File outputfile, boolean buildIndex) {
+		if(!(csvfile.exists() && csvfile.canRead()))
+		{
+			println "csvfile does not exist"
+			return false;
+		}
+		
+		if(!outputfile.getParentFile().exists())
+		{
+			println "outpurfile's parent does not exist"
+			return false;
+		}
+		
+		CsvReader csvreader = new CsvReader(csvfile.getAbsolutePath(), separator.charAt(0), Charset.forName(encoding))
+		
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		FileOutputStream output = new FileOutputStream(outputfile)
+		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8");
+		
+		if(csvreader == null)
+		{
+			println "failed to build csvreader"
+			return false;
+		}
+		
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeStartElement("html");
+		//header
+		writer.writeStartElement("head");
+		//writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.writeStartElement("meta");
+		writer.writeAttribute("http-equiv","Content-Type");
+		writer.writeAttribute("content","text/html");
+		writer.writeAttribute("charset","UTF-8");
+		writer.writeEndElement();//meta
+		writer.writeStartElement("title");
+		writer.writeCharacters("Fiches biblio BFM")
+		writer.writeEndElement();//title
+		writer.writeEndElement();//head
+		
+		//get titles
+		csvreader.readHeaders();
+		def titles = [];
+		for(String title : csvreader.getHeaders())
+		{
+			titles << title
+		}
+		
+		writer.writeStartElement("body");
+		
+		if(buildIndex)
+		{
+		writer.writeStartElement("ul");
+		while (csvreader.readRecord()){
+			
+			writer.writeStartElement("li");
+			writer.writeStartElement("a");
+			writer.writeAttribute("href", "#"+csvreader.get("id"))
+			writer.writeCharacters(csvreader.get("id"))
+			writer.writeEndElement();//a
+			writer.writeEndElement();//li			
+		}
+		writer.writeEndElement();//ul
+		}
+		
+		println "Metadata declared: "+titles
+		csvreader = new CsvReader(csvfile.getAbsolutePath(), separator.charAt(0), Charset.forName(encoding))
+		csvreader.readHeaders();
+		while (csvreader.readRecord()){
+			writer.writeStartElement("div");
+			writer.writeAttribute("class", "fiche");
+			
+			writer.writeStartElement("a")
+			writer.writeAttribute("name", csvreader.get("id"));
+			writer.writeEndElement();//a
+			
+			writer.writeStartElement("h1");
+			String auteur = csvreader.get("Auteur")
+			String titre = csvreader.get("Titre")
+			if(auteur != null && auteur != "anonyme")
+				writer.writeCharacters(auteur+", ")
+			writer.writeCharacters(titre);
+			writer.writeEndElement();//h1
+			
+			writer.writeStartElement("ul");
+			for (int i = 0 ; i < titles.size() ; i++) {
+				writer.writeStartElement("li");
+				writer.writeStartElement("b");
+				writer.writeCharacters(titles.get(i)+" : ")
+				writer.writeEndElement();
+				
+				writer.writeCharacters(csvreader.get(titles.get(i)))
+				writer.writeEndElement();//li
+			}
+			
+			writer.writeEndElement();//ul
+			writer.writeEndElement();//div
+		}
+		
+		writer.writeEndElement();//body
+		writer.close();
+		output.close()
+		return true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File csvfile = new File("C:\\Documents and Settings\\alavrent\\Mes documents\\bfm - local\\Bases bibliographiques (copie)\\Fiches_biblio_TXM.txt")
+		File outputfile = new File("C:\\Documents and Settings\\alavrent\\Mes documents\\bfm - local\\Bases bibliographiques (copie)\\Fiches_biblio_TXM.html")
+		String separator = "\t"
+		String encoding = "cp1252"
+		
+		if(!BuildBiblio.process(csvfile, separator, encoding, outputfile, false))
+			println("Error.")
+		else
+			println "Success";
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/BuildBiblioFromHeaders.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/BuildBiblioFromHeaders.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/BuildBiblioFromHeaders.groovy	(revision 321)
@@ -0,0 +1,125 @@
+package org.txm.importer.bfm;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import org.txm.utils.*;
+import org.txm.importer.*;
+
+import javax.xml.stream.*;
+import java.net.URL;
+	
+
+String xslfile = "G:\\Feuilles de style\\teiHeader2ficheBiblio.xsl";
+ApplyXsl a = new ApplyXsl(xslfile);
+
+File srcdirectory = new File("C:\\Documents and Settings\\alavrent\\TXM-SRC\\corptef")
+File outdirectory = new File("C:\\Documents and Settings\\alavrent\\TXM-SRC\\corptef-biblio")
+outdirectory.mkdir();
+assert(srcdirectory.exists())
+assert(outdirectory.exists())
+assert(srcdirectory.isDirectory())
+assert(srcdirectory.canRead())
+assert(srcdirectory.canExecute())
+
+List<File> files = srcdirectory.listFiles();
+
+files = files.findAll{item-> item.getName().endsWith(".xml")}
+
+long bigstart = System.currentTimeMillis()
+for(File infile : files)
+{
+	String outfile = outdirectory.getAbsolutePath()+"/"+infile.getName().substring(0,infile.getName().length()-4)+"-biblio.html";
+	if(infile.canRead())
+	{
+		long start = System.currentTimeMillis()
+		println "Process : "+infile.getName()
+		// String[] params = ["pbval1",1,"pbval2",2];
+		a.process(infile.getAbsolutePath(), outfile); // no parameters
+		// a.process( infile.getAbsolutePath(),outfile,params);
+		println "Done : "+(System.currentTimeMillis()-start)+"ms"
+	}
+}
+println "Total time : "+(System.currentTimeMillis()-bigstart)+"ms"
+/*
+class BuildBiblioFromHeaders {
+	
+	public static boolean process(File dir, File xslfile, File outputfile) {
+		if(!(xslfile.exists() && xslfile.canRead()))
+		{
+			println "xslfile does not exist"
+			return false;
+		}
+		
+		if(!outputfile.getParentFile().exists())
+		{
+			println "outputfile's parent does not exist"
+			return false;
+		}
+		
+		
+		
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		FileOutputStream output = new FileOutputStream(outputfile)
+		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8");
+		
+		
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeStartElement("html");
+		//header
+		writer.writeStartElement("head");
+		//writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.writeStartElement("meta");
+		writer.writeAttribute("http-equiv","Content-Type");
+		writer.writeAttribute("content","text/html");
+		writer.writeAttribute("charset","UTF-8");
+		writer.writeEndElement();//meta
+		writer.writeStartElement("title");
+		writer.writeCharacters("Fiches biblio BFM")
+		writer.writeEndElement();//title
+		writer.writeEndElement();//head
+		
+		writer.writeStartElement("body");
+		
+		String rootDir = dir.getAbsolutePath()+"/";
+		
+		List<File> files = new File(rootDir,"").listFiles();//scan directory split
+		
+		for(File f : files) {
+			
+//			ApplyXsl a = new ApplyXsl(xslfile.getAbsolutePath());
+//			String[] args = ["outputDir",xxx];
+//			a.process(f.getAbsolutePath(), "./HTML",args);
+			
+			
+
+		}
+		
+		
+		writer.writeEndElement();//body
+		writer.close();
+		return true;
+	}
+	
+	
+	
+	public static void main(String[] args)
+	{
+	
+		String rootDir;
+		String htmloutdir;
+		File dir = new File(rootDir);
+		
+		rootDir = "C:\\Documents and Settings\\alavrent\\TXM-SRC\\corptef";
+		File xslfile = new File("G:\\Feuilles de style\\teiHeader2ficheBiblio.xsl")
+		File outputfile = new File("C:\\Documents and Settings\\alavrent\\Mes documents\\bfm - local\\Bases bibliographiques (copie)\\Fiches_biblio_TXM_test.html")
+
+		
+		if(!BuildBiblioFromHeaders.process(xslfile, outputfile, false))
+			println("Error.")
+		else
+			println "Success";
+	}
+}
+*/
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/importer.groovy	(revision 321)
@@ -0,0 +1,303 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-05-11 08:38:47 +0200 (Wed, 11 May 2016) $
+// $LastChangedRevision: 3211 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.bfm;
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Properties;
+
+import org.txm.*;
+
+import filters.CutHeader.*;
+import filters.ReunitBrokenWords.*;
+import filters.Tokeniser.*;
+import filters.TagSentences.*;
+import filters.FusionHeader.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Convert TEI-BFM to TEI-TXM: <br/>
+ * 1- tokenize : manage BFM's &lt;choice> policy (see <a
+ * href="http://bfm.ens-lsh.fr">BFM website</a>).<br/>
+ * 2- launch XML2TEITXM.groovy with no annotation to import. <br/>
+ *
+ * @param dir contains the sources files
+ * @param basename the basename
+ * @return true, if successful
+ * @author mdecorde
+ */
+class importer {
+	
+	public boolean run(File srcDir, File binDir, File txmDir, String basename, Properties metadataXPath)
+	{
+		new File(binDir, "ptokenized").deleteDir();
+		new File(binDir, "ptokenized").mkdir();
+		new File(binDir, "tokenized").deleteDir();
+		new File(binDir, "tokenized").mkdir();
+		new File(binDir, "headers").deleteDir();
+		new File(binDir, "headers").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();
+		milestones.add("lb");
+		milestones.add("pb");
+		milestones.add("milestone");
+
+		List<File> files = txmDir.listFiles();
+		if (files == null || files.size() == 0) return false;
+
+		for (int i = 0 ; i < files.size() ; i++)
+			if (!(files.get(i).getName().endsWith(".xml"))
+			|| files.get(i).getName().equals("import.xml"))
+				files.remove(i--);
+
+		for (File f : files) {
+			//System.out.println("Get metadata from file $xpathfile");
+			HashMap<String, String> metadataValues = new HashMap<String, String>();
+			if (metadataXPath != null) {
+				def xpathprocessor = new XPathResult(f)
+				for (String name : metadataXPath.keySet()) {
+					String value = xpathprocessor.getXpathResponse(metadataXPath.get(name), "N/A");
+					value = value.trim().replace("\n", " ");
+					//println "xpath: "+metadataXPath.get(name)+" >> "+value.trim()
+					metadataValues.put(name, value);
+				}
+				xpathprocessor.close();
+			}
+			
+			//forme is mandatory !!!
+			String forme = "prose";
+			if (metadataValues.containsKey("forme")) {
+				String tmp = metadataValues.get("forme");
+				if (tmp.startsWith("#forme_"))
+					tmp = tmp.substring(7)
+				metadataValues.put("forme", tmp)
+				forme = metadataValues.get("forme");
+			}
+
+			if (metadataValues.containsKey("morphosynt")) {
+				//println "MORPHOSYNT: "+metadataValues.get("morphosynt");
+				if (metadataValues.get("morphosynt").equals("N/A"))
+					metadataValues.put("morphosynt","non vérifié");
+				else
+					metadataValues.put("morphosynt","vérifié");
+			}
+
+			String txtname = f.getName().substring(0, f.getName().length()-4);
+			String titreId = txtname;
+			if (metadataValues.containsKey("sigle")) {
+				titreId = metadataValues.get("sigle");
+			} else if (metadataValues.containsKey("idbfm")) {
+				titreId = metadataValues.get("idbfm");
+			}
+			metadataValues.put("sigle", titreId);
+			
+			// inject attributes in $f
+			AddAttributeInXml builder = new AddAttributeInXml(f, "text", metadataValues);
+			builder.onlyOneElement();
+			//println "INJECTING $metadataValues"
+			File outfile = new File(txmDir, "copy"+f.getName())
+			
+			if (!builder.process(outfile)) {
+				println "Error while injecting metadata in $f"
+				return false;
+			}
+			builder= null;
+			if (f.delete()) {
+				outfile.renameTo(f)
+			} else {
+				println "ERROR could not delete $f for metadatainjection"
+				return false
+			}
+		}
+
+		files = txmDir.listFiles();
+		if (files == null || files.size() == 0) return false;
+		
+		//return false; // TEMP TEMP TEMP
+		
+		//PREPARE EACH FILE TO BE TOKENIZED
+		println("preparing "+files.size()+" files for the tokenizer");
+
+		for (File f : files) {
+			if (f.isDirectory()) continue;
+			print "."//+f.getAbsolutePath()
+			File srcfile = f;
+			File resultfile = new File(binDir,"ptokenized/"+f.getName());
+
+			def builder2 = new OneTagPerLine(srcfile.toURI().toURL(), milestones);
+			if (!builder2.process(resultfile)) {
+				println "Failed to tokenize file: "+f;
+				resultfile.delete();
+			}
+		}
+		println ""
+
+		//TOKENIZE FILES
+		files = new File(binDir, "ptokenized").listFiles();//scan directory split
+		println("Tokenizing "+files.size()+" files")
+		for (File infile : files) {
+			print "."
+			try {
+				Sequence S = new Sequence();
+				Filter F1 = new CutHeader();
+				Filter F2 = new ReunitBrokenWords();
+				Filter F6 = new Tokeniser(infile);
+				Filter F7 = new TagSentences();
+				Filter F11 = new FusionHeader();
+				S.add(F1);
+				//S.add(F2);
+				S.add(F6);
+				//S.add(F7);
+				S.add(F11);
+
+				File xmlfile = new File(binDir,"tokenized/"+infile.getName());
+
+				S.SetInFileAndOutFile(infile.getAbsolutePath(), xmlfile.getAbsolutePath());
+				S.setEncodages("UTF-8","UTF-8");
+
+				Object[] arguments1 = [new File(binDir,"/headers/"+infile.getName()+"header.xml").getAbsolutePath()];
+				F1.SetUsedParam(arguments1);
+				Object[] arguments2 = [new File(binDir,"/headers/"+infile.getName()+"header.xml").getAbsolutePath(),F1];
+				F11.SetUsedParam(arguments2);
+
+				S.proceed();
+				S.clean();
+			} catch(Exception e) {
+				org.txm.utils.logger.Log.printStackTrace(e);
+				return false;
+			}
+		}
+		println ""
+
+
+		files = new File(binDir,"tokenized").listFiles()
+		println "Validating XML of "+files.size()+" files"
+		for (File f : files) {
+			print "."
+			if (!ValidateXml.test(f)) {
+				System.out.println("XML NOT VALID "+f);
+				new File(binDir, "tokenizerError").mkdir();
+				FileCopy.copy(f,new File(binDir, "tokenizerError/"+f.getName()));
+				f.delete();
+			}
+		}
+		println("");
+
+		//TRANSFORM INTO XML-TEI-TXM
+		files = new File(binDir, "tokenized").listFiles()
+		println("Building xml-tei-txm "+files.size()+ " files")
+		if (files.size() == 0) return false;
+		for (File file : files) {
+			print "."
+			String txmfile = file.getName();
+
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+
+			//lance le traitement
+			def builder3 = new Xml2Ana(file);
+			builder3.setCorrespondances(correspRef, correspType);
+			builder3.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			if (!builder3.process(new File(txmDir,txmfile))) {
+				println "Failed to build xml-txm file from file: "+file;
+				return false;
+			}
+		}
+		println("")
+		return true;
+	}
+
+	/**
+	 * for test purpose.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		long start = System.currentTimeMillis();
+		File dir = new File("/home/mdecorde/xml/qgraal")
+		File srcfile = new File(dir, "qgraal_cm-detoken.xml");
+		File resultfile = new File(dir,"QGRAALFRO-pt1.xml");
+		File xmlfile = new File(dir,"QGRAALFRO-t1.xml");
+
+		def builder2 = new OneTagPerLine(srcfile.toURI().toURL(), []);
+		if (!builder2.process(resultfile)) {
+			println "Failed to tokenize file: "+srcfile;
+			resultfile.delete();
+		}
+
+		try {
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F2 = new ReunitBrokenWords();
+			Filter F6 = new Tokeniser(resultfile);
+			Filter F7 = new TagSentences();
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			//S.add(F2);
+			S.add(F6);
+			//S.add(F7);
+			//S.add(F11);
+
+			S.SetInFileAndOutFile(resultfile.getAbsolutePath(), xmlfile.getAbsolutePath());
+			S.setEncodages("UTF-8","UTF-8");
+
+			Object[] arguments1 = [new File(dir,"header.xml").getAbsolutePath()];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [new File(dir,"header.xml").getAbsolutePath(),F1];
+			F11.SetUsedParam(arguments2);
+
+			S.proceed();
+			S.clean();
+		} catch(Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+		}
+		println "time: "+(System.currentTimeMillis()-start)/1000
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/compiler.groovy	(revision 321)
@@ -0,0 +1,996 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.bfm
+
+import org.txm.Toolbox;
+import org.txm.importer.cwb.*
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Produce WTC files from the TEI-TXM files. <br/>
+ * - Read texts metadata with XPath queries <br/>
+ * - Add the following word properties : sic, abbr, orig, lb and pb <br/>
+ * - Keep &lt;front>, &lt;body> and &lt;back> for each text <br/>
+ * - Text enclosed in &lt;q> is tokenized <br/>
+ * 
+ * @author mdecorde
+ *
+ */
+class compiler {
+	/** The debug. */
+	private boolean debug= false;
+
+	/** The annotate_status. */
+	private boolean annotate_status=true;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private def output;
+
+	/** The url. */
+	private def url;
+
+	/** The anahash. */
+	static boolean firstWord = true;
+	static private def anaTypes = [];
+	private HashMap<String,String> anahash = new HashMap<String,String>() ;
+
+	private static SAttributesListener sattrsListener;
+	private static HashMap<String,ArrayList<String>> structs;
+	private static HashMap<String, Integer> structsProf;
+
+	/** The text. */
+	private String text="";
+
+	/** The base. */
+	private String base="";
+
+	/** The project. */
+	private String project="";
+
+	/** The cwb loc. */
+	private String cwbLoc;
+
+	/** The lang. */
+	private String lang ="fr";
+
+	/**
+	 * contains the metadata xpath organize per name
+	 */
+	Properties metadataXPath;
+
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){
+		firstWord = true;
+		anaTypes = [];
+	}
+
+	/**
+	 * initialize the compiler.
+	 *
+	 * @param url the file to process
+	 * @param text the Texte's name
+	 * @param base the base's name
+	 * @param project the Project's name
+	 */
+	public compiler(URL url,String text,String base, String project, Properties metadataXPath)
+	{
+		this.metadataXPath = metadataXPath;
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+			if (sattrsListener == null)
+				sattrsListener = new SAttributesListener(parser);
+			else
+				sattrsListener.start(parser)
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the path to cwb utils binaries
+	 */
+	public void setCwbPath(String path) {
+		if (!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+
+	ArrayList<File> orderedFiles;
+	public ArrayList<File> getOrderedTxmFiles() {
+		return orderedFiles;
+	}
+
+	/**
+	 * Sets the lang.
+	 *
+	 * @param lang the new lang
+	 */
+	public void setLang(String lang)
+	{
+		this.lang = lang;
+	}
+
+	/**
+	 * Sets the annotation done.
+	 *
+	 * @param done the new annotation done
+	 */
+	public void setAnnotationDone(boolean done)
+	{
+		this.annotate_status = done;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f) {
+		try {
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.END_ELEMENT)
+				if (parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+
+	/**
+	 * Increment.
+	 *
+	 * @param parser the parser
+	 * @param value the value
+	 * @return the java.lang. object
+	 */
+	private def increment(XMLStreamReader parser, int value)
+	{
+		String n=null;
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			//System.out.println("attr name "+parser.getAttributeLocalName(i));
+			if (parser.getAttributeLocalName(i) == "n") {
+				n = parser.getAttributeValue(i);
+				break;
+			}
+		}
+		//System.out.println("inc n "+n);
+		if (n != null)
+			try {
+				value = Integer.parseInt(n);
+				return value;
+			}
+			catch (Exception e) {return value+1;}
+
+		value = value+1;
+		return value;
+	}
+
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean transfomFileWtc(File wtcFile)
+	{
+		try {
+			if (!createOutput(wtcFile)) return false;
+
+			String headvalue = ""
+			String vAna = "";
+			String vForm = "";
+			String wordid = "";
+			String vHead = "";
+
+			Integer p_id = 0;
+			Integer s_id = 0;
+			Integer q_id = 0;
+			int sp_id = 0;
+			Integer body_id = 0;
+			Integer front_id = 0;
+			Integer back_id = 0;
+			Integer lb_id = 0;
+			Integer pb_id = 0;
+			Integer ab_id = 0;
+			int foreign_id = 0;
+			int name_id = 0;
+
+			boolean captureword = false;
+
+			String vExpan = "";
+			String vCorr = "";
+			String vReg = "";
+			String vOrig = "";
+			String vSic = "";
+			String vAbbr = "";
+			String givenpos = "";
+			String pb_n = "";
+			String foreign_lang = "";
+			String nameType = "";
+			String anaType;
+			//String abType = "";
+			
+			boolean foundtei=false, foundtext=false;
+
+			boolean flaglg = false;
+			int levelq = 0;
+			//boolean flagq = false;
+			boolean flaghead = false;
+			//Added:
+			boolean flagSp = false;
+			boolean flagAuthor = false;
+			boolean flagDate = false;
+			boolean flagWord = false;
+			boolean flagForm = false;
+			boolean flagAna = false;
+
+			boolean flagchoice = false;
+			boolean flagcorr = false;
+			boolean flagsic = false;
+			boolean flagreg = false;
+			boolean flagexpan = false;
+			boolean flagorig = false;
+			boolean flagabbr = false;
+			boolean flagfw = false;
+			//boolean flagSupplied = false;
+			int levelSupplied = 0;
+			//boolean flagSurplus = false;
+			boolean flagForeign = false;
+			//boolean flagName = false;
+
+			this.GoToText();
+			int missingId= 0
+			boolean USEVERSE = false; // switch default reference to verse references
+			String titreId; // the title to use in the reference
+			
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+
+						String localname = parser.getLocalName();
+						if (foundtext) sattrsListener.startElement(localname);
+						
+						switch (localname) {
+							case "TEI":
+								foundtei = true;
+								break;
+							case "text":
+								foundtext = true;
+								sattrsListener.startElement(localname);
+								output.write("<text id=\""+text+"\"")
+
+								for (int i = 0; i < parser.getAttributeCount() ; i++) {
+									String name = parser.getAttributeLocalName(i);
+									if ("id" == name || "base" == name || "project" == name) continue;
+									output.write(" "+name+"=\""+parser.getAttributeValue(i)+"\"");
+									
+									if (name == "forme") {
+										USEVERSE = (parser.getAttributeValue(i).contains("vers"))
+									} else if (name == "sigle") {
+										titreId = parser.getAttributeValue(i)
+									}
+								}
+								
+								output.write(" base=\""+base+"\" project=\""+project+"\">\n");
+								captureword=true;
+								break;
+
+							case "div":
+							//output.write("<div type=\""+parser.getAttributeValue(null,"type")+"\">\n");
+								String divType = "NA";
+								String divSubtype = "NA";
+								String divN = "NA";
+								String divId ="NA";
+								for(int i = 0 ; i < parser.getAttributeCount(); i++) {
+									if(parser.getAttributeLocalName(i) == "type") {
+										divType = parser.getAttributeValue(i);
+									} else if(parser.getAttributeLocalName(i) == "subtype") {
+										divSubtype = parser.getAttributeValue(i);
+									} else if(parser.getAttributeLocalName(i) == "n") {
+										divN = parser.getAttributeValue(i);
+									} else if(parser.getAttributeLocalName(i) == "id") {
+										divId = parser.getAttributeValue(i);
+										break;
+									}
+								}
+								output.write("<div type=\""+divType+"\" subtype=\""+divSubtype+"\" n=\""+divN+"\" id=\""+divId+"\">\n");
+								break;
+							case "p":
+								p_id = increment(parser, p_id);
+								output.write("<p n=\""+p_id+"\">\n");
+								break;
+							case "ab":
+								ab_id = increment(parser, ab_id)
+								output.write("<ab n=\""+(ab_id)+"\" type=\""+parser.getAttributeValue(null,"type")+"\" subtype=\""+parser.getAttributeValue(null,"subtype")+"\" rend=\""+parser.getAttributeValue(null,"rend")+"\">\n");
+								break;
+							case "q":
+								q_id = increment(parser, q_id)
+								output.write("<q n=\""+(q_id)+"\">\n");
+							//flagq=true;
+								levelq = levelq + 1;
+								break;
+							case "sp":
+								sp_id = increment(parser, sp_id)
+								output.write("<sp n=\""+(sp_id)+"\">\n");
+								flagSp = true;
+								break;
+							case "front":
+								front_id = increment(parser, front_id)
+								output.write("<front n=\""+front_id+"\">\n");
+								break;
+							case "body":
+								body_id= increment(parser, body_id)
+								output.write("<body n=\""+body_id+"\">\n");
+								break;
+							case "back":
+								back_id = increment(parser, back_id)
+								output.write("<back n=\""+back_id+"\">\n");
+								break;
+							case "lb":
+								lb_id = increment(parser, lb_id)
+								break;
+							case "pb":
+								pb_id = increment(parser, pb_id)
+								for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+									if (parser.getAttributeLocalName(i) == "n") {
+										pb_n = parser.getAttributeValue(i);
+									}
+								}
+								break;
+							case "s":
+								s_id = increment(parser, s_id)
+								output.write("<s n=\""+s_id+"\">\n");
+								break;
+							case "choice":
+								flagchoice = true;
+								break;
+							case "corr":
+								flagcorr = true;
+								vCorr= "";
+								break;
+							case "reg":
+								flagreg = true;
+								vReg= "";
+								break;
+							case "expan":
+								flagexpan = true;
+								vExpan= "";
+								break;
+							case "orig":
+								flagreg = true;
+								vOrig= "";
+								break;
+							case "sic":
+								flagsic = true;
+								vSic= "";
+								break;
+							case "abbr":
+								flagreg = true;
+								vAbbr= "";
+								break;
+							case "foreign":
+								flagForeign = true;
+								for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+									if (parser.getAttributeLocalName(i) == "lang") {
+										lang = parser.getAttributeValue(i);
+										break;
+									}
+								}
+
+								output.write("<foreign n=\""+(foreign_id++)+"\" lang=\""+lang+"\">\n");
+							//vForeign = "";
+								break;
+
+							case "name":
+							//flagName = true;
+								for(int i = 0 ; i < parser.getAttributeCount(); i++)
+									if(parser.getAttributeLocalName(i) == "type")
+								{
+									nameType = parser.getAttributeValue(i);
+									break;
+								}
+
+								output.write("<name n=\""+(name_id++)+"\" type=\""+nameType+"\">\n");
+								break;
+							case "supplied":
+							//flagSupplied = true;
+								levelSupplied = levelSupplied + 1;
+								break;
+
+							case "surplus":
+								flagfw = true;
+								break;
+
+							case "del":
+								flagfw = true;
+								break;
+
+							case "w":
+								givenpos = "";
+								wordid = "w_"+text+"_m"+missingId++
+								for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+									if (parser.getAttributeLocalName(i) == "id") {
+										wordid = parser.getAttributeValue(i);
+									} else if (parser.getAttributeLocalName(i) == "type") {
+										givenpos = parser.getAttributeValue(i);
+									}
+								}
+								if (wordid.startsWith("w")) {
+									if (!wordid.startsWith("w_"))
+										wordid = "w_"+wordid.substring(1)
+								} else {
+									wordid = "w_"+wordid;
+								}
+
+								if (givenpos == null || givenpos == "")
+									givenpos = "NA";
+								vForm = "";
+								anahash.clear(); // remove previous word ana values
+								flagWord = true;
+								break;
+							case "form":
+								flagForm = true;
+								vForm = "";
+								break;
+
+							case "ana":
+								flagAna = true;
+								anaType = parser.getAttributeValue(null, "type")
+								anahash.put(anaType, "");
+								if (firstWord) {
+									anaTypes << anaType;
+								}
+								break;
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						String localname = parser.getLocalName();
+						if (foundtext) sattrsListener.endElement(localname);
+
+						switch (localname) {
+							case "div":
+								output.write("</div>\n");
+								break;
+							case "text":
+								output.write("</text>\n");
+								captureword=false;
+								break;
+							case "p":
+								output.write("</p>\n");
+								break;
+							case "s":
+								output.write("</s>\n");
+								break;
+							case "ab":
+								output.write("</ab>\n");
+								break;
+							case "q":
+								output.write("</q>\n");
+							//flagq= false;
+								levelq = levelq - 1;
+								break;
+							case "sp":
+								output.write("</sp>\n");
+								flagSp = false;
+								break;
+							case "front":
+								output.write("</front>\n");
+								break;
+							case "body":
+								output.write("</body>\n");
+								break;
+							case "back":
+								output.write("</back>\n");
+								break;
+
+							//							case "fw":
+							//							flagfw = false;
+							//							break;
+
+							case "choice":
+								if(vOrig == "")
+									vOrig="NA";
+								if(vSic == "")
+									vSic="NA";
+								if(vAbbr == "")
+									vAbbr="NA";
+
+								String ref;
+								if(USEVERSE)
+									ref = titreId+", p."+pb_n+", v."+lb_id;
+								else
+									ref = titreId+", p."+pb_n;
+
+								if (flagfw) {
+									// on est hors texte
+								} else {
+									String vFormToWrite = vForm;
+									if (vCorr != "") {
+										vFormToWrite = vCorr;
+									} else if(vReg != "") {
+										vFormToWrite = vReg;
+									} else if(vExpan != "") {
+										vFormToWrite = vExpan
+									}
+									firstWord = false;
+									output.write( vFormToWrite +"\t"+wordid+"\t"+levelq.toString().substring(0,1)+"\t"+flagSp.toString().substring(0,1)+"\t"+pb_n+"\t"+lb_id+"\t"+vOrig+
+											"\t"+vSic+"\t"+vAbbr+"\t"+ref+"\t"+givenpos+"\t"+levelSupplied.toString().substring(0,1)+"\t"+lang+"\t"+nameType);
+									for(String type : anaTypes) {
+										output.write("\t"+anahash.get(type));
+									}
+									output.write("\n")
+								}
+								flagchoice = false;
+								vCorr= "";
+								vSic= "";
+								break;
+							case "corr":
+								flagcorr = false;
+
+								break;
+							case "reg":
+								flagreg = false;
+								vReg = "";
+								break;
+							case "expan":
+								flagexpan = false;
+								vExpan= "";
+								break;
+							case "orig":
+								flagreg = false;
+								vOrig= "";
+								break;
+							case "sic":
+								flagsic = false;
+
+								break;
+							case "abbr":
+								flagreg = false;
+								vAbbr= "";
+								break;
+
+							case "foreign":
+								flagForeign = false;
+								lang = "";
+								output.write("</foreign>\n");
+								break;
+
+							case "name":
+							//flagName = false;
+								nameType = "";
+								output.write("</name>\n");
+								break;
+
+							case "supplied":
+							//flagSupplied = false;
+								levelSupplied = levelSupplied - 1;
+								break;
+
+							case "surplus":
+								flagfw = false;
+								break;
+
+							case "del":
+								flagfw = false;
+								break;
+
+							case "w":
+								if (captureword) {
+									if (flagchoice) {
+
+									} else if(flagfw) {
+
+									} else {
+										if (vOrig == "")
+											vOrig="NA";
+										if(vSic == "")
+											vSic="NA";
+										if(vAbbr == "")
+											vAbbr="NA";
+										if (nameType == "")
+											nameType = "NA";
+										if(lang == "")
+											lang="fr"
+
+										String ref;
+										if(USEVERSE)
+											ref = titreId+", p."+pb_n+", v."+lb_id;
+										else
+											ref = titreId+", p."+pb_n;
+
+										firstWord = false;
+										output.write(vForm.replaceAll("&", "&amp;").replaceAll("<", "&lt;") +"\t"+wordid+"\t"+levelq.toString().substring(0,1)+"\t"+flagSp.toString().substring(0,1)+"\t"+pb_n+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\t"+ref+"\t"+givenpos+"\t"+levelSupplied.toString().substring(0,1)+"\t"+lang+"\t"+nameType);
+										for(String type : anaTypes) {
+											output.write("\t"+anahash.get(type));
+										}
+										output.write("\n")
+									}
+
+									flagWord = false;
+								}
+								break;
+
+							case "form":
+								flagForm = false;
+								break;
+
+							case "ana":
+								flagAna = false;
+								anahash.put(anaType, vAna);
+								vAna = "";
+								break;
+						}
+						break; // end elem
+
+					case XMLStreamConstants.CHARACTERS:
+						if (flagAna) {
+							vAna += parser.getText().trim()
+						}
+
+						if (flagForm) {
+							vForm += parser.getText().trim();
+							if (flagchoice) {
+								if (flagsic) {
+									vSic += parser.getText().trim();
+								}
+								if (flagorig) {
+									vOrig += parser.getText().trim();
+								}
+								if (flagabbr) {
+									vAbbr += parser.getText().trim();
+								}
+								if (flagcorr) {
+									vCorr += parser.getText().trim();
+								}
+							}
+						}
+				}
+			}
+			//output.write("</txmcorpus>");
+			output.close();
+			parser.close();
+			inputData.close();
+		} catch (Exception ex) {
+			System.out.println("Exception while parsing " + inputData);
+			ex.printStackTrace();
+			return false;
+		}
+
+		return true;
+	}
+
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile contains the TEI-TXM files
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String corpusname, Properties metadataXPath)
+	{
+		sattrsListener = null; // reset SAttribute Listener for each new import
+		this.metadataXPath = metadataXPath;
+		if (cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if (!txmDir.exists()) {
+			println ("binary directory does not exists: "+txmDir)
+			return false;
+		}
+
+		File wtcFile = new File(binDir, "wtc/${corpusname}.wtc");
+		new File(binDir, "/wtc/").deleteDir();
+		new File(binDir, "/wtc/").mkdir();
+		new File(binDir, "/data/${corpusname}").deleteDir();
+		new File(binDir, "/data/${corpusname}").mkdir();
+		new File(binDir, "registry/").mkdir();
+
+		String textid = "";
+		int counttext = 0;
+		List<File> files = txmDir.listFiles();
+		
+		// get text siecles to be able to sort with it
+		HashMap<File,Integer[]> filesiecle = new HashMap<File, Integer[]>()
+		for (File f : files) {
+			Integer[] date = new Integer[3];
+			date[0] = date[1] = date[2] = 0;
+			String xpath = "//tei:TEI/tei:teiHeader/tei:profileDesc/tei:creation/tei:date[@type='compo']/@when"
+			if (metadataXPath.containsKey("datecompo"))
+				xpath = metadataXPath.get("datecompo")
+			String datecompo = XPathResult.getXpathResponse(f, xpath);
+			if (datecompo != null) {
+				//println f.getName()+" > "+datecompo
+				String[] split = datecompo.split("-"); // yyyy-mm-dd
+				if (split.length == 3) {
+					date[0] = Integer.parseInt(split[0]);
+					date[1] = Integer.parseInt(split[1]);
+					date[2] = Integer.parseInt(split[2]);
+				}
+				else if (split.length == 1) { // yyyy
+					date[0] = Integer.parseInt(split[0]);
+					date[1] = 1;
+					date[2] = 1;
+				}				
+			}
+			filesiecle.put(f, date);
+		}
+		//println "date compos: "+filesiecle
+		Collections.sort(files); // Alpha order
+		Collections.sort(files, new Comparator<File>() { // Date order
+					@Override
+					public int compare(File o1, File o2) {
+						Integer[] date1 = filesiecle.get(o1);
+						Integer[] date2 = filesiecle.get(o2);
+						if (date1[0] < date2[0]) {
+							return -1;
+						} else if(date1[0] > date2[0]) {
+							return 1;
+						}
+
+						if (date1[1] < date2[1]) {
+							return -1;
+						} else if(date1[1] > date2[1]) {
+							return 1;
+						}
+
+						if (date1[2] < date2[2]) {
+							return -1;
+						} else if(date1[2] > date2[2]) {
+							return 1;
+						}
+
+						return 0;
+					}
+				});
+			
+		this.orderedFiles = files;
+		println("process "+files.size()+" files ")
+		//println("files: $files")
+		//write txmcorpus
+		if (!createOutput(wtcFile)) {
+			println "Error: could not write wtc file"
+			return false;
+		} else {
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+		
+		//1- Transform into WTC file
+		for (File f : files) {
+			counttext++;
+			if (!f.exists()) {
+				println("file "+f+ " does not exists")
+			} else {
+				print "."
+				String txtname = f.getName().substring(0, f.getName().length()-4);
+				def builder = new compiler(f.toURI().toURL(), txtname, corpusname, "default", metadataXPath);
+				builder.setLang(lang)
+				if (!builder.transfomFileWtc(wtcFile)) {
+					println "Failed to compile "+f
+				}
+				builder.setAnnotationDone(this.annotate_status);
+			}
+		}
+
+		//close txmcorpus
+		if (!createOutput(wtcFile)) {
+			println "Error: could not write wtc file"
+			return false;
+		} else {
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+		println ""
+		
+		//2- Import into CWB
+		def outDir = binDir.getAbsolutePath();;
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		
+		def pAttrs = ["id","q","sp","pb","lb","orig","sic","abbr","ref","pos","supplied","lang","nametype"];
+		for(String type : anaTypes)
+			pAttrs.add(type.substring(1)); // remove #
+
+		structs = sattrsListener.getStructs();
+		structsProf = sattrsListener.getProfs();
+		if (debug) {
+			println structs
+			println structsProf
+		}
+		// add structures+properties found in sources
+		List<String> sargs = new ArrayList<String>();
+		for (String name : structs.keySet()) {
+			if ( name == "text") continue; // added after
+			//if ( name == "q") continue; // added after
+			//if ( name == "foreign") continue; // added after
+			String concat = name+":"+structsProf.get(name); // append the depth
+			for (String value : structs.get(name)) // append the attributes
+				concat += "+"+value;
+			if ((name == "p" || name == "body" || name == "back" || name == "front") &&
+			!(concat.endsWith("+n") || concat.contains("+n+")))
+				concat += "+n"
+			sargs.add(concat);
+		}
+
+		String textSAttributes = "text:0+id+base+project";
+		if (metadataXPath != null) {
+			for (String meta : metadataXPath.keySet()) // text property declarations from metadata.csv
+				textSAttributes+="+"+meta;
+		}
+		if (!metadataXPath.keySet().contains("sigle"))
+			textSAttributes+="+sigle";
+			
+		sargs.add(textSAttributes)
+		sargs.add("txmcorpus:0+lang")
+		//sargs.add("q:0+n+lang")
+		for (int c = 0 ; c < sargs.size() ; c++) {
+			String sarg = sargs.get(c);
+			if (sarg.startsWith("q:")) {
+				if (! sarg.contains("+n")) sarg +="+n"
+				if (! sarg.contains("+lang")) sarg +="+lang"
+
+				sargs.set(c, sarg);
+			} else if(sarg.startsWith("foreign:")) {
+				if (! sarg.contains("+n")) sarg +="+n"
+				if (! sarg.contains("+lang")) sarg +="+lang"
+				sargs.set(c, sarg);
+			} else if(sarg.startsWith("ab:") || sarg.startsWith("sp:")) {
+				if (! sarg.contains("+n")) sarg +="+n"
+				if (! sarg.contains("+subtype")) sarg +="+subtype"
+				if (! sarg.contains("+rend")) sarg +="+rend"
+				sargs.set(c, sarg);
+			}  else if(sarg.startsWith("div:")) {
+				if (! sarg.contains("+n")) sarg +="+n"
+				if (! sarg.contains("+id")) sarg +="+id"
+				if (! sarg.contains("+type")) sarg +="+type"
+				if (! sarg.contains("+subtype")) sarg +="+subtype"
+				sargs.set(c, sarg);
+			} else if(sarg.startsWith("name:")) {
+				if (! sarg.contains("+n")) sarg +="+n"
+				if (! sarg.contains("+type")) sarg +="+type"
+				sargs.set(c, sarg);
+			}
+		}
+		sargs.sort();
+		
+		String[] sAttributes = sargs;
+		String[] pAttributes = pAttrs;
+		println "P-attributes: "+pAttributes
+		println "S-attributes: "+sargs
+
+		try {
+			String regPath = outDir + "/registry/"+corpusname.toLowerCase(); // CQP wants lower case registry files
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), outDir + "/data/${corpusname}", outDir + "/wtc/"+corpusname+".wtc", regPath,pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname, outDir + "/registry");
+		} catch (Exception ex) {System.out.println(ex);return false;}
+
+		return true;
+	}
+
+	/**
+	 * show cwb utils messages.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+
+	/**
+	 * test purpose.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/bfm");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/package.html	(revision 321)
@@ -0,0 +1,12 @@
+<html>
+<body>
+<p>The BFM import module<br/>
+The bfmLoader.groovy script is the main entry point.<br/>
+Process : <br/>
+1- Read a directory containing XML files who follow the BFM encoding guidelines (<a href="http://bfm.ens-lsh.fr/">BFM website</a>) <br/>
+2- Add annotations on words <br/>
+3- Build a TXM corpus with the same name as the directory <br/>
+4- Build a HTML edition <br/>
+</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/bfmLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/bfmLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/bfmLoader.groovy	(revision 321)
@@ -0,0 +1,219 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate: 2016-03-01 10:10:58 +0100 (Tue, 01 Mar 2016) $
+// $LastChangedRevision: 3133 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.bfm;
+
+import javax.xml.stream.XMLStreamReader;
+
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.bfm.importer;
+import org.txm.importer.bfm.compiler;
+import org.txm.importer.bfm.pager;
+import org.txm.objects.*;
+import org.txm.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.scripts.teitxm.*;
+import org.txm.importer.*;
+import org.w3c.dom.Element
+import org.txm.utils.*;
+
+String userDir = System.getProperty("user.home");
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+def MONITOR;
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/bfm/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		//rootDir = userDir+"/xml/TESTS/alceste";  // directory which contains the source file
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+// BFM XPATH PARAMETERS
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "READ XPATH PARAMETERS")
+File paramDeclarationFile = new File(srcDir, "import.properties");
+Properties metadataXPath = new Properties();
+if (paramDeclarationFile.exists() && paramDeclarationFile.canRead()) {
+	InputStreamReader input = new InputStreamReader(new FileInputStream(paramDeclarationFile) , "UTF-8");
+	metadataXPath.load(input);
+	input.close();
+
+	if (!metadataXPath.containsKey("titre"))
+		println "Warning: parameters property file does not contain the 'titre' metadata"
+	if (!metadataXPath.containsKey("forme"))
+		println "Warning: parameters property file does not contain the 'forme' metadata in param values. The default value is 'prose'"
+}
+else
+	println "No '$paramDeclarationFile' file found"
+
+// Apply XSL
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "APPLYING XSL")
+if (xsl != null && xslParams != null && xsl.trim().length() > 0) {
+	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))
+	// return; // error during process
+	srcDir = new File(binDir, "src");
+	println ""
+}
+
+// copy txm files
+println "-- VALIDATION - checking XML source files well-formedness"
+List<File> srcfiles = srcDir.listFiles();
+for (File f : srcfiles) { // check XML format, and copy file into binDir
+	if (f.isHidden() || f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties"))
+		continue;
+	if (ValidateXml.test(f)) {
+		FileCopy.copy(f, new File(txmDir, f.getName()));
+	} else {
+		println "Won't process file "+f;
+	}
+}
+
+if (txmDir.listFiles() == null) {
+	println "No txm file to process"
+	return;
+}
+
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+def imp = new importer()
+if (!imp.run(srcDir, binDir, txmDir, basename, metadataXPath)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools - $model model"
+boolean annotate_status = true;
+if (annotate && !(new Annotate().run(binDir, txmDir,model+".par"))) {
+	println "Skipping Annotate process";
+	annotate_status = false;
+}
+
+//if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+//if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+//println "-- ANNOTATE - Running NLP tools - fr model"
+//annotate_status = true;
+//if (annotate && !(new Annotate().run(binDir, txmDir,"fr.par"))) {
+//	println "Skipping Annotate process";
+//	annotate_status = false;
+//}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if (debug) c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setLang(lang);
+c.setAnnotationDone(annotate_status)
+if (!c.run(binDir, txmDir, corpusname, metadataXPath)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION"
+	if (MONITOR != null) MONITOR.worked(25, "EDITION")
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	files = c.getOrderedTxmFiles();
+
+	println("Building editions: "+files.size()+" files" );
+	for (File txmFile : files) {
+		print "."
+		String txtname = txmFile.getName();
+		int idx = txtname.lastIndexOf(".");
+		if(idx > 0) txtname = txtname.substring(0, idx);
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);//["'","(","[","{","«"];
+
+		Element text = params.addText(corpusElem, txtname, txmFile);
+
+		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, corpusname, metadataXPath);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bfm/pager.groovy	(revision 321)
@@ -0,0 +1,545 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.bfm;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Build BFM texts HTML edition from the TEI-TXM files. <br/>
+ * The result is similar to the one of the XSLT script made by Alexis Lavrentiev
+ * (BFM & ICAR3). <br/>
+ * 
+ * @author mdecorde
+ */
+class pager {
+	private List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	private List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	private int wordcount = 0;
+
+	/** The pagecount. */
+	private int pagecount = 0;
+
+	/** The wordmax. */
+	private int wordmax = 0;
+
+	/** The wordid. */
+	private String wordid;
+
+	/** The first word. */
+	private boolean firstWord = true;
+
+	/** The wordvalue. */
+	private String wordvalue;
+
+	/** The interpvalue. */
+	private String interpvalue;
+
+	/** The lastword. */
+	private String lastword = " ";
+
+	/** The wordtype. */
+	private String wordtype;
+
+	/** The flagform. */
+	private boolean flagform = false;
+
+	/** The flaginterp. */
+	private boolean flaginterp = false;
+
+	/** The url. */
+	private URL url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	private OutputStreamWriter writer;
+
+	/** The multiwriter. */
+	private OutputStreamWriter multiwriter = null;
+	StaxStackWriter pagedWriter = null;
+
+	/** The infile. */
+	private File infile;
+
+	/** The outfile. */
+	private File outfile;
+	private File outDir;
+
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+
+	/** The titre id. */
+	private String titreId;
+
+	private String basename;
+	private String textname;
+
+	private def xpathProperties;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the file to convert
+	 * @param outfile the result file
+	 * @param NoSpaceBefore the punctuation marks that don't have a space before
+	 * @param NoSpaceAfter the punctuation marks that don't have a space after
+	 * @param max the max number of word per page
+	 */
+	pager(File infile, File outDir, String textname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String basename, Properties xpathprops) {
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.basename = basename;
+		this.textname = textname;
+		this.outDir = outDir;
+		this.xpathProperties = xpathprops;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		process();
+	}
+
+	private def closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+
+			if (firstWord) { // there was no words
+				this.idxstart.add("w_0")
+				pagedWriter.write("<span id=\"w_0\"/>");
+			}
+			pagedWriter.writeEndElements();
+			pagedWriter.close();
+			return tags;
+		} else {
+			return [];
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		wordcount = 0;
+		try {
+			def tags = closeMultiWriter();
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+			File outfile = new File(outDir, textname+"_"+(++pagecount)+".html")
+			pages.add(outfile);
+			firstWord = true; // waiting for next word
+
+			pagedWriter = new StaxStackWriter(outfile, "UTF-8");
+
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html")
+			pagedWriter.writeEmptyElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"]);
+			pagedWriter.writeStartElement("head")
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(basename.toUpperCase()+" - "+textname+" - Edition - Page "+pagecount)
+			pagedWriter.writeEndElement() // </title>
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+			pagedWriter.writeStartElements(tags);
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the pages create during the processing
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the word's indexes
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * process the infile.
+	 */
+	void process() {
+		String localname = "";
+
+		boolean closeTitre = false;
+		boolean closeAmen = false;
+
+		boolean flagchoice = false;
+		boolean flagcorr = false;
+		boolean flagsic = false;
+		boolean flagreg = false;
+		boolean flagexpan = false;
+		boolean flagorig = false;
+		boolean flagabbr = false;
+		boolean flagforeign = false;
+		int levelSupplied = 0;
+		boolean flagSurplus = false;
+		boolean flagDel = false;
+
+		// option for metadata from BFM TEI header
+		def mValues = [:];
+		def xpathprocessor = new XPathResult(infile)
+		for (String name : xpathProperties.keySet()) {
+			String value = xpathprocessor.getXpathResponse(xpathProperties.get(name), "N/A");
+			mValues.put(name, value.replace("\n", " "));
+		}
+		if (mValues.containsKey("forme") && !mValues.get("forme").equals("N/A")) {
+			if (mValues.get("forme").startsWith("#forme_")) {
+				mValues.put("forme", mValues.get("forme").substring(7));
+			} else if (mValues.get("forme").startsWith("#")) {
+				mValues.put("forme", mValues.get("forme").substring(1));
+			} else {
+				mValues.put("forme", mValues.get("forme"));
+			}
+		}
+		createNextOutput();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							pagedWriter.writeStartElement("h1",
+							["align":"center", "style":"color:darkviolet;font-family:Times;font-weight:bold;font-style:italic;font-size=200%", "class":"text"])
+							pagedWriter.writeCharacters(mValues.get("titre"))
+							pagedWriter.writeEndElement(); // h1
+							pagedWriter.writeStartElement("p")
+							pagedWriter.writeEmptyElement("hr")
+							pagedWriter.writeCharacters("auteur : "+mValues.get("auteur"))
+							pagedWriter.writeEmptyElement("br");
+							pagedWriter.writeCharacters("date de composition : "+mValues.get("datecompolibre"));
+							pagedWriter.writeEmptyElement("br");
+							pagedWriter.writeCharacters("domaine : "+mValues.get("domaine"))
+							pagedWriter.writeEmptyElement("br");
+							pagedWriter.writeCharacters("genre : "+mValues.get("genre"))
+							pagedWriter.writeEmptyElement("br");
+							pagedWriter.writeCharacters("forme : "+mValues.get("forme"))
+							pagedWriter.writeEmptyElement("br");
+							pagedWriter.writeCharacters("dialecte : "+mValues.get("dialecte"))
+							pagedWriter.writeEmptyElement("br");
+							pagedWriter.writeEmptyElement("hr")
+							pagedWriter.writeEndElement(); // p
+							break;
+						case "head":
+							pagedWriter.writeStartElement("h2")
+							break;
+						case "lg":
+						case "p":
+							pagedWriter.writeStartElement("p")
+							break;
+						case "pb":
+							createNextOutput();
+							if (parser.getAttributeValue(null,"n") != null) {
+								pagedWriter.writeStartElement("p", ["style":"color:red", "align":"center"]);
+								pagedWriter.writeCharacters("- "+parser.getAttributeValue(null,"n")+" -")
+								pagedWriter.writeEndElement() // p
+							}
+							break;
+
+						case "ab":
+							if (parser.getAttributeValue(null,"n") != null) {
+								pagedWriter.writeStartElement("p", ["align":"center"]);
+								pagedWriter.writeCharacters(parser.getAttributeValue(null,"n"))
+								pagedWriter.writeEndElement() // p
+							}
+							break;
+						case "l":
+							pagedWriter.writeEmptyElement("br");
+							break;
+						case "div":
+							if (parser.getAttributeValue(null,"type") == "titre") {
+								pagedWriter.writeStartElement("h3")
+								closeTitre = true;
+							} else if (parser.getAttributeValue(null,"type") == "amen") {
+								pagedWriter.writeStartElement("h3")
+								closeAmen = true;
+							}
+							break;
+						case "w":
+							interpvalue = "";
+							for (int i = 0 ; i < parser.getAttributeCount(); i++) {
+								if (parser.getAttributeLocalName(i).equals("type")) {
+									interpvalue = " pos: "+parser.getAttributeValue(i);
+								}
+							}
+							wordid = parser.getAttributeValue(null, "id");
+
+							if (firstWord) {
+								firstWord = false;
+								this.idxstart.add(wordid);
+							}
+							wordcount++;
+							if (wordcount >= wordmax) {
+								createNextOutput();
+							}
+							break;
+
+						case "choice":
+							flagchoice = true;
+							break;
+						case "corr":
+							flagcorr = true;
+							break;
+						case "sic":
+							flagsic = true;
+							break;
+						case "reg":
+							flagreg = true;
+							break;
+						case "orig":
+							flagorig = true;
+							break;
+						case "foreign":
+							flagforeign = true;
+							break;
+						case "supplied":
+							levelSupplied = levelSupplied +1;
+							if(flagform)
+								wordvalue = wordvalue+"[";
+							break;
+
+						case "surplus":
+							flagSurplus = true;
+							pagedWriter.writeStartElement("span", ["class": "surplus", "style":"color:red;"])
+							break;
+
+						case "del":
+							flagDel = true;
+							pagedWriter.writeStartElement("span", ["class": "del", "style":"color:red;text-decoration:line-through;"])
+							break;
+
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+
+						case "form":
+							wordvalue="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "head":
+							pagedWriter.writeEndElement() // h2
+							break;
+
+						case "div":
+							if (closeTitre) {
+								pagedWriter.writeEndElement() // h3
+								closeTitre = false;
+							} else if(closeAmen) {
+								pagedWriter.writeEndElement() // h3
+								closeTitre = false;
+							}
+							break;
+
+						case "lb":
+							pagedWriter.writeEmptyElement("br")
+							break;
+						case "lg":
+						case "p":
+							pagedWriter.writeEndElement() // p
+							break;
+
+						case "choice":
+							flagchoice = false;
+							break;
+						case "corr":
+							flagcorr = false;
+							break;
+						case "sic":
+							flagsic = false;
+							break;
+						case "reg":
+							flagreg = false;
+							break;
+						case "orig":
+							flagorig = false;
+							break;
+						case "foreign":
+							flagforeign = false;
+							break;
+						case "supplied":
+							levelSupplied = levelSupplied -1;
+							if (flagform)
+								wordvalue = wordvalue+"]";
+							break;
+						case "surplus":
+							flagSurplus = false;
+							pagedWriter.writeCharacters(")")
+							pagedWriter.writeEndElement() // span
+							break;
+
+						case "del":
+							flagDel = false;
+							pagedWriter.writeEndElement() // span
+							break;
+
+						case "form":
+							flagform = false
+							break;
+
+						case "ana":
+							flaginterp = false
+							break;
+
+						case "w":
+							int l = lastword.length();
+							String color = "";
+							if (flagcorr)
+								color = "color: green;";
+							else if (flagreg)
+								color = "color: darkgreen;";
+							else if (flagforeign)
+								color = "color: darkred;";
+							else if (levelSupplied == 1)
+								color = "color: blue;";
+							else if (levelSupplied == 2)
+								color = "color: darkblue;";
+							if (!flagchoice || flagcorr || flagreg) {
+								String endOfLastWord = "";
+								if (l > 0)
+									endOfLastWord = lastword.subSequence(l-1, l);
+
+								if (NoSpaceBefore.contains(wordvalue) ||
+								NoSpaceAfter.contains(lastword) ||
+								wordvalue.startsWith("-") ||
+								NoSpaceAfter.contains(endOfLastWord))
+								{
+									//multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\" "+color+">");
+								} else {
+									//multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\" "+color+">");
+									pagedWriter.writeCharacters(" ")
+								}
+								
+								pagedWriter.writeStartElement("span", ["title":interpvalue, "id":wordid, "style":color]) // span
+								pagedWriter.writeCharacters(wordvalue) 
+								pagedWriter.writeEndElement() // span
+								lastword=wordvalue;
+							}
+							break;
+					}
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					if (flagform && parser.getText().length() > 0)
+							wordvalue += (parser.getText());
+					if (flaginterp && parser.getText().length() > 0)
+							interpvalue += (parser.getText());
+					break;
+			}
+		}
+		closeMultiWriter();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/teicmc/SplitTEICMRPerLocutor.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/teicmc/SplitTEICMRPerLocutor.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/teicmc/SplitTEICMRPerLocutor.groovy	(revision 321)
@@ -0,0 +1,200 @@
+package org.txm.importer.teicmc
+
+import java.io.File;
+import java.net.URL;
+
+import javax.xml.stream.*;
+
+import org.txm.importer.StaxIdentityParser;
+import org.txm.importer.StaxStackWriter;
+
+public class SplitTEICMRPerLocutor extends StaxIdentityParser {
+
+	File outputDirectory;
+	def writers = [:];
+
+	public SplitTEICMRPerLocutor(File inputFile, File outputDirectory) {
+		super(inputFile);
+		this.outputDirectory = outputDirectory;
+		outputDirectory.mkdir()
+	}
+
+	protected void processStartElement() {
+		if (parser.getLocalName() == "posting") {
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				if (parser.getAttributeLocalName(i) == "who") {
+					String loc = parser.getAttributeValue(i)
+					writer = writers.get(loc) // switch writer on locutor
+					break;
+				}
+			}
+		}
+
+		if (writer != null) {
+			processStartElement(writer);
+		} else {
+			for (def swriter : writers.values()) processStartElement(swriter);
+		}
+	}
+
+	protected void processStartElement(def swriter)
+	{
+		String prefix = parser.getPrefix();
+
+		if (prefix != null && prefix.length() > 0)
+			swriter.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+		else
+			swriter.writeStartElement(localname);
+
+		for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {
+			swriter.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+		}
+
+		writeAttributes(swriter);
+	}
+
+	private void _processStartElement() {
+		String prefix = parser.getPrefix();
+
+		if (prefix != null && prefix.length() > 0)
+			writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+		else
+			writer.writeStartElement(localname);
+
+		for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {
+			writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+		}
+
+		writeAttributes();
+	}
+
+
+	protected void processNamespace() {
+		if (writer != null) {
+			writer.writeNamespace(parser.getPrefix(), parser.getNamespaceURI());
+		} else {
+			for (def swriter : writers.values()) swriter.writeNamespace(parser.getPrefix(), parser.getNamespaceURI());
+		}
+	}
+
+	protected void writeAttributes() {
+		if (writer != null) {
+			writeAttributes(writer);
+		} else {
+			for (def swriter : writers.values()) writeAttributes(swriter);
+		}
+	}
+
+	protected void writeAttributes(def swriter) {
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			String attrPrefix = parser.getAttributePrefix(i);
+			if (attrPrefix != null && attrPrefix.length() > 0)
+				swriter.writeAttribute(attrPrefix+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+			else
+				swriter.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+		}
+	}
+
+	protected void processCharacters()
+	{
+		if (writer != null) {
+			writer.writeCharacters(parser.getText());
+		} else {
+			for (def swriter : writers.values()) swriter.writeCharacters(parser.getText());
+		}
+	}
+
+	protected void processProcessingInstruction()
+	{
+		if (writer != null) {
+			writer.writeProcessingInstruction(parser.getPITarget(), parser.getPIData());
+		} else {
+			for (def swriter : writers.values()) swriter.writeProcessingInstruction(parser.getPITarget(), parser.getPIData());
+		}
+	}
+
+	protected void processDTD()
+	{
+		if (writer != null) {
+			writer.writeDTD(parser.getText());
+		} else {
+			for (def swriter : writers.values()) swriter.writeDTD(parser.getText());
+		}
+	}
+
+	protected void processCDATA()
+	{
+		if (writer != null) {
+			writer.writeCData(parser.getText())
+		} else {
+			for (def swriter : writers.values()) swriter.writeCData(parser.getText())
+		}
+	}
+
+	protected void processComment()
+	{
+		if (writer != null) {
+			writer.writeComment(parser.getText());
+		} else {
+			for (def swriter : writers.values()) swriter.writeComment(parser.getText());
+		}
+	}
+
+	protected void processEndElement()
+	{
+		if (writer != null) {
+			writer.writeEndElement();
+		} else {
+			for (def swriter : writers.values()) swriter.writeEndElement();
+		}
+		if (parser.getLocalName() == "posting") writer = null;
+	}
+
+	protected void processEndDocument() {
+		if (writer != null) {
+			writer.writeEndDocument();
+		} else {
+			for (def swriter : writers.values()) swriter.writeEndDocument();
+		}
+	}
+
+	protected void processEntityReference() {
+		if (writer != null) {
+			writer.writeEntityRef(parser.getLocalName());
+		} else {
+			for (def swriter : writers.values()) swriter.writeEntityRef(parser.getLocalName());
+		}
+	}
+
+	public boolean processLocutors(def locutors) {
+		for (String loc : locutors) {
+			println "Create writer for $loc"
+			writers.put(loc, new StaxStackWriter(new File(outputDirectory, loc+"-teicmr.xml"), "UTF8"))
+		}
+		println "writers: $writers"
+
+		for (def swriter : writers.values()) {
+			swriter.writeStartDocument("UTF-8", "1.0");
+			swriter.writeCharacters("\n");
+		}
+
+		boolean ret = process(writer);
+
+		for (def swriter : writers.values()) {
+			try {swriter.close();} catch(Exception e){println "close writer exep: "+e}
+		}
+
+		if (parser != null)
+			try {parser.close()} catch(Exception e){println "parser exep: "+e}
+
+		return ret;
+	}
+
+	public static void main(String[] args) {
+		File inputFile = new File("/home/mdecorde/xml/comere/ismael-textchat.xml")
+		File outputDirectory = new File("/home/mdecorde/xml/comere/split_out")
+
+		def p = new SplitTEICMRPerLocutor(inputFile, outputDirectory)
+		println p.processLocutors(["s_1", "s_10", "s_11", "s_12", "s_13", "s_14", "s_3", "s_4", "s_5", "s_6", "s_7", "s_8", "s_9", "tt_1", "tt_12", "tt_2", "tt_3", "tt_4", "tt_5", "tt_6", "tt_7", "tt_8"])
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CleanFile.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CleanFile.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CleanFile.groovy	(revision 321)
@@ -0,0 +1,96 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import java.io.BufferedWriter;
+import java.io.File;
+
+// TODO: Auto-generated Javadoc
+/**
+ * remove all ctrl char and replace \t by ' '.
+ *
+ * @param infile : the file to process
+ * @param encoding : the encoding of the file
+ */
+class CleanFile {
+	public CleanFile(File infile, String encoding){
+		Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(infile),encoding));
+		File outfile = File.createTempFile("filecleaner","123456", infile.getParentFile());
+		Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8"));
+		String line = reader.readLine();
+		while(line != null)
+		{
+			writer.write(clean(line) +"\n")
+			line = reader.readLine();
+		}
+		writer.close();
+		reader.close();
+		
+		infile.delete(); // erase old file
+		outfile.renameTo(infile); // rename the copy
+	}
+
+	/**
+	 * Clean.
+	 *
+	 * @param str the str
+	 * @return the string
+	 */
+	public static String clean(String str) {
+		str = str.replace("\t", " ");// replace \t by ' '
+		str = str.replaceAll("\\p{Cntrl}", ""); // remove all ctrl char
+		return str;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args){
+		File infile = new File("C:\\Documents and Settings\\H\\xml\\cleaner\\test.txt")		
+		String encoding = "CP1252";
+		new CleanFile(infile, encoding);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Alceste2Xml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Alceste2Xml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Alceste2Xml.groovy	(revision 321)
@@ -0,0 +1,294 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.txm.utils.AsciiUtils;
+import org.txm.utils.CharsetDetector;
+import org.txm.utils.i18n.DetectBOM;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * Create a file per text of an alceste file.
+ */
+class Alceste2Xml {
+
+	/** The textno. */
+	String textno = "0001";
+
+	/** The textcount. */
+	int textcount = 0;
+	int nbOfStarNumbers;
+	static def TEXTNUMBERREGEX = /^[0-9]{4,6} .+/
+	static def TEXTSTARREGEX = /^[*]{4,6} .+/
+	/**
+	 * run the script.
+	 *
+	 * @param infile the infile
+	 * @param outdir the outdir
+	 * @param encoding the encoding
+	 */
+	public boolean run(File infile, File outdir, String encoding) {
+		if (encoding == "??") {
+			encoding = new CharsetDetector(infile).getEncoding();
+			println "Guessing encoding of $infile : $encoding"
+		}
+		int noline = 0;
+		int p_n = 1;
+		int lb_n = 1;
+		def duplicatedAttributes = new HashSet<String>();
+		try {
+			outdir.mkdir();
+			boolean star = guessStar(infile, encoding);
+
+			String textname = infile.getName().substring(0,infile.getName().length()-4);// creates file with prefix the name of the alceste file
+
+			FileInputStream input = new FileInputStream(infile);
+			Reader reader = new InputStreamReader(input , encoding);
+
+			DetectBOM bomdetector = new DetectBOM(infile);
+			for (int ibom = 0 ; ibom < bomdetector.getBOMSize() ; ibom++) input.read()
+			
+			XMLOutputFactory factory = XMLOutputFactory.newInstance();
+			BufferedOutputStream output;
+			XMLStreamWriter writer;
+
+			String line = reader.readLine();
+			while (line != null) {
+				line = CleanFile.clean(line);
+				
+				if (((line ==~ TEXTNUMBERREGEX) || line ==~ TEXTSTARREGEX) && line.contains("*")) { // we found a text declaration with star attributes
+					if (writer != null) { // end previous text if any
+						writer.writeEndElement(); // p
+						writer.writeEndElement(); // text
+						writer.close();
+						output.close()
+					}
+					
+					lb_n = 1; // reset line counter
+					p_n = 1; // reset paragraph counter
+					int idx = line.indexOf(" "); // star metadata starts after the first ' '
+					
+					if (!star) {
+						textno = line.substring(0, idx)
+						textcount++;
+					} else {
+						this.buildNextId(); // update textno
+					}
+
+					String filename = textno;
+					def attributes = new LinkedHashMap<String, String>();
+					filename = filename.trim();
+					output = new BufferedOutputStream(new FileOutputStream(new File(outdir, filename+".xml")))
+					writer = factory.createXMLStreamWriter(output, "UTF-8"); //create a new file
+					writer.writeStartDocument("UTF-8", "1.0")
+					writer.writeStartElement("text");
+					
+					attributes["id"] = filename // writen at the end
+					// get text metadata
+					line = line.substring(idx + 1); // remove the text marker
+					
+					String[] splited = line.split("\\*");
+					for (String s1 : splited) { // for each text star metadata
+						s1 = s1.trim();
+						if (s1.length() == 0) continue;
+
+						String[] splited2 = s1.split("_", 2);
+						if (splited2.length == 1) { // attribute without value
+							
+							if (s1.endsWith("0")) {
+								String attrname = s1.substring(0, s1.length()-1); // remove "0"
+								String attrnamebase = attrname = AsciiUtils.buildId(attrname);
+								int c = 2;
+								while (attributes.containsKey(attrname)) {
+									attrname = attrnamebase+(c++)
+								}
+								if (c > 2) duplicatedAttributes.add(attrname);
+								attributes.put(attrname, "N/A")
+							} else {
+								println "Error: attribute: "+s1+ " with line "+line
+							}
+						} else if (splited2.length == 2) {
+							
+							String attrname = splited2[0].toLowerCase()
+							String attrnamebase = attrname = AsciiUtils.buildId(attrname);
+								int c = 2;
+								while (attributes.containsKey(attrname)) {
+									attrname = attrnamebase+(c++)
+								}
+								if (c > 2) duplicatedAttributes.add(attrname);
+							
+							String attrvalue = splited2[1]
+								
+							attributes.put(attrname, attrvalue)
+						} else {
+						
+							print("wrong size: name_value : "+splited2.length+" : ");
+							for (String s : splited ) print s+""
+							println ""
+						}
+					}
+					
+					for (def attname: attributes.keySet()) { // finally write the XML attributes
+						writer.writeAttribute(attname, attributes[attname]);
+					}
+					writer.writeCharacters("\n");
+					writer.writeStartElement("p"); // we create a tag <p>
+					writer.writeAttribute("n", ""+(p_n++));// then we add attributes to the tag
+
+				} else if (line.startsWith("-*")) {
+					println "ignored locution : "+line;
+				} else {
+					if (writer != null) {
+						if (line.length() > 0) {
+							writer.writeEmptyElement("lb");
+							writer.writeAttribute("n", ""+(lb_n++))
+							writer.writeCharacters("\n");
+						} else if (line.trim().length() == 0) {
+							writer.writeEndElement(); // p
+							writer.writeStartElement("p");
+							writer.writeAttribute("n", ""+(p_n++))
+							writer.writeCharacters("\n");
+						}
+						
+						writer.writeCharacters(line);
+					}
+				}
+				line = reader.readLine();
+				noline++;
+			}
+			// the end
+			if (writer != null) { // end last text
+				writer.writeEndElement(); // p
+				writer.writeEndElement(); // text
+
+				reader.close();
+				writer.close();
+				output.close();
+			}
+		} catch(Exception e) {
+			println "Error line : "+noline;
+			org.txm.utils.logger.Log.printStackTrace(e);
+		}
+		
+		if (duplicatedAttributes.size() > 0){
+			println "Some attributes were duplicated: "+duplicatedAttributes;
+		}
+		
+		if (textcount == 0) {
+			println "Error: no text found in "+infile;
+			return false;
+		}
+		
+		return true;
+	}
+
+	/**
+	 * Builds the next id.
+	 *
+	 * @return the string
+	 */
+	private String buildNextId() {
+		textcount++;
+		textno = ""+textcount;
+		while (textno.length() < nbOfStarNumbers) {
+			textno="0"+textno;
+		}
+		return textno;
+	}
+
+	/**
+	 * allow to guess if the file use **** or 000x format.
+	 *
+	 * @param infile the infile
+	 * @param encoding the file encoding
+	 * @return true, if the text marker is "****"
+	 */
+	private boolean guessStar(File infile, String encoding)
+	{
+		Reader input = new InputStreamReader(new FileInputStream(infile) , encoding);
+		String line = input.readLine();
+		while (line != null) {
+			if (line ==~ /^[0-9]{4,6} .*/) { // 0000
+				int idx = line.indexOf(" ");
+				nbOfStarNumbers = idx;
+				//println "NB NUMBER:" + nbOfStarNumbers
+				input.close();
+				return false;
+			} else if (line ==~ /^[*]{4,6} .*/) { 
+				input.close();
+				int idx = line.indexOf(" ");
+				nbOfStarNumbers = idx;
+				//println "NB STAR:" + nbOfStarNumbers
+				return true;
+			}
+			line = input.readLine();
+		}
+		input.close();
+	}
+
+	/**
+	 * Main.
+	 *
+	 * @param args the args
+	 */
+	static main(args)
+	{
+		//File infile = new File(System.getProperty("user.home")+"/xml/nature/Nature.txt");
+		File infile = new File(System.getProperty("user.home")+"/xml/TESTS2/alceste/Voeux.txt");
+		//File infile = new File("~/xml/geo/src/rivières.txt");
+		File outdir = new File(infile.getParentFile(), "out");
+		outdir.mkdir()
+		new Alceste2Xml().run(infile, outdir, "cp1252");
+		println "done: $infile -> $outdir"
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/compiler.groovy	(revision 321)
@@ -0,0 +1,684 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.tmx;
+
+import org.txm.importer.cwb.BuildAlignOut;
+import org.txm.importer.cwb.CwbAlign;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.cwb.PatchCwbRegistry;
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.logger.Log;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+
+import java.net.URL;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler
+{
+	/** The debug. */
+	private boolean debug= false;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private def output;
+
+	/** The url. */
+	private def url;
+
+	/** The text. */
+	String text="";
+
+	/** The base. */
+	String base="";
+
+	/** The project. */
+	String project="";
+
+	/** The anahash. */
+	static boolean firstWord = true;
+	private def anaTypes = [];
+	public def getAnaTypes() { return anaTypes; }
+	private HashMap<String,String> anahash = new HashMap<String,String>() ;
+
+	private static SAttributesListener sattrsListener;
+	private static HashMap<String,ArrayList<String>> structs;
+	private static HashMap<String, Integer> structsProf;
+	def headerAttrs; // only for TMX
+
+	/** The tuprops. */
+	List<String> tuprops;
+
+	/** The cwb loc. */
+	String cwbLoc;
+
+	/** The corporanames. */
+	List<String> corporanames = new ArrayList<String>();
+
+	/** The align structure. */
+	String alignStructure;
+
+	/** The align attribute. */
+	String alignAttribute;
+
+	/** The do align. */
+	boolean doAlign = false;
+	public static HashMap<String, Integer> segs_id = [:]; // static so the id are uniques
+	public static int seg_id = 0;
+
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 * @param tuprops the tuprops
+	 */
+	public compiler(URL url, String text, String base, String project, List<String> tuprops)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.tuprops = tuprops;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+
+			if (sattrsListener == null)
+				sattrsListener = new SAttributesListener(parser);
+			else
+				sattrsListener.start(parser)
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			//We don't want to concatenate the wtc files
+			output = new OutputStreamWriter(new FileOutputStream(f, f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+
+			return false;
+		}
+	}
+
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		createOutput(dirPathName, fileName);
+
+		GoToText();
+		firstWord = true;
+		String headvalue = ""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+
+		boolean captureword = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+
+		String anaType;
+		String tuRef;
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						sattrsListener.startElement(parser.getLocalName());
+						switch (parser.getLocalName()) {
+							case "header": // capture header attributes
+								headerAttrs = [:];
+								for ( int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									headerAttrs[parser.getAttributeLocalName(i)] =  parser.getAttributeValue(i);
+								}
+								break;
+							case "text":
+								output.write("<text id=\""+text+"\" base=\""+base+"\"" +
+								" project=\""+project+"\"");
+								def textAttrs = [];
+								for ( int i = 0 ; i < parser.getAttributeCount() ; i++)
+								{
+									String attrname = parser.getAttributeLocalName(i);
+									String attrvalue = parser.getAttributeValue(i);
+									if(attrname != "id")
+										output.write(" "+attrname+"=\""+attrvalue+"\"")
+									textAttrs << attrname
+								}
+								if (headerAttrs != null)
+									for( String key : headerAttrs.keySet()) {
+										if (!textAttrs.contains(key))
+											output.write(" "+key+"=\""+headerAttrs[key]+"\"")
+									}
+								output.write(">\n");
+
+								break;
+
+							case "tu":
+								output.write("<tu");
+								for( String attrname : tuprops)
+								{
+									String attrvalue = parser.getAttributeValue(null, attrname);
+									attrname = attrname.toLowerCase();
+									if(attrvalue == null)
+										output.write(" "+attrname+"=\"N/A\"")
+									else
+										output.write(" "+attrname+"=\""+attrvalue+"\"")
+								}
+								tuRef = text;
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"")
+									if (parser.getAttributeLocalName(i) == "ref")
+										tuRef = parser.getAttributeValue(i);
+								}
+								output.write(">\n");
+								break;
+							case "seg":
+								output.write("<seg")
+								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+									output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"")
+								}
+								output.write(">\n")
+								break;
+
+							case "w":
+								for(int i = 0 ; i < parser.getAttributeCount(); i++)
+									if(parser.getAttributeLocalName(i).equals("id"))
+								{
+									wordid = parser.getAttributeValue(i);
+								}
+								break;
+							case "form":
+								flagForm = true;
+								vForm = "";
+								anahash.clear();
+								break;
+
+							case "ana":
+								flagAna = true;
+								anaType = parser.getAttributeValue(null, "type")
+								if (anaType.length() > 0) anaType = anaType.substring(1);
+								//println "anatype $anaType"
+								anahash.put(anaType, "");
+								if (firstWord) {
+									anaTypes << anaType;
+								}
+								break;
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						sattrsListener.endElement(parser.getLocalName());
+						switch (parser.getLocalName())
+						{
+							case "text":
+								output.write("</text>\n");
+								break;
+
+							case "tu":
+								output.write("</tu>\n")
+								break;
+							case "seg":
+								output.write("</seg>\n")
+								break;
+
+							case "w":
+								firstWord = false;
+								output.write( vForm.replaceAll("&", "&amp;").replaceAll("<", "&lt;")
+									 +"\t"+wordid+"\t"+tuRef);
+								for(String type : anaTypes) {
+									output.write("\t"+anahash.get(type));
+								}
+								output.write("\n")
+
+								vForm = "";
+								break;
+
+							case "form":
+								flagForm = false;
+								break;
+
+							case "ana":
+								anahash.put(anaType, vAna);
+								vAna = "";
+								flagAna = false;
+								break;
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						if(flagForm)
+							vForm += parser.getText().trim();
+						if (flagAna) {
+							vAna += parser.getText().trim()
+						}
+						break;
+				}
+			}
+			output.close();
+			parser.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+
+		return true;
+	}
+
+	/**
+	 * Sets the alignment.
+	 *
+	 * @param structure the structure
+	 * @param attribute the attribute
+	 */
+	public void setAlignment(String structure, String attribute)
+	{
+		doAlign = true;
+		this.alignStructure = structure;
+		this.alignAttribute = attribute;
+	}
+
+	/**
+	 * Gets the tu props.
+	 *
+	 * @param files the files
+	 * @return the tu props
+	 */
+	public List<String> getTuProps(List<File> files)
+	{
+		Set<String> tuprops = new HashSet<String>();
+		for (File f : files) {
+			def inputData = f.toURI().toURL().openStream();
+			def factory = XMLInputFactory.newInstance();
+			def parser = factory.createXMLStreamReader(inputData);
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						switch (parser.getLocalName()) {
+							case "tu":
+							for ( int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								tuprops.add(parser.getAttributeLocalName(i));
+							}
+							break;
+						}
+				}
+			}
+		}
+		return new ArrayList<String>(tuprops);
+	}
+
+	public static HashMap<String, String> langs;
+	public void setLangs(HashMap<String, String> langs)
+	{
+		this.langs = langs;
+	}
+
+	public static HashMap<Integer, ArrayList<String>> langGroups;
+	public void setLangGroups(HashMap<Integer, ArrayList<String>> groups)
+	{
+		this.langGroups = groups;
+	}
+	
+	public static HashMap<Integer, ArrayList<String>> corpusIDS;
+	public void setCorpusIDS(HashMap<Integer, ArrayList<String>> corpusIDS)
+	{
+		this.corpusIDS = corpusIDS;
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @param textAttributes the text attributes
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String basename)
+	{
+		sattrsListener = null; // reset SAttribute Listener for each new import
+		String rootDir = binDir.getAbsolutePath();
+		seg_id=1;
+		if (cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if (!new File(rootDir).exists()) {
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+
+		new File(binDir,"/wtc/").deleteDir();
+		new File(binDir,"/wtc/").mkdir();
+		new File(binDir,"/data/").deleteDir();
+		new File(binDir,"/data/").mkdir();
+		new File(binDir,"registry/").mkdir();
+
+		String textid="";
+		int counttext =0;
+		List<File> files = txmDir.listFiles();
+
+		//0- get all tu props
+		//List<String> tuprops = getTuProps(files);
+
+		if (corpusIDS == null || corpusIDS.size() == 0) {
+			corpusIDS = [:];
+			for (int group : langGroups.keySet()) {
+				String lang = langs.get(langGroups.get(group)[0]);
+				if (lang ==null) {
+					println "ERROR: no lang defined for group $group . Aborting."
+				}
+				corpusIDS[group] = (lang+group).toLowerCase();
+			}
+		}
+		println "Using corpus ID: $corpusIDS"
+		
+		String wtcName;
+		for (int group : langGroups.keySet()) {
+			String lang = langs.get(langGroups.get(group)[0]);
+			segs_id.put(group, 0);
+			wtcName = basename+"_"+corpusIDS.get(group);
+			createOutput(rootDir+"/wtc", "${wtcName}.wtc");
+			output.write("<txmcorpus id=\"${wtcName}\" lang=\"$lang\">\n")
+			output.close();
+
+			//create txmDirs
+			new File(rootDir, "txm/"+wtcName.toUpperCase()).mkdir();
+		}
+
+		//1- Transform into WTC file and build a corpus per file
+		Collections.sort(files);
+		def anaTypesPerWtc = [:];
+		def builder = null;
+		for (int group : langGroups.keySet()) {
+			//String lang = langs.get(langGroups.get(group)[0]);
+			wtcName = basename+"_"+corpusIDS.get(group);
+			def filenames = langGroups.get(group);
+			filenames.sort()
+			//println("Process group no $group of files "+filenames)
+			for (String filename : filenames) {
+				File f = new File(txmDir, filename);
+				print "."
+				if (!f.exists()) {
+					println("COMPILER: file "+f+ " does not exists -> stop")
+					return false;
+				}
+
+				filename = filename.substring(0, filename.length()-4);
+				String corpusname = (wtcName).toLowerCase();
+				counttext++;
+
+				String txtname = f.getName();
+				txtname = txtname.substring(0, txtname.lastIndexOf("_"));
+				seg_id = segs_id.get(group);
+				builder = new compiler(f.toURI().toURL(), txtname, basename, "default", tuprops);
+				builder.transfomFileWtc(rootDir+"/wtc",wtcName+".wtc");
+				anaTypesPerWtc[wtcName] = builder.getAnaTypes();
+				segs_id.put(group, seg_id);
+
+				// move xml-txm file
+				File txmCorpusDir = new File(rootDir, "txm/"+wtcName.toUpperCase())
+				f.renameTo(new File(txmCorpusDir, f.getName()));
+			}
+		}
+		println ""
+		
+		for (int group : langGroups.keySet()) {
+			//String lang = langs.get(langGroups.get(group)[0]);
+			createOutput(rootDir+"/wtc", basename+"_"+corpusIDS.get(group)+".wtc");
+			output.write("</txmcorpus>")
+			output.close();
+		}
+
+		if(builder == null) {
+			System.out.println("No TXM files to process Stop import");
+			return false;
+		}
+		//2- Import into CWB
+		def outDir = rootDir;
+		def outDirTxm = rootDir;
+
+		def wtcFiles = [:]
+		for (File wtcfile : new File(rootDir, "wtc").listFiles()) {
+			String corpusname = wtcfile.getName();
+			
+			corpusname = corpusname.substring(0, corpusname.length()-4);
+			def corpusAnaTypes = anaTypesPerWtc[corpusname];
+			
+			CwbEncode cwbEn = new CwbEncode();
+			cwbEn.setDebug(debug);
+			CwbMakeAll cwbMa = new CwbMakeAll();
+			cwbMa.setDebug(debug);
+
+			def pAttrs = ["id", "ref"];
+			for(String type : corpusAnaTypes) {
+				int i = 2;
+				while (pAttrs.contains(type))
+					type = type+(i++);
+				pAttrs.add(type);
+			}
+
+			structs = sattrsListener.getStructs();
+			structsProf = sattrsListener.getProfs();
+			// add structures+properties found in sources
+			List<String> sargs = new ArrayList<String>();
+
+			for (String name : structs.keySet()) {
+				if (name == "header") continue;
+				//if ( name == "text") continue; // added after
+				//if ( name == "q") continue; // added after
+				//if ( name == "foreign") continue; // added after
+				String concat = name+":"+structsProf.get(name); // append the depth
+				for (String value : structs.get(name)) // append the attributes
+					concat += "+"+value;
+				if ((name == "text") &&
+				!(concat.endsWith("+id") || concat.contains("+id+")))
+					concat += "+id"
+				if ((name == "text") &&
+				!(concat.endsWith("+base") || concat.contains("+base+")))
+					concat += "+base"
+				if ((name == "text") &&
+				!(concat.endsWith("+project") || concat.contains("+project+")))
+					concat += "+project"
+				sargs.add(concat);
+			}
+
+			sargs.add("txmcorpus:0+id+lang")
+			sargs.sort();
+
+			String[] sAttributes = sargs;
+			String[] pAttributes = pAttrs;
+			println "P-attributes: "+pAttrs
+			println "S-attributes: "+sargs
+
+			try {
+				wtcFiles[corpusname.toLowerCase()] = outDir + "/wtc/"+wtcfile.getName();
+				String regPath = outDirTxm + "/registry/"+corpusname.toLowerCase()
+				cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), outDirTxm + "/data/"+corpusname.toUpperCase()+"/", outDir + "/wtc/"+wtcfile.getName(), regPath, pAttributes, sAttributes);
+				if (!new File(regPath).exists()) {
+					println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+					return false;
+				}
+				cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname.toUpperCase(), outDirTxm + "/registry");
+			} catch (Exception ex) {System.out.println(ex); return false;}
+		}
+
+		// 3- do alignement
+		if (doAlign) {
+			File registryDirectory = new File(rootDir,"registry");
+			def registryFiles = registryDirectory.listFiles();
+			for (File corpusName : registryFiles) {
+				for (File targetName : registryFiles) {
+					if (!corpusName.equals(targetName)) {
+						try {
+							PatchCwbRegistry.patchAlignment(new File(registryDirectory, corpusName.getName()), targetName.getName());
+							
+							File wtcFile1 = new File(wtcFiles[corpusName.getName()]);
+							File wtcFile2 = new File(wtcFiles[targetName.getName()]);
+							File alignOutFile = new File(outDir, "align.out");
+							
+							BuildAlignOut bao = new BuildAlignOut(wtcFile1, wtcFile2);
+							if (!bao.process(alignOutFile, "seg", "id")) {
+								println "Error while creating alignement file of $corpusName. Aborting."
+								return false;
+							}
+							
+							CwbAlign tt = new CwbAlign(cwbLoc);
+							// ./cwb-align-encode -D -r ~/TXM/corpora/tmxtest/registry/ -v out.align
+							tt = new CwbAlign(cwbLoc);
+							tt.setD();
+							tt.setv();
+							tt.setr(new File(rootDir,"registry").getAbsolutePath());
+							tt.cwbalignencode(alignOutFile.getAbsolutePath());
+						} catch (IOException e) {
+							Log.severe("Error while calling TreeTagger: "+e.getLocalizedMessage());
+							return false;
+						}
+					}
+				}
+			}
+		}
+
+		return true;
+	}
+
+	/**
+	 * Gets the corpora names.
+	 *
+	 * @return the corpora names
+	 */
+	public List<String> getCorporaNames()
+	{
+		return corporanames;
+	}
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>TMX import module - not yet finished </p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/Tmx2XmlFiles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/Tmx2XmlFiles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/Tmx2XmlFiles.groovy	(revision 321)
@@ -0,0 +1,468 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-12-17 12:11:39 +0100 (Thu, 17 Dec 2015) $
+// $LastChangedRevision: 3087 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.tmx;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.stream.*;
+
+import org.txm.objects.Base;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+
+import java.net.URL;
+
+import org.txm.objects.*;
+// TODO: Auto-generated Javadoc
+
+/**
+ * Split a TMX file into N xml files, N the number of lang declared in the <tuv> tags.
+ *
+ * @author mdecorde
+ */
+class Tmx2XmlFiles {
+
+	/** The lang. */
+	String lang;
+
+	/** The type. */
+	String type;
+
+	/** The infile. */
+	File infile;
+
+	/** The outdir. */
+	File outdir;
+
+	/** The writers. */
+	ArrayList<XMLStreamWriter> writers;
+	def outputs;
+
+	/** The textslangs. */
+	HashMap<String, String> textslangs;
+	HashMap<Integer, ArrayList<String>> langGroups;
+	Set<String> langs;
+	def header = [:];
+
+	/**
+	 * Instantiates a new tmx2 xml files.
+	 *
+	 * @param base the base
+	 */
+	public Tmx2XmlFiles(HashMap<String, String> textslangs, HashMap<Integer, ArrayList<String>> langGroups)
+	{
+		this.textslangs = textslangs;
+		this.langGroups = langGroups;
+	}
+
+	/**
+	 * Gets the text langs.
+	 *
+	 * @return the langs of the TMX files
+	 */
+	public HashMap<String, String> getTextLangs()
+	{
+		return textslangs;
+	}
+
+	/**
+	 * Gets the original texts
+	 *
+	 * @return the langs of the TMX files
+	 */
+	public HashMap<String, String> getLangGroups()
+	{
+		return langGroups;
+	}
+	
+	def corpusIDS = [:];
+	int noCorpusID = 0;
+	public def getCorpusIDS() {
+		return corpusIDS;
+	}
+
+	int TOTALTUV = 0;
+	public def initWriters(File tmxFile) {
+		println "initialize writers for : $tmxFile"
+		def inputData = infile.toURI().toURL().openStream();
+		def inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+
+		TOTALTUV = 0;
+		writers = new ArrayList<XMLStreamWriter>(); // one writer per text
+		outputs = []; // one writer per text
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if (parser.getLocalName() == "tuv") {
+
+					String lang = "fr";
+					for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+						if (parser.getAttributeLocalName(i) == "lang")
+					{
+						lang = (parser.getAttributeValue(i)).toLowerCase()
+						break;
+					}
+
+					buildWriter(TOTALTUV, lang);
+					TOTALTUV++
+				}
+			} else if (event == XMLStreamConstants.END_ELEMENT) {
+				if (parser.getLocalName() == "tu") {
+					break; // stop at the end of the first </tu>
+				}
+			}
+		}
+
+		parser.close()
+		inputData.close();
+	}
+
+	String textname;
+	int noTuv = -1;
+	XMLOutputFactory factory = XMLOutputFactory.newInstance();
+	public def buildWriter(int no, String lang) {
+		//println "build Writer : $no $lang"
+		if (writers.size() <= no) {
+			File outfile = new File(outdir, textname+no+".xml");
+			textLangs.put(outfile.getName(), lang);
+			if (!langGroups.containsKey(no)) langGroups[no] = [];
+			langGroups.get(no).add(outfile.getName());
+
+			FileOutputStream output = new FileOutputStream(outfile)
+			def writer = factory.createXMLStreamWriter(output, "UTF-8")
+			writers.add(no, writer);
+			outputs.add(no, output);
+
+			writer.writeStartDocument("UTF-8","1.0");
+			writer.writeStartElement ("TEI");
+			writer.writeStartElement ("teiHeader");
+			writer.writeEndElement();
+		}
+		return writers.get(no);
+	}
+
+	XMLStreamReader parser;
+	int tuCounter = 0;
+	/**
+	 * start the processing.
+	 *
+	 * @param infile the infile
+	 * @param outdir the outdir
+	 */
+	public void run(File indir, File outdir)
+	{
+		boolean isTuv = false;
+		boolean isProp = false;
+		boolean isTu = false;
+		boolean flagHeader = false;
+		String propvalue;
+		String localname;
+
+		def tuvProps = [:];
+		def tuProps = [:];
+		def corpusTITLES = [];
+		def corpusSUBTITLES = [];
+		def corpusAUTHORS = [];
+		def corpusTRANSLATORS = [];
+		
+		def tuTitles = [];
+		boolean endOfTuProp = false;
+
+		for (File infile : indir.listFiles()) {
+			if (!infile.canRead() || infile.getName().startsWith("import") || infile.getName().endsWith(".css"))  {
+				println "skip file : "+infile;
+				continue;
+			}
+			this.infile = infile;
+			this.outdir = outdir;
+			outdir.mkdir(); // ensure the directory exists
+			// get the filename without ".xml"
+			textname = infile.getName().substring(0,infile.getName().length()-4)+"_";
+			initWriters(infile);
+
+			// create XML reader
+			def inputData = infile.toURI().toURL().openStream();
+			def inputfactory = XMLInputFactory.newInstance();
+			parser = inputfactory.createXMLStreamReader(inputData);
+			def headerProps = [:];
+			noTuv = -1;
+			//getWriter(noTuv);
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+
+						if (localname == "tuv") {
+							noTuv++;
+							
+							if (endOfTuProp) {
+								endOfTuProp = false;
+								//println "localname: "+localname+" $tuTitles "//+parser.getLocation()
+								for(int i = 0 ; i < writers.size() && i < tuTitles.size() ; i++) {
+									writers.get(i).writeAttribute("title", tuTitles.get(i));
+								}
+							}
+							
+							this.writeStartElement("seg");
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) // get the lang attribute of the tuv
+								if (parser.getAttributeLocalName(i) == "lang")
+							{
+								lang = parser.getAttributeValue(i)
+								break;
+							}
+							
+							
+							
+							this.writeAttribute("id", "seg_$tuCounter");
+						} else if (localname == "seg") {
+							// do nothing :)
+						} else if (localname == "prop") {
+							type = null;
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) { // get the type attribute of the prop
+								if (parser.getAttributeLocalName(i) == "type") {
+									type = parser.getAttributeValue(i)
+									break;
+								}
+							}
+							if (type == null) {
+								println "ERROR NO PROP TYPE : "+parser.getLocation()
+							}
+							propvalue= "";
+							isProp= true;
+						} else if (localname == "header") {
+							flagHeader = true;
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) { // get the type attribute of the prop
+								headerProps[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i)
+							}
+						} else {
+
+							if (endOfTuProp && localname == "note") {
+								//println "localname: note $tuTitles "
+								for (int i = 0 ; i < writers.size() ; i++) {
+									writers.get(i).writeAttribute("title", tuTitles.get(i));
+								}
+								endOfTuProp = false;
+							}
+							
+							if (localname == "tmx") continue;
+							if (localname == "tu") {
+								tuTitles = [];
+								isTu= true;
+								noTuv = -1;
+								endOfTuProp = true; // wait for the next start element that is not note or prop
+								tuCounter++;
+							}
+							if (localname == "body")
+								localname = "text"
+							this.writeStartElement(localname); // create text tag in each xml file
+
+							for (int i= 0 ; i < parser.getAttributeCount() ;i++) {
+								this.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+							}
+							if (localname == "text") {
+								println "add header : $headerProps"
+								//for(String key : headerProps.keySet())	this.writeAttribute(key, headerProps[key]);
+
+								for (int i = 0 ; i < writers.size() ; i++) {
+									if (corpusIDS.size() > i)
+										writers.get(i).writeAttribute("corpusid", corpusIDS[i]);
+									if (corpusTITLES.size() > i)
+										writers.get(i).writeAttribute("title", corpusTITLES.get(i));
+									if (corpusSUBTITLES.size() > i)
+										writers.get(i).writeAttribute("subtitle", corpusSUBTITLES.get(i));
+									if (corpusAUTHORS.size() > i)
+										writers.get(i).writeAttribute("author", corpusAUTHORS.get(i));
+									if (corpusTRANSLATORS.size() > i)
+										writers.get(i).writeAttribute("translator", corpusTRANSLATORS.get(i));
+								}
+							}
+						}
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName();
+						switch (localname) {
+							case "tuv":
+								lang = null
+								this.writeEndElement();
+								break
+							case "seg":
+								break;
+							case "prop":
+								//println "flagHeader $flagHeader isTu $isTu type $type"
+								if (flagHeader) {
+									if (type == "corpusId") {
+										corpusIDS[noCorpusID++] = propvalue.toLowerCase();
+									} else if (type == "title") {
+										corpusTITLES << propvalue;
+									} else if (type == "subtitle") {
+										corpusSUBTITLES << propvalue;
+									} else if (type == "author") {
+										corpusAUTHORS << propvalue;
+									} else if (type == "translator") {
+										corpusTRANSLATORS << propvalue;
+									}
+								} else if (isTu && type == "tuTitle") {
+									tuTitles << propvalue
+								} else {
+									writeAttribute(type, propvalue); // transform all properties to attributes
+								}
+								isProp= false;
+								break;
+							case "tmx":
+								break;
+							case "tu":
+								isTu= false;
+								endOfTuProp = false;
+								noTuv = -1;
+								this.writeEndElement();
+							case "header":
+								flagHeader = false;
+								break;
+							default:
+								this.writeEndElement();
+								break;
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						if (isProp) {
+							propvalue += parser.getText(); // get prop tag text
+						} else {
+							this.writeCharacters(parser.getText());
+						}
+						break;
+
+				}
+			}
+
+			this.writeEndElement(); // close TEI
+			parser.close();
+			inputData.close();
+
+			for (XMLStreamWriter writer : writers) { // close all xml file output streamS
+				writer.close();
+			}
+			for (def output : outputs) { // close all xml file output streamS
+				output.close();
+			}
+		}
+	}
+
+	/**
+	 * call writeEndElement in all outputstream
+	 * if lang field is set, write only in the stream of the file with the corresponding lang.
+	 */
+	private void writeEndElement()
+	{
+		if (noTuv == -1) {
+			for (def writer : writers) writer.writeEndElement();
+		} else
+			writers.get(noTuv).writeEndElement();
+	}
+
+	/**
+	 * call writeStartElement in all outputstream
+	 * if lang field is set, write only in the stream of the file with the corresponding lang.
+	 *
+	 * @param localname the localname
+	 */
+	private void writeStartElement(String localname)
+	{
+		if (noTuv == -1) {
+			for (def writer : writers) writer.writeStartElement(localname);
+		} else
+			writers.get(noTuv).writeStartElement(localname);
+	}
+
+	/**
+	 * call writeStartDocument in all outputstream
+	 * if lang field is set, write only in the stream of the file with the corresponding lang.
+	 *
+	 * @param text the text
+	 */
+	private void writeCharacters(String text)
+	{
+		//System.err.println "write chars elem : '"+text+"' >> "+lang
+		text = text.trim();
+		if (text.length() > 0) {
+			if (noTuv == -1) {
+				for (def writer : writers) writer.writeCharacters(text);
+			} else
+				writers.get(noTuv).writeCharacters(text);
+		}
+	}
+
+	/**
+	 * call writeAttribute in all outputstream
+	 * if lang field is set, write only in the stream of the file with the corresponding lang.
+	 *
+	 * @param name the name
+	 * @param value the value
+	 */
+	private void writeAttribute(String name, String value)
+	{
+		try {
+			if (noTuv == -1) {
+				for (def writer : writers) writer.writeAttribute(name, value);
+			} else
+				writers.get(noTuv).writeAttribute(name, value);
+		} catch (Exception e) { org.txm.utils.logger.Log.printStackTrace(e); System.err.println("ERROR: "+parser.getLocation());}
+	}
+
+	/**
+	 * Main.
+	 *
+	 * @param args the args
+	 */
+	static main(args)
+	{
+		File infile = new File(System.getProperty("user.home"),"xml/tmx/sample.tmx");
+		File outfile = new File(System.getProperty("user.home"),"TXM/corpora/tmx/");
+		new Tmx2XmlFiles().run(infile, outfile);
+		println "done"
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/pager.groovy	(revision 321)
@@ -0,0 +1,449 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-06-01 17:47:31 +0200 (ven., 01 juin 2012) $
+// $LastChangedRevision: 2185 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.tmx;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** Build GEO corpus simple edition from a xml-tei. @author mdecorde */
+class pager {
+	List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename = "";
+	String txtname = "";
+	File outdir;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue;
+
+	/** The interpvalue. */
+	String interpvalue;
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	OutputStreamWriter writer;
+
+	/** The multiwriter. */
+	OutputStreamWriter multiwriter = null;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	String editionPage;
+	ArrayList<Integer> splitTUs; // contains the tu ids used to split pages
+	boolean shouldSplit = false;
+	boolean useSplitTUs = false;
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager(File infile, File outdir, String txtname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String basename, String editionPage, ArrayList<Integer> splitTUs) {
+		this.editionPage = editionPage;
+		this.basename = basename;
+		this.txtname = txtname;
+		this.outdir = outdir;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+		this.splitTUs = splitTUs;
+		//println "spliting pages with : "+splitTUs
+		useSplitTUs = splitTUs.size() > 0;
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		process();
+	}
+
+	private void closeMultiWriter()
+	{
+		if (multiwriter != null) {
+			if (firstWord) { // there was no words
+				this.idxstart.add("w_0")
+				multiwriter.write("<span id=\"w_0\"/>");
+			}
+			multiwriter.write("</div>\n")
+			multiwriter.write("</body>");
+			multiwriter.write("</html>");
+			multiwriter.close();
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		wordcount = 0;
+		shouldSplit = false;
+		try {
+			closeMultiWriter();
+			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")
+			pages.add(outfile);
+			firstWord = true; // waiting for next word
+
+			multiwriter = new OutputStreamWriter(new FileOutputStream(outfile) , "UTF-8");
+
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"   \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n")
+			multiwriter.write("<html>");
+			multiwriter.write("<head>");
+			multiwriter.write("<title>"+basename.toUpperCase()+" $txtname Edition - Page "+pagecount+"</title>");
+			multiwriter.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"tmx.css\"/>");
+			multiwriter.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\"/>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			multiwriter.write("<div class=\"tmx\">\n")
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput() {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * Process.
+	 */
+	void process() {
+
+		String vNote = ""
+		String cssNote = "note"
+		boolean flagNote = false;
+		
+		String tuCSS = "tu"
+		
+		int tuCounter = 0;
+		String localname = "";
+		createNextOutput();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							boolean grey = "true"
+							String title = txtname;
+							String subtitle = "";
+							String author = "";
+							String translator = "";
+							String table = "";
+							table += ("<table>\n");
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								String name = parser.getAttributeLocalName(i);
+								grey = !grey;
+								if (grey) table += ("<tr  style=\"background-color:lightgrey;\">");
+								else table += ("<tr>");
+								
+								table += ("<td>"+parser.getAttributeLocalName(i)+": </td>");
+								table += ("<td>"+parser.getAttributeValue(i)+"</td>");
+								table += ("</tr>\n");
+								
+								if (name == "title") title = parser.getAttributeValue(i);
+								else if (name == "subtitle") subtitle = parser.getAttributeValue(i);
+								else if (name == "author") author = parser.getAttributeValue(i);
+								else if (name == "translator") translator = parser.getAttributeValue(i);
+							}
+							multiwriter.write("<div class=\"header\">")
+							multiwriter.write("<span class=\"authorStyle\"> "+author+" </span>")
+							multiwriter.write("<span class=\"titleStyle\"> "+title+" </span>")
+							multiwriter.write("<span class=\"subtitleStyle\"> "+subtitle+" </span>")
+							if (translator.size() > 0)
+								multiwriter.write("<span class=\"translatorStyle\"> "+translator+" </span>")
+							multiwriter.write("</div>")
+//							multiwriter.write(table)
+//							multiwriter.write("</table><br/>\n");
+							break;
+						case "head":
+							multiwriter.write("<h2>\n")
+							break;
+						case "graphic":
+							String url = parser.getAttributeValue(null, "url")
+						//println "URL: "+url
+							if (url != null) multiwriter.write("<div><img src=\"$url\"/></div>");
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							String rend = parser.getAttributeValue(null, "rend")
+							if (rend == null) rend = "normal"
+							multiwriter.write("<p class=\"$rend\">\n")
+							break;
+						//case "pb":
+						case editionPage:
+							createNextOutput();
+							wordcount=0;
+							if (parser.getAttributeValue(null,"n") != null) {
+								multiwriter.write("<p style=\"color:red\" align=\"center\">- "+parser.getAttributeValue(null,"n")+" -</p>\n")
+							}
+							break;
+						case "lb":
+						case "br":
+							multiwriter.write("<br/>\n")
+							break;
+						case "tu":
+							tuCounter++;
+							if (useSplitTUs) {
+								if (splitTUs.contains(tuCounter))
+									createNextOutput();
+							} else if (shouldSplit) {
+								createNextOutput();
+								this.splitTUs << tuCounter
+							}
+							String str = "";
+							String title;
+							tuCSS = "tu"
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								str += " "+parser.getAttributeLocalName(i)+"="+parser.getAttributeValue(i)
+								if (parser.getAttributeLocalName(i).equals("cssClass")) {
+									tuCSS = parser.getAttributeValue(i)
+								} else if (parser.getAttributeLocalName(i).equals("title")) {
+									title = parser.getAttributeValue(i)
+								}
+							}
+							if (title != null) str = title;
+							multiwriter.write("<h5 class=\"${tuCSS}Title\">$str</h5>\n")
+							break;
+						case "note":
+						cssNote = "note"
+						for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+							if (parser.getAttributeLocalName(i) == "cssClass") {
+								cssNote = parser.getAttributeValue(i)
+								break
+							}
+						}
+							vNote = "";
+							flagNote = true;
+							break;
+						case "seg":
+						String css = "seg"
+							if (tuCSS != "tu") css = tuCSS;
+							multiwriter.write("<p class=\"${css}\">\n");
+							break;
+						case "w":
+							wordid = parser.getAttributeValue(null,"id");
+
+							if (firstWord) {
+								firstWord = false;
+								this.idxstart.add(wordid);
+							}
+							wordcount++;
+							if (wordcount >= wordmax) {
+								//createNextOutput();
+								shouldSplit = true;
+								//println "should split: "+wordcount
+							}
+							break;
+
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "head":
+							multiwriter.write("</h2>\n")
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							multiwriter.write("</p>\n")
+							break;
+
+						case "seg":
+							multiwriter.write("</p>\n");
+							break;
+						case "note":
+							flagNote = false;
+							multiwriter.write("<div class=\"${cssNote}\">${vNote}</div>")
+							break;
+
+						case "form":
+							flagform = false
+							break;
+						case "ana":
+							flaginterp = false
+							break;
+						case "w":
+
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if (l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+
+							if (interpvalue != null)
+								interpvalue = interpvalue.replace("&", "&amp;").replace("<", "&lt;").replace("\"","&quot;");
+
+							if (NoSpaceBefore.contains(wordvalue) ||
+							NoSpaceAfter.contains(lastword) ||
+							wordvalue.startsWith("-") ||
+							NoSpaceAfter.contains(endOfLastWord)) {
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							} else {
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+						//writer.write(wordvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+							break;
+					}
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					if (flagform)
+						if (parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if (flaginterp)
+						if (parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					if (flagNote)
+						if (parser.getText().length() > 0)
+							vNote+=(parser.getText());
+					break;
+			}
+		}
+		//writer.write("</body>");
+		//writer.write("</html>");
+		//writer.close();
+		closeMultiWriter();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/tmxLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/tmxLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/tmxLoader.groovy	(revision 321)
@@ -0,0 +1,194 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2015-08-31 13:45:22 +0200 (Mon, 31 Aug 2015) $
+// $LastChangedRevision: 3028 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.tmx;
+import org.txm.searchengine.cqp.corpus.MainCorpus;
+
+import java.io.File;
+
+import org.txm.importer.tmx.importer;
+import org.txm.importer.tmx.compiler;
+import org.txm.importer.tmx.pager;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+def MONITOR;
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/tmx/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang;
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);;
+int wordsPerPage = params.getWordsPerPage("default")
+String page_element = params.getPageElement("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir, "txm");
+txmDir.deleteDir();
+txmDir.mkdirs();
+HashMap<String, String> textLangs = [:];
+def langGroups = [:];
+
+println "-- IMPORTER - Reading source files"
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "IMPORTER - Reading source files");
+def imp = new importer()
+imp.run(srcDir, binDir, txmDir, textLangs, langGroups);
+def corpusIDS = imp.getCorpusIDS()
+println "-- ANNOTATE - Running NLP tools"
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE - Running NLP tools");
+new Annotate().run(binDir, txmDir, textLangs);
+
+println "langs : "+textLangs
+println "texts : "+langGroups
+
+println "-- COMPILING - Building Search Engine indexes"
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "COMPILING - Building Search Engine indexes")
+def c = new compiler();
+if (debug) c.setDebug();
+c.setAlignment("seg","id");
+c.setLangs(textLangs);
+c.setCorpusIDS(corpusIDS);
+c.setLangGroups(langGroups);
+if (!c.run(binDir, txmDir, basename)) {
+	println "Compiler failed"
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+new File(binDir,"HTML").deleteDir();
+new File(binDir,"HTML").mkdirs();
+if (build_edition) {
+	
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(20, "EDITION - Building edition")
+	
+	List<File> filelist = new File(binDir,"txm").listFiles();
+	def second = 0
+
+	def splitTUsPerText = [:]
+
+	for (File txmCorpusDir : txmDir.listFiles()) {
+		def txmFiles = txmCorpusDir.listFiles();
+		txmFiles.sort() // same order
+		//println "paging txmFiles : "+txmFiles
+
+		File outdir = new File(binDir,"/HTML/"+txmCorpusDir.getName()+"/default/");
+		outdir.mkdirs();
+		//println "processing pages of corpus "+txmCorpusDir.getName()
+
+		Element paraCorpusElem = corpusElem.cloneNode(true);
+		paraCorpusElem.setAttribute("name", txmCorpusDir.getName());
+
+		for (File srcfile : txmFiles) {
+			print "."
+			String txtname = srcfile.getName();
+			int i = txtname.lastIndexOf("_");
+			if(i > 0) txtname = txtname.substring(0, i);
+
+			//println ""+srcfile.getName()+" -> "+splitTUsPerText[txtname]
+			def splitTUs = splitTUsPerText[txtname];
+			if (splitTUs == null) {
+				splitTUs = [];
+				splitTUsPerText[txtname] = splitTUs;
+			}
+
+			String l = textLangs.get(srcfile.getName())
+			if (l == null) l = "fr";
+			List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(l);
+			List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(l);
+
+			Element text = params.addText(paraCorpusElem, txtname, srcfile);
+
+			def ed = new pager(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element,splitTUs);
+			Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+			for (i = 0 ; i < ed.getPageFiles().size();) {
+				File f = ed.getPageFiles().get(i);
+				String wordid = ed.getIdx().get(i);
+				params.addPage(edition, ""+(++i), wordid);
+			}
+		}
+
+		params.corporaElement.appendChild(paraCorpusElem);
+
+		File cssfile = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "css/tmx.css")
+		if (cssfile.exists()) {
+			FileCopy.copy(cssfile, new File(outdir, "tmx.css"));
+		}
+		File stylefile = new File(binDir, "style.css")
+		if (stylefile.exists()) {
+			FileCopy.copy(stylefile, new File(outdir, "style.css"));
+		}
+		println ""
+	}
+}
+params.corporaElement.removeChild(corpusElem);
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/tmx/importer.groovy	(revision 321)
@@ -0,0 +1,151 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-12-01 17:48:31 +0100 (Thu, 01 Dec 2016) $
+// $LastChangedRevision: 3351 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.tmx
+
+
+
+//import filters.TabulatedToXml.*;
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.*;
+//import org.txm.utils.treetagger.TreeTagger;
+import org.txm.utils.io.FileCopy;
+
+import javax.xml.stream.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.net.URL;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.FusionHeader.*;
+import filters.TagSentences.*;
+
+import org.txm.objects.*;
+import org.txm.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+	static Tmx2XmlFiles converter;
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param base the base
+	 */
+	public static void run(File srcDir, File binDir, File txmDir, HashMap<String, String> langs, HashMap<String, String> originalTexts)
+	{
+		String filename = binDir.getName();
+		
+		new File(binDir, "ptokenized").deleteDir();
+		new File(binDir, "ptokenized").mkdir();
+		new File(binDir, "tokenized").deleteDir();
+		new File(binDir, "tokenized").mkdir();
+		new File(binDir, "split").deleteDir();
+		new File(binDir, "split").mkdir();
+
+		//build xml files
+		if(srcDir.listFiles() == null || srcDir.listFiles().size() == 0)
+			return;
+		//File tmxfile = rootDirFile.listFiles()[0]
+		converter = new Tmx2XmlFiles(langs, originalTexts) // fill langs and originalTexts dictionaries
+		converter.run(srcDir, new File(binDir, "split"));
+			
+		List<File> srcfiles = new File(binDir, "split").listFiles();
+		println("Tokenizing "+srcfiles.size()+" files")
+		for (File f : srcfiles) {
+			
+			String lang = langs.get(f.getName());
+						
+			print "."
+			File outfile = new File(binDir,"tokenized/"+f.getName());
+			SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(f, outfile, lang)
+			tokenizer.setOutSideTextTagsAndKeepContent("note");
+			if (!tokenizer.process()) {
+				println("Failed to tokenize $f")
+				outfile.delete()
+			}
+		}
+		println ""
+		
+		// copy css file if any
+		File cssSrc = new File(srcDir, "style.css");
+		File cssBin = new File(binDir, "style.css");
+		if (cssSrc.exists()) {
+			FileCopy.copy(cssSrc, cssBin)
+		}
+		
+		//TRANSFORM INTO XML-TEI-TXM
+		List<File> tokenfiles = new File(binDir,"tokenized").listFiles()
+		println("Building xml-tei-txm ("+tokenfiles.size()+" files)")
+		for (File f : tokenfiles) {
+			print "."
+			String txmfile = f.getName();
+			
+			def correspType = new HashMap<String, String>()
+			def correspRef = new HashMap<String, String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String, HashMap<String,String>>();	
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+			//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String, String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String, HashMap<String, String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String, String[]>();
+			//lance le traitement
+			String wordprefix = "w_";
+			def builder = new Xml2Ana(f);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			//builder.setAddTEIHeader();
+			if (!builder.process(new File(txmDir, txmfile))) {
+				println("Failed to process file "+f)
+				new File(srcDir, txmfile).delete()
+			}
+		}
+		println ""
+	}
+	
+	public def getCorpusIDS() {
+		converter.getCorpusIDS()
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/FactivaMail2Alceste.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/FactivaMail2Alceste.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/FactivaMail2Alceste.groovy	(revision 321)
@@ -0,0 +1,153 @@
+package org.txm.importer.factiva
+/**
+ * 
+ * This is a Groovy port of the converter made by Pierre Ratinaud
+ * #Author: Pierre Ratinaud
+ * #Copyright (c) 2012 Pierre Ratinaud
+ * #Lisense: GNU/GPL
+ * 
+ * @author mdecorde
+ *
+ */
+class FactivaMail2Alceste {
+
+	def txtdir;
+	def fileout;
+	def encodage_in = 'UTF-8'
+	def encodage_out = 'UTF-8'
+
+	def metadata = ["BY":"Author",
+		"ART":"CaptionsDescriptionGraphics",
+		"CLM":"Column",
+		"CT":"Contact",
+		"CY":"Copyright",
+		"CX":"Correction",
+		"CR":"Credit",
+		"ED":"Edition",
+		"HD":"Headline",
+		"LA":"Language",
+		"PG":"Page",
+		"PD":"PublicationDate",
+		"ET":"PublicationTime",
+		"RBBCM":"ReutersCodes",
+		"SE":"Section",
+		"SC":"SourceCode",
+		"GC":"SourceGroupCode",
+		"NGC":"SourceGroupName",
+		"SN":"SourceName",
+		"TD":"TextFollowingLeadParagraphs",
+		"VOL":"Volume",
+		"WC":"WordCount"]
+
+	def starts = ["LP":"LeadParagraph"]
+
+	def ends = ["RF":"Reference",
+		"AN":"AccessionNumber",
+		"PUB":"PublisherName",
+		"RE":"RegionCodeDescriptor",
+		"IPC":"InformationProviderCodes",
+		"IN":"IndustryCodeDescriptor",
+		"CO":"CompanyCodeName",
+		"IPD":"InformationProviderDescriptors",
+		"NS":"SubjectCodeDescriptor",
+		"DJIC":"DowJonesCodes",
+		"DJID":"DowJonesDescriptors"]
+	
+	def ucis = []
+	def writeTextOK = false
+	def nextLineIsAnEndMetadataValue = false
+	def endsKey
+
+	FactivaMail2Alceste(File txtdir, File fileout, String encodage_in, String encodage_out) {
+		this.txtdir = txtdir;
+		this.fileout = fileout;
+		this.encodage_in = encodage_in;
+		this.encodage_out = encodage_out;
+	}
+
+	def toAttribute(String str) {
+		return str.replaceAll(' ','')
+//		.replaceAll('\'','').replaceAll(":", "").replaceAll("\\|", "").replaceAll("_","")
+//			 .replaceAll('´','').replaceAll('’','').replaceAll('-','').toLowerCase().replaceAll("/", "").replaceAll("\\.", "")
+//			 .replaceAll(" \"", "")
+	}
+	
+	def writeText(String str) {
+		if (writeTextOK) ucis[-1][1] += str+"\n";
+	}
+	
+	/**
+	 * parser de texte pour factiva
+	 * @return liste d'uci (= list header + corp)
+	 */
+	def parsetxt(txt) {
+		def lines = txt.split("\n") // met le texte dans une liste de lignes
+		def keepline = false
+		
+		for (String line : lines)  { //pour chaque ligne du texte...
+			if (line.length() == 0) continue;
+			
+			if (nextLineIsAnEndMetadataValue) {
+				//println "DATA: "+line
+				ucis[-1][0] += " *${endsKey}_"+toAttribute(line)
+				nextLineIsAnEndMetadataValue = false;
+			} else if (line.startsWith('---------------------------------------------------------------')) {// si la ligne commence avec...
+				ucis << ['****',''] // nouveau texte
+			} else {
+				int idxSpace = line.indexOf(" ")
+				if (0 <= idxSpace && idxSpace < 5) { // maybe a key
+					String head = line.substring(0, idxSpace)
+					String tail = line.substring(idxSpace+1)
+					String tail2;
+					int idxTab = tail.indexOf("\t");
+
+					if (metadata.containsKey(head)) {
+						if (idxTab == 0 && tail.length() > 0) {
+							ucis[-1][0] += " *${head}_"+toAttribute(tail.substring(1));
+						} else if (tail.length() > 0){
+							writeText(line)
+						}
+					} else if (ends.containsKey(head)) {
+						if (idxTab == 0) nextLineIsAnEndMetadataValue = true;
+						endsKey = head;
+						writeTextOK = false; // stop writing text
+					} else if (starts.containsKey(head)) {
+						writeTextOK = true; // start writing text
+					} else {
+						writeText(line)
+					}
+				} else {
+					writeText(line)
+				}
+			}
+		}
+		return ucis
+	}
+
+	def doparse() {
+		def files = txtdir.listFiles() // liste des fichiers dans txtdir
+		fileout.withWriter("UTF-8") { writer ->
+			for (File f : files) { // pour chaque fichier en entree...
+				if (f.isHidden() || !f.canRead() || !f.isFile() || f.getName() == "import.xml") {
+					continue;
+				}
+				println "parse file: $f"
+				for (def uci : parsetxt(f.getText("UTF-8"))) { // on récupère les lignes
+					writer.println uci[0] // header
+					writer.println uci[1] // corp
+				}
+			}
+		}
+		return true;
+	}
+
+	public static void main(String[] args) {
+		File  txtdir = new File("/home/mdecorde/xml/factivatxt/") // repertoire des textes
+		File fileout = new File("/home/mdecorde/xml/factivatxt/montagne-out.txt")
+		String encodage_in = "UTF-8"
+		String encodage_out = "UTF-8"
+		def converter = new FactivaMail2Alceste(txtdir, fileout, encodage_in, encodage_out);
+
+		println 'Done: '+converter.doparse()
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/factivaLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/factivaLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/factivaLoader.groovy	(revision 321)
@@ -0,0 +1,290 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-02-13 11:42:24 +0100 (lun., 13 févr. 2012) $
+// $LastChangedRevision: 2126 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.factiva;
+
+import javax.xml.stream.XMLStreamReader;
+
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.Factiva2XmlTxm;
+import org.txm.importer.ValidateXml;
+import org.txm.importer.xml.importer;
+import org.txm.importer.xml.compiler;
+import org.txm.importer.xml.pager;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.txm.metadatas.*;
+
+import javax.xml.stream.*;
+
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true;
+	params = new BaseParameters(new File(userDir, "xml/TESTS/factiva/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+String page_element = params.getPageElement("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+File propertyFile = new File(rootDir, "import.properties")//default
+Properties props = new Properties();
+String[] metadatasToKeep;
+
+String textSortAttribute = null;
+String paginationElement = null;
+boolean normalizeMetadata = false;
+String ignoredElements = null;
+boolean stopIfMalformed = false;
+
+println "Trying to read import properties file: "+propertyFile
+if (propertyFile.exists() && propertyFile.canRead()) {
+	InputStreamReader input = new InputStreamReader(new FileInputStream(propertyFile) , "UTF-8");
+	props.load(input);
+	input.close();
+	if(props.getProperty("sortmetadata") != null)
+		textSortAttribute = props.get("sortmetadata").toString();
+	if(props.getProperty("editionpage") != null)
+		paginationElement = props.get("editionpage").toString();
+	if(props.getProperty("normalizemetadata") != null)
+		normalizeMetadata = Boolean.parseBoolean(props.get("normalizemetadata").toString());
+	if(props.getProperty("ignoredelements") != null)
+		ignoredElements = props.get("ignoredelements").toString();
+	if(props.getProperty("stopIfMalformed") != null)
+		stopIfMalformed = Boolean.parseBoolean(props.get("stopIfMalformed").toString());
+
+	println "import properties: "
+	println " sort metadata: "+textSortAttribute
+	println " edition page tag: "+paginationElement
+	println " normalize attributes: "+normalizeMetadata
+	println " ignored elements: "+ignoredElements
+	println " stop if a XML source is malformed: "+stopIfMalformed
+}
+
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+
+// Apply Factiva2XMLTXM
+println "Factiva -> xml/w"
+File factivaOutDir = new File(binDir, "src")
+new Factiva2XmlTxm().run(srcDir, factivaOutDir.getAbsolutePath());
+if (factivaOutDir.listFiles() == null) {
+	println "Error: no file to process"
+	return;
+}
+srcDir = factivaOutDir;
+
+// Apply XSL
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "APPLYING XSL")
+if (xsl != null && xslParams != null && xsl.trim().length() > 0) {
+	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))
+	// return; // error during process
+	srcDir = new File(binDir, "src");
+	println ""
+}
+
+// copy txm files
+List<File> srcfiles = srcDir.listFiles();
+for (int i = 0 ; i < srcfiles.size() ; i++) {// check XML format, and copy file into binDir
+	File f = srcfiles.get(i)
+	if (f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties")) {
+		srcfiles.remove(i);
+		i--;
+		continue;
+	}
+	if (ValidateXml.test(f)) {
+		FileCopy.copy(f, new File(txmDir, f.getName()));
+	} else {
+		println "Won't process file "+f;
+	}
+}
+
+if (txmDir.listFiles() == null) {
+	println "No txm file to process"
+	return;
+}
+
+//get metadata values from CSV
+Metadatas metadatas; // text metadata
+println "Trying to read metadata from: "+allmetadatasfile
+if (allmetadatasfile.exists()) {
+	File copy = new File(binDir, "metadata.csv")
+	if (!FileCopy.copy(allmetadatasfile, copy)) {
+		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
+		return;
+	}
+	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+} else
+	println "no metadata file: "+allmetadatasfile
+
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+def imp = new importer();
+imp.doValidation(true) // change this to not validate xml
+imp.doTokenize(true) // change this, to not tokenize xml
+imp.setStopIfMalformed(stopIfMalformed);
+if (!imp.run( srcDir, binDir, txmDir, basename, null, lang)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- INJECTING METADATA - from csv file: "+allmetadatasfile
+if (metadatas != null) {
+	println("Injecting metadata: "+metadatas.getHeadersList()+" in texts of directory "+new File(binDir,"txm"))
+	for (File infile : txmDir.listFiles()) {
+		print "."
+		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {
+			outfile.delete();
+		} else {
+			if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+			if (!infile.exists()) {
+				println "Error: could not replace $infile by $outfile"
+				return false;
+			}
+		}
+	}
+	println ""
+}
+
+List<File> files = txmDir.listFiles()
+if (files == null || files.size() == 0) {
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(10, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = true;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if(debug) c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setOptions(textSortAttribute, normalizeMetadata);
+c.setAnnotationSuccess(annotationSuccess)
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname, null, srcfiles, metadatas)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(20, "EDITION")
+	
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+	def second = 0
+
+	println "Paginating texts: "
+
+	for (File txmFile : filelist) {
+		print "."
+		String txtname = txmFile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(corpusElem, txtname, txmFile);
+
+		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/factivamailLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/factivamailLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/factivamailLoader.groovy	(revision 321)
@@ -0,0 +1,158 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-07-20 17:33:42 +0200 (ven., 20 juil. 2012) $
+// $LastChangedRevision: 2221 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.factiva;
+
+import org.txm.importer.alceste.importer;
+import org.txm.importer.alceste.compiler;
+import org.txm.importer.xml.pager_old;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception) {
+	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/factivatxt/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+// TRANSFORM FACTIVA TO ALCESTE HERE
+File newSrcDir = new File(binDir, "src")
+newSrcDir.mkdir();
+File alcesteFile = new File(newSrcDir, srcDir.getName()+".txt");
+FactivaMail2Alceste converter = new FactivaMail2Alceste(srcDir, alcesteFile, "UTF-8", "UTF-8")
+if (!converter.doparse()) {
+	println "Failed to convert files in $srcDir to one Alceste file"
+	return false;
+}
+if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+println "-- IMPORTER - Reading source files"
+
+if (!(new importer().run(newSrcDir, binDir, txmDir, encoding, basename, lang))) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = true;
+}
+
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+c.setAnnotationSuccess(annotationSuccess)
+if (debug) c.setDebug();
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null) MONITOR.worked(20, "EDITION")
+println "-- EDITION - Building edition"
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+File outdir = new File(binDir,"/HTML/$corpusname/default/");
+outdir.mkdirs();
+List<File> filelist = txmDir.listFiles();
+Collections.sort(filelist);
+def second = 0
+
+println "Paginating texts: "
+for (File srcfile : filelist) {
+	print "."
+	String txtname = srcfile.getName();
+	int i = txtname.lastIndexOf(".");
+	if(i > 0) txtname = txtname.substring(0, i);
+	
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+	Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);
+		
+	def ed = new pager_old(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, 500, basename, "pb");
+	Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+	
+	for (i = 0 ; i < ed.getPageFiles().size();) {
+		File f = ed.getPageFiles().get(i);
+		String wordid = ed.getIdx().get(i);
+		params.addPage(edition, ""+(++i), wordid);
+	}
+}
+
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Factiva XML and mail import modules</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/factiva/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SetNewXmlDocumentRoot.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SetNewXmlDocumentRoot.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/SetNewXmlDocumentRoot.groovy	(revision 321)
@@ -0,0 +1,43 @@
+package org.txm.importer
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.xpath.*
+
+import org.txm.importer.graal.PersonalNamespaceContext
+import org.txm.utils.xml.DomUtils
+import org.w3c.dom.Document
+import org.w3c.dom.Element
+import org.w3c.dom.Node
+
+class SetNewXmlDocumentRoot {
+	public static boolean process(File xmlfile, String xpath) {
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		Document doc = builder.parse(xmlfile);
+		Element root = doc.getDocumentElement();
+		
+		XPathFactory xpathfactory = XPathFactory.newInstance();
+		def _xpath = xpathfactory.newXPath();
+		_xpath.setNamespaceContext(new PersonalNamespaceContext());
+		def expr = _xpath.compile(xpath);
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+
+		for(Node node : nodes) {
+
+			Document document = builder.newDocument();
+			document.setXmlVersion("1.0");
+			document.setXmlStandalone(true);
+			Node adopted = document.adoptNode(node)
+			document.appendChild(adopted)
+			return DomUtils.save(document, xmlfile);
+		}
+		return false;
+	}
+	
+	//		File xmlfile = new File("/home/mdecorde/xml/txmrefman/TEI/refman.xml")
+	//		File outfile = new File("/home/mdecorde/xml/txmrefman/TEI/refman-o.xml")
+	//		String xpath = "//body"
+	
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/INARecords.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/INARecords.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/INARecords.groovy	(revision 321)
@@ -0,0 +1,39 @@
+package org.txm.importer
+
+import org.txm.utils.CsvReader;
+import java.nio.charset.Charset;
+
+File csvFile = new File("/home/mdecorde/xml/notices/MARTICE_20130304_TF1_IMAGO_1994-12-31.csv")
+CsvReader reader = new CsvReader(csvFile.getAbsolutePath(), ";".charAt(0), Charset.forName("UTF-8"))
+
+reader.readHeaders()
+def headers = reader.getHeaders()
+int nheader = headers.size()
+int nlines = 0;
+def counts = new int[nheader]
+def values = [:]
+for (String key : headers) values[key] = new HashSet()
+
+while (reader.readRecord()) {
+	nlines++;
+	for (int i = 0 ; i < nheader ; i++) {
+		String key = headers[i]
+		String str = " "+reader.get(key);
+		//print str
+		if (str.trim().length() > 0) {
+			counts[i] = counts[i] + 1
+			values[key] << str.trim()
+		}
+	}
+	//println ""
+}
+
+println "N lines: "+nlines
+println "Empty cols: N <= 20"
+for (int i = 0 ; i < nheader ; i++) {
+	
+	if (counts[i] < 30) {
+		println ( headers[i]+ " : "+counts[i])
+		println values[headers[i]]
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/pager.groovy	(revision 321)
@@ -0,0 +1,374 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.bvh;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Build BVH texts simple edition from the TEI-TXM files. <br/>
+ * The result is similar to the edition of the BVH site. <br/>
+ * Segment edition by page at &lt;pb> tags. <br/>
+ * 
+ * @author mdecorde
+ * 
+ */
+class pager {
+	private List<String> NoSpaceBefore;
+	
+	/** The No space after. */
+	private List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	private int wordcount = 0;
+	
+	/** The pagecount. */
+	private int pagecount = 0;
+	
+	/** The wordmax. */
+	private int wordmax = 0;
+
+	/** The wordid. */
+	private String wordid;
+	
+	/** The first word. */
+	private boolean firstWord = true;
+	
+	/** The wordvalue. */
+	private String wordvalue;
+	
+	/** The interpvalue. */
+	private String interpvalue;
+	
+	/** The lastword. */
+	private String lastword = " ";
+	
+	/** The wordtype. */
+	private String wordtype;
+	
+	/** The flagform. */
+	private boolean flagform = false;
+	
+	/** The flaginterp. */
+	private boolean flaginterp = false;
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The writer. */
+	private OutputStreamWriter writer;
+	
+	/** The multiwriter. */
+	private OutputStreamWriter multiwriter = null;
+	
+	/** The infile. */
+	private File infile;
+	
+	/** The outfile. */
+	private File outfile;
+	
+	/** The pages. */
+	private ArrayList<File> pages = new ArrayList<File>();
+	
+	/** The idxstart. */
+	private ArrayList<String> idxstart = new ArrayList<String>();
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the file to convert
+	 * @param outfile the result file
+	 * @param NoSpaceBefore the punctuation marks that don't have a space before
+	 * @param NoSpaceAfter the punctuation marks that don't have a space after
+	 * @param max the max number of word per page
+	 */
+	pager(File infile, File outfile, List<String> NoSpaceBefore,
+			List<String> NoSpaceAfter, int max) {
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		createOutput(outfile);
+		process();
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		try {
+			if(multiwriter != null)
+			{
+				multiwriter.write("</body>");
+				multiwriter.write("</html>");
+				multiwriter.close();
+			}
+			pagecount++;
+			File f = new File(outfile.getParent()+"/multi/",outfile.getName().substring(0,outfile.getName().length()-5)+"_"+pagecount+".html");
+			pages.add(f);
+			idxstart.add(wordid)
+			multiwriter = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			multiwriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			multiwriter.write("<html>");
+			multiwriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			multiwriter.write("<head>");
+			multiwriter.write("<title>BVH Edition - Page "+pagecount+"</title>");
+			multiwriter.write("</head>");
+			multiwriter.write("<body>");
+			
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			this.outfile = outfile;
+
+			writer = new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8");
+			createNextOutput();
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the geenrated edition page path
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the id of the edition
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	/**
+	 * start processing.
+	 */
+	void process()
+	{
+		String localname = "";
+		
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		writer.write("<html>");
+		writer.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+		writer.write("<head>");
+		writer.write("<title>BVH edition</title>");
+		writer.write("</head>");
+		writer.write("<body>");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+			
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "head":
+							writer.write("<h2>\n")
+							multiwriter.write("<h2>\n")
+							break;
+						
+						case "lg":
+							writer.write("<p>\n")
+							multiwriter.write("<p>\n")
+							break;
+							
+						case "pb":
+							createNextOutput();
+						break;
+						
+						case "l":
+							writer.write("<br/>\n")
+							multiwriter.write("<br/>\n")
+							break;
+						
+						case "p":
+						case "q":
+							writer.write("<p>\n")
+							multiwriter.write("<p>\n")
+							break;
+						
+						case "w":
+							wordid=parser.getAttributeValue(null,"id");
+							if(firstWord)
+							{
+								firstWord=false;
+								this.idxstart.set(0,wordid);
+							}
+							wordcount++;
+							if(wordcount >= wordmax)
+							{
+								
+								wordcount=0;
+							}
+							break;
+						
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+						
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "head":
+							writer.write("</h2>\n")
+							multiwriter.write("</h2>\n")
+							break;
+						
+						case "lg":
+							writer.write("</p>\n")
+							multiwriter.write("</p>\n")
+							break;
+						
+						case "lb":
+							writer.write("<br/>\n")
+							multiwriter.write("<br/>\n")
+							break;
+												
+						case "p":
+						case "q":
+							writer.write("</p>\n")
+							multiwriter.write("</p>\n")
+							break;
+							
+						case "form":
+							flagform = false
+						
+							break;
+						
+						case "ana":
+							flaginterp = false
+							break;
+						
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if(l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+							
+							if(interpvalue != null)
+								interpvalue = interpvalue.replace("\"","&quot;");
+							
+							if(NoSpaceBefore.contains(wordvalue) || 
+									NoSpaceAfter.contains(lastword) || 
+									wordvalue.startsWith("-") || 
+									NoSpaceAfter.contains(endOfLastWord))
+							{
+								writer.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							else
+							{
+								writer.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							}
+							writer.write(wordvalue.replace("<", "&lt;")+"</span>");
+							multiwriter.write(wordvalue.replace("<", "&lt;")+"</span>");
+							lastword=wordvalue;
+							break;
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if(flagform)
+						if(parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if(flaginterp)
+						if(parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					break;
+			}
+		}	
+		writer.write("</body>");
+		writer.write("</html>");
+		writer.close();
+		multiwriter.write("</body>");
+		multiwriter.write("</html>");
+		multiwriter.close();
+		inputData.close();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/importer.groovy	(revision 321)
@@ -0,0 +1,173 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.bvh
+
+import org.txm.importer.*;
+import org.txm.importer.filters.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import javax.xml.stream.*;
+import java.io.File;
+import java.net.URL;
+
+import org.txm.*;
+
+import filters.CutHeader.*;
+import filters.Tokeniser.*;
+import filters.FusionHeader.*;
+import filters.TagSentences.TagSentences;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Convert TEI-BVH to TEI-TXM <br/>
+ * 1- tokenize<br/>
+ * 2- launch XML2TEITXM with no annotation import <br/>.
+ *
+ * @author mdecorde
+ */
+class importer {
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile contains the source files
+	 */
+	public static void run(File rootDirFile, String basename)
+	{
+		
+		File fullfile;
+		String rootDir =rootDirFile.getAbsolutePath();
+		
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		
+		File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+		binDir.deleteDir();
+		binDir.mkdir();
+		new File(binDir,"ptokenized").deleteDir();
+		new File(binDir,"ptokenized").mkdir();
+		new File(binDir,"tokenized").deleteDir();
+		new File(binDir,"tokenized").mkdir();
+		new File(binDir,"txm").deleteDir();
+		new File(binDir,"txm").mkdir();
+		
+		//PREPARE EACH SPLITED FILE TO BE TOKENIZED
+		List<File> srcfiles = rootDirFile.listFiles();
+		println("Preparing Tokenizing "+srcfiles.size()+" files")
+		for(File f : srcfiles)
+		{
+			print "."
+			File srcfile = f;
+			File resultfile = new File(binDir,"ptokenized/"+f.getName());
+			//println("prepare tokenizer file : "+srcfile+" to : "+resultfile );
+			def builder = new OneTagPerLine(srcfile.toURL(), milestones);
+			builder.process(resultfile);
+		}
+		println ""
+		rootDir = binDir.getAbsolutePath();
+		
+		//TOKENIZE FILES
+		List<File> ptokenfiles = new File(rootDir,"ptokenized").listFiles()	
+		println("Tokenizing "+ptokenfiles.size()+" files")
+		for(File f : ptokenfiles)
+		{
+			print "."
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F6 = new Tokeniser(f);
+			Filter F7 = new TagSentences();
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			S.add(F6);
+			S.add(F7);
+			S.add(F11);
+			File infile = new File(rootDir+"/ptokenized",f.getName());
+			File xmlfile = new File(rootDir+"/tokenized",f.getName());
+			File headerfile = new File(rootDir+"/ptokenized/",f.getName()+"header.xml");
+			
+			S.SetInFileAndOutFile(infile.getPath(), xmlfile.getPath());
+			S.setEncodages("UTF-8","UTF-8");
+			Object[] arguments1 = [headerfile.getAbsolutePath()];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [headerfile.getAbsolutePath(),F1];
+			F11.SetUsedParam(arguments2);
+			S.proceed();
+			
+			S.clean();
+			//headerfile.delete();//remove the prepared file to clean
+		}
+		println ""
+		
+		//TRANSFORM INTO XML-TEI-TXM
+		List<File> tokenfiles = new File(rootDir,"tokenized").listFiles()	
+		println("Building "+tokenfiles.size()+" xml-tei-txm files")
+		for(File f : tokenfiles)
+		{
+			print "."
+			File file = f; 
+			String txmfile = f.getName();
+			
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			//il faut lister les id de tous les respStmt
+			def respId = [];
+			//fait la correspondance entre le respId et le rapport d'execution de l'outil
+			def applications = new HashMap<String,HashMap<String,String>>();	
+			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+					//pour construire les ref vers les taxonomies
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			//associe un id d'item avec sa description et son URI
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			//informations de respStmt
+			//resps (respId <voir ci-dessus>, [description, person, date])
+			def resps = new HashMap<String,String[]>();
+			//lance le traitement
+			String wordprefix = "w_";
+			def builder = new Xml2Ana(file);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			builder.process(new File(new File(rootDir,"txm"),txmfile));
+		}
+		println ""
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		new importer().run(new File("~/xml/bvh"));
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/bvhLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/bvhLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/bvhLoader.groovy	(revision 321)
@@ -0,0 +1,145 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-09-16 14:31:48 +0200 (lun., 16 sept. 2013) $
+// $LastChangedRevision: 2535 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.bvh;
+
+import java.io.File;
+import org.txm.importer.bvh.importer;
+import org.txm.importer.bvh.compiler;
+import org.txm.importer.bvh.annotate;
+import org.txm.importer.bvh.pager;
+import org.txm.objects.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Run.
+ *
+ * @return the java.lang. object
+ * @author mdecorde
+ */
+
+String userDir = System.getProperty("user.home");
+String lang;
+String encoding;
+String rootDir;
+String model;
+String basename;
+try{rootDir = rootDirBinding;lang=langBinding;encoding=encodingBinding;model=modelBinding;basename=basenameBinding}
+catch(Exception)
+{	println "DEV MODE";//exception means we debug
+	if(!org.txm.Toolbox.isInitialized())
+	{
+		rootDir = userDir+"/xml/bvh/";
+		lang="fr";
+		encoding= "UTF-8";// not used by bvh
+		model="rgaqcj"; //not used by bvh
+		basename="bvh"
+		Toolbox.workspace = new Workspace(new File(userDir,"TXM/workspaces/default.xml"));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File(userDir,"TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+println "IMPORTER"
+new importer().run(new File(rootDir), basename);
+
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+rootDir = binDir.getAbsolutePath();
+
+println "ANNOTATE - fr "
+new Annotate().run(new File(rootDir),"fr.par");//the files in ./txm
+
+println "ANNOTATE - afr"
+def secondannotator = new Annotate()
+secondannotator.run(new File(rootDir),"afr.par");//the files in ./txm
+
+println "--COMPILING"
+File dir = new File(rootDir);
+def c = new compiler();
+c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setLang(lang);
+if (!c.run(dir)) {
+	println "Compiler failed"
+	return;
+}
+
+//move registry file to cwb registry dir
+File registryfile = new File(rootDir+"/registry",basename);
+if(registryfile.exists())
+	org.txm.utils.io.FileCopy.copy(registryfile,new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"registry/"+basename))
+
+Workspace w = org.txm.Toolbox.workspace;
+Project p = w.getProject("default")
+p.removeBase(basename)
+Base b = p.addBase(basename);
+b.addDirectory(new File(rootDir,"txm"));
+b.setAttribute("lang", lang)
+b.propagateAttribute("lang")
+
+println "-- EDITION"
+new File(rootDir+"/HTML/").deleteDir()
+new File(rootDir+"/HTML/").mkdir();
+new File(rootDir+"/HTML/default/").mkdir();
+files = new File(rootDir,"txm").listFiles();
+
+for(Text text : b.getTexts())
+{
+	File srcfile = text.getSource();
+	File resultfile = new File(rootDir+"/HTML",srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+	println("Building edition  : "+srcfile+" to : "+resultfile );
+	
+	def ed = new pager(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter,600);
+	Edition editionweb = text.addEdition("default","html",resultfile);
+
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+	
+//	Edition editionbp = text.addEdition("onepage","html",resultfile);
+//	editionbp.addPage(resultfile,ed.getIdx().get(0));
+}
+
+w.save();
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/annotate.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/annotate.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/annotate.groovy	(revision 321)
@@ -0,0 +1,181 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-11-08 13:38:06 +0100 (ven., 08 nov. 2013) $
+// $LastChangedRevision: 2569 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.bvh
+
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+import org.txm.Toolbox;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Add treetagger annotations with a choosen model file<br/>
+ * If a word is unknowned the lemma will be set to the word's form <br/>
+ * Use the &lt;s> tags to build sentences boundaries <br/>.
+ *
+ * @author mdecorde
+ */
+class annotate {
+	/*
+	 * @param rootDirFile contains the TEI-TXM files
+	 * @param model the name of the model file to use
+	 * @param pos the name of the pos attribute added
+	 * @param lemme the name of the lemme attribute added
+	 */
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param model the model
+	 * @param pos the pos
+	 * @param lemme the lemme
+	 */
+	public static void run(File rootDirFile,String model,String pos, String lemme)
+	{
+		String rootDir = rootDirFile.getAbsolutePath()+"/";
+		
+		//cleaning
+		new File(rootDir,"annotations").deleteDir();
+		new File(rootDir,"annotations").mkdir();
+		new File(rootDir,"treetagger").deleteDir();
+		new File(rootDir,"treetagger").mkdir();
+		new File(rootDir,"ptreetagger").deleteDir();
+		new File(rootDir,"ptreetagger").mkdir();
+		
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		
+		List<File> files = new File(rootDir,"txm").listFiles()	
+		//BUILD TT FILE READY TO BE TAGGED
+		for(File f : files)
+		{
+			File srcfile = f;
+			File resultfile = new File(rootDir+"ptreetagger/",f.getName()+".tt");
+			new BuildTTSrc(srcfile.toURL()).process(resultfile)
+		}
+		
+		//APPLY TREETAGGER
+		files = new File(rootDir,"ptreetagger").listFiles()	
+		for(File f : files)
+		{
+			File modelfile = new File(Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH),"/afr.par");
+			File infile = f
+			File outfile = new File(rootDir+"/treetagger",f.getName());
+			println("3- APPLY TT on : "+infile+" with : "+modelfile +" >>  "+outfile);
+			
+			TreeTagger tt = new TreeTagger(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/");
+			tt.settoken();
+			tt.setlemma();
+			tt.setquiet();
+			tt.setsgml();
+			tt.setnounknown();
+			tt.seteostag("<s>");
+			tt.treetagger( modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())
+			//infile.delete();
+		}
+		
+		//BUILD STAND-OFF FILES
+		//contains txm:application/txm:commandLine
+		File reportFile = new File(rootDir,"NLPToolsParameters.xml");
+		
+		String respPerson = System.getProperty("user.name");
+		String respId = "txm";
+		String respDesc = "NLP annotation tool";
+		String respDate = "";
+		String respWhen = ""
+		
+		String appIdent = "TreeTagger";
+		String appVersion = "3.2";
+		
+		String distributor = "";
+		String publiStmt = """""";
+		String sourceStmt = """""";
+		
+		def types = [pos,lemme];
+		def typesTITLE = ["",""];
+		def typesDesc = ["",""];
+		def typesTAGSET = ["",""];
+		def typesWEB = ["",""];
+		String idform ="w_";
+		
+		files = new File(rootDir,"treetagger").listFiles()	
+		for(File f : files)
+		{
+			String target = f.getAbsolutePath();
+			File ttfile = f
+			File posfile = new File(rootDir+"annotations/",f.getName()+"STOFF.xml");
+			
+			def encoding ="UTF-8";
+			def transfo = new CSV2W_ANA();
+			println("build w-interp "+ttfile.getName()+ ">>"+posfile.getName())
+			transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+			transfo.setResp(respId, respDesc,respDate, respPerson, respWhen);
+			transfo.setApp(appIdent, appVersion);
+			transfo.setTarget(target, reportFile);
+			transfo.setInfos(distributor,  publiStmt, sourceStmt);
+			transfo.process( ttfile, posfile, encoding );
+		}
+		
+		files = new File(rootDir,"annotations").listFiles();
+		List<File> txmfiles = new File(rootDir,"txm").listFiles();
+		files.sort();
+		txmfiles.sort();
+		for(int i = 0 ; i< files.size();i++)
+		{
+			File srcfile = txmfiles.get(i);
+			File pos1file = files.get(i);
+			File temp = new File(rootDir,"temp"); 
+			
+			println("5- inject annotation in file : "+srcfile+" with : "+pos1file );
+			
+			def builder = new org.txm.scripts.teitxm.AnnotationInjection(srcfile.toURL(), pos1file.toURL(), milestones);
+			builder.transfomFile(temp.getParent(),temp.getName());
+			
+			if (!(srcfile.delete() && temp.renameTo(srcfile))) println "Warning can't rename file "+temp+" to "+srcfile
+		}
+		
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File rootDir = new File("~/xml/bvh/");
+		new annotate().run(rootDir,);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/compiler.groovy	(revision 321)
@@ -0,0 +1,642 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.bvh
+
+
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Produce WTC files from TEI-TXM files. <br/>
+ * Add the following word properties : "frpos","frlemme","afrpos","afrlemme","id","page","line","orig","sic","abbr". <br/>
+ * 
+ * @author mdecorde
+ */
+class compiler 
+{
+	
+	/** The debug. */
+	private boolean debug= false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash =new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * Instantiates a new compiler.
+	 */
+	public compiler(){}
+	
+	/**
+	 * initialize the compiler.
+	 *
+	 * @param url the fiel to read
+	 * @param text the Texte's name
+	 * @param base the Base's name
+	 * @param project the Project's name
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the path to cwb utils binaries
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			File f = new File(dirPathName, fileName)
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+	
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean transfomFileWtc(String dirPathName, String fileName)
+	{
+		createOutput(dirPathName, fileName);
+		
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+		
+		String vExpan = "";
+		String vCorr = "";
+		String vReg = "";
+		String vOrig = "";
+		String vSic = "";
+		String vAbbr = "";
+		
+		String divtype = "";
+		
+		int p_id = 0;
+		int q_id = 0;
+		int lg_id = 0;
+		int l_id = 0;
+		int pb_id = 0;
+		int lb_id = 0;
+		int said_id = 0;
+		int foreign_id = 0;
+		int sp_id = 0;
+		int speaker_id = 0;
+		int stage_id = 0;
+		
+		boolean captureword = false;
+		boolean fwcapture = false;
+		boolean choicecapture = false;
+		boolean divcapture = false;
+		
+		boolean flagchoice = false;
+		boolean flagcorr = false;
+		boolean flagsic = false;
+		boolean flagreg = false;
+		boolean flagexpan = false;
+		boolean flagorig = false;
+		boolean flagabbr = false;
+		boolean flagfw = false;
+		
+		boolean flaglg = false;
+		boolean flaghead = false;
+		boolean flagAuthor = false;
+		boolean flagDate = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+		boolean
+		
+		this.GoToText()
+		output.write("<txmcorpus lang=\""+lang+"\">\n"); 
+		try 
+		{
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+						switch (parser.getLocalName()) 
+						{				
+							case "text":
+							captureword=true;
+							output.write("<text id=\""+text+"\" type=\""+parser.getAttributeValue(null,"type")+"\" base=\""+base+"\" project=\""+project+"\">\n");
+							break;
+							
+							case "div":
+								divtype = parser.getAttributeValue(null,"type")
+							switch(divtype)
+							{
+								case "toa":
+								case "toc":
+								break;
+								case null:
+								break;
+								default:
+								output.write("<div id=\""+parser.getAttributeValue(null,"n")+"\" type=\""+parser.getAttributeValue(null,"type")+"\">\n");		
+							}
+							
+							break;
+							
+							case "p":
+							output.write("<p id=\""+(p_id++)+"\">\n");
+							break;
+							
+							case "lg":
+							output.write("<lg id=\""+(lg_id++)+"\">\n");
+							break;
+							
+							case "l":
+							output.write("<l id=\""+(l_id++)+"\">\n");
+							break;
+							
+							case "q":
+							output.write("<q id=\""+(q_id++)+"\">\n");
+							break;
+							
+							case "said":
+							output.write("<said id=\""+(said_id++)+"\">\n");
+							break;
+							
+							case "foreign":
+							output.write("<foreign id=\""+(foreign_id++)+"\">\n");
+							break;
+							
+							case "stage":
+							output.write("<stage id=\""+(stage_id++)+"\">\n");
+							break;
+							
+							case "sp":
+							output.write("<sp id=\""+(sp_id++)+"\">\n");
+							break;
+							
+							case "speaker":
+							output.write("<speaker id=\""+(speaker_id++)+"\">\n");
+							break;
+							
+							case "fw":
+							flagfw = true;
+							break;
+							case "choice":
+							flagchoice = true;
+							break;
+							case "corr":
+							flagcorr = true;
+							vCorr= "";
+							break;
+							case "reg":
+							flagreg = true;
+							vReg= "";
+							break;
+							case "expan":
+							flagexpan = true;
+							vExpan= "";
+							break;
+							case "orig":
+							flagreg = true;
+							vOrig= "";
+							break;
+							case "sic":
+							flagsic = true;
+							vSic= "";
+							break;
+							case "abbr":
+							flagreg = true;
+							vAbbr= "";
+							break;
+							
+							case "w":
+							for(int i = 0 ; i < parser.getAttributeCount(); i++)
+								if(parser.getAttributeLocalName(i).equals("id"))
+								{	
+									wordid = parser.getAttributeValue(i);
+								}
+							break;
+							case "form":
+							flagForm = true;
+							vForm = "";
+							vAna ="";
+							break;
+							
+							case "ana":
+							flagAna = true;
+							break;
+						}
+						break;
+					
+					case XMLStreamConstants.END_ELEMENT:
+						switch (parser.getLocalName()) 
+						{	
+							case "body":
+							captureword=false;
+							break;
+							
+							case "text":
+							output.write("</text>\n");
+							break;
+							
+							case "div":
+							switch(divtype)
+							{
+								case "toa":
+								case "toc":
+								break;
+								default:
+									output.write("</div>\n");		
+							}
+							
+							break;
+							
+							case "p":
+							output.write("</p>\n");
+							break;
+							
+							case "q":
+							output.write("</q>\n");
+							break;
+							
+							case "said":
+							output.write("</said>\n");
+							break;
+							
+							case "foreign":
+							output.write("</foreign>\n");
+							break;
+							
+							case "lg":
+							output.write("</lg>\n");
+							break;
+							
+							case "pb":
+							pb_id++;
+							break;
+							
+							case "lb":
+								lb_id++;
+								break;
+							
+							case "l":
+							output.write("</l>\n");
+							break;
+							
+							case "stage":
+							output.write("</stage>\n");
+							break;
+							
+							case "sp":
+							output.write("</sp>\n");
+							break;
+							
+							case "speaker":
+							output.write("</speaker>\n");
+							break;
+							
+							
+							case "fw":
+							flagfw = false;
+							break;
+							case "choice":
+							flagchoice = false;
+							break;
+							case "corr":
+							flagcorr = false;
+							vCorr= "";
+							break;
+							case "reg":
+							flagreg = false;
+							vReg= "";
+							break;
+							case "expan":
+							flagexpan = false;
+							vExpan= "";
+							break;
+							case "orig":
+							flagreg = false;
+							vOrig= "";
+							break;
+							case "sic":
+							flagsic = false;
+							vSic= "";
+							break;
+							case "abbr":
+							flagreg = false;
+							vAbbr= "";
+							break;
+							
+							case "w":
+							if(vAna != null)
+								if(captureword)
+								{
+									if(flagchoice)
+									{
+										if(flagcorr)
+										{
+											output.write( vCorr +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+										}
+										else if(flagreg)
+										{
+											output.write( vReg +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+										}
+										else if(flagsic)
+										{
+											
+										}
+										else if(flagexpan)
+										{
+											output.write( vExpan +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+										}	
+									}
+									else if(flagfw)
+									{
+										
+									}
+									else
+									{
+										output.write( vForm +"\t"+vAna+wordid+"\t"+pb_id+"\t"+lb_id+"\t"+vOrig+"\t"+vSic+"\t"+vAbbr+"\n");
+									}
+									
+								}
+							vAna = "";
+							vForm = "";
+							break;
+							
+							case "form":
+							flagForm = false;
+							break;
+							
+							case "ana":
+							vAna += "\t";
+							flagAna = false;
+							break;
+						}
+						break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						if(flagForm)
+						{	
+							vForm += parser.getText().trim();
+						}
+						if(flagAna)
+						{
+							vAna += parser.getText().trim();
+						}
+						if(flagsic)
+						{
+							vSic += parser.getText().trim();
+						}
+						if(flagorig)
+						{
+							vOrig += parser.getText().trim();
+						}
+						if(flagabbr)
+						{
+							vAbbr += parser.getText().trim();
+						}
+						break;
+				}
+			}
+			output.write("</txmcorpus>"); 
+			output.close();
+			parser.close();
+			inputData.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+		}
+		
+		return true;
+	}
+	
+	
+	/**
+	 * start processing.
+	 *
+	 * @param rootDirFile the directory containing the TEI-TXM files
+	 * @return true, if successful
+	 */
+	public boolean run(File rootDirFile) 
+	{
+		String rootDir =rootDirFile.getAbsolutePath();
+		
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!new File(rootDir).exists())
+		{
+			println ("binary directory does not exists: "+rootDir)
+			return false;
+		}
+		new File(rootDir+"/wtc/","bvh.wtc").delete();//cleaning&preparing
+		new File(rootDir,"/wtc/").deleteDir();
+		new File(rootDir,"/wtc/").mkdir();
+		new File(rootDir,"/data/").deleteDir();
+		new File(rootDir,"/data/").mkdir();
+		new File(rootDir,"registry/").mkdir();
+		
+		String textid="";
+		int counttext =0;
+		List<File> files = new File(rootDirFile,"txm").listFiles();
+		Collections.sort(files);
+		//1- Transform into WTC file
+		for(File f : files)
+		{
+			counttext++;
+			if(!f.exists())
+			{
+				println("file "+f+ " does not exists")	
+			}
+			else
+			{	
+				println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				def builder = new compiler(f.toURL(),txtname, "bvh", "default");
+				builder.setLang lang
+				builder.transfomFileWtc(rootDir+"/wtc","bvh.wtc");
+			}
+		}
+		
+		//2- Import into CWB
+		def outDir =rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		String[] pAttributes = ["frpos","frlemme","afrpos","afrlemme","id","page","line","orig","sic","abbr"];
+		String[] sAttributes = ["txmcorpus:0+lang", "text:2+id+type+base+project","div:2+id+type","p:0+id","q:0+id","said:0+id","foreign:0+id","lg:0+id","l:0+id","sp:0+id","speaker:0+id","stage:0+id"];
+		try {
+			String regPath = outDirTxm + "/registry/"+"bvh"
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(), outDirTxm + "/data", outDir + "/wtc/"+"bvh.wtc", regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), "BVH", outDirTxm + "/registry");
+			
+		} catch (Exception ex) {System.out.println(ex); return false}
+		
+		System.out.println("Done.") 
+		
+		return true;
+	}
+	
+	/**
+	 * show cwb-encode and cwb-makeall messages.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+	
+	/**
+	 * test purpose.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/bvh");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/package.html	(revision 321)
@@ -0,0 +1,12 @@
+<html>
+<body>
+<p>The BVH import module<br>
+The bvhLoader.groovy script is the main entry point.<br>
+Process : <br>
+1- Reads a directory containing XML files who follow the BVH encoding guidelines, see <a href="http://www.bvh.univ-tours.fr/">BVH website</a>.<br/>
+2- Annotates with <b>rgaqcj.par</b> and <b>afr.par</b> models.<br> 
+3- Builds a TXM corpus with the name of the directory. <br/>
+4- Builds an edition similar to the BVH's one.<br/><big>Hello</big><big><br></big><big>
+</big></p><big>
+</big></body><big>
+</big></html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/bvh/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/MetadataGetter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/MetadataGetter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/MetadataGetter.groovy	(revision 321)
@@ -0,0 +1,35 @@
+package org.txm.importer
+
+import javax.xml.stream.*;
+
+class MetadataGetter {
+	public static String get(File xmlFile, String struct, String prop) {
+
+		def inputData = xmlFile.toURI().toURL().openStream();
+		def factory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if ("text".equals(parser.getLocalName())) {
+					for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+						if(prop.equals(parser.getAttributeLocalName(i))) {
+							String tmp = parser.getAttributeValue(i)
+							parser.close()
+							return tmp
+						}
+					}
+				}
+			}
+		}
+		parser.close()
+		return null;
+	}
+
+	public static void main(String[] args) {
+		File xmlFile = new File("/home/mdecorde/TXM/corpora/superphenix/txm/SUPERPHENIX/CreysSuper_04_0175.xml")
+		String struct = "text"
+		String prop = "date"
+		println MetadataGetter.get(xmlFile, struct, prop)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/MileStoneProjection.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/MileStoneProjection.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/MileStoneProjection.groovy	(revision 321)
@@ -0,0 +1,111 @@
+package org.txm.importer
+
+import java.io.File;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.File;
+import java.net.URL;
+import javax.xml.stream.*;
+import org.txm.importer.filters.*;
+import org.txm.importer.graal.PersonalNamespaceContext;
+
+/**
+ * add 2 attributes per element to encode the distance to the previous milestone and next milestone
+ * TODO: only the first attribute is implemented
+ * 
+ * @author mdecorde
+ *
+ */
+class MileStoneProjection extends StaxIdentityParser {
+	String wordTag
+	String mileStoneTag
+	String startTag
+	boolean start = false
+
+	int mileStoneDistance = 0
+
+	String mileStoneID = ""
+	def milestonesLength = []
+	int milestonesCounter = 0
+	boolean secondPass = false
+	
+	String msIdAttributeName
+	String msStartAttributeName
+	String msEndAttributeName
+
+	public MileStoneProjection(File inputFile, String startTag, String wordTag, String mileStoneTag) {
+		super(inputFile)
+
+		this.wordTag = wordTag
+		this.mileStoneTag = mileStoneTag
+		mileStoneID = mileStoneTag+"_0"
+		this.startTag = startTag
+		this.start = false;
+		
+		msIdAttributeName = mileStoneTag+"id";
+		msStartAttributeName = mileStoneTag+"start";
+		msEndAttributeName = mileStoneTag+"end";
+
+		fetchMilestoneLengths();
+		start = false // reset
+	}
+
+	public void fetchMilestoneLengths() {
+		def inputData = inputurl.openStream();
+		def factory = XMLInputFactory.newInstance();
+		def parser = factory.createXMLStreamReader(inputData);
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName()
+					if (start && localname == mileStoneTag) {
+						milestonesLength << mileStoneDistance;
+						mileStoneDistance = 0
+						milestonesCounter++;
+					} else if (start && localname == wordTag) {
+						mileStoneDistance++
+					} else if (localname == startTag) {
+						start = true
+					}
+					break;
+			}
+		}
+		milestonesLength[milestonesCounter] = mileStoneDistance;
+		parser.close()
+		milestonesCounter = 0;
+	}
+
+
+	
+	public void processStartElement() {
+		super.processStartElement();
+
+		if (start && localname == mileStoneTag) {
+			mileStoneDistance = 0
+			mileStoneID = parser.getAttributeValue(null, "id")
+			if (mileStoneID == null) mileStoneID = parser.getAttributeValue("xml", "id");
+			if (mileStoneID == null) mileStoneID = "0";
+			
+			milestonesCounter++;
+		} else if (start && localname == wordTag) {
+			// println "end of $milestonesCounter len="+milestonesLength[milestonesCounter]+" dist="+mileStoneDistance
+			writer.writeAttribute(msEndAttributeName, Integer.toString((milestonesLength[milestonesCounter] - mileStoneDistance - 1)))
+			writer.writeAttribute(msStartAttributeName, Integer.toString(mileStoneDistance))
+			writer.writeAttribute(msIdAttributeName, mileStoneID)
+
+			mileStoneDistance++
+		} else if (localname == startTag) {
+			start = true
+		}
+	}
+
+	public static void main(String[] args) {
+		File inputFile = new File("/home/mdecorde/TXM/corpora/BVHEPISTEMON2016/tokenized/1538_MarotAdole.xml")
+		File outputFile = new File("/home/mdecorde/TEMP/tmp.xml")
+
+		MileStoneProjection msp = new MileStoneProjection(inputFile, "body", "w", "lb");
+		println "Sucess: "+msp.process(outputFile)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WExtract.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WExtract.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WExtract.groovy	(revision 321)
@@ -0,0 +1,334 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2015-12-17 12:11:39 +0100 (Thu, 17 Dec 2015) $
+// $LastChangedRevision: 3087 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.stream.*;
+import java.io.File;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Extract w tags from a tei file
+ * not finished.
+ *
+ * @author mdecorde
+ */
+class WExtract 
+{
+	
+	/**
+	 * Process.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param max the max
+	 * @return the java.lang. object
+	 */
+	public process(File infile, File outfile, int max)
+	{
+		println "Process "+infile.getName()+", keep $max words"
+		int count = this.countW(infile);
+		if(count < max)
+		{
+			println "can't extract $max words, the file "+infile.getName()+" contains only $count words"
+			return;
+		}
+		String ms = "#ms_K"
+		int tier = max/3;
+		int from1 = 0 
+		int to1 = tier;
+		int from2 = (count/2) - (tier/2);
+		int to2 =(count/2) + (tier/2);
+		int from3 = count -tier;
+		int to3= count-1;
+		boolean isSic = false;
+		boolean isW = false;
+		boolean isText = false;
+		boolean printW = true;
+		int wcount=0;
+		
+		println " count : "+count
+		println "  get from "+from1+" to "+to1
+		println "  get from "+from2+" to "+to2
+		println "  get from "+from3+" to "+to3
+		
+		
+		String localname;
+		String prefix;
+		InputStream inputData = infile.toURI().toURL().openStream();
+		XMLInputFactory inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		
+		FileOutputStream output = new FileOutputStream(outfile);
+		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8");
+		
+		writer.writeStartDocument("utf-8", "1.0");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(isText)
+			{
+				if((wcount >= from1 && wcount <= to1 )||
+				(wcount >= from2 && wcount <= to2) ||
+				(wcount >= from3 && wcount <= to3))
+					printW = true;
+				else
+					printW = false;
+			}
+			else
+				printW = true;
+			
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					prefix = parser.getPrefix();
+				
+				/*
+				 if(localname == "supplied")
+				 if(parser.getAttributeValue(null,"source") != null)
+				 ms = parser.getAttributeValue(null,"source")
+				 if(localname == "sic")
+				 {
+				 isSic= true;
+				 }
+				 */
+					if(localname == "text")
+						isText = true;
+				
+					if(localname == "w")
+					{
+						isW= true;
+						wcount++;
+						
+						if(isText)
+						{
+							if((wcount >= from1 && wcount <= to1 )||
+							(wcount >= from2 && wcount <= to2) ||
+							(wcount >= from3 && wcount <= to3))
+								printW = true;
+							else
+								printW = false;
+						}
+						else
+							printW = true;
+					}
+				
+				/*if(!isSic)
+				 if(localname != "choice" && localname != "corr" && localname != "sic" && localname != "supplied" && localname != "seg")
+				 {*/
+					if(localname == "w")
+					{
+						if(printW)
+						{
+							if(prefix != null && prefix.length() > 0)
+								writer.writeStartElement(prefix+":"+localname);
+							else
+								writer.writeStartElement(localname);
+							
+							for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+							{
+								if(parser.getAttributePrefix(i)!= "")
+									writer.writeAttribute(parser.getAttributePrefix(i)+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+								else
+									writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+							}
+							//writer.writeAttribute("srcmf:src", ms);
+						}
+					}
+					else
+					{
+						if(prefix != null && prefix.length() > 0)
+							writer.writeStartElement(prefix+":"+localname);
+						else
+							writer.writeStartElement(localname);
+						
+						if(localname == "teiHeader")
+						{
+							writer.writeAttribute("xmlns:me", "http://www.menota.org/ns/1.0");
+							writer.writeAttribute("xmlns:bfm", "http://bfm.ens-lsh.fr/ns/1.0");
+							//writer.writeAttribute("xmlns:srcmf", "https://listes.cru.fr/wiki/srcmf/index");
+						}
+						
+						if(localname == "TEI")
+						{
+							writer.writeAttribute("xmlns","http://www.tei-c.org/ns/1.0");
+						}
+						
+						for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+						{
+							if(parser.getAttributePrefix(i)!= "")
+								writer.writeAttribute(parser.getAttributePrefix(i)+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+							else
+								writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+						}
+					}
+				//}
+					break;
+				
+				case XMLStreamConstants.END_ELEMENT:
+					localname =parser.getLocalName()
+				
+				/*if(localname == "sic")
+				 isSic= false;
+				 if(localname == "w")
+				 isW= false;
+				 if(localname == "supplied" && ms != "#ms_K")
+				 ms = "#ms_K";
+				 if(!isSic)
+				 if(localname != "choice" && localname != "corr" && localname != "sic" && localname != "supplied" && localname != "seg")
+				 {*/
+					if(localname == "w")
+					{
+						if(printW)
+						{
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						}
+					}
+					else
+					{	
+						writer.writeEndElement();
+						writer.writeCharacters("\n");
+					}
+				//	}
+				
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+				//if(!isSic)
+					if(isW)
+					{
+						if(printW)
+						{
+							writer.writeCharacters(parser.getText().trim());
+						}
+					}
+					else
+						writer.writeCharacters(parser.getText().trim());
+					break;
+			}
+		}
+		writer.flush();
+		writer.close();
+		output.close()
+		inputData.close();
+	}
+	
+	/**
+	 * Count w.
+	 *
+	 * @param infile the infile
+	 * @return the int
+	 */
+	public int countW(File infile)
+	{
+		InputStream inputData = infile.toURI().toURL().openStream();
+		XMLInputFactory inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+		
+		int count = 0;
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					if(parser.getLocalName() == "w")
+						count++;
+			}
+		}
+		inputData.close()
+		return count;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String userDir = System.getProperty("user.home");
+		
+		File directory = new File(userDir+"/xml/extract/");
+		File outdir = new File(userDir+"/xml/extract/","results");
+		outdir.mkdir();
+		
+		File maxfile = new File(userDir+"/xml/extract/maxfile");
+		/*
+		 * maxfile format:
+		 * 
+		 * filename1.xml	45000
+		 * filename2.xml	22500
+		 * filename3.xml	45000
+		 */
+		HashMap<File, Integer> maxperfile = new HashMap<File, Integer>();
+		maxfile.eachLine{it->
+			String[] split = it.split("\t");
+			if(split.length == 2)
+			{
+				try
+				{
+				String filename = it.split("\t")[0];
+				int max = Integer.parseInt(it.split("\t")[1])
+				maxperfile.put(filename, max);
+				}catch(Exception e ){}
+			}
+		}
+		println maxperfile;
+		
+		def files = directory.listFiles();
+		for(File infile : files)
+		{
+			
+			if(maxperfile.containsKey(infile.getName()))
+			{
+				File outfile = new File(outdir, infile.getName());
+				int max= maxperfile.get(infile.getName());
+				new WExtract().process(infile, outfile, max)
+			}
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WExtractWithMode.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WExtractWithMode.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WExtractWithMode.groovy	(revision 321)
@@ -0,0 +1,376 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2011-10-19 17:50:26 +0200 (mer., 19 oct. 2011) $
+// $LastChangedRevision: 2038 $
+// $LastChangedBy: alavrentev $ 
+//
+package org.txm.importer
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.stream.*;
+import java.io.File;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Extract w tags from a tei file
+ * not finished.
+ *
+ * @author mdecorde
+ */
+class WExtractWithMode 
+{
+	
+	/**
+	 * Process.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param max the max
+	 * @return the java.lang. object
+	 */
+	public process(File infile, File outfile, String modemax)
+	{
+		println "Process "+infile.getName()+", keep $modemax words"
+		int count = this.countW(infile);
+		
+		int max = 0
+		String mode = ""
+		
+		try {
+		mode = modemax.split("/")[0]
+		max = Integer.parseInt(modemax.split("/")[1])		
+		}catch(Exception e ){}
+		
+		if(count < max)
+		{
+			println "can't extract $max words, the file "+infile.getName()+" contains only $count words"
+			return;
+		}
+		//String ms = "#ms_K"
+		int part = 0; 
+		if (mode == "3")
+		{
+			part = max/3		
+		}
+		else if (mode == "2")
+		{
+			part = max/2
+		}
+		else if (mode == "1a" || mode == "1m" || mode == "1z")
+		{
+			part = max
+		}
+		else
+		{
+			println "mode must be 1a, 1m, 1z, 2 or 3"
+			return
+		}
+		int from1 = 0 
+		int to1 = 0
+		if (mode != "1m" && mode != "1z")
+		{
+			to1 = part
+		}
+		int from2 = 0
+		int to2 = 0
+		if (mode == "3" || mode == "1m")
+		{
+			from2 = (count/2) - (part/2);
+			to2 =(count/2) + (part/2);			
+		}
+		int from3 = 0
+		int to3 = 0
+		if (mode != "1a" && mode != "1m")
+		{
+			from3 = count -part;
+			to3= count-1;			
+		}
+		boolean isSic = false;
+		boolean isW = false;
+		boolean isText = false;
+		boolean printW = true;
+		int wcount=0;
+		
+		println " count : "+count
+		println "  get from "+from1+" to "+to1
+		println "  get from "+from2+" to "+to2
+		println "  get from "+from3+" to "+to3
+		
+		
+		String localname;
+		String prefix;
+		InputStream inputData = infile.toURI().toURL().openStream();
+		XMLInputFactory inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		
+		FileOutputStream output = new FileOutputStream(outfile)
+		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8");
+		
+		writer.writeStartDocument("utf-8", "1.0");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(isText)
+			{
+				if((wcount >= from1 && wcount <= to1 )||
+				(wcount >= from2 && wcount <= to2) ||
+				(wcount >= from3 && wcount <= to3))
+					printW = true;
+				else
+					printW = false;
+			}
+			else
+				printW = true;
+			
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					prefix = parser.getPrefix();
+				
+				/*
+				 if(localname == "supplied")
+				 if(parser.getAttributeValue(null,"source") != null)
+				 ms = parser.getAttributeValue(null,"source")
+				 if(localname == "sic")
+				 {
+				 isSic= true;
+				 }
+				 */
+					if(localname == "text")
+						isText = true;
+				
+					if(localname == "w")
+					{
+						isW= true;
+						wcount++;
+						
+						if(isText)
+						{
+							if((wcount >= from1 && wcount <= to1 )||
+							(wcount >= from2 && wcount <= to2) ||
+							(wcount >= from3 && wcount <= to3))
+								printW = true;
+							else
+								printW = false;
+						}
+						else
+							printW = true;
+					}
+				
+				/*if(!isSic)
+				 if(localname != "choice" && localname != "corr" && localname != "sic" && localname != "supplied" && localname != "seg")
+				 {*/
+					if(localname == "w")
+					{
+						if(printW)
+						{
+							if(prefix != null && prefix.length() > 0)
+								writer.writeStartElement(prefix+":"+localname);
+							else
+								writer.writeStartElement(localname);
+							
+							for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+							{
+								if(parser.getAttributePrefix(i)!= "")
+									writer.writeAttribute(parser.getAttributePrefix(i)+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+								else
+									writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+							}
+							//writer.writeAttribute("srcmf:src", ms);
+						}
+					}
+					else
+					{
+						if(prefix != null && prefix.length() > 0)
+							writer.writeStartElement(prefix+":"+localname);
+						else
+							writer.writeStartElement(localname);
+						
+						if(localname == "teiHeader")
+						{
+							writer.writeAttribute("xmlns:me", "http://www.menota.org/ns/1.0");
+							writer.writeAttribute("xmlns:bfm", "http://bfm.ens-lsh.fr/ns/1.0");
+							//writer.writeAttribute("xmlns:srcmf", "https://listes.cru.fr/wiki/srcmf/index");
+						}
+						
+						if(localname == "TEI")
+						{
+							writer.writeAttribute("xmlns","http://www.tei-c.org/ns/1.0");
+						}
+						
+						for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+						{
+							if(parser.getAttributePrefix(i)!= "")
+								writer.writeAttribute(parser.getAttributePrefix(i)+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+							else
+								writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+						}
+					}
+				//}
+					break;
+				
+				case XMLStreamConstants.END_ELEMENT:
+					localname =parser.getLocalName()
+				
+				/*if(localname == "sic")
+				 isSic= false;
+				 if(localname == "w")
+				 isW= false;
+				 if(localname == "supplied" && ms != "#ms_K")
+				 ms = "#ms_K";
+				 if(!isSic)
+				 if(localname != "choice" && localname != "corr" && localname != "sic" && localname != "supplied" && localname != "seg")
+				 {*/
+					if(localname == "w")
+					{
+						if(printW)
+						{
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						}
+					}
+					else
+					{	
+						writer.writeEndElement();
+						writer.writeCharacters("\n");
+					}
+				//	}
+				
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+				//if(!isSic)
+					if(isW)
+					{
+						if(printW)
+						{
+							writer.writeCharacters(parser.getText().trim());
+						}
+					}
+					else
+						writer.writeCharacters(parser.getText().trim());
+					break;
+			}
+		}
+		writer.flush();
+		writer.close();
+		output.close()
+		inputData.close();
+	}
+	
+	/**
+	 * Count w.
+	 *
+	 * @param infile the infile
+	 * @return the int
+	 */
+	public int countW(File infile)
+	{
+		InputStream inputData = infile.toURI().toURL().openStream();
+		XMLInputFactory inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+		
+		int count = 0;
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					if(parser.getLocalName() == "w")
+						count++;
+			}
+		}
+		inputData.close()
+		return count;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String userDir = System.getProperty("user.home");
+		
+		File directory = new File(userDir+"/xml/extract/");
+		File outdir = new File(userDir+"/xml/extract/","results");
+		outdir.mkdir();
+		
+		File maxfilemode = new File(userDir+"/xml/extract/maxfilemode");
+		/*
+		 * maxfilemode format:
+		 * 
+		 * filename1.xml	3	45000
+		 * filename2.xml	1a	15000
+		 * filename3.xml	1m	15000
+		 * filename4.xml	1z	15000
+		 * filename5.xml	2	22500
+		 */
+		HashMap<File, String> maxperfile = new HashMap<File, String>();
+		maxfilemode.eachLine{it->
+			String[] split = it.split("\t");
+			if(split.length == 3)
+			{
+				try
+				{
+				String filename = it.split("\t")[0];
+				String modemax = it.split("\t")[1]+"/"+it.split("\t")[2]
+				maxperfile.put(filename, modemax);
+				}catch(Exception e ){}
+			}
+		}
+		println maxperfile;
+		
+		def files = directory.listFiles();
+		for(File infile : files)
+		{
+			
+			if(maxperfile.containsKey(infile.getName()))
+			{
+				File outfile = new File(outdir, infile.getName());
+				String modemax = maxperfile.get(infile.getName());
+				new WExtractWithMode().process(infile, outfile, modemax)
+			}
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/StaxStackWriter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/StaxStackWriter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/StaxStackWriter.groovy	(revision 321)
@@ -0,0 +1,259 @@
+package org.txm.importer
+
+import java.nio.charset.Charset;
+
+import javax.xml.namespace.NamespaceContext;
+import javax.xml.stream.*;
+
+import org.eclipse.core.runtime.Platform;
+
+public class StaxStackWriter implements XMLStreamWriter {
+
+	XMLStreamWriter writer;
+	OutputStream output;
+	def events = [];
+	
+	public StaxStackWriter(File file, String encoding) {
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		this.output = new BufferedOutputStream(new FileOutputStream(file));
+		this.writer = factory.createXMLStreamWriter(output, encoding); //create a new file
+	}
+	
+	public StaxStackWriter(OutputStream output, String encoding) {
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		this.output = output
+		this.writer = factory.createXMLStreamWriter(output, encoding);//create a new file
+	}
+	
+	public StaxStackWriter(OutputStream output) {
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		this.output = output
+		this.writer = factory.createXMLStreamWriter(output, "UTF-8");//create a new file
+	}
+
+	Charset UTF8 = Charset.forName("UTF-8");
+	public void write(String txt) {
+		output.write(txt.getBytes(UTF8))
+	}
+	
+	public void writeEndElements() {
+		int size = events.size();
+		for (int i = 0 ; i < size ; i++) {
+			writeEndElement();
+		}
+		events = [];
+	}
+
+	public void writeStartElements(def tagsToWrite) {
+		for (String tag : tagsToWrite)
+			writeStartElement(tag);
+	}
+
+	public def getTagStack() {
+		return events;
+	}
+
+	@Override
+	public void close () throws XMLStreamException {
+		writer.close();
+		output.close();
+	}
+
+	@Override
+	public void flush () throws XMLStreamException {
+		writer.flush();
+
+	}
+
+	@Override
+	public NamespaceContext getNamespaceContext() {
+		return writer.getNamespaceContext();
+	}
+
+	@Override
+	public String getPrefix (String uri) throws XMLStreamException {
+		return writer.getPrefix(uri);
+	}
+
+	@Override
+	public Object getProperty (String name) throws IllegalArgumentException {
+		return writer.getProperty(name);
+	}
+
+	@Override
+	public void setDefaultNamespace (String uri) throws XMLStreamException {
+		writer.setDefaultNamespace(uri);
+	}
+
+	@Override
+	public void setNamespaceContext(NamespaceContext context)
+	throws XMLStreamException {
+		writer.setNamespaceContext(context);
+	}
+
+	@Override
+	public void setPrefix (String prefix, String uri) throws XMLStreamException {
+		writer.setPrefix(prefix, uri);
+	}
+
+	@Override
+	public void writeAttribute (String localName, String value) throws XMLStreamException {
+		writer.writeAttribute(localName, value);
+	}
+
+	@Override
+	public void writeAttribute (String namespaceURI, String localName, String value) throws XMLStreamException {
+		writer.writeAttribute(namespaceURI, localName, value);
+	}
+
+	@Override
+	public void writeAttribute (String prefix, String namespaceURI, String localName, String value) throws XMLStreamException {
+		writer.writeAttribute(prefix, namespaceURI, localName, value);
+	}
+
+	@Override
+	public void writeCData (String data) throws XMLStreamException {
+		writer.writeCData(data);
+	}
+
+	@Override
+	public void writeCharacters (String text) throws XMLStreamException {
+		writer.writeCharacters(text);
+	}
+
+	@Override
+	public void writeCharacters (char[] text, int start, int len) throws XMLStreamException {
+		writer.writeCharacters(text, start, len);
+	}
+
+	@Override
+	public void writeComment (String data) throws XMLStreamException {
+		writer.writeComment(data);
+	}
+
+	@Override
+	public void writeDTD (String dtd) throws XMLStreamException {
+		writer.writeDTD(dtd);
+	}
+
+	@Override
+	public void writeDefaultNamespace(String namespaceURI)
+	throws XMLStreamException {
+		writer.writeDefaultNamespace(namespaceURI);
+	}
+
+	public void writeElement (String localName, String text) throws XMLStreamException {
+		writer.writeStartElement(localName);
+		writer.writeCharacters(text);
+		writer.writeEndElement();
+	}
+	
+	public void writeElement (String localName, def map, String text) throws XMLStreamException {
+		writer.writeStartElement(localName);
+		for (def key : map.keySet()) writeAttribute(key.toString(), map[key].toString());
+		writer.writeCharacters(text);
+		writer.writeEndElement();
+	}
+
+	@Override
+	public void writeEmptyElement (String localName) throws XMLStreamException {
+		writer.writeEmptyElement(localName);
+	}
+
+	public void writeEmptyElement (String localName, def map) throws XMLStreamException {
+		writer.writeEmptyElement(localName);
+		for (def key : map.keySet())
+			writeAttribute(key.toString(), map[key].toString());
+	}
+
+	@Override
+	public void writeEmptyElement(String namespaceURI, String localName)
+	throws XMLStreamException {
+		writer.writeEmptyElement(namespaceURI, localName);
+	}
+
+	@Override
+	public void writeEmptyElement (String prefix, String localName, String namespaceURI) throws XMLStreamException {
+		writer.writeEmptyElement(prefix, localName, namespaceURI);
+	}
+
+	@Override
+	public void writeEndDocument () throws XMLStreamException {
+		writer.writeEndDocument();
+	}
+
+	@Override
+	public void writeEndElement () throws XMLStreamException {
+		writer.writeEndElement();
+		events.pop()
+
+	}
+
+	@Override
+	public void writeEntityRef (String name) throws XMLStreamException {
+		writer.writeEntityRef(name);
+	}
+
+	@Override
+	public void writeNamespace(String prefix, String namespaceURI)
+	throws XMLStreamException {
+		writer.writeNamespace(prefix, namespaceURI);
+	}
+
+	@Override
+	public void writeProcessingInstruction(String target)
+	throws XMLStreamException {
+		writer.writeProcessingInstruction(target);
+	}
+
+	@Override
+	public void writeProcessingInstruction(String target, String data)
+	throws XMLStreamException {
+		writer.writeProcessingInstruction(target, data);
+	}
+
+	@Override
+	public void writeStartDocument () throws XMLStreamException {
+		writer.writeStartDocument();
+
+	}
+
+	@Override
+	public void writeStartDocument (String version) throws XMLStreamException {
+		writer.writeStartDocument(version);
+	}
+
+	@Override
+	public void writeStartDocument(String encoding, String version)
+	throws XMLStreamException {
+		writer.writeStartDocument(encoding, version);
+	}
+
+	@Override
+	public void writeStartElement (String localName) throws XMLStreamException {
+		writer.writeStartElement(localName);
+		events << localName
+
+	}
+	
+	public void writeStartElement (String localName, def map) throws XMLStreamException {
+		writeStartElement(localName);
+		for (def key : map.keySet()) {
+			writeAttribute(key.toString(), map[key].toString());
+		}
+	}
+
+	@Override
+	public void writeStartElement(String namespaceURI, String localName)
+	throws XMLStreamException {
+		writer.writeStartElement(namespaceURI, localName);
+		events << localName
+	}
+
+	@Override
+	public void writeStartElement(String prefix, String localName,
+	String namespaceURI) throws XMLStreamException {
+		writer.writeStartElement(prefix, localName, namespaceURI);
+		events << localName
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RenameTag.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RenameTag.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/RenameTag.groovy	(revision 321)
@@ -0,0 +1,146 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package org.txm.importer
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.xpath.*
+
+import org.txm.importer.graal.PersonalNamespaceContext
+import org.txm.utils.xml.DomUtils
+import org.w3c.dom.Document
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class RenameTag : using Stax or DOM depending on the file size
+ */
+class RenameTag extends StaxIdentityParser{
+
+	public static int DOMMAXSIZE = 1000000;
+	String newname, oldname;
+	File xmlfile;
+
+	public RenameTag(File xmlfile, String oldname, String newname)
+	{
+		super(xmlfile.toURI().toURL());
+		this.oldname = oldname;
+		this.newname = newname;
+		this.xmlfile = xmlfile;
+	}
+	/**
+	 * Rename an element - DOM method
+	 *
+	 * @param xmlfile the xmlfile
+	 * @param oldname the oldname
+	 * @param newname the newname
+	 * @return true, if successful
+	 */
+	public static boolean rename(File xmlfile, String oldname, String newname)
+	{
+		if (!xmlfile.exists()) {
+			println "Error: $xmlfile does not exists"
+		}
+
+		if (!(xmlfile.canRead() && xmlfile.canWrite())) {
+			println "Error: $xmlfile is not readable or writable"
+		}
+
+		if (xmlfile.length() <= DOMMAXSIZE) {
+			String xpath = "//"+oldname;
+			DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+			domFactory.setNamespaceAware(true); // never forget this!
+			DocumentBuilder builder = domFactory.newDocumentBuilder();
+			Document doc = builder.parse(xmlfile);
+
+			XPathFactory xpathfactory = XPathFactory.newInstance();
+			def _xpath = xpathfactory.newXPath();
+			_xpath.setNamespaceContext(new PersonalNamespaceContext());
+			def expr = _xpath.compile(xpath);
+			def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+
+			for(def node : nodes) {
+				doc.renameNode(node, null, newname)
+			}
+			return DomUtils.save(doc, xmlfile)
+		}
+		else {
+			RenameTag builder = new RenameTag(xmlfile, oldname, newname);
+			File temp = File.createTempFile("temp", ".xml", xmlfile.getParentFile());
+			if(builder.process(temp)) {
+				if (!(xmlfile.delete() && temp.renameTo(xmlfile))) println "Warning can't rename file "+temp+" to "+xmlfile
+				return true;
+			}
+			else
+				return false;
+		}
+	}
+
+	/**
+	 * Rewrite processStartElement to update the localname if localname == oldname
+	 */
+	public void processStartElement()
+	{
+		String prefix = parser.getPrefix();
+		if (INCLUDE == localname && XI == prefix) {
+			processingXInclude();
+			return;
+		}
+
+		if (localname == oldname) {
+			localname = newname;
+
+			if(prefix.length() > 0)
+				writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+			else
+				writer.writeStartElement(localname);
+
+			for(int i = 0 ; i < parser.getNamespaceCount() ; i++)
+			{
+				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+			}
+
+			for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+				writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+		} else {
+			super.processStartElement();
+		}
+	}
+
+	public static void main(String[] args)
+	{
+		File xmlfile = new File("/home/mdecorde/xml/xmlçàé/essai.xml");
+		File temp = new File("/home/mdecorde/xml/xmlçàé/renamed.xml");
+		String oldname = "machin"
+		String newname = "titre"
+		long time = System.currentTimeMillis();
+		RenameTag.rename(xmlfile, oldname, newname);
+		//		RenameTag builder = new RenameTag(xmlfile, oldname, newname);
+		//		builder.process(temp);
+		println "elapsed: "+(System.currentTimeMillis()-time);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HTML2XHTML.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HTML2XHTML.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HTML2XHTML.groovy	(revision 321)
@@ -0,0 +1,42 @@
+package org.txm.importer
+
+import org.txm.utils.CharsetDetector;
+import org.txm.importer.ValidateXml;
+
+File infile = new File("/home/mdecorde/Bureau/matrice/témoignages/CONVERSIONS/jod/odt.html")
+File outfile = new File("/home/mdecorde/Bureau/matrice/témoignages/CONVERSIONS/jod/odt.xml")
+
+String encoding = new CharsetDetector(infile).getEncoding();
+println "Encoding: $encoding"
+String text = infile.getText(encoding);
+
+//lower case tags
+text = text.replaceAll(/(<[^!][^>]*>)/, 
+	{ full, word -> 
+		//fix attributes TRUC=sdf234
+		word = word.replaceAll("([A-Z]+=)([^\" >]+)([ >])",'$1"$2"$3' )
+		word.toLowerCase() // bourrin
+	} )
+
+//lower case <.> tags
+text = text.replaceAll(/(<.>)/,
+	{ full, word ->
+		word.toLowerCase()
+	} )
+
+//resolve entities
+text = text.replaceAll(/&nbsp;/," ")
+
+//close tags
+text = text.replaceAll(/<br>/,"<br/>")
+text = text.replaceAll(/<meta([^>]*)>/,'<meta$1/>')
+text = text.replaceAll(/<img([^>]*)>/,'<img$1/>')
+
+//remove doctype declaration
+text = text.replace('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">', '')
+
+//write&Validate
+outfile.withWriter(encoding) { writer -> writer.write(text) }
+if (!ValidateXml.test(outfile)) {
+	println "FILE: $outfile"
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/GetPagesFromURL.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/GetPagesFromURL.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/GetPagesFromURL.groovy	(revision 321)
@@ -0,0 +1,74 @@
+package org.txm.importer
+
+import org.w3c.tidy.Tidy
+
+def workDir = new File("/home/mdecorde/TEMP")
+def outdir = new File("/home/mdecorde/TEMP/out")
+
+def src = "http://txm.sourceforge.net/doc/manual/manual1.xhtml";
+def base = src.substring(0, src.lastIndexOf("/"));
+
+def url_xpath = """//*/a[@href]/@href""";
+def url_regexp = /.+/;
+def duplicates = false; // 'true' not implemented
+def filename_regexp = /zer/;
+def content_xpath = "//*/div[@id=\"content\"]";
+
+def xhtmlFile = new File(workDir, "src.xhtml")
+URL2XHTMLFILE(src, xhtmlFile, workDir);
+
+println "finding urls of $xhtmlFile"
+XPathResult xpathResult = new XPathResult(xhtmlFile);
+def urls = new LinkedHashSet<String>();
+
+for (String url : xpathResult.getXpathResponses(url_xpath))
+	if (urls ==~ url_regexp ) urls << url
+println urls
+
+int i = 1;
+for (String url : urls) {
+	//url = url.replaceAll()
+	def subxHtmlFile = new File(workDir, url)
+	URL2XHTMLFILE(base+"/"+url, subxHtmlFile, workDir);
+	
+	def txtFile = new File(outdir, url+".xhtml")
+	def writer = txtFile.newWriter("UTF-8");
+	for (def node : xpathResult.getNodes(content_xpath)) {
+		writer.println(node)
+	}
+	writer.close()
+	i++
+}
+
+def URL2XHTMLFILE(String url, File outfile, File workDir) {
+	println "getting src: $url"
+	def htmlFile = new File(workDir, "base.html")
+	def filestream = new FileOutputStream(htmlFile)
+	def out = new BufferedOutputStream(filestream)
+	out << new URL(url).openStream()
+	out.close()
+	
+	println "xHTMLize HTML $htmlFile"
+	Tidy tidy = new Tidy(); // obtain a new Tidy instance
+	tidy.setXHTML(true); // set desired config options using tidy setters
+	tidy.setInputEncoding("UTF-8")
+	tidy.setOutputEncoding("UTF-8")
+	tidy.setShowErrors(0)
+	tidy.setTabsize(10)
+	tidy.setShowWarnings(false)
+	File xhtmlFile = new File(workDir, "base.xhtml")
+	xhtmlFile.withWriter("UTF-8") { out2 ->
+		tidy.parse(htmlFile.toURI().toURL().newInputStream([readTimeout: 10000]), out2); // run tidy, providing an input and output stream
+	}
+	
+	println "remove entities and doctype"
+	String text = xhtmlFile.getText("UTF-8")
+	text = text.replaceAll("&nbsp;", " ")
+	text = text.replaceAll("xmlns=\"http://www.w3.org/1999/xhtml\"", " ")
+	text = text.replaceAll("""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">""", "")
+	outfile.withWriter("UTF-8") { writer ->
+		writer.write(text);
+	}
+}
+println "done"
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Doc2Transcription.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Doc2Transcription.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Doc2Transcription.groovy	(revision 321)
@@ -0,0 +1,122 @@
+package org.txm.importer
+
+import javax.xml.stream.*
+import java.net.URL
+
+class Doc2Transcription {
+
+	String encoding
+	File txtfile
+	File outfile
+	Reader reader
+	XMLOutputFactory factory
+	FileOutputStream output
+	XMLStreamWriter writer
+
+	int noline = 1
+	boolean question = false
+	int idquestion = 1
+	int idresponse = 1
+	boolean opened = false
+
+	public Doc2Transcription(File txtfile, File outfile, String encoding)
+	{
+		this.txtfile = txtfile
+		this.outfile = outfile
+		this.encoding = encoding
+
+		reader = new InputStreamReader(new FileInputStream(txtfile) , encoding)
+		factory = XMLOutputFactory.newInstance()
+		output = new FileOutputStream(outfile)
+		writer = factory.createXMLStreamWriter(output, "UTF-8")//create a new file
+	}
+
+	public boolean process()
+	{
+		if(reader == null || writer == null)
+		{
+			println "I/O error"
+			return false
+		}
+		
+		// build text id
+		String filename = txtfile.getName()
+		int idx = filename.indexOf(".")
+		if(idx > 0)
+			filename = filename.substring(0, idx)
+
+		//write Start doc
+		writer.writeStartDocument("UTF-8", "1.0")
+		writer.writeStartElement("text") // we create a tag <text>
+		writer.writeAttribute("id",filename)// and set its id
+
+		//write first block
+		writeStartQuestionResponse()
+		
+		String line = reader.readLine()
+		while(line != null)
+		{
+			if(line.length() == 0) // saut de ligne
+			{
+				question = !question
+				writer.writeEmptyElement("br")
+				writeStartQuestionResponse()
+			}
+			else
+			{
+				writer.writeCharacters(line+" \n")
+				writer.writeEmptyElement("br")
+			}
+
+			noline++
+			line = reader.readLine()
+		}
+
+		//close xml doc and question/response if needed
+		if(opened)
+			writer.writeEndElement()// question/response
+		writer.writeEndElement()// text
+
+		reader.close()
+		writer.close()
+		output.close()
+		return true
+	}
+
+	public void writeStartQuestionResponse()
+	{
+		if(opened)
+		{
+			writer.writeEndElement()
+			opened = false
+		}
+		opened = true
+		if(question)
+		{
+			writer.writeStartElement("question") // we create a tag <text>
+			writer.writeAttribute("id","q"+idquestion++)// and set its id
+		}
+		else
+		{
+			writer.writeStartElement("response") // we create a tag <text>
+			writer.writeAttribute("id","r"+idresponse++)// and set its id
+		}
+	}
+
+	public void setInitialeState(boolean state)
+	{
+		question = state
+	}
+
+	static main(args) {
+		File infile = new File("/home/matt/xml/txttranscription/test.txt")
+		File outfile = new File("/home/matt/xml/txttranscription/test.xml")
+		String encoding = "UTF-8"
+
+		def builder = new Doc2Transcription(infile, outfile, encoding)
+		if(builder.process())
+			println "ok"
+		else
+			println "fail"
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/ValidateXmlTXM.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/ValidateXmlTXM.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/ValidateXmlTXM.groovy	(revision 321)
@@ -0,0 +1,219 @@
+package org.txm.importer.xmltxmpara
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+class ValidateXmlTXM {
+	
+	public static String XMLNS = "http://www.w3.org/XML/1998/namespace";
+	public static String TXMNS = "http://textometrie.org/1.0";
+	public static String TEINS = "http://www.tei-c.org/ns/1.0";
+	
+
+	boolean checkteiCorpus = false;
+	boolean checkTEI = true;
+	boolean checkAlignStruct = false;
+	boolean checkW = true;
+
+	boolean hasTeiCorpus = false;
+	boolean hasTeiCorpusID = false;
+	boolean hasTeiCorpusHeader = false;
+	boolean hasTeiCorpusHeaderAppDesc = false;
+	boolean hasTeiCorpusHeaderVersion = false;
+	int hasTei = 0;
+	int hasTeiID = 0;
+	boolean hasAlignStruct = false
+	String alignStruct = "";
+	boolean hasTXMW = false;
+
+	public boolean validate(File sourceDir)
+	{
+		boolean ret = true;
+		//get infos from import.xml if any
+		File alignxml = sourceDir.listFiles().find{it.getName() == "align.xml"}
+		if(alignxml != null)
+		{
+			infosFromImportXML(alignxml)
+			checkteiCorpus = true;
+			// checkAlignStruct = true;
+			// mainLang = ...
+			// corpusnames = ...
+			// alignStructs = 
+		}
+		
+		sourceDir.eachFileMatch(~/.+.xml/){f->
+
+			if (f.getName() != "import.xml" && f.getName() != "align.xml" && f.isFile())
+				ret = ret & validateXMLTXM(f);
+		}
+
+		return ret;
+	}
+	
+	HashMap<String, ArrayList<String>> links = [:];
+	protected boolean infosFromImportXML(File alignxml)
+	{
+		this.url = xmlfile.toURI().toURL();;
+		inputData = url.openStream();
+
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			if(event == XMLStreamConstants.START_ELEMENT)
+			{
+				if(parser.getLocalName() == "link")
+				{
+					links.put(parser.getAttributeValue(null, "target"), 
+						[parser.getAttributeValue(null, "alignElement"), parser.getAttributeValue(null, "alignLevel")])
+					
+				}
+			}
+		}
+		parser.close();
+		println "get infos from $alignxml"
+		return true;
+	}
+	
+	private void printAttributes(def parser)
+	{
+		for(int i = 0 ; i < parser.getAttributeCount(); i++)
+		println( "$path "+parser.getAttributeLocalName(i)+
+		 " = "+ parser.getAttributeValue(i))
+	}
+	
+	def url;
+	def inputData;
+	def factory;
+	XMLStreamReader parser;
+	String path = ""
+	protected boolean validateXMLTXM(File xmlfile)
+	{
+		println "validate $xmlfile"
+		boolean inTeiCorpus = false;
+		boolean inTeiHeader = false;
+		boolean inMetadata = false;
+		boolean inTEI = false;
+		boolean inW = false;
+		boolean inForm = false;
+
+		try {
+			this.url = xmlfile.toURI().toURL();;
+			inputData = url.openStream();
+
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+			return false;
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+			return false
+		}
+		String localname = ""
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			if(event == XMLStreamConstants.START_ELEMENT)
+			{
+				localname = parser.getLocalName();
+				path += "/"+localname
+				switch (localname)
+				{
+					case "teiCorpus":
+						inTeiCorpus = true;
+						hasTeiCorpus = true;
+						//printAttributes(parser);
+						if(parser.getAttributeValue(XMLNS, "id") != null)
+							hasTeiCorpusID = true;
+						break;
+					case "teiHeader":
+						inTeiHeader = true;
+						if(inTeiCorpus)
+							hasTeiCorpusHeader = true;
+						break;
+					case "metadata":
+						inMetadata = true;
+						//printAttributes(parser);
+						if(parser.getAttributeValue(null, "name") == "version")
+							hasTeiCorpusHeaderVersion = true;
+						break;
+					case "TEI":
+						inTEI = true;
+						hasTei++;
+						if(parser.getAttributeValue(XMLNS, "id") != null)
+							hasTeiID++;
+						break;
+					case "w":
+						inW = true;
+						String id = parser.getAttributeValue(XMLNS, "id");
+						if(!(id != null && id.matches("w_.+_.+")))
+							hasTXMW = false;
+						break;
+					case "form":
+						inForm = true;
+						break;
+					case alignStruct:
+					if(parser.getAttributeValue(null, "align") == null)
+						hasAlignStruct = false;
+						break;
+				}
+			}
+			else if(event == XMLStreamConstants.END_ELEMENT)
+			{
+				localname = parser.getLocalName()
+				if(path.lastIndexOf("/") > 0)
+					path = path.substring(0, path.lastIndexOf("/"))
+				switch (localname)
+				{
+					case "teiCorpus":
+						inTeiCorpus = false;
+						break;
+					case "teiHeader":
+						inTeiHeader = false;
+						break;
+					case "metadata":
+						inMetadata = false;
+						break;
+					case "TEI":
+						inTEI = false;
+						break;
+					case "w":
+						inW = false;
+						break;
+					case "form":
+						inForm = false;
+						break;
+				}
+			}
+		}
+		parser.close();
+		
+		if(checkteiCorpus &
+			!(hasTeiCorpus & hasTeiCorpusID & hasTeiCorpusHeader & hasTeiCorpusHeaderVersion))
+		{	println "wrong <teiCorpus> format: $hasTeiCorpus & $hasTeiCorpusID & $hasTeiCorpusHeader & $hasTeiCorpusHeaderVersion"
+			return false;
+		}
+		
+		if(checkAlignStruct &
+			!hasAlignStruct)
+		{   println "wrong align structure format: $hasAlignStruct"
+			return false;
+		}
+		
+		if(checkTEI & !(hasTei == hasTeiID))
+		{	println "wrong <TEI> format: $hasTei == $hasTeiID"
+			return false;
+		}
+		
+		return true;
+	}
+	
+	public static void main(String[] args)
+	{
+		File srcdir = new File("/home/mdecorde/xml/xmltxmpara");
+		def checker = new ValidateXmlTXM();
+		println checker.validate(srcdir)
+	}
+}
+
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/xmltxmparaLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/xmltxmparaLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/xmltxmparaLoader.groovy	(revision 321)
@@ -0,0 +1,243 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+// $LastChangedDate: 2011-06-03 12:37:57 +0200 (Fri, 03 Jun 2011) $
+// $LastChangedRevision: 1867 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.xmltxmpara;
+
+import org.txm.importer.xmltxmpara.compiler;
+import org.txm.importer.xml.pager_old;
+import org.txm.objects.*;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.importer.*;
+import org.txm.importer.cwb.CwbAlign;
+import org.txm.importer.cwb.PatchCwbRegistry;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+String rootDir;
+String lang;
+String encoding;
+String model;
+String basename;
+String xsl;
+
+try {rootDir = rootDirBinding;}
+catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	if (!org.txm.Toolbox.isInitialized()) {
+		rootDir = userDir+"/xml/qgraal";
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"),"TXM"));
+	}
+}
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+srcDir.getName());
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File srcParamFile = new File(rootDir, "import.xml");
+File paramFile = new File(binDir, "import.xml")
+if (!srcParamFile.exists()) {
+	println "Stop import: can't find params file: "+srcParamFile
+	return;
+}
+FileCopy.copy(srcParamFile, paramFile);
+if (!paramFile.exists()) {
+	println "Stop import: failed to copy param file: "+srcParamFile+" to "+paramFile
+	return;
+}
+BaseParameters params = new BaseParameters(paramFile);
+params.load();
+basename = params.name;
+
+File paraDir = new File(binDir, "paracorpus/");
+paraDir.deleteDir();
+paraDir.mkdir();
+
+// Apply XSL
+if (xsl != null && xsl.trim().length() > 0) {
+	if (!ApplyXsl2.processImportSources(new File(xsl, new File(rootDir), srcDir)))
+	return; // error durring process
+	srcDir = new File(binDir, "src");
+}
+
+//copy txm files
+List<File> srcfiles = srcDir.listFiles();
+for (File f : srcfiles) {// check XML format, and copy file into binDir
+	if (f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties"))
+	continue;
+	if (ValidateXml.test(f)) {
+		FileCopy.copy(f, new File(paraDir, f.getName()));
+	} else {
+		println "Won't process file "+f;
+	}
+}
+
+if (paraDir.listFiles() == null) {
+	println "No txm file to process"
+	return;
+}
+
+//contains the text files associated to a corpus
+println "Getting lang of each corpus and splitting corpus: "+params.corpora.keySet()
+for (String corpusname : params.corpora.keySet()) {
+	
+	lang = params.corpora.get(corpusname).getAttribute("lang");
+	encoding = params.corpora.get(corpusname).getAttribute("encoding");
+	model = lang
+	
+	print "."
+	println "split in"
+	File txmDir = new File(binDir,"txm/$corpusname");
+	txmDir.deleteDir();
+	txmDir.mkdirs();
+	
+	def corpusElem = params.corpora.get(corpusname)
+	String corpuslang = corpusElem.getAttribute("lang")
+	
+	println "split corpus $corpusname by TEI@id"
+	File f = new File(paraDir, corpusname+".xml")
+	if (!f.exists()) {
+		println "source file is missing: "+f;
+	}
+	def splitter = new SplitBy(f);
+	if (splitter.process(txmDir, "TEI", "id")) {
+		for (File textfile : splitter.getFiles()) {
+			params.addText(corpusElem, basename, textfile)
+		}
+	} else {
+		println "failed to split corpus "+corpusname;
+		return false;
+	}
+}
+
+println "-- COMPILING - Building Search Engine indexes"
+for (String corpusname : base.getCorpora().keySet()) {
+	println "Build corpus "+corpusname
+	def c = new compiler();
+	c.setDebug();
+	//c.setCwbPath("~/TXM/cwb/bin");
+	if (!c.run(corpusfiles.get(corpusname), binDir, corpusname, basename)) {
+		println "Compiler failed"
+		return;
+	}
+}
+
+println "-- ALIGN - aligning CQP corpus with each others"
+for (String alignement : parameters.getAlignements()) {
+	println "aligning..."+alignement
+		String alignStructure = parameters.getStructure(alignement);
+		String alignAttribute = "align";
+		
+		String regpath = new File(binDir,"registry").getAbsolutePath();
+		String cwbLoc;
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		
+		println "aligned corpora: "+parameters.getTargets(alignement);
+		for (String corpusName : parameters.getTargets(alignement)) {
+			for (String targetName : parameters.getTargets(alignement)) {
+				if (corpusName != targetName) {
+					try {
+						println "build align index: "+corpusName+" with "+targetName
+						//./cwb-align -V seg_id -r ~/TXM/corpora/tmxtest/registry/ tmxen tmxfr seg
+						CwbAlign tt = new CwbAlign(cwbLoc);
+						tt.setV(alignStructure+"_"+alignAttribute);
+						tt.setr(regpath);
+						tt.cwbalign(corpusName, targetName, alignStructure);
+						
+						//patch registry
+						println " patch reg : "+new File(regpath, corpusName.toLowerCase())
+						if (PatchCwbRegistry.patchAlignment(new File(regpath, corpusName.toLowerCase()), targetName.toLowerCase()))
+						{
+							// ./cwb-align-encode -D -r ~/TXM/corpora/tmxtest/registry/ -v out.align
+							tt = new CwbAlign(cwbLoc);
+							tt.setD();
+							tt.setv();
+							tt.setr(regpath);
+							tt.cwbalignencode("out.align");
+						}
+					} catch (IOException e) {
+						org.txm.utils.logger.Log.printStackTrace(e);
+						return;
+					}
+				}
+			}
+		}
+
+}
+
+println "-- EDITION - Building edition"
+new File(binDir,"HTML").deleteDir();
+new File(binDir,"HTML").mkdir();
+new File(binDir,"HTML/default").mkdir();
+List<File> filelist = new File(binDir,"txm").listFiles();
+def second = 0
+
+println "Paginating text: "
+for(String textname : base.getTextsID())
+{
+	Text text = base.getText(textname);
+	File srcfile = text.getSource();
+	File resultfile = new File(binDir,"HTML/"+srcfile.getName().substring(0,srcfile.getName().length()-4)+".html");
+	lang = text.getAttribute("lang").toString();
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+		
+	if (second) { print(", ") }
+	if (second > 0 && (second % 5) == 0) println ""
+	print(srcfile.getName());
+	second++
+	
+	def ed = new pager_old(srcfile,resultfile, NoSpaceBefore, NoSpaceAfter, 500, basename, "pb");
+	
+	Edition editionweb = text.addEdition("default", "html", resultfile);
+	for(int i = 0 ; i < ed.getPageFiles().size();i++)
+	{
+		File f = ed.getPageFiles().get(i);
+		String idx = ed.getIdx().get(i);
+		editionweb.addPage(f,idx);
+	}
+}
+
+DomUtils.print(params.root.getOwnerDocument())
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/compiler.groovy	(revision 321)
@@ -0,0 +1,511 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2011-06-03 12:37:57 +0200 (Fri, 03 Jun 2011) $
+// $LastChangedRevision: 1867 $
+// $LastChangedBy: mdecorde $ 
+//
+
+
+package org.txm.importer.xmltxmpara;
+
+import java.util.ArrayList;
+
+import org.txm.importer.cwb.BuildCwbEncodeArgs;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler 
+{
+	
+	/** The debug. */
+	private boolean debug= false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private Writer output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The text attributes. */
+	String[] textAttributes = null;
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/** The anatypes. */
+	private static ArrayList<String> anatypes;
+	
+	/** The s attribs. */
+	private static HashMap<String, List<String>> sAttribs;
+	
+	/**
+	 * initialize.
+	 *
+	 */
+	public compiler(){}
+	
+	/**
+	 * Instantiates a new compiler.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.textAttributes = textAttributes;
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.err.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if(!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f){
+		try {
+			output = new OutputStreamWriter(new FileOutputStream(f,f.exists()) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			
+			return false;
+		}
+	}
+	
+	/**
+	 * Go to text.
+	 */
+	private void GoToText()
+	{
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			if(event == XMLStreamConstants.END_ELEMENT)
+				if(parser.getLocalName().equals("teiHeader"))
+					return;
+		}
+	}
+	
+	/**
+	 * Transfom file wtc.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFileWtc(File wtcfile)
+	{
+		createOutput(wtcfile);
+		String headvalue=""
+		String vAna = "";
+		String vForm = "";
+		String wordid= "";
+		String vHead = "";
+		
+		int p_id = 0;
+		int s_id = 0;
+		
+		boolean captureword = false;
+		boolean flagForm = false;
+		boolean flagAna = false;
+		boolean inW = false;
+		int wcounter = 1;
+		GoToText();
+		
+		try 
+		{
+			boolean stop = false;
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT && !stop; event = parser.next()) 
+			{
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+					
+						
+						switch (parser.getLocalName()) 
+						{							
+							case "w":
+								inW = true;
+								wordid = parser.getAttributeValue(null, "id")
+								if(wordid == null)
+									wordid = "w_"+text+"_"+(wcounter++)
+							
+							vAna ="";
+							break;
+							
+							case "form":
+								String type = parser.getAttributeValue(null, "type");
+								if(type == null)
+								{
+									flagForm = true;
+								}
+								else if(type.equals("default"))
+								{
+									flagForm = true;
+								}
+								else
+								{
+									flagAna = true;
+									vAna += "\t";
+									if(!anatypes.contains(type))
+										anatypes << type;	
+								}
+							vForm = "";
+							break;
+							
+							case "ana":
+							flagAna = true;
+							vAna += "\t";
+							String type = parser.getAttributeValue(null, "type");
+
+								if(type != null)
+								{	
+									if(type.startsWith("#"))
+										type = type.substring(1)
+									if(!anatypes.contains(type))
+										anatypes << type;
+									break;
+								}
+							break;
+							
+							default:
+								if(!inW)
+								{
+									output.write("<"+parser.getLocalName().toLowerCase());
+									if(!sAttribs.containsKey(parser.getLocalName()))
+										sAttribs.put(parser.getLocalName().toLowerCase(), []);
+									
+									for( int i = 0 ; i < parser.getAttributeCount() ; i++)
+									{
+										String attrname = parser.getAttributeLocalName(i).toLowerCase();
+										String attrvalue = parser.getAttributeValue(i);
+										if(!(parser.getLocalName() == "text" && attrname == "id"))
+											output.write(" "+attrname+"=\""+attrvalue.replace("\"", "'")+"\"");
+										
+										if(!sAttribs.get(parser.getLocalName().toLowerCase()).contains(attrname))
+											sAttribs.get(parser.getLocalName().toLowerCase()).add(attrname)
+									}
+							
+									if(parser.getLocalName() == "text")
+									{ // add some infos
+										output.write(" id=\""+text+"\" base=\""+base+"\" project=\""+project+"\"");
+									}
+									output.write(">\n");
+								}
+							}
+						break;
+					
+					case XMLStreamConstants.END_ELEMENT:
+						
+						switch (parser.getLocalName()) 
+						{	
+							case "w":
+							output.write( vForm.replaceAll("&", "&amp;").replaceAll("<", "&lt;") +"\t"+wordid+vAna+"\n");
+							vAna = "";
+							vForm = "";
+							inW = false;
+							break;
+							
+							case "form":
+							flagForm = false;
+							flagAna = false;
+							break;
+							
+							case "ana":
+							flagAna = false;
+							break;
+							
+							default:
+								if(!inW)
+									output.write("</"+parser.getLocalName().toLowerCase()+">\n");
+								if(parser.getLocalName() == "text")
+									stop = true;
+						}
+						break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						if(inW)
+						{
+						if(flagForm)
+							if(flagAna)
+								vAna += parser.getText().trim();
+							else
+								vForm += parser.getText().trim();
+						if(flagAna)
+							vAna += parser.getText().trim();
+						}
+						break;
+				}
+			}
+			
+			output.close();
+			parser.close();
+		}
+		catch (XMLStreamException ex) {
+			System.out.println(ex);
+			return false;
+		}
+		catch (IOException ex) {
+			System.out.println("IOException while parsing " + inputData);
+			return false;
+		}
+		return true;
+	}
+	
+	
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @param textAttributes the text attributes
+	 * @return true, if successful
+	 */
+	public boolean run(ArrayList<File> files, File binDir, String corpusname, String basename, String[] textAttributes) 
+	{
+		anatypes = new ArrayList<String>();// init only 1 time
+		sAttribs = new HashMap<String, List<String>>();// init only 1 time
+		String rootDir = binDir.getAbsolutePath();
+		
+		if(cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if(!new File(cwbLoc).exists())
+		{
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if(!binDir.exists())
+		{
+			println ("binary directory does not exists: "+binDir)
+			return false;
+		}
+		
+		if(files == null || files.size() == 0)
+		{
+			println "Error: no file to process"
+			return false;
+		}
+				
+		String textid = "";
+		int counttext = 0;
+		File wtcdir = new File(binDir,"wtc");
+		File wtcfile = new File(wtcdir, corpusname.toLowerCase()+".wtc");
+		//0 set Lang
+		if(createOutput(wtcfile))
+		{
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+		
+		//1- Transform into WTC file
+		def builder = null;
+		for (File f : files) {
+			counttext++;
+			if (!f.exists()) {
+				println("The file "+f+ " does not exists")	
+			} else {	
+				//println("process file "+f)
+				String txtname = f.getName().substring(0,f.getName().length()-4);
+				builder = new compiler(f.toURL(), txtname, basename, "default");
+				builder.setLang(lang);
+				if (!builder.transfomFileWtc(wtcfile))
+					return false;
+			}
+		}
+		
+		//end corpus
+		if(createOutput(wtcfile))
+		{
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+		
+		//2- Import into CWB
+		def outDir = rootDir;
+		def outDirTxm = rootDir;
+		
+		CwbEncode cwbEn = new CwbEncode();
+		cwbEn.setDebug(debug);
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbMa.setDebug(debug);
+		List<String> pargs = ["id"];
+		for(String ana : anatypes)
+			pargs.add(ana);
+		
+		List<String> sargs = [];
+		//println "Found Sattributes "+this.sAttribs;
+		if (sAttribs.containsKey("text")) {
+			if (!sAttribs.get("text").contains("id"))
+				sAttribs.get("text").add("id");
+			if (!sAttribs.get("text").contains("base"))
+				sAttribs.get("text").add("base");
+			if (!sAttribs.get("text").contains("project"))
+				sAttribs.get("text").add("project");
+		} else {
+			sargs.add("text:0+id+base+project")
+		}
+		
+		if(sAttribs.containsKey("txmcorpus"))
+		{
+			if(!sAttribs.get("txmcorpus").contains("lang"))
+				sAttribs.get("txmcorpus").add("lang");
+		} else {
+			sargs.add("txmcorpus:0+lang")
+		}
+		
+		for(String tag : this.sAttribs.keySet())
+		{
+			String sAttr = tag;
+			if(sAttribs.get(tag).size() > 0)
+				sAttr += ":";
+			for(String attr : sAttribs.get(tag))
+				sAttr +="+"+attr;
+			sargs.add(sAttr)
+		}
+		
+		String[] sAttributes = sargs;
+		String[] pAttributes = pargs;
+		println "Corpus structural attributes: "+sAttributes;
+		println "Corpus lexical attributes: "+pAttributes;
+		try {
+			String regPath = outDirTxm + "/registry/"+corpusname.toLowerCase();
+			cwbEn.run(new File(cwbLoc,"cwb-encode").getAbsolutePath(),
+				outDirTxm + "/data/"+corpusname+"/", 
+				wtcfile.getAbsolutePath(), 
+				regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(new File(cwbLoc,"cwb-makeall").getAbsolutePath(), corpusname.toUpperCase(), outDirTxm + "/registry");
+			
+		} catch (Exception ex) {System.out.println(ex); return false;}
+		
+		System.out.println("Done.") 
+		
+		return true;
+	}
+	
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug()
+	{
+		this.debug = true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/geo");
+		def c = new compiler();
+		c.setDebug();
+		c.setCwbPath("~/TXM/cwb/bin");
+		c.run(dir,"geo");
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>XML-TXM import module specialized with aligned corpus</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xmltxmpara/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/compiler.groovy	(revision 321)
@@ -0,0 +1,221 @@
+
+
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-01-05 14:27:34 +0100 (jeu., 05 janv. 2012) $
+// $LastChangedRevision: 2096 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.wtc
+
+import org.txm.Toolbox;
+import org.txm.importer.cwb.*
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.utils.treetagger.TreeTagger;
+
+import java.util.ArrayList;
+
+import javax.xml.stream.*;
+
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler
+{
+	/** The debug. */
+	boolean debug = false;
+
+	/** The dir. */
+	private def dir;
+
+	/** The cwb loc. */
+	String cwbLoc;
+	File srcWTCFile, srcRegistryFile;
+
+	public def pAttributesList = [];
+	public def sAttributesList = [];
+
+	public compiler(File wtcFile, File registryFile) {
+		this.srcWTCFile = wtcFile;
+		this.srcRegistryFile = registryFile
+	}
+
+	/**
+	 * Sets the debug.
+	 *
+	 * @return the java.lang. object
+	 */
+	public setDebug()
+	{
+		debug =true;
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if (!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, String corpusname)
+	{
+		if (cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if (!binDir.exists()) {
+			println ("binary directory does not exists: "+binDir)
+			return false;
+		}
+
+		if (!srcWTCFile.exists()) {
+			println "Error: cannot find the WTC file $srcWTCFile"
+			return false;
+		}
+
+		File wtcFile = new File(binDir,"wtc/"+corpusname+".wtc");
+		new File(binDir, "wtc").deleteDir();
+		new File(binDir, "wtc").mkdir();
+		new File(binDir, "data").deleteDir();
+		new File(binDir, "data").mkdir();
+		new File(binDir, "registry").mkdir();
+
+		FileCopy.copy(srcWTCFile, wtcFile);
+
+		//2- Import into CWB
+		def outDir = binDir.getAbsolutePath()+"/";
+
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbEn.setDebug(debug);
+		cwbMa.setDebug(debug);
+
+		String[] pAttributes;
+		String[] sAttributes;
+
+		if (srcRegistryFile == null) {
+			println "WARNING: No registry file found in source directory"
+			println "We'll search for positional attributes and structural attributes in the WTC file"
+
+			// s attributes
+			BuildCwbEncodeArgsFromWTC argsgetter = new BuildCwbEncodeArgsFromWTC(); // XML stream
+			argsgetter.process(wtcFile); //$NON-NLS-1$
+			sAttributesList = argsgetter.getSAttributes();
+
+			// p attributes
+			int nbAttr = -1;
+			File tmp = File.createTempFile("txm", ".wtc", wtcFile.getParentFile());
+			int wcounter = 1;
+			println "Adding the 'id' property to the WTC file and getting word properties number."
+			tmp.withWriter("UTF-8") { writer ->
+				wtcFile.eachLine("UTF-8") { line ->
+					if (!line.startsWith("<")) {
+						if (nbAttr == -1) nbAttr = line.split("\t").size();
+						writer.println(line+"\tw_"+(wcounter++))
+					} else {
+						writer.println(line)
+					}
+					writer.flush();
+				}
+			}
+			wtcFile.delete()
+			tmp.renameTo(wtcFile)
+
+			System.out.println("Found "+(nbAttr-1)+" word properties, $nbAttr with the 'id'");
+			for (int i = 1; i < nbAttr ; i++) {
+				pAttributesList << "p$i";
+			}
+			pAttributesList << "id"
+		} else {
+			ReadRegistryFile reader = new ReadRegistryFile(srcRegistryFile);
+			pAttributesList = reader.getPAttributes();
+			sAttributesList = reader.getSAttributes();
+			pAttributesList.remove(0) // remove word
+
+			if (!pAttributesList.contains("id")) {
+				System.out.println("Error: The registry file does not declare the 'id' word property");
+				return false;
+			}
+		}
+
+		pAttributes = pAttributesList; // cast to array
+		sAttributes = sAttributesList; // cast to array
+
+		println "pAttrs : "+Arrays.toString(pAttributes)
+		println "sAttrs : "+Arrays.toString(sAttributes)
+
+		try {
+			cwbEn.setDebug(debug);
+			cwbMa.setDebug(debug);
+			String regPath =outDir + "/registry/"+corpusname.toLowerCase()
+			cwbEn.run(cwbLoc + "cwb-encode", outDir + "/data/$corpusname", 
+					wtcFile.getAbsolutePath(),
+					regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(cwbLoc + "cwb-makeall", corpusname, outDir + "/registry");
+			return true;
+		} catch (Exception ex) {System.out.println(ex); return false;}
+
+		return true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/perrault/txm/");
+		List<File> files = dir.listFiles();
+		new compiler().run(files);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>CWB import module.</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/pager.groovy	(revision 321)
@@ -0,0 +1,350 @@
+package org.txm.importer.wtc;
+//Copyright © - ANR Textométrie - http://textometrie.ens-lyon.fr
+//
+//This file is part of the TXM platform.
+//
+//The TXM platform is free software: you can redistribute it and/or modif y
+//it under the terms of the GNU General Public License as published by
+//the Free Software Foundation, either version 3 of the License, or
+//(at your option) any later version.
+//
+//The TXM platform is distributed in the hope that it will be useful,
+//but WITHOUT ANY WARRANTY; without even the implied warranty of
+//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//GNU General Public License for more details.
+//
+//You should have received a copy of the GNU General Public License
+//along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+//$LastChangedDate: 2012-06-01 17:47:31 +0200 (ven., 01 juin 2012) $
+//$LastChangedRevision: 2185 $
+//$LastChangedBy: mdecorde $
+//
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+/** Build WTC corpus simple edition from a WTC file. 
+ * 
+ * @author mdecorde
+ */
+class pager {
+	List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename = "";
+	String txtname = "";
+	File outdir;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue;
+
+	/** The interpvalue. */
+	String interpvalue;
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	OutputStreamWriter writer;
+
+	/** The pagedWriter. */
+	OutputStreamWriter pagedWriter = null;
+
+	/** The wtcFile. */
+	File wtcFile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The pages. */
+	def pages = [:];
+
+	/** The idxstart. */
+	def idxstart = [:]
+	String editionPage;
+	ArrayList<Integer> splitTUs; // contains the tu ids used to split pages
+	boolean shouldSplit = false;
+	boolean useSplitTUs = false;
+	boolean hasWordId = false;
+	int noWordIdProperty = -1;
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param wtcFile the wtcFile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager(File wtcFile, File outdir, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String basename, boolean hasWordId, List pAttrs) {
+		this.editionPage = editionPage;
+		this.basename = basename;
+		this.txtname = "text1";
+		this.outdir = outdir;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.wtcFile = wtcFile;
+		this.hasWordId = hasWordId;
+		if (hasWordId) {
+			noWordIdProperty = pAttrs.indexOf("id") + 1;
+			if (noWordIdProperty == 0)
+			hasWordId = false;
+		}
+		
+		process();
+	}
+
+	private void closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			if (firstWord) { // there was no words
+				this.idxstart[txtname] = ["w_0"]
+				pagedWriter.write("<span id=\"w_0\"/>");
+			}
+			pagedWriter.write("</p>\n")
+			pagedWriter.write("</body>");
+			pagedWriter.write("</html>");
+			pagedWriter.close();
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		wordcount = 0;
+		shouldSplit = false;
+		try {
+			closeMultiWriter();
+			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")
+			//println "outfile: "+outfile
+			if (pages[txtname] == null) pages[txtname] = []
+			pages[txtname] << outfile;
+			firstWord = true; // waiting for next word
+
+			pagedWriter = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(outfile)) , "UTF-8");
+			pagedWriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+			pagedWriter.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"   \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n")
+			pagedWriter.write("<html>");
+			pagedWriter.write("<head>");
+			pagedWriter.write("<title>"+basename.toUpperCase()+" $txtname Edition - Page "+pagecount+"</title>");
+			pagedWriter.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"tmx.css\"/>");
+			pagedWriter.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\"/>");
+			pagedWriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");
+			pagedWriter.write("</head>");
+			pagedWriter.write("<body>");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput() {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public def getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public def getIdx() {
+		return idxstart;
+	}
+
+	def texts = []
+
+	/**
+	 * Process.
+	 */
+	void process() {
+
+		String localname = "";
+		String lastword= "";
+		createNextOutput();
+		int wcounter = 1;
+		int txtwcounter = 1;
+
+		BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(wtcFile) , "UTF-8"));
+		String line = reader.readLine();
+		int cline = 0;
+		while (line != null) {
+			pagedWriter.flush();
+			cline++;
+			//println "line: "+line
+			if (line.startsWith("<")) {
+				if (line.startsWith("<lb/") || line.startsWith("<br/") || line.startsWith("</p>") || 
+					line.startsWith("<lb>") || line.startsWith("<br>")) {
+					pagedWriter.write("<br/>");
+				} 
+				if (line.startsWith("<text>") || line.startsWith("<text ")) {
+					String tmp = line;
+					int idx = tmp.indexOf("id=\"");
+					if (idx > 0) {
+						tmp = tmp.substring(idx+4);
+						//println "tmp1: "+tmp
+						int idx2 = tmp.indexOf("\"");
+						//println "tmp2: "+tmp
+						if (idx2 > 0) {
+							tmp = tmp.substring(0, idx2);
+							txtname = tmp
+							texts << txtname;
+							pagecount = 0;
+							createNextOutput();
+						}
+					}
+				} else if (line.startsWith("<s>") || line.startsWith("<s ")) {
+					pagedWriter.write(line.replaceAll("<s", "<sent "));
+				} else if (line.startsWith("</s>")) {
+					pagedWriter.write("</sent>");
+				} else {
+					pagedWriter.write(line);
+				}
+			} else { // word
+				if (txtwcounter > wordmax) {
+					txtwcounter = 1;
+					createNextOutput();
+				}
+				def split = line.split("\t");
+				if (split == null || split.size() == 0) {
+					// empty line
+				} else {
+					String wordid;
+					if (hasWordId) {
+						if (split.size() <= noWordIdProperty) { // +1 since line contains word
+							println "Error: line $cline"
+						} else {
+							wordid = split[noWordIdProperty];
+						}
+					} else {
+						wordid = "w_"+(wcounter++);
+					}
+					txtwcounter++;
+					String wordvalue = split[0];
+
+					if (firstWord) {
+						firstWord = false;
+						if (this.idxstart[txtname] == null) {
+							this.idxstart[txtname] = []
+						}
+						this.idxstart[txtname] << [wordid];
+					}
+
+					int l = lastword.length();
+					String endOfLastWord = "";
+					if (l > 0) {
+						endOfLastWord = lastword.subSequence(l-1, l);
+					}
+
+					String interpvalue = "";
+					if (split.size() > 1) {
+						interpvalue = split[1..split.size()-1].join(" ")
+						interpvalue = interpvalue.replace("&", "&amp;").replace("<", "&lt;").replace("\"","&quot;");
+					}
+
+
+					if (NoSpaceBefore.contains(wordvalue) ||
+					NoSpaceAfter.contains(lastword) ||
+					wordvalue.startsWith("-") ||
+					NoSpaceAfter.contains(endOfLastWord)) {
+						pagedWriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+					} else {
+						pagedWriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+					}
+
+					pagedWriter.write(wordvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;")+"</span>\n");
+					pagedWriter.flush()
+					lastword = wordvalue;
+				}
+			}
+			line = reader.readLine();
+		}
+
+		closeMultiWriter();
+	}
+
+	def getTextNames() {
+		return texts;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/wtcLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/wtcLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/wtc/wtcLoader.groovy	(revision 321)
@@ -0,0 +1,145 @@
+package org.txm.importer.wtc
+
+import org.txm.*
+import org.txm.metadatas.*
+import org.txm.objects.*
+import org.txm.scripts.teitxm.*
+import org.txm.utils.i18n.*
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/wtc/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+// FAKE: will contains no XML-TXM files
+File txmDir = new File(binDir, "txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+// Will contains the edition files
+File htmlDir = new File(binDir, "HTML/$corpusname");
+htmlDir.deleteDir();
+htmlDir.mkdirs();
+
+// get the wtc file and the registry file
+File wtcFile;
+File registryFile;
+println "Looking for files to process..."
+for(File f : srcDir.listFiles()) {
+	if (f.isHidden()) continue;
+	if (f.isDirectory()) continue;
+	if (f.getName().endsWith(".wtc")) {
+		if (wtcFile == null) {
+			wtcFile = f;
+			println "Using WTC file : $wtcFile"
+		}
+	} else if (f.getName().endsWith(".xml")) {
+
+	} else if (!f.getName().contains(".")){ // its the registry file
+		if (registryFile == null) {
+			registryFile = f;
+			println "Using registry file : $f"
+		}
+	}
+}
+
+if (wtcFile == null) {
+	println "No WTC file in source directory: "+srcDir;
+	return false;
+}
+
+// get pAttributes an run cwb-encode + cwb-makeall
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler(wtcFile, registryFile)
+c.setDebug(debug)
+if (!c.run(binDir, corpusname)) {
+	println "Import stopped"
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+	
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(20, "EDITION - Building edition")
+
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+
+	print "."
+
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+	File binWtcFile = new File(binDir,"wtc/"+corpusname+".wtc");
+	def ed = new pager(binWtcFile, outdir, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, registryFile !=null, c.pAttributesList);
+	def allIndexes = ed.getIdx();
+	def allPages = ed.getPageFiles();
+	for (String txtname : ed.getTextNames()) {
+		def txtIndexes = allIndexes[txtname];
+		def txtPages = allPages[txtname];
+
+		Element text = params.addText(params.corpora.get(corpusname), txtname, wtcFile);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < txtPages.size();) {
+			File f = txtPages.get(i);
+			String wordid = txtIndexes.get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+println ""
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ApplyXsl.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ApplyXsl.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ApplyXsl.groovy	(revision 321)
@@ -0,0 +1,244 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2016-03-01 10:10:58 +0100 (Tue, 01 Mar 2016) $
+// $LastChangedRevision: 3133 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+/**
+ * @author mdecorde
+ */
+
+import java.io.File;
+import javax.xml.transform.Transformer
+import javax.xml.transform.TransformerFactory
+import javax.xml.transform.stream.StreamResult
+import javax.xml.transform.stream.StreamSource;
+
+import org.txm.searchengine.cqp.clientExceptions.CqiClientException;
+import net.sf.saxon.*;
+import net.sf.saxon.s9api.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * apply a xsl file to a file : use process(File infile, File outfile, String[]
+ * args) to apply the xslt file loaded by the constructor args =
+ * ["name1","value1","name2",value2, ...]
+ * 
+ * @author mdecorde
+ */
+
+public class ApplyXsl {
+	
+//	/** The proc. */
+//	private Processor proc;
+//	
+//	/** The comp. */
+//	private XsltCompiler comp;
+//	
+//	/** The exp. */
+//	private XsltExecutable exp;
+//
+//	/** The source. */
+//	private XdmNode source;
+//	
+//	/** The out. */
+//	private Serializer out;
+//	
+//	/** The trans. */
+//	private XsltTransformer trans;
+
+	TransformerFactory tFactory;
+	Transformer transformer;
+	
+	/**
+	 * initialize with the xslt file.
+	 *
+	 * @param xsltfile the xsltfile
+	 */
+	public ApplyXsl(String xsltfile) {
+//		proc = new Processor(false);
+//		comp = proc.newXsltCompiler();
+//		exp = comp.compile(new StreamSource(new File(xsltfile)));
+		tFactory = new net.sf.saxon.TransformerFactoryImpl();
+		transformer = tFactory.newTransformer(new StreamSource(xsltfile));
+
+	}
+	
+	/**
+	 * initialize with the xslt file.
+	 *
+	 * @param xsltfile the xsltfile
+	 */
+	public ApplyXsl(File xsltfile) {
+//		proc = new Processor(false);
+//		comp = proc.newXsltCompiler();
+//		exp = comp.compile(new StreamSource(xsltfile));
+		tFactory = new net.sf.saxon.TransformerFactoryImpl();
+		transformer = tFactory.newTransformer(new StreamSource(xsltfile));
+	}
+
+	/**
+	 * Set in and out file.
+	 *
+	 * @param xmlinfile the xmlinfile
+	 * @param xmloutfile the xmloutfile
+	 * @return true, if successful
+	 */
+	private boolean SetInOutSource(String xmlinfile, String xmloutfile) {
+//		if (proc == null || exp == null || comp == null)
+//			return false;
+//
+//		source = proc.newDocumentBuilder().build(
+//				new StreamSource(new File(xmlinfile)));
+//		out = new Serializer();
+//		out.setOutputFile(new File(xmloutfile));
+//
+//		trans = exp.load();
+//		transformer.set.setInitialContextNode(source);
+//		transformer.setDestination(out);
+		return true;
+	}
+
+	/**
+	 * Set a xslt param.
+	 *
+	 * @param name the name
+	 * @param value the value
+	 * @return true, if successful
+	 */
+	public boolean SetParam(String name, Object value) {
+//		if (trans != null)
+//			trans.setParameter(new QName(name), new XdmAtomicValue("" + value));
+//		else
+//			return false;
+		return true;
+	}
+
+	/**
+	 * Force java to clean memory after processing a xslt. If not we might catch
+	 * a JavaHeapspace Exception
+	 * 
+	 */
+	private void cleanMemory() {
+//		source = null;
+//		trans = null;
+//		out = null;
+	}
+
+	/**
+	 * Process files with xslt args.
+	 *
+	 * @param xmlinfile file to be processed
+	 * @param xmloutfile output file
+	 * @param args xslt args ["arg1","arg1value","arg2","arg2value"...]
+	 * @return true, if successful
+	 */
+
+	public boolean process(String xmlinfile, String xmloutfile, String[] args) {
+		if (!this.SetInOutSource(xmlinfile, xmloutfile))
+			return false;
+		for (int i = 0; i < args.length; i = i + 2)// set params
+		{
+			if (!this.SetParam(args[i], args[i + 1]))
+				return false;
+		}
+
+		transformer.transform(new StreamSource(xmlinfile), new StreamResult(new BufferedOutputStream(new FileOutputStream(xmloutfile))));
+		cleanMemory(); // save memory
+		// System.out.println("Done");
+		return true;
+	}
+
+	/**
+	 * Process files without xslt args.
+	 *
+	 * @param xmlinfile the xmlinfile
+	 * @param xmloutfile the xmloutfile
+	 * @return true, if successful
+	 */
+	public boolean process(String xmlinfile, String xmloutfile) {
+		if (!this.SetInOutSource(xmlinfile, xmloutfile))
+			return false;
+
+		transformer.transform(new StreamSource(xmlinfile), new StreamResult(new BufferedOutputStream(new FileOutputStream(xmloutfile))));
+		cleanMemory();
+		// System.out.println("Done");
+		return true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String xslfile = "pathtoxslfile";
+		ApplyXsl a = new ApplyXsl(xslfile);
+		
+		File srcdirectory = new File("pathtodirectorytoprocess")
+		assert(srcdirectory.exists())
+		assert(srcdirectory.isDirectory())
+		assert(srcdirectory.canRead())
+		assert(srcdirectory.canExecute())
+		
+		List<File> files = srcdirectory.listFiles();
+		
+		files = files.findAll{item-> item.getName().endsWith(".xml")}
+		
+		long bigstart = System.currentTimeMillis()
+		for(File infile : files)
+		{
+			String outfile = infile.getAbsolutePath()+".out"
+			if(infile.canRead())
+			{
+				long start = System.currentTimeMillis()
+				println "Process : "+infile.getName()
+				// String[] params = ["pbval1",1,"pbval2",2];
+				// a.process(infile.getAbsolutePath(),outfile); // no parameters
+				// a.process( infile.getAbsolutePath(),outfile,params);
+				println "Done : "+(System.currentTimeMillis()-start)+"ms"
+			}
+		}
+		println "Total time : "+(System.currentTimeMillis()-bigstart)+"ms"
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CopyXMLFiles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CopyXMLFiles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/CopyXMLFiles.groovy	(revision 321)
@@ -0,0 +1,63 @@
+package org.txm.importer;
+
+import java.io.File;
+import java.io.InputStream;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLResolver;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import javax.xml.stream.events.DTD;
+import javax.xml.stream.events.EntityReference;
+
+import org.txm.utils.*
+import org.txm.utils.io.FileCopy;
+
+public class CopyXMLFiles {
+	File xmlFile;
+	File outDir;
+	def dtdFiles = []
+	
+	public CopyXMLFiles(File xmlFile) {
+		this.xmlFile = xmlFile;
+	}
+	
+	public boolean copy(File outDir) {
+		XMLInputFactory factory;
+		XMLStreamReader parser;
+		InputStream inputData = xmlFile.toURI().toURL().openStream();
+		factory = XMLInputFactory.newInstance();
+		factory.setXMLResolver(new XMLResolver() {
+			@Override
+			public Object resolveEntity(String publicID, String systemID,
+					String baseURI, String namespace) throws XMLStreamException {
+				//println "Resolving $publicID $systemID $baseURI $namespace"
+				File srcFile = new File(xmlFile.getParentFile(), systemID);
+				dtdFiles << srcFile
+				return srcFile.toURI().toURL().openStream();
+			}
+		});
+	
+		parser = factory.createXMLStreamReader(inputData);
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			
+		}
+		
+		dtdFiles << xmlFile
+		for (File dtd : dtdFiles) {
+			File cpy = new File(outDir, dtd.getName());
+			FileCopy.copy(dtd, cpy);
+		}
+		return outDir.listFiles().size() == dtdFiles.size();
+	}
+	
+	public static void main(def args) {
+		File xmlFile = new File("/home/mdecorde/Téléchargements/Inscriptions1.xml")
+		File outDir = new File("/home/mdecorde/Téléchargements/test")
+		outDir.mkdir()
+		CopyXMLFiles cdf = new CopyXMLFiles(xmlFile)
+		println cdf.copy(outDir)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WriteIdAndNAttributes.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WriteIdAndNAttributes.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/WriteIdAndNAttributes.groovy	(revision 321)
@@ -0,0 +1,231 @@
+package org.txm.importer
+
+class WriteIdAndNAttributes extends StaxIdentityParser {
+
+	String textname
+
+	int nMileStone = 1, nPb = 1, nCb = 1, nLb = 1, nW = 1, nSeg = 1, nRejet = 1;
+	String previousMileStone, previousPb, previousCb, previousW;
+
+
+	String PB = "pb", CB = "cb", LB = "lb", ID = "id", TYPE = "type",
+	N = "n", CORRESP = "corresp", FACS="facs", W="w", PC="pc", SEG="seg",
+	UNIT="unit", XML="xml", WP="wp", SURFACE="surface", POINT = ".", REJET ="rejet";
+
+	public WriteIdAndNAttributes(File xmlFile, String textname) {
+		super(xmlFile);
+
+		this.textname = textname
+	}
+
+	protected void writeAttributes() {
+		// do nothing
+	}
+
+	protected void processStartElement() {
+		String id = null
+		String n = null
+		String type = null
+		String corresp = null
+		String facs = null
+
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			if (parser.getAttributeLocalName(i) == ID) {
+				id = parser.getAttributeValue(i)
+			} else if (parser.getAttributeLocalName(i) == N) {
+				n = parser.getAttributeValue(i)
+			} else if (parser.getAttributeLocalName(i) == TYPE) {
+				type = parser.getAttributeValue(i)
+			} else if (parser.getAttributeLocalName(i) == CORRESP) {
+				corresp = parser.getAttributeValue(i)
+			} else if (parser.getAttributeLocalName(i) == FACS) {
+				facs = parser.getAttributeValue(i)
+			}
+		}
+
+		super.processStartElement(); // attributes are not written because super.writeAttributes() is not called
+
+		if (localname == "milestone" && parser.getAttributeValue(null, UNIT) == SURFACE) {
+			if (n == null) {
+				n = nMileStone
+			} else {
+				int tmp = Integer.parseInt(n)
+				if (nMileStone < tmp) {
+					nMileStone = tmp
+					n = tmp
+				} else {
+					n = nMileStone;
+				}
+			}
+
+			if (id == null) { // don't rewrite "id"
+				if (facs == null) {
+					println "Error: found milestone@type=\"surface\" with no @facs at "+parser.getLocation()
+					throw new Exception("no facs attribute")
+				}
+				if (facs.lastIndexOf(POINT) > 0) facs = facs.substring(0, facs.lastIndexOf(POINT))
+				previousMileStone = "surf_$textname"+"_"+facs
+				id = previousMileStone
+			} else {
+				previousMileStone = id
+			}
+
+			nMileStone++
+		} else if (localname == PB) {
+			if (n == null) {
+				n = nPb
+			} else {
+				try {
+					int tmp = Integer.parseInt(n)
+					if (nPb < tmp) {
+						nPb = tmp
+					} else {
+						n = nPb;
+					}
+				} catch(Exception e) { }
+			}
+
+			if (id == null) { // don't rewrite "id"
+				id = "page_${textname}_"+nPb
+				previousPb = "${textname}_"+nPb
+			} else {
+				previousPb = id
+			}
+
+			nRejet = nLb = nCb = 1
+			nPb++
+		} else if (localname == CB) {
+			if (n == null) {
+				n = nCb
+			} else {
+				try {
+					int tmp = Integer.parseInt(n)
+					if (nCb < tmp) {
+						nCb = tmp
+					} else {
+						n = nCb;
+					}
+				} catch(Exception e) { }
+			}
+
+			if (id == null) { // don't rewrite "id"
+				previousCb = "${previousPb}_"+nCb
+				id = "col_${previousPb}_"+nCb
+			} else {
+				previousCb = id
+			}
+
+			nLb = nRejet = 1
+			nCb++
+		} else if (localname == LB) {
+			if (n == null) {
+				if (REJET == type) {
+					println "Warning: no 'n' attribute provided for 'rejet' line break at "+parser.getLocation()
+				}
+				
+				n = nLb
+			} else {
+				if (REJET != type) {
+					try { // if n is numeric update nLgRelative
+						int tmp = Integer.parseInt(n)
+						nLb = tmp;
+					} catch(Exception e) {
+						// n is not numeric use it
+					}
+				}
+			}
+
+			if (id == null) { // don't rewrite "id"
+				if (REJET == type) {
+					id = "line_${previousCb}_"+n+"_r"+nRejet
+				} else {
+					id = "line_${previousCb}_"+n
+				}
+			}
+
+			if (corresp == null && REJET == type) {
+				//println "Warning: no 'corresp' attribute provided for 'rejet' line break at "+parser.getLocation()
+				corresp = "#line_${previousCb}_"+n; // next word id in the same column
+			}
+
+			if (REJET != type) {
+				nLb++;
+			} else {
+				nRejet++;
+			}
+
+		} else if (localname == W || localname == PC) {
+			if (n == null) {
+				n = nW
+			} else {
+				try {
+					int tmp = Integer.parseInt(n)
+					if (nW < tmp) {
+						nW = tmp
+					} else {
+						n = nW;
+					}
+				} catch(Exception e) { }
+			}
+
+			if (id == null) { // don't rewrite "id"
+				//previousW = "${previousPb}_"+nW
+				id = localname+"_${textname}_"+nW
+			} 
+			previousW = id
+			nSeg = 1
+			nW++
+		} else if (localname == SEG && WP == type) {
+			if (n == null) {
+				n = nSeg
+			} else {
+				try {
+					int tmp = Integer.parseInt(n)
+					if (nSeg < tmp) {
+						nSeg = tmp
+					} else {
+						n = nSeg;
+					}
+				} catch(Exception e) { }
+			}
+
+			if (id == null) { // don't rewrite "id"
+				id = "w_p_"+previousW+"_"+nSeg
+			}
+			nSeg++
+		}
+
+		// write attributes except ID, N and CORRESP
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			if (parser.getAttributeLocalName(i) == ID) {
+
+			} else if (parser.getAttributeLocalName(i) == N) {
+
+			}  else if (parser.getAttributeLocalName(i) == CORRESP) {
+
+			} else {
+				writeAttribute(parser.getAttributePrefix(i), parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+			}
+		}
+		//if (localname == "lb") println "write id: "+["xml", "id", id]
+		if (id != null)
+			writeAttribute(XML, ID, id)
+		if (n != null)
+			writeAttribute(null, N, n)
+		if (corresp != null)
+			writeAttribute(null, CORRESP, corresp)
+	}
+
+	public static void main(String[] args) {
+		File xmlFile = new File("/home/mdecorde/xml/bugrejet/Psautier5-or28.xml")
+		File outFile = new File("/home/mdecorde/xml/bugrejet/Psautier5-or28-o.xml")
+
+		WriteIdAndNAttributes wiana = new WriteIdAndNAttributes(xmlFile, "qgraal_cm")
+		println wiana.process(outFile)
+
+		String text = xmlFile.getText().replaceAll(">", ">\n");
+		new File("/home/mdecorde/xml/bugrejet/Psautier5-or28-p.xml").write(text);
+		text = outFile.getText().replaceAll(">", ">\n");
+		new File("/home/mdecorde/xml/bugrejet/Psautier5-or28-o-p.xml").write(text);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/FortranColumns2XML.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/FortranColumns2XML.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/FortranColumns2XML.groovy	(revision 321)
@@ -0,0 +1,325 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2016-12-16 17:28:35 +0100 (Fri, 16 Dec 2016) $
+// $LastChangedRevision: 3377 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import org.txm.utils.i18n.DetectBOM;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class FortranColumns2XML.
+ */
+class FortranColumns2XML {
+	
+	/** The infile. */
+	
+	File infile;
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The encoding. */
+	String encoding;
+	
+	/** The column names. */
+	List<String> columnNames
+	
+	/** The word tag. */
+	String wordTag;
+	
+	/** The wordcolumn. */
+	int wordcolumn;
+	
+	/** The linecounter. */
+	int linecounter = 0;
+	
+	/** The check line number. */
+	int checkLineNumber;
+	
+	/** The check line number tag. */
+	String checkLineNumberTag;
+	
+	/** The columnindex. */
+	private List<Integer> columnindex;
+	
+	/** The endcolumnindex. */
+	private List<Integer> endcolumnindex;
+	
+	/** The line. */
+	private String line;
+	
+	/** The writer. */
+	private XMLStreamWriter writer;
+	
+	/** The reader. */
+	private Reader reader;
+	
+	/** The output. */
+	private FileOutputStream output;
+	
+	String textname;
+	
+	/**
+	 * Instantiates a new fortran columns2 xml.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param encoding the encoding
+	 */
+	FortranColumns2XML(File infile, File outfile, String encoding)
+	{
+		this.infile = infile;
+		this.outfile = outfile;
+		
+		textname = infile.getName();
+		int idx = -1;
+		idx = textname.indexOf(".");
+		if (idx > 0)
+			textname = textname.substring(0, idx);
+
+		def input = new FileInputStream(infile) 		
+		reader = new InputStreamReader(input , encoding);
+		DetectBOM bomdetector = new DetectBOM(infile);
+		for (int ibom = 0 ; ibom < bomdetector.getBOMSize() ; ibom++) input.read()
+		
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		output = new FileOutputStream(outfile)
+		writer = factory.createXMLStreamWriter(output, "UTF-8");
+	}
+	
+	/**
+	 * Process.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean process()
+	{
+		line = reader.readLine(); // use the first line to find columns 
+		
+		if(columnindex == null || endcolumnindex == null)
+		{
+			searchNumberOfColumns();
+		}
+		
+		if(columnindex.size() != columnNames.size())
+		{
+			System.err.println("Number of column("+columnindex.size()+") different of number of column name("+columnNames.size()+")");
+			return false;
+		}
+		
+		//write start document
+		writer.writeStartDocument("UTF-8", "1.0");
+		writer.writeStartElement("text")
+		writer.writeAttribute("id",infile.getName())
+		
+		boolean gogogo = true;
+		while(gogogo)
+		{	
+			//println "line : "+line
+			processLine();//split line accordinly to columnindexes
+			linecounter++;
+			
+			//continue to read the document
+			line = reader.readLine();
+			if(line == null)
+				gogogo= false;
+		}
+		
+		writer.writeEndElement();
+		writer.writeEndDocument();
+		writer.close();
+		output.close();
+		
+		return true;
+	}
+	
+	/**
+	 * Process line.
+	 */
+	int wordcount = 1;
+	private void processLine()
+	{
+		String pcdata;
+		if(line.length() > 0)
+		{
+			writer.writeStartElement(wordTag)
+			if(setIds)
+				writer.writeAttribute("id", "w_"+textname+"_"+(wordcount++));
+			for(int i = 0 ; i < columnindex.size(); i++)
+			{
+				if(line.length() > endcolumnindex.get(i))
+				{
+					String substring = line.substring(columnindex.get(i), endcolumnindex.get(i))
+					if(i != wordcolumn)
+					{
+						writer.writeAttribute(columnNames.get(i), substring.trim())
+					}
+					else
+					{
+						pcdata = substring.trim()
+					}
+				}
+				else
+				{
+					println "Error: line $linecounter: len="+line.length()+" maxidx="+endcolumnindex.get(i)
+					println line;
+				}
+			}
+			writer.writeCharacters(pcdata);
+			writer.writeEndElement();
+		}
+	}
+	
+	/**
+	 * Sets the column indexes.
+	 *
+	 * @param starts the starts
+	 * @param ends the ends
+	 */
+	private void setColumnIndexes(List<Integer> starts, List<Integer> ends)
+	{
+		columnindex = starts;
+		endcolumnindex = ends;
+	}
+	
+	boolean setIds = false;
+	public void setAddIds(boolean setIds)
+	{
+		this.setIds = setIds;
+	}
+	
+	/**
+	 * Search number of columns.
+	 */
+	private void searchNumberOfColumns()
+	{
+		columnindex = new ArrayList<Integer>();
+		endcolumnindex = new ArrayList<Integer>();
+		columnindex.add(0);
+		int ic = 0
+		char c;
+		char[] cline = line.toCharArray();
+		int state = 0; // 0 lettre, 1 espace
+		
+		for(ic = 0 ; ic < cline.length ; ic++)
+		{
+			c = cline[ic];
+			if(state == 0)// zone lettre
+			{
+				if(c == " ")
+				{
+					state = 1;
+					
+				}
+			}
+			else // zone espace
+			{
+				if(c != " ")
+				{
+					columnindex.add(ic);
+					endcolumnindex.add(ic-1);
+					state = 0;
+				}
+			}
+		}
+		endcolumnindex.add(cline.length);
+		println columnindex;
+		println endcolumnindex;
+	}
+	
+	/**
+	 * Sets the columns lines names.
+	 *
+	 * @param columnNames the column names
+	 * @param unitTag the unit tag
+	 * @param wordcolumn the wordcolumn
+	 */
+	public void setColumnsLinesNames(List<String> columnNames, String unitTag, int wordcolumn)
+	{
+		this.columnNames = columnNames;
+		this.wordTag = unitTag;
+		this.wordcolumn = wordcolumn;
+	}
+	
+	/**
+	 * Sets the line check.
+	 *
+	 * @param colindex the colindex
+	 * @param tagname the tagname
+	 */
+	public void setLineCheck(int colindex, String tagname)
+	{
+		this.checkLineNumber = colindex;
+		this.checkLineNumberTag = tagname;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File fortranfile = new File(System.getProperty("user.home"),"\\xml\\lasla\\lasla\\L01PlaAmphi.APN");
+		File outfile = new File(System.getProperty("user.home"),"\\xml\\lasla\\out.xml");
+		FortranColumns2XML builder = new FortranColumns2XML(fortranfile, outfile, "iso-8859-1");
+		builder.setColumnIndexes([0, 3, 4, 8, 29, 55, 67],	[3, 4, 8, 28, 54, 66, 79]);
+		builder.setColumnsLinesNames(["ref","s","sent","word","lemme","line","pos"], "w", 3);
+		builder.setLineCheck(3, "l");
+		
+		builder.process();
+	}
+}
+
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ReunitShortLines.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ReunitShortLines.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ReunitShortLines.groovy	(revision 321)
@@ -0,0 +1,93 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer
+
+import java.io.File;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ReunitShortLines.
+ */
+class ReunitShortLines {
+	
+	/**
+	 * Instantiates a new reunit short lines.
+	 *
+	 * @param file the file
+	 * @param maxsize the maxsize
+	 * @param encoding the encoding
+	 */
+	ReunitShortLines(File file, int maxsize, String encoding)
+	{
+		File temp = new File( file.getParentFile(), "reu.txt");
+		Reader input = new InputStreamReader(new FileInputStream(file) , encoding);
+		OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(temp) , encoding);
+		
+		String line = input.readLine();
+		boolean oklength = true;
+		boolean previousok = false;
+		while(line != null)
+		{
+			if(line.length() == 0)
+			{
+				writer.write("\n");
+				oklength = true;
+			}
+			else if(line.startsWith("**** "))
+			{
+				writer.write("\n"+line);
+				oklength = true;
+			}
+			else
+			{
+			previousok = oklength;
+			oklength = line.length() > maxsize;
+			
+			if(previousok && oklength)
+				writer.write("\n");
+			
+			writer.write(line);
+			
+			if(previousok && ! oklength)
+				writer.write("\n");
+			
+			}
+			line = input.readLine();
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		new ReunitShortLines(new File("/home/mdecorde/xml/voeux/Voeux.txt"), 30, "cp1252")
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/FixBreaksAndAutomaticStyles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/FixBreaksAndAutomaticStyles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/FixBreaksAndAutomaticStyles.groovy	(revision 321)
@@ -0,0 +1,120 @@
+package org.txm.importer.doc
+
+import org.txm.utils.NameSpaceXmlNodePrinter
+import org.txm.utils.NameSpaceXmlNodePrinter.NamespaceContext
+import org.txm.utils.io.IOUtils;
+
+import groovy.xml.QName
+import groovy.xml.StreamingMarkupBuilder
+
+class FixBreaksAndAutomaticStyles {
+
+	def nsText = new groovy.xml.Namespace("urn:oasis:names:tc:opendocument:xmlns:text:1.0", 'tei')
+	def sytleAt;
+	def pbAt;
+
+	FixBreaksAndAutomaticStyles() {
+
+	}
+	def befores
+	def afters
+	def parents
+
+	public boolean run(File xmlFile, def befores, def afters, def parents) {
+
+		sytleAt = nsText.get("rend")
+		pbAt = nsText.get("soft-page-break")
+
+
+		def parser = new XmlParser(false, true)
+		def root = parser.parse(xmlFile)
+
+		this.befores = befores
+		this.afters = afters
+		this.parents = parents
+		process(root, null)
+
+
+		//println root.children()
+		File tmp = new File(xmlFile.getParentFile(), "test.txt")
+		def writer = IOUtils.getWriter(tmp, "UTF-8")
+		writer.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+		def printer = new NameSpaceXmlNodePrinter(new PrintWriter(writer))
+		//create namespace context
+		NamespaceContext namespaceContext = new NamespaceContext(printer);
+		namespaceContext.registerNamespacePrefix("office", "urn:oasis:names:tc:opendocument:xmlns:office:1.0");
+		namespaceContext.registerNamespacePrefix("style", "urn:oasis:names:tc:opendocument:xmlns:style:1.0");
+		namespaceContext.registerNamespacePrefix("text", "urn:oasis:names:tc:opendocument:xmlns:text:1.0");
+		namespaceContext.registerNamespacePrefix("table", "urn:oasis:names:tc:opendocument:xmlns:table:1.0");
+		namespaceContext.registerNamespacePrefix("draw", "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0");
+		namespaceContext.registerNamespacePrefix("fo", "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0");
+		namespaceContext.registerNamespacePrefix("xlink", "http://www.w3.org/1999/xlink");
+		namespaceContext.registerNamespacePrefix("dc", "http://purl.org/dc/elements/1.1/");
+		namespaceContext.registerNamespacePrefix("meta", "urn:oasis:names:tc:opendocument:xmlns:meta:1.0");
+		namespaceContext.registerNamespacePrefix("number", "urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0");
+		namespaceContext.registerNamespacePrefix("svg", "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0");
+		namespaceContext.registerNamespacePrefix("chart", "urn:oasis:names:tc:opendocument:xmlns:chart:1.0");
+		namespaceContext.registerNamespacePrefix("dr3d", "urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0");
+		namespaceContext.registerNamespacePrefix("math", "http://www.w3.org/1998/Math/MathML");
+		namespaceContext.registerNamespacePrefix("form", "urn:oasis:names:tc:opendocument:xmlns:form:1.0");
+		namespaceContext.registerNamespacePrefix("script", "urn:oasis:names:tc:opendocument:xmlns:script:1.0");
+		namespaceContext.registerNamespacePrefix("ooo", "http://openoffice.org/2004/office");
+		namespaceContext.registerNamespacePrefix("ooow", "http://openoffice.org/2004/writer");
+		namespaceContext.registerNamespacePrefix("oooc", "http://openoffice.org/2004/calc");
+		namespaceContext.registerNamespacePrefix("dom", "http://www.w3.org/2001/xml-events");
+		namespaceContext.registerNamespacePrefix("xforms", "http://www.w3.org/2002/xforms");
+		namespaceContext.registerNamespacePrefix("xsd", "http://www.w3.org/2001/XMLSchema");
+		namespaceContext.registerNamespacePrefix("xsi", "http://www.w3.org/2001/XMLSchema-instance");
+		namespaceContext.registerNamespacePrefix("rpt", "http://openoffice.org/2005/report");
+		namespaceContext.registerNamespacePrefix("of", "urn:oasis:names:tc:opendocument:xmlns:of:1.2");
+		namespaceContext.registerNamespacePrefix("xhtml", "http://www.w3.org/1999/xhtml");
+		namespaceContext.registerNamespacePrefix("grddl", "http://www.w3.org/2003/g/data-view#");
+		namespaceContext.registerNamespacePrefix("tableooo", "http://openoffice.org/2009/table");
+		namespaceContext.registerNamespacePrefix("field", "urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0");
+		namespaceContext.registerNamespacePrefix("formx", "urn:openoffice:names:experimental:ooxml-odf-interop:xmlns:form:1.0");
+		namespaceContext.registerNamespacePrefix("css3t", "http://www.w3.org/TR/css3-text/");
+
+		printer.setNamespaceAware(true);
+		printer.print(root, namespaceContext)
+		writer.close();
+
+		if (xmlFile.delete()) tmp.renameTo(xmlFile);
+		else println "FixBreaksAndAutomaticStyles: Error: could not replace "+xmlFile
+	}
+
+	public void process(def node, def parent) {
+		if (!(node instanceof groovy.util.Node)) return;
+
+		def list = new ArrayList(node.children())
+		for( def child : list) {
+			process(child, node)
+		}
+		String name = node.name().getLocalPart()
+		if ("p" == name || "div" == name) {
+			//println "P: "+node.attribute("rend")
+			String style = node.attribute("rend")
+			if (befores.contains(style)) {
+				def newChild = new Node(null, "pb")
+				newChild.parent = parent
+				def idx = parent.children().indexOf(node)
+				//println "b idx: $idx"
+				parent.children().add(idx, newChild)
+			} else if (afters.contains(style)) {
+				def newChild = new Node(null, "pb")
+				newChild.parent = parent
+				def idx = parent.children().indexOf(node)
+				//println "a idx: $idx"
+				parent.children().add(idx+1, newChild)
+			}
+			if (parents.containsKey(style)) {
+				//println "parent of "+style+ " = "+parents[style]
+				node.attributes()["rend"] = parents[style]
+			}
+		}
+	}
+
+	public static void main(String[] args) {
+		File xmlFile = new File("/home/mdecorde/xml/temoignages/content.xml")
+		new FixBreaksAndAutomaticStyles().run(xmlFile, ["P1", "P2"], ["P3"], ["P1": "Standard", "P2":"Standard", "P3":"Standard"])
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/docLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/docLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/docLoader.groovy	(revision 321)
@@ -0,0 +1,362 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-04-24 14:30:47 +0200 (mar., 24 avr. 2012) $
+// $LastChangedRevision: 2174 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.doc;
+
+import org.txm.importer.*;
+
+import javax.xml.stream.XMLStreamReader;
+
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.ValidateXml;
+import org.txm.importer.xml.importer;
+import org.txm.importer.xml.compiler;
+import org.txm.objects.*;
+import org.txm.tokenizer.TokenizerClasses;
+import org.txm.utils.*;
+import org.txm.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.i18n.*;
+import org.txm.metadatas.*;
+
+import javax.xml.stream.*;
+
+import org.w3c.dom.Element;
+import org.txm.sw.RemoveTag;
+import org.txm.doc.*;
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "xml/doc/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		TokenizerClasses.loadFromNode(params.getTokenizerElement(params.getCorpusElement()));
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir, "txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+String textSortAttribute = null; // a property in metadata.csv file
+String paginationElement = "pb";
+boolean normalizeMetadata = true;
+
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+
+if (txmDir.listFiles() == null) {
+	println "No file to process"
+	return;
+}
+
+def srcfiles = [];
+for (File f : srcDir.listFiles()) {
+	String name = f.getName().toLowerCase();
+	if (name.endsWith(".properties")) continue;
+	if (name.equals("import.xml")) continue;
+	if (name.endsWith(".csv")) continue;
+	if (f.isDirectory()) continue;
+	if (f.isHidden()) continue;
+	if (name.endsWith(".doc") || name.endsWith(".docx") || name.endsWith(".odt") || name.endsWith(".rtf")) {
+		srcfiles.add(f)
+	}
+}
+
+if (srcfiles.size() == null) {
+	println "No file to process"
+	return;
+}
+
+// convert to TEI
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "CREATING TEI FILES")
+println "-- CONVERTER - Converting source files"
+File xsldir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "xsl")
+if (!DocumentToTei.processFiles(srcfiles, txmDir, xsldir)) {
+	println "Abord import"
+	return;
+}
+println ""
+
+// move data folders and build css file
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "RETRIEVE STYLES")
+println "Retrieving data folders and style files"
+println txmDir.listFiles()
+File docfiles = new File(binDir, "docfiles");
+docfiles.mkdir()
+for (File infile : txmDir.listFiles()) {
+	if (infile.isDirectory()) {
+		File unzipDir = new File(docfiles, infile.getName())
+		infile.renameTo(unzipDir)
+
+		//println "zipdir "+unzipDir
+		StylesToCSS converter = new StylesToCSS(unzipDir);
+		if (!converter.process(new File(unzipDir, "style.css"))) {
+			println "WARNING: Failed to build css file of $unzipDir"
+		}
+		// and get the soft page breaks and styles parents
+		def parentStyles = converter.parentStyles;
+		def beforebreaks = converter.beforebreaks
+		def afterbreaks = converter.afterbreaks
+
+		//println "BEFORES: "+beforebreaks
+		//println "AFTERS: "+afterbreaks
+		//println "PARENTS: "+parentStyles
+
+		// se servir de ça pour insérer <pb/> et remplacer styles automatiques
+		File xmlFile = new File(txmDir, unzipDir.getName().substring(6))
+		//println "PATCH : $xmlFile"
+		new FixBreaksAndAutomaticStyles().run(xmlFile, beforebreaks, afterbreaks, parentStyles)
+	}
+}
+
+// set new Root
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FIXING TEI")
+println "Setting new XML root element"
+for (File infile : txmDir.listFiles()) {
+	if (infile.isHidden() || infile.isDirectory() || infile.getName().startsWith(".")) continue;
+	print "."
+	if (!SetNewXmlDocumentRoot.process(infile, "//tei:text")) {
+		println "Failed to reset XML root $infile"
+		return
+	}
+}
+println ""
+
+// filtering
+println "Filtering XML files with xpaths: [//tei:term]"
+def xpaths = params.getExcludeXpaths()
+if (xpaths == null) xpaths = ["//tei:term"]
+else xpaths << "//tei:term"
+
+if (xpaths != null) {
+	println "Filtering XML files with xpaths: $xpaths"
+	for (File infile : txmDir.listFiles()) {
+		if (infile.isHidden() || infile.isDirectory() || infile.getName().startsWith(".")) continue;
+		print "."
+		if (!RemoveTag.xpath(infile, xpaths)) {
+			println "Failed to filter $infile"
+			return
+		}
+	}
+	println ""
+}
+
+//get metadata values from CSV
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "READ METADATA")
+Metadatas metadatas; // text metadata
+//println "Trying to read metadata from: "+allmetadatasfile
+if (allmetadatasfile.exists()) {
+	File copy = new File(binDir, "metadata.csv")
+	if (!FileCopy.copy(allmetadatasfile, copy)) {
+		println "Error: could not create a copy of the metadata.csv file "+allmetadatasfile.getAbsoluteFile();
+		return;
+	}
+	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+} else {
+	println "no $allmetadatasfile metadata file found."
+}
+
+for (File infile : txmDir.listFiles()) {
+	if (infile.isHidden()) infile.delete();
+	if (infile.getName().startsWith(".")) infile.delete();
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(5, "IMPORTER")
+println "-- IMPORTER - Reading source files"
+def imp = new importer();
+imp.doValidation(true) // change this to not validate xml
+imp.doTokenize(true) // change this, to not tokenize xml
+if (!imp.run(srcDir, binDir, txmDir, basename, "", lang)) {
+	println "import process stopped";
+	return;
+}
+
+if (metadatas != null) {
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA")
+	println "-- INJECTING METADATA - from csv file: "+allmetadatasfile
+	
+	println("Injecting metadata: "+metadatas.getHeadersList()+" in texts of directory "+new File(binDir,"txm"))
+	for (File infile : txmDir.listFiles()) {
+		if (infile.isHidden() || infile.isDirectory() || infile.getName().startsWith(".")) continue;
+		print "."
+
+		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {
+			outfile.delete();
+		} else {
+			if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+			if (!infile.exists()) {
+				println "Error: could not replace $infile by $outfile"
+				return false;
+			}
+		}
+	}
+	println ""
+}
+
+List<File> files = txmDir.listFiles()
+if (files == null || files.size() == 0) {
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = true;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if(debug) c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setOptions(textSortAttribute, normalizeMetadata);
+c.setAnnotationSuccess(annotationSuccess)
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname, null, srcfiles, metadatas)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(25, "EDITION")
+
+	File outdir = new File(binDir,"/HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+	def second = 0
+
+	println "Paginating texts: "
+
+	for (File txmFile : filelist) {
+		if (txmFile.isHidden() || txmFile.isDirectory() || txmFile.getName().startsWith(".")) continue;
+		print "."
+
+		String txtname = txmFile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(corpusElem, txtname, txmFile);
+
+		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, "pb");
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+
+	for (File unzipDir : docfiles.listFiles()) {
+		File css = new File(unzipDir, "style.css")
+		String textname = unzipDir.getName();
+		textname = textname.substring(6, textname.lastIndexOf(".")) // remove "files-" and ".xml"
+		File newcss = new File(outdir, textname+".css")
+		css.renameTo(newcss)
+	}
+	File doc = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "css/doc.css")
+	doc_copy = new File(outdir, "doc.css")
+	FileCopy.copy(doc, doc_copy);
+
+
+	// copy images : images externaly linked must be added manually in the html/CORPUS/default directory
+	println "Copying internal images..."
+	for (File txmFile : filelist) {
+		if (txmFile.isHidden() || txmFile.isDirectory() || txmFile.getName().startsWith(".")) continue;
+		print "."
+		File picturesDirectory = new File(binDir, "docfiles/files-"+txmFile.getName()+"/Pictures")
+		println picturesDirectory
+		if (picturesDirectory.exists()) {
+			File picturesDirectory_copy = new File(outdir, "Pictures")
+			if (!FileCopy.copyFiles(picturesDirectory, picturesDirectory_copy)) {
+				println "Failed to copy $picturesDirectory to $picturesDirectory_copy"
+			}
+		}
+	}
+	println ""
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/pager.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/pager.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/pager.groovy	(revision 321)
@@ -0,0 +1,472 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-04-24 14:30:47 +0200 (mar., 24 avr. 2012) $
+// $LastChangedRevision: 2174 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.doc;
+
+import org.txm.importer.*;
+import org.xml.sax.Attributes;
+import org.txm.importer.filters.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+// TODO: Auto-generated Javadoc
+/** 
+ * Build edition from xml-tei-odt
+ * @author mdecorde
+ * 
+ */
+class pager {
+	List<String> NoSpaceBefore;
+
+	/** The No space after. */
+	List<String> NoSpaceAfter;
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The pagecount. */
+	int pagecount = 0;
+
+	/** The wordmax. */
+	int wordmax = 0;
+
+	/** The basename. */
+	String basename = "";
+	String txtname = "";
+	File outdir;
+
+	/** The wordid. */
+	String wordid;
+
+	/** The first word. */
+	boolean firstWord = true;
+
+	/** The wordvalue. */
+	String wordvalue;
+
+	/** The interpvalue. */
+	String interpvalue;
+
+	/** The lastword. */
+	String lastword = " ";
+
+	/** The wordtype. */
+	String wordtype;
+
+	/** The flagform. */
+	boolean flagform = false;
+
+	/** The flaginterp. */
+	boolean flaginterp = false;
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The writer. */
+	OutputStreamWriter writer;
+
+	/** The multiwriter. */
+	OutputStreamWriter multiwriter = null;
+	
+	/** The pagedWriter. */
+	StaxStackWriter pagedWriter = null;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The pages. */
+	ArrayList<File> pages = new ArrayList<File>();
+
+	/** The idxstart. */
+	ArrayList<String> idxstart = new ArrayList<String>();
+	String editionPage;
+
+	/**
+	 * Instantiates a new pager.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param NoSpaceBefore the no space before
+	 * @param NoSpaceAfter the no space after
+	 * @param max the max
+	 * @param basename the basename
+	 */
+	pager(File infile, File outdir, String txtname, List<String> NoSpaceBefore,
+	List<String> NoSpaceAfter, int max, String basename, String editionPage) {
+		this.editionPage = editionPage;
+		this.basename = basename;
+		this.txtname = txtname;
+		this.outdir = outdir;
+		this.wordmax = max;
+		this.NoSpaceBefore = NoSpaceBefore;
+		this.NoSpaceAfter = NoSpaceAfter;
+		this.url = infile.toURI().toURL();
+		this.infile = infile;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+		process();
+	}
+
+	private def closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+			if (firstWord) { // there was no words
+				this.idxstart.add("w_0")
+				pagedWriter.write("<span id=\"w_0\"/>");
+			}
+
+			pagedWriter.writeEndElements();
+			pagedWriter.close();
+			return tags;
+
+			//			pagedWriter.write("</body>");
+			//			pagedWriter.write("</html>");
+			//			pagedWriter.close();
+		} else {
+			return [];
+		}
+	}
+
+	/**
+	 * Creates the next output.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean createNextOutput()
+	{
+		wordcount = 0;
+		try {
+			def tags = closeMultiWriter()
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")
+			pages.add(outfile)
+			firstWord = true // waiting for next word
+
+			pagedWriter = new StaxStackWriter(outfile , "UTF-8");
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html")
+			pagedWriter.writeEmptyElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"])
+			pagedWriter.writeStartElement("head")
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "href":txtname+".css"])
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "href":"doc.css"])
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(basename.toUpperCase()+" Edition - Page "+pagecount)
+			pagedWriter.writeEndElement() // </title>
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+			pagedWriter.writeStartElements(tags)
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput() {
+		try {
+			return createNextOutput();
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the page files.
+	 *
+	 * @return the page files
+	 */
+	public ArrayList<File> getPageFiles() {
+		return pages;
+	}
+
+	/**
+	 * Gets the idx.
+	 *
+	 * @return the idx
+	 */
+	public ArrayList<String> getIdx() {
+		return idxstart;
+	}
+
+	void gotoText() {
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.END_ELEMENT:
+					if(parser.getLocalName() == "teiHeader")
+						return;
+			}
+		}
+	}
+	/**
+	 * Process.
+	 */
+	void process() {
+		try {
+		String localname = "";
+		String figurerend = ""
+		boolean unordered = false;
+		createNextOutput();
+		
+		gotoText();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "textunit":
+							if (parser.getAttributeValue(null,"id") != null) {
+								pagedWriter.writeStartElement("h3")
+								pagedWriter.writeCharacters(parser.getAttributeValue(null,"id"))
+								pagedWriter.writeEndElement() // h3
+							}
+							pagedWriter.writeStartElement("table")
+							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+								pagedWriter.writeStartElement("tr")
+								pagedWriter.writeStartElement("td")
+								pagedWriter.writeCharacters(parser.getAttributeLocalName(i))
+								pagedWriter.writeEndElement() // td
+								pagedWriter.writeStartElement("td")
+								pagedWriter.writeCharacters(parser.getAttributeValue(i))
+								pagedWriter.writeEndElement() // td
+								pagedWriter.writeEndElement() // tr
+							}
+							pagedWriter.writeEndElement() // table
+							pagedWriter.writeEmptyElement("br")
+							break;
+						case "head":
+							String rend = parser.getAttributeValue(null, "rend")
+							if (rend == null) rend = "normal"
+							pagedWriter.writeStartElement("h2", ["class":rend])
+							break;
+						case "figure":
+							figurerend = parser.getAttributeValue(null, "rend")
+							break;
+						case "graphic":
+							String url = parser.getAttributeValue(null, "url")
+							if (url.startsWith("..//")) url = url.substring(4);
+							else if (url.startsWith("../")) url = url.substring(3);
+
+							if (url != null) { 
+								pagedWriter.writeStartElement("div")
+								pagedWriter.writeEmptyElement("img", ["class":figurerend, "src":url])
+								pagedWriter.writeEndElement() // div
+							}
+							figurerend = "";
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							String rend = parser.getAttributeValue(null, "rend")
+							if (rend == null) rend = "normal"
+							pagedWriter.writeStartElement("p", ["class":rend])
+							break;
+						//case "pb":
+						case editionPage:
+							createNextOutput();
+							wordcount=0;
+							if (parser.getAttributeValue(null,"n") != null) {
+								pagedWriter.writeStartElement("p", ["style":"color:red", "align":"center"])
+								pagedWriter.writeCharacters("- "+parser.getAttributeValue(null,"n")+" -")
+								pagedWriter.writeEndElement() // p
+							}
+							break;
+						case "table":
+							pagedWriter.writeStartElement("table", ["class":parser.getAttributeValue(null, "rend")])
+							break;
+						case "row":
+							pagedWriter.writeStartElement("tr")
+							break;
+						case "cell":
+							pagedWriter.writeStartElement("td")
+							break;
+						case "lb":
+						case "br":
+							pagedWriter.writeEmptyElement("br")
+							break;
+						case "w":
+							wordid = parser.getAttributeValue(null,"id");
+
+							if (firstWord) {
+								firstWord = false;
+								this.idxstart.add(wordid);
+							}
+							wordcount++;
+
+							break;
+						case "list":
+							String type = parser.getAttributeValue(null,"type");
+							if ("unordered" == type) {
+								unordered = true;
+								pagedWriter.writeStartElement("ul")
+							} else {
+								unordered = false
+								pagedWriter.writeStartElement("ol")
+							}
+							break
+						case "item":
+							pagedWriter.writeStartElement("li")
+							break;
+						case "hi":
+							pagedWriter.writeStartElement("b")
+							break;
+						case "emph":
+							pagedWriter.writeStartElement("i")
+							break;
+						case "ana":
+							flaginterp=true;
+							interpvalue+=" "+parser.getAttributeValue(null,"type").substring(1)+":"
+							break;
+
+						case "form":
+							wordvalue="";
+							interpvalue ="";
+							flagform=true;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					switch(localname)
+					{
+						case "head":
+							pagedWriter.writeEndElement(); // h2
+							break;
+						case "lg":
+						case "p":
+						case "q":
+							pagedWriter.writeEndElement(); // p
+							if (wordcount >= wordmax) {
+								createNextOutput();
+							}
+							break;
+						case "list":
+							pagedWriter.writeEndElement(); // ul or ol
+							break
+						case "item":
+							pagedWriter.writeEndElement(); // li
+							break;
+						case "hi":
+							pagedWriter.writeEndElement(); // b
+							break;
+						case "emph":
+							pagedWriter.writeEndElement(); // i
+							break;
+						case "table":
+							pagedWriter.writeEndElement(); // table
+							break;
+						case "row":
+							pagedWriter.writeEndElement(); // tr
+							break;
+						case "cell":
+							pagedWriter.writeEndElement(); // td
+							break;
+						case "form":
+							flagform = false
+							break;
+						case "ana":
+							flaginterp = false
+							break;
+						case "w":
+							int l = lastword.length();
+							String endOfLastWord = "";
+							if (l > 0)
+								endOfLastWord = lastword.subSequence(l-1, l);
+
+							if (interpvalue != null)
+								interpvalue = interpvalue;
+
+							if (NoSpaceBefore.contains(wordvalue) ||
+								NoSpaceAfter.contains(lastword) ||
+								wordvalue.startsWith("-") ||
+								NoSpaceAfter.contains(endOfLastWord))
+							{
+								// multiwriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+							} else {
+								// multiwriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");
+								pagedWriter.writeCharacters(" ");
+							}
+							pagedWriter.writeStartElement("span", ["title":interpvalue, "id":wordid]);
+							pagedWriter.writeCharacters(wordvalue);
+							pagedWriter.writeEndElement() // span
+							lastword=wordvalue;
+							break;
+					}
+					break;
+				case XMLStreamConstants.CHARACTERS:
+					if (flagform)
+						if (parser.getText().length() > 0)
+							wordvalue+=(parser.getText());
+					if (flaginterp)
+						if (parser.getText().length() > 0)
+							interpvalue+=(parser.getText());
+					break;
+			}
+		}
+		//writer.write("</body>");
+		//writer.write("</html>");
+		//writer.close();
+		closeMultiWriter();
+	} catch(Exception e) { org.txm.utils.logger.Log.printStackTrace(e); pagedWriter.close(); }
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/ODTMovePicturesFiles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/ODTMovePicturesFiles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/ODTMovePicturesFiles.groovy	(revision 321)
@@ -0,0 +1,15 @@
+package org.txm.importer.doc
+
+File dir = new File("/home/mdecorde/TXM/corpora/temoignages/docfiles/")
+File outdir = new File("/home/mdecorde/TXM/corpora/temoignages/HTML/TEMOIGNAGES/default/Pictures")
+outdir.deleteDir()
+outdir.mkdir()
+
+for(def subdir : dir.listFiles()) {
+	if (subdir.isDirectory()) {
+		File picturesDir = new File(subdir, "Pictures")	
+		for (def img : picturesDir.listFiles()) {
+			img.renameTo(new File(outdir, img.getName()))
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/StylesToCSS.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/StylesToCSS.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/StylesToCSS.groovy	(revision 321)
@@ -0,0 +1,194 @@
+package org.txm.importer.doc
+
+class StylesToCSS {
+
+	File cssFile, styleFile, contentFile;
+	def rootStyles, rootContent;
+
+	def styles = [:]
+	def translatedStyles = [:]
+	
+	def beforebreaks = []
+	def afterbreaks = []
+	def parentStyles = [:]
+
+	public StylesToCSS(File unzipDir) {
+		this.styleFile= new File(unzipDir, "styles.xml");
+		this.contentFile= new File(unzipDir, "content.xml");
+		rootStyles = new XmlSlurper().parse(styleFile);
+		rootContent = new XmlSlurper().parse(contentFile);
+	}
+
+	public boolean process(File css) {
+		//println "StylesToCSS: "+css
+		this.cssFile = cssFile;
+		
+		//INDEXING STYLES BY ".NAME_FAMILY"
+		// for each style of styles.xml
+		for( def style : rootStyles.styles.style) {
+			String family = ""+style.@family;
+			String name = ""+style.@name
+			if ("paragraph".equals(family)) {
+				styles[name+"_"+family] = style
+			} else if ("text".equals(family)) {
+				if (!name.startsWith("WW8Num"))
+					styles[name+"_"+family] = style
+			} else if ("graphic".equals(family)) {
+				styles[name+"_"+family] = style
+			}
+		}
+		
+		// for each automatic style of content.xml
+		for( def style : rootContent['automatic-styles'].style) {
+			//println "style: "+style.@name
+			String family = ""+style.@family;
+			String name = ""+style.@name
+			String parent = ""+style['@parent-style-name'] // get parent style name
+			String breakpage = ""; // get soft page break
+			for (def pproperty : style['paragraph-properties']) {
+				//println " paragraph-properties: "+pproperty['@break-before']
+				//println " paragraph-properties: "+pproperty['@break-after']
+				if ("page" == pproperty['@break-before'].toString())
+					beforebreaks << name
+				else if ("page" == pproperty['@break-after'].toString())
+					afterbreaks << name
+			}
+			
+			parentStyles[name] = parent
+			
+			//println "auto $name $family"
+			if ("paragraph".equals(family)) {
+				styles[name+"_"+family] = style
+			} else if ("text".equals(family)) {
+				if (!name.startsWith("WW8Num"))
+					styles[name+"_"+family] = style
+			} else if ("graphic".equals(family)) {
+				styles[name+"_"+family] = style
+			}
+		}
+		
+		// for each default style of styles.xml
+		for( def style : rootStyles.styles['default-style']) {
+			String family = ""+style.@family;
+			String name = ""+style.@name
+			if ("paragraph".equals(family)) {
+				styles[name+"_"+family] = style
+			}
+		}
+		//println styles.keySet()
+		
+		//BUILDING CSS
+		// build styles.xml css
+		for( def style : rootStyles.styles.style) {
+			String name = ""+style.@name
+			if (!name.startsWith("WW8Num")) buildStyle(style);
+		}
+		
+		// for each default style of styles.xml
+		for( def style : rootStyles.styles['default-style']) {
+			String family = ""+style.@family;
+			String name = ""+style.@name
+			if ("paragraph".equals(family)) {
+				buildStyle(style)
+			}
+		}
+		
+		// build content.xml css
+		for( def style : rootContent['automatic-styles'].style) {
+			String name = ""+style.@name
+			if (!name.startsWith("WW8Num")) buildStyle(style);
+		}
+		
+		def writer = css.newPrintWriter("UTF-8")
+		def keys = new ArrayList<String>(translatedStyles.keySet());
+		keys.sort()
+		//println keys
+		for (String name : keys) {
+			if (name.equals("")) {
+				writer.println("body{\n")
+			} else {
+				writer.println(".$name{")
+			}
+			def props = translatedStyles[name]
+			for (def key : props.keySet())
+				writer.println(""+key+":"+props[key]+";")
+			writer.println("}\n")
+		}
+		writer.close()
+
+		return true;
+	}
+	
+	private void buildDefaultStyle(def style) {
+		
+	}
+
+	private void buildStyle (def style) {
+		
+		StringBuffer str = new StringBuffer();
+		String name = style.@name
+		//if (name.length() == 1) name = ".body"
+		if (translatedStyles.containsKey(name)) return
+		def props = [:];
+		
+		String family = ""+style.@family
+		String parentname = style['@parent-style-name']
+		if ( parentname != null && parentname.length() > 0) {
+			String fullparentname = parentname+"_"+family
+			//println "build parent: "+parentname+" for $name"
+			buildStyle(styles[fullparentname]); // build its parent if any
+			def parentprops = translatedStyles[parentname]
+			for (def key : parentprops.keySet())
+				props[key] = parentprops[key]
+		}
+
+		//writer.println(".$name {")
+		for (def property : style['paragraph-properties']) {
+			for (def attr : property.attributes()) {
+				//println attr.getClass()
+				switch (attr.getKey()) {
+					case "text-align": props[attr.getKey()] = attr.getValue(); break;
+					case "margin": props[attr.getKey()] = attr.getValue(); break;
+					case "margin-left": props[attr.getKey()] = attr.getValue(); break;
+					case "margin-right": props[attr.getKey()] = attr.getValue(); break;
+					case "margin-top": props[attr.getKey()] = attr.getValue(); break;
+					case "margin-bottom": props[attr.getKey()] = attr.getValue(); break;
+					case "padding": props[attr.getKey()] = attr.getValue(); break;
+					case "padding-left": props[attr.getKey()] = attr.getValue(); break;
+					case "padding-right": props[attr.getKey()] = attr.getValue(); break;
+					case "padding-top": props[attr.getKey()] = attr.getValue(); break;
+					case "padding-bottom": props[attr.getKey()] = attr.getValue(); break;
+					//case "writing-mode": println(attr); break;
+				}
+			}
+			break;
+		}
+
+		for (def property : style['text-properties']) {
+			//println property.attributes()
+			for (def attr : property.attributes()) {
+				switch (attr.getKey()) {
+					//case "fo:language": println(attr); break;
+					case "font-size": props[attr.getKey()] = attr.getValue(); break;
+					case "font-name": props["font-family"] = "\""+attr.getValue()+"\""; break;
+					case "font-style": props[attr.getKey()] = attr.getValue(); break;
+					case "font-weight": props[attr.getKey()] = attr.getValue(); break;
+					//case "text-position": str.append(""+attr+"\n"); break;
+					case "color": props[attr.getKey()] = attr.getValue(); break;
+				}
+			}
+			break;
+		}
+				
+		translatedStyles[name] = props;
+		//println "builded $name"
+	}
+
+	static main(args) {
+		File unzipDir = new File("/home/mdecorde/xml/txmrefman/TEI/files-Manuel de Référence TXM 0.6_FR.odt.xml")
+		File cssFile = new File("/home/mdecorde/xml/txmrefman/TEI/files-Manuel de Référence TXM 0.6_FR.odt.xml", "style.css")
+
+		StylesToCSS converter = new StylesToCSS(unzipDir);
+		println converter.process(cssFile);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/package.html	(revision 321)
@@ -0,0 +1,90 @@
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<title>IMPORT WORD-ODT</title>
+</head>
+<body>
+<h3>IMPORT WORD-ODT </h3>
+
+<h4>Processus :</h4> 
+Pour l'instant, l'import WORD-ODT repose grandement sur l'import XML/w
+<ul>
+ <li>transformation en ODT de tous les documents : doc et docx</li>
+ <li>transformation en TEI avec XSL de Ratz</li>
+ <li>retire &lt;term> et &lt;teiHeader> (de toute façon vide)</li>
+ <li>import XML/w</li>
+</ul>
+
+<h4>Sauts de pages</h4>
+ Les pages sont paginées en fonction des soft-page-break d'ODT<br/>
+ repérage des &lt;xsl:template match="text:soft-page-break"><br/>
+
+<h4>Images</h4>
+ Le chemin des images de la source sont conservés jusqu'en l'édition dans l'attribut "url" des balises "graphic".<br/>
+ Il faut placer le dossier "Pictures" à coté des pages d'édition.<br/>
+ Il n'y a aucune différence dans le codage d'image linkée ou pas.<br/>
+ On peut différentier si une image est linké par son url : <br/>
+<ul>
+  <li>incorporée : "Pictures/xxx", Pictures est un dossier de l'archive ODT</li>
+  <li>linkée : "../xxx"</li>
+  <li>web : "http://" "ftp://" ...</li>
+</ul>
+
+<h4>Tokenisation</h4>
+ Pour cet import, j'ai branché le Tokenizer paramétrable en variables et en tests.<br/>
+ Les mots sont typés à l'import grâce aux paramétrages : protocole, tag, path, number<br/>
+ corrections de bugs de tokenisation du "SimpleXmlTokenizer"<br/>
+
+<h3>MODIFS XSL RATZ</h3>
+<h4>Paragraphes</h4>
+ styles automatiques PN -> prend style parent<br/>
+ erreur xsl avec certains odt<br/>
+ récupération du style parent des styles automatiques<br/>
+
+<h4>Sauts de page</h4>
+ text:soft-page-break -> pb<br/>
+
+<h4>gestion par défaut de certains [[[UNTRANSLATED...]]]</h4>
+Affichage du content des éléments :<br/>
+<ul>
+ <li>text:editing-cycles</li>
+ <li>text:page-count</li>
+ <li>text:word-count</li>
+ <li>text:image-count</li>
+ <li>text:table-count</li>
+ <li>text:modification-date</li>
+ <li>text:modification-time</li>
+ <li>text:sequence-ref</li>
+</ul>
+<h3>A FAIRE</h3>
+
+<h4>Images</h4>
+ Automatiser la déplacement des images lors de l'import pour que les url pointent bien les fichiers.<br/>
+ xxxx-files -> $TXMHOME/corpora/$CORPUS/HTML/default<br/>
+
+<h4>Tokenisation</h4>
+ PB: les mots sont dabord découpés par blanc (ex: pas bon pour <tag attr="" atrt2="">)<br/>
+
+<h4>UNTRANSLATED</h4>
+Faut-il marquer ces infos : <br/>
+<ul>
+ <li>text:editing-cycles</li>
+ <li>text:page-count</li>
+ <li>text:word-count</li>
+ <li>text:image-count</li>
+ <li>text:table-count</li>
+ <li>text:modification-date</li>
+ <li>text:modification-time</li>
+ <li>text:sequence-ref</li>
+</ul>
+<h4>Sauts de page durs</h4>
+ Ils sont codés dans les styles de paragraphe **automatiques**<br/>
+ &lt;style:paragraph-properties fo:break-before="page"/><br/>
+
+<h4>Typage des paragraphes</h4>
+ but atteindre un typage comme celui fait par l'export xHTML d'oo<br/>
+ lire styles.xml et créer une css<br/>
+ Unifier les styles identiques ?<br/>
+ Reclasser les @rend ?<br/>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/doc/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XPathResult.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XPathResult.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XPathResult.groovy	(revision 321)
@@ -0,0 +1,235 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-03-29 09:51:35 +0200 (Tue, 29 Mar 2016) $
+// $LastChangedRevision: 3185 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import org.txm.importer.graal.PersonalNamespaceContext;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import org.w3c.dom.*;
+import org.xml.sax.SAXException;
+
+import javax.xml.bind.helpers.DefaultValidationEventHandler;
+import javax.xml.parsers.*;
+import javax.xml.xpath.*;
+import javax.xml.stream.*;
+import java.net.URL;
+import java.util.Iterator;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class XPathResult.
+ *
+ * @authors ayepdieu, mdecorde, vchabanis
+ * 
+ * return the id of a bfm tag <milestone/>
+ */
+public class XPathResult {
+	
+	/** The doc. */
+	Document doc;
+	XPath xpath;
+
+	/**
+	 * Instantiates a new x path result.
+	 *
+	 * @param file the file
+	 */
+	public XPathResult(File xmlfile) {
+		this(xmlfile, true)
+	}
+	
+	/**
+	 * Instantiates a new x path result.
+	 *
+	 * @param file the file
+	 */
+	public XPathResult(File xmlfile, boolean namespaceAware) {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		factory.setNamespaceAware(namespaceAware); // never forget this!
+		
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		doc = builder.parse(xmlfile);
+		
+		XPathFactory xfactory = XPathFactory.newInstance();
+		xpath = xfactory.newXPath();
+		xpath.setNamespaceContext(new PersonalNamespaceContext());
+	}
+	
+	public def getNodes(String query) {
+		def rnodes = [];
+		
+		XPathExpression expr = xpath.compile(query);
+		Object result = expr.evaluate(doc.getDocumentElement(), XPathConstants.NODESET);
+		
+		NodeList nodes = (NodeList) result;
+		for (int i = 0; i < nodes.getLength(); i++) {
+			rnodes.add(nodes.item(i));
+		}
+		return rnodes;
+	}
+	
+	public String getXpathResponse(String query) {
+		XPathExpression expr = xpath.compile(query);
+		Object result = expr.evaluate(doc.getDocumentElement(), XPathConstants.NODESET);
+		
+		NodeList nodes = (NodeList) result;
+		for (int i = 0; i < nodes.getLength(); i++) {
+			//println nodes.item(i)
+			return (nodes.item(i).getNodeValue());
+		}
+	}
+	
+	public ArrayList<String> getXpathResponses(String query) {
+		ArrayList<String> xresult = new ArrayList<String>();
+		XPathExpression expr = xpath.compile(query);
+		Object result = expr.evaluate(doc.getDocumentElement(), XPathConstants.NODESET);
+		
+		NodeList nodes = (NodeList) result;
+		for (int i = 0; i < nodes.getLength(); i++) {
+			//println nodes.item(i)
+			xresult.add((nodes.item(i).getNodeValue()));
+		}
+		return xresult
+	}
+	
+	public String getXpathResponse(String query, String devaultValue) {
+		String rez = getXpathResponse(query);
+		if (rez == null)
+			return devaultValue;
+		return rez;
+	}
+	
+	public void close() {
+		xpath = null;
+		doc = null;
+	}
+
+	/**
+
+	 * OBSOLETE VERSION FOR TXM return the node text content given a XPath
+
+	 * "//path.../.../@attr"
+	 * 
+	 * @param path
+	 * @param attr
+	 *            null or "" return the text of the tag targeted
+	 * @return
+	 */
+	/*
+	 * public String getXpathResponse(String path, String attr) throws
+	 * ParserConfigurationException, SAXException, IOException,
+	 * XPathExpressionException { String Query = path;
+	 * 
+	 * //println "XPath : "+Query; XPathFactory factory =
+	 * XPathFactory.newInstance(); XPath xpath = factory.newXPath();
+	 * xpath.setNamespaceContext(new PersonalNamespaceContext());
+	 * XPathExpression expr = xpath.compile(Query);
+	 * 
+	 * Object result = expr.evaluate(doc, XPathConstants.NODESET); // don't work
+	 * on windows NodeList nodes = (NodeList) result; //println
+	 * "result xpath : "+nodes.getLength(); for(int i = 0 ; i <
+	 * nodes.getLength() ; i++) { //println
+	 * "item : "+nodes.item(i).getLocalName(); if(attr != null &&
+	 * !attr.equals("")) return
+	 * nodes.item(i).getAttributes().getNamedItem(attr).getNodeValue(); else
+	 * return nodes.item(i).getTextContent();
+	 * 
+	 * } return ""; }
+	 */
+	
+	static public String getXpathResponse(File xmlfile, String query, String devaultValue) {
+		String rez = getXpathResponse(xmlfile, query);
+		if (rez == null)
+			return devaultValue;
+		return rez;
+	}
+	
+	static public String getXpathResponse(File xmlfile, String query, String devaultValue, boolean namespaceAware) {
+		String rez = getXpathResponse(xmlfile, query, namespaceAware);
+		if (rez == null)
+			return devaultValue;
+		return rez;
+	}
+
+	/**
+	 * Gets the xpath response.
+	 *
+	 * @param xmlfile the xmlfile
+	 * @param query the query
+	 * @return the xpath response
+	 */
+	static public String getXpathResponse(File xmlfile, String query) {
+		return getXpathResponse(xmlfile, query, true);
+	}
+	
+	/**
+	 * Gets the xpath response.
+	 *
+	 * @param xmlfile the xmlfile
+	 * @param query the query
+	 * @return the xpath response
+	 */
+	static public String getXpathResponse(File xmlfile, String query, boolean namespaceAware) {
+		XPathResult result = new XPathResult(xmlfile);
+		return result.getXpathResponse(query);
+//		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+//		factory.setNamespaceAware(namespaceAware); // never forget this!
+//		
+//		DocumentBuilder builder = factory.newDocumentBuilder();
+//		Document doc = builder.parse(xmlfile);
+//		
+//		XPathFactory xfactory = XPathFactory.newInstance();
+//		XPath xpath = xfactory.newXPath();
+//		xpath.setNamespaceContext(new PersonalNamespaceContext());
+//		
+//		XPathExpression expr = xpath.compile(query);
+//		Object result = expr.evaluate(doc, XPathConstants.NODESET);
+//		
+//		NodeList nodes = (NodeList) result;
+//		println "size: "+result.getLength()
+//		for (int i = 0; i < nodes.getLength(); i++) {
+//			return (nodes.item(i).getNodeValue());
+//		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		println("main of XPathResult")
+		File xmlFile = new File("/home/mdecorde/TXM/corpora/superphenix/txm/SUPERPHENIX/CreysSuper_04_0175.xml")
+		String value = XPathResult.getXpathResponse(xmlFile, "TEI/text/@date");
+		println "value: $value"
+		//value = XPathResult.getXpathResponse(new File("/home/mdecorde/xml/bfm/strasb.xml"), "tei:TEI/tei:teiHeader/tei:revisionDesc/tei:change[contains(.,'étiquetage morpho')]");
+		//println "value: $value"
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HTMLIndexer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HTMLIndexer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/HTMLIndexer.groovy	(revision 321)
@@ -0,0 +1,170 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.importer
+
+import java.util.Collections;
+import java.io.File;
+
+import org.txm.utils.*;
+import org.txm.utils.io.FileCopy;
+
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.io.File;
+
+import javax.xml.stream.*;
+
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class HTMLIndexer.
+ */
+class HTMLIndexer {
+	
+	/** The idxprefix. */
+	static String idxprefix = "IDX-";
+	
+	/** The index. */
+	HashMap<String, ArrayList<String>> index = new HashMap<String, ArrayList<String>>();
+	
+	/**
+	 * Process html dir.
+	 *
+	 * @param htmlDirectory the html directory
+	 * @return true, if successful
+	 */
+	private boolean processHTMLDir(File htmlDirectory)
+	{
+		ArrayList<File> htmlfiles = DeleteDir.scanDirectory(htmlDirectory, true)
+		Collections.sort(htmlfiles);
+		
+		for(File htmlFile : htmlfiles)//get all indexes
+		{
+			if(htmlFile.getName().endsWith(".html"))
+				processHTMLFile(htmlFile);
+		}
+		
+		ArrayList<String> tokens = new ArrayList<String>(index.keySet());
+		Collections.sort(tokens);
+		
+		//fix doubles like étiquette&étiquettes
+		for(int i = 0 ; i < tokens.size() ; i++)
+		{
+			String t1 = tokens.get(i);
+			String t2 = tokens.get(i+1);
+			if(t1.equals(t2.substring(0, t2.length() -1)))
+			{
+				tokens.remove(i+1);
+				index.get(t1).addAll(index.get(t2));
+				//i--;
+			}
+		}
+		
+		tokens = new ArrayList<String>(index.keySet());
+		for(String token : tokens)
+		{
+			println("Token: "+token);
+			println(index.get(token));
+		}
+	}
+	
+	/**
+	 * Process html file.
+	 *
+	 * @param htmlFile the html file
+	 * @return true, if successful
+	 */
+	private boolean processHTMLFile(File htmlFile)
+	{
+		
+		
+		String lasttoken;
+		String page;
+		
+		try
+		{
+			URL url = htmlFile.toURI().toURL();
+			println "process html file "+url;
+			def inputData = url.openStream();
+			def factory = XMLInputFactory.newInstance();
+			//factory.setXMLResolver resolver
+			//factory.setProperty(factory.IS_VALIDATING, false)
+			factory.setProperty("javax.xml.stream.supportDTD", false);
+			factory.setProperty("javax.xml.stream.isReplacingEntityReferences", false);
+			
+			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+			{
+				//println "parse"
+				switch (event) 
+				{
+					case XMLStreamConstants.START_ELEMENT:
+						//println "elem "+parser.getLocalName()
+						if(parser.getLocalName() == "div")
+							if(parser.getAttributeValue(null, "id") != null && parser.getAttributeValue(null, "id").startsWith("index-body"))
+								page = parser.getAttributeValue(null, "id")
+								
+						String id = parser.getAttributeValue(null, "id");
+						if(id != null && id.startsWith(idxprefix))
+						{
+							if(!index.containsKey(lasttoken))
+								index.put(lasttoken, new ArrayList<String>());
+							index.get(lasttoken).add(htmlFile.getName()+"#"+id)
+						}
+					
+						break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						String text = parser.getText().trim();
+						if(text.length() > 0)
+						{
+							def texts = text.split(" ");
+							lasttoken = texts[texts.size()-1];
+							if(lasttoken.endsWith("."))
+								lasttoken = lasttoken.substring(0, lasttoken.length() -1)
+						}
+				}
+			}
+			
+		}
+		catch(Exception e){println("File "+htmlFile+"\n"+e); return false;}
+		return true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File htmlDirectory = new File("/home/mdecorde/xml/html")
+		new HTMLIndexer().processHTMLDir(htmlDirectory);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Hyperbase2Xml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Hyperbase2Xml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/Hyperbase2Xml.groovy	(revision 321)
@@ -0,0 +1,185 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-12-16 17:28:35 +0100 (Fri, 16 Dec 2016) $
+// $LastChangedRevision: 3377 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer
+
+import org.txm.utils.CharsetDetector
+import org.txm.utils.i18n.DetectBOM;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Create a xml file per text declared in a Hyperbase file (old format).
+ *
+ * @author mdecorde
+ */
+class Hyperbase2Xml {
+	
+	/**
+	 * Run.
+	 *
+	 * @param infile the hyperbase file (old format)
+	 * @param outdir where the xml files are created
+	 * @param encoding the encoding of the hyperbase file
+	 * @return true, if successful
+	 */
+	public boolean run(File infile, File outdir, String encoding)
+	{
+		if (encoding == "??") {
+			encoding = new CharsetDetector(infile).getEncoding();
+			println "Guessing encoding of $infile : $encoding"
+		}
+		outdir.mkdir();
+		
+		String textname = null
+		String partname = null;
+		int textcount = 1;
+		def input = new FileInputStream(infile)
+		Reader reader = new InputStreamReader(input , encoding);
+		DetectBOM bomdetector = new DetectBOM(infile);
+		for (int ibom = 0 ; ibom < bomdetector.getBOMSize() ; ibom++) input.read()
+		
+		XMLOutputFactory factory = X	MLOutputFactory.newInstance();
+		BufferedOutputStream output = null;
+		XMLStreamWriter writer = null;
+		
+		boolean firsttext=true;
+		
+		String line = reader.readLine();
+		while(line != null)
+		{
+			line = CleanFile.clean(line);
+			if((line.startsWith("&&&"))) // text declaration &&& longtitle, title, shorttitle &&&
+			{
+				if(partname != null) // close previous part
+				{
+					//println("close last part")
+					writer.writeEndElement();
+					partname= null;
+				}
+				if(textname != null) // close previous text
+				{
+					//println("close text "+textname)
+					writer.writeEndElement();
+					writer.close();
+					output.close();
+				}
+				textname = line.substring(3, line.length()-3);
+				if(textname.contains(","))
+				{
+					String[] textnames = textname.split(",");
+					if(textnames.length > 1)
+						textname = textnames[1].trim();
+					
+				}
+				// create a new text
+				output = new BufferedOutputStream(new FileOutputStream(new File(outdir,textname+".xml")))
+				writer = factory.createXMLStreamWriter(output, "UTF-8");
+				writer.writeStartElement("text");
+				writer.writeAttribute("id",textname);
+				println "text : "+new File(outdir,textname+".xml");
+			}
+			else if((line.startsWith("%%%%"))) // an other type of text declaration
+			{
+				if(partname != null)
+				{
+					//println("close last part")
+					writer.writeEndElement();
+					partname= null;
+				}
+				if(textname != null)
+				{
+					//println("close text "+textname)
+					writer.writeEndElement();
+					writer.close();
+					output.close();
+				}
+				textname = infile.getName().substring(0, infile.getName().length()-4)+(textcount++);
+				output =  new BufferedOutputStream(new FileOutputStream(new File(outdir,textname+".xml")))
+				writer = factory.createXMLStreamWriter(output, "UTF-8");
+				writer.writeStartElement("text");
+				writer.writeAttribute("id",textname);
+				println "text : "+new File(outdir,textname+".xml");
+			}
+			else if((line.startsWith("\$"))) // paragraph declaration
+			{
+				if(partname != null)
+				{
+					//println("close part "+partname)
+					writer.writeEndElement();
+				}
+				partname = line.substring(1);
+				
+				writer.writeStartElement("p");
+				writer.writeAttribute("id",partname);
+			}
+			else // brut text > write chars
+			{
+				if(writer != null)
+					writer.writeCharacters(line+"\n");
+			}
+			line = reader.readLine();
+		}
+		
+		if(partname != null)
+		{
+			//println("close last part")
+			writer.writeEndElement();
+		}
+		if(writer != null)
+			writer.writeEndElement();
+		
+		reader.close();
+		if(output != null)
+			output.close();
+		
+		return true;
+	}
+	
+	/**
+	 * Main.
+	 *
+	 * @param args the args
+	 */
+	static main(args) 
+	{
+		File infile = new File(System.getProperty("user.home")+"/xml/presdiscfra/src/PresDiscFranc");
+		File outfile = new File(System.getProperty("user.home")+"/xml/presdiscfra/split/");
+		new Hyperbase2Xml().run(infile, outfile,"ISO-8859-1");
+		println "done"
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ImportXmlTag.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ImportXmlTag.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ImportXmlTag.groovy	(revision 321)
@@ -0,0 +1,216 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class ImportXmlTag.
+ *
+ * @author mdecorde
+ */
+public class ImportXmlTag {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The output. */
+	private def output;
+	
+	/**
+	 * Instantiates a new import xml tag.
+	 *
+	 * @param url the url
+	 */
+	public ImportXmlTag(URL url){
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+			
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile){
+		try {
+			File f = outfile;
+			output = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+			
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/** The maxprof. */
+	int maxprof = 0;
+	
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile){
+		LinkedList<Integer> writechilds = new LinkedList<Integer>();
+		writechilds.add(0);
+		String prof = "";
+		int wordid =0
+		
+		def children = 0;
+		if(createOutput(outfile)){
+			
+			String localname = "";
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+					
+					String prefix = parser.getPrefix();
+					if(prefix == null || prefix == "")
+						prefix = "none";
+					
+					switch (event) {
+					
+						case XMLStreamConstants.START_ELEMENT:
+							localname = parser.getLocalName();
+							if (writechilds.getLast() == 0) {
+								children++;
+								output.write(prof+"<children n="+children+">\n");
+							}
+							
+							int cpt = writechilds.pop();
+							cpt++;
+							writechilds.add(cpt);
+
+							prof+=" ";
+							maxprof = Math.max(maxprof,prof.length());
+							output.write(prof+localname+"\t"+(wordid++)+"\t"+prefix+"\t"+prof.length()+"\t"+parser.getAttributeCount()+"\n");
+
+							writechilds.add(0);
+							break;	
+						
+						case XMLStreamConstants.END_ELEMENT:
+							localname = parser.getLocalName();
+							if (writechilds.getLast() > 0) {
+								output.write(prof+"</children>\n");
+							}
+							writechilds.removeLast();
+							prof = prof.subSequence(0, prof.length() -1);
+
+							break;
+						
+						case XMLStreamConstants.CHARACTERS:		
+							break;
+					}
+				}
+				output.write(prof+"</children>\n");
+				output.close();
+				parser.close();
+			}
+			catch (XMLStreamException ex) {
+				System.out.println(ex);
+			}
+			catch (IOException ex) {
+				System.out.println("IOException while parsing " + inputData);
+			}
+		}
+	}
+	
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir+"/wtc/").mkdir();
+		new File(rootDir,"/registry/").mkdir();
+		
+		File srcfile = new File(rootDir+"/src/","roland.xml");
+		File resultfile = new File(rootDir+"/wtc/","roland-xmltag.wtc");
+		println("importXmlTag : "+srcfile+" to : "+resultfile );
+		
+		def builder = new ImportXmlTag(srcfile.toURL());
+		builder.process(resultfile);
+		
+		def cwbLoc ="~/Bureau/textometrie/CWB/cwb-3.0/utils/";//chemin vers executable cqp
+		
+		def inDir = rootDir;
+		def outDir =rootDir+"wtc/";
+		def outDirTxm = rootDir;
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		
+		String[] pAttributes = ["id","prefix","prof","nbattr"];
+		String[] sAttributes = ["children:"+builder.maxprof+"+n"];
+		
+		try {
+			if (System.getProperty("os.name").contains("Windows")) {
+				cwbEn.run(cwbLoc + "cwb-encode.exe", outDirTxm + "data/"+"ROLANDXML", outDir +"roland-xmltag.wtc", outDirTxm + "registry/"+"rolandxml",pAttributes, sAttributes);
+				cwbMa.run(cwbLoc + "cwb-makeall.exe", "ROLANDXML", outDirTxm + "registry");
+			} else {
+				cwbEn.run(cwbLoc + "cwb-encode", outDirTxm + "data/"+"ROLANDXML", outDir +"roland-xmltag.wtc", outDirTxm + "registry/"+"rolandxml",pAttributes, sAttributes);
+				cwbMa.run(cwbLoc + "cwb-makeall", "ROLANDXML", outDirTxm + "registry");
+			}
+		} catch (Exception ex) {System.out.println(ex);}
+		
+		return;
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/quickLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/quickLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/quickLoader.groovy	(revision 321)
@@ -0,0 +1,195 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-05-11 08:38:47 +0200 (Wed, 11 May 2016) $
+// $LastChangedRevision: 3211 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.quick;
+
+import org.txm.*
+import org.txm.importer.xml.pager
+import org.txm.metadatas.*
+import org.txm.objects.*
+import org.txm.scripts.teitxm.*
+import org.txm.utils.i18n.*
+import org.txm.utils.io.FileCopy;
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+def MONITOR;
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	params = new BaseParameters(new File(userDir, "TXM/clipboard/clipboard2/import.xml"))
+	params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:/Program Files/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH, new File("C:/Program Files/TreeTagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH, new File("C:/Users/mdecorde/TXM/treetagger"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(userDir, "TXM"));
+	}
+}
+if (params == null) { println "No parameters file. Aborting"; return;}
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+int wordsPerPage = params.getWordsPerPage("default")
+boolean build_edition = params.getDoEdition("default")
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir, "txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+//get metadata values from CSV
+Metadatas metadatas; // text metadata
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+
+if (allmetadatasfile.exists()) {
+	println "Trying to read metadata from: "+allmetadatasfile
+	File copy = new File(binDir, "metadata.csv")
+	if (!FileCopy.copy(allmetadatasfile, copy)) {
+		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
+		return;
+	}
+	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+} else {
+	println "No metadata file: "+allmetadatasfile
+}
+
+def suffixes = ["txt", "TXT"]
+println "-- IMPORTER - Reading source files with extension "+suffixes
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "IMPORTER - Reading source files with extension "+suffixes)
+if (!new importer().run(srcDir, binDir, txmDir,encoding, suffixes, basename, lang)) {
+	println "Import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA - from csv file: "+allmetadatasfile)
+if (metadatas != null) {
+	println "-- INJECTING METADATA - from csv file: "+allmetadatasfile
+	println("Injecting metadata: "+metadatas.getHeadersList()+" in texts of directory "+txmDir)
+	for (File infile : txmDir.listFiles()) {
+		print "."
+		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+
+		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {
+			outfile.delete();
+		} else {
+			if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+			if (!infile.exists()) {
+				println "Error: could not replace $infile by $outfile"
+				return false;
+			}
+		}
+	}
+	println ""
+}
+
+println "-- ANNOTATE - Running NLP tools"
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE - Running NLP tools")
+boolean annotationSuccess = true;
+if (annotate && !new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = false;
+}
+
+println "-- COMPILING - Building Search Engine indexes"
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "COMPILING - Building Search Engine indexes")
+def c = new compiler(null, "", corpusname, "default");
+//c.setCwbPath(userDir+"/TXM/cwb/bin/")// for developers
+if (metadatas != null)
+	c.setMetadataAttributes(metadatas.getSattributes())
+c.setLang(lang);
+c.setAnnotationSuccess(annotationSuccess)
+if (debug) c.setDebug();
+if (!c.run(binDir, txmDir, corpusname)) {
+	println "Import process stopped";
+	return;
+}
+//println "basename :"+basename;
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+if (build_edition) {
+
+	println "-- EDITION - Building edition"
+	if (MONITOR != null) MONITOR.worked(20, "EDITION - Building edition")
+	
+	File outdir = new File(binDir, "HTML/$corpusname/default/");
+	outdir.mkdirs();
+	List<File> filelist = txmDir.listFiles();
+	Collections.sort(filelist);
+	def second = 0
+
+
+	for (File srcfile : filelist) {
+		print "."
+		String txtname = srcfile.getName();
+		int i = txtname.lastIndexOf(".");
+		if (i > 0) txtname = txtname.substring(0, i);
+
+		List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+		List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+		Element text = params.addText(params.corpora.get(corpusname), txtname, srcfile);
+
+		def ed = new pager(srcfile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, null);
+		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+		for (i = 0 ; i < ed.getPageFiles().size();) {
+			File f = ed.getPageFiles().get(i);
+			String wordid = ed.getIdx().get(i);
+			params.addPage(edition, ""+(++i), wordid);
+		}
+	}
+}
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/compiler.groovy	(revision 321)
@@ -0,0 +1,288 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.quick
+
+import org.txm.Toolbox;
+import org.txm.importer.cwb.CwbEncode
+import org.txm.importer.cwb.CwbMakeAll
+import org.txm.importer.cwb.PatchCwbRegistry;
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.BuildTTSrc;
+import org.txm.scripts.teitxm.*;
+import org.txm.utils.treetagger.TreeTagger;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler {
+	
+	/** The debug. */
+	boolean debug = false;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The dir. */
+	private def dir;
+	
+	/** The output. */
+	private def output;
+	
+	/** The url. */
+	private def url;
+	
+	/** The anahash. */
+	private HashMap<String,String> anahash = new HashMap<String,String>() ;
+	
+	/** The text. */
+	String text="";
+	
+	/** The base. */
+	String base="";
+	
+	/** The project. */
+	String project="";
+	
+	/** The cwb loc. */
+	String cwbLoc;
+	
+	/** The lang. */
+	private String lang ="fr";
+	
+	/**
+	 * initialize.
+	 *
+	 * @param url the url
+	 * @param text the text
+	 * @param base the base
+	 * @param project the project
+	 */
+	
+	public compiler(URL url,String text,String base, String project)
+	{
+		this.text = text
+		this.base = base;
+		this.project = project;
+		this.url = url;
+	}
+	
+	/**
+	 * Sets the debug.
+	 *
+	 * @return the java.lang. object
+	 */
+	public setDebug()
+	{
+		debug =true;
+	}
+	
+	/** The annotation success. */
+	boolean annotationSuccess = false;
+	
+	/**
+	 * Sets the annotation success.
+	 *
+	 * @param val the new annotation success
+	 */
+	public void setAnnotationSuccess(boolean val)
+	{
+		this.annotationSuccess = val;
+	}
+	
+	/** The metadata s attributes. */
+	String metadataSAttributes = "";
+	
+	/**
+	 * Sets the metadata attributes.
+	 *
+	 * @param attrsdecl the new metadata attributes
+	 */
+	public void setMetadataAttributes(String attrsdecl)
+	{
+		metadataSAttributes = attrsdecl;
+	}
+	
+	/**
+	 * set the language of the corpus.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if (!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param f the f
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File f) {
+		try {
+			output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f,f.exists())) , "UTF-8");
+			return true;
+		} catch (Exception e) { 
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String corpusname) 
+	{
+		if (cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+		
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false;
+		}
+		if (!binDir.exists()) {
+			println ("Binary directory does not exists: "+binDir)
+			return false;
+		}
+		
+		File wtcFile = new File(binDir,"wtc/"+corpusname+".wtc");
+		new File(binDir, "wtc").deleteDir();
+		new File(binDir, "wtc").mkdir();
+		new File(binDir, "data").deleteDir();
+		new File(binDir, "data").mkdir();
+		new File(binDir, "registry").mkdir();
+		
+		String textid="";
+		int counttext =0;
+		
+		//start corpus
+		if (createOutput(wtcFile)) {
+			output.write("<txmcorpus lang=\""+lang+"\">\n");
+			output.close();
+		}
+		
+		//1- Transform into WTC file
+		println("Compiling "+txmDir.listFiles().length+" files")
+		XMLTXM2WTC wtcbuilder = null;
+		ArrayList<File> files = txmDir.listFiles();
+		Collections.sort(files);
+		for (File txmfile : files) {
+			print "."
+			wtcbuilder = new XMLTXM2WTC(txmfile.toURI().toURL());
+			String txtname = txmfile.getName().substring(0,txmfile.getName().length()-4);
+			wtcbuilder.setTextInfo(txtname, this.base, this.project);
+
+			wtcbuilder.setBalisesToKeep(["text","s"]);
+			wtcbuilder.setSendToPAttributes(["lb":["n"]]);
+			wtcbuilder.setLang(lang);
+			if (!wtcbuilder.transformFile(wtcFile)) {
+				println("Failed to compile "+txmfile)
+			}
+		}
+		println ""
+		
+		//end corpus
+		if (createOutput(wtcFile)) {
+			output.write("</txmcorpus>\n");
+			output.close();
+		}
+		
+		//2- Import into CWB
+		def outDir = binDir.getAbsolutePath()+"/";
+		
+		CwbEncode cwbEn = new CwbEncode();
+		CwbMakeAll cwbMa = new CwbMakeAll();
+		cwbEn.setDebug(debug);
+		cwbMa.setDebug(debug);
+		
+		if(wtcbuilder == null)
+			return false;
+		
+		List<String> pAttributesList = wtcbuilder.getpAttributs();
+		List<String> sAttributesList = wtcbuilder.getsAttributs();
+		println "pAttrs : "+pAttributesList
+		println "sAttrs : "+sAttributesList
+		String[] pAttributes = pAttributesList.toArray(new String[pAttributesList.size()])
+		String[] sAttributes = sAttributesList.toArray(new String[sAttributesList.size()])
+		
+		try {
+			String regPath = outDir + "/registry/"+corpusname.toLowerCase();
+			cwbEn.run(cwbLoc + "cwb-encode", outDir + "/data/$corpusname", outDir + "/wtc/"+corpusname+".wtc", regPath, pAttributes, sAttributes);
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(cwbLoc + "cwb-makeall", corpusname, outDir + "/registry");
+		} catch (Exception ex) {System.out.println("CWB error: "+ex); return false;}
+		
+		return true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/perrault/txm/");
+		List<File> files = dir.listFiles();
+		new compiler().run(files);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Quick import module : text files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/quick/importer.groovy	(revision 321)
@@ -0,0 +1,291 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-12-16 17:28:35 +0100 (Fri, 16 Dec 2016) $
+// $LastChangedRevision: 3377 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.quick
+
+import javax.xml.stream.*
+
+import org.txm.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+import org.txm.scripts.*
+import org.txm.scripts.teitxm.*
+import org.txm.utils.*
+import org.txm.utils.i18n.DetectBOM
+
+import filters.CutHeader.*
+import filters.FusionHeader.*
+import filters.TagSentences.*
+import filters.Tokeniser.*
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/**
+	 * Run.
+	 *
+	 * @param rootfile the rootfile
+	 * @param encoding the encoding
+	 * @param suffixes the suffixes
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public static boolean run(File srcDir, File binDir, File txmDir, String encoding, List<String> suffixes, String basename, String lang)
+	{
+		//String rootDir = srcDir.getAbsolutePath();// ./xml/quick
+
+		File ptokenizedDir = new File(binDir,"ptokenized");
+		ptokenizedDir.deleteDir();
+		ptokenizedDir.mkdir();
+		File tokenizedDir = new File(binDir,"tokenized");
+		tokenizedDir.deleteDir();
+		tokenizedDir.mkdir();
+
+		List<File> files = null;
+
+		ArrayList<String> milestones = new ArrayList<String>();//the tags who stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+
+		// Build the list of src files
+		List<File> srcfiles = DeleteDir.scanDirectory(srcDir, true)
+		for (int i = 0; i < srcfiles.size(); i++) {
+			File srcfile = srcfiles.get(i);
+			int point = srcfile.getName().indexOf(".");
+			String filename, fileext;
+			if (point >= 1 && point < srcfile.getName().length()) {
+				filename = srcfile.getName().substring(0, point)
+				fileext = srcfile.getName().substring(point+1, srcfile.getName().length());
+			} else {
+				filename = srcfile.getName()
+				fileext = "";
+			}
+
+			if (!suffixes.contains(fileext)) {
+				srcfiles.remove(i);
+				i--;
+			}
+		}
+		
+		if (srcfiles != null && srcfiles.size() == 0) {
+			println "No 'txt' file to process in directory $srcDir"
+			return false;
+		}
+
+		//CREATE SIMPLE XML FILE
+		HashSet<String> existingfiles = new HashSet<String>();
+		String currentpath = srcDir.getAbsolutePath()
+
+		String encodingAll = null;
+		if (encoding == "??") {
+			encodingAll = new CharsetDetector(srcDir).getEncoding();
+			println "Guessed encoding: $encodingAll"
+		}
+
+		println("Create simple XML files ("+srcfiles.size()+" files)")
+		for (File srcfile : srcfiles) {
+			print "."
+			File xmlfile;
+			try {
+				int point = srcfile.getName().indexOf(".");
+				String filename;
+				String fileext;
+				if (point >= 1 && point < srcfile.getName().length() ) {
+					filename = srcfile.getName().substring(0, point)
+					fileext = srcfile.getName().substring(point+1, srcfile.getName().length());
+				} else {
+					filename = srcfile.getName()
+					fileext = "";
+				}
+
+				String filepath = srcfile.getParentFile().getAbsolutePath();
+				filepath = filepath.substring(currentpath.length());
+				xmlfile = new File(txmDir, filename+".xml");
+
+				while (existingfiles.contains(xmlfile.getName()))//to not erase a file with the same name
+				{
+					xmlfile = new File(txmDir, "_"+xmlfile.getName());
+				}
+				existingfiles.add(xmlfile.getName());
+
+				//println "ENCODING "+encoding
+				String tmpEncoding = encoding;
+				if (encodingAll != null) {
+					tmpEncoding = encodingAll
+					if (srcfile.length() > CharsetDetector.MINIMALSIZE) {
+						tmpEncoding = new CharsetDetector(srcfile).getEncoding();
+						//println "file encoding: $tmpEncoding"
+					}
+				}
+				def input = new FileInputStream(srcfile)
+				Reader reader = new InputStreamReader(input , tmpEncoding);
+				DetectBOM bomdetector = new DetectBOM(srcfile);
+				for (int ibom = 0 ; ibom < bomdetector.getBOMSize() ; ibom++) input.read()
+
+				int lb = 1;
+				int np = 1;
+				int countlb = 0;
+
+				BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(xmlfile))
+				XMLOutputFactory factory = XMLOutputFactory.newInstance();
+				XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8");
+
+				writer.writeStartDocument("UTF-8","1.0");
+				writer.writeStartElement ("TEI");
+				writer.writeDefaultNamespace("http://www.tei-c.org/ns/1.0");
+				writer.writeNamespace("txm", "http://textometrie.org/1.0");
+				writer.writeStartElement ("teiHeader");
+				writer.writeStartElement("fileDesc");
+				writer.writeStartElement("titleStmt");
+				writer.writeStartElement("title")
+				writer.writeCharacters(xmlfile.getName())
+				writer.writeEndElement(); // title
+				writer.writeEndElement(); //titleStmt
+				writer.writeStartElement("publicationStmt");
+				writer.writeEndElement(); // publicationStmt
+				writer.writeStartElement("sourceDesc");
+				writer.writeStartElement("p");
+				writer.writeCharacters("Generated by TXM TXT+CSV import module - TXM project - http://textometrie.org");
+				writer.writeEndElement(); // p
+				writer.writeEndElement(); // sourceDesc
+				writer.writeEndElement(); // fileDesc
+				writer.writeStartElement("encodingDesc")
+				writer.writeStartElement("http://textometrie.org/1.0", "applicationDesc")
+				writer.writeEndElement();
+				writer.writeEndElement();
+				writer.writeEndElement (); // teiHeader
+				writer.writeStartElement ("text");
+				writer.writeAttribute("id",filename)
+
+				reader.eachLine{String line ->
+					line = CleanFile.clean(line);
+					
+					//writer.writeStartElement("p");
+					//writer.writeAttribute("id",""+np++)
+					line = FixString.normalize(line);
+					writer.writeStartElement("lb")
+					writer.writeAttribute("n",""+lb++)
+					writer.writeEndElement(); // close lb
+					writer.writeCharacters(line.replace("\t"," "))
+					//writer.writeEndElement(); // p
+					writer.writeCharacters("\n"); // XML readability
+					//}
+				}
+
+				writer.writeEndElement();//text
+				writer.writeEndElement();// TEI
+				
+				writer.close();
+				output.close();
+			}
+			catch(Exception e) {
+				println("Failed to process file "+srcfile);
+				xmlfile.delete()
+			}
+		}
+		println ""
+
+		println("Tokenizing "+txmDir.listFiles().length+" files")
+		for (File pfile : txmDir.listFiles()) {
+			print "."
+			File tfile = new File(tokenizedDir, pfile.getName());
+
+			SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(pfile, tfile, lang);
+			tokenizer.setStartTag("text")
+			if (!tokenizer.process()) {
+				println("Failed to tokenize file "+pfile)
+				tfile.delete()
+			}
+		}
+		println ""
+		
+		// Tag sentences
+		List<File> stokenfiles = tokenizedDir.listFiles()
+		File stokenizeDir = new File(tokenizedDir.getParentFile(), "stokenized")
+		stokenizeDir.mkdir();
+		println("Tagging sentences of "+stokenfiles.size()+" files")
+		for (File f : stokenfiles) {
+			print "."
+			Sequence S = new Sequence();
+			Filter F1 = new CutHeader();
+			Filter F7 = new TagSentences();
+			Filter F11 = new FusionHeader();
+			S.add(F1);
+			S.add(F7);
+			S.add(F11);
+			File infile = new File(tokenizedDir, f.getName());
+			File xmlfile = new File(stokenizeDir, f.getName());
+			File headerfile = new File(tokenizedDir, f.getName()+"header.xml");
+
+			S.SetInFileAndOutFile(infile.getPath(), xmlfile.getPath());
+			S.setEncodages("UTF-8","UTF-8");
+			Object[] arguments1 = [headerfile.getAbsolutePath()];
+			F1.SetUsedParam(arguments1);
+			Object[] arguments2 = [headerfile.getAbsolutePath(), F1];
+			F11.SetUsedParam(arguments2);
+			if (!S.proceed()) {
+				println "Failed to tag file: "+f
+			}
+			
+			S.clean();
+			S = F1 = F7 = F11 = null;
+			headerfile.delete();//remove the prepared file to clean
+		}
+		println ""
+
+		//TRANSFORM INTO XML-TEI-TXM
+		println("Building xml-tei-txm ("+stokenizeDir.listFiles().length+" files)")
+		for (File tfile : stokenizeDir.listFiles()) {
+			print  "."
+			File xmlfile = new File(txmDir, tfile.getName());
+
+			def correspType = new HashMap<String,String>()
+			def correspRef = new HashMap<String,String>()
+			def respId = [];
+			def applications = new HashMap<String,HashMap<String,String>>();
+			def taxonomiesUtilisees = new HashMap<String,String[]>();
+			def itemsURI = new HashMap<String,HashMap<String,String>>();
+			def resps = new HashMap<String,String[]>();
+
+			//println "file : $tfile"
+			Xml2Ana builder = new Xml2Ana(tfile);
+			builder.setCorrespondances(correspRef, correspType);
+			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+			if (!builder.process(xmlfile)) {
+				println("Failed to process file "+tfile)
+				new File(xmlfile.getParent(),xmlfile.getName()).delete()
+			}
+		}
+		println ""
+
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/TabulatedToXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/TabulatedToXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/TabulatedToXml.groovy	(revision 321)
@@ -0,0 +1,244 @@
+package org.txm.importer 
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamWriter;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Tool to convert Tabulated file to xml file. Such as CWB files or TreeTagger ouput
+ */
+public class TabulatedToXml {
+	String[] colnames;
+	
+	/** The texttag. */
+	String texttag;
+	
+	/** The tag. */
+	boolean tag;
+	
+	/** The no col form. */
+	int noColForm;
+	
+	/**
+	 * Instantiates a new tabulated to xml.
+	 *
+	 * @param colnames the word properties name
+	 * @param texttag the element that represents a text unit
+	 * @param tag if true, split the file by the texttag, if false wrap the file with an element texttag
+	 */
+	public TabulatedToXml(String[] colnames, String texttag, boolean tag) throws Exception
+	{
+		this.colnames = colnames;
+		this.texttag = texttag;
+		this.tag = tag;
+		this.noColForm = 0;
+	}
+	
+	/**
+	 * Instantiates a new tabulated to xml.
+	 *
+	 * @param colnames the colnames
+	 * @param texttag the texttag
+	 * @param tag the tag
+	 * @param noColForm the no col form
+	 */
+	public TabulatedToXml(String[] colnames, String texttag, boolean tag, int noColForm) throws Exception
+	{
+		this.colnames = colnames;
+		this.texttag = texttag;
+		this.tag = tag;
+		this.noColForm = noColForm;
+	}
+	
+	/**
+	 * Process.
+	 *
+	 * @param tabulatedfile the tabulatedfile
+	 * @param xmlfile the xmlfile
+	 * @param encoding the encoding
+	 * @return true, if successful
+	 */
+	public boolean process(File tabulatedfile, File xmlfile, String encoding)
+	{
+		BufferedReader reader = new BufferedReader(new FileReader(tabulatedfile));
+		FileOutputStream output = new FileOutputStream(xmlfile)
+		
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8"); 
+		
+		writer.writeStartDocument("UTF-8", "1.0")
+		
+		if(!tag) // there is no tag, we must wrap the content in an element
+		{
+			if(texttag.length() == 0)
+			{
+				println "no text element given"
+				return false;
+			}
+			writer.writeStartElement(texttag);
+			String filename = tabulatedfile.getName()
+			filename = filename.substring(0, filename.lastIndexOf("."));
+			writer.writeAttribute("id", filename);
+		}
+		
+		String line = reader.readLine();
+		int noline = 0;
+		while(line != null)
+		{
+			if(tag && line.startsWith("</")) // closing xml tag
+			{
+				writer.writeEndElement();
+			}
+			else if(tag && line.startsWith("<")) // opening xml tag
+			{
+				int firstblank = line.indexOf(" ");
+				String tagname;
+				String[] attrnamevalues = null;
+				if(firstblank > 0) // contains attributes
+				{
+					tagname = line.substring(1, firstblank);
+					line = line.substring(firstblank+1, line.length() -1);
+					attrnamevalues = line.split(" ");
+				}
+				else // no attr
+				{
+					tagname = line.substring(1, line.length() -1);
+				}
+				
+				writer.writeStartElement(tagname);
+				if(attrnamevalues != null)
+				{
+					for(String namevalue : attrnamevalues)
+					{
+						int egalidx = namevalue.indexOf("=");
+						String name = namevalue.substring(0, egalidx);
+						String value = namevalue.substring(egalidx+1, namevalue.length());
+						
+						writer.writeAttribute(name, value.replace("\"",""));
+					}
+				}
+			}
+			else // it's a word
+			{
+				String[] split = line.split("\t");
+				if(split.length != colnames.length)
+				{
+					System.out.println("Error line "+noline+" : missing columns :"+split.length+" instead of "+ colnames.length);
+					println "line: "+line
+				}
+				String form = "";
+				writer.writeStartElement("w");
+				for(int i = 0 ; i < colnames.length ; i++)
+				{
+					if(i == noColForm)
+					{
+						form = split[0];
+					}
+					else
+					{
+						if(split.length <= i)
+						writer.writeAttribute(colnames[i], "N/A");
+						else
+						writer.writeAttribute(colnames[i], split[i]);
+					}
+				}
+				writer.writeCharacters(form);
+				writer.writeEndElement();
+				
+			}
+			noline++;
+			line = reader.readLine();
+		}
+		
+		if(!tag) // there is no tag, we close the wrapping element
+		{
+			writer.writeEndElement();
+		}
+		
+		writer.close();
+		output.close();
+		reader.close();
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String home = System.getProperty("user.home");
+		// sample_TXM_LA.vrt
+		/*
+		 File tabulatedfile = new File(home, "xml/tabulated/sample_TXM_LA.vrt");
+		 File xmlfile = new File(home, "xml/tabulated/sample_TXM_LA.xml");
+		 String[] colnames = ["form", "pos", "lem", "func"]; // TreeTagger
+		 boolean tag = true;
+		 String encoding = "UTF-8";
+		 try {
+		 TabulatedToXml ttx = new TabulatedToXml(colnames, "", tag);
+		 ttx.process(tabulatedfile, xmlfile, encoding)
+		 } catch (Exception e) {
+		 // TODO Auto-generated catch block
+		 org.txm.utils.logger.Log.printStackTrace(e);
+		 }*/
+		File srcdir = new File(home, "xml/Bendinelli");
+		File outdir = new File(home, "xml/Bendinelli/xml");
+		println "srcdir: "+srcdir
+		outdir.deleteDir();
+		outdir.mkdir();
+		
+		String[] colnames = ["form", "pos", "lem"]; // TreeTagger
+		boolean tag = false;
+		String encoding = "UTF-8";
+		
+		TabulatedToXml ttx = new TabulatedToXml(colnames, "debat", tag);
+		for(File tabulatedfile : srcdir.listFiles())
+		{
+			if(tabulatedfile.getName().endsWith(".txt"))
+			{
+				String filename = tabulatedfile.getName()
+				filename = filename.substring(0, filename.lastIndexOf("."));
+				File xmlfile = new File(outdir, filename+".xml")
+				println "process: "+tabulatedfile
+				ttx.process(tabulatedfile, xmlfile, encoding)
+			}
+		}
+		
+		println "Done"
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XMLTXM2WTC.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XMLTXM2WTC.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/XMLTXM2WTC.groovy	(revision 321)
@@ -0,0 +1,509 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-03-29 09:51:35 +0200 (Tue, 29 Mar 2016) $
+// $LastChangedRevision: 3185 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class XMLTXM2WTC.
+ *
+ * @author mdecorde
+ * simple transofmration of a xml-tei-txm file into wtc file
+ */
+
+class XMLTXM2WTC
+{
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The output. */
+	private def output;
+	
+	/** The hashmap of txm:form and txm:ana values and the attributes hash*/
+	LinkedHashMap<String, String> anahash = new LinkedHashMap<String, String>();
+	LinkedHashMap<String, String> formhash = new LinkedHashMap<String, String>();
+	LinkedHashMap<String, String> wordattributes = new LinkedHashMap<String, String>();
+	
+	/** The balisesfound. */
+	HashMap<String, List<String>> balisesfound;// = new HashMap<String, List<String>>();
+
+	/** The balises to keep. */
+	List<String> balisesToKeep;
+	
+	/** The send to p attributes. */
+	HashMap <String, List<String>> sendToPAttributes;// = new HashMap<String, List<String>>();
+	
+	/** The injected p attributes. */
+	List<String> injectedPAttributes = new ArrayList<String>();
+	
+	/** The default reference : a pattern + the properties to use */
+	List<String> defaultReferences = new ArrayList<String>();
+	String defaultReferencePattern;
+	
+	/** The injected p attributes values. */
+	HashMap <String, String> injectedPAttributesValues;// = new ArrayList<String>();
+	
+	/** The addinfos. */
+	boolean addinfos = false;
+	
+	/** The txtname. */
+	String txtname;
+	
+	/** The base. */
+	String base;
+	
+	/** The project. */
+	String project;
+	
+	/** The lang. */
+	public String lang= "fr";
+	public String currentForm;
+	public String currentAna;
+	
+	/**
+	 * Sets the lang.
+	 *
+	 * @param lang the lang
+	 * @return the java.lang. object
+	 */
+	public setLang(String lang)
+	{
+		this.lang = lang;
+	}
+	
+	/**
+	 * Instantiates a new xMLTX m2 wtc.
+	 *
+	 * @param url the url
+	 */
+	public XMLTXM2WTC(URL url){
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			
+			parser = factory.createXMLStreamReader(inputData);
+			
+			
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * Sets the text info.
+	 *
+	 * @param name the name
+	 * @param base the base
+	 * @param project the project
+	 */
+	public void setTextInfo(String name, String base, String project)
+	{
+		this.addinfos = true;
+		this.txtname= name;
+		this.base = base;
+		this.project = project;
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile)
+	{
+		try {
+			output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile, outfile.exists()) , "UTF-8"));
+			return true;
+		} catch (Exception e) {
+			System.err.println(e);
+			return false;
+		}
+	}
+	
+	/** The haspb. */
+	boolean haspb = false;
+	
+	/** The haslb. */
+	boolean haslb = false;
+	
+	/**
+	 * Transform file.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	public boolean transformFile(File outfile)
+	{
+		if(balisesToKeep == null)
+		{
+			println "no element has been defined to be keeped"
+			return false;
+		}
+		
+		haspb = false;
+		haslb = false;	
+		
+		boolean flagAna;
+		boolean flagForm;
+		boolean flagWord;
+		String vWord = "";
+		String vForm = "";
+		String vAna = "";
+		
+		String lb_id = "";
+		String pb_id = "";
+		
+		wordattributes = [:];
+		balisesfound = new HashMap<String, List<String>>();
+		
+		
+		if(!createOutput(outfile))
+			return false;
+		
+		if(sendToPAttributes != null)
+		{
+			for(String tag: sendToPAttributes.keySet())
+				for(String attr : sendToPAttributes.get(tag))
+					injectedPAttributes.add(tag+attr);
+			injectedPAttributesValues = [:];
+		}
+		
+		//output.write("<txmcorpus lang=\""+lang+"\">\n");
+		balisesfound.put("txmcorpus",["lang"]);
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) 
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					String localname = parser.getLocalName().toLowerCase();
+				
+				// we will only declare found tags in cwb registry
+					if(balisesToKeep.contains(localname))
+					{
+						if(!balisesfound.containsKey(localname))
+						{
+							balisesfound.put(localname,[]);
+						}
+						
+						List<String> attrlist = balisesfound.get(localname);
+						for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+							if(!attrlist.contains(parser.getAttributeLocalName(i)))
+								attrlist.add(parser.getAttributeLocalName(i));
+						
+					}
+					switch (localname) 
+					{						
+						case "w": // get word id !!
+							wordattributes.put("id", parser.getAttributeValue(null, "id"));
+							break;
+						
+						case "form":
+							flagForm = true;
+							currentForm = parser.getAttributeValue(null, "type");
+							if(currentForm == null)
+								currentForm = "default";
+							vForm = "";
+							break;
+						
+						case "ana":
+							flagAna = true;
+							vAna ="";
+						
+							currentAna = (parser.getAttributeValue(null,"type"));
+							if(currentAna != null)
+								currentAna = currentAna.substring(1)// remove the #
+							else
+								flagAna = false;
+							break;
+						
+						default:
+							
+							if (sendToPAttributes != null) {
+								//println "should store $localname ? with "+sendToPAttributes.keySet()
+								if (sendToPAttributes.keySet().contains(localname)) {
+									//println "store attr of "+localname
+									List<String> attrs = sendToPAttributes.get(localname);
+									for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {
+										if (attrs.contains(parser.getAttributeLocalName(i))) {
+											injectedPAttributesValues.put(localname+parser.getAttributeLocalName(i).toLowerCase(),parser.getAttributeValue(i)) 
+										}
+									}
+								}
+							}
+							
+							if (balisesToKeep.contains(localname)) {
+								output.write("<"+localname);
+								//println "write <"+localname+"..."
+								//write attributes
+								boolean idwritten = false;
+								boolean basewritten = false;
+								boolean projectwritten = false;
+								for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {
+									String attrname = parser.getAttributeLocalName(i).toLowerCase();
+									if (attrname == "id")
+										idwritten = true;
+									if (attrname == "base")
+										basewritten = true;
+									if (attrname == "project")
+										projectwritten = true;
+									output.write(" "+attrname+"=\""+parser.getAttributeValue(i).replace("\"", "'")+"\"" );
+								}
+								
+								if (localname.equals("text"))
+									if (addinfos) {
+										List<String> attrlist = balisesfound.get(localname);
+	
+										if (!idwritten) {
+											output.write(" id=\""+txtname+"\"")
+											attrlist.add("id");
+										}
+										if (!basewritten) {
+											output.write(" base=\""+base+"\"");
+											attrlist.add("base");
+										}
+										if (!projectwritten) {
+											output.write(" project=\""+project+"\"");
+											attrlist.add("project");
+										}											
+									}	
+											
+								// finalize tag
+								output.write(">\n");
+							}
+					}
+					break;
+				
+				case XMLStreamConstants.END_ELEMENT:
+					String localname = parser.getLocalName().toLowerCase();
+					switch (localname) {
+						case "form":
+							if(flagForm)
+								formhash.put(currentForm, vForm);
+							flagForm = false; 
+							break;
+						
+						case "ana":
+							if(flagAna)
+								anahash.put(currentAna, vAna);
+							flagAna = false;
+							break;
+						
+						case "w":
+							vWord = "";
+							vWord = formhash.get("default").replaceAll("&", "&amp;").replaceAll("<", "&lt;"); // get default form
+							for (String form : formhash.keySet()) // and the others
+								if (form != "default")
+									vWord += "\t"+formhash.get(form);
+							
+							for (String type : wordattributes.keySet()) // only word id ?
+								vWord+="\t"+wordattributes.get(type)
+						
+							if (sendToPAttributes != null) // word attributes from structure properties
+							{
+								//println "injectedPAttributesValues: "+injectedPAttributesValues
+								for(String pattr : injectedPAttributes)
+									vWord+="\t"+injectedPAttributesValues.get(pattr) ;//les attributs injecter
+							}
+							
+							for (String type : anahash.keySet()) // word annotations in txm:ana
+								vWord+="\t"+anahash.get(type)
+							
+							output.write(vWord+"\n");
+							vWord= "";
+							break;
+						
+						default:
+							if (sendToPAttributes != null) // reset structure properties
+							{
+								if (sendToPAttributes.keySet().contains(localname)) {
+									for (String attr : sendToPAttributes.get(localname)) {
+										injectedPAttributesValues.put(attr, "N/A") 
+									}
+								}
+							}
+							
+							if (balisesToKeep.contains(localname)) {
+								output.write("</"+localname+">\n");
+							}
+					}
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if (flagForm) {
+						vForm += parser.getText().trim();
+					}
+					if (flagAna) {
+						vAna += parser.getText().trim();
+					}
+					break;
+			}
+		}
+		//output.write("</txmcorpus>\n"); 
+		output.close();
+		parser.close();
+		
+		return true;
+	}
+	
+	/**
+	 * Gets the p attributs.
+	 *
+	 * @return the p attributs
+	 */
+	public List<String> getpAttributs()
+	{
+		def pAttributs = [];
+		
+		for (String wordattr : wordattributes.keySet()) {
+			pAttributs.add(wordattr);
+		}
+		
+		if (sendToPAttributes != null)
+			for (String pAttr : this.injectedPAttributes)
+				pAttributs.add(pAttr);
+		
+		for (String anakey : anahash.keySet()) {
+			pAttributs.add(anakey);
+		}
+		
+		return pAttributs;
+	}
+	
+	/**
+	 * Gets the s attributs.
+	 *
+	 * @return the s attributs
+	 */
+	public List<String> getsAttributs()
+	{
+		def sAttributs = [];
+		for (String balise : this.balisesfound.keySet()) {
+			List<String> sAtt = this.balisesfound.get(balise);
+			String attributes = "";
+			for (String attr : sAtt) {
+				attributes+="+"+attr;
+			}
+			
+			if (sAtt.size() > 0)
+				sAttributs.add(balise +":"+attributes);
+			else
+				sAttributs.add(balise);
+		}
+		return sAttributs;
+	}
+	
+	/**
+	 * Sets the balises to keep.
+	 *
+	 * @param balisesToKeep the new balises to keep
+	 */
+	public void setBalisesToKeep(List<String> balisesToKeep)
+	{
+		if (balisesToKeep != null)
+			this.balisesToKeep = balisesToKeep;
+		else
+			println("Warning: the list of elements to keep is null")
+	}
+	
+	/**
+	 * Sets the defautl reference pattern
+	 * TODO: not implemented
+	 *
+	 * @param balisesToKeep the new balises to keep
+	 */
+	public void setDefaultReference(String pattern, List<String> strucProperties)
+	{
+		if (defaultReferencePattern != null) {
+			this.defaultReferences = defaultReferences;
+			defaultReferencePattern = pattern;
+		}
+	}
+	
+	
+	/**
+	 * Sets the send to p attributes.
+	 *
+	 * @param sendus the sendus
+	 */
+	public void setSendToPAttributes(HashMap<String, List<String>> sendus)
+	{
+		if (sendus != null)
+			this.sendToPAttributes = sendus;
+		else
+			println("Warning: the pAttributes to inject is null")
+	}
+	
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		
+		String rootDir = "C:/Documents and Settings/H/xml/quick/txm";
+		
+		File srcfile = new File(rootDir,"quick.xml");
+		File wtcfile = new File(rootDir+"/wtc","quick.wtc");
+		new File(rootDir,"wtc").deleteDir()
+		new File(rootDir,"wtc").mkdir()
+		
+		System.out.println("XMLTXM2WTC : "+srcfile+" >> "+wtcfile);
+		def builder = new XMLTXM2WTC(srcfile.toURL());
+		def balises = ["text","lb","pb"];
+		builder.setBalisesToKeep(balises);
+		builder.transformFile wtcfile;
+		
+		println(builder.getsAttributs());
+		println(builder.getpAttributs());
+		return;
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/compiler.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/compiler.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/compiler.groovy	(revision 321)
@@ -0,0 +1,203 @@
+
+
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2012-01-05 14:27:34 +0100 (jeu., 05 janv. 2012) $
+// $LastChangedRevision: 2096 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.importer.limsi
+
+import org.txm.Toolbox;
+import org.txm.importer.cwb.*
+import org.txm.importer.*;
+import org.txm.scripts.*;
+import org.txm.scripts.teitxm.*;
+import org.txm.stat.utils.ConsoleProgressBar
+import org.txm.utils.io.FileCopy;
+import org.txm.utils.treetagger.TreeTagger;
+
+import java.util.ArrayList;
+
+import javax.xml.stream.*;
+
+import java.net.URL;
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class compiler.
+ */
+class compiler
+{
+	/** The debug. */
+	boolean debug = false
+
+	/** The dir. */
+	private def dir
+
+	/** The cwb loc. */
+	String cwbLoc
+	boolean annotationSuccess = false
+
+	public def pAttributesList = []
+	public def sAttributesList = []
+
+	/**
+	 * Sets the debug.
+	 *
+	 * @return the java.lang. object
+	 */
+	public setDebug()
+	{
+		debug = true
+	}
+
+	/**
+	 * Sets the cwb path.
+	 *
+	 * @param path the new cwb path
+	 */
+	public void setCwbPath(String path)
+	{
+		if (!new File(path).exists())
+			System.err.println("CWB Path : "+path+" does not exists")
+		cwbLoc = path
+	}
+
+	/**
+	 * Sets the annotation success.
+	 *
+	 * @param val the new annotation success
+	 */
+	public void setAnnotationSuccess(boolean val)
+	{
+		this.annotationSuccess = val
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param basename the basename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, String corpusname)
+	{
+		if (cwbLoc == null)
+			cwbLoc = org.txm.Toolbox.getParam(org.txm.Toolbox.CQI_SERVER_PATH_TO_CQPLIB)+File.separator;
+
+		if (!new File(cwbLoc).exists()) {
+			println ("CWB path error: "+cwbLoc)
+			return false
+		}
+		if (!binDir.exists()) {
+			println ("binary directory does not exists: "+binDir)
+			return false
+		}
+
+		//File wtcFile = new File(binDir,"wtc/"+corpusname+".wtc")
+		new File(binDir, "wtc").deleteDir()
+		new File(binDir, "wtc").mkdir()
+		new File(binDir, "data").deleteDir()
+		new File(binDir, "data").mkdir()
+		new File(binDir, "registry").mkdir()
+
+		//1- Merge WTC files
+		if (annotationSuccess) {
+			File annotationDir = new File(binDir, "annotations")
+			if (annotationDir.exists()) txmDir = annotationDir;
+		}
+		def files = txmDir.listFiles()
+		files.sort(); // sort files by name
+
+		//2- Import into CWB
+		def outDir = binDir.getAbsolutePath()+"/";
+
+		CwbEncode cwbEn = new CwbEncode()
+		CwbMakeAll cwbMa = new CwbMakeAll()
+		cwbEn.setDebug(debug)
+		cwbMa.setDebug(debug)
+
+		def pAttrs = []; // cast to array
+		if (annotationSuccess) {
+			pAttrs << "frpos"
+			pAttrs << "frlemma"
+		}
+		def sAttrs = ["text:0+id", "txmcorpus:0+lang"]; // cast to array
+
+
+		String[] pAttributes = pAttrs // cwb-encode needs String[]
+		String[] sAttributes = sAttrs // cwb-encode needs String[]
+		println "pAttrs : "+pAttrs
+		println "sAttrs : "+sAttrs
+
+		try {
+			cwbEn.setDebug(debug)
+			cwbMa.setDebug(debug)
+			String regPath =outDir + "/registry/"+corpusname.toLowerCase()
+			cwbEn.run(cwbLoc + "cwb-encode", outDir + "/data/$corpusname",
+					null,
+					regPath, pAttributes, sAttributes);
+
+			def output = new OutputStreamWriter(cwbEn.getOutputStream());
+			if (output == null) {println "CWB-ENCODE OUTPUT NULL"; return false;}
+			ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+			println "processing "+files.size()+" files."
+			output.write("<txmcorpus lang=\"fr\">\n")
+			for (File f : files) {
+				cpb.tick()
+				output.write(f.getText("UTF-8"))
+				output.flush();
+				output.write("\n")
+			}
+			output.write("</txmcorpus>\n")
+			output.close()
+			cwbEn.endProcess();
+			
+			if (!new File(regPath).exists()) {
+				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"
+				return false;
+			}
+			cwbMa.run(cwbLoc + "cwb-makeall", corpusname, outDir + "/registry")
+			return true
+		} catch (Exception ex) {System.out.println("Error while compiling indexes: $ex"); ex.printStackTrace(); return false;}
+
+		return true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File("~/xml/perrault/txm/")
+		List<File> files = dir.listFiles()
+		new compiler().run(files)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Limsi format transformation scripts</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/importer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/importer.groovy	(revision 321)
@@ -0,0 +1,229 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+
+package org.txm.importer.limsi
+
+import java.util.ArrayList;
+import org.txm.utils.Pair;
+
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.txm.utils.logger.Log;
+import org.txm.scripts.teitxm.*;
+import org.txm.stat.utils.ConsoleProgressBar
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import org.w3c.dom.*;
+import org.xml.sax.SAXException;
+import javax.xml.parsers.*;
+import javax.xml.xpath.*;
+
+import java.util.HashMap;
+import org.txm.importer.*;
+import org.txm.utils.*;
+import org.txm.metadatas.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class importer.
+ */
+class importer {
+
+	/** The speakers. */
+	HashMap<String, String> speakers = ["spk1":"//Speaker[@id='spk1']", "spk2":"//Speaker[@id='spk2']"];
+
+	/** The topics. */
+	HashMap<String, String> topics = ["to1":"//Topic[@id='to1']", "to2":"//Topic[@id='to2']"];
+
+	/** The trans. */
+	HashMap<String, String> trans = ["trans":"//Trans"];
+
+	/** The doc. */
+	def doc;
+
+	/** The infile. */
+	File infile;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The outdir. */
+	File txmDir;
+	File binDir;
+
+	/** The trsfiles. */
+	ArrayList<String> trsfiles;
+
+	/** The metadata. */
+	Metadatas metadatas;
+
+	/**
+	 * Instantiates a new importer.
+	 *
+	 * @param trsfiles the trsfiles
+	 * @param outdir the outdir
+	 * @param metadata the metadata
+	 */
+	public importer(ArrayList<File> trsfiles, File binDir, File txmDir)
+	{
+		this.trsfiles = trsfiles;
+		this.txmDir = txmDir;
+		this.binDir = binDir;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean run()
+	{
+		if (trsfiles == null) {
+			println "no files to process"
+			return false;
+		}
+		txmDir.mkdir();
+		if (!txmDir.exists()) {
+			println "can't create txmDir: "+txmDir.getAbsolutePath()
+		}
+
+		println "Convert from LIMSI files to CQP files ("+trsfiles.size()+")."
+		ConsoleProgressBar cpb = new ConsoleProgressBar(trsfiles.size());
+		for (File infile : trsfiles) {
+			cpb.tick();
+
+			String filename = infile.getName();
+			int idx = filename.indexOf(".");
+			if (idx > 0) filename = filename.substring(0, idx)
+
+			File outfile = new File(txmDir, filename+".wtc")
+			
+			if (outfile.exists() && outfile.lastModified() >= infile.lastModified()) {
+				// skip
+			} else {
+				def processor = new LimsiToCQP(infile);
+				processor.process(outfile);
+			}
+		}
+
+		println ""
+		return txmDir.listFiles() != null;
+	}
+
+	/**
+	 * Process.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 * @param metas the metas
+	 * @return true, if successful
+	 */
+	public boolean process(File infile, File outfile, ArrayList<Pair<String, String>> metas)
+	{
+		//inject metadata into
+		this.infile = infile;
+		this.outfile = outfile;
+		def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+		doc = builder.parse(infile)
+		insert(trans.get("trans"), metas);
+		return save();
+	}
+
+	/**
+	 * Insert.
+	 *
+	 * @param xpath the xpath
+	 * @param pairs the pairs
+	 */
+	public void insert(String xpath, List<Pair<String, String>> pairs)
+	{
+		println ("insert $pairs into $xpath")
+		def expr = XPathFactory.newInstance().newXPath().compile(xpath)
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET)
+
+		for (Node node : nodes) {
+			Element elem = (Element)node;
+			for (Pair<String, String> p : pairs) {
+				elem.setAttribute(p.getFirst(), p.getSecond());
+			}
+		}
+	}
+
+	/**
+	 * Save.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean save()
+	{
+		try {
+			// Création de la source DOM
+			Source source = new DOMSource(doc);
+
+			// Création du fichier de sortie
+			Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8"));
+			Result resultat = new StreamResult(writer);
+
+			// Configuration du transformer
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			Transformer transformer = fabrique.newTransformer();
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
+
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			return true;
+		} catch (Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			return false;
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String userhome = System.getProperty("user.home");
+		File metadatasfile = new File(userhome,"/xml/transcriber/metadatas.xml");
+		File infile = new File(userhome,"/xml/transcriber/dialogue.trs");
+		File outfile = new File(userhome,"/xml/transcriber/dialogue-out.trs");
+		new importer().run(infile, outfile, metadatasfile);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/LimsiToCQP.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/LimsiToCQP.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/LimsiToCQP.groovy	(revision 321)
@@ -0,0 +1,78 @@
+package org.txm.importer.limsi
+
+import javax.xml.stream.*
+import java.net.URL
+
+class LimsiToCQP {
+
+	File xmlfile;
+
+	public LimsiToCQP(File xmlfile) {
+		this.xmlfile = xmlfile;
+	}
+
+	public boolean process(File outfile) {
+		if (!xmlfile.exists()) return false;
+		
+		URL url = xmlfile.toURI().toURL();
+		String filename = outfile.getName()
+		filename = filename.substring(0, filename.length()-4); // remove ".wtc"
+		def inputData = url.openStream();
+		XMLInputFactory factory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+
+		def output = new OutputStreamWriter(new FileOutputStream(outfile) , "UTF-8");
+		
+		boolean flagWord = false
+		String word = ""
+		
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event)
+			{
+				case XMLStreamConstants.START_ELEMENT:
+					String localname = parser.getLocalName()
+					//println "start $localname"
+					switch(localname) {
+						case "AudioDoc":
+							output.println "<text id=\"$filename\">"
+							break
+						case "Word":
+							flagWord = true
+							word = ""
+						break;
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					String localname = parser.getLocalName()
+					switch(localname) {
+						case "AudioDoc":
+						output.println "</text>"
+						break
+						case "Word":
+							flagWord = false
+							output.println word.trim()
+						//	println "WORD: $word"
+						break;
+					}
+					break
+				case XMLStreamConstants.CHARACTERS:
+					if (flagWord) {
+						word += parser.getText();
+					}
+					break
+			}
+		}
+		output.flush()
+		output.close()
+		//println "$xmlfile -> $outfile"
+		return true;
+	}
+
+	public static void main(String[] args) {
+		File infile = new File("/home/mdecorde/xml/limsi","20071220_1900_1920_inter.xml")
+		File outfile = new File("/home/mdecorde/xml/limsi","out.wtc")
+		def processor = new LimsiToCQP(infile);
+		println processor.process(outfile);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/limsiLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/limsiLoader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/limsi/limsiLoader.groovy	(revision 321)
@@ -0,0 +1,222 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package org.txm.importer.limsi;
+
+import org.eclipse.swt.widgets.*;
+import org.txm.importer.transcriber.compiler.*;
+import org.txm.importer.transcriber.pager.*;
+import org.txm.importer.transcriber.*;
+
+import java.io.File;
+
+import org.txm.scripts.teitxm.*;
+import org.txm.stat.utils.ConsoleProgressBar
+import org.txm.*;
+import org.txm.objects.*;
+import org.txm.utils.i18n.*;
+import org.txm.utils.io.FileCopy;
+import org.txm.utils.*;
+import org.txm.importer.*;
+import org.w3c.dom.Element
+import org.txm.rcpapplication.commands.*
+
+
+//PARAMETERS
+boolean removeInterviewer = false;//if true the transcription of speakers (en1 and enq2) defined in metadatas file will be ignored
+boolean includeComments = false;
+boolean ignoreTranscriberMetadata = false;
+int csvHeaderNumber = 1;
+int maxlines = 200;
+
+String userDir = System.getProperty("user.home");
+
+def MONITOR;
+boolean debug = org.txm.utils.logger.Log.isPrintingErrors();
+BaseParameters params;
+try {params = paramsBinding;MONITOR=monitor} catch (Exception)
+{	println "DEV MODE";//exception means we debug
+	debug = true
+	//
+		params = new BaseParameters(new File(userDir, "xml/limsi/files/import.xml"))
+		println "loading "+params.paramFile
+		params.load()
+	if (!org.txm.Toolbox.isInitialized()) {
+		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));
+		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));
+		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));
+		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");
+		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");
+		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");
+		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));
+	}
+}
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME),"corpora/"+basename);
+//binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir,"txm/$corpusname");
+//txmDir.deleteDir();
+txmDir.mkdirs();
+
+try {
+	// select only xml files
+	String ext = ".xml";
+	ArrayList<File> limsiFiles = srcDir.listFiles(); //find all trs files
+	if (limsiFiles  == null) {
+		println ("No files in "+srcDir.getAbsolutePath())
+		return false;
+	}
+	
+	// remove non XML files
+	for (int i = 0 ; i < limsiFiles.size() ; i++) {
+		File f = limsiFiles.get(i);
+		if (f.getName().equals("import.xml") || !f.getName().endsWith(ext) || !f.canRead() || f.isHidden()) {
+			limsiFiles.remove(i)
+			i--;
+		}
+	}
+
+	if (limsiFiles.size() == 0) {
+		println ("No transcriptions in "+srcDir.getAbsolutePath())
+		return false;
+	} else {
+		println "Number of transcriptions: "+limsiFiles.size();
+	}
+
+	if (MONITOR != null) MONITOR.worked(1, "IMPORTER")
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	println "-- IMPORTER"
+	def imp = new importer(limsiFiles, binDir, txmDir) //put result in the txm folder of binDir
+	if (!imp.run()) {
+		println "Failed to prepare files - Aborting";
+		return;
+	}
+	if (MONITOR != null) MONITOR.worked(20)
+
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+	println "-- ANNOTATE CQP- Running NLP tools"
+	boolean annotationSuccess = false;
+	if (annotate && new AnnotateWTC().run(binDir, txmDir, model+".par")) {
+		annotationSuccess = true;
+	}
+
+	if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+	if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+	println "--COMPILING - Building Search Engine indexes"
+	limsiFiles = txmDir.listFiles();
+
+	def comp = new compiler()
+	if(debug) comp.setDebug();
+	comp.setAnnotationSuccess(annotationSuccess)
+	if (!comp.run(binDir, txmDir, corpusname)) {
+		println "Failed to compile files";
+		return;
+	}
+	
+	// create HTML directory but don't build an edition
+	File htmlDir = new File(binDir,"HTML/$corpusname");
+	htmlDir.mkdirs();
+	
+	def filelist;
+	if (annotationSuccess)
+		filelist = new File(binDir, "annotations").listFiles();
+	else 
+		filelist = txmDir.listFiles();
+	filelist.sort();
+	def second = 0
+	
+	println "Registering texts: "+filelist.size()
+	ConsoleProgressBar cpb = new ConsoleProgressBar(filelist.size());
+	for (File txmFile : filelist) {
+		cpb.tick()
+		
+		String txtname = txmFile.getName();
+		int i = txtname.lastIndexOf(".");
+		if(i > 0) txtname = txtname.substring(0, i);
+		
+		Element text = params.addText(corpusElem, txtname, txmFile);
+	}
+}
+catch (Exception e){org.txm.utils.logger.Log.printStackTrace(e);}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
+
+
+// load corpus
+println "Loading corpus..."
+try {LoadBinaryCorpus.loadBase(binDir)} catch(Exception e777){try {AddBases.loadBase(binDir, MONITOR)} catch(Exception e){}} // LoadBinaryCorpus does not exist if TXM version is < 0.7.7
+
+Toolbox.restartWorkspace();
+Toolbox.restartSearchEngine();
+
+Display.getDefault().syncExec(new Runnable() {
+	@Override
+	public void run() {
+		println "Reloading corpora view..."
+		RestartTXM.reloadViews();
+		println "import done."
+	}
+});
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ReadImportParameters.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ReadImportParameters.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/ReadImportParameters.groovy	(revision 321)
@@ -0,0 +1,277 @@
+package org.txm.importer
+
+import javax.xml.stream.*;
+import java.net.URL;
+/**
+ * Read the import.xml file
+ * @author mdecorde
+ */
+class ReadImportParameters {
+	HashMap<String, ArrayList<String>> links = new HashMap<String, ArrayList<String>>();;
+	HashMap<String, String> structs = new HashMap<String, String>();
+	HashMap<String, Integer> levels = new HashMap<String, Integer>();
+	
+	HashMap<String, HashMap<String, String>> metadatas = [:];
+	HashMap<String, HashMap<String, String>> pattributes = [:];
+	HashMap<String, HashMap<String, String>> sattributes = [:];
+	
+	HashMap<String, String> biblios = [:];
+	
+	HashMap<String, String> uis = [:];
+	
+	HashMap<String, ArrayList<HashMap<String, String>>> editions = [:];
+	
+	String basename;
+	String encoding;
+	String language;
+	String rootDir;
+	String script;
+	String xslt;
+	String author;
+	String description;
+	
+	boolean debug;
+
+	public ReadImportParameters()
+	{
+
+	}
+
+	public boolean read(File infile)
+	{
+		try {
+			def url = infile.toURI().toURL();
+			def inputData = url.openStream();
+			def factory = XMLInputFactory.newInstance();
+			def parser = factory.createXMLStreamReader(inputData);
+			return read(parser);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	public boolean read(def parser)
+	{
+		boolean startAlignInfos = false;
+		boolean startEditions = false;
+		boolean startMetadatas = false;
+		boolean startPAttributes = false;
+		boolean startSAttributes = false;
+		boolean startTokenizer = false;
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event) {
+
+				case XMLStreamConstants.END_ELEMENT:
+					String localname = parser.getLocalName();
+					if(localname == "linkGrp")
+					{
+						startAlignInfos = false;
+					}
+					else if(localname == "metadatas")
+					{
+						startMetadatas = false;
+					}
+					else if(localname == "editions")
+					{
+						startEditions = false;
+					}
+					else if(localname == "pattributes")
+					{
+						startPAttributes = false;
+					}
+					else if(localname == "sattributes")
+					{
+						startSAttributes = false;
+					}
+					else if(localname == "tokenizer")
+					{
+						startSAttributes = false;
+					}
+					break;
+
+				case XMLStreamConstants.START_ELEMENT:
+					String localname = parser.getLocalName();
+					if (localname == "import")
+					{
+						basename = parser.getAttributeValue(null, "name");
+						encoding = parser.getAttributeValue(null, "encoding");
+						language = parser.getAttributeValue(null, "lang");
+						rootDir = parser.getAttributeValue(null, "rootDir");
+						script = parser.getAttributeValue(null, "script");
+						xslt = parser.getAttributeValue(null, "xslt");
+						author = parser.getAttributeValue(null, "author")
+						description = parser.getAttributeValue(null, "desc");
+						
+						boolean debug;
+					}
+					else if(localname == "editions")
+					{
+						startEditions = true;
+					}
+					else if(localname == "edition")
+					{
+						String corpus = parser.getAttributeValue(null, "corpus")
+						if(!editions.containsKey(corpus))
+							editions.put(corpus, []);
+						def list = editions.get(corpus);
+						def map = [:]
+						list.add(map)
+						
+						map.put("name", parser.getAttributeValue(null, "name"))
+						map.put("mode", parser.getAttributeValue(null, "mode"))
+						map.put("script", parser.getAttributeValue(null, "script"))
+					}
+					else if (localname == "metadatas") {
+						startMetadatas = true;
+					}
+					else if (localname == "metadata" && startMetadatas) {
+						metadatas.put(parser.getAttributeValue(null, "id"), 
+							["id":parser.getAttributeValue(null, "id"),
+							"shortname":parser.getAttributeValue(null, "shortname"),
+							"longname":parser.getAttributeValue(null, "longname"),
+							"type":parser.getAttributeValue(null, "type"),
+							"selection":parser.getAttributeValue(null, "selection"),
+							"partition":parser.getAttributeValue(null, "partition"),
+							"display":parser.getAttributeValue(null, "display"),])
+					}
+					else if (localname == "pattributes") {
+						startPAttributes = true;
+					} else if(localname == "pattribute" && startPAttributes) {
+						pattributes.put(parser.getAttributeValue(null, "id"), 
+							["id":parser.getAttributeValue(null, "id"),
+							"shortname":parser.getAttributeValue(null, "shortname"),
+							"longname":parser.getAttributeValue(null, "longname"),
+							"type":parser.getAttributeValue(null, "type"),
+							"renderer":parser.getAttributeValue(null, "renderer"),
+							"tooltip":parser.getAttributeValue(null, "tooltip"),
+							"display":parser.getAttributeValue(null, "display"),
+							"min":parser.getAttributeValue(null, "min"),
+							"max":parser.getAttributeValue(null, "max"),
+							"inputFormat":parser.getAttributeValue(null, "inputFormat"),
+							"outputFormat":parser.getAttributeValue(null, "outputFormat"),
+							"position":parser.getAttributeValue(null, "position"),
+							"sortBy":parser.getAttributeValue(null, "sortBy")])
+					}
+					else if (localname == "sattributes") {
+						startSAttributes = true;
+					}
+					else if (localname == "sattribute" && startSAttributes) {
+						sattributes.put(parser.getAttributeValue(null, "id"), 
+							["id":parser.getAttributeValue(null, "id"),
+							"shortname":parser.getAttributeValue(null, "shortname"),
+							"longname":parser.getAttributeValue(null, "longname"),
+							"type":parser.getAttributeValue(null, "type"),
+							"renderer":parser.getAttributeValue(null, "renderer"),
+							"tooltip":parser.getAttributeValue(null, "tooltip"),
+							"pattern":parser.getAttributeValue(null, "renderer"),
+							"import":parser.getAttributeValue(null, "import"),
+							"min":parser.getAttributeValue(null, "min"),
+							"max":parser.getAttributeValue(null, "max"),
+							"mandatory":parser.getAttributeValue(null, "mandatory"),
+							"order":parser.getAttributeValue(null, "order"),
+							"inputFormat":parser.getAttributeValue(null, "inputFormat"),
+							"outputFormat":parser.getAttributeValue(null, "outputFormat"),])
+					} else if (localname == "link" && startAlignInfos) {
+						String target = parser.getAttributeValue(null, "target");
+						String struct = parser.getAttributeValue(null, "alignElement");
+						String level = parser.getAttributeValue(null, "alignLevel")
+						ArrayList<String> corpora = target.split("#");
+						for (int i = 0 ; i < corpora.size() ; i++) {
+							corpora[i] = corpora[i].trim()
+							if (corpora[i].length() == 0) {
+								corpora.remove(i)
+								i--
+							}
+						}
+
+						if (level == null)
+							level = "1";
+
+						if (corpora.size() == 0) {
+							println "no corpus aligned: "+parser.getLocation()
+							return false;
+						}
+						if (struct == null) {
+							println "a sturcture is missing: "+parser.getLocation()
+							return false;
+						}
+						links.put(target, corpora)
+						structs.put(target, struct)
+						levels.put(target, level.toInteger())
+					} else if(localname == "linkGrp" && parser.getAttributeValue(null, "type") == "align") {
+						startAlignInfos = true;
+					} else if(localname == "tokenizer") {
+						startSAttributes = true;
+					}
+			}
+		}
+	}
+
+	public Set<String> getAlignements()
+	{
+		return links.keySet();
+	}
+
+	public ArrayList<String> getTargets(String alignement)
+	{
+		return links.get(alignement);
+	}
+
+	public String getStructure(String alignement)
+	{
+		return structs.get(alignement);
+	}
+
+	public int getLevel(String alignement)
+	{
+		return levels.get(alignement);
+	}
+
+	public void dump()
+	{
+		println "LINKS"
+		for(String link : links.keySet())
+		{
+			println link
+			println " corpus: "+links.get(link)
+			println " struct: "+structs.get(link)
+			println " level: "+levels.get(link)
+		}
+		
+		println "PROPERTIES"
+		println "metadatas $metadatas"
+		println " pattributes $pattributes"
+		println " sattributes $sattributes"
+		
+		println " biblios $biblios"
+		
+		println "uis $uis"
+		
+		println " editions $editions"
+		
+		println "HEADER"
+		println " basename $basename"
+		println " encoding $encoding"
+		println " language $language"
+		println " rootDir $rootDir"
+		println " script $script"
+		println " xslt $xslt"
+		println " author $author"
+		println " description $description"
+		println "DEBUG: $debug"
+	}
+
+	public static void main(String[] args)
+	{
+		String userhome = System.getProperty("user.home");
+		File alignfile = new File(userhome, "xml/xmltxmpara/align.xml")
+		ReadImportParameters parameters = new ReadImportParameters();
+		parameters.read(alignfile);
+		parameters.dump();
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/StaxParser.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/StaxParser.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/StaxParser.groovy	(revision 321)
@@ -0,0 +1,166 @@
+package org.txm.importer
+
+import groovy.transform.CompileStatic
+import java.io.File;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.File;
+import java.net.URL;
+import javax.xml.stream.*;
+import org.txm.importer.filters.*;
+import org.txm.importer.graal.PersonalNamespaceContext;
+
+@CompileStatic
+class StaxParser {
+
+	protected URL inputurl;
+	protected InputStream inputData;
+	protected XMLInputFactory factory;
+	protected XMLStreamReader parser;
+	
+	
+	protected static PersonalNamespaceContext Nscontext = new PersonalNamespaceContext();
+	
+	protected String localname;
+	
+	public StaxParser(File infile)
+	{
+		this(infile.toURI().toURL());
+	}
+	
+	public StaxParser(URL inputurl)
+	{
+		this.inputurl = inputurl;
+		inputData = inputurl.openStream();
+		factory = XMLInputFactory.newInstance();
+		
+		parser = factory.createXMLStreamReader(inputData);
+	}
+	
+	protected void before()
+	{
+		
+	}
+	
+	protected void after()
+	{
+		factory = null;
+		if (parser !=null) parser.close();
+		parser = null;
+	}
+	
+	protected void closeForError() {
+		parser.close();
+	}
+
+	public boolean process()
+	{
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				switch (event) {
+					case XMLStreamConstants.NAMESPACE:
+						processNamespace();
+						break;
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+						processStartElement();
+						break;		
+					case XMLStreamConstants.CHARACTERS:
+						processCharacters();
+						break;
+					case XMLStreamConstants.PROCESSING_INSTRUCTION:
+						processProcessingInstruction();
+						break;
+					case XMLStreamConstants.DTD:
+						processDTD();
+						break;
+					case XMLStreamConstants.CDATA:
+						processCDATA();
+						break;
+					case XMLStreamConstants.COMMENT:
+						processComment();
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName();
+						processEndElement();	
+						break;
+					case XMLStreamConstants.END_DOCUMENT:
+						processEndDocument();
+						break;
+					case XMLStreamConstants.ENTITY_REFERENCE:
+						processEntityReference();
+						break;
+				}
+			}
+		} catch(Exception e) {
+			println("Unexpected error while parsing file "+inputurl+" : "+e);
+			println("Location line: "+parser.getLocation().getLineNumber()+" character: "+parser.getLocation().getColumnNumber());
+			org.txm.utils.logger.Log.printStackTrace(e);
+			parser.close();
+			return false;
+		}
+		
+		after(); // if you need to do something before closing the parser();
+		return true;
+	}
+
+	public def getLocation() {
+		if (parser != null)
+			return "Line: "+parser.getLocation().getLineNumber()+" Col: "+parser.getLocation().	getColumnNumber()
+		return null;
+	}
+	
+	protected void processNamespace() {
+	}
+	
+	protected void processStartElement()
+	{
+	}
+
+	protected void processCharacters()
+	{
+	}
+	
+	protected void processProcessingInstruction()
+	{
+	}
+	
+	protected void processDTD()
+	{
+	}
+	
+	protected void processCDATA()
+	{
+	}
+	
+	protected void processComment()
+	{
+	}
+	
+	protected void processEndElement()
+	{
+	}
+	
+	protected void processEndDocument() {
+	}
+	
+	protected void processEntityReference() {
+	}
+	
+	public static void main(String[] args)
+	{
+		File input = new File("/home/mdecorde/xml/xiinclude/master.xml")
+		File output = new File("/home/mdecorde/xml/xiinclude/merged.xml")
+		if (!(input.exists() && input.canRead())) {
+			println "cannot found $input";
+			return;
+		}
+		def builder = new StaxParser(input.toURI().toURL());
+		if (builder.process()) {
+			println "success ? "+ValidateXml.test(output);
+		} else {
+			println "failure !"
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/AnnotationInjectionFilter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/AnnotationInjectionFilter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/AnnotationInjectionFilter.groovy	(revision 321)
@@ -0,0 +1,176 @@
+package org.txm.macro.annotation
+
+import javax.xml.stream.*
+
+import org.txm.importer.StaxIdentityParser
+import org.txm.importer.filters.*
+import org.txm.importer.graal.PersonalNamespaceContext
+
+/**
+ * The Class AnnotationInjection.
+ *
+ * @author mdecorde
+ *
+ * inject annotation from a stand-off file into a xml-tei-txm
+ * file
+ */
+
+public class AnnotationInjectionFilter extends StaxIdentityParser {
+
+	public static String TXMNS = "http://textometrie.org/1.0"
+
+	/** The xml reader factory. */
+	private def factory;
+
+	def xmlFile;
+	def records;
+	def properties;
+	int n = 0;
+	boolean debug = false
+
+	public AnnotationInjectionFilter() {
+		super(null, null, null);
+	}
+	
+	/**
+	 * Instantiates a new annotation injection.
+	 *
+	 * @param url the xml-tei-txm file
+	 * @param anaurl the stand-off file
+	 */
+	public AnnotationInjectionFilter(File xmlFile, def records, def properties, def debug) {
+		super(xmlFile.toURI().toURL()); // init reader and writer
+		
+		this.debug = debug
+		this.xmlFile = xmlFile
+		this.records = records
+		this.properties = properties
+		this.n = 0;
+		//println ""+records.size()+" lines to process..."
+
+		try {
+			factory = XMLInputFactory.newInstance();
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	def data; // the word id properties to add/replace
+	String newform = null
+	String wordId;
+	String anaType;
+	boolean start = false;
+	boolean replaceAnnotation = false;
+	protected void processStartElement() {
+		// this block don't write but only do tests
+		switch (localname) {
+			case "form":
+				if (start) {
+					def f = data.get("word")
+					if (f != null && f.length() > 0)
+						newform = f
+				}
+			case "w":
+				for (int i= 0 ; i < parser.getAttributeCount() ; i++ ) {
+					if (parser.getAttributeLocalName(i) == "id") {
+						wordId = parser.getAttributeValue(i);
+						data = records.get(wordId)
+						if (records.containsKey(wordId)) {
+							n++;
+							start = true; // found a word to update !
+							//println "found word $wordId"
+						}
+						break;
+					}
+				}
+				break;
+			case "ana":
+				if (start) { // found a word to update
+					for (int i= 0 ; i < parser.getAttributeCount() ; i++) { // look for the "type" attribute
+						if (parser.getAttributeLocalName(i) == "type") { 
+							anaType = parser.getAttributeValue(i);
+							anaType = anaType.substring(1)
+							//if (debug) println "type: $anaType"
+							replaceAnnotation = properties.contains(anaType) // there is a new value for this property
+							//if (replaceAnnotation) println " replace annotation $anaType"
+							break;
+						}
+					}
+				}
+				break;
+		}
+
+		super.processStartElement();
+	}
+
+	@Override
+	protected void processCharacters() {
+		if (start && replaceAnnotation) return; // don't write annotation chars if we are currently replacing
+		if (newform != null) { // don't rewrite form content
+			// we are currently in a <w> tags
+		} else {
+			super.processCharacters();
+		}
+	}
+
+	protected void processEndElement() {
+		if (start) { // write the new content of the <w> tags
+			switch (localname) {
+				case "w":
+					start = false
+					if (data.keySet().size() > 0) writer.writeCharacters("\n");
+					// write remaining properties
+					if (debug) if (data.size() > 0) println "new ana: " +data
+					for (def prop : data.keySet()) { // (1)
+						// write prop
+						writer.writeStartElement(TXMNS, "ana")
+						writer.writeAttribute("resp", "txm")
+						writer.writeAttribute("type", "#$prop")
+						writer.writeCharacters(data.get(prop))
+						writer.writeEndElement()
+						writer.writeCharacters("\n")
+						println " create annotation $prop with "+data.get(prop)
+					}
+					break;
+				case "form":
+					if (start) newform = null
+				case "ana":
+					if (replaceAnnotation) {
+						writer.writeCharacters(data.get(anaType))
+						if (debug) println " replace annotation $anaType content with "+data.get(anaType)
+						data.remove(anaType) // the anaType is removed because it is now written. I don't need to write it here (1)
+					}
+					anaType = ""
+					replaceAnnotation = false;
+					break;
+			}
+		}
+		super.processEndElement();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		File xmlFile = new File("/home/mdecorde/TEMP/annotation/01_DeGaulle.xml")
+		def properties = ["p1", "lemma"]
+		def records = [
+			"w_1":["p1":"a", "lemma":"b"],
+			"w_224":["p1":"c", "lemma":"d"],
+			"w_400":["p1":"e", "lemma":"f"],
+			"w_750":["p1":"g", "lemma":"h"],
+			"w_753":["p1":"i", "lemma":"j"],
+			"w_756":["p1":"k", "lemma":"l"]
+		]
+
+		def builder = new AnnotationInjectionFilter(xmlFile, records, properties);
+		builder.process(new File("/home/mdecorde/TEMP", "rez.xml"));
+
+		return;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/InjectWordPropTableMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/InjectWordPropTableMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/InjectWordPropTableMacro.groovy	(revision 321)
@@ -0,0 +1,102 @@
+package org.txm.macro.annotation;
+
+import org.txm.utils.*
+import org.txm.utils.io.FileCopy;
+
+import java.nio.charset.Charset
+
+import org.kohsuke.args4j.*
+
+import groovy.transform.Field
+
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+@Field @Option(name="inputDirectory", usage="The XML-TXM folder", widget="Folder", required=true, def="")
+def inputDirectory = new File(System.getProperty("user.home"), "xml/anapovoas/HELPTXM-02-131124/ite02")
+
+@Field @Option(name="outputDirectory", usage="The result folder", widget="Folder", required=true, def="")
+def outputDirectory = new File(System.getProperty("user.home"), "xml/anapovoas/HELPTXM-02-131124/ite03")
+
+@Field @Option(name="tsvFile", usage="The annotation TSV file", widget="File", required=true, def="")
+def tsvFile = new File(System.getProperty("user.home"), "xml/anapovoas/HELPTXM-02-131124/TSV-ITE02.csv")
+
+@Field @Option(name="properties", usage="columns to inject separated by commas", widget="String", required=true, def="p1, p2, ... , pn")
+def properties = "verificado"
+
+@Field @Option(name="debug", usage="Debug de the macro", widget="Boolean", required=true, def="false")
+def debug = true
+
+if (!ParametersDialog.open(this)) return;
+
+def split = properties.split(",")
+if (split.length == 0) {
+	println "ERROR: no property given"
+	return false
+}
+properties = []
+for (def p : split) properties << p.trim()
+
+println "Injecting annotations with the following parameters:"
+println "- inputDirectory: $inputDirectory"
+println "- outputDirectory: $outputDirectory"
+println "- tsvFile: $tsvFile"
+println "- properties: $properties"
+
+outputDirectory.mkdir()
+if (!outputDirectory.exists()) {
+	println "ERROR: could not create or read $outputDirectory directory"
+	return false
+}
+
+CsvReader records = new CsvReader(tsvFile.getAbsolutePath(), "\t".charAt(0), Charset.forName("UTF-8"))
+records.setTextQualifier((char)0)
+records.readHeaders();
+
+def header = Arrays.asList(records.getHeaders())
+if (debug) println "CSV header: $header"
+for (def s : properties) if (!header.contains(s)) {
+	println "Missing annotation property : $s"
+	return;
+}
+for (def s : ["text_id", "id"]) if (!header.contains(s)) {
+	println "Missing annotation property : $s"
+	return;
+}
+
+def injections = [:]
+int no = 1;
+while (records.readRecord()) {
+	String textid = records.get("text_id")
+	if (!injections.containsKey(textid)) injections[textid] = [:]
+	
+	def props = [:]
+	for (def s : properties) props[s] = records.get(s)
+	if (records.get("id") == "") {
+		 print "ERROR in record "+no
+	} else {
+		injections[textid][records.get("id")] = props
+	}
+	no++;
+}
+
+println "Injecting..."
+//for (String textid : injections.keySet()) {
+for (File currentXMLFile : inputDirectory.listFiles()) {
+	String textid = currentXMLFile.getName();
+	int idx = textid.indexOf(".")
+	if (idx > 0) textid = textid.substring(0, idx);
+	
+	if (injections.containsKey(textid)) {	
+		if (debug) println "Injections of "+injections[textid].size()+" lines."
+		
+		def builder = new AnnotationInjectionFilter(currentXMLFile, injections[textid], properties, debug)
+		println "injecting in $currentXMLFile $properties"
+		builder.process(new File(outputDirectory, currentXMLFile.getName()));
+		println ""+builder.n+" lines injected."
+	} else {
+		println "copying file $currentXMLFile"
+		FileCopy.copy(currentXMLFile, new File(outputDirectory, currentXMLFile.getName()))
+	}
+}
+
+println "New XML-TXM files are saved in ${outputDirectory}."
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/BuildWordPropTableMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/BuildWordPropTableMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/annotation/BuildWordPropTableMacro.groovy	(revision 321)
@@ -0,0 +1,179 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+// $LastChangedDate: 2012-10-15 17:35:36 +0200 (lun., 15 oct. 2012) $
+// $LastChangedRevision: 2279 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.macro.annotation
+
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.concordances.*
+import org.txm.functions.concordances.comparators.*
+import org.txm.functions.ReferencePattern
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+def corpus = corpusViewSelection
+if (!(corpus instanceof Corpus)) {
+	println "Error: you must select a corpus or a subcorpus to export properties"
+	return false;
+}
+
+@Field @Option(name="properties", usage="columns to inject separated by commas", widget="String", required=true, def="p1, p2, ... , pn")
+		def properties = "pos"
+
+@Field @Option(name="query", usage="The query to select words to annotate", widget="Query", required=true, def="[]")
+		def query
+
+@Field @Option(name="leftcontextsize", usage="The context sizes", widget="Integer", required=true, def="10")
+		def leftcontextsize = 10
+
+@Field @Option(name="rightcontextsize", usage="The context sizes", widget="Integer", required=true, def="10")
+		def rightcontextsize = 10
+
+@Field @Option(name="references", usage="references to show", widget="String", required=true, def="text_id")
+		def references = "text_id, p_id, s_id"
+
+@Field @Option(name="tsvFile", usage="The result TSV file", widget="File", required=true, def="export.tsv")
+		def tsvFile
+
+if (!ParametersDialog.open(this)) return;
+
+def split = properties.split(",")
+if (split.length == 0) {
+	println "ERROR: no property given"
+	return false
+}
+properties = []
+for (def p : split) properties << p.trim()
+
+println "Building Annotation Table with the following parameters: "
+println "- corpus: $corpus"
+println "- query: $query"
+println "- annotation properties: $properties"
+println "- TSV file: $tsvFile"
+
+
+if (!tsvFile.getParentFile().canWrite()) {
+	println "Error: "+tsvFile.getParentFile()+" is not writable."
+	return;
+}
+
+def annots = []
+for (def p : properties) {
+	def prop = corpus.getProperty(p)
+	if (prop == null) {
+		println "No such property $p in the $corpus corpus"
+		return false;
+	}
+	annots << prop
+}
+def id = corpus.getProperty("id")
+def word = corpus.getProperty("word")
+def text = corpus.getStructuralUnit("text")
+def text_id = text.getProperty("id")
+def refs = references.split(",")
+
+if (annots == null || annots.size() == 0) {
+	println "No such property given"//$annots in the $corpus corpus"
+	return false;
+}
+
+println "Exporting ..."
+def start = System.currentTimeMillis()
+
+// define the references pattern for each concordance line
+def referencePattern = new ReferencePattern()
+referencePattern.addProperty(text_id)
+referencePattern.addProperty(id)
+for (def annot : annots) referencePattern.addProperty(annot)
+
+def refProperties = []
+for (def ref : refs) {
+	ref = ref.trim()
+	try {
+		if (ref.contains("_")) {
+			def split2 = ref.split("_",2)
+			def refp = corpus.getStructuralUnit(split2[0].trim()).getProperty(split2[1].trim())
+			if (ref != "text_id")
+				referencePattern.addProperty(refp)
+			refProperties << refp
+		} else {
+			def p = corpus.getProperty(ref)
+			if (p != null) referencePattern.addProperty(p)
+		}
+	} catch(Exception e) { println "Error while parsing references: "+ref+" "+e}
+}
+
+// compute the concordance with contexts of 15 words on each side of the keyword
+//query = new Query(Query.fixQuery(query))
+def viewprops = [word]
+viewprops.addAll(annots)
+def concordance = new Concordance(corpus, query, word, viewprops, referencePattern, referencePattern, leftcontextsize, rightcontextsize)
+
+//println "Conc done "+(System.currentTimeMillis()-start)
+
+def writer = tsvFile.newWriter("UTF-8");
+
+//println "Writing lines..."
+start = System.currentTimeMillis()
+String annotHeader = ""
+for (def annot : annots) annotHeader += "\t$annot"
+writer.write("N"+"\t"+"Références"+"\t"+"ContexteGauche"+"\t"+"Pivot"+annotHeader+"\t"+"ContexteDroit"+"\t"+"id"+"\t"+"text_id\n")
+
+// define which occurrence properties will be displayed
+concordance.setViewProperties([word])
+int NLines = concordance.getNLines();
+int chunk = 1000;
+for (int i = 0 ; i < concordance.getNLines() ; i += chunk) {
+	List<Line> lines
+	if (i+chunk > NLines)
+		lines = concordance.getLines(i, concordance.getNLines()-1)
+	else
+		lines = concordance.getLines(i, i+chunk-1) // e.g. from 0 to 999
+
+	int n = 1
+	for (Line l : lines) {
+		String ntext = l.getViewRef().getValue(text_id)
+		String lcontext = l.leftContextToString()
+		String pivot = l.keywordToString()
+		String rcontext = l.rightContextToString()
+		String ids = l.getViewRef().getValue(id)
+		String refValue = "";
+		for (def refp : refProperties) refValue += " "+l.getViewRef().getValue(refp)
+
+		String poss = "";
+		for (def annot : annots) {
+			poss += "\t"+l.getViewRef().getValue(annot)
+		}
+
+		//println l.getKeywordsViewProperties().get(pos)
+		writer.write(""+(l.getKeywordPosition())+"\t"+refValue+"\t"+lcontext+"\t"+pivot+poss+"\t"+rcontext+"\t"+ids+"\t"+ntext+"\n")
+	}
+	writer.flush()
+}
+println "Saved in "+tsvFile.getAbsolutePath()
+writer.close()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/csv/CSV2XMLMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/csv/CSV2XMLMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/csv/CSV2XMLMacro.groovy	(revision 321)
@@ -0,0 +1,113 @@
+package org.txm.macro.csv
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import java.nio.charset.Charset
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.utils.*
+import javax.xml.stream.*
+import java.net.URL
+
+@Field @Option(name="inputFile", usage="CSV File", widget="File", required=false, def="file.csv")
+File inputFile;
+
+@Field @Option(name="columnSeparator", usage="column columnSeparator", widget="String", required=false, def="\t")
+def columnSeparator;
+
+@Field @Option(name="characterEncoding", usage="File characterEncoding", widget="String", required=false, def="UTF-8")
+def characterEncoding;
+
+@Field @Option(name="rootTag", usage="Root tag name", widget="String", required=false, def="root")
+def rootTag;
+
+@Field @Option(name="textTag", usage="Text tag name", widget="String", required=false, def="unit")
+def textTag;
+
+@Field @Option(name="metadataColumnList", usage="metadataColumnList column list separated with comma", widget="String", required=false, def="meta1,meta2")
+def metadataColumnList;
+
+@Field @Option(name="textColumnList", usage="textColumnList column list separated with comma", widget="String", required=false, def="textColumnList1,textColumnList2")
+def textColumnList;
+
+if (!ParametersDialog.open(this)) return;
+
+metadataColumnList = metadataColumnList.split(",")
+textColumnList = textColumnList.split(",")
+textTag = textTag.trim()
+rootTag = rootTag.trim()
+
+
+
+CsvReader reader = new CsvReader(inputFile.getAbsolutePath(), columnSeparator.charAt(0), Charset.forName(characterEncoding));
+if (!reader.readHeaders()) {
+	println "Error: no header"
+	return
+}
+def headers = Arrays.asList(reader.getHeaders()) 
+
+println "root tag: $rootTag"
+println "textColumnList tag: $textTag"
+println "columns: $headers"
+println "metadataColumnList columns: $metadataColumnList"
+println "textColumnList columns: $textColumnList"
+
+def ok = true
+for (String m : metadataColumnList) {
+	m = m.trim()
+	if (!headers.contains(m)) {
+		println "Error: missing metadataColumnList column: $m"
+		ok = false
+	}
+}
+for (String t : textColumnList) {
+	t = t.trim()
+	if (!headers.contains(t)) {
+		println "Error: missing textColumnList column: $t"
+	}
+}
+
+if (!ok) { return; }
+
+String name = inputFile.getName()
+int idx = name.indexOf(".")
+if (idx > 0) name = name.substring(0, idx)
+File outputfile = new File(inputFile.getParentFile(), name+".xml")
+XMLOutputFactory factory = XMLOutputFactory.newInstance()
+FileOutputStream output = new FileOutputStream(outputfile)
+XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8")
+
+writer.writeStartDocument("UTF-8","1.0")
+writer.writeCharacters("\n") // simple XML formating
+writer.writeStartElement(rootTag)
+
+int pb_n = 1;
+while (reader.readRecord()) {
+	
+	writer.writeCharacters("\n") // simple XML formating
+	
+	writer.writeEmptyElement("pb") // <pb/>
+	writer.writeAttribute("n", ""+pb_n++)
+	
+	writer.writeCharacters("\n") // simple XML formating
+	
+	writer.writeStartElement(textTag)
+	
+	for (String m : metadataColumnList) {
+		m = m.trim()
+		writer.writeAttribute(m, reader.get(m))
+	}
+	
+	for (String t : textColumnList) {
+		t = t.trim()
+		writer.writeStartElement(t)
+		writer.writeCharacters(reader.get(t)) // get textColumnList content
+		writer.writeEndElement() // t
+	}
+	
+	writer.writeEndElement() // textTag
+}
+
+writer.writeEndElement() // rootTag
+writer.close()
+output.close()
+reader.close()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/RemoveTag.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/RemoveTag.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/RemoveTag.groovy	(revision 321)
@@ -0,0 +1,135 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.macro.edition;
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.transform.dom.DOMSource
+import javax.xml.transform.stream.StreamResult
+import javax.xml.xpath.*
+
+import org.txm.metadatas.*
+import org.txm.utils.*
+import org.w3c.dom.Document
+import org.w3c.dom.Element
+
+/**
+ * Removes tags of XML file given a XPath. 
+ * @author mdecorde
+ *
+ */
+public class RemoveTag {
+	File outfile
+	String xpath
+	Document doc
+	
+	public RemoveTag(def root, def outfile, def xpath)
+	{
+		this.doc = root
+		this.outfile = outfile
+		this.xpath = xpath
+		
+		process()
+	}
+	
+	/**
+	 *
+	 * @param xmlfile the xmlfile
+	 * @param outfile the outfile
+	 * @param xpath the XPath
+	 */
+	public RemoveTag(File xmlfile, File outfile, String xpath)
+	{
+		this.outfile = outfile
+		this.xpath = xpath
+		
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		this.doc = builder.parse(xmlfile);
+		
+		process()
+	}
+	
+	private void process() {
+		def expr = XPathFactory.newInstance().newXPath().compile(xpath);
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+		
+		if (nodes != null)
+		for(def node : nodes)
+		{
+			//println "Remove node "+node
+			Element elem = (Element)node;
+			elem.getParentNode().removeChild(node);
+		}
+		save()
+		doc = null
+	}
+	
+	/**
+	 * Save.
+	 *
+	 * @return true, if successful
+	 */
+	private boolean save()
+	{
+		if (outfile == null) return true;
+	
+		try {
+			// Création de la source DOM
+			Source source = new DOMSource(doc);
+			
+			// Création du fichier de sortie
+				Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8")); 
+			Result resultat = new StreamResult(writer);
+			
+			// Configuration du transformer
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			Transformer transformer = fabrique.newTransformer();
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes"); 
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); 
+			
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			return true;
+		} catch (Exception e) {
+			org.txm.utils.logger.Log.printStackTrace(e);
+			return false;
+		}
+	}
+	
+	public static void main(String[] args) {
+		RemoveTag rt = new RemoveTag(
+			new File("/home/mdecorde/TXM/corpora/graal/import.xml"),
+			new File("/home/mdecorde/TXM/corpora/graal/import-o.xml"),
+			"//edition[@name='courante']"
+			)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/AddFacsAttributeMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/AddFacsAttributeMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/AddFacsAttributeMacro.groovy	(revision 321)
@@ -0,0 +1,90 @@
+package org.txm.macro.edition
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+try { println "ARGS=$args";} catch(Exception e) {args = [:]}
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="sourceDirectory", usage="Directory containig XML-TXM or XML SRC files", widget="Folder", required=true, def="src")
+def sourceDirectory
+
+@Field @Option(name="imageDirectory", usage="directory containing the ordered images files in subdirectories", widget="Folder", required=true, def="img")
+def imageDirectory
+
+@Field @Option(name="outputDirectory", usage="Output directory", widget="Folder", required=true, def="out")
+def outputDirectory
+
+@Field @Option(name="element", usage="The element to upgrade", widget="String", required=true, def="pb")
+def element
+
+@Field @Option(name="attribute", usage="The attribute to add", widget="String", required=true, def="facs")
+def attribute
+
+@Field @Option(name="prefix", usage="The image path prefix, if empty then the absolute file path is used", widget="String", required=true, def="../img/")
+def prefix
+
+// Open the parameters input dialog box
+if (args.size() == 0) {
+	if (!ParametersDialog.open(this)) return;
+} else {
+	monitor = args["monitor"];
+	sourceDirectory = args["sourceDirectory"]
+	imageDirectory = args["imageDirectory"]
+	outputDirectory = args["outputDirectory"]
+	element = args["element"]
+	attribute = args["attribute"]
+	prefix = args["prefix"]
+}
+// END OF PARAMETERS
+
+println "Parameters: "
+println " sourceDirectory: $sourceDirectory"
+println " imageDirectory: $imageDirectory"
+println " outputDirectory: $outputDirectory"
+println " element: $element"
+println " attribute: $attribute"
+
+assert(sourceDirectory.exists())
+assert(imageDirectory.exists())
+
+outputDirectory.deleteDir()
+outputDirectory.mkdir()
+
+if (!outputDirectory.exists()) {
+	println "Error: failed to create $outputDirectory"
+	return;
+}
+
+def srcFiles = []
+sourceDirectory.eachFile() { file ->
+	if (!file.isDirectory() && !file.isHidden() && file.getName().endsWith(".xml") && file.getName() != "import.xml") srcFiles << file
+}
+println "srcFiles=$srcFiles"
+
+for (def subdir : imageDirectory.listFiles()) {
+	if (!subdir.isDirectory()) continue;
+	
+	String name = subdir.getName();
+	File srcFile = new File(sourceDirectory, name+".xml")
+	if (!srcFile.exists()) { println "Warning: missing source file: $srcFile"; continue }
+	
+	def imgFiles = subdir.listFiles().sort()
+	if (imgFiles.size() == 0) { println "Warning: no image in $subdir"; continue }
+	
+	def imgPaths = []
+	for (def img : imgFiles) {
+		if (img.isFile() && !img.isHidden()) {
+			if (prefix != null && prefix.length() > 0)
+				imgPaths << prefix+name+"/"+img.getName()
+			else
+				imgPaths << img.getAbsolutePath()
+		}
+	}
+	
+	println "Processing '$name' directory with "+imgPaths.size()+" images."
+	
+	AddAttributeValuesInXML builder = new AddAttributeValuesInXML(srcFile, element, attribute, imgPaths);
+	builder.process(new File(outputDirectory, srcFile.getName()));
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/BuildFacsEditions.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/BuildFacsEditions.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/BuildFacsEditions.groovy	(revision 321)
@@ -0,0 +1,166 @@
+package org.txm.macro.edition
+
+import org.txm.importer.*
+import org.xml.sax.Attributes
+import org.txm.importer.filters.*
+import java.util.ArrayList
+import javax.xml.parsers.*
+import javax.xml.stream.*
+import java.net.URL
+import org.xml.sax.InputSource
+import org.xml.sax.helpers.DefaultHandler
+
+class BuildFacsEditions {
+
+	private def url
+	private def inputData
+	private def factory
+	private XMLStreamReader parser
+	OutputStreamWriter writer
+	StaxStackWriter pagedWriter = null
+
+	File editionDir
+	File xmlFile
+	File htmlFile
+	def pages = []
+	def tag, attribute, txtname, corpusname
+	boolean firstWord
+	boolean cutBefore = true;
+	boolean debug = false;
+	
+	public BuildFacsEditions(File xmlFile, File editionDir, String corpusname, String txtname, String tag, String attribute, boolean debug) {
+		inputData = xmlFile.toURI().toURL().openStream()
+		factory = XMLInputFactory.newInstance()
+		parser = factory.createXMLStreamReader(inputData)
+		
+		this.xmlFile = xmlFile
+		this.editionDir = editionDir
+		this.tag = tag
+		this.attribute = attribute
+		this.txtname = txtname
+		this.debug = debug
+	}
+	
+	int n = 1;
+	private boolean createNextOutput()
+	{
+		try {
+			def tags = closeMultiWriter();
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+			if (wordid != null) {//wordid = "w_0";
+				//println " add page $n $wordid, page=$pages"
+				pages << ["$n", wordid]
+				n++
+			}
+			// Page suivante
+			
+			htmlFile = new File(editionDir, "${txtname}_${n}.html")
+			firstWord = true
+	//println "SET FIRST WORD=true"
+			pagedWriter = new StaxStackWriter(htmlFile, "UTF-8");
+			if (debug) println "Create file $htmlFile"
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html");
+			pagedWriter.writeEmptyElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"]);
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"txm.css"]);
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"${corpusname}.css"]);
+			pagedWriter.writeStartElement("head");
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(corpusname+" Edition - Page "+n)
+			pagedWriter.writeEndElement(); // </title>
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+
+			pagedWriter.writeStartElements(tags);
+			
+			wordid = "w_0"; // default value if no word is found
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	private def closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+
+			if (firstWord) { // there was no words
+				pagedWriter.writeCharacters("");
+				pagedWriter.write("<span id=\"w_0\"/>");
+			}
+			pagedWriter.writeEndElements();
+			pagedWriter.close();
+			return tags;
+		} else {
+			return [];
+		}
+	}
+	
+	private writeImg(String src) {
+		pagedWriter.writeStartElement("div");
+		pagedWriter.writeEmptyElement("img", ["src":src, "width":"100%"]);
+		pagedWriter.writeEndElement(); // </div>
+	}
+	
+	String wordid = null;
+	public def process() {
+		
+		boolean start = false
+		String localname
+		
+		createNextOutput();
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							start = true
+						break;
+						case "w":
+							if (firstWord) {
+								wordid = parser.getAttributeValue(null, "id");
+								//println "found word: $wordid"
+								firstWord = false;
+							}
+						break;
+						case tag:
+							if (debug) println "** TAG $tag $attribute : "+parser.getAttributeValue(null, "id");
+							String imgPath = parser.getAttributeValue(null, attribute);
+							if (imgPath == null) {
+								println "ERROR in $xmlFile no value found for $tag@$attribute at location "+parser.getLocation().getLineNumber()
+							} else {
+								if (cutBefore) {
+									if (debug) println " cut before"
+									createNextOutput()
+									if (debug) println " write img $imgPath"
+									writeImg(imgPath)
+								} else {
+									if (debug) println " write img $imgPath"
+									writeImg(imgPath)
+									if (debug) println " cut after"
+									createNextOutput()
+								}
+							}
+							//firstWord = true;
+						break;
+					}
+				break;
+			}
+		}
+		closeMultiWriter()
+		pages << ["$n", wordid] // add the last page (no pb encountered
+		return pages
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/FacsEditionBuilderMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/FacsEditionBuilderMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/FacsEditionBuilderMacro.groovy	(revision 321)
@@ -0,0 +1,206 @@
+package org.txm.macro.edition
+
+// STANDARD DECLARATIONS
+
+import groovy.transform.Field
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.transform.dom.DOMSource
+import javax.xml.transform.stream.StreamResult
+
+import org.kohsuke.args4j.*
+import org.txm.Toolbox
+import org.txm.objects.*
+import org.txm.rcpapplication.commands.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.utils.xml.DomUtils
+import org.w3c.dom.*
+
+
+try { println "ARGS=$args";} catch(Exception e) {args = [:]}
+
+def corpus = null
+if (args.size() == 0) {
+	if (!(corpusViewSelection instanceof MainCorpus)) {
+		println "This marcro works with a MainCorpus selection. Aborting"
+		return;
+	}
+	println "Working on $corpusViewSelection corpus"
+	corpus = corpusViewSelection
+} else {
+	monitor = args["monitor"];
+}
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="editionName", usage="the edition name to create", widget="String", required=true, def="facs")
+def editionName
+
+@Field @Option(name="tag", usage="The tag to upgrade", widget="String", required=true, def="pb")
+def tag
+
+@Field @Option(name="attribute", usage="The attribute to add", widget="String", required=true, def="facs")
+def attribute
+
+@Field @Option(name="debug", usage="Debug mode", widget="Boolean", required=false, def="true")
+def debug = false
+
+// END OF params
+if (args.size() == 0) {
+	if (!ParametersDialog.open(this)) return;
+} else {
+	monitor = args["monitor"];
+	attribute = args["attribute"]
+	tag = args["tag"]
+	editionName = args["editionName"]
+}
+
+if (args.size() == 0) { // the "corpus" variable exists
+	binDirectory = corpus.getBase().getBaseDirectory()
+	txmDirectory = new File(binDirectory, "txm/"+corpus.getName().toUpperCase())
+	tokenizedDirectory = new File(binDirectory, "tokenized")
+	HTMLDirectory = new File(binDirectory, "HTML")
+	newEditionDirectory = new File(HTMLDirectory, corpus.getName().toUpperCase()+"/"+editionName)
+	params = corpus.getBase().params
+} else { // the "corpus" variable does not exist, get informations from the "args" map
+	binDirectory = args["binDirectory"]
+	txmDirectory = args["txmDirectory"]
+	params = args["params"]
+	tokenizedDirectory = new File(binDirectory, "tokenized")
+	HTMLDirectory = new File(binDirectory, "HTML")
+	corpusName = params.getCorpusName()
+	newEditionDirectory = new File(HTMLDirectory, corpusName.toUpperCase()+"/"+editionName)
+}
+
+println "Parameters:"
+println "	attribute = $attribute"
+println "	tag =  $tag"
+println "	editionName =  $editionName"
+
+if (!HTMLDirectory.exists()) {
+	println "ERROR: can't find this corpus 'HTML' directory: $HTMLDirectory. Aborting"
+	return false;
+}
+
+File workDirectory = txmDirectory
+if (!workDirectory.exists()) {
+	println "XML-TXM directory ($txmDirectory) not found. Using XML tokenized directory instead: "+tokenizedDirectory
+	workDirectory = tokenizedDirectory
+}
+if (!workDirectory.exists()) {
+	println "XML tokenized directory not found: "+tokenizedDirectory
+	println "Aborting."
+	return false
+}
+
+println "Working directory=$workDirectory"
+
+//0- clean previous edition if any : html files, import.xml
+if (newEditionDirectory.exists()) {
+	println "** Old version of $editionName edition found."
+	println " removing the 'edition' reference from the corpus configuration."
+	File tempParam = new File(binDirectory, "import.xml.cpy")
+	RemoveTag rt = new RemoveTag(
+			params.root.getOwnerDocument(), // will be updated
+			null, // don't create a new import.xml
+			"//edition[@name='$editionName']"
+			)
+	println " delete $newEditionDirectory"
+	newEditionDirectory.deleteDir()
+	
+	//printDOM(params.root.getOwnerDocument())
+}
+
+//2- fix import.xml
+println "** Updating corpus configuration..."
+// for edition list
+def corpusElem = params.getCorpusElement()
+params.addEditionDefinition(corpusElem, editionName, "groovy", "FacsEditionBuilderMacro");
+
+//1- create HTML files
+println "** Building new edition HTML files..."
+
+println " Creating edition '$editionName' directory: '$newEditionDirectory'"
+newEditionDirectory.mkdir()
+for (def xmlFile : workDirectory.listFiles()) {
+	if (xmlFile.isHidden() || xmlFile.isDirectory()) continue // ignore
+	String txtname = xmlFile.getName()
+	int idx = txtname.indexOf(".")
+	if (idx > 0) txtname = txtname.substring(0,idx)
+
+	// create a page per "pb" element found in the xml-txm file
+	println " Building HTML pages of text=$txtname"
+	BuildFacsEditions builder = new BuildFacsEditions(xmlFile, newEditionDirectory, params.name, txtname, tag, attribute, debug);
+	def newPages = builder.process()
+	if (newPages == null || newPages.size() == 0) {
+		println "WARNING: no edition files created with $xmlFile"
+	}
+		
+	println " Building edition references in corpus configuration"
+	
+	Element corpusElement = params.getCorpusElement();
+	//println "corpusElement $corpusElement"
+	Element textsElem = params.getTextsElement(corpusElement);
+	//println "textsElem $textsElem"
+	Element textElem = params.getTextElement(textsElem, txtname);
+	if (textElem == null) { // just in case
+		textElem = params.addText(textsElem, txtname, xmlFile)
+	}
+	//println "textElem $textElem with txtname=$txtname"
+	Element editionElem = params.addEdition(textElem, editionName, newEditionDirectory.getAbsolutePath(), "html"); // add a new edition to the "text" node
+	for (def pagedef : newPages) {
+		params.addPage(editionElem, pagedef[0], pagedef[1]);
+	}
+}
+
+//printDOM(params.root.getOwnerDocument())
+
+//3- Save and reload the corpus
+println " Saving corpus configuration..."
+File paramFile = new File(binDirectory, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);
+
+
+//4- Reload Corpora
+if (args.size() == 0) {
+	Toolbox.restart();
+	monitor.syncExec(new Runnable() {
+				public void run() {
+					RestartTXM.reloadViews();
+				}
+			});
+}
+
+//5- Done
+println "New edition created."
+
+//printDOM(params.root.getOwnerDocument())
+
+
+
+def printDOM(def doc) {
+	if (!debug) return;
+	try {
+		// Création de la source DOM
+		Source source = new DOMSource(doc);
+		
+		// Création du fichier de sortie
+		StreamResult resultat = new StreamResult(new PrintWriter(System.out));
+		
+		// Configuration du transformer
+		TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+		Transformer transformer = fabrique.newTransformer();
+		transformer.setOutputProperty(OutputKeys.METHOD, "xml"); //$NON-NLS-1$
+		transformer.setOutputProperty(OutputKeys.INDENT, "yes");  //$NON-NLS-1$
+		transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");  //$NON-NLS-1$
+		
+		// Transformation
+		transformer.transform(source, resultat);
+		// writer.close();
+		return true;
+	} catch (Exception e) {
+		e.printStackTrace();
+		return false;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/EditionUpdaterMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/EditionUpdaterMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/EditionUpdaterMacro.groovy	(revision 321)
@@ -0,0 +1,259 @@
+package org.txm.macro.edition
+// STANDARD DECLARATIONS
+
+import groovy.transform.Field
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+
+import org.kohsuke.args4j.*
+import org.txm.Toolbox
+import org.txm.importer.*
+import org.txm.objects.*
+import org.txm.rcpapplication.commands.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.utils.io.FileCopy;
+import org.w3c.dom.*
+
+try { println "ARGS=$args";} catch(Exception e) {args = [:]}
+
+def corpus = null
+if (args.size() == 0) {
+	if (!(corpusViewSelection instanceof MainCorpus)) {
+		println "This marcro works with a MainCorpus selection. Aborting"
+		return;
+	}
+	println "Working on $corpusViewSelection corpus"
+	corpus = corpusViewSelection
+} else {
+	monitor = args["monitor"];
+}
+
+// BEGINNING OF params
+
+@Field @Option(name="xslEdition", usage="XSL to build the HTML edition - if empty uses 'txm-edition-xtz.xsl'", widget="File", required=false, def="txm-edition-xtz.xsl")
+def xslEdition
+
+@Field @Option(name="xslPages", usage="XSL to build the edition pages - if empty uses 'txm-edition-page-split.xsl'", widget="File", required=false, def="txm-edition-page-split.xsl")
+def xslPages
+
+@Field @Option(name="editionName", usage="The edition name to produce", widget="String", required=false, def="default")
+String editionName
+
+@Field @Option(name="useTokenizedDirectory", usage="Use the 'XML/w' of the 'tokenized' directory instead of the 'XML-TXM' files", widget="Boolean", required=false, def="false")
+def useTokenizedDirectory = false
+
+@Field @Option(name="debug", usage="Enable debug mode: temporary files are not deleted", widget="Boolean", required=false, def="false")
+def debug = false
+
+// END OF params
+if (args.size() == 0) {
+	if (!ParametersDialog.open(this)) return;
+} else {
+	monitor = args["monitor"];
+	xslEdition = new File(args["xslEdition"])
+	xslPages = new File(args["xslPages"])
+	editionName = args["editionName"]
+	useTokenizedDirectory = args["useTokenizedDirectory"]
+}
+
+if (args.size() == 0) { // corpus informations
+	defaultEditionName = corpus.getDefaultEdition()
+	corpusName = corpus.getName()
+	binDirectory = corpus.getBase().getBaseDirectory()
+	txmDirectory = new File(binDirectory, "txm/"+corpusName.toUpperCase())
+	params = corpus.getBase().params
+} else { // the "corpus" variable does not exists
+	defaultEditionName = "default"
+	binDirectory = args["binDirectory"]
+	txmDirectory = args["txmDirectory"]
+	params = args["params"]
+	corpusName = params.getCorpusName()
+}
+
+if (editionName == null || editionName.length() == 0) {
+	editionName = defaultEditionName
+}
+println "Parameters:"
+println "	xslEdition = $xslEdition"
+println "	xslPages = $xslPages"
+println "	editionName = $editionName"
+println "	useTokenizedDirectory = $useTokenizedDirectory"
+
+File TXMHOME = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME))
+File xslDirectory = new File(TXMHOME, "xsl")
+if (xslEdition == null || xslEdition.getName() == "txm-edition-xtz.xsl")
+	xslEdition = new File(xslDirectory, "txm-edition-xtz.xsl")
+if (xslPages == null || xslPages.getName() == "txm-edition-page-split.xsl")
+	xslPages = new File(xslDirectory, "txm-edition-page-split.xsl")
+
+println "XSLs: "+xslEdition.getName()+" & "+ xslPages.getName()
+println ""
+
+
+if (useTokenizedDirectory) {
+	println "Using the 'tokenized' directory instead of the 'txm' directory to get XML files"
+	txmDirectory = new File(binDirectory, "tokenized")
+}
+File HTMLDirectory = new File(binDirectory, "HTML")
+File HTMLCorpusDirectory = new File(HTMLDirectory, corpusName.toUpperCase())
+File defaultEditionDirectory = new File(HTMLCorpusDirectory, editionName)
+
+File cssDirectory = new File(TXMHOME, "css")
+File cssDefaultEditionDirectory = new File(defaultEditionDirectory, "css")
+File cssTXM = new File(cssDirectory, "txm.css")
+File cssTEI = new File(cssDirectory, "tei.css")
+boolean newEdition = false;
+
+if (!txmDirectory.exists()) {
+	println "ERROR: can't find this corpus 'txm' directory: $txmDirectory. Aborting"
+	return false;
+}
+if (!defaultEditionDirectory.exists()) {
+	println "This is a new edition"
+	newEdition = true;
+	defaultEditionDirectory.mkdir()
+	if (!defaultEditionDirectory.exists()) {
+		println "HTML directory could be created: $defaultEditionDirectory. Aborting"
+		return false
+	}
+}
+if (!cssTXM.exists()) {
+	println "WARNING: can't find the $cssTXM CSS file. Aborting"
+	return false;
+}
+if (!cssTEI.exists()) {
+	println "WARNING: can't find the $cssTEI CSS file. Aborting"
+	return false;
+}
+if (!xslEdition.exists()) {
+	println "Error: can't find $xslEdition XSL file"
+	return false;
+}
+if (!xslPages.exists()) {
+	println "Error: can't find $xslPages XSL file"
+	return false;
+}
+
+defaultEditionDirectory.mkdir()
+cssDefaultEditionDirectory.mkdir()
+FileCopy.copy(cssTXM, new File(cssDefaultEditionDirectory, cssTXM.getName()))
+FileCopy.copy(cssTEI, new File(cssDefaultEditionDirectory, cssTEI.getName()))
+
+//1- Back up current "HTML" directory
+if (!newEdition) {
+	File backupDirectory = new File(binDirectory, "HTML-"+defaultEditionDirectory.getName()+"-back")
+	backupDirectory.mkdir()
+	println "Backup of $defaultEditionDirectory directory to $backupDirectory..."
+	for (File f : defaultEditionDirectory.listFiles()) {
+		String name = f.getName()
+		if (f.isDirectory() || f.isHidden()) continue
+		
+		File rez = new File(backupDirectory, f.getName())
+		
+		if (debug) println " file $f >> $rez"
+		else print "."
+		
+		if (!FileCopy.copy(f, rez)) {
+			println "Error: failed to backup $f"
+			return false;
+		}
+	}
+	println ""
+}
+
+//2- Apply edition XSL
+println "Applying XSL 1: $xslEdition..."
+ApplyXsl2 applier = new ApplyXsl2(xslEdition);
+def htmlFiles = []
+for (File f : txmDirectory.listFiles()) {
+	String name = f.getName()
+	String txtname = name.substring(0, name.indexOf("."));
+	File rez = new File(HTMLCorpusDirectory, txtname+".html")
+
+	if (!f.isDirectory() && !f.isHidden() && name.endsWith(".xml") && !name.equals("import.xml")) {
+	
+		if (debug) println " file $f >> $rez"
+		else print "."
+	
+		if (!applier.process(f, rez)) {
+			println "Error: failed to process $f"
+			return false
+		} else {
+			htmlFiles << rez
+		}
+	}
+}
+println ""
+
+//3- Apply pages XSL
+println "Applying XSL 2: $xslPages..."
+ApplyXsl2 applier2 = new ApplyXsl2(xslPages);
+applier2.setParam("editionname", editionName)
+applier2.setParam("cssname", corpusName)
+for (File f : htmlFiles) {
+	String name = f.getName()
+	String txtname = name.substring(0, name.indexOf("."));
+	File rez = new File(defaultEditionDirectory, txtname+"-pages.html")
+	
+	if (debug) println " file $f >> $rez"
+	else print "."
+	
+	if (!applier2.process(f, rez)) {
+		println "Error: failed to process $f"
+		return false
+	} else {
+		if (!debug) rez.delete()
+	}
+}
+println ""
+
+// clean temp files
+if (!debug) {
+	for (File f : htmlFiles) {
+		f.delete()
+	}
+}
+
+//4- register new edition if any (copy edition)
+
+if (editionName != defaultEditionName) {
+	println "Update corpus configuration"
+	
+	// remove edition declaration if any
+	RemoveTag rt = new RemoveTag(
+			params.root.getOwnerDocument(), // will be updated
+			null, // don't create a new import.xml
+			"//edition[@name='$editionName']"
+			)
+
+	def corpusElem = params.getCorpusElement()
+	params.addEditionDefinition(corpusElem, editionName, "xsl", "XSLEditionBuilder"); // declare the new edition
+	
+	for (def text : corpus.getTexts()) {
+		Element textElem = text.getSelfElement()
+		def defaultEdition = text.getEdition(defaultEditionName)
+		if (defaultEdition == null) { println "Error: no default edition with name="+defaultEditionName; return false} 
+		Element editionElem = params.addEdition(textElem, editionName, defaultEditionDirectory.getAbsolutePath(), "html");
+		def pages = defaultEdition.getPages()
+		for (int i = 1 ; i <= pages.size() ; i++) {
+			def page = pages[i-1]
+			params.addPage(editionElem, "$i", page.getWordId());
+		}
+	}
+	
+	File paramFile = new File(binDirectory, "import.xml");
+	DomUtils.save(params.root.getOwnerDocument(), paramFile);
+}
+
+//5- Reload Corpora
+if (args.size() == 0) { // the user run the macro
+	Toolbox.restart();
+	monitor.syncExec(new Runnable() {
+			public void run() {
+				RestartTXM.reloadViews();
+			}
+		});
+}
+println "New edition created."
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/AddAttributeValuesInXML.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/AddAttributeValuesInXML.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/AddAttributeValuesInXML.groovy	(revision 321)
@@ -0,0 +1,88 @@
+package org.txm.macro.edition
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import org.txm.importer.*
+
+/**
+ * Add a attribute value map in a XML file
+ * Warning: if an attribute already exists its value won't be changed
+ */
+class AddAttributeValuesInXML extends StaxIdentityParser {
+	File xmlFile;
+	String tag, attribute;
+	List<String> values;
+	HashMap<String, String> attributesMap;
+	boolean result;
+
+	public AddAttributeValuesInXML(File xmlFile, String tag, String attribute, List<String> values)
+	{
+		super(xmlFile.toURI().toURL());
+		this.xmlFile = xmlFile;
+		this.tag = tag;
+		this.attribute = attribute;
+		this.values = values;
+	}
+
+	public boolean process(File outfile) {
+		this.result = false;
+		boolean ret = super.process(outfile)
+		return this.result & ret;
+	}
+
+	
+	/**
+	 * Rewrite the processStartElement() to update/add attributes
+	 */
+	int n = 0;
+	public void processStartElement()
+	{
+		if (localname != tag) {
+			super.processStartElement()
+		} else {
+			String prefix = parser.getPrefix();
+//TODO: uncomment for TXM 0.7.6
+//			if (INCLUDE == localname && XI == prefix) {
+//				processingXInclude();
+//				return;
+//			}
+
+			if (prefix.length() > 0)
+				writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+			else
+				writer.writeStartElement(localname);
+
+			for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {
+				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+			}
+
+			// get attributes
+			HashMap<String, String> attributes = new HashMap<String, String>();
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				attributes[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i);
+			}
+			// add/update the value
+			if (n < values.size()) {
+				attributes[attribute] = values[n]; 
+			} else {
+				println "ERROR: not enough values to insert for file $xmlFile, at XML parser location: l="+parser.getLocation().getLineNumber()+",c="+parser.getLocation().getColumnNumber()+")."
+			}
+			n++
+			
+			// write attributes
+			for (def k : attributes.keySet()) {
+				writer.writeAttribute(k, attributes[k])
+			}
+		}
+	}
+	
+	@Override
+	public void after() {
+		super.after();
+		
+		if (n != values.size()) {
+			println "ERROR: number of $tag ("+n+") missmatch the number of values to insert: "+values.size()
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/BuildXTZEditions.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/BuildXTZEditions.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/edition/BuildXTZEditions.groovy	(revision 321)
@@ -0,0 +1,144 @@
+package org.txm.macro.edition
+
+import org.txm.importer.*
+import org.xml.sax.Attributes
+import org.txm.importer.filters.*
+import java.util.ArrayList
+import javax.xml.parsers.*
+import javax.xml.stream.*
+import java.net.URL
+import org.xml.sax.InputSource
+import org.xml.sax.helpers.DefaultHandler
+
+class BuildXTZEditions {
+
+	private def url
+	private def inputData
+	private def factory
+	private XMLStreamReader parser
+	OutputStreamWriter writer
+	StaxStackWriter pagedWriter = null
+
+	File editionDir
+	File xmlFile
+	File htmlFile
+	def pages = []
+	def txtname, corpusname
+	int wordsPerPage = 500
+	boolean firstWord
+	boolean cutBefore = true;
+	
+	public BuildFacsEditions(File xmlFile, File editionDir, String corpusname, String txtname, int wordsPerPage) {
+		inputData = xmlFile.toURI().toURL().openStream()
+		factory = XMLInputFactory.newInstance()
+		parser = factory.createXMLStreamReader(inputData)
+		
+		this.xmlFile = xmlFile
+		this.editionDir = editionDir
+		this.txtname = txtname
+		this.wordsPerPage = wordsPerPage
+	}
+	
+	int n = 0;
+	private boolean createNextOutput()
+	{
+		try {
+			def tags = closeMultiWriter();
+			for (int i = 0 ; i < tags.size() ; i++) {
+				String tag = tags[i]
+				if ("body" != tag) {
+					tags.remove(i--)
+				} else {
+					tags.remove(i--) // remove "body"
+					break; // remove elements until "body tag
+				}
+			}
+			n++
+			htmlFile = new File(editionDir, "${txtname}_${n}.html")
+			firstWord = true
+
+			pagedWriter = new StaxStackWriter(htmlFile, "UTF-8");
+
+			pagedWriter.writeStartDocument("UTF-8", "1.0")
+			pagedWriter.writeStartElement("html");
+			pagedWriter.writeEmptyElement("meta", ["http-equiv":"Content-Type", "content":"text/html","charset":"UTF-8"]);
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"txm.css"]);
+			pagedWriter.writeEmptyElement("link", ["rel":"stylesheet", "type":"text/css","href":"${corpusname}.css"]);
+			pagedWriter.writeStartElement("head");
+			pagedWriter.writeStartElement("title")
+			pagedWriter.writeCharacters(corpusname+" Edition - Page "+n)
+			pagedWriter.writeEndElement(); // </title>
+			pagedWriter.writeEndElement() // </head>
+			pagedWriter.writeStartElement("body") //<body>
+
+			pagedWriter.writeStartElements(tags);
+			return true;
+		} catch (Exception ee) {
+			System.out.println(ee);
+			return false;
+		}
+	}
+	
+	private def closeMultiWriter()
+	{
+		if (pagedWriter != null) {
+			def tags = pagedWriter.getTagStack().clone();
+
+			if (firstWord) { // there was no words
+				pagedWriter.writeCharacters("");
+				pagedWriter.write("<span id=\"w_0\"/>");
+				pages << ["$n", "w_0"]
+			}
+			pagedWriter.writeEndElements();
+			pagedWriter.close();
+			return tags;
+		} else {
+			return [];
+		}
+	}
+	
+	private writeImg(String src) {
+		pagedWriter.writeStartElement("div");
+		pagedWriter.writeEmptyElement("img", ["src":src]);
+		pagedWriter.writeEndElement(); // </div>
+	}
+	
+	public def process() {
+		String wordid = "w_0"
+		boolean start = false
+		String localname
+		
+		createNextOutput();
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					switch (localname) {
+						case "text":
+							start = true
+						break;
+						case "w":
+							if (firstWord) {
+								wordid = parser.getAttributeValue(null, "id");
+								firstWord = false;
+							}
+						break;
+						case "pb":
+							if (cutBefore) {
+								pages << ["$n", wordid]
+								// WRITE PB
+								createNextOutput()
+							} else {
+								createNextOutput()
+								pages << ["$n", wordid]
+								// WRITE PB
+							}
+						break;
+					}
+				break;
+			}
+		}
+		closeMultiWriter()
+		return pages
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/EuroPress2XMLMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/EuroPress2XMLMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/EuroPress2XMLMacro.groovy	(revision 321)
@@ -0,0 +1,299 @@
+package org.txm.macro.misc
+// STANDARD DECLARATIONS
+
+import groovy.transform.Field
+import groovy.xml.QName
+
+import java.text.DecimalFormat
+
+import org.kohsuke.args4j.*
+import org.txm.importer.ValidateXml
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.w3c.tidy.Tidy
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="rootDir", usage="The directory contains the 'orig' directory which contains the html files", widget="Folder", required=false, def="/path")
+File rootDir = new File("");
+
+@Field @Option(name="encoding", usage="HTML encoding", widget="String", required=false, def="iso-8859-1")
+String encoding = "iso-8859-1" // HTML files encoding
+
+@Field @Option(name="debug", usage="show debug messages. Values = true, false", widget="String", required=false, def="false")
+def debug = "true" // set true to debug the script
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+debug = ("true" == debug)
+// END OF PARAMETERS
+
+String corpusName = rootDir.getName()
+File srcDir = new File(rootDir, "orig");
+File outDir = new File(rootDir, "xhtml");
+File outDir2 = new File(rootDir, corpusName);
+File rejected = new File(rootDir, "duplicates");
+File tmpDir = new File(rootDir, "tmp");
+
+
+if (!srcDir.exists()) {
+	println "STOP, srcDir does not exists $srcDir"
+	return;
+}
+
+outDir.deleteDir()
+outDir.mkdir()
+outDir2.deleteDir()
+outDir2.mkdir()
+rejected.deleteDir()
+rejected.mkdir()
+tmpDir.deleteDir()
+tmpDir.mkdir()
+
+def allTags = new HashSet<String>();
+def allStyles = new HashSet<String>();
+def allClasses = new HashSet<String>();
+def newPrefix = "Numéro de document : "
+int itext = 1;
+def formater = new DecimalFormat("0000");
+int LIMITDIFF = 10
+def metadatas = ["DocPublicationName", "DocHeader"]
+def files = []
+srcDir.eachFileMatch(~/.*\.html/){ htmlFile -> files << htmlFile}
+files = files.sort()
+
+def done = new HashSet<String>();
+def ignored = []
+def allTitles = [:]
+def dones = [:]
+def ignoreds = []
+
+def getText(def node) {
+	//if (debug) println "node: "+node
+	String s = " ";
+	if (node instanceof String) {
+		s += " "+node
+	} else {
+		for(def c : node.children())
+			s += " "+getText(c)
+	}
+	//println " "+s.replace("\n", " ").trim();
+	return " "+s.replace("\n", " ").trim();
+}
+
+println "Nb of HTML files: "+files.size()
+for (File htmlFile : files) {
+	println "Processing file $htmlFile"
+	File tmpHTML = new File(tmpDir, htmlFile.getName())
+	tmpHTML.withWriter("UTF-8") { writer -> 
+		String txt = htmlFile.getText(encoding)
+		txt = txt.replaceAll("<p></p>", " ");
+		txt = txt.replaceAll("<p> </p>", " ");
+		txt = txt.replaceAll("<br>", "<br> ");
+		writer.write(txt)
+	}
+	
+	String name = htmlFile.getName()
+	name = name.substring(0, name.indexOf("."));
+
+	File xhtmlFile = new File(outDir, name+".xhtml")
+
+	Tidy tidy = new Tidy(); // obtain a new Tidy instance
+	tidy.setXHTML(true); // set desired config options using tidy setters
+	tidy.setInputEncoding("UTF-8")
+	tidy.setOutputEncoding("UTF-8")
+	tidy.setShowErrors(0)
+	tidy.setShowWarnings(false)
+	xhtmlFile.withWriter("UTF-8") { out ->
+		tidy.parse(tmpHTML.toURI().toURL().newInputStream(), out); // run tidy, providing an input and output stream
+	}
+
+	if (ValidateXml.test(xhtmlFile)) {
+		def root = new XmlParser().parse(xhtmlFile)
+		def tables = root.body.table.tbody.tr.td
+		if (tables.size() == 0) tables = root.body.table.tr.td
+		//println "Nb of txt : "+tables.size()
+
+		for (def text : tables) {
+			String sign = ""
+		
+			if (! text instanceof groovy.util.Node) { println "NOT NODE: "+text; continue; }
+			//println "TEXT "
+			//text.setName("text")
+			boolean endOfText = false;
+			def textMetadatas = [:]
+			for (String metadata : metadatas) {
+				textMetadatas[metadata] = ""
+			}
+			for (def subtable : text.table) text.remove(subtable)	
+			
+			for (def p : text.table.p) p.addChild(" ")		
+			
+			for (def child : text.span) {
+				if ("color:red; font-weight:bold".equals(child.@style)) {
+					//text.remove(child)
+					if (debug) "Found bold: "+child
+					child.replaceNode { node -> w(expFound: "y", child.text())}
+				}
+			}
+			
+			def startIgnoringText = false
+			def tmp =""
+			def ichar = 0
+			String title = "";
+			def ignoredText = ""
+			def children = text.children()
+			for (int idx = 0 ; idx < children.size() ; idx++) {
+				
+				def child  = children[idx]
+				if (debug) println "child: $child"
+			
+				if (startIgnoringText) {
+					if (debug) println "Ignoring text : "+ignoredText
+					if (child instanceof String) ignoredText += child
+					else ignoredText += child.text()
+					
+					def t = text.children().remove(idx);
+					//if (tmp.length() > 0) println "removing : "+t
+					idx--
+					continue; // next child
+				}
+			
+				if (child instanceof String) {
+					//println " "+child
+					ichar += child.length()
+				} else {
+					ichar += child.text().length()
+					//		allTags.add(child.name().getLocalPart())
+					//		allClasses.add(child.@class)
+					//		allStyles.add(child.@style)
+					def nn = child.name()
+					
+					try {nn = nn.getLocalPart()} catch(Exception e) {}
+					switch (nn) {
+						case "br": break;
+						case "span":
+						if (debug) println "Found span $child"
+							String classV = child.@class
+							String style = child.@style
+							if (classV != null) {
+								if (metadatas.contains(classV)) {
+									textMetadatas[classV] = (textMetadatas[classV] + " " + child.text()).trim()
+									text.remove(child);
+									idx--
+									//println "METADATA: "+classV + " = "+child.text().trim().length();
+								} else if ("TitreArticleVisu" == classV) {
+									title += getText(child)//.text().trim().replaceAll("\n", " ")+ " "
+									child.name = new QName(child.name().getNamespaceURI(), "head", child.name().getPrefix())
+								} else {
+									println "UNKNOWED CLASS: "+classV + " = "+child.text().trim().length();
+								}
+							} else if (style != null) {
+								if ("color:red; font-weight:bold".equals(style)) {
+									//child.replaceNode { node -> w(expFound: "test")	}
+									//println "KEYWORD: "+child.text().trim();
+								} else {
+									println "UNKNOWED STYLE: '"+style+"' = "+child.text().trim();
+								}
+							} else {
+								println "UNKNOWED SPAN: "+child.text().trim();
+							}
+							break;
+
+						case "a": break
+						case "w": break;
+						case "b":
+							startIgnoringText = true;
+							tmp = child.text()
+							//if (tmp.length() > 0) println "START REMOVING FROM : "+tmp
+							text.remove(child);
+							idx--
+							break;
+						case "i": break;
+						case "font":
+							if (debug) println "Found font $child"
+							String style = child.@style
+							if ("font-style:italic;" == style) {
+								if (debug) println "ITALIC: "+getText(child).trim();
+								child.replaceNode { node -> i(getText(child))}
+							} else if ("font-weight:bold;") {
+								if (debug) println "BOLD: "+getText(child).trim();
+								child.replaceNode { node -> b(getText(child))}
+							} else {
+								println "FSTYLE: '"+style+"' = "+getText(child).trim();
+							}
+							break;
+						default: println child.name()
+					}
+				}
+			}
+			
+			//rename td to text
+			text.name = new QName(text.name().getNamespaceURI(), "text", text.name().getPrefix())
+			
+			//Write metadatas
+			for( String metadata : metadatas) {
+				text.attributes().put(metadata, textMetadatas[metadata])
+				//sign+= " "+textMetadatas[metadata].trim()
+			}
+			
+			// get document number
+			ignoredText = ignoredText.replaceAll("\n", " ")
+			int iNo= ignoredText.indexOf(newPrefix);
+			//println ignoredText
+			if (iNo >= 0) {
+				String no =ignoredText.substring(iNo+newPrefix.length()).trim()
+				text.attributes().put("idnews", no)
+				//sign += " "+no
+				text.attributes().put("date", no.substring(5,9)+"-"+no.substring(9,11)+"-"+no.substring(11,13))
+				text.attributes().put("yyyymmdd", no.substring(5,13))
+				text.attributes().put("yyyymm", no.substring(5,11))
+				text.attributes().put("yyyy", no.substring(5,9))
+				text.attributes().put("mm", no.substring(9,11))
+				text.attributes().put("dd", no.substring(11,13))
+			}
+			
+			//sign += " "+ichar
+			sign += " "+title
+			
+			if (allTitles[title] == null) allTitles[title] = ichar
+			if (Math.abs(allTitles[title] - ichar) > LIMITDIFF) {
+				sign += " "+ichar
+			}
+			File xmlFile;
+			if (done.contains(sign)) {
+				ignored << sign
+				xmlFile = new File(rejected, name+"_"+formater.format((itext++))+".xml")
+				ignoreds << xmlFile.getName()
+			} else {
+				done << sign;
+				xmlFile = new File(outDir2, name+"_"+formater.format((itext++))+".xml")
+				dones[sign] = xmlFile
+			}
+			def writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(xmlFile) , "UTF-8")); //$NON-NLS-1$
+				writer.println "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+				new XmlNodePrinter(writer).print(text)
+		}
+	}
+}
+
+if (ignored.size() > 0) {
+	File ignoredFile = new File (rejected, "ignored.txt");
+	ignoredFile.withWriter("UTF-8") { writer ->
+		writer.println "TOTAL: "+ignored.size()
+		for (int i = 0 ; i < ignored.size() ; i++) {
+			def sign = ignored[i]
+			writer.println "\n**DUPLICATE\n "
+			writer.println "keeped="+dones[sign];
+			writer.println "rejected="+ignoreds[i];
+			writer.println "SIGN="+sign
+			writer.println "\n"
+		}
+	}
+	println "TOTAL IGNORED: "+ignored.size()
+}
+
+println "TOTAL TEXT: $itext"
+if (!debug) {
+	outDir.deleteDir()
+	tmpDir.deleteDir()
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/TXM2CoNLL2009Macro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/TXM2CoNLL2009Macro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/TXM2CoNLL2009Macro.groovy	(revision 321)
@@ -0,0 +1,43 @@
+package org.txm.macro.misc
+import org.txm.export.conll2009.ToCoNLL2009
+
+import java.io.File;
+import org.txm.searchengine.cqp.corpus.*
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+if (!(corpusViewSelection instanceof Corpus)) {
+	println "Select a corpus or a sub-corpus"
+	return;
+}
+def corpus = corpusViewSelection
+
+@Field @Option(name="outputFile", usage="an example file", widget="File", required=true, def="conn_export.tsv")
+File outputFile = new File("conn_export.tsv")
+@Field @Option(name="encoding", usage="sentenceProperty", widget="String", required=true, def="UTF-8")
+String encoding = "UTF-8"
+
+@Field @Option(name="sentenceProperty", usage="sentenceProperty", widget="String", required=true, def="s_id")
+String sentenceProperty = "s_id"
+@Field @Option(name="posProperty", usage="sentenceProperty", widget="String", required=true, def="pos")
+String posProperty = "frpos"
+@Field @Option(name="lemmaProperty", usage="sentenceProperty", widget="String", required=true, def="lemma")
+String lemmaProperty = "frlemma"
+
+if (!ParametersDialog.open(this)) return;
+
+def split = sentenceProperty.split("_", 2)
+StructuralUnitProperty s = corpus.getStructuralUnit(split[0]).getProperty(split[1])
+Property word = corpus.getProperty("word")
+Property lemma = corpus.getProperty(lemmaProperty)
+Property pos = corpus.getProperty(posProperty)
+
+if (s == null) { println "Error sentence property: $sentenceProperty"; return}
+if (word == null) { println "Error no word property"; return}
+if (pos == null) { println "Error pos property: $posProperty"; return}
+if (lemma == null) { println "Error lemma property: $lemmaProperty"; return}
+
+
+ToCoNLL2009 processor = new ToCoNLL2009()
+processor.process(outputFile, corpus, s, word, lemma, pos, encoding)
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/Specif2CoinMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/Specif2CoinMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/misc/Specif2CoinMacro.groovy	(revision 321)
@@ -0,0 +1,171 @@
+// Copyright © 2016 ENS de Lyon
+//
+// Authors:
+// - Serge Heiden
+//
+// Licence:
+// This file is part of the TXM platform.
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+// Version:
+// $LastChangedDate: 2014-11-01 16:00:01 +0100 (sam., 1 nov. 2014) $
+// $LastChangedRevision: XXXX $
+// $LastChangedBy: sheiden $
+//
+
+/* --------------   Specif2Coin    --------------
+
+FR:
+
+Macro affichant la probabilité a priori (avant de faire les lancés) d'obtenir N faces 'pile' consécutives en lançant une pièce au pile ou face.
+
+On considère qu'une pièce a 50% de chances (1 chance sur 2) de tomber sur la face 'pile' à chaque lancé (les lancés sont indépendants).
+
+La probabilité est mise en regard avec la spécificité équivalente.
+
+La macro prend un seul paramètre :
+
+* nthrows : le nombre de lancés 'pile' consécutifs
+
+La macro affiche - selon les options :
+
+* displayProba : toutes les probabilités de 1 à nthrows lancés en détaillant :
+-  n : le nombre de 'pile' consécutifs
+-  p : la probabilité correspondante
+-  % : le pourcentage correspondant
+- pe : la probabilité exprimée en notation avec exposant en base 10
+-  s : la spécificité équivalente (l'exposant de la probabilité, soit son logarithme en base 10)
+
+* onlyMainRanks : s'utilise avec l'option displayProba pour n'afficher que les probabilités des rangs décimaux principaux (1, 2..., 10, 20..., 100, 200...)
+
+* displayCoins : plutôt que le détail de la probabilité, liste pour une spécificité S- donnée, le nombre équivalent de lancés 'pile' consécutifs correspondants exprimé sous la forme d'un intervalle : nombre de lancés minimum - nombre de lancés maximum
+Par exemple '-10 = 30-33' signifie : une spécificité S- de '-10' équivaut à entre 30 et 33 lancés 'pile' consécutifs
+
+Les limites du calcul sont celles de la machine. Une machine 64-bit peut typiquement calculer la probabilité de 1023 lancés consécutifs.
+
+*/
+
+// STANDARD DECLARATIONS
+package org.txm.macro.misc
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.eclipse.ui.console.*
+
+// BEGINNING OF PARAMETERS
+
+@Field @Option(name="nthrows", usage="maximum number of throws", widget="Integer", required=true, def="99")
+def nthrows
+
+@Field @Option(name="displayProba", usage="display probabilities", widget="Boolean", required=false, def="true")
+def displayProba
+
+@Field @Option(name="onlyMainRanks", usage="only display main rank lines", widget="Boolean", required=false, def="false")
+def onlyMainRanks
+
+@Field @Option(name="displayCoins", usage="display equivalent of coin flipping", widget="Boolean", required=false, def="true")
+def displayCoins
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return
+
+// END OF PARAMETERS
+
+def clearConsole = { ->
+	// clear the console
+	(ConsolePlugin.getDefault().getConsoleManager().getConsoles())[0].clearConsole()
+}
+
+clearConsole()
+
+min = [:]
+max = [:]
+
+pow = 2**(nthrows)
+powd = pow.doubleValue()
+
+if (powd == Double.POSITIVE_INFINITY || powd == Double.NEGATIVE_INFINITY) {
+	println sprintf("** Impossible to calculate probabilities for this value, try a lower value (impossible to represent 2 to the power of %d with this machine). Aborting.", nthrows)
+	return
+}
+
+d = 1/powd
+
+significandSize = ((-java.lang.Math.log10(d) as double)-1 as int)+4
+
+format = sprintf("%%4d = %%%d.%df = %%0%d.%df%%%% = %%9.2e = %%4d", significandSize, significandSize-2, significandSize-1, significandSize-4)
+
+unit = 1
+dec = 1
+
+if (displayProba) {
+	println """Légende :
+-  n : le nombre de 'pile' consécutifs
+-  p : la probabilité correspondante
+-  % : le pourcentage correspondant
+- pe : la probabilité exprimée en notation avec exposant en base 10
+-  s : la spécificité équivalente (l'exposant de la probabilité, soit son logarithme en base 10)
+"""
+	spc = ' ' * (significandSize+2)
+	println "   n    p"+spc+" %"+spc+"  pe         s"
+}
+
+nthrows.times {
+    n = it+1
+	p = 1/2**(n) as double
+	s = java.lang.Math.log10(p)-1 as int
+	if (min[s]) { if (min[s] > n) min[s] = n } else min[s] = n
+	if (max[s]) { if (max[s] < n) max[s] = n } else max[s] = n
+	if (displayProba) {
+		s = sprintf(format, n, p, p*100, p, s).replaceAll(/ = 0([^,])/, ' =  $1')
+		m0 = s =~ /(....)0+ = ([^-])/
+		if (m0.count > 0) {
+			spc0 = m0[0][1] + (' ' * (m0[0][0].size()-8)) + ' = ' + m0[0][2]
+			s = m0.replaceFirst(spc0)
+		}
+		m0p = s =~ /(....)0+% = /
+		if (m0p.count > 0) {
+			spc0p = m0p[0][1] + '%' + (' ' * (m0p[0][0].size()-8)) + ' = '
+			s = m0p.replaceFirst(spc0p)
+		}
+
+		if (!onlyMainRanks || ((n % dec) == 0) || (n == nthrows)) {
+			println s
+		}
+		
+		unit = unit+1
+		if (unit >= dec * 10) {
+			dec = dec * 10
+			unit = 1
+		}
+	}
+}
+
+if (displayCoins) {
+
+	if (displayProba) {
+		println ""
+	}
+
+	println """Légende :
+- -S      : spécificité équivalente
+-  lancés : intervalle des lancés 'pile' consécutifs correspondant
+"""
+	println " -S   lancés"
+	min.each { key, value -> if (min[key] != max[key]) println sprintf("%3d = %d-%d", key, min[key], max[key]) else println sprintf("%3d = %d", key, min[key])}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains various macro samples</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/stats/Specif2ThrowMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/stats/Specif2ThrowMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/stats/Specif2ThrowMacro.groovy	(revision 321)
@@ -0,0 +1,187 @@
+// Copyright © 2016 ENS de Lyon, CNRS, University of Franche-Comté
+// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)
+// @author sheiden
+
+/* --------------   Specif2Throw    --------------
+
+FR:
+
+Macro affichant la probabilité a priori (avant de faire les lancés ou les tirages) d'obtenir N succès consécutifs
+dans des jeux de lancés ou de tirages simples en regard avec des valeurs de spécificités.
+Il s'agit de jeux de lancés ou tirages connus de tous : le lancé de pièce pour obtenir pile ou face, le lancé de dé
+à 6 faces, tirer une carte dans un jeu de 32 ou 52 cartes, etc.
+On considère qu'une pièce a 50% de chances (1 chance sur 2) de tomber sur la face 'pile' à chaque lancé.
+On considère qu'un dé a 16,6% de chances (1 chance sur 6) de tomber sur '6' à chaque lancé.
+On considère qu'on a 3,1% de chances (1 chance sur 32) de tirer un as de trèfle à chaque tirage dans un jeu de 32 cartes.
+On considère qu'on a 1,9% de chances (1 chance sur 52) de tirer un as de trèfle à chaque tirage dans un jeu de 52 cartes.
+etc.
+On considère que les lancés ou les tirages sont indépendants et que les objets ne sont pas biaisés (eg le dé n'est pas pipé).
+
+La probabilité est mise en regard avec la spécificité équivalente pour mettre en lien
+des valeurs de spécificités calculées par TXM et l'intuition que l'on a de chances de succès
+dans un jeu de lancé ou de tirage.
+Il s'agit bien sûr uniquement d'illustrer l'ordre de grandeur de la surprise à avoir en
+interprétant directement le modèle des spécificités. Il ne s'agit en aucun cas d'illustrer
+une quelconque assimilation de l'apparition de mots dans des textes à un jeu de hasard.
+
+La macro prend deux paramètres :
+
+* cardOmega : le nombre d'issues possibles pour le jeu de lancé ou de tirage considéré
+              (Omega est l'univers de l'expérience aléatoire, cardOmega son cardinal)
+ - pour un lancé de pièce : choisir 2 pour cardOmega
+   (Omega = {pile, face})
+ - pour un lancé de dé : choisir 6 pour cardOmega
+   (Omega = {1, 2, 3, 4, 5, 6})
+ - pour un tirage de carte dans un jeu de 32 cartes : choisir 32 pour cardOmega
+   (Omega = {1 de trèfle, 2 de trèfle, 3 de trèfle, 4 de trèfle, 5 de trèfle, 6 de trèfle, 7 de trèfle,
+   8 de trèfle, 9 de trèfle, 10 de trèfle, valet de trèfle, dame de trèfle, roi de trèfle, 1 de carreau,
+   2 de carreau, etc.})
+ - pour un tirage de carte dans un jeu de 52 cartes : choisir 52 pour cardOmega
+ - etc.
+* nthrows : le nombre maximum de lancés ou de tirages avec succès consécutifs
+
+La macro affiche - selon les options :
+
+* displayProba : toutes les probabilités de 1 à nthrows lancés ou tirages en détaillant :
+-  n : le nombre de succès consécutifs
+-  p : la probabilité correspondante
+-  % : le pourcentage correspondant
+- pe : la probabilité exprimée en notation avec exposant en base 10
+-  s : la spécificité équivalente (l'exposant de la probabilité, soit son logarithme en base 10)
+
+* onlyMainRanks : s'utilise avec l'option displayProba pour n'afficher que les probabilités des rangs décimaux principaux (1, 2..., 10, 20..., 100, 200...)
+
+* displayThrows : plutôt que le détail de la probabilité, liste pour une spécificité S- donnée, le nombre équivalent de lancés ou de tirages avec succès consécutifs correspondants exprimé sous la forme d'un intervalle : nombre de lancés ou tirages minimum - nombre de lancés ou tirages maximum
+Par exemple '-10 = 30-33' signifie : une spécificité S- de '-10' équivaut à entre 30 et 33 lancés ou tirages avec succès consécutifs (cas du lancé de pièce, cardOmega = 2)
+
+Les limites du calcul sont liées au modèle de la mémoire de la machine. Une machine 64-bit peut typiquement calculer la probabilité de 1023 lancés de pièce avec succès consécutifs.
+
+*/
+
+// STANDARD DECLARATIONS
+package org.txm.macro.stats
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.eclipse.ui.console.*
+
+// BEGINNING OF PARAMETERS
+
+@Field @Option(name="cardOmega", usage="card(Ω): 2=coin, 6=dice, 32=32-card deck, 52=52-card deck, etc.", widget="Integer", required=true, def="2")
+def cardOmega
+
+@Field @Option(name="nthrows", usage="maximum number of successful throws or draws", widget="Integer", required=true, def="99")
+def nthrows
+
+@Field @Option(name="displayProba", usage="display probabilities", widget="Boolean", required=false, def="true")
+def displayProba
+
+@Field @Option(name="onlyMainRanks", usage="only display main rank lines", widget="Boolean", required=false, def="false")
+def onlyMainRanks
+
+@Field @Option(name="displayThrows", usage="display equivalent specificity value for each number or interval number of throws or draws", widget="Boolean", required=false, def="true")
+def displayThrows
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return
+
+// END OF PARAMETERS
+
+def clearConsole = { ->
+	// clear the console
+	(ConsolePlugin.getDefault().getConsoleManager().getConsoles())[0].clearConsole()
+}
+
+clearConsole()
+
+min = [:]
+max = [:]
+
+pow = cardOmega**(nthrows)
+powd = pow.doubleValue()
+
+if (powd == Double.POSITIVE_INFINITY || powd == Double.NEGATIVE_INFINITY) {
+	println sprintf("** Impossible to calculate probabilities for this value, try a lower value (impossible to represent cardOmega to the power of %d with this machine). Aborting.", nthrows)
+	return
+}
+
+d = 1/powd
+
+significandSize = ((-java.lang.Math.log10(d) as double)-1 as int)+4
+
+format = sprintf("%%4d = %%%d.%df = %%0%d.%df%%%% = %%9.2e = %%4d", significandSize, significandSize-2, significandSize-1, significandSize-4)
+
+unit = 1
+dec = 1
+
+println "cardOmega = "+cardOmega
+
+if (displayProba) {
+	println """Légende :
+-  n : le nombre de lancés ou tirages avec succès consécutifs
+-  p : la probabilité correspondante
+-  % : le pourcentage correspondant
+- pe : la probabilité exprimée en notation avec exposant en base 10
+-  s : la spécificité équivalente (l'exposant de la probabilité, soit son logarithme en base 10)
+"""
+	spc = ' ' * (significandSize+2)
+	println "   n    p"+spc+" %"+spc+"  pe         s"
+}
+
+nthrows.times {
+    n = it+1
+	p = (1/cardOmega)**(n) as double
+	s = java.lang.Math.log10(p)-1 as int
+	if (min[s]) { if (min[s] > n) min[s] = n } else min[s] = n
+	if (max[s]) { if (max[s] < n) max[s] = n } else max[s] = n
+	if (displayProba) {
+		s = sprintf(format, n, p, p*100, p, s).replaceAll(/ = 0([^,])/, ' =  $1')
+		m0 = s =~ /(....)0+ = ([^-])/
+		if (m0.count > 0) {
+			spc0 = m0[0][1] + (' ' * (m0[0][0].size()-8)) + ' = ' + m0[0][2]
+			s = m0.replaceFirst(spc0)
+		}
+		m0p = s =~ /(....)0+% = /
+		if (m0p.count > 0) {
+			spc0p = m0p[0][1] + '%' + (' ' * (m0p[0][0].size()-8)) + ' = '
+			s = m0p.replaceFirst(spc0p)
+		}
+
+		if (!onlyMainRanks || ((n % dec) == 0) || (n == nthrows)) {
+			println s
+		}
+		
+		unit = unit+1
+		if (unit >= dec * 10) {
+			dec = dec * 10
+			unit = 1
+		}
+	}
+}
+
+if (displayThrows) {
+
+	if (displayProba) {
+		println ""
+	}
+
+	def x = []
+	def y = []
+	
+	println """Légende :
+- -S      : spécificité équivalente
+-  lancés : intervalle des nombres de lancés ou tirages avec succès consécutifs correspondant
+"""
+	println " -S   lancés"
+	min.each { key, value ->
+	 if (min[key] != max[key]) println sprintf("%3d = %d-%d", key, min[key], max[key])
+	 else println sprintf("%3d = %d", key, min[key])
+	 x.push(key)
+	 y.push(min[key])
+	}
+	
+//	println "x = "+x
+//	println "y = "+y
+
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/stats/BasicVocabularyMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/stats/BasicVocabularyMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/stats/BasicVocabularyMacro.groovy	(revision 321)
@@ -0,0 +1,133 @@
+package org.txm.macro.stats
+// Copyright © 2010-2016 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// @author mdecorde
+// @author sheiden
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+// This macro computes the basic vocabulary of a partition
+// given a specificities table. The result is saved in a
+// TSV file.
+
+import org.txm.Toolbox
+import org.txm.functions.ca.*
+import org.txm.searchengine.cqp.clientExceptions.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.stat.*
+import org.txm.stat.data.*
+import org.txm.stat.engine.r.*
+import org.txm.functions.specificities.*
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+//TODO temporary uncommented
+//if (!(corpusViewSelection instanceof SpecificitesResult)) {
+//	println "** You need to select a Specificities result icon in the Corpus view before calling this macro. Aborting."
+//	return
+//}
+
+// PARAMETERS
+
+@Field @Option(name="outputDirectory", usage="output directory", widget="Folder", required=true, def=".")
+def outputDirectory
+
+@Field @Option(name="outputFile", usage="TSV result file", widget="String", required=true, def="banal.tsv")
+def outputFile
+
+@Field @Option(name="scoreMax", usage="score maximum threshold", widget="Float", required=false, def="1.5")
+def scoreMax
+
+//@Field @Option(name="suseSum", usage="Use the row sums : y/n", widget="String", required=false, def="n")
+def suseSum // not used
+
+// END OF PARAMETERS
+if (!ParametersDialog.open(this)) return
+
+if (suseSum == null || suseSum.length() == 0) suseSum = "n"
+boolean useSum = suseSum.toLowerCase().charAt(0) == "y"
+
+if (useSum) {
+	println "Using line sums..."
+}
+
+def output = new File(outputDirectory.toString()+"/"+outputFile)
+def writer = output.newWriter("UTF-8")
+def specif = corpusViewSelection
+def indices = specif.getSpecificitesIndex()
+def freqs = specif.getFrequency()
+def rownames = specif.getTypeNames()
+def colnames = specif.getPartShortNames()
+
+def selected = []
+
+writer.print "unit"
+writer.print "\tF"
+writer.print "\tscore_max"
+
+for (int j = 0; j < colnames.size() ; j++) {
+	writer.print "\t"+colnames[j]
+	writer.print "\tscore"
+}
+
+writer.println ""
+
+for (int i = 0; i < rownames.length ; i++) {
+	boolean add = true
+	def totF = 0
+	def totscore = 0.0
+	def maxS = 0
+
+	for (int j = 0; j < colnames.size() ; j++) {
+
+		def sp = Math.abs(indices[i][j])
+		if (sp >= scoreMax) {
+			add = false
+		} else {
+			if (sp >= maxS) maxS = sp
+		}
+		totF += freqs[i][j]
+		totscore += sp
+	}
+	
+	if (useSum) {
+		if (totscore >= scoreMax) add = false
+		else add = true
+	}
+	
+	if (add) {
+		selected << rownames[i]
+		writer.print rownames[i]
+		writer.print "\t$totF"
+		writer.print "\t$maxS"
+		for (int j = 0; j < colnames.size() ; j++) {
+			writer.print "\t"+freqs[i][j]
+			writer.print "\t"+indices[i][j]
+		}
+		writer.println ""
+	}
+}
+
+println "Selected: "+selected
+println "Saved in $output."
+
+writer.close()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TextTranscription2TRSMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TextTranscription2TRSMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TextTranscription2TRSMacro.groovy	(revision 321)
@@ -0,0 +1,54 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import org.kohsuke.args4j.*
+
+import groovy.transform.Field
+
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+
+@Field @Option(name="odtDir", usage="Dossier qui contient les fichiers ODT", widget="Folder", required=true, def="")
+File odtDir;
+
+@Field @Option(name="debug", usage="Don't clean files, 'y' to enable debug", widget="String", required=false, def="n")
+def debug = "n";
+
+if (!ParametersDialog.open(this)) return;
+
+File outDir = new File(odtDir, "out") // write result in "out" folder
+
+outDir.deleteDir()
+outDir.mkdir()
+
+println "out: $outDir"
+println "odtDir: $odtDir"
+
+println "DEBUG: "+("y" == debug || "yes" == debug)
+
+if (!odtDir.exists()) {
+	println "$odtDir does not exists. Aborting"
+}
+def monitor = null
+TextTranscription2TRS trs = new TextTranscription2TRS(outDir, "y" == debug, monitor)
+trs.process(odtDir)
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/Transana2TRS.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/Transana2TRS.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/Transana2TRS.groovy	(revision 321)
@@ -0,0 +1,267 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import javax.xml.stream.*
+
+import org.kohsuke.args4j.*
+import org.txm.doc.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.w3c.tidy.Tidy
+
+class Transana2TRS {
+	boolean debug = false;
+	boolean isDirectory = false;
+	File outDir;
+	File dtd;
+	ConvertDocument converter;
+	def monitor
+
+	Transana2TRS(File outDir, File dtd, boolean debug, def monitor) {
+		this.dtd = dtd;
+		this.outDir = outDir
+		this.monitor = monitor
+		this.debug = debug
+	}
+
+	boolean process(File file) {
+		try {
+			isDirectory = file.isDirectory();
+			if (isDirectory) {
+				return processDir(file)
+			} else {
+				return processFile(file, 100)
+			}
+		} catch(Exception e) {
+			println "Error while processing: "+e
+			if (debug) e.printStackTrace()
+		}
+	}
+
+	boolean processDir(File dir) {
+		boolean ret = true
+		try {
+			println "* Processing $dir directory"
+			converter = new ConvertDocument();
+			def files = dir.listFiles()
+			def okfiles = [];
+			if (files != null)
+				for (File file : files) {
+					String fname = file.getName().toLowerCase()
+					if ( file.isFile() &&
+					(fname.endsWith(".odt") || fname.endsWith(".doc") || fname.endsWith(".rtf"))) {
+						okfiles << file
+					}
+
+				}
+
+			if (okfiles.size() == 0) {
+				println "No ODT/DOC/RTF file found in $dir"
+				return false
+			}
+
+			int delta = 100 / okfiles.size()
+			for (File file : okfiles) {
+				monitor.worked(0, "Processing file: $file")
+				if (!processFile(file, delta)) {
+					ret = false;
+					println "ERROR: Failed to process $file"
+				}
+
+				if (monitor.isCanceled()) {
+					break; // stop looping
+				}
+			}
+		} catch(Exception e) {
+			println "Error while processing directory: "+e;
+			if (debug) e.printStackTrace();
+		}
+		if (converter != null) converter.stop();
+		return ret;
+	}
+
+	boolean processFile(File docFile, int work) {
+		if (docFile == null) return false;
+		int delta = work / 5
+		println "** Processing $docFile file"
+		String filename = docFile.getName()
+		int idx = filename.indexOf(".")
+		if (idx < 0) return false;
+		String ext = filename.substring(idx)
+		filename = filename.substring(0, idx)
+
+		//File docFile = new File(srcDir, "${filename}$ext")
+		File htmlFile = new File(outDir, "${filename}.html")
+		File htmlFile2 = new File(outDir, "${filename}_temp.html")
+		File xhtmlFile = new File(outDir, "${filename}.xhtml")
+		File xmlFile = new File(outDir, "${filename}.xml")
+		File trsFile = new File(outDir, "${filename}-tmp.trs")
+		File finalTrsFile = new File(outDir, "${filename}.trs")
+
+		monitor.worked(0, "DOCtoHTML: "+docFile.getName())
+
+		boolean ret = DOCtoHTML(docFile, htmlFile)
+		monitor.worked(delta, "HTMLtoHTMLforTidy: "+docFile.getName())
+		if (monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		ret = ret && HTMLtoHTMLforTidy(htmlFile, htmlFile2)
+		monitor.worked(delta, "HTMLtoXHTML: "+docFile.getName())
+		if (monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		ret = ret && HTMLtoXHTML(htmlFile2, xhtmlFile)
+		monitor.worked(delta, "XHTMLtoTRS: "+docFile.getName())
+		if (monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		ret = ret && XHTMLtoTRS(xhtmlFile, trsFile)
+		monitor.worked(delta, "TRStoFixedTRS: "+docFile.getName())
+		if (monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		ret = ret && TRStoFixedTRS(trsFile, finalTrsFile);
+		monitor.worked(delta, "Done: "+docFile.getName())
+		if (monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		// cleaning
+		System.gc()
+		if (!debug) {
+			htmlFile.delete();htmlFile2.delete();xhtmlFile.delete();xmlFile.delete();trsFile.delete()
+		}
+
+		return ret
+	}
+
+	boolean DOCtoHTML(File docFile, File htmlFile) {
+		println "*** ODT -> HTML"
+		try {
+			if (!isDirectory) converter = new ConvertDocument();
+			converter.setDebug(debug)
+			converter.autoFile(docFile, htmlFile, "html")
+		} catch(Exception e) {
+			println "Error while converting $docFile : $e"
+			if (debug) e.printStackTrace()
+		} finally {
+			if (!isDirectory && converter != null) converter.stop()
+		}
+		return htmlFile.exists() && htmlFile.length() > 0
+	}
+
+	boolean HTMLtoHTMLforTidy(File htmlFile, File htmlFile2) {
+		println "*** HTML -> HTML for tidy"
+		try {
+			println "replace TABS with 4 spaces"
+			String text2 = htmlFile.getText("UTF-8")
+			text2 = text2.replaceAll("&nbsp;", " ")
+			text2 = text2.replaceAll("’", "'")
+			text2 = text2.replaceAll("&rsquo;", "'")
+			text2 = text2.replaceAll("\t", "&nbsp;&nbsp;&nbsp;&nbsp;")
+			text2 = text2.replaceAll("\n", '&nbsp;')
+			text2 = text2.replaceAll("\r\n", '&nbsp;')
+			htmlFile2.withWriter("UTF-8") { writer ->
+				writer.write(text2);
+			}
+		} catch(Exception e) {
+			println "Error while preparing HTML of $htmlFile : $e"
+			if (debug) e.printStackTrace()
+		}
+		return htmlFile2.exists() && htmlFile2.length() > 0
+	}
+
+	boolean HTMLtoXHTML(File htmlFile2, File xhtmlFile) {
+		println "*** HTML for tidy -> XHTML"
+		try {
+			Tidy tidy = new Tidy(); // obtain a new Tidy instance
+			tidy.setXHTML(true); // set desired config options using tidy setters
+			tidy.setInputEncoding("UTF-8")
+			tidy.setOutputEncoding("UTF-8")
+			tidy.setShowErrors(100)
+			tidy.setShowWarnings(debug)
+			tidy.setTabsize(10)
+			tidy.setWraplen(9999)
+			tidy.setForceOutput(true) // Tidy won't stop if error are found
+			xhtmlFile.withWriter("UTF-8") { out ->
+				def input = new InputStreamReader(htmlFile2.toURI().toURL().newInputStream(), "UTF-8")
+				tidy.parse(input, out); // run tidy, providing an input and output stream
+			}
+			if (xhtmlFile.exists()) {
+				// JTidy produced a "0x0" char. removing them
+				// fix separated < and / ???
+				def c = Character.toChars(0)[0]
+				String txttmp = xhtmlFile.getText("UTF-8");
+				xhtmlFile.withWriter("UTF-8") { out ->
+					out.write(txttmp.replace("<\n/", "</").replace("<\r\n/", "</"))
+				}
+			}
+		} catch(Exception e) {
+			println "Error while applying JTidy: "+e
+			if (debug) e.printStackTrace()
+		}
+		return xhtmlFile.exists() && xhtmlFile.length() > 0
+	}
+
+	boolean XHTMLtoTRS(File xhtmlFile, File trsFile) {
+		println "*** XHTML -> TRS"
+		try {
+			HTML2TRS ht = new HTML2TRS();
+			ht.process(xhtmlFile, trsFile);
+		} catch(Exception e) {
+			println "Error while creating TRS file: "+e
+			if (debug) e.printStackTrace()
+		}
+		return trsFile.exists() && trsFile.length() > 0
+	}
+
+	boolean TRStoFixedTRS(File trsFile, File finalTrsFile) {
+		println "*** TRS -> FIXED TRS"
+		try {
+			def timingfixer = new FixTransanaTimings(trsFile, finalTrsFile);
+			if (!timingfixer.process()) {
+				println "Error while fixing timing"
+			}
+		} catch(Exception e) {
+			println "Error while fixing TRS file: "+e
+			if (debug) e.printStackTrace()
+		}
+		return finalTrsFile.exists() && finalTrsFile.length() > 0
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/FixTransanaTimings.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/FixTransanaTimings.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/FixTransanaTimings.groovy	(revision 321)
@@ -0,0 +1,220 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+class FixTransanaTimings {
+
+	File trsInFile = new File("/home/mdecorde/xml/transana/out.xml")
+	File trsOutFile = new File("/home/mdecorde/xml/transana/TrP1S8_08102010.trs")
+
+	public FixTransanaTimings(File trsInFile, File trsOutFile) {
+		this.trsInFile = trsInFile;
+		this.trsOutFile = trsOutFile;
+	}
+
+	public boolean process() {
+		def slurper = new XmlParser();
+		def trs = slurper.parse(trsInFile)
+
+		def speakers = new HashSet<String>();
+		for (def turn : trs.Episode.Section.Turn) {
+			String tmp = turn.@speaker.toUpperCase();
+			speakers << tmp
+			turn.@speaker = tmp
+		}
+		for (def speakersNode : trs.Speakers) {
+			for (String spk : speakers) {
+				def speakerNode = new Node(speakersNode, 'Speaker',
+				[id:spk, name:spk, type:'unknown', check:'yes', dialect:'native', accent:'no', scope:'local'])
+			}
+			def speakerNode = new Node(speakersNode, 'Speaker',
+				[id:"none", name:"none", type:'unknown', check:'yes', dialect:'native', accent:'no', scope:'local'])
+		}
+		
+		String currentEnd = null;
+		for (def turn : trs.Episode.Section.Turn) {
+			def children = turn.children()
+	
+			if (children.size() > 0) {
+				//println "fixing turn start="+turn.@startTime+" end="+turn.@endTime+" previous end="+currentEnd
+				if (currentEnd != null && currentEnd.length() > 0 && turn.@startTime.length() == 0) { // using last Sync
+					//println "fixing with previous Turn $currentEnd"
+					turn.@startTime = currentEnd
+					setStartTime(children[0], currentEnd)
+				} else if (!(children[0] instanceof String)) { // no previous Sync, using next Sync
+					def start = getStartTime(children[0])
+					if (start != null && start.length() > 0)	turn.@startTime = start
+				}
+				currentEnd = null
+	
+				if (!(children[children.size() - 1] instanceof String)) { // this is the last Sync
+					def end = getStartTime(children[children.size() - 1])
+					if (end != null && end.length() > 0)  {
+						turn.@endTime = end
+						currentEnd = end;
+					}
+				}
+			}
+		}
+	
+		def nodes = []
+		// get Sync nodes to fix
+		for (def section : trs.Episode.Section) { // all Section
+			for (def child : section.children()) { // all Section children
+				if (!(child instanceof String)) { // is a Tag
+					switch(child.name()) {
+						case "Turn": // get Sync in Turn
+							//nodes << child;
+							for (def sync : child.Sync)
+								nodes << sync
+							break;
+						case "Sync":
+							nodes << child;
+							break;
+						default: break;
+					}
+				}
+			}
+		}
+		
+		//Fixing Sync@time
+		for (int i = 0 ; i < nodes.size() ; i++) {
+			def node = nodes[i]
+			String time = getStartTime(node)	
+			
+			if (time.length() == 0) {				
+				def list = [];
+				String previous;
+				if (i > 0) { previous = getStartTime(nodes[i-1]);
+				} else { previous = "0.0" }
+				
+				String next = null;
+				while ((next == null || next.length() == 0) && i <= nodes.size()) {
+					list << node
+					node = nodes[++i]
+					next = getStartTime(node)
+				}
+				if (next != null) {
+					if (node != null) list << node
+					fixSyncTimes(list, previous, next)
+				} else {
+					println "ERROR: no end time in the transcription"
+				}
+			}
+		}
+	
+		// fixing startTime of Turn using next Sync.@time
+		def turns = trs.Episode.Section.Turn
+		for (int i = 0 ; i < turns.size() ; i++) {
+			def turn = turns[i]
+			def syncs = turn.Sync
+			if (syncs.size() > 0) {
+				turn.@startTime = syncs[0].@time
+			} else {
+				println "Error: Turn with no Sync milestone"
+			}
+		}
+	
+		// fixing endTimes of Turn using next Turn.@startTime
+		for (int i = 0 ; i < turns.size() ; i++) {
+			def turn = turns[i]
+			//println "turn start="+turn.@startTime+" end="+turn.@endTime
+			if (i < turns.size() - 1) {
+				if (turn.@endTime.length() == 0 && turns[i+1].@startTime.length() > 0) {
+					//println "fixing turn.@endTime "+turn.@endTime+" with turns[i+1].@startTime "+(turns[i+1]);
+					turn.@endTime = turns[i+1].@startTime
+				}
+			}  else if (turn.@endTime.length() == 0) { // last turn has no timing, aproximate it 
+				Float e = Float.parseFloat(turn.@startTime)
+				e += turn.text().length()*1.0f 
+				turn.@endTime = e.toString()
+			}
+		}
+		
+		// fixing endTimes of Section using last Turn.@endTime
+		// fixing startTimes of Section using first Turn.@startTime
+		def sections = trs.Episode.Section
+		for (int i = 0 ; i < sections.size() ; i++) {
+			def t = sections[i].Turn
+			if (t.size() == 0) continue;
+			sections[i].@startTime = t[0].@startTime
+			sections[i].@endTime = t[-1].@endTime
+		}
+		
+		trsOutFile.withWriter("UTF-8") { writer ->
+			writer.write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE Trans SYSTEM "trans-14.dtd">\n')
+			new groovy.util.XmlNodePrinter(new PrintWriter(writer)).print(trs) }
+		
+		return true;
+	}
+	
+	def getStartTime(def node) {
+		def ret = null;
+		if (node == null) return "0.0"
+		if (node.name() == null) return "0.0"
+		switch(node.name()) {
+			case "Turn":
+			case "Section":
+			//			println "Turn"
+				ret = node.@startTime
+				break
+			case "Sync":
+			//		println "Sync"
+				ret = node.@time
+				break
+			default: break;
+		}
+		//println "getStartTime "+node.name()+" $ret"
+		return ret
+	}
+
+	def setStartTime(def node, def value) {
+		switch(node.name()) {
+			case "Turn":
+			case "Section":
+				node.@startTime = ""+value
+				break;
+			
+			case "Sync":
+				return node.@time = ""+value
+			default: break;
+		}
+	}
+
+	def fixSyncTimes(def list, def start, def end) {
+		//println "Nfix: "+list.size()+" "+list
+		//println "$start $end"
+		
+		def startf = Double.parseDouble(start)
+		def endf = Double.parseDouble(end)
+		def delta = (endf-startf)/list.size()
+		//println delta
+		float c = startf;
+		for (int i = 0 ; i < list.size() ; i++) {
+			c += delta;
+			list[i].@time = ""+c
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/SetTRSTurnTimingsAndSpeakers.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/SetTRSTurnTimingsAndSpeakers.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/SetTRSTurnTimingsAndSpeakers.groovy	(revision 321)
@@ -0,0 +1,220 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+class SetTRSTurnTimingsAndSpeakers {
+
+	File trsInFile = new File("/home/mdecorde/xml/transana/out.xml")
+	File trsOutFile = new File("/home/mdecorde/xml/transana/TrP1S8_08102010.trs")
+
+	public SetTRSTurnTimingsAndSpeakers(File trsInFile, File trsOutFile) {
+		this.trsInFile = trsInFile;
+		this.trsOutFile = trsOutFile;
+	}
+
+	public boolean process() {
+		def slurper = new XmlParser();
+		def trs = slurper.parse(trsInFile)
+
+		// collect speakers from Turn@speaker values
+		def speakers = new HashSet<String>();
+		for (def turn : trs.Episode.Section.Turn) {
+			String tmp = turn.@speaker.toUpperCase();
+			speakers << tmp
+			turn.@speaker = tmp
+		}
+		for (def speakersNode : trs.Speakers) {
+			for (String spk : speakers) {
+				def speakerNode = new Node(speakersNode, 'Speaker',
+				[id:spk.replace(" ", "_"), name:spk, type:'unknown', check:'yes', dialect:'native', accent:'no', scope:'local'])
+			}
+			def speakerNode = new Node(speakersNode, 'Speaker',	[id:"none", name:"none", type:'unknown', check:'yes', dialect:'native', accent:'no', scope:'local'])
+		}
+				
+		String currentEnd = null;
+		for (def turn : trs.Episode.Section.Turn) {
+			def children = turn.children()
+	
+			if (children.size() > 0) {
+				//println "fixing turn start="+turn.@startTime+" end="+turn.@endTime+" previous end="+currentEnd
+				if (currentEnd != null && currentEnd.length() > 0 && turn.@startTime.length() == 0) { // using last Sync
+					//println "fixing with previous Turn $currentEnd"
+					turn.@startTime = currentEnd
+					setStartTime(children[0], currentEnd)
+				} else if (!(children[0] instanceof String)) { // no previous Sync, using next Sync
+					def start = getStartTime(children[0])
+					if (start != null && start.length() > 0)	turn.@startTime = start
+				}
+				currentEnd = null
+	
+				if (!(children[children.size() - 1] instanceof String)) { // this is the last Sync
+					def end = getStartTime(children[children.size() - 1])
+					if (end != null && end.length() > 0)  {
+						turn.@endTime = end
+						currentEnd = end;
+					}
+				}
+			}
+		}
+	
+		def nodes = []
+		// get Sync nodes to fix
+		for (def section : trs.Episode.Section) { // all Section
+			for (def child : section.children()) { // all Section children
+				if (!(child instanceof String)) { // is a Tag
+					switch(child.name()) {
+						case "Turn": // get Sync in Turn
+							//nodes << child;
+							for (def sync : child.Sync) {
+								nodes << sync
+							}
+							break;
+						case "Sync":
+							nodes << child;
+							break;
+						default: break;
+					}
+				}
+			}
+		}
+
+		//Fixing Sync@time
+		String previousValue = "0.0"
+		for (int i = 0 ; i < nodes.size() ; i++) {
+			def node = nodes[i]
+			String time = getStartTime(node)	
+			
+			if (time.length() == 0 || time == previousValue) {				
+				def list = [];
+				String previous;
+				if (i > 0) { previous = getStartTime(nodes[i-1]);
+				} else { previous = "0.0" }
+				
+				String next = null;
+				while ((next == null || next.length() == 0 || next == previousValue) && i <= nodes.size()) {
+					list << node
+					node = nodes[i]
+					next = getStartTime(node)
+					i++
+				}
+				if (next != null) {
+					if (node != null) list << node
+					fixSyncTimes(list, previous, next)
+					previousValue = next;
+				} else {
+					println "ERROR: no end time in the transcription"
+				}
+			}
+		}
+	
+		// fixing startTime of Turn using next Sync.@time
+		def turns = trs.Episode.Section.Turn
+		for (int i = 0 ; i < turns.size() ; i++) {
+			def turn = turns[i]
+			def syncs = turn.Sync
+			if (syncs.size() > 0) {
+				turn.@startTime = syncs[0].@time
+			} else {
+				println "Error: Turn with no Sync milestone"
+			}
+		}
+	
+		// fixing endTimes of Turn using next Turn.@startTime
+		for (int i = 0 ; i < turns.size() ; i++) {
+			def turn = turns[i]
+			//println "turn start="+turn.@startTime+" end="+turn.@endTime
+			if (i < turns.size() - 1) {
+				if (turn.@endTime.length() == 0 && turns[i+1].@startTime.length() > 0) {
+					//println "fixing turn.@endTime "+turn.@endTime+" with turns[i+1].@startTime "+(turns[i+1]);
+					turn.@endTime = turns[i+1].@startTime
+				}
+			}
+		}
+		
+		// fixing endTimes of Section using last Turn.@endTime
+		// fixing startTimes of Section using first Turn.@startTime
+		def sections = trs.Episode.Section
+		for (int i = 0 ; i < sections.size() ; i++) {
+			def t = sections[i].Turn
+			if (t.size() == 0) continue;
+			sections[i].@startTime = t[0].@startTime
+			sections[i].@endTime = t[-1].@endTime
+		}
+		
+		trsOutFile.withWriter("UTF-8") { writer ->
+			writer.write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE Trans SYSTEM "trans-14.dtd">\n')
+			new groovy.util.XmlNodePrinter(new PrintWriter(writer)).print(trs) }
+		
+		return true;
+	}
+	
+	def getStartTime(def node) {
+		def ret = null;
+		if (node == null) return "0.0"
+		if (node.name() == null) return "0.0"
+		switch(node.name()) {
+			case "Turn":
+			case "Section":
+			//			println "Turn"
+				ret = node.@startTime
+				break
+			case "Sync":
+			//		println "Sync"
+				ret = node.@time
+				break
+			default: break;
+		}
+		//println "getStartTime "+node.name()+" $ret"
+		return ret
+	}
+
+	def setStartTime(def node, def value) {
+		switch(node.name()) {
+			case "Turn":
+			case "Section":
+				node.@startTime = ""+value
+				break;
+			
+			case "Sync":
+				return node.@time = ""+value
+			default: break;
+		}
+	}
+
+	def fixSyncTimes(def list, def start, def end) {
+		println "Nfix: "+list.size()+" "+list
+		
+		def startf = Double.parseDouble(start)
+		def endf = Double.parseDouble(end)
+		def delta = (endf-startf)/list.size()
+		println "start=$start end=$end delta=$delta"
+
+		float c = startf;
+		for (int i = 0 ; i < list.size() ; i++) {
+			c += delta;
+			list[i].@time = ""+c
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TXT2TRS.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TXT2TRS.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TXT2TRS.groovy	(revision 321)
@@ -0,0 +1,345 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import java.text.DecimalFormat
+import java.text.SimpleDateFormat
+
+import javax.xml.stream.*
+
+import org.txm.doc.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+
+class TXT2TRS {
+
+	boolean inTurn = false
+	def locs = new HashSet<String>()
+	def lastTiming = "none"
+	String localname = null
+	String allTxt
+	String header = ""
+	String type = ""
+	boolean startOfP = false
+	boolean firstTurn = true
+	boolean firstSync = true
+	def loc = null
+	int time = 0;
+	boolean start= false;
+	boolean startTranscription=false
+	boolean isThemeOpened = false
+
+	def formater
+	def formater2
+	StaxStackWriter pagedWriter
+	
+	
+	//                 /\([0-9]+:[0-9][0-9]?:[0-9][0-9]?(\.[0-9])?\)|(¤<[0-9]+>)/
+	def bullet_regex = /\([0-9]+:[0-9][0-9]?:[0-9][0-9]?(\.[0-9])?\)|(¤<[0-9]+>)/
+	def turn_with_bullet = /.+(\([0-9]+:[0-9][0-9]?:[0-9][0-9]?(\.[0-9])?\)|(¤<[0-9]+>))/
+	//                  /([a-zA-Z]+)="([^"]+)"/
+	def section_regex = /([a-zA-Z]+)="([^"]+)"/
+	
+	public TXT2TRS() { }
+
+	public boolean process(File txtFile, File trsFile) {
+
+		def lines = txtFile.readLines("UTF-8")
+
+		formater = new SimpleDateFormat("h:mm:ss.S");
+		formater.setTimeZone(TimeZone.getTimeZone("GMT"));
+		formater2 = new DecimalFormat("#######.0")
+		formater2.setMaximumFractionDigits(2)
+
+		//println "Start writing TRS file..."
+		pagedWriter = new StaxStackWriter(trsFile, "UTF-8");
+		pagedWriter.writeStartDocument("UTF-8", "1.0")
+		//pagedWriter.writeDTD("<!DOCTYPE Trans SYSTEM \"trans-14.dtd\">") // removed because parser has the "no DTD" option set
+		pagedWriter.writeStartElement("Trans");
+		pagedWriter.writeAttribute("scribe", "SCRIBE");
+		pagedWriter.writeAttribute("audio_filename", "");
+		pagedWriter.writeAttribute("version", "1.0");
+		pagedWriter.writeAttribute("version_date", "110110");
+		pagedWriter.writeStartElement("Speakers");
+		pagedWriter.writeEndElement(); // Speakers
+		
+		pagedWriter.writeStartElement("Episode");
+		pagedWriter.writeAttribute("program", "NA");
+		pagedWriter.writeAttribute("air_date", "NA");
+		isThemeOpened = false
+		def firstTurn = true
+
+		// test if the transcription structure is : comment, start line, end line
+		def trslines = []
+		boolean started = false
+		for (String line : lines) {
+			line = line.trim()
+			if (line.matches("\\(.+:..:..\\..+\\)")) {
+				started = true;
+			}
+
+			if (started) { // get ttranscription lines
+				if (line.length() > 0) trslines << line
+			}
+		}
+
+		if (!started) {
+			println "The $txtFile file does not contain the first time bullet"
+			return false;
+		}
+		String lastLine = trslines[-1]
+		if (!lastLine.matches("\\(.+:..:..\\..+\\)")) {
+			// check if last turn has a bullet at the end
+			boolean bulletIsMissing = true;
+			for (int i = trslines.size() - 1 ; i > 0 ; i--) {
+				String line = trslines[i].trim()
+				def split = line.split("\t", 2)
+				//println "$line "+split.length+ " VS "+turn_with_bullet
+				if (split.length == 2) {
+				 	if (line.matches(turn_with_bullet)) {
+						bulletIsMissing = false;
+					} 
+					println "END $bulletIsMissing"
+					i=-1
+				}
+			}
+			if (bulletIsMissing) {
+				println "Error: The $txtFile file last line or last turn does not contain the end time bullet: $bullet_regex"
+				return false;
+			}
+		}
+
+		lines = trslines
+
+		def currentTime = lines[0] // get starting time
+
+		// group lines per Turn, Event, Comment...
+		def groups = [["sync", lines[0]]]
+		lines.remove(0)
+		def currentGroup = []
+
+		for (String line : lines) {
+			if (line.indexOf("\t") > 0) { // start of Turn
+				def split = line.split("\t", 2)
+				def loc = split[0]
+				def content = split[1]
+
+				if (currentGroup.size() > 0) {groups << currentGroup} // store previous group
+				currentGroup = ["turn", loc, content];
+
+			} else if (line.matches("(\\([^)]+\\) *)+")) {//("(") && line.endsWith(")")) { // comment or timing
+				if (line.matches(bullet_regex)) { // Sync in a Turn
+					if (currentGroup.size() > 0) {
+						currentGroup << line
+					} else {
+						if (currentGroup.size() > 0) {groups << currentGroup} // store previous group
+						groups << ["sync", line]
+					}
+				} else { // comment -> ignored
+					if (currentGroup.size() > 0) {groups << currentGroup} // store previous group
+					currentGroup = ["comment", line];
+				}
+			} else if (line.startsWith("[") && line.endsWith("]")) { // section, end of Turn
+				if (currentGroup.size() > 0) groups << currentGroup
+				groups << ["section", line]
+			} else { // something that happens during a Turn
+				currentGroup << line
+				
+			}
+		}
+
+		for (def g : groups) {
+			//println g
+			if (g[0] == "turn") {
+				String loc = g[1]
+				//println "process turn: "+g
+				def tokens = []
+				for (int i = 2 ; i < g.size() ; i++) {// speach starts for position 2
+					//println " process speach: "+g[i]
+					// 	tokenize time bullets, comment and speach
+
+
+					def turntokens = []
+					def toProcess = [g[i]]
+					def lastMatch = 0
+					// find Sync
+					while (toProcess.size() > 0) {
+						String txt = toProcess.pop()
+						boolean nomatch = true
+						def m = (txt =~ bullet_regex) // time
+						//if (m.size() > 0) {
+						while (m.find()) {
+							nomatch = false
+							//println "found: "+m.group()
+							// grab the string in between the end of the last match
+							// and the start of the current one (empty string if none)
+							String before = txt.substring(lastMatch, m.start())
+							if (before.length() > 0) turntokens << before
+							// grab the delimiter
+							turntokens << m.group()
+							// keep looking from the end of the current match
+							lastMatch = m.end()
+						}
+						if (nomatch) turntokens << txt
+						if (lastMatch > 0 && lastMatch < txt.length()) turntokens << txt.substring(lastMatch)
+					}
+					//println "  speach+syncs="+turntokens
+
+					// find Comments (....)
+					toProcess = turntokens
+					turntokens = []
+					
+					while (toProcess.size() > 0) {
+						String txt = toProcess.remove(0)
+						
+						boolean nomatch = true
+						def m = (txt =~ /(\([^)]+\))/) // time
+						//if (m.size() > 0) {
+						lastMatch = 0
+						while (m.find()) {
+							nomatch = false
+							//println "found: "+m.group()
+							// grab the string in between the end of the last match
+							// and the start of the current one (empty string if none)
+							String before = txt.substring(lastMatch, m.start())
+							if (before.length() > 0) turntokens << before
+							// grab the delimiter
+							turntokens << m.group()
+							// keep looking from the end of the current match
+							lastMatch = m.end()
+						}
+						if (nomatch) turntokens << txt
+						if (lastMatch > 0 && lastMatch < txt.length()) turntokens << txt.substring(lastMatch)
+					}
+					//println "  speach+syncs+comments="+turntokens
+					//					if (!processCommentAndSync(g[i])) {
+					//						println "ERROR $lastTiming 'contains ([]) but malformed' error line "+parser.getLocation().getLineNumber()+" : $allTxt"
+					//					}
+					tokens.addAll(turntokens)
+				}
+
+				if (firstTurn && !isThemeOpened) {
+					isThemeOpened = true;
+					
+					pagedWriter.writeStartElement("Section");
+					pagedWriter.writeAttribute("type", "report");
+					pagedWriter.writeAttribute("topic", "");
+					pagedWriter.writeAttribute("startTime", "0.0");
+					pagedWriter.writeAttribute("endTime", "");
+				}
+				firstTurn = false
+				pagedWriter.writeCharacters("\n")
+				pagedWriter.writeStartElement("Turn");
+				pagedWriter.writeAttribute("speaker", loc);
+				pagedWriter.writeAttribute("startTime", "");
+				pagedWriter.writeAttribute("endTime", "");
+				pagedWriter.writeCharacters("\n")
+				writeSync(currentTime)
+
+				String previousToken = ""
+				for (def t : tokens) {
+					if (t.matches(bullet_regex)) {
+						writeSync(t)
+						currentTime = t
+					}  else if (t.startsWith("(") && t.endsWith(")") ){
+						pagedWriter.writeCharacters("\n")
+						pagedWriter.writeEmptyElement("Comment")
+						pagedWriter.writeAttribute("desc", t.substring(1, t.length() -1))
+					}else {
+						pagedWriter.writeCharacters(" "+t)
+					}
+				}
+
+				pagedWriter.writeCharacters("\n")
+				pagedWriter.writeEndElement(); // Turn
+			} else if (g[0] == "sync") {
+				currentTime = g[1]
+				writeSync(currentTime)
+			} else if (g[0] == "comment") {
+//				println "process comment: "+g[1]
+				pagedWriter.writeCharacters("\n")
+				pagedWriter.writeEmptyElement("Comment")
+				pagedWriter.writeAttribute("desc", g[1].substring(1, g[1].length() -1))
+			} else if (g[0] == "section") {
+//				println "process section: "+g[1]
+				
+				def m = (g[1] =~ section_regex) // time
+				if (m.size() > 0) {
+					if (isThemeOpened) {
+						pagedWriter.writeCharacters("\n")
+						pagedWriter.writeEndElement(); // previous Section
+					}
+					isThemeOpened = true;
+					pagedWriter.writeCharacters("\n")
+					pagedWriter.writeStartElement("Section");
+					def attr = ["type":"report", "topic":"", "startTime":"", "endTime":""]
+					
+					for (def match : m) {
+						if (match.size() == 3) {
+							attr[match[1]]=match[2];
+						} else {
+							println "ERROR $lastTiming malformed section attribute "+g[1]+ "near $currentTime"
+						}
+					}
+					
+					// write attributes
+					for (def k : attr.keySet())	pagedWriter.writeAttribute(k, attr[k]);
+				} else {
+					println "ERROR $lastTiming malformed section "+g[1]
+				}
+			} else {
+				println "Error: found malformed line: "+g
+			}
+		}
+
+		boolean ret = true
+		if (isThemeOpened) { // close previous theme
+			pagedWriter.writeCharacters("\n")
+			pagedWriter.writeEndElement(); // Section
+		}
+
+		//pagedWriter.writeEndElement(); // Section
+		pagedWriter.writeCharacters("\n")
+		pagedWriter.writeEndElement(); // Episode
+		pagedWriter.writeCharacters("\n")
+		pagedWriter.writeEndElement(); // Trans
+
+		ret = true
+
+		pagedWriter.close()
+		return ret
+	}
+
+	def writeSync(String txt) {
+		//println "write time code $txt"
+		pagedWriter.writeEmptyElement("Sync");
+		if (txt.startsWith("¤")) { // ¤<5648>
+			String time = txt.substring(2, txt.length()-1)
+			pagedWriter.writeAttribute("time", time.substring(0,time.length()-3)+"."+time.substring(time.length()-3));
+		} else {
+			String str = txt.substring(1, txt.length()-1)
+			Date date = formater.parse(str);
+			pagedWriter.writeAttribute("time", ""+(date.getTime()/1000.0f));
+		}
+		pagedWriter.writeCharacters("\n")
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Macros which transform transcriptions written in ODT, WORD or RTF (like Transana) files to the Transcriber format</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/HTML2TRS.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/HTML2TRS.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/HTML2TRS.groovy	(revision 321)
@@ -0,0 +1,331 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import java.text.DecimalFormat
+import java.text.SimpleDateFormat
+
+import javax.xml.stream.*
+
+import org.txm.doc.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+
+class HTML2TRS {
+
+	boolean inTurn = false
+	def locs = new HashSet<String>()
+	def lastTiming = "none"
+	String localname = null
+	String allTxt
+	String header = ""
+	String type = ""
+	boolean startOfP = false
+	boolean firstTurn = true
+	boolean firstSync = true
+	def loc = null
+	int time = 0;
+	boolean start= false;
+	boolean startTranscription=false
+	boolean isThemeOpened = false
+
+	def inputData
+	def factory
+	def parser
+	def formater
+	def formater2
+	StaxStackWriter pagedWriter
+	public HTML2TRS() { }
+	
+	public process(File xhtmlFile, File trsFile) {
+	
+		println "remove entities and doctype"
+		String text = xhtmlFile.getText("UTF-8")
+		text = text.replaceAll("&nbsp;", " ")
+		text = text.replaceAll("""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">""", "")
+		xhtmlFile.withWriter("UTF-8") { writer ->
+			writer.write(text);
+		}
+		
+		inputData = xhtmlFile.toURI().toURL().openStream();
+		factory = XMLInputFactory.newInstance();
+		factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
+		parser = factory.createXMLStreamReader(inputData);
+		
+		formater = new SimpleDateFormat("h:mm:ss.S");
+		formater.setTimeZone(TimeZone.getTimeZone("GMT"));
+		formater2 = new DecimalFormat("#######.0")
+		formater2.setMaximumFractionDigits(2)
+	
+		println "start writing TRS file"
+		pagedWriter = new StaxStackWriter(trsFile, "UTF-8");
+		pagedWriter.writeStartDocument("UTF-8", "1.0")
+		//pagedWriter.writeDTD("<!DOCTYPE Trans SYSTEM \"trans-14.dtd\">") // removed because parser has the "no DTD" option set
+		pagedWriter.writeStartElement("Trans");
+		pagedWriter.writeAttribute("scribe", "SCRIBE");
+		pagedWriter.writeAttribute("audio_filename", "");
+		pagedWriter.writeAttribute("version", "1.0");
+		pagedWriter.writeAttribute("version_date", "110110");
+		pagedWriter.writeStartElement("Speakers");
+		pagedWriter.writeEndElement(); // Speakers
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName()
+					//println "start elem: "+localname
+					switch (localname) {
+						case "body": println "START OF BODY";start = true;
+							break;
+						case "p":
+							loc = null
+							allTxt = ""
+							type = "";
+							startOfP = true;
+							break
+					}
+					break;
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName()
+					switch (localname) {
+						case "body": start = false;
+							break
+						case "p":
+							//if (allTxt.length() > 0) allTxt = allTxt.substring(1)
+							//println "PTEXT '$allTxt'"
+							if (allTxt.indexOf("####") > 0) {
+								//println "SAY $txt"
+								if (allTxt.startsWith("\"")) allTxt = allTxt.substring(1)
+								loc = allTxt.substring(0, allTxt.indexOf("####")).trim()
+								locs.add(loc)
+								allTxt = allTxt.substring(allTxt.indexOf("####")+4) // remove " [A-Z]"
+								//println "LOC: '$loc'"
+								//println "LOC: '$allTxt'"
+							}
+							String trimedTxt = allTxt.trim()
+							if (startTranscription)
+							if (loc) {
+								inTurn = true;
+								pagedWriter.writeStartElement("Turn");
+								pagedWriter.writeAttribute("speaker", loc);
+								pagedWriter.writeAttribute("startTime", "");
+								pagedWriter.writeAttribute("endTime", "");
+								
+								// write Sync
+								if (firstTurn) {
+									writeTimeCode("(0:00:00.0)")
+									firstTurn = false;
+								} else {
+									pagedWriter.writeEmptyElement("Sync")
+									pagedWriter.writeAttribute("time", "");
+								}
+								
+								if (!processCommentAndSync(allTxt)) {
+									if (allTxt.trim().length() > 0)
+										println "ERROR $lastTiming 'contains ([]) but malformed' error line "+parser.getLocation().getLineNumber()+" : $allTxt"
+								}
+								//pagedWriter.writeCharacters(allTxt.replaceAll("\\(.+:..:..\\..+\\)", ""))
+								pagedWriter.writeEndElement(); // endTurn
+								
+								inTurn = false
+								time++;
+							} else { // Sync or Comment or Section : out of Turn 
+								if (allTxt.length() > 0)
+								if (trimedTxt.startsWith("(") && trimedTxt.endsWith(")")) {
+									if (!processCommentAndSync(allTxt)) {
+										if (allTxt.trim().length() > 0)
+											println "ERROR $lastTiming 'contains ([]) but malformed' line "+parser.getLocation().getLineNumber()+" : $allTxt"
+									}
+								} else if (trimedTxt.startsWith("[") && trimedTxt.endsWith("]")) {
+									println "SECTION: "+trimedTxt
+									def section_regex = /([a-zA-Z]+)="([^"]+)"/
+									def m = (trimedTxt =~ section_regex) // time
+									if (m.size() > 0) {
+										if (isThemeOpened)
+											pagedWriter.writeEndElement(); // previous Section
+										isThemeOpened = true;
+										pagedWriter.writeStartElement("Section");
+										pagedWriter.writeAttribute("type", "report");
+										pagedWriter.writeAttribute("topic", "");
+										pagedWriter.writeAttribute("startTime", "");
+										pagedWriter.writeAttribute("endTime", "");
+										for (def match : m) {
+											if (match.size() == 3) {
+												pagedWriter.writeAttribute(match[1], match[2]);
+											} else {
+												println "ERROR $lastTiming malformed section attribute "+trimedTxt
+											}
+										}
+									} else {
+										println "ERROR $lastTiming malformed section "+trimedTxt
+									}
+								} else if (allTxt.trim().length() > 0) {
+									println "ERROR $lastTiming 'not in Turn and not a comment' line "+parser.getLocation().getLineNumber()+" : $allTxt"
+								}
+							}
+		
+							allTxt = ""
+							break
+					}
+					break;
+					
+				case XMLStreamConstants.CHARACTERS:
+					String txt = parser.getText().replaceAll("\n", " ")
+					allTxt += txt.replaceAll("    ", "####")
+					//println allTxt+"\n\n"
+					if (!startTranscription && start && txt.matches("\\(.+:..:..\\..+\\)")) {
+						println "START with $txt"
+						startTranscription = true
+						
+						pagedWriter.writeStartElement("Episode");
+						pagedWriter.writeAttribute("program", "NA");
+						pagedWriter.writeAttribute("air_date", "NA");
+						isThemeOpened = true
+						pagedWriter.writeStartElement("Section");
+						pagedWriter.writeAttribute("type", "report");
+						pagedWriter.writeAttribute("topic", "");
+						pagedWriter.writeAttribute("startTime", "0.0");
+						pagedWriter.writeAttribute("endTime", "");
+					}
+					//println "$start $startTranscription "+txt.length()+" $startOfP"
+					if (start && startTranscription && txt.length() > 0 && startOfP) {
+						//println "first P: $txt $loc"
+						startOfP = false
+					}
+					break;
+				default: break;
+			}
+		}
+		boolean ret = true
+		if (startTranscription) {
+			if (isThemeOpened) { // close previous theme
+				pagedWriter.writeEndElement(); // Section
+			}
+			
+			//pagedWriter.writeEndElement(); // Section
+			pagedWriter.writeEndElement(); // Episode
+			pagedWriter.writeEndElement(); // Trans
+
+			ret = true
+		} else {
+			println "ERROR6: header not found"
+			ret = false;
+		}
+		pagedWriter.close()
+		return ret
+	}
+
+	def writeTimeCode(String txt) {
+		//println "write time code $txt"
+		String str = txt.substring(1, txt.length()-1)
+		Date date = formater.parse(str);
+		lastTiming = str;
+		if (inTurn || firstSync) {
+			pagedWriter.writeEmptyElement("Sync");
+			pagedWriter.writeAttribute("time", ""+(date.getTime()/1000.0f));
+		} else {
+			pagedWriter.writeStartElement("Turn");
+			pagedWriter.writeAttribute("speaker", "none");
+			pagedWriter.writeAttribute("startTime", "");
+			pagedWriter.writeAttribute("endTime", "");
+			inTurn = true
+			writeTimeCode(txt);
+			inTurn = false	
+			pagedWriter.writeEndElement(); // endTurn
+			//println "ERROR: can't write Sync out of Turn $txt at "+parser.getLocation().getLineNumber()
+		}
+		firstSync = false
+	}
+	
+	//                 /\([0-9]+:[0-9][0-9]?:[0-9][0-9]?(\.[0-9])?\)/
+	def bullet_regex = /\([0-9]+:[0-9][0-9]?:[0-9][0-9]?(\.[0-9])?\)/
+	def processCommentAndSync(String txt) {
+		if (txt.length() == 0) return true;
+		//println "process sync text: "+txt
+		boolean ret = true;
+		//find timing
+		//find comments
+		def allBySync = []
+		def all = []
+		def splits = []
+		def lastMatch = 0
+		
+		// split bullet from txt
+		def m = (txt =~ bullet_regex) // time
+		while (m.find()) {
+			//println "found: "+m.group()
+	   		// grab the string in between the end of the last match
+	   		// and the start of the current one (empty string if none)
+	   		allBySync << txt.substring(lastMatch, m.start())
+	   		// grab the delimiter
+	   		allBySync << m.group()
+	   		// keep looking from the end of the current match
+	   		lastMatch = m.end()
+		}
+		all = allBySync
+		// grab everything after the end of the last match
+		all << txt.substring(lastMatch)
+		//println "** $txt >> $all"
+		for (String str : all) {
+			if (str.trim().length() == 0) continue
+			if (str.matches(bullet_regex)) {
+				//println "TIMING: $str"
+				writeTimeCode(str)
+			} else {
+				int idxOpen = str.indexOf("(");
+				int idxClose = str.indexOf(")")
+				
+				while (idxOpen >=0 && idxClose > idxOpen) {
+					if (str.substring(0, idxOpen).length() > 0) {
+						if (loc != null) {
+							pagedWriter.writeCharacters(str.substring(0, idxOpen))
+							//println "SAY: "+str.substring(0, idxOpen)
+						} else {
+							if (str.trim().length() > 0)
+								println "WARNING $lastTiming no locutor at line "+parser.getLocation().getLineNumber()+" : $str"
+						}
+					}
+					pagedWriter.writeEmptyElement("Comment")
+					pagedWriter.writeAttribute("desc", str.substring(idxOpen+1, idxClose))
+					//println "COMMENT: "+str.substring(idxOpen+1, idxClose)
+					str = str.substring(idxClose+1)
+					idxOpen = str.indexOf("(");
+					idxClose = str.indexOf(")")
+				}
+				if (str.length() > 0) {
+					if (loc != null) {
+						pagedWriter.writeCharacters(str)
+					//	println "SAY: "+str
+					} else {
+						if (str.trim().length() > 0)
+							println "ERROR $lastTiming 'not in turn and not a comment' line "+parser.getLocation().getLineNumber()+" : $str"
+					}
+				}
+				if (str.contains("(") || str.contains("[") || str.contains(")") || str.contains("]")) {
+					ret = false;
+				}
+		  }
+		}
+		return ret;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TextTranscription2TRS.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TextTranscription2TRS.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/transcription/TextTranscription2TRS.groovy	(revision 321)
@@ -0,0 +1,257 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.transcription
+
+import javax.xml.stream.*
+
+import org.kohsuke.args4j.*
+import org.txm.doc.*
+import org.txm.importer.*
+import org.txm.importer.filters.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+class TextTranscription2TRS {
+	boolean debug = false;
+	boolean isDirectory = false;
+	File outDir;
+	ConvertDocument converter;
+	def monitor
+
+	TextTranscription2TRS(File outDir, boolean debug, def monitor) {
+		this.outDir = outDir
+		this.monitor = monitor
+		this.debug = debug
+	}
+
+	boolean process(File file) {
+		try {
+			isDirectory = file.isDirectory();
+			if (isDirectory) {
+				return processDir(file)
+			} else {
+				return processFile(file, 100)
+			}
+		} catch(Exception e) {
+			println "Error while processing: "+e
+			if (debug) e.printStackTrace()
+		}
+	}
+
+	boolean processDir(File dir) {
+		boolean ret = true
+		try {
+			println "* Processing $dir directory"
+			converter = new ConvertDocument();
+			def files = dir.listFiles()
+			def okfiles = [];
+			if (files != null)
+				for (File file : files) {
+					String fname = file.getName().toLowerCase()
+					if ( file.isFile() &&
+					(fname.endsWith(".odt") || fname.endsWith(".doc") || fname.endsWith(".rtf") || fname.endsWith(".txt"))) {
+						okfiles << file
+					}
+
+				}
+
+			if (okfiles.size() == 0) {
+				println "No ODT/DOC/RTF file found in $dir"
+				return false
+			}
+
+			int delta = 100 / okfiles.size()
+			for (File file : okfiles) {
+				if (monitor != null) monitor.worked(0, "Processing file: $file")
+				if (!processFile(file, delta)) {
+					ret = false;
+					println "ERROR: Failed to process $file"
+				}
+
+				if (monitor != null && monitor.isCanceled()) {
+					break; // stop looping
+				}
+			}
+		} catch(Exception e) {
+			println "Error while processing directory: "+e;
+			if (debug) e.printStackTrace();
+		}
+		if (converter != null) converter.stop();
+		return ret;
+	}
+
+	boolean processFile(File docFile, int work) {
+		if (docFile == null) return false;
+		int delta = work / 5
+		println "** Processing $docFile file"
+		String filename = docFile.getName()
+		int idx = filename.indexOf(".")
+		if (idx < 0) return false;
+		String ext = filename.substring(idx)
+		filename = filename.substring(0, idx)
+
+		//File docFile = new File(srcDir, "${filename}$ext")
+		//		File htmlFile = new File(outDir, "${filename}.html")
+		//		File htmlFile2 = new File(outDir, "${filename}_temp.html")
+		//		File xhtmlFile = new File(outDir, "${filename}.xhtml")
+		//		File xmlFile = new File(outDir, "${filename}.xml")
+		File txtFile = new File(outDir, "${filename}.txt")
+		File trsFile = new File(outDir, "${filename}-tmp.trs")
+		File finalTrsFile = new File(outDir, "${filename}.trs")
+		boolean ret = false;
+		
+		if (!docFile.getName().toLowerCase().endsWith("txt")) {
+			if (monitor != null) monitor.worked(0, "DOCtoTXT: "+docFile.getName())
+
+			ret = DOCtoTXT(docFile, txtFile)
+			if (monitor != null) monitor.worked(delta, "DOCtoTXT: "+docFile.getName())
+			if (monitor != null && monitor.isCanceled()) {
+				println "Process interrupted by user"
+				return false;
+			}
+			if (!ret) return ret;
+		} else { // the document is already a TXT file
+			org.txm.utils.io.FileCopy.copy(docFile, txtFile)
+			ret = true
+		}
+
+		ret = ret && TXTtoTRS(txtFile, trsFile)
+		if (monitor != null) monitor.worked(delta, "TXTtoTRS: "+docFile.getName())
+		if (monitor != null && monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		ret = ret && TRStoFixedTRS(trsFile, finalTrsFile);
+		if (monitor != null) monitor.worked(delta, "Done: "+docFile.getName())
+		if (monitor != null && monitor.isCanceled()) {
+			println "Process interrupted by user"
+			return false;
+		}
+		if (!ret) return ret;
+
+		// cleaning
+		System.gc()
+		if (!debug) {
+			//htmlFile.delete();htmlFile2.delete();xhtmlFile.delete();xmlFile.delete();
+			txtFile.delete()
+			trsFile.delete()
+		}
+
+		return ret
+	}
+
+	boolean DOCtoTXT(File docFile, File txtFile) {
+		println "*** ODT -> TXT"
+		try {
+			if (!isDirectory) converter = new ConvertDocument();
+			converter.setDebug(debug)
+			converter.autoFile(docFile, txtFile, "txt")
+		} catch(Exception e) {
+			println "Error while converting $docFile : $e"
+			if (debug) e.printStackTrace()
+		} finally {
+			if (!isDirectory && converter != null) converter.stop()
+		}
+		return txtFile.exists() && txtFile.length() > 0
+	}
+
+//	boolean HTMLtoHTMLforTidy(File htmlFile, File htmlFile2) {
+//		println "*** HTML -> HTML for tidy"
+//		try {
+//			println "replace TABS with 4 spaces"
+//			String text2 = htmlFile.getText("UTF-8")
+//			text2 = text2.replaceAll("&nbsp;", " ")
+//			text2 = text2.replaceAll("’", "'")
+//			text2 = text2.replaceAll("&rsquo;", "'")
+//			text2 = text2.replaceAll("\t", "&nbsp;&nbsp;&nbsp;&nbsp;")
+//			text2 = text2.replaceAll("\n", '&nbsp;')
+//			text2 = text2.replaceAll("\r\n", '&nbsp;')
+//			htmlFile2.withWriter("UTF-8") { writer ->
+//				writer.write(text2);
+//			}
+//		} catch(Exception e) {
+//			println "Error while preparing HTML of $htmlFile : $e"
+//			if (debug) e.printStackTrace()
+//		}
+//		return htmlFile2.exists() && htmlFile2.length() > 0
+//	}
+//
+//	boolean HTMLtoXHTML(File htmlFile2, File xhtmlFile) {
+//		println "*** HTML for tidy -> XHTML"
+//		try {
+//			Tidy tidy = new Tidy(); // obtain a new Tidy instance
+//			tidy.setXHTML(true); // set desired config options using tidy setters
+//			tidy.setInputEncoding("UTF-8")
+//			tidy.setOutputEncoding("UTF-8")
+//			tidy.setShowErrors(100)
+//			tidy.setShowWarnings(debug)
+//			tidy.setTabsize(10)
+//			tidy.setWraplen(9999)
+//			tidy.setForceOutput(true) // Tidy won't stop if error are found
+//			xhtmlFile.withWriter("UTF-8") { out ->
+//				def input = new InputStreamReader(htmlFile2.toURI().toURL().newInputStream(), "UTF-8")
+//				tidy.parse(input, out); // run tidy, providing an input and output stream
+//			}
+//			if (xhtmlFile.exists()) {
+//				// JTidy produced a "0x0" char. removing them
+//				// fix separated < and / ???
+//				def c = Character.toChars(0)[0]
+//				String txttmp = xhtmlFile.getText("UTF-8");
+//				xhtmlFile.withWriter("UTF-8") { out ->
+//					out.write(txttmp.replace("<\n/", "</").replace("<\r\n/", "</"))
+//				}
+//			}
+//		} catch(Exception e) {
+//			println "Error while applying JTidy: "+e
+//			if (debug) e.printStackTrace()
+//		}
+//		return xhtmlFile.exists() && xhtmlFile.length() > 0
+//	}
+
+	boolean TXTtoTRS(File txtFile, File trsFile) {
+		println "*** TXT -> TRS"
+		try {
+			TXT2TRS ht = new TXT2TRS();
+			ht.process(txtFile, trsFile);
+		} catch(Exception e) {
+			println "Error while creating TRS file: "+e
+			if (debug) e.printStackTrace()
+		}
+		return trsFile.exists() && trsFile.length() > 0
+	}
+
+	boolean TRStoFixedTRS(File trsFile, File finalTrsFile) {
+		println "*** TRS -> FIXED TRS"
+		try {
+			def timingfixer = new SetTRSTurnTimingsAndSpeakers(trsFile, finalTrsFile);
+			if (!timingfixer.process()) {
+				println "Error while fixing timing"
+			}
+		} catch(Exception e) {
+			println "Error while fixing TRS file: "+e
+			if (debug) e.printStackTrace()
+		}
+		return finalTrsFile.exists() && finalTrsFile.length() > 0
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/CharListMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/CharListMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/CharListMacro.groovy	(revision 321)
@@ -0,0 +1,30 @@
+package org.txm.macro.txt;
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+// BEGINNING OF PARAMETERS
+
+// Declare each parameter here
+// (available widget types: Query, File, Folder, String, Text, Boolean, Integer, Float and Date)
+
+@Field @Option(name="inputFile", usage="The file to read", widget="File", required=true, def="C:/Temp/foo.txt")
+def inputFile
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+// END OF PARAMETERS
+
+String s = inputFile.getText()
+
+def dic = [:]
+
+for (char c : s) {
+	if (!dic.containsKey(c)) { dic[c] = 0 }
+	dic[c]++
+}
+
+for (def e : dic) { println(sprintf("%c\t%d", [e.getKey(), e.getValue()])) }
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/ChangeEncodingMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/ChangeEncodingMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/ChangeEncodingMacro.groovy	(revision 321)
@@ -0,0 +1,76 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.txt
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.utils.CharsetDetector
+
+@Field @Option(name="inputDirectory", usage="Dossier qui contient les fichiers à traiter", widget="Folder", required=true, def="")
+File inputDirectory
+
+@Field @Option(name="inputEncoding", usage="Input encoding", widget="String", required=false, def="")
+String inputEncoding
+
+@Field @Option(name="outputEncoding", usage="Output encoding", widget="String", required=true, def="UTF-8")
+String outputEncoding
+
+if (!ParametersDialog.open(this)) return;
+
+File outDir = new File(inputDirectory, "out") // write result in "out" folder
+if (outDir.exists()) {
+	if (!outDir.deleteDir()) {
+		println "Could not delete directory: $outDir"
+		return;
+	}
+}
+if (!outDir.mkdir()) {
+	println "Could not create directory: $outDir"
+	return;
+}
+println "Writing result in : $outDir"
+
+def processFile(File file, File outfile, String fromEncoding, String toutputEncoding) {
+	String text = file.getText(fromEncoding)
+	outfile.withWriter(toutputEncoding) {writer -> // read with the "fromEncoding"
+		writer.write(text) // write with the "toutputEncoding"
+	}
+}
+
+try {
+inputDirectory.eachFile { file ->
+	if (!file.isHidden() && !file.isDirectory()) {
+		String e = inputEncoding
+		if (inputEncoding == null || inputEncoding.length() == 0) {
+			CharsetDetector detector = new CharsetDetector(file)
+			e = detector.getEncoding()
+		}
+		
+		File outfile = new File(outDir, file.getName())
+		println "$e -> $outputEncoding : "+outfile.getName()
+		processFile(file, outfile, e, outputEncoding)
+	}
+}
+} catch (UnsupportedEncodingException e) {
+	println "ERROR: Unsupported encoding: "+e
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/SearchReplaceInDirectoryMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/SearchReplaceInDirectoryMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/SearchReplaceInDirectoryMacro.groovy	(revision 321)
@@ -0,0 +1,104 @@
+package org.txm.macro.txt
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author sheiden
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 11:28:42 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2378 $
+// $LastChangedBy: sheiden $
+//
+
+// imports
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+// parameters
+
+// **change this parameter**
+@Field @Option(name="inputDirectory", usage="Dossier qui contient les fichiers à modifier", widget="Folder", required=true, def='voeux')
+		inputDirectory = new File(System.getProperty("user.home"), "Bureau/voeux")
+
+// **change this parameter**
+@Field @Option(name="extension", usage="Regexp de l'extension des fichiers à modifier", widget="String", required=true, def='\\.txt')
+		extension = "\\.txt"
+
+// **change this parameter**
+@Field @Option(name="find", usage="Expression régulière", widget="String", required=true, def='’')
+		find = "’"
+
+// **change this parameter**
+@Field @Option(name="replaceWith", usage="Chaîne de remplacement", widget="String", required=false, def='\'')
+		replaceWith = "'"
+
+// **change this parameter**
+@Field @Option(name="encoding", usage="Encodage des fichiers", widget="String", required=true, def='UTF-8')
+		encoding = "utf-8"
+
+// **change this parameter**
+// 'true' = only display matching lines, 'false' = replace in matching files
+//@Field @Option(name="showMatchingFilesOnly", usage="Montrer seulement les matchs", widget="Boolean", required=false, def='false')
+		showMatchingFilesOnly = false;
+
+if (!ParametersDialog.open(this)) return;
+
+println "Working with $inputDirectory files with extension="+/.*$extension/
+if (showMatchingFilesOnly) {
+	println "Search '$find'"
+} else {
+	println "Replace '$find' with '$replaceWith'"
+}
+
+//find = /date="([0-9]+)-([0-9]+-[0-9]+)"/
+// **change this parameter**
+//replaceWith = 'date="$1-$2" year="$1"'
+// **change this parameter** (warning: '$1', '$2'... can be interpreted by Groovy in "..." strings)
+process(inputDirectory, extension, encoding, find, replaceWith, showMatchingFilesOnly)
+
+// main body
+def process(File inputDirectory, extension, encoding, find, replaceWith, showMatchingFilesOnly) {
+	
+	inputDirectory.eachFileMatch(~/.*$extension/) { file ->               // for each file matching extension
+		println "Processing: "+file.getName()
+//		if (showMatchingFilesOnly) {
+//			printMatches(file, encoding, find)
+//		} else {
+			replaceInFile(file, encoding, find,replaceWith)
+//		}
+	}
+}
+
+//def printMatches(File file, encoding, find) {
+//	file.eachLine(encoding) { line ->                  // for each line
+//		if (line.matches("$find")) { println " *match* : "+line }
+//	}
+//}
+
+def replaceInFile(File file, encoding,  find, replaceWith) {
+	def tmp = File.createTempFile("SearchReplaceInDirectoryTemp", ".tmp", file.getParentFile()) // create temporary file
+	tmp.write('')                                // create empty file
+	tmp.withWriter(encoding) { writer ->
+		String txt = file.getText(encoding);
+		writer.println txt.replaceAll(find, replaceWith)
+//		file.eachLine(encoding) { line ->                  // for each line
+//			writer.println line.replaceAll(find, replaceWith) // find&replace and print
+//		}
+		writer.close()
+	}
+	file.delete()
+	tmp.renameTo(file)                           // save results
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/SearchInDirectoryMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/SearchInDirectoryMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/SearchInDirectoryMacro.groovy	(revision 321)
@@ -0,0 +1,45 @@
+package org.txm.macro.txt
+
+import org.kohsuke.args4j.*
+
+import groovy.transform.Field
+
+import java.nio.charset.Charset
+
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.utils.*
+
+import javax.xml.stream.*
+
+import java.net.URL
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Field @Option(name="inputDirectory", usage="TXT directory", widget="Folder", required=false, def="txt")
+File inputDirectory;
+@Field @Option(name="regexp", usage="Regular expression to match", widget="String", required=false, def="p")
+String regexp;
+@Field @Option(name="encoding", usage="File encoding", widget="String", required=false, def="UTF-8")
+String encoding;
+
+if (!ParametersDialog.open(this)) return;
+
+def p = /$regexp/
+
+println "processing: "+inputDirectory.listFiles()
+for (File inputfile : inputDirectory.listFiles()) {
+	if (inputfile.isDirectory()) continue // ignore
+	
+	def lines = []
+	inputfile.eachLine("UTF-8") { line, n ->
+		def m = line =~ p
+		if ( m.size() > 0) {
+			lines << "  line $n: $line"
+		}
+	}
+	
+	println inputfile.getName() + " "+lines.size() + " match" +((lines.size() > 1)?"s":"")
+	if (lines.size() > 0) {
+		for (String s : lines ) println s
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/TXT2XMLMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/TXT2XMLMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/txt/TXT2XMLMacro.groovy	(revision 321)
@@ -0,0 +1,47 @@
+package org.txm.macro.txt
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import java.nio.charset.Charset
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.utils.*
+import javax.xml.stream.*
+import java.net.URL
+
+@Field @Option(name="inputDirectory", usage="TXT directory", widget="Folder", required=false, def="txt")
+File inputDirectory;
+
+@Field @Option(name="encoding", usage="File encoding", widget="String", required=false, def="UTF-8")
+String encoding;
+
+@Field @Option(name="rootTag", usage="rootTag", widget="String", required=false, def="doc")
+String rootTag;
+
+if (!ParametersDialog.open(this)) return;
+rootTag = rootTag.trim()
+encoding = encoding.trim()
+
+File outputDirectory = new File(inputDirectory, "out")
+outputDirectory.mkdir()
+println "processing: "+inputDirectory
+for (File inputfile : inputDirectory.listFiles()) {
+	if (inputfile.isDirectory() || inputfile.isHidden() || !inputfile.getName().endsWith(".txt")) continue // ignore
+	println " file: "+inputfile
+	
+	String name = inputfile.getName()
+	int idx = name.indexOf(".")
+	if (idx > 0) name = name.substring(0, idx)
+	File outputfile = new File(outputDirectory, name+".xml")
+	
+	XMLOutputFactory factory = XMLOutputFactory.newInstance()
+	FileOutputStream output = new FileOutputStream(outputfile)
+	XMLStreamWriter writer = factory.createXMLStreamWriter(output)
+
+	writer.writeStartDocument("UTF-8","1.0")
+	writer.writeCharacters("\n") // manage XML reserved characters
+	writer.writeStartElement(rootTag)
+	writer.writeCharacters(inputfile.getText(encoding)) // manage XML reserved characters
+	writer.writeEndElement() // rootTag
+	writer.close()
+	output.close()
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/xml/XMLStatisticsMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/xml/XMLStatisticsMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/xml/XMLStatisticsMacro.groovy	(revision 321)
@@ -0,0 +1,28 @@
+package org.txm.macro.xml
+
+import org.txm.scripts.*
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+@Field @Option(name="inputDirectory", usage="Dossier qui contient les fichiers à parser", widget="Folder", required=true, def="src")
+File inputDirectory;
+
+@Field @Option(name="tsvFile", usage="FichierTSV résultat", widget="File", required=true, def="file.tsv")
+File tsvFile;
+
+@Field @Option(name="usePaths", usage="FichierTSV résultat", widget="Boolean", required=true, def="false")
+def usePaths = true
+@Field @Option(name="useAttributes", usage="FichierTSV résultat", widget="Boolean", required=true, def="false")
+def useAttributes = true
+@Field @Option(name="useAttributeValues", usage="FichierTSV résultat", widget="Boolean", required=true, def="false")
+def useAttributeValues = true
+
+if (!ParametersDialog.open(this)) return;
+println "Input directory: "+inputDirectory
+println "TSV file: "+tsvFile
+println "use paths $usePaths"
+println "use attributes $useAttributes"
+println "use attributes values $useAttributeValues"
+
+XMLStatistics.processDirectory(inputDirectory, tsvFile, ".xml", usePaths, useAttributes, useAttributeValues);
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/commands/CrossedPartitionBuilderMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/commands/CrossedPartitionBuilderMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/commands/CrossedPartitionBuilderMacro.groovy	(revision 321)
@@ -0,0 +1,337 @@
+package org.txm.macro.commands
+
+import groovy.transform.Field
+
+import org.kohsuke.args4j.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.rcpapplication.views.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.utils.logger.Log
+
+//BEGINNING OF PARAMETERS
+def corpus = corpusViewSelection
+if (!(corpus instanceof Corpus)) {
+	println "Error: this macro should be run with a Corpus selected"
+	return
+}
+
+@Field @Option(name="structuralUnit", usage="the structural Unit to use", widget="String", required=true, def="text")
+String structuralUnit = "text"
+
+@Field @Option(name="structuralUnitPropertiesList", usage="the structural Unit properties list separated with commas", widget="String", required=true, def="p1,p2")
+def structuralUnitPropertiesList = "loc, type";
+		
+//@Field @Option(name="expandTarget", usage="Expand structure", widget="String", required=false, def="")
+def expandTarget = "";
+
+//@Field @Option(name="structuralUnitToIgnore", usage="the structural Unit properties list to ignore separated with commas", widget="String", required=false, def="")
+def structuralUnitToIgnore = "";
+
+//@Field @Option(name="partitionName", usage="The partition name to use", widget="String", required=false, def="")
+def partitionName = ""
+
+//@Field @Option(name="partNamePrefix", usage="the part prefix to use", widget="String", required=true, def="")
+def partNamePrefix = ""
+
+@Field @Option(name="debug", usage="If debug, then show only queries", widget="Boolean", required=false, def="false")
+def debug = true;
+
+//Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+//END OF PARAMETERS
+
+/**
+ * Create partition with advanced CQL queries and autoname the parts.
+ * Can create partitions by defining a multi-level structural units hierarchy or by defining several properties values for one structural unit.
+ * Can define some structural units to remove from the resulting parts subcorpus.
+ * 
+ */
+
+// TODO: add this variables to the macro parameters
+
+def structuralUnits = []
+def structuralUnitProperties = []
+structuralUnit = structuralUnit.trim()
+structuralUnitPropertiesList = structuralUnitPropertiesList.trim()
+
+// Build lists
+def split = structuralUnit.split(",")
+def split2 = structuralUnitPropertiesList.split(",")
+if (split.size() == 1) {
+	for (String p : split2) {
+		p = p.trim();
+		structuralUnits << structuralUnit
+		structuralUnitProperties << p
+	}
+} else {
+	if (split.size() == split2.size()) {
+		for (int i = 0 ; i < split.size() ; i++) {
+			def su = split[i]
+			def p = split2[i];
+			structuralUnits << su
+			structuralUnitProperties << p
+		}
+	} else {
+		println "Error: structuralUnitList size is different from structuralUnitPropertiesList"
+		println "structuralUnitList=structuralUnitList=$structuralUnit structuralUnitPropertiesList=$structuralUnitPropertiesList"
+		return false
+	}
+}
+
+// check properties existance
+try {
+	boolean ok = true;
+	for (int i = 0 ; i < structuralUnits.size() ; i++) {
+		def suName = structuralUnits[i]
+		def pName = structuralUnitProperties[i]
+		
+		if (corpus.getStructuralUnit(suName) == null) {
+			println "Error: no '$suName' structure in the '$corpus' corpus"
+			ok = false;
+		} else {
+			def su = corpus.getStructuralUnit(suName)
+			for (String p : structuralUnitProperties) {
+				if (su.getProperty(pName) == null) {
+					println "Error: no '$pName' structure property in the '$corpus' corpus"
+					ok = false;
+				}
+			}
+		}
+	}
+	if (!ok) return;
+} catch(Exception e) {
+	println "Error: $e"
+	Log.printStackTrace(e)
+	return
+}
+
+
+
+//Running
+PartitionQueriesGenerator pqg = new PartitionQueriesGenerator();
+try {
+	def partition = pqg.createPartition(corpus, debug,
+			partitionName, partNamePrefix,
+			structuralUnits, structuralUnitProperties,
+			structuralUnitToIgnore, expandTarget);
+
+	if (partition != null)	{
+		monitor.syncExec(new Runnable() {
+					public void run() {
+						CorporaView.refresh();
+						CorporaView.expand(partition.getParent());
+					}
+				});
+	}
+} catch(Exception e) {
+	println "Exception $e"
+	Log.printStackTrace(e);
+	return;
+}
+
+/**
+ * Create a list of queries and part names regarding the structural units, structural units properties, structural units to ignore user defined lists and expand target value specified.
+ * @author s
+ *
+ */
+public class PartitionQueriesGenerator	{
+
+	public boolean DEBUG = false;					// si DEBUG, alors les requêtes sont affichées mais la partition n'est pas créée
+
+	public String PARTITION_NAME = "";				// Nom de la partition (optionnel)
+	public def STRUCTURAL_UNITS = [];				// Liste des unités structurelles sur lesquelles effectuer la partition, ex: ['text', 'div1']
+	public def STRUCTURAL_UNITS_PROPERTIES = [];	// Propriétés des unités structurelles sur lesquelles effectuer la partition, ex : ['id', 'name']
+	public def STRUCTURAL_UNITS_TO_IGNORE = [];	// Structure à ignorer, ex. CQL : !speaker (optionnel)
+	public String PART_NAMES_PREFIX = '';			// Prefix pour les noms de partie (optionnel)
+	public String EXPAND_TARGET = null;			// Expand to target, englobe les empans jusqu'à la balise parente spécifiée.
+	// NOTE : Le expand entre en conflit avec les sections à ignorer.
+	// Si la target est à un niveau supérieur aux balises à ignorer, il les remet dans liste de résultats CWB et elles ne sont donc pas ignorées
+
+	public def queries = [];
+	public def partNames = [];
+
+	/**
+	 * Init the generator and process.
+	 * @param corpusName
+	 */
+	public Partition createPartition(Corpus corpus, boolean debug,
+			String partitionName, String partNamePrefix,
+			def structuralUnits, def structuralUnitProperties,
+			def structuralUnitToIgnore, String expandTarget) 	{
+
+		PARTITION_NAME = partitionName
+		STRUCTURAL_UNITS = structuralUnits
+		STRUCTURAL_UNITS_PROPERTIES = structuralUnitProperties
+		STRUCTURAL_UNITS_TO_IGNORE = structuralUnitToIgnore
+		PART_NAMES_PREFIX = partNamePrefix
+		EXPAND_TARGET = expandTarget
+		DEBUG = debug;
+
+		if (DEBUG) {
+			println "Arguments: "
+			println "PARTITION_NAME = $PARTITION_NAME"
+			println "STRUCTURAL_UNITS = $STRUCTURAL_UNITS"
+			println "STRUCTURAL_UNITS_PROPERTIES = $STRUCTURAL_UNITS_PROPERTIES"
+			println "STRUCTURAL_UNITS_TO_IGNORE = $STRUCTURAL_UNITS_TO_IGNORE"
+			println "PART_NAMES_PREFIX = $PART_NAMES_PREFIX"
+			println "EXPAND_TARGET = $EXPAND_TARGET"
+			println "DEBUG = $DEBUG"
+		}
+
+		if (STRUCTURAL_UNITS.size() > 0 && STRUCTURAL_UNITS.size() == STRUCTURAL_UNITS_PROPERTIES.size())	{
+
+			if (DEBUG) println 'Creating the queries on corpus "' + corpus + "'" ;
+			if (DEBUG) println 'PARTITION_NAME: ' + PARTITION_NAME;
+
+			//Corpus corpus = CorpusManager.getCorpusManager().getCorpus(corpusName);
+
+			// Recursing through the corpus and subcorpus
+			process(corpus, 0, '', '');
+
+			// Displaying the queries
+			if (DEBUG) {
+				println "Queries processed: "
+				for (int i = 0 ; i < queries.size() ; i++) {
+					println partNames[i] + " = " + queries[i]
+				}
+			}
+
+			// Finalizing the queries
+			finalizeQueries();
+
+			// Displaying the queries
+			if (DEBUG) {
+				println "Queries finalized: "
+				for (int i = 0 ; i < queries.size() ; i++) {
+					println partNames[i] + " = " + queries[i]
+				}
+			}
+
+			if (DEBUG) println 'Queries created.';
+
+			// Creating the partition
+			if (!DEBUG  && queries.size() == partNames.size()) {
+				return corpus.createPartition(PARTITION_NAME, queries, partNames);
+			}
+		} else {
+			println "Error: Structural units count or structural units properties count error.";
+			return null
+		}
+	}
+
+	/**
+	 * Recurse through structural units and structural units properties of corpus and create the queries and the part names. 
+	 * @param corpus the corpus or subcorpus
+	 * @param index the index for recursion
+	 * @param tmpQuery the temporary query for creating subcorpus part
+	 * @param tmpPartName the temporary part name of the subcorpus part
+	 */
+	protected void process(Corpus corpus, int index, String tmpQuery, String tmpPartName)	{
+		// End of array
+		if (index >= STRUCTURAL_UNITS.size()) {
+
+			queries.add(tmpQuery);
+			partNames.add(PART_NAMES_PREFIX + tmpPartName);
+
+			return;
+		}
+
+		StructuralUnit su = corpus.getStructuralUnit(STRUCTURAL_UNITS[index]);
+		StructuralUnitProperty sup = su.getProperty(STRUCTURAL_UNITS_PROPERTIES[index]);
+
+		if (DEBUG) {
+			if (index == 0)	{
+				println 'Pocessing Structural Unit Property "' + sup.getFullName() + '" on mother corpus "' + corpus.getName() + '"';
+			} else {
+				println 'Pocessing Structural Unit Property "' + sup.getFullName() + '" on subcorpus part "' + tmpPartName + '"';
+			}
+		}
+
+
+		// Creating the queries parts for each structural units properties values
+		// for (supValue in sup.getOrderedValues()) { // TODO : signaler bug Matthieu, on ne devrait pas être obligé de repasser le sous-corpus à la méthode car sup a déjà été créée depuis le sous-corpus ? getValues() bugge aussi
+		for (supValue in sup.getOrderedValues(corpus)) {
+
+			// TODO : Log
+			if (DEBUG) println 'Value "' + supValue + '"';
+
+			// Getting the subcorpus linked to the structural unit property value
+			Subcorpus tmpSubcorpus = corpus.createSubcorpusWithQueryString(su, sup, supValue, "tmp" + UUID.randomUUID());
+
+			// Partition conditions and part name separators
+			String and = '';
+			String underscore = '';
+			if (tmpQuery != '')	{
+				underscore = ' ∩ ';
+				and = ' & ';
+			}
+
+			process(tmpSubcorpus, index + 1, (tmpQuery + and + '_.' + sup.getFullName() + '="' + supValue + '"'), tmpPartName + underscore + supValue);
+
+			// Deleting the temporary subcorpus
+			// TODO : bug : cette méthode ne supprime pas le corpus sans doute car il faut que le sous-corpus ne contienne pas d'autres sous-corpus ? le delete() en revanche fonctionne.
+			//			corpus.dropSubcorpus(tmpSubcorpus);
+			tmpSubcorpus.delete();
+		}
+	}
+
+
+	/**
+	 * 	Autoname the partition.
+	 * @param partitionName
+	 */
+	protected void autoNamePartition(String partitionName) {
+
+		// Structural units names and properties
+		for (int i = 0; i < STRUCTURAL_UNITS.size(); i++) {
+			partitionName +=  STRUCTURAL_UNITS[i] + '_' + STRUCTURAL_UNITS_PROPERTIES[i] + ' x ';
+		}
+
+		// Structural units to ignore
+		for (int i = 0; i < STRUCTURAL_UNITS_TO_IGNORE.size(); i++) {
+			partitionName +=  'NOT_' + STRUCTURAL_UNITS_TO_IGNORE[i] + '.';
+		}
+
+		// Removing last 'x' in partition name
+		PARTITION_NAME = partitionName.substring(0, partitionName.length() - 3);
+	}
+
+	/**
+	 * Finalize the queries.
+	 */
+	protected void finalizeQueries() {
+
+		String expandTo = '';
+		// Expanding to user defined target
+		if (EXPAND_TARGET != null && EXPAND_TARGET.length() > 0) {
+			expandTo = ' expand to ' + EXPAND_TARGET;
+		}
+		// Expanding to last child structural unit in user defined hierarchy
+		else if(STRUCTURAL_UNITS_TO_IGNORE.size() == 0)	 {
+			expandTo = ' expand to ' + STRUCTURAL_UNITS[STRUCTURAL_UNITS.size() - 1];
+		}
+
+		// Autonaming the partition
+		if (PARTITION_NAME != null && PARTITION_NAME.length() == 0) {
+			autoNamePartition(PARTITION_NAME);
+			// Finalizing partition name
+			if (EXPAND_TARGET != null && EXPAND_TARGET.length() > 0)
+				PARTITION_NAME += expandTo.replace(' expand to', '.EXPAND TO').replace(' ', '_');
+		}
+
+		// Finalizing queries
+		for (int j = 0; j < queries.size(); j++) {
+
+			String queryEnd = '';
+
+			// Removing some sections
+			for (sectionToIgnore in STRUCTURAL_UNITS_TO_IGNORE) {
+				queryEnd += ' & !' + sectionToIgnore;
+			}
+
+			queryEnd += ']' + expandTo;
+			queries.set(j, '[' +  queries.get(j) + queryEnd);
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/r/PlotSpecifMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/r/PlotSpecifMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/r/PlotSpecifMacro.groovy	(revision 321)
@@ -0,0 +1,101 @@
+package org.txm.macro.r;
+// Copyright © 2013-2016 ENS de Lyon, University of Franche-Comté.
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+
+import groovy.transform.Field
+
+import java.lang.invoke.MethodHandleImpl.BindCaller.T
+
+import org.kohsuke.args4j.*
+import org.txm.Toolbox
+import org.txm.rcpapplication.commands.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+/**
+ * 
+ * @authors Matthieu Decorde, Serge Heiden, Achille Falaise.
+ * 
+ * Sample default values from:
+ *  Lafon, P. “Sur la variabilité de la fréquence des formes dans un corpus.” Mots, no. 1 (1980):
+ *  Robespierre 'peuple' word in D9 part example (see Figure 1), pp 140-141.
+ *  <http://www.persee.fr/web/revues/home/prescript/article/mots_0243-6450_1980_num_1_1_1008>.
+ *  
+**/
+
+@Field @Option(name="f", usage="La fréquence de la forme dans la partie", widget="Integer", required=true, def="11")
+def f;
+
+@Field @Option(name="F", usage="La fréquence totale de la forme dans le corpus", widget="Integer", required=true, def="296")
+def F;
+
+@Field @Option(name="t", usage="Le nombre d'occurrences de la partie", widget="Integer", required=true, def="1084")
+def t;
+
+@Field @Option(name="T", usage="Le nombre total d'occurrences du corpus", widget="Integer", required=true, def="61449")
+def T;
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+// END OF PARAMETERS
+
+def r = RWorkspace.getRWorkspaceInstance()
+def file = File.createTempFile("txm", ".svg", new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "results"))
+
+/// BEGIN SCRIPTS
+def script ="""
+library(textometry)
+res <- specificities.distribution.plot($f, $F, $t, $T)
+"""
+/// END SCRIPTS
+r.plot(file, script)
+
+def px = r.eval('res$px').asDouble()
+def mode = r.eval('res$mode').asInteger()
+def pfsum = r.eval('res$pfsum'+"[$f+1]").asDouble()
+def isPositiveSpecificity = (f > mode.abs())  // True if S+
+def proba = pfsum.abs()  // Compute actual probability (should be a positive number, even for negative specificities, where pfsum = -probability)
+def specifSign = (isPositiveSpecificity ? "+" : "-")
+def specif = Math.log10(proba).abs()  // Compute actual specificity
+def operatorForDisplay = isPositiveSpecificity ? '>=' : '<='
+
+println """
+P(f'=f) et P(f' $operatorForDisplay f)
+f	F	t	T
+$f	$F	$t $T
+P(f' = $f) = $px
+mode = $mode
+P(f' $operatorForDisplay $f) = $proba
+Specificity S$specifSign = $specif
+"""
+println "Result saved in: "+file.getAbsolutePath()
+
+//display the graphic
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {	OpenSVGGraph.OpenSVGFile(file.getAbsolutePath(), "Specificity distribution") }
+});
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/CreateCQPListMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/CreateCQPListMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/CreateCQPListMacro.groovy	(revision 321)
@@ -0,0 +1,118 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+package org.txm.macro.cqp
+
+/*
+This macro allows you to define CQL lists which you can call in CQP queries during the current session.
+To use this macro for your own lists,
+1. Find below an example of the type of list you want to create
+   (case 1 = explicit list, or case 2 = list from file),
+2. Copy then paste this example (5 lines) after the given examples
+3. Give the name for your list as a string after list_name=
+4. Give the content of the list as a string after
+either list_value= (for case 1, explicit list)
+or list_path= (for case 2, list from file).
+5. Save this file (macro code)
+6. Run the macro by double-clicking on it.
+The lists are defined only for the current session.
+To use the same lists in a new session, just run the macro to generate the lists again.
+*/
+
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.*
+
+def CQI = Toolbox.getCqiClient();
+if ((CQI instanceof NetCqiClient)) {
+	println "Error: only available in CQP memory mode"
+	return;
+}
+
+def list_name=""
+def list_value=""
+def list_path=""
+def cqp_value=""
+File f;
+try {
+
+// BEGIN CONFIGURATION HERE
+
+// 1st case : explicit lists
+list_name = "negatif"
+list_value = "rien non ne pas sans"
+
+cqp_statement = "define \$$list_name = \"$list_value\";"
+println "CQP executes : $cqp_statement"
+CQI.query(cqp_statement)
+
+// two other examples of 1st case (explicit list)
+// 1st explicit list
+list_name = "evaluation"
+list_value = "trop assez peu bien très beaucoup vraiment"
+
+cqp_statement = "define \$$list_name = \"$list_value\";"
+println "CQP executes : $cqp_statement"
+CQI.query(cqp_statement)
+
+// 2nd explicit list
+list_name = "categoriesmotsvides"
+list_value = "ABR ADV DET:ART DET:POS INT KON NUM PRO PRO:DEM PRO:IND PRO:PER PRO:POS PRO:REL PRP PRP:det PUN PUN:cit SENT SYM"
+
+cqp_statement = "define \$$list_name = \"$list_value\";"
+println "CQP executes : $cqp_statement"
+CQI.query(cqp_statement)
+
+// one example of regular expression list to be used with RE()
+list_name = "regexp_categoriesmotsvides"
+list_value = "ABR ADV DET.* INT KON NUM PRO.* PRP.* PUN.* SENT SYM"
+
+cqp_statement = "define \$$list_name = \"$list_value\";"
+println "CQP executes : $cqp_statement"
+CQI.query(cqp_statement)
+
+// 2nd case : list from a file
+
+list_name = "motsvides"
+list_path = System.getProperty("user.home")+"/TXM/cqp/listes/stopwords_fr_ranknl.txt"
+// Windows users must replace "/" with "\\". 
+// for instance : 
+// list_path = System.getProperty("user.home")+"\\TXM\\cqp\\listes\\stopwords_fr_ranknl.txt"
+f = new File(list_path);
+if (f.exists()) {
+	cqp_statement = "define \$$list_name < \"$list_path\";"
+	println "CQP executes : $cqp_statement"
+	CQI.query(cqp_statement)
+} else {
+	println "Error: \"$f\" file does not exists"
+}
+
+/*
+You can comment a group of several lines of code
+using stars and slash like this.
+*/
+
+// END OF CONFIGURATION
+
+} catch (Exception e) {
+	println "Error during the \"$list_name\" list creation with the \"$list_value\" value"
+	println "CPQ statement was \"$cqp_statement\""
+	if (f != null) println "Last CQL file path was \"$f\""
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/ExecCQPMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/ExecCQPMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/ExecCQPMacro.groovy	(revision 321)
@@ -0,0 +1,173 @@
+package org.txm.macro.cqp
+// Copyright © 2014 ENS de Lyon
+//
+// Authors:
+// - Matthieu Decorde
+// - Serge Heiden
+//
+// Licence:
+// This file is part of the TXM platform.
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+// Version:
+// $LastChangedDate: 2014-11-01 16:00:01 +0100 (sam., 1 nov. 2014) $
+// $LastChangedRevision: XXXX $
+// $LastChangedBy: sheiden $
+//
+
+//Title:
+// EN:
+// Macro to call directly internal CQP corpus engine statements
+//
+// Warning: This macro must be used by advanced users,
+// at the risk of compromising CQP engine stability for the current TXM session
+//
+//FR:
+// Macro de demande d'exécution directe d'instructions au moteur de corpus interne CQP
+//
+// Avertissement : Cette macro doit être utilisée par des utilisateurs avertis,
+// au risque de compromettre la stabilité du moteur CQP pour la session TXM courante
+
+// Necessary declarations - Déclarations préalables
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.*
+
+// Sanity check - Vérification de cohérence
+def CQI = Toolbox.getCqiClient();
+if ((CQI instanceof NetCqiClient)) {
+	println "Error: CQP eval only available in CQP memory mode"
+	return;
+}
+
+// Parameter declaration - Déclaration du paramètre
+@Field @Option(name="statement", usage="CQP statement to execute (mind the ';' at the end)", widget="String", required=true, def="set da enpos;")
+String statement;
+
+// Usefull CQP statements - Instructions CQP Utiles //
+//
+// 1] Set a CQP variable - régler une variable CQP :
+// "set variable valeur;"
+//
+// Useful CQP variables
+// A) da = DefaultNonbrackAttr variable setting: the implicit word property in subsequent CQL queries
+//    For example, when DefaultNonbrackAttr=word, the "the" CQL query is interpreted by CQP as the '[word="the"]' query
+// Example Statements:
+// - set da word;     set implicit word property to "word" ('word' meaning graphical form of the word)
+// - set da enlemma;	set implicit word property to "enlemma" ('enlemma' meaning TreeTagger based 'En'glish 'Lemma')
+// - set da enpos;	set implicit word property to "enpos" ('enpos' meaning TreeTagger based 'En'glish 'P'art 'O'f 'S'peech)
+//
+// B) ms = MatchingStrategy variable setting: the CQL query word level iterators (?, *, +) resolution strategy used by CQP
+// Possible Statements:
+// - set ms shortest: ?, *, + word level operators should match sequences with the minimal number of words.
+//                    Optional words at the begining or at the end of the query are ignored.
+// - set ms standard: early match of ?, *, + word level operators are considered.
+//                    Optional words at the end of the query are ignored.
+// - set ms longest: ?, *, + word level operators should match sequences with the maximum number of words.
+// - set ms traditional: early match of ?, *, + word level operators are considered. Every sequence is returned.
+//
+// Example resolutions for each MatchingStrategy type 
+// (from The CQP Query Language Tutorial, (CWB version 2.2.b90), Stefan Evert, 10 July 2005)
+//
+// for the query:
+// [enpos="DET"]? [enpos="ADJ"]* [enpos="NN"] ([enpos="PREP"] [enpos="DET"]? [enpos="ADJ"]* [enpos="NN"])*
+//
+// Input:
+//     the old book on the table in the room
+//
+// - 'shortest' match strategy: (3 matches)
+// r1=         book
+// r2=                     table
+// r3=                                  room
+//
+// - 'longest' match strategy: (1 match)
+// r1= the old book on the table in the room
+//
+// - 'standard' matching strategy: (3 matches)
+// r1= the old book
+// r2=                 the table
+// r3=                              the room
+//
+// - 'traditional' matching strategy: (7 overlapping matches)
+// r1= the old book
+// r2=     old book
+// r3=         book
+// r4=                 the table
+// r5=                     table
+// r6=                              the room
+// r7=                                  room
+//
+// C) sr = StrictRegions variable setting: CQL queries must match a single structure
+// Possible Statements:
+// - set sr on: ?, *, + word level operators cannot cross a boundary of the structure involved in the query
+// - set sr off: ?, *, + word level operators can cross any number of boundaries of the structure involved in the query
+
+// Parameters settings UI
+if (!ParametersDialog.open(this)) {
+	println("** ExecCQLMacro error: Impossible to open Parameters settings UI dialog box.")
+	return
+}
+
+// Actual CQP call
+CQI.query(statement)
+
+// End of the macro
+
+/////////////////////////////////////
+//// CQP variables documentation ////
+/////////////////////////////////////
+
+//// Useful CQP variables ///
+// [da]	DefaultNonbrackAttr - implicit word property in queries ('word' by default)
+// [ms]	MatchingStrategy    - shortest | standard | longest | traditional
+// [sr]	StrictRegions       - single structure match
+
+//// Maybe useful variables ////
+// [o]	Optimize - simple regular expressions infixe optimizer
+// [hf]	HistoryFile
+// [wh]	WriteHistory
+
+//// Useless variables ////
+// [p]	 Paging
+// [pg]	 Pager
+// [h]	 Highlighting
+// [col] Colour
+// [pb]	 ProgressBar
+// [pp]	 PrettyPrint
+// [c]	 Context
+// [lc]	 LeftContext
+// [rc]	 RightContext
+// [ld]	 LeftKWICDelim
+// [rd]	 RightKWICDelim
+// [pm]	 PrintMode
+// [po]	 PrintOptions
+// [ps]	 PrintStructures
+// [sta] ShowTagAttributes
+// [st]	 ShowTargets
+// [as]	 AutoShow
+// [es]	 ExternalSort
+// [esc] ExternalSortCommand
+// 	     AutoSave
+// 	     SaveOnExit
+// 	     Timing
+
+//// Variables that must not be changed ////
+// [r]	 Registry
+// [dd]	 DataDirectory
+// [sub] AutoSubquery
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/SetMatchingStrategyMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/SetMatchingStrategyMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/cqp/SetMatchingStrategyMacro.groovy	(revision 321)
@@ -0,0 +1,42 @@
+package org.txm.macro.cqp
+
+// STANDARD DECLARATIONS
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.*
+
+def current = System.getProperty("cqp_matching_strategy")
+if (current != null)
+	println "Current strategy is: $current"
+
+def CQI = Toolbox.getCqiClient();
+if ((CQI instanceof NetCqiClient)) {
+	println "Error: only available in CQP memory mode"
+	return;
+}
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="matchingStrategy", usage="CQL matching strategy", widget="String", required=false, def="shortest | standard | longest | traditional")
+def matchingStrategy 
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+// END OF PARAMETERS
+
+matchingStrategy = matchingStrategy.trim().toLowerCase();
+switch(matchingStrategy ) {
+	case "shortest":
+	case "standard":
+	case "longest":
+	case "traditional":
+		println "Setting Greedy to $matchingStrategy "
+		CQI.query("set MatchingStrategy ${matchingStrategy };")
+		System.setProperty("cqp_matching_strategy",matchingStrategy ) 
+	break
+	case "shortest | standard | longest | traditional":
+		println "You must choose between shortest, standard, longest or traditional"
+	break
+	default:
+		println "Unknown matching strategy: ${matchingStrategy}. Aborting "
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/text/Text2TXTMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/text/Text2TXTMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/text/Text2TXTMacro.groovy	(revision 321)
@@ -0,0 +1,40 @@
+package org.txm.macro.text;
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.doc.*;
+import org.txm.importer.*;
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="inputDirectory", usage="the directory containing the DOC/ODT/RTF files to convert", widget="Folder", required=true, def="")
+File inputDirectory
+
+@Field @Option(name="extension", usage="an example file", widget="String", required=true, def="(odt|doc|rtf|html)")
+String extension = "doc"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+boolean debug = false;
+
+ConvertDocument converter;
+def files = []
+try {
+	converter = new ConvertDocument();
+	inputDirectory.eachFileMatch(~/.+\.$extension/) { docFile ->
+		String name = docFile.getName()
+		name = name.substring(0, name.indexOf("."))
+		def txtFile = new File(docFile.getParentFile(), name+".txt")
+		converter.autoFile(docFile, txtFile, "txt")
+		files << docFile
+	}
+} catch(Exception e) {
+	println "Error while processing directory: "+e;
+	if (debug) e.printStackTrace();
+}
+if (converter != null) converter.stop();
+
+println "Processed directory: $inputDirectory"
+println "files: "+files
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macro/xsl/ExecXSLMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macro/xsl/ExecXSLMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macro/xsl/ExecXSLMacro.groovy	(revision 321)
@@ -0,0 +1,68 @@
+package org.txm.macro.xsl;
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.importer.ApplyXsl2;
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="XSLFile", usage="an example file", widget="File", required=true, def="file.xsl")
+def XSLFile = new File(System.getProperty("user.home"),"TXM/xsl/identity.xsl")
+
+@Field @Option(name="intputDirectory", usage="an example folder", widget="Folder", required=true, def="in")
+def intputDirectory = new File(System.getProperty("user.home"),"xml/TESTS2/xml")
+
+@Field @Option(name="outputDirectory", usage="an example folder", widget="Folder", required=true, def="out")
+def outputDirectory = new File("/tmp/xsltest")
+
+//@Field @Option(name="parameters", usage="an example folder", widget="Text", required=false, def="")
+def parameters = ""
+
+@Field @Option(name="debug", usage="Show debug messages, value = true|false", widget="String", required=true, def="false")
+def debug
+
+if (!ParametersDialog.open(this)) return;
+// END OF PARAMETERS
+
+debug = "true" == debug
+
+outputDirectory.mkdir()
+
+parameters = parameters.trim()
+if (parameters.length() == 0) {
+	parameters = null
+} else {
+	def split = parameters.split("\n")
+	parameters = []
+	for (def str : split) {
+		def split2 = str.split("=", 2)
+		if (split2.size() == 2) {
+			parameters << split2
+		}
+	}
+}
+
+println "Use XSL $XSLFile with parameters $parameters"
+println "Processed directory: $intputDirectory"
+
+def files = [] 
+ApplyXsl2 a = new ApplyXsl2(XSLFile.getAbsolutePath());
+intputDirectory.eachFileMatch(~/.+\.(xml|XML)/) { XMLFile ->
+	String name = XMLFile.getName()
+	try {
+		
+		def outFile = new File(outputDirectory, name)
+		if (parameters != null) {
+			for(def param : parameters) {
+				a.setParam(param[0], param[1])
+			}
+		}
+		a.process(XMLFile, outFile);
+		a.resetParams()
+		files << XMLFile
+	} catch (Exception e) {
+		println "Warning: XSL transformation of '$name' failed with error=$e with "
+		if (debug) e.printStackTrace(); 
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/ant/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/ant/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains prototypes of Ant Groovy scripts</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build.java	(revision 321)
@@ -0,0 +1,184 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-12-06 16:57:35 +0100 (ven., 06 déc. 2013) $
+// $LastChangedRevision: 2583 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.ant;
+
+import groovy.util.ResourceException;
+import groovy.util.ScriptException;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.types.FileSet;
+import org.apache.tools.ant.types.Path;
+import org.apache.tools.ant.types.selectors.FileSelector;
+import org.codehaus.groovy.ant.Groovyc;
+
+// TODO: Auto-generated Javadoc
+/**
+ *<taskdef name="groovyc"
+         classname="org.codehaus.groovy.ant.Groovyc"
+         classpathref=classpath/>
+ **/
+public class Build
+{
+
+	/**
+	 * Run.
+	 * @throws IOException 
+	 * @throws ScriptException 
+	 * @throws ResourceException 
+	 */
+	public void run() throws IOException, ResourceException, ScriptException
+	{
+		String userdir = System.getProperty("user.home");
+
+		Groovyc groovyc = new Groovyc();
+
+		groovyc.setCaseSensitive(true);
+		groovyc.setListfiles(true);
+		groovyc.setStacktrace(true);
+		groovyc.setVerbose(true);
+
+		System.out.println("Build project"); 
+		Project project = new Project();
+		project.setBaseDir(new File(userdir, "scripts"));
+		project.setName("Scripts");
+		project.setDescription("The TXM groovy scripts");
+		groovyc.setProject(project);
+		
+		System.out.println("Set libs classpath"); 
+		File libdir = new File(userdir, "Bureau/exportRCP/linux.gtk.x86/TXM/plugins");
+		Path classpath = groovyc.createClasspath();
+		FileSet fs = new FileSet();
+		fs.setDir(libdir);
+		fs.add(new FileSelector() {
+			@Override
+			public boolean isSelected(File arg0, String arg1, File arg2)
+					throws BuildException {
+					return true;
+			}
+		});
+		classpath.addFileset(fs);
+		classpath.setProject(project);	
+		
+		// src dir
+		Path srcdir = groovyc.createSourcepath();
+		srcdir.setLocation(new File(userdir, "scripts"));
+		srcdir.setProject(project);
+		
+		groovyc.setEncoding("UTF-8");
+		groovyc.setSrcdir(srcdir);
+		groovyc.setDestdir(new File(userdir, "scripts/bin"));
+		groovyc.setClasspath(classpath);
+
+		System.out.println("srcdir "+groovyc.getSrcdir());
+		System.out.println("destdir "+groovyc.getDestdir());
+		System.out.println("classpath "+groovyc.getClasspath());
+		System.out.println("encoding "+groovyc.getEncoding());
+
+		groovyc.execute();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String args[])
+	{
+		try {
+			new Build().run();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (ResourceException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (ScriptException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}
+/*
+class Build {
+String srcdirectory ="~/TXM/scripts/src"
+
+	AntBuilder ant = new AntBuilder().sequential {
+	def webinf = "."
+
+	taskdef name: "groovyc", classname: "org.codehaus.groovy.ant.Groovyc"
+	groovyc srcdir: "src", destdir: "${webinf}", encoding: "UTF-8", {
+		classpath {
+			fileset dir: "${webinf}/plugins", {
+		    	include name: "*.jar"
+			}
+			pathelement path: "${webinf}"
+		}
+		javac source: "1.5", target: "1.5", debug: "on"
+	}
+}
+
+
+   def base_dir = "./"
+   def src_dir = base_dir + "src"
+   def lib_dir = base_dir + "lib"
+   def build_dir = base_dir + "bin"
+   def dist_dir = base_dir + "dist"
+   def file_name = "mdecorde"
+
+   def clean() {
+      ant.delete(dir: "${build_dir}")
+      ant.delete(dir: "${dist_dir}")
+   }
+   def build() {
+      ant.mkdir(dir: "${build_dir}")
+      ant.javac(destdir: "${build_dir}", srcdir: "${src_dir}", classpath: "${classpath}")
+      ant.groovyc(destdir: "${build_dir}", srcdir: "${src_dir}", classpath: "${classpath}")              
+   }
+   def jar() {
+      clean()
+      build()
+      ant.mkdir(dir: "${dist_dir}")
+      ant.jar(destfile: "${dist_dir}/${file_name}.jar", basedir: "${build_dir}")
+   }
+   def static void main(args) {
+      Build b = new Build()
+      b.run(args)
+   }
+   void run(args) {
+      if ( args.size() > 0 ) { 
+         invokeMethod(args[0], null )
+      }
+      else {
+        // build()
+      } 
+   }
+}*/

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build2.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build2.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build2.java	(revision 321)
@@ -0,0 +1,19 @@
+package org.txm.ant;
+
+import groovy.lang.Binding;
+import groovy.util.GroovyScriptEngine;
+ 
+class Build2
+{
+	public void run() throws Exception
+	{
+		String userdir = System.getProperty("user.home");
+		String[] roots = new String[] { userdir+"/scripts"};
+		GroovyScriptEngine gse = new GroovyScriptEngine(roots, this.getClass().getClassLoader());
+		gse.run("A.groovy", new Binding());
+	}
+	public static void main(String args[]) throws Exception
+	{
+		new Build2().run();
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build2.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/AntTest.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/ant/AntTest.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/ant/AntTest.groovy	(revision 321)
@@ -0,0 +1,20 @@
+package org.txm.ant;
+
+String userdir = System.getProperty("user.home");
+File libdir = new File(userdir, "Bureau/exportRCP/linux.gtk.x86/TXM/plugins");
+File srcdir = new File(userdir, "TXM/scripts/test")
+File destdir = new File(userdir, "TXM/scripts/bin")
+def ant = new AntBuilder().sequential {
+	//webinf = libdir.getAbsolutePath()
+	taskdef name: "groovyc", classname: "org.codehaus.groovy.ant.Groovyc"
+	groovyc srcdir: srcdir.getAbsolutePath(), destdir: destdir.getAbsolutePath(), {
+		classpath {
+			fileset dir: libdir.getAbsolutePath(), {
+		    	include name: "*.jar"
+			}
+			pathelement path: libdir.getAbsolutePath()+"/classes"
+		}
+		javac source: "1.5", target: "1.5", debug: "on"
+	}
+}
+println "fin"
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build3.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build3.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/ant/Build3.groovy	(revision 321)
@@ -0,0 +1,19 @@
+package org.txm.ant;
+
+import java.io.IOException;
+
+import groovy.lang.Binding;
+import groovy.util.GroovyScriptEngine;
+import groovy.util.ResourceException;
+import groovy.util.ScriptException;
+
+		String userdir = System.getProperty("user.home");
+		String[] roots = [ userdir+"/TXM/scripts", userdir+"/TXM/scripts/test"];
+		GroovyScriptEngine gse = new GroovyScriptEngine(roots, this.getClass().getClassLoader());
+		//gse.loadScriptByName("test/B.groovy");
+		//System.out.println("B: " +gse.getResourceConnection("test/B.groovy"));
+		Binding binding = new Binding();
+		binding.setVariable("input", "world");
+		gse.run("test/A.groovy", binding);
+		
+	
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/doc/DocumentToTei.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/doc/DocumentToTei.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/doc/DocumentToTei.groovy	(revision 321)
@@ -0,0 +1,175 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-03-14 17:07:32 +0100 (Mon, 14 Mar 2016) $
+// $LastChangedRevision: 3165 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.doc
+
+import java.io.File;
+
+import org.txm.importer.ApplyXsl2;
+import org.txm.importer.ConvertDocument;
+import org.txm.utils.zip.Zip;
+
+/**
+ * The Class DocumentToTei.
+ */
+class DocumentToTei {
+
+	static boolean DEBUG = true;
+
+	public static boolean processFiles(def files, File outdir, File xsldir) {
+		boolean ret = true;
+
+		File xslOdtTei = new File(xsldir, "tei/odt/odttotei.xsl");
+
+		if (!xsldir.exists()) { println "XslDir does not exists: "+xsldir; return false;}
+		if (!xslOdtTei.exists()) { println "xslOdtTei file does not exists: "+xslOdtTei; return false;}
+
+		ConvertDocument converter;
+		for (File file : files) {
+			print "."
+			String name = file.getName();
+			if (file.getName().endsWith(".odt")) {
+				name = name.substring(0, name.indexOf(".odt"));
+				File teifile = new File(outdir, name+".xml");
+				try {
+					if (!new DocumentToTei().run(file, xslOdtTei, teifile)) {
+						println "ODT to TEI failed: "+file
+						//						ret = false;
+						//						break;
+					}
+				} catch(Exception e) { println "ODT to TEI failed: $file: $e"; }
+			} else if (file.getName().endsWith(".docx")) {
+				name = name.substring(0, name.indexOf(".docx"));
+				File teifile = new File(outdir, name+".xml");
+				File odtFile = File.createTempFile("workflowdocx", "sfsdf.odt", outdir);
+				
+				try {
+					converter = new ConvertDocument();
+					converter.setDebug(DEBUG)
+					converter.autoFile(file, odtFile, "odt")
+					if (!new DocumentToTei().run(odtFile, xslOdtTei, teifile)) {
+						println "Docx to Odt to Tei failed: "+file
+						odtFile.delete();
+					}
+				} catch(Exception e) { println "DOCX to ODT to TEI failed: $file: $e"; }
+				finally { 
+					odtFile.delete();
+					if (converter != null) converter.stop();
+				}
+				
+			} else if (file.getName().endsWith(".doc")) {
+				name = name.substring(0, name.indexOf(".doc"));
+				File teifile = new File(outdir, name+".xml");
+				File odtFile = File.createTempFile("workflowdoc", "sfsdf.odt", outdir);
+				
+				try {
+					converter = new ConvertDocument();
+					converter.setDebug(DEBUG)
+					converter.autoFile(file, odtFile, "odt")
+					if (!new DocumentToTei().run(odtFile, xslOdtTei, teifile)) {
+						println "Doc to ODT to TEI failed: "+file
+						odtFile.delete();
+					}
+				} catch(Exception e) { println "DOC to ODT to TEI failed: $file: $e"; }
+				finally {
+					odtFile.delete();
+					if (converter != null) converter.stop();
+				}
+				
+			}
+		}
+
+		return ret;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param odtfile the odtfile
+	 * @param xslfile the xslfile
+	 * @param teifile the teifile
+	 * @return true, if successful
+	 */
+	public boolean run(File odtfile, File xslfile, File teifile)
+	{
+		println "** ODT TO TEI $odtfile $teifile"
+		File unzipdir = new File(teifile.getParent(), "files-"+teifile.getName());
+		unzipdir.deleteDir()
+		Zip.decompress(odtfile.getAbsolutePath(), unzipdir.getAbsolutePath(), false);
+
+		//println "get content.xml"
+		File content = new File(unzipdir, "content.xml");
+		if (!content.exists()) {
+			println "OdtToTei: failed to get 'content.xml' file from "+unzipdir
+			return false;
+		}
+
+		//println "apply xsl oo-to-tei"
+		ApplyXsl2 a = new ApplyXsl2(xslfile.getAbsolutePath());
+		a.process(content.getAbsolutePath(), teifile.getAbsolutePath());
+
+		//unzipdir.deleteDir()
+		return true;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param odtfile the odtfile
+	 * @param xslfile the xslfile
+	 * @param teifile the teifile
+	 * @return true, if successful
+	 */
+	public boolean rundocx(File docxfile, File xslfile, File teifile)
+	{
+		//println "** ODT TO TEI $docxfile $teifile"
+		File unzipdir = new File(docxfile.getParent(), "files-"+teifile.getName());
+		unzipdir.deleteDir()
+		Zip.decompress(docxfile.getAbsolutePath(), unzipdir.getAbsolutePath(), false);
+
+		//println "get content.xml"
+		File content = new File(unzipdir, "word/document.xml");
+		if (!content.exists()) {
+			println "failed to get $content"
+			return false;
+		}
+
+		//println "apply xsl oo-to-tei"
+		ApplyXsl2 a = new ApplyXsl2(xslfile.getAbsolutePath());
+		a.setParam("word-directory", unzipdir.getAbsolutePath())
+		a.process(content.getAbsolutePath(), teifile.getAbsolutePath());
+
+		return true;
+	}
+	
+	public static void main(String[] args) {
+		File odtfile = new File("/home/mdecorde/xml/manual/Manuel de TXM 0.7 FR.odt")
+		File xslfile = new File("/home/mdecorde/TXM/xsl/tei/odt/odttotei.xsl")
+		File teifile = new File("/home/mdecorde/xml/manual/Manuel de TXM 0.7 FR.odt.xml")
+		new DocumentToTei().run(odtfile, xslfile, teifile)
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/doc/TeiCorrector.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/doc/TeiCorrector.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/doc/TeiCorrector.groovy	(revision 321)
@@ -0,0 +1,51 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.doc
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class TeiCorrector.
+ */
+class TeiCorrector {
+	
+	/**
+	 * Run.
+	 *
+	 * @param teifile the teifile
+	 * @param headerfile the headerfile
+	 * @return true, if successful
+	 */
+	public boolean run(File teifile, File headerfile)
+	{
+		println "** TEICORRECTOR $teifile"
+		println "get header file"
+		println "paste header"
+		println "remove false header (up to <div rend=\"P92\">"
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/doc/TeiToHtml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/doc/TeiToHtml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/doc/TeiToHtml.groovy	(revision 321)
@@ -0,0 +1,77 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.doc
+
+import java.io.File;
+import org.txm.importer.ApplyXsl;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class TeiToHtml.
+ */
+class TeiToHtml {
+	
+	/** The imgdirname. */
+	static String imgdirname = "ImagesTXM"
+		
+	/**
+	 * Run.
+	 *
+	 * @param teifile the teifile
+	 * @param xslfile the xslfile
+	 * @param htmloutdir the htmloutdir
+	 * @return true, if successful
+	 */
+	public boolean run(File teifile, File xslfile, File htmloutdir)
+	{
+		println "** TEI TO HTML $teifile $htmloutdir"
+		
+		println "Check Pictures directory"
+		if(!new File(htmloutdir.getParentFile(), imgdirname).exists())
+		{
+			println "$imgdirname directory does not exists: "+new File(htmloutdir.getParentFile(), imgdirname).getAbsolutePath()
+			return false;
+		}
+		
+		println "create html outdirectory with folders HTML and ImgTXM"
+		if(!htmloutdir.mkdir())
+		{
+			if(!htmloutdir.exists())
+			{println "Failed to create "+htmloutdir.getAbsolutePath();
+			return false;
+			}
+		}
+		
+		println "apply xsl tei to HTML"
+		ApplyXsl a = new ApplyXsl(xslfile.getAbsolutePath());
+		String[] args = ["outputDir",htmloutdir];
+		a.process(teifile.getAbsolutePath(), "./HTML",args);
+
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/doc/Workflow.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/doc/Workflow.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/doc/Workflow.groovy	(revision 321)
@@ -0,0 +1,92 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.doc
+
+import org.txm.importer.ConvertDocument;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String home = System.getProperty("user.home");
+File srcdir = new File(home, "xml/txmrefman")
+File outdir = new File(home, "xml/txmrefman/TEI")
+File xsldir = new File(home, "xsl")
+File xslOdtTei = new File(xsldir, "tei/odt/odttotei.xsl");
+File xslDocxTei = new File(xsldir, "tei/docx/from/docxtotei.xsl");
+
+if (!xslOdtTei.exists()) {
+	println "$xslOdtTei file does not exists"
+	return;
+}
+
+outdir.deleteDir()
+outdir.mkdirs()
+
+println "start"
+for (File file : srcdir.listFiles()) {
+	ConvertDocument converter = new ConvertDocument();
+	try {
+		File teifile = new File(outdir, file.getName()+".xml");
+		if (file.getName().endsWith(".odt")) {
+			if (!new DocumentToTei().run(file, xslOdtTei, teifile)) {
+				println "Odt to Tei failed"
+				return;
+			}
+		} else if (file.getName().endsWith(".docx")) {
+			File odtFile = File.createTempFile("workflowodt", "sfsdf.odt", srcdir);
+			// convert doc to odt
+			converter.autoFile(file, odtFile, "odt")
+			if (!new DocumentToTei().run(odtFile, xslOdtTei, teifile)) {
+				println "Docx to Odt to Tei failed: "+file
+				odtFile.delete();
+				return;
+			}
+			odtFile.delete();
+		} else if (file.getName().endsWith(".doc")) {
+			File odtFile = File.createTempFile("workflowodt", "sfsdf.odt", srcdir);
+			// convert doc to odt
+			converter.autoFile(file, odtFile, "odt")
+			if (!new DocumentToTei().run(odtFile, xslOdtTei, teifile)) {
+				println "Doc to Odt to Tei failed: "+file
+				odtFile.delete();
+				return;
+			}
+			odtFile.delete();
+		}
+	}
+	catch(Exception e) { e.printStackTrace() }
+	finally { converter.stop(); }
+}
+
+println "--Done"
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/doc/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/doc/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/doc/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains scripts to manipulates Word or ODT or RTF files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/doc/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/RenameFilesMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/RenameFilesMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/RenameFilesMacro.groovy	(revision 321)
@@ -0,0 +1,37 @@
+// STANDARD DECLARATIONS
+package org.txm.macroproto
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+// BEGINNING OF PARAMETERS
+
+@Field @Option(name="inputDirectory", usage="TXT directory", widget="Folder", required=false, def="txt")
+File inputDirectory;
+
+// **change this parameter**
+@Field @Option(name="extension", usage="Regexp de l'extension des fichiers à modifier", widget="String", required=true, def='\\.txt')
+def extension = "\\.trs"
+
+// **change this parameter**
+@Field @Option(name="find", usage="Expression régulière", widget="String", required=true, def='’')
+def find = "Bobine"
+
+// **change this parameter**
+@Field @Option(name="replaceWith", usage="Chaîne de remplacement", widget="String", required=false, def='\'')
+def replaceWith = ""
+
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+// END OF PARAMETERS
+
+println "In $inputDirectory..."
+inputDirectory.eachFileMatch(~/.*$extension/) { file ->               // for each file matching extension
+		println " renaming: "+file.getName()
+		String name = file.getName()
+		name = name.replaceAll(find, replaceWith)
+		file.renameTo(new File(file.getParentFile(), name))
+	}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/StanfordSegmenterMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/StanfordSegmenterMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/StanfordSegmenterMacro.groovy	(revision 321)
@@ -0,0 +1,121 @@
+package org.txm.macroproto;
+// STANDARD DECLARATIONS
+/*
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import edu.stanford.nlp.international.arabic.process.*
+import edu.stanford.nlp.international.arabic.*
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import edu.stanford.nlp.ie.crf.CRFClassifier;
+import edu.stanford.nlp.io.IOUtils;
+import edu.stanford.nlp.ling.CoreLabel;
+import edu.stanford.nlp.ling.HasWord;
+import edu.stanford.nlp.ling.Sentence;
+import edu.stanford.nlp.ling.TaggedWord;
+import edu.stanford.nlp.ling.CoreAnnotations;
+import edu.stanford.nlp.objectbank.ObjectBank;
+import edu.stanford.nlp.process.TokenizerFactory;
+import edu.stanford.nlp.process.WordSegmenter;
+import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
+import edu.stanford.nlp.sequences.SeqClassifierFlags;
+import edu.stanford.nlp.stats.ClassicCounter;
+import edu.stanford.nlp.stats.Counter;
+import edu.stanford.nlp.trees.Tree;
+import edu.stanford.nlp.util.Generics;
+import edu.stanford.nlp.util.PropertiesUtils;
+import edu.stanford.nlp.util.StringUtils;
+import edu.stanford.nlp.util.concurrent.MulticoreWrapper;
+import edu.stanford.nlp.util.concurrent.ThreadsafeProcessor;
+
+@Field @Option(name="inputDir", usage="input directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe")
+File inputDir
+@Field @Option(name="outputDir", usage="output directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe/out-s")
+File outputDir
+
+@Field @Option(name="model", usage="'.tagger' model file", widget="File", required=false, def="data/arabic-segmenter-atbtrain.ser.gz")
+File model
+@Field @Option(name="extension", usage="Regexp de l'extension des fichiers à modifier", widget="String", required=true, def='\\.txt')
+String extension = "\\.txt"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+outputDir.mkdir()
+if (!outputDir.exists()) {
+	println("Could not create $outputDir")
+	return;
+}
+
+// END OF PARAMETERS
+//File inputFile = new File("/home/mdecorde/xml/testarabe/test.txt")
+inputDir.eachFileMatch(~/.*$extension/) { inputFile ->
+println "Processing file: $inputFile"
+File outputFile = new File(outputDir, inputFile.getName())
+//File model = new File("/home/mdecorde/LIBRAIRIES/stanford-segmenter-2014-01-04/data/arabic-segmenter-atbtrain.ser.gz")
+Properties options = new Properties();
+//options.put("prefixMarker", "#")
+//options.put("suffixMarker", "#")
+options.put("loadClassifier", model.getAbsolutePath())
+ArabicSegmenter segmenter = new ArabicSegmenter(options);
+segmenter.flags.inputEncoding = "UTF-8";
+segmenter.loadSegmenter(segmenter.flags.loadClassifier, options);
+
+def br = inputFile.newReader("UTF-8")
+OutputStreamWriter out = new OutputStreamWriter( new FileOutputStream(outputFile), "UTF-8");
+PrintWriter pw = new PrintWriter(out, true);
+double charsPerSec = decode(segmenter, br, pw, 5);
+IOUtils.closeIgnoringExceptions(br);
+System.err.printf("Done! Processed input text at %.2f input characters/second%n", charsPerSec);
+
+}
+
+def decode(ArabicSegmenter segmenter, BufferedReader br,
+                               PrintWriter pwOut, int nThreads) {
+    assert nThreads > 0;
+    long nChars = 0;
+    final long startTime = System.nanoTime();
+    if (nThreads > 1) {
+      MulticoreWrapper<String,String> wrapper = new MulticoreWrapper<String,String>(nThreads, segmenter);
+      try {
+        for (String line; (line = br.readLine()) != null;) {
+          nChars += line.length();
+          wrapper.put(line);
+          while (wrapper.peek()) {
+            pwOut.println(wrapper.poll());
+          }
+        }
+
+        wrapper.join();
+        while (wrapper.peek()) {
+          pwOut.println(wrapper.poll());
+        }
+
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+
+    } else {
+      nChars = segmenter.segment(br, pwOut);
+    }
+    long duration = System.nanoTime() - startTime;
+    double charsPerSec = (double) nChars / (duration / 1000000000.0);
+    return charsPerSec;
+  }
+*/
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/StanfordTaggerMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/StanfordTaggerMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/StanfordTaggerMacro.groovy	(revision 321)
@@ -0,0 +1,67 @@
+package org.txm.macroproto;
+/*
+
+import edu.stanford.nlp.ling.*
+import edu.stanford.nlp.process.*
+import edu.stanford.nlp.tagger.maxent.*;
+
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+@Field @Option(name="inputDir", usage="input directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe/out-s")
+File inputDir
+@Field @Option(name="outputDir", usage="output directory", widget="Folder", required=true, def="/home/mdecorde/xml/testarabe/out-t")
+File outputDir
+
+@Field @Option(name="model", usage="'.tagger' model file", widget="File", required=false, def="taggers/arabic-train.tagger")
+File model
+@Field @Option(name="extension", usage="Regexp de l'extension des fichiers à modifier", widget="String", required=true, def='\\.txt')
+String extension = "\\.txt"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+def modelPath = model.getAbsolutePath()
+
+outputDir.mkdir()
+if (!outputDir.exists()) {
+	println("Could not create $outputDir")
+	return;
+}
+
+inputDir.eachFileMatch(~/.*$extension/) { inputFile ->
+	String name = inputFile.getName()
+	int idx = name.indexOf(".")
+	if (idx > 0) name = name.substring(0, idx)
+	
+	new File(outputDir, name+".xml").withWriter("UTF-8") { writer ->
+		
+		writer.println('<?xml version="1.0" encoding="UTF-8"?>')
+		writer.println('<text>')
+		
+		MaxentTagger tagger = new MaxentTagger(modelPath);
+		TokenizerFactory<CoreLabel> ptbTokenizerFactory = PTBTokenizer.factory(new CoreLabelTokenFactory(),
+				"untokenizable=noneKeep");
+	
+		DocumentPreprocessor documentPreprocessor = new DocumentPreprocessor(inputFile.newReader());
+		documentPreprocessor.setTokenizerFactory(ptbTokenizerFactory);
+		int n = 0
+		
+		for (List<HasWord> sentence : documentPreprocessor) {
+			writer.println("<s n=\"$n\">")
+			n++;
+			List<TaggedWord> tSentence = tagger.tagSentence(sentence);
+			//println(Sentence.listToString(tSentence, false));
+			for (def TaggedWord w : tSentence) {
+				def form = w.word();
+				def pos = w.tag();
+				writer.println("<w pos=\"$pos\">$form</w>")
+			}
+			writer.println('</s>')
+		}
+		writer.println('</text>')
+	}
+}
+*/
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/SelectBanal.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/SelectBanal.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/SelectBanal.groovy	(revision 321)
@@ -0,0 +1,84 @@
+package org.txm.macroproto;
+import groovy.transform.Field
+
+import org.kohsuke.args4j.*
+import org.txm.*
+import org.txm.functions.ca.*
+import org.txm.functions.specificities.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.clientExceptions.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.stat.data.*
+import org.txm.stat.engine.r.*
+
+//TODO: temporary uncommented
+//if (!(corpusViewSelection instanceof SpecificitesResult)) {
+//	println "Selection if not a Specificites Result"
+//	return;
+//}
+
+// PARAMETERS
+@Field @Option(name="outfile", usage="CSV result file ", widget="File", required=true, def="result.csv")
+def outfile;// = new File("/home/mdecorde/Bureau/test.csv")
+
+@Field @Option(name="MAX", usage="score limit", widget="Integer", required=false, def="1")
+def MAX;// = 1.0
+
+@Field @Option(name="suseSum", usage="Use the row sums : y/n", widget="String", required=false, def="n")
+String suseSum;// = false
+
+// END OF PARAMETERS
+if (!ParametersDialog.open(this)) return;
+
+if (suseSum == null || suseSum.length() == 0) suseSum = "n"
+boolean useSum = suseSum.toLowerCase().charAt(0) == "y"
+println "useSum $useSum"
+
+def writer = outfile.newWriter("UTF-8")
+def specif = corpusViewSelection
+def indices = specif.getSpecificitesIndex()
+def freqs = specif.getFrequency()
+def rownames = specif.getTypeNames();
+def colnames = specif.getPartShortNames() ;
+
+def selected = []
+
+writer.print "unit"
+writer.print "\tF"
+for (int j = 0; j < colnames.size() ; j++) {
+	writer.print "\t"+colnames[j]
+	writer.print "\tscore"
+}
+writer.println ""
+
+for (int i = 0; i < rownames.length ; i++) {
+	boolean add = true
+	def totF = 0;
+	def totscore = 0.0
+
+	for (int j = 0; j < colnames.size() ; j++) {
+		if (Math.abs(indices[i][j]) >= MAX) add = false
+		totF += freqs[i][j]
+		totscore += Math.abs(indices[i][j])
+	}
+	
+	if (useSum) {
+		if (totscore >= MAX) add = false
+		else add = true
+	}
+	
+	if (add) {
+		selected << rownames[i]
+		writer.print rownames[i]
+		writer.print "\t$totF"
+		for (int j = 0; j < colnames.size() ; j++) {
+			writer.print "\t"+freqs[i][j]
+			writer.print "\t"+indices[i][j]
+		}
+		writer.println ""
+	}
+}
+println selected
+println "Saved in $outfile"
+writer.close()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/export/ExportTextsContentMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/export/ExportTextsContentMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/export/ExportTextsContentMacro.groovy	(revision 321)
@@ -0,0 +1,80 @@
+// STANDARD DECLARATIONS
+package org.txm.macroproto.export
+
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.Query
+import org.txm.Toolbox
+import org.txm.utils.i18n.LangFormater;
+import org.apache.commons.lang.StringUtils;
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+println "corpora selection: "+corpusViewSelection
+
+if (!(corpusViewSelection instanceof Corpus)) {
+	println "Please select a corpus"
+	return;
+}
+
+// PARAMETERS
+
+@Field @Option(name="exportDir", usage="Result directory ", widget="Folder", required=true, def="export")
+File exportDir
+
+@Field @Option(name="lineSeparatorStructureName", usage="line separator structure", widget="String", required=false, def="p")
+def lineSeparatorStructureName
+
+if (!ParametersDialog.open(this)) return
+
+// BEGINNING
+
+if (!exportDir.exists()) exportDir.mkdirs()
+
+Corpus corpus = corpusViewSelection
+CQI = Toolbox.getCqiClient();
+
+def lineSeparatorStructure = corpus.getStructuralUnit(lineSeparatorStructureName)
+
+if (lineSeparatorStructure == null) {
+	println "The $lineSeparatorStructureName structure does not exists in the $corpus corpus"
+	return;
+}
+
+def breaks_pos = Arrays.asList(corpus.query(new Query("[]</"+lineSeparatorStructureName+">"),"test", false).getEnds())
+println breaks_pos
+
+println "Exporting $corpus text content to $exportDir"
+
+def wordProperty = corpus.getWordProperty()
+def textidProperty = corpus.getStructuralUnit("text").getProperty("id")
+def textStartBoundaries = corpus.getTextStartLimits();
+def textEndBoundaries = corpus.getTextEndLimits();
+int[] struct_pos = CQI.cpos2Struc(textidProperty.getQualifiedName(), textStartBoundaries)
+String[] textids =  CQI.struc2Str(textidProperty.getQualifiedName(), struct_pos)
+println ""+textStartBoundaries.size()+" texts"
+
+for (int i = 0 ; i < textStartBoundaries.size() ; i++) {
+	int start = textStartBoundaries[i];
+	int end = textEndBoundaries[i]
+
+	File txtFile = new File(exportDir, textids[i]+".txt")
+	print ".."
+	def writer = txtFile.newWriter("UTF-8")
+	int[] positions = new int[end - start + 1]
+	int c = 0;
+	for (int p : start..end) {
+		positions[c++] = p
+	}
+	int[] idx = CQI.cpos2Id(wordProperty.getQualifiedName(), positions)
+	def words = CQI.id2Str(wordProperty.getQualifiedName(), idx)
+	for (int j = 0 ; j < positions.length ; j++) {
+		int p = positions[j]
+		if (breaks_pos.contains(p)) words[j] = words[j] +"\n" 
+	}
+	writer.println LangFormater.format(StringUtils.join(words, " "),
+				corpus.getAttribute("lang"));
+	writer.close();
+}  
+
+println "\nDone, result saved in "+exportDir.getAbsolutePath()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/export/ExportTextsMetadataMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/export/ExportTextsMetadataMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/export/ExportTextsMetadataMacro.groovy	(revision 321)
@@ -0,0 +1,83 @@
+// STANDARD DECLARATIONS
+package org.txm.macroproto.export
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.Toolbox
+
+if (!(corpusViewSelection instanceof Corpus)) {
+	println "Please select a corpus first"
+	return
+}
+
+// PARAMETERS
+
+@Field @Option(name="tsvFile", usage="CSV file path", widget="File", required=true, def="/tmp/metadata.csv")
+def File tsvFile
+
+@Field @Option(name="columnSeparator", usage="column columnSeparator", widget="String", required=false, def="\t")
+def columnSeparator
+
+if (!ParametersDialog.open(this)) return
+
+// BEGINNING
+def Corpus corpus = corpusViewSelection
+def CQI = Toolbox.getCqiClient()
+def writer = tsvFile.newWriter("UTF-8")
+def internalTextProperties = ["project", "base", "path"]
+
+println "Exporting $corpus metadata to $tsvFile file."
+
+def text = corpus.getStructuralUnit("text")
+def properties = text.getProperties()
+
+// putting the "id" property in the first position
+def idi = properties.findIndexOf { prop -> prop.toString()=="id" }
+if (idi != 0 && idi > 0) {
+	def tmp = properties[0]
+	properties[0] = properties[idi]
+	properties[idi] = tmp
+} else if (idi == -1) {
+println sprintf("** Warning: incoherent metadata content found for %s corpus - no 'id' metadata found", corpus)
+println "** Aborting"
+return
+}
+
+// getting values for all texts and all text metadata
+def propertyValues = [:]
+def numberOfTexts = CQI.attributeSize(properties[0].getQualifiedName())
+def int[] struct_pos = new int[numberOfTexts]
+
+struct_pos.eachWithIndex { c, i -> struct_pos[i] = i }
+
+properties.each { property ->
+	def values = CQI.struc2Str(property.getQualifiedName(), struct_pos)
+	propertyValues[property] = values
+}
+
+// writing result to tsvFile
+def c = 0
+properties.each { property ->
+	if (!internalTextProperties.contains(property.getName())) {
+		if (c++ > 0) writer.print columnSeparator
+		writer.print "$property"
+	}
+}
+writer.println ""
+
+numberOfTexts.times {
+	c = 0
+	properties.each { property ->
+		if (!internalTextProperties.contains(property.getName())) {
+			if (c++ > 0) writer.print columnSeparator
+			writer.print propertyValues[property][it]
+		}
+	}
+	writer.println ""
+}
+
+writer.close()
+println sprintf("Wrote %d different metadata for %d texts.", c, numberOfTexts)
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/importer/XTZImporterMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/importer/XTZImporterMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/importer/XTZImporterMacro.groovy	(revision 321)
@@ -0,0 +1,384 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2013-05-15 14:30:48 +0200 (mer., 15 mai 2013) $
+// $LastChangedRevision: 2392 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.macroproto.importer;
+
+import groovy.transform.Field
+
+import javax.xml.stream.*
+
+import org.kohsuke.args4j.*
+import org.txm.*
+import org.txm.importer.ApplyXsl2
+import org.txm.importer.ValidateXml
+import org.txm.importer.xml.compiler
+import org.txm.importer.xml.importer
+import org.txm.importer.xml.pager
+import org.txm.metadatas.*
+import org.txm.objects.*
+import org.txm.rcpapplication.commands.*
+import org.txm.rcpapplication.commands.workspace.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.scripts.teitxm.*
+import org.txm.tokenizer.TokenizerClasses
+import org.txm.utils.*
+import org.txm.utils.i18n.*
+import org.txm.utils.xml.DomUtils
+import org.w3c.dom.Element
+
+String userDir = System.getProperty("user.home");
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="srcDirectory", usage="the directory containing the DOC/ODT/RTF files to convert", widget="Folder", required=true, def="srcDirectory")
+File srcDirectory
+
+@Field @Option(name="SpannedEdition", usage="edition xsl", widget="Boolean", required=false, def="true")
+Boolean SpannedEdition // call EditionUpdater
+// calls EditionUpdater : 
+// Defaults corpus=imported corpus, xslEdition:XTZ xsl, xslPages=XTZ xsl2, editionname=default, useTokenizedDirectory=false
+
+@Field @Option(name="synoptic", usage="syn", widget="Boolean", required=false, def="true")
+Boolean synoptic 
+// calls AddFacsAttribute + FacsEditionBuilder. 
+// Defaults AddFacsAttribute : imageDirectory=$SRC/img, outputDirectory=temp, element=pb, attribute=n, prefix=mediaPath
+// Default FacsEditionBuilder : editionName=facs, element=pb, attribute=facs
+ 
+@Field @Option(name="mediaPath", usage="absolute or relative path to images directory", widget="String", required=false, def="img")
+String mediaPath
+
+@Field @Option(name="facsEditionName", usage="facs edition name to create", widget="String", required=false, def="facs")
+String facsEditionName
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+if (!mediaPath.endsWith("/")) mediaPath += "/"
+
+println "Parameters: "
+println "	srcDirectory: $srcDirectory"
+println "	SpannedEdition: $SpannedEdition"
+println "	synoptic: $synoptic"
+println "	mediaPath: $mediaPath"
+println "	facsEditionName: $facsEditionName"
+
+def MONITOR;
+boolean debug = false;
+File importFile = new File(srcDirectory, "import.xml")
+if (!importFile.exists()) {
+	println "No import.xml file found in $srcDirectory"
+	println "Aborting."
+	return;
+}
+BaseParameters params = new BaseParameters(importFile);
+params.load()
+params.rootDir = srcDirectory.getAbsolutePath();
+TokenizerClasses.loadFromNode(params.getTokenizerElement(params.getCorpusElement()));
+if (params == null) { println "no parameters. Aborting"; return; }
+
+String corpusname = params.getCorpusName();
+Element corpusElem = params.corpora.get(corpusname);
+String basename = params.name;
+String rootDir = params.rootDir;
+String lang = corpusElem.getAttribute("lang");
+String model = lang
+String encoding = corpusElem.getAttribute("encoding");
+boolean annotate = "true" == corpusElem.getAttribute("annotate");
+String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")
+def xslParams = params.getXsltParams(corpusElem);
+
+File srcDir = new File(rootDir);
+File binDir = new File(Toolbox.getParam(Toolbox.USER_TXM_HOME), "corpora/"+basename);
+binDir.deleteDir();
+binDir.mkdirs();
+if (!binDir.exists()) {
+	println "Could not create binDir "+binDir
+	return;
+}
+
+File txmDir = new File(binDir, "txm/$corpusname");
+txmDir.deleteDir();
+txmDir.mkdirs();
+
+File propertyFile = new File(rootDir, "import.properties")//default
+Properties props = new Properties();
+String[] metadatasToKeep;
+
+String textSortAttribute = null;
+String paginationElement = "pb";
+boolean normalizeMetadata = false;
+String ignoredElements = null;
+boolean stopIfMalformed = false;
+
+println "Trying to read import properties file: "+propertyFile
+if (propertyFile.exists() && propertyFile.canRead()) {
+	InputStreamReader input = new InputStreamReader(new FileInputStream(propertyFile) , "UTF-8");
+	props.load(input);
+	input.close();
+	if(props.getProperty("sortmetadata") != null)
+		textSortAttribute = props.get("sortmetadata").toString();
+	if (props.getProperty("editionpage") != null)
+		paginationElement = props.get("editionpage").toString();
+	if (props.getProperty("normalizemetadata") != null)
+		normalizeMetadata = Boolean.parseBoolean(props.get("normalizemetadata").toString());
+	if (props.getProperty("ignoredelements") != null)
+		ignoredElements = props.get("ignoredelements").toString();
+	if (props.getProperty("stopifmalformed") != null)
+		stopIfMalformed = Boolean.parseBoolean(props.get("stopifmalformed").toString());
+
+	println "import properties: "
+	println " sort metadata: "+textSortAttribute
+	println " edition page tag: "+paginationElement
+	println " normalize attributes: "+normalizeMetadata
+	println " ignored elements: "+ignoredElements
+	println " stop if a XML source is malformed: "+stopIfMalformed
+}
+
+File allmetadatasfile = new File(srcDir, "metadata.csv");
+
+// Apply XSL
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(1, "APPLYING XSL")
+if (xsl != null && xslParams != null && xsl.trim().length() > 0) {
+	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))
+	// return; // error during process
+	srcDir = new File(binDir, "src");
+	println ""
+}
+
+String[] roots = [ System.getProperty("user.home")+"/TXM/scripts/macro/" ];
+GroovyScriptEngine gse = new GroovyScriptEngine(roots, monitor.getClass().getClassLoader());
+File jardir = new File(System.getProperty("user.home")+"/TXM/scripts/lib"); //$NON-NLS-1$
+if (jardir.exists() && jardir.isDirectory()) {
+	for (File f: jardir.listFiles(new FilenameFilter() {
+		public boolean accept(File dir, String name) {
+			return name.endsWith(".jar"); //$NON-NLS-1$
+		}
+	})) {
+		gse.getGroovyClassLoader().addURL(f.toURI().toURL());
+	}
+}
+// Call AddFacsAttribute - must be called before starting the importer step
+// Defaults AddFacsAttribute : imageDirectory=$SRC/img, outputDirectory=temp, element=pb, attribute=facs, prefix=mediaPath
+if (synoptic) {
+	println "-- Adding the $paginationElement@facs attributes"
+	File srcDirTemp = new File(binDir, "facs_src")
+	srcDirTemp.mkdir() // create a new src directory that contains the modified src files
+	Binding b = new Binding();
+	b.setVariable("args", ["monitor":monitor, "sourceDirectory":srcDir, "imageDirectory":new File(srcDir, "img"), "outputDirectory":srcDirTemp, "element":paginationElement, "attribute":"facs", "prefix":mediaPath]);
+	gse.run "org/txm/macro/edition/AddFacsAttributeMacro.groovy", b
+	println "switching srcDir to $srcDirTemp"
+	srcDir = srcDirTemp;
+}
+
+// copy xml+dtd files
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+List<File> srcfiles = srcDir.listFiles();
+if (srcfiles != null)
+for (int i = 0 ; i < srcfiles.size() ; i++) {// check XML format, and copy file into binDir
+	File f = srcfiles.get(i)
+	if (f.getName().equals("import.xml") || f.getName().equals("metadata.csv") || f.getName().endsWith(".properties")) {
+		srcfiles.remove(i);
+		i--;
+		continue;
+	}
+	if (ValidateXml.test(f)) {
+		FileCopy.copy(f, new File(txmDir, f.getName()));
+	} else {
+		println "Won't process file "+f;
+	}
+}
+
+if (txmDir.listFiles() == null) {
+	println "No txm file to process"
+	return;
+}
+
+//get metadatas values from CSV
+Metadatas metadatas; // text metadatas
+
+println "Trying to read metadatas from: "+allmetadatasfile
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (allmetadatasfile.exists()) {
+	File copy = new File(binDir, "metadata.csv")
+	if (!FileCopy.copy(allmetadatasfile, copy)) {
+		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
+		return;
+	}
+	metadatas = new Metadatas(copy, Toolbox.getParam(Toolbox.METADATA_ENCODING), Toolbox.getParam(Toolbox.METADATA_COLSEPARATOR), Toolbox.getParam(Toolbox.METADATA_TXTSEPARATOR), 1)
+} else {
+	println "no metadata file: "+allmetadatasfile
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(5, "IMPORTER")
+println "-- IMPORTER - Reading source files"
+def imp = new importer();
+imp.doValidation(true) // change this to not validate xml
+imp.doTokenize(true) // change this, to not tokenize xml
+imp.setStopIfMalformed(stopIfMalformed);
+if (!imp.run( srcDir, binDir, txmDir, basename, ignoredElements, lang)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "INJECTING METADATA")
+if (metadatas != null) {
+	println("-- INJECTING METADATA - "+metadatas.getHeadersList()+" in texts of directory "+new File(binDir,"txm"))
+	for (File infile : txmDir.listFiles()) {
+		print "."
+		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());
+		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {
+			outfile.delete();
+		} else {
+			if (!(infile.delete() && outfile.renameTo(infile))) println "Warning can't rename file "+outfile+" to "+infile
+			if (!infile.exists()) {
+				println "Error: could not replace $infile by $outfile"
+				return false;
+			}
+		}
+	}
+	println ""
+}
+List<File> files = txmDir.listFiles()
+if (files == null || files.size() == 0) {
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
+println "-- ANNOTATE - Running NLP tools"
+boolean annotationSuccess = false;
+if (annotate && new Annotate().run(binDir, txmDir, model+".par")) {
+	annotationSuccess = true;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "COMPILING")
+println "-- COMPILING - Building Search Engine indexes"
+def c = new compiler();
+if(debug) c.setDebug();
+//c.setCwbPath("~/TXM/cwb/bin");
+c.setOptions(textSortAttribute, normalizeMetadata);
+c.setAnnotationSuccess(annotationSuccess)
+c.setLang(lang);
+if (!c.run(binDir, txmDir, corpusname, null, srcfiles, metadatas)) {
+	println "import process stopped";
+	return;
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(25, "EDITION")
+println "-- EDITION - Building edition"
+new File(binDir,"HTML/$corpusname").deleteDir();
+new File(binDir,"HTML/$corpusname").mkdirs();
+File outdir = new File(binDir,"/HTML/$corpusname/default/");
+outdir.mkdirs();
+List<File> filelist = txmDir.listFiles();
+Collections.sort(filelist);
+def second = 0
+
+println "Paginating texts: "+filelist
+for (File txmFile : filelist) {
+	print "."
+	String txtname = txmFile.getName();
+	int i = txtname.lastIndexOf(".");
+	if(i > 0) txtname = txtname.substring(0, i);
+
+	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);
+	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);
+
+	Element text = params.addText(corpusElem, txtname, txmFile);
+
+	def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, 30000, basename, paginationElement);
+	Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");
+
+	for (i = 0 ; i < ed.getPageFiles().size();) {
+		File f = ed.getPageFiles().get(i);
+		String wordid = ed.getIdx().get(i);
+		params.addPage(edition, ""+(++i), wordid);
+	}
+}
+
+if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
+if (MONITOR != null) MONITOR.worked(20, "FINALIZING")
+File paramFile = new File(binDir, "import.xml");
+DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;
+
+// Call FacsEditionBuilder
+// Default FacsEditionBuilder : editionName=facs, element=pb, attribute=facs
+if (synoptic) {
+	println "-- Building the '$facsEditionName' image edition..."
+	b = new Binding();
+	b.setVariable("args", ["monitor":monitor,"params":params, "binDirectory":binDir, "txmDirectory":txmDir,"corpus":srcDirectory, "editionName":facsEditionName, "tag":paginationElement, "attribute":"facs"]);
+	gse.run "org/txm/macro/edition/FacsEditionBuilderMacro.groovy", b
+	
+	//copy images into the binary corpus
+	if (mediaPath.length() > 0 && !mediaPath.startsWith("http")) {
+		File binImageDirectory = new File(binDir,"HTML/$basename/$facsEditionName/$mediaPath")
+		println "copying images into binary corpus: $binImageDirectory ..."
+		FileCopy.copyFiles(new File(srcDirectory, "img"), binImageDirectory)
+		println "copy done"
+	} else {
+		println "Note: the $facsEditionName edition images are not embedded in the binary corpus"
+	}
+}
+
+// Call EditionUpdater
+// corpus=imported corpus, xslEdition:XTZ xsl, xslPages=XTZ xsl2, editionname=default, useTokenizedDirectory=false
+if (SpannedEdition) {
+	println "-- Building Spanned edition..."
+	
+	Binding b = new Binding();
+	b.setVariable("args", ["monitor":monitor,"params":params, "binDirectory":binDir, "txmDirectory":txmDir,"corpus":srcDirectory, "xslEdition":"txm-edition-xtz.xsl", "xslPages":"txm-edition-page-split.xsl", "editionname":"default", "useTokenizedDirectory":false]);
+	gse.run "org/txm/macro/edition/EditionUpdaterMacro.groovy", b
+}
+// // create the ".txm" file
+// try {
+// 	File exportzip = new File(binDir.getParent(), binDir.getName()+".txm")
+// 	println "creating the $exportzip binary corpus file..."
+// 	Zip.compress(binDir, exportzip);
+// 	println "done."
+// } catch (IOException e) {
+// 	println "Error during binary file creation: $e"
+// }
+
+// load corpus
+println "Loading corpus..."
+try {LoadBinaryCorpus.loadBase(binDir)} catch(Exception e777){AddBases.loadBase(binDir, monitor)} // LoadBinaryCorpus does not exist if TXM version is < 0.7.7
+
+Toolbox.restartWorkspace();
+Toolbox.restartSearchEngine();
+
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {
+		println "Reloading corpora view..."
+		RestartTXM.reloadViews();
+		println "import done."
+	}
+});
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/FindMultiLineRegExpMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/FindMultiLineRegExpMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/FindMultiLineRegExpMacro.groovy	(revision 321)
@@ -0,0 +1,80 @@
+package org.txm.macroproto
+// Copyright © 2015 - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author sheiden
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 11:28:42 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2378 $
+// $LastChangedBy: sheiden $
+//
+
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+
+import groovy.transform.Field
+
+import java.nio.charset.Charset
+
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.utils.*
+
+import javax.xml.stream.*
+
+import java.net.URL
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// PARAMETERS
+
+@Field @Option(name="inputDirectory", usage="TXT directory", widget="Folder", required=false, def="")
+File inputDirectory;
+@Field @Option(name="regexp", usage="Regular expression to match", widget="String", required=false, def="")
+String regexp;
+@Field @Option(name="encoding", usage="File encoding", widget="String", required=false, def="UTF-8")
+String encoding;
+
+// PARAMETERS
+
+if (inputDirectory == null) {
+	if (!ParametersDialog.open(this)) return;
+}
+
+// SANITY CHECK
+
+if (inputDirectory==null || regexp==null) { println "** FindMultiLineRegExpMacro: the input directory and the regular expression must be specified."; return}
+
+if (!inputDirectory.exists()) { println "** FindMultiLineRegExpMacro: impossible to access the '$inputDirectory' input directory."; return}
+
+// MAIN BODY
+
+def p = /$regexp/
+
+println "-- looking for: $regexp"
+
+for (File inputfile : inputDirectory.listFiles()) {
+
+	if (inputfile.isDirectory()) continue // ignore
+
+	def matcher = inputfile.getText() =~ p
+	
+	println "\n-- file:"+inputfile.getName() + " "+matcher.size() + " match" +((matcher.size() > 1)?"s":"")
+
+	if (matcher.size() > 0) {
+		matcher.each { println it }
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/RunImportMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/RunImportMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/RunImportMacro.groovy	(revision 321)
@@ -0,0 +1,85 @@
+package org.txm.macroproto
+import groovy.transform.Field
+
+import org.kohsuke.args4j.*
+import org.txm.Toolbox
+import org.txm.objects.BaseParameters
+import org.txm.rcpapplication.commands.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+// PARAMETERS
+@Field @Option(name="inputDirectory", usage="Dossier qui contient les fichiers à importer", widget="Folder", required=true, def='')
+File inputDirectory
+
+@Field @Option(name="importName", usage="Nom du module d'import", widget="String", required=true, def='TXT+CSV | XML/w+CSV | XML-Factiva | XML-TEI-BFM | XML-TEI-Frantext | XML-TEI-TXM | XML-Transcriber+CSV | XML-TMX | CNR+CSV | Hyperbase | Alceste | CWB | DOC')
+// choose between : 
+def importName
+
+if (!ParametersDialog.open(this)) return;
+// END OF PARAMETERS
+// TXT+CSV | XML/w+CSV | XML-Factiva | XML-TEI-BFM | XML-TEI-Frantext | XML-TEI-TXM | XML-Transcriber+CSV | XML-TMX | CNR+CSV | Hyperbase | Alceste | CWB
+
+// fix possible miss write
+importName = importName.toUpperCase()
+importName = importName.replaceAll(" ", "")
+importName = importName.trim()
+importName = importName.replaceAll("\\+", "")
+importName = importName.replaceAll("-", "")
+importName = importName.replaceAll("/", "")
+
+def scripts = [:]; // 12
+scripts["TXTCSV"] = "quickLoader.groovy"
+scripts["XMLWCSV"] = "xmlLoader.groovy"
+scripts["XMLFACTIVA"] = "factivaLoader.groovy"
+scripts["XMLTEIBFM"] = "bfmLoader.groovy"
+scripts["XMLTEIFrantex"] = "frantextLoader.groovy"
+scripts["XMLTEITXM"] = "xmltxmLoader.groovy"
+scripts["XMLTranscriberCSV"] = "transcriberLoader.groovy"
+scripts["XMLTMX"] = "tmxLoader.groovy"
+scripts["CNRCSV"] = "discoursLoader.groovy"
+scripts["HYPERBASE"] = "hyperbaseLoader.groovy"
+scripts["ALCESTE"] = "alcesteLoader.groovy"
+scripts["CWB"] = "wtcLoader.groovy"
+scripts["DOC"] = "docLoader.groovy"
+
+def scriptName = scripts.get(importName)
+if (scriptName == null) {
+	println "Error: no import script for name $importName"
+	return
+}
+
+String txmhome = Toolbox.getParam(Toolbox.USER_TXM_HOME);
+File scriptsdir = new File(txmhome, "scripts/import")
+File script = new File(scriptsdir, scriptName)
+
+if (!script.exists()) {
+	println "Error: no script found for import name ${importName}: $script"
+	return
+}
+
+if (inputDirectory.exists()) {
+	File paramFile = new File(inputDirectory, "import.xml")
+	if (!paramFile.exists()) { // create, if it does not exists
+		BaseParameters.createEmptyParams(paramFile, inputDirectory.getName())
+	}
+
+	final BaseParameters params = new BaseParameters(paramFile);
+	params.load();
+	params.rootDir = inputDirectory.getAbsolutePath(); // fix
+	params.saveToElement();
+
+	if (!params.save()) {
+		System.out.println("ERROR: could not save import configuration: "+ params.paramFile);
+		return;
+	}
+
+	monitor.syncExec(new Runnable() {
+		public void run() {
+			ExecuteImportScript.executeScript(script.getAbsolutePath(), params);
+		}
+	});
+} else {
+	System.out.println("Warning: Missing inputDirectory directory: "+ inputDirectory);
+}
+
+	
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreatePartitionByMetadataMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreatePartitionByMetadataMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreatePartitionByMetadataMacro.groovy	(revision 321)
@@ -0,0 +1,62 @@
+// STANDARD DECLARATIONS
+package org.txm.macroproto.partition
+
+import org.kohsuke.args4j.*
+
+import groovy.transform.Field
+
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.utils.logger.Log;
+import org.txm.rcpapplication.views.*
+import org.txm.stat.engine.r.*
+import org.txm.Toolbox
+
+//BEGINNING OF PARAMETERS
+def corpus = corpusViewSelection
+if (!(corpus instanceof Corpus)) {
+	println "Error: this macro should be run with a Corpus selected"
+	return
+}
+
+@Field @Option(name="values_file", usage="txt file", widget="File", required=true, def="C:\\Users\\Fanny\\Desktop\\values.txt")
+def values_file
+
+@Field @Option(name="property", usage="the structural Unit properties list separated with commas", widget="String", required=true, def="emotionalfeelingthe911")
+def property = "emotionalfeelingthe911"
+
+@Field @Option(name="expand_struct", usage="expand_struct", widget="String", required=true, def="question")
+def expand_struct = "question"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+println "Partition du corpus '$corpus' à partir de la métadonnée '$property' "
+
+def domain = []
+for (def s :  values_file.getText("UTF-8").split("\n")) domain << s.trim()
+
+def uniq_domain = new HashSet(domain).sort() 
+println "domain size "+domain.size()
+println "uniq domain size "+uniq_domain.size()
+
+def names = []
+def queries = []
+
+for (def d : uniq_domain) {
+	queries <<  "[_.text_"+property+"=\"$d\"] expand to $expand_struct"
+	names << d	
+}
+
+corpus.createPartition(property+"_non_balanced", queries, names);
+
+println queries
+println names
+
+//display the graphic
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {	
+	org.txm.rcpapplication.commands.RestartTXM.reloadViews();
+	}
+});
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreateCrossedPartitionByMetadataMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreateCrossedPartitionByMetadataMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreateCrossedPartitionByMetadataMacro.groovy	(revision 321)
@@ -0,0 +1,67 @@
+// STANDARD DECLARATIONS
+package org.txm.macroproto.partition
+
+import org.kohsuke.args4j.*
+
+import groovy.transform.Field
+
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.utils.logger.Log;
+import org.txm.rcpapplication.views.*
+import org.txm.stat.engine.r.*
+import org.txm.Toolbox
+
+//BEGINNING OF PARAMETERS
+def corpus = corpusViewSelection
+if (!(corpus instanceof Corpus)) {
+	println "Error: this macro should be run with a Corpus selected"
+	return
+}
+
+def struct_property2 = "survey_type"
+def struct_property_values = ["t1","t2","t3","t4"]
+
+@Field @Option(name="values_file", usage="txt file", widget="File", required=true, def="C:\\Users\\Fanny\\Desktop\\values.txt")
+def values_file
+
+@Field @Option(name="property", usage="the structural Unit properties list separated with commas", widget="String", required=true, def="emotionalfeelingthe911")
+def property = "emotionalfeelingthe911"
+
+
+
+@Field @Option(name="expand_struct", usage="expand_struct", widget="String", required=true, def="question")
+def expand_struct = "question"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+println "Partition du corpus '$corpus' à partir de la métadonnée '$property' "
+
+def domain = []
+for (def s :  values_file.getText("UTF-8").split("\n")) domain << s.trim()
+
+def uniq_domain = new HashSet(domain).sort() 
+println "domain size "+domain.size()
+println "uniq domain size "+uniq_domain.size()
+
+def names = []
+def queries = []
+for (def d2 : struct_property_values) {
+	for (def d : uniq_domain) {
+		queries <<  "[_."+struct_property2+"=\"$d2\" & _.text_"+property+"=\"$d\"] expand to $expand_struct"
+		names << d2+"_"+d	
+	}
+}
+corpus.createPartition(struct_property2+" x "+property+"_non_balanced", queries, names);
+
+println queries
+println names
+
+//display the graphic
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {	
+	org.txm.rcpapplication.commands.RestartTXM.reloadViews();
+	}
+});
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreatePartitionByNumericMetadataMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreatePartitionByNumericMetadataMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/partition/CreatePartitionByNumericMetadataMacro.groovy	(revision 321)
@@ -0,0 +1,127 @@
+// STANDARD DECLARATIONS
+package org.txm.macroproto.partition
+
+import groovy.transform.Field
+
+import org.kohsuke.args4j.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.rcpapplication.views.*
+import org.txm.searchengine.cqp.corpus.*
+
+//BEGINNING OF PARAMETERS
+def corpus = corpusViewSelection
+if (!(corpus instanceof Corpus)) {
+	println "Error: this macro should be run with a Corpus selected"
+	return
+}
+
+@Field @Option(name="property", usage="the structural Unit properties list separated with commas", widget="String", required=true, def="sadnesst1")
+def property = "sadnesst1"
+
+@Field @Option(name="values_file", usage="txt file", widget="File", required=true, def="C:\\Users\\Fanny\\Desktop\\values_num.txt")
+def values_file
+
+@Field @Option(name="balance", usage="the structural Unit properties list separated with commas", widget="Boolean", required=true, def="true")
+def balance = false
+
+@Field @Option(name="expand_struct", usage="expand_struct", widget="String", required=true, def="question")
+def expand_struct = "question"
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+def property_values = values_file.getText("UTF-8").replaceAll("\n", ",").replaceAll("\r", "")
+
+println "Balance de la métadonnée '$property' du corpus '$corpus'"
+
+def domain = []
+for (def s : property_values.split(",")) {
+	s = s.trim()
+	try {
+		domain << Integer.parseInt(s)
+	} catch(Exception e) {
+		domain << Float.parseFloat(s)
+	}
+}
+def uniq_domain = new HashSet(domain).sort() 
+println "domain size "+domain.size()
+
+def names = []
+def queries = []
+def p = 2
+String values = ""
+
+if (!balance) {
+
+for (def d : uniq_domain) {
+	if (d < p) {
+		values += "|$d"
+	} else {
+		if (values.startsWith("|")) values = values.substring(1)
+		queries <<  "[_.text_"+property+"=\"$values\"] expand to $expand_struct"
+		names << values
+		values = d
+		p++
+	}
+}
+queries <<  "[_.text_"+property+"=\"$values\"] expand to $expand_struct"
+names << values
+
+
+corpus.createPartition(property+"_non_balanced", queries, names);
+
+} else {
+//String v = domain.toString()
+//v = "c("+v.substring(1, v.length() -1) + ")"
+String v  = "c($property_values)"
+println "=========================="
+//RWorkspace.getRWorkspaceInstance().addVectorToWorkspace("domain", domain.toArray(new double[domain.size()]))
+def res = RWorkspace.getRWorkspaceInstance().eval("quantile($v)")
+def ranges = res.asDoubles()
+println "quantiles : $ranges"
+values = ""
+p = ranges[1]
+i = 1
+names = []
+queries = []
+for (def d : uniq_domain) {
+	//println "d=$d, p=$p"
+	if (d < p) {
+		values += "|$d"
+	} else {
+		if (values.startsWith("|")) values = values.substring(1)
+		println """[_.text_${property}="$values"]"""
+		queries <<  "[_.text_"+property+"=\"$values\"] expand to $expand_struct"
+		names << values
+		values = d
+		i++
+		p = ranges[i]
+	}
+}
+
+queries <<  "[_.text_"+property+"=\"$values\"] expand to $expand_struct"
+names << values
+
+corpus.createPartition(property+"_balanced", queries, names);
+}
+
+
+println queries
+println names
+
+
+//display the graphic
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {	
+	org.txm.rcpapplication.commands.RestartTXM.reloadViews();
+	}
+});
+
+/*
+int[] counts = new int[51]
+for (def d : domain) {
+	counts[(int)(10*d)]++
+}
+println counts
+*/
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/GetXPathMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/GetXPathMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/GetXPathMacro.groovy	(revision 321)
@@ -0,0 +1,84 @@
+package org.txm.macroproto
+// Copyright © 2015 - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author sheiden
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 11:28:42 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2378 $
+// $LastChangedBy: sheiden $
+//
+
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+import org.txm.importer.*
+
+// BEGINNING OF PARAMETERS
+
+// Declare each parameter here
+// (available widget types: Query, File, Folder, String, Text, Boolean, Integer, Float and Date)
+
+@Field @Option(name="srcDirectory", usage="XML source directory", widget="Folder", required=false, def="")
+def srcDirectory
+
+@Field @Option(name="srcFile", usage="XML source file", widget="File", required=false, def="")
+def srcFile
+
+@Field @Option(name="xPath", usage="XPath expression", widget="String", required=true, def="//tei:title/text()")
+def xPath
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+// END OF PARAMETERS
+
+if (srcDirectory==null && srcFile==null) { println "** GetXPathMacro: at least a source file or a source directory must be specified."; return}
+
+if (srcDirectory!=null && srcDirectory.exists()) {
+
+	def files = srcDirectory.listFiles()
+	if (files == null || files.size() == 0) {
+		println "No files in $srcDirectory"
+		return;
+	}
+	files.sort()
+
+	for (File xmlFile : files) {
+		String name = xmlFile.getName();
+		if (!name.endsWith(".xml")) continue;
+	
+		println "-- file '$xmlFile':"
+		
+		XPathResult xpathProcessor = new XPathResult(xmlFile)
+		def value = xpathProcessor.getXpathResponse(xPath)
+		if (value != null) { println(value) }
+}
+
+} else if (srcFile!=null && srcFile.exists()) {
+	def xmlFile = srcFile
+	String name = xmlFile.getName();
+	if (!name.endsWith(".xml")) { println "** GetXPathMacro: source file extension must be '.xml'."; return}
+		
+	println "-- file '$xmlFile':"
+
+	XPathResult xpathProcessor = new XPathResult(xmlFile)
+	def value = xpathProcessor.getXpathResponse(xPath)
+	if (value != null) { println(value) } else { println "No result." }
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/XMLText2MetadataCSVMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/XMLText2MetadataCSVMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/XMLText2MetadataCSVMacro.groovy	(revision 321)
@@ -0,0 +1,83 @@
+package org.txm.macroproto
+
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+import org.txm.importer.*
+
+// BEGINNING OF PARAMETERS
+
+// Declare each parameter here
+// (available widget types: Query, File, Folder, String, Text, Boolean, Integer, Float and Date)
+
+@Field @Option(name="xpathFile", usage="'.properties' file", widget="File", required=true, def="metadata.properties")
+def xpathFile
+
+@Field @Option(name="srcDirectory", usage="XML source directory", widget="Folder", required=false, def="srcDirectory")
+def srcDirectory
+
+@Field @Option(name="metadataFile", usage="'.properties' file", widget="File", required=false, def="metadata.csv")
+def metadataFile
+
+@Field @Option(name="columnSeparator", usage="", widget="String", required=false, def=",")
+def columnSeparator
+
+@Field @Option(name="txtSeparator", usage="", widget="String", required=false, def="\"")
+def txtSeparator
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+// END OF PARAMETERS
+
+if (!srcDirectory.exists()) { println "$srcDirectory does not exist"; return}
+if (!xpathFile.exists()) { println "$xpathFile does not exist"; return}
+
+// get Xpath queries
+def xpaths = [:]
+for (String line : xpathFile.readLines("UTF-8")) {
+	def split = line.split("=", 2);
+	if (split.size() == 2) {
+		xpaths[split[0]] = split[1]
+	}
+}
+println "XPATH queries: $xpaths"
+
+if (metadataFile == null)
+	metadataFile = new File(srcDirectory, "metadata.csv")
+
+def writer = metadataFile.newWriter("UTF-8")
+println "Creating file: $metadataFile"
+writer.print "id"
+for (String p : xpaths.keySet()) {writer.print columnSeparator+p}
+writer.println ""
+
+def files = srcDirectory.listFiles()
+if (files == null || files.size() == 0) {
+	println "No files in $srcDirectory"
+	return;
+}
+files.sort()
+
+for (File xmlFile : files) {
+	String name = xmlFile.getName();
+	if (!name.endsWith(".xml")) continue;
+	
+	println "Processing file $xmlFile..."
+	name = name.substring(0, name.indexOf(".xml"))
+	
+	writer.print "$name"
+	
+	XPathResult xpathProcessor = new XPathResult(xmlFile)
+	for (String p : xpaths.keySet()) {
+		def value = xpathProcessor.getXpathResponse(xpaths[p]);
+		if (value == null) value = "N/A"
+		writer.print columnSeparator+txtSeparator+value.replaceAll(txtSeparator,txtSeparator+txtSeparator)+txtSeparator
+	}
+	writer.println ""
+}
+
+writer.close()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/edition/XTZEditionBuilderMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/edition/XTZEditionBuilderMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/edition/XTZEditionBuilderMacro.groovy	(revision 321)
@@ -0,0 +1,160 @@
+package org.txm.macroproto.edition
+
+// STANDARD DECLARATIONS
+
+import groovy.transform.Field
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.transform.dom.DOMSource
+import javax.xml.transform.stream.StreamResult
+
+import org.kohsuke.args4j.*
+import org.txm.Toolbox
+import org.txm.macro.edition.BuildFacsEditions
+import org.txm.macro.edition.RemoveTag
+import org.txm.objects.*
+import org.txm.rcpapplication.commands.*
+import org.txm.rcpapplication.swt.widget.parameters.*
+import org.txm.searchengine.cqp.corpus.*
+import org.w3c.dom.*
+
+println "Corpus view selection is: "+corpusViewSelection
+if (!(corpusViewSelection instanceof MainCorpus)) {
+	println "This marcro works with a MainCorpus selection. Aborting"
+	return;
+}
+def corpus = corpusViewSelection
+debug = false
+def editionName = corpus.getDefaultEdition()
+
+// PARAMETERS
+@Field @Option(name="wordsPerPage", usage="Number of words per page", widget="Integer", required=true, def="500")
+int wordsPerPage
+if (!ParametersDialog.open(this)) return;
+// END OF PARAMETERS
+
+File binDirectory = corpus.getBase().getBaseDirectory()
+File txmDirectory = new File(binDirectory, "txm/"+corpus.getName().toUpperCase())
+
+File HTMLDirectory = new File(binDirectory, "HTML")
+File newEditionDirectory = new File(HTMLDirectory, corpus.getName().toUpperCase()+"/"+editionName)
+
+BaseParameters parameters = corpus.getBase().params
+
+if (!HTMLDirectory.exists()) {
+	println "ERROR: can't find this corpus 'HTML' directory: $HTMLDirectory. Aborting"
+	return false;
+}
+
+File workDirectory = txmDirectory
+if (!workDirectory.exists()) {
+	println "XML-TXM directory ($txmDirectory) not found. Using XML tokenized directory instead: "+tokenizedDirectory
+	workDirectory = tokenizedDirectory
+}
+if (!workDirectory.exists()) {
+	println "XML tokenized directory not found: "+tokenizedDirectory
+	println "Aborting."
+	return false
+}
+
+println "Work directory=$workDirectory"
+
+//0- clean previous edition if any : html files, import.xml
+if (newEditionDirectory.exists()) {
+	println "** Old version of $editionName edition found."
+	println " removing the 'edition' reference from the corpus configuration."
+	File tempParam = new File(binDirectory, "import.xml.cpy")
+	RemoveTag rt = new RemoveTag(
+			parameters.root.getOwnerDocument(), // will be updated
+			null, // don't create a new import.xml
+			"//edition[@name='$editionName']"
+			)
+	println " delete $newEditionDirectory"
+	newEditionDirectory.deleteDir()
+	
+	printDOM(parameters.root.getOwnerDocument())
+}
+
+//2- fix import.xml
+println "** Updating corpus configuration..."
+// for edition list
+def corpusElem = parameters.getCorpusElement()
+parameters.addEditionDefinition(corpusElem, editionName, "groovy", "FacsEditionBuilderMacro");
+
+//1- create HTML files
+println "** Building new edition HTML files..."
+
+println " Creating edition $editionName directory: $newEditionDirectory"
+newEditionDirectory.mkdir()
+for (def xmlFile : workDirectory.listFiles()) {
+	if (xmlFile.isHidden() || xmlFile.isDirectory()) continue // ignore
+	String txtname = xmlFile.getName()
+	int idx = txtname.indexOf(".")
+	if (idx > 0) txtname = txtname.substring(0,idx)
+
+	println " Build HTML pages of text=$txtname"
+	BuildFacsEditions builder = new BuildFacsEditions(xmlFile, newEditionDirectory, parameters.name, txtname, wordsPerPage);
+	def newPages = builder.process()
+	if (newPages == null || newPages.size() == 0) {
+		println "WARNING: no edition files created with $xmlFile"
+	} 
+		
+	println " Build edition references in corpus configuration"
+	
+	Element textElem = corpus.getText(txtname).getSelfElement()
+	Element editionElem = parameters.addEdition(textElem, editionName, newEditionDirectory.getAbsolutePath(), "html");
+	//println "$textElem $editionElem"
+	for (def pagedef : newPages) {
+		parameters.addPage(editionElem, pagedef[0], pagedef[1]);
+	}
+}
+
+	printDOM(parameters.root.getOwnerDocument())
+
+//3- Save and reload the corpus
+println " Saving corpus configuration..."
+File paramFile = new File(binDirectory, "import.xml");
+DomUtils.save(parameters.root.getOwnerDocument(), paramFile);
+
+
+//4- Reload Corpora
+Toolbox.restart();
+monitor.syncExec(new Runnable() {
+			public void run() {
+				println "RELOAD VIEWS"
+				RestartTXM.reloadViews();
+				println "RELOAD VIEWS2"
+			}
+		});
+
+//5- Done
+println "New edition created. Be sure to copy images files in $newEditionDirectory"
+
+printDOM(parameters.root.getOwnerDocument())
+
+def printDOM(def doc) {
+	if (!debug) return;
+	try {
+		// Création de la source DOM
+		Source source = new DOMSource(doc);
+		
+		// Création du fichier de sortie
+		StreamResult resultat = new StreamResult(new PrintWriter(System.out));
+		
+		// Configuration du transformer
+		TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+		Transformer transformer = fabrique.newTransformer();
+		transformer.setOutputProperty(OutputKeys.METHOD, "xml"); //$NON-NLS-1$
+		transformer.setOutputProperty(OutputKeys.INDENT, "yes");  //$NON-NLS-1$
+		transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");  //$NON-NLS-1$
+		
+		// Transformation
+		transformer.transform(source, resultat);
+		// writer.close();
+		return true;
+	} catch (Exception e) {
+		e.printStackTrace();
+		return false;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/CollatorTesterMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/CollatorTesterMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/CollatorTesterMacro.groovy	(revision 321)
@@ -0,0 +1,37 @@
+package org.txm.macroproto;
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+import java.text.Collator
+
+// BEGINNING OF PARAMETERS
+@Field @Option(name="string", usage="an example string", widget="String", required=false, def="δὲ ξυνεχὴς εἶχε εἶχεν βληχρός ἐπέβαλε ἔλαβε οὐ ἐπέλαβε ἐπεγίνετο")
+def string
+
+@Field @Option(name="localeName", usage="an example string", widget="String", required=false, def="el")
+def localeName
+
+@Field @Option(name="strengh", usage="0 1 or 2 for PRIMARY, SECONDARY and TERTIARY", widget="Integer", required=false, def="2")
+def strengh
+
+@Field @Option(name="decomposition", usage="0 1 or 2 for NO_DECOMPOSITION, CANONICAL_DECOMPOSITION, FULL_DECOMPOSITION", widget="Integer", required=false, def="2")
+def decomposition
+
+if (!ParametersDialog.open(this)) return;
+// END OF PARAMETERS
+
+Locale locale = new Locale(localeName)
+
+def collator = Collator.getInstance(locale)
+
+if (strengh < 0) strengh = 0
+if (strengh > 2) strengh = 2
+collator.setStrength(strengh)
+
+if (decomposition < 0) decomposition = 0
+if (decomposition > 2) decomposition = 2
+collator.setDecomposition(decomposition)
+
+println "RESULT"
+Collections.sort(string.tokenize(), collator);
+println tokenize
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/SelectSpecifMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/SelectSpecifMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/SelectSpecifMacro.groovy	(revision 321)
@@ -0,0 +1,94 @@
+package org.txm.macroproto;
+import org.txm.Toolbox
+import org.txm.functions.ca.*
+import org.txm.searchengine.cqp.clientExceptions.*
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.stat.*
+import org.txm.stat.data.*
+import org.txm.stat.engine.r.*
+import org.txm.functions.specificities.*
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+//TODO temporary uncommented
+//if (!(corpusViewSelection instanceof SpecificitesResult)) {
+//	println "Selection if not a Specificites Result"
+//	return;
+//}
+
+@Field @Option(name="seuil", usage="score limit", widget="Integer", required=false, def="2.0")
+def seuil;// = 1.0
+
+@Field @Option(name="NombreDeMots", usage="Use the row sums : y/n", widget="Integer", required=false, def="10")
+def NombreDeMots;
+
+// END OF PARAMETERS
+if (!ParametersDialog.open(this)) return;
+
+if (NombreDeMots <= 0) {
+	println "Error: le nombre de mots est insuffisant."
+	return
+}
+println "seuil $seuil"
+println "NombreDeMots $NombreDeMots"
+
+
+def specif = corpusViewSelection
+def indices = specif.getSpecificitesIndex()
+def rownames = specif.getTypeNames();
+def colnames = specif.getPartShortNames() ;
+
+def result = new HashSet<String>();
+
+// for each specif column find the N first indices
+for (int i = 0; i < colnames.size() ; i++) {
+	
+	// get indices
+	def idx = []
+	for (int i2 = 0 ; i2 < rownames.length ; i2++) {
+		idx << [indices[i2][i], rownames[i2]];
+	}
+	
+	// sort indices
+//	idx = idx.sort(true, new Comparator() {
+//		int compare(def o1, def o2) {
+//			if (o1 == o2) return 0;
+//			return o1[0] - o2[0]
+//		}
+//	});
+//	
+//	idx = idx.reverse();
+	
+	def idxsorted = [];
+	for (def o : idx) {
+		//if (idxsorted.size() > 0)
+		for (int i4 = 0 ; i4 < idxsorted.size() ; i4++) {
+			if (idxsorted[i4][0] < o[0]) {
+				idxsorted.add(i4, o)
+				break;
+			}
+		}
+		idxsorted << o
+	}
+	idx = idxsorted
+	
+	//println idx
+	// get N first indices
+	int N = 0
+	for (def i3 : idx) {
+		result.add(i3[1]) // add index in result
+		N++
+		if (N >= NombreDeMots) break;
+		if (i3[0] < seuil) break;
+	}
+	println "Selection de $N dans "+colnames[i]
+}
+
+println "Fin de la sélection des spécificités : "+result.size()
+StringBuffer buffer = new StringBuffer();
+for (def s : result.sort()) {
+	buffer.append("|"+s.replace(".", "\\.").replace("|", "\\|"))
+}
+println buffer.toString().substring(1)
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/txt/CharListDirectoryMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/txt/CharListDirectoryMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/txt/CharListDirectoryMacro.groovy	(revision 321)
@@ -0,0 +1,55 @@
+package org.txm.macroproto.txt;
+// STANDARD DECLARATIONS
+
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+// BEGINNING OF PARAMETERS
+
+// Declare each parameter here
+// (available widget types: Query, File, Folder, String, Text, Boolean, Integer, Float and Date)
+
+@Field @Option(name="inputFile", usage="The file to read", widget="File", required=true, def="C:/Temp/foo.txt")
+def inputFile
+
+@Field @Option(name="inputDir", usage="The file to read", widget="Folder", required=false, def="")
+def inputDir
+
+
+// Open the parameters input dialog box
+if (!ParametersDialog.open(this)) return;
+
+def dic = [:]
+
+// END OF PARAMETERS
+if (inputDir != null && inputDir.exists()) {
+if (inputDir.listFiles() == null) {println "no files..."; return;}
+	println "processing "+inputDir.listFiles().size()+" files"
+	for (def f : inputDir.listFiles()) {
+		if (f.isDirectory()) continue;
+		print "."
+		count(f, dic)
+	}
+	println ""
+} else {
+	count(inputFile, dic)
+}
+int total = 0
+for (def e : dic.sort()) {
+	println(sprintf("%c\t%d", [e.getKey(), e.getValue()]))
+	total += e.getValue()
+}
+println "TOTAL\t$total"
+
+def count(def f, def dic) {
+	String s = f.getText()
+	for (char c : s) {
+		if (c == '\r' || c == '\n' || c == '\t' || c == ' ' || c == ' ') continue;
+		if (!dic.containsKey(c)) { dic[c] = 0 }
+		dic[c]++
+	}
+
+	return dic
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/txt/MultiLineSearchReplaceInDirectoryMacro.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/txt/MultiLineSearchReplaceInDirectoryMacro.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/macroproto/txt/MultiLineSearchReplaceInDirectoryMacro.groovy	(revision 321)
@@ -0,0 +1,87 @@
+package org.txm.macroproto.txt
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author sheiden
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 11:28:42 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2378 $
+// $LastChangedBy: sheiden $
+//
+
+// STANDARD DECLARATIONS
+
+// imports
+import org.kohsuke.args4j.*
+import groovy.transform.Field
+import org.txm.rcpapplication.swt.widget.parameters.*
+
+// PARAMETERS
+
+// **change this parameter**
+@Field @Option(name="inputDirectory", usage="Dossier contenant les fichiers à modifier", widget="Folder", required=true, def='/home')
+		inputDirectory = new File(System.getProperty("user.home"), "Bureau/voeux")
+
+// **change this parameter**
+@Field @Option(name="extension", usage="extension des fichiers à modifier (expression régulière)", widget="String", required=true, def='\\.txt')
+		extension = "\\.txt"
+
+// **change this parameter**
+@Field @Option(name="find", usage="Expression régulière à chercher", widget="String", required=true, def='’')
+		find = "’"
+
+// **change this parameter**
+@Field @Option(name="replaceWith", usage="Chaîne de remplacement", widget="String", required=false, def='\'')
+		replaceWith = "'"
+
+// **change this parameter**
+@Field @Option(name="encoding", usage="Encodage des caractères des fichiers", widget="String", required=true, def='UTF-8')
+		encoding = "utf-8"
+
+// PARAMETERS DIALOG
+
+if (!ParametersDialog.open(this)) return;
+
+// SANITY CHECK
+
+if (inputDirectory==null || find==null || replaceWith==null) { println "** MultiLineSearchReplaceInDirectoryMacro: the input directory, the regular expression to search and replacement must be specified."; return}
+
+if (!inputDirectory.exists()) { println "** MultiLineSearchReplaceInDirectoryMacro: impossible to access the '$inputDirectory' input directory."; return}
+
+// MAIN BODY
+
+println "-- working in $inputDirectory directory with files of '$extension' extension"
+println "-- replacing '$find' with '$replaceWith'"
+
+// find = /date="([0-9]+)-([0-9]+-[0-9]+)"/
+// **change this parameter**
+// replaceWith = 'date="$1-$2" year="$1"'
+// **change this parameter** (warning: '$1', '$2'... can be interpreted by Groovy in "..." strings)
+//
+// RECETTE
+// <tei:pb [^>]+>\n   <tei:p>[0-9]+</tei:p>\n   <tei:p>[^ ]+ 18[5-6][0-9]\.</tei:p>\n
+
+def p = /$find/
+
+inputDirectory.eachFileMatch(~/.*$extension/) { file ->               // for each file matching extension
+	println "\n-- processing: "+file.getName()
+	def tmp = File.createTempFile("SearchReplaceInDirectoryTemp", ".tmp", file.getParentFile()) // create temporary file
+	tmp.withWriter(encoding) { writer ->
+		writer.print(file.getText(encoding).replaceAll(p, replaceWith))
+	}
+	file.delete()
+	tmp.renameTo(file)
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/svn/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/svn/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/svn/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains Groovy scripts used to manipulate SVN repositories</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/svn/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/XMLStatistics.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/XMLStatistics.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/XMLStatistics.groovy	(revision 321)
@@ -0,0 +1,207 @@
+package org.txm.scripts
+
+import java.text.Collator;
+import java.util.Locale;
+
+import org.txm.importer.StaxIdentityParser;
+
+class XMLStatistics extends StaxIdentityParser {
+
+	static String UNICATTRS = "id|src|name|href"
+	String path = "";
+	def paths = [:];
+
+	boolean usePaths=false, useAttributes=false, useAttributeValues=false;
+
+	public XMLStatistics(URL url) {
+		super(url);
+	}
+
+	public void setUsePaths(boolean b) {
+		this.usePaths = b
+	}
+
+	public void setUseAttributes(boolean b) {
+		this.useAttributes = b
+	}
+
+	public void setUseAttributeValues(boolean b) {
+		this.useAttributeValues = b;
+	}
+
+	public void processCharacters() {
+		//super.processCharacters()// don't write :)
+	}
+	
+	public void processEndElement() {
+		if (usePaths) {
+			path = path.substring(0, path.lastIndexOf("/"))
+		}
+		//super.processEndElement()// don't write :)
+	}
+
+	public void processStartElement() {
+		String localname = this.parser.getLocalName();
+		if (usePaths) {
+			path = path + "/$localname"
+		} else {
+			path = localname;
+		}
+
+		if (this.useAttributes) {
+			if (this.useAttributeValues) {
+				for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+					if (!parser.getAttributeLocalName(i).matches(UNICATTRS))
+						path += "@"+parser.getAttributeLocalName(i)+"="+parser.getAttributeValue(i)
+			} else {
+				for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+					path += "@"+parser.getAttributeLocalName(i)
+			}
+		}
+
+
+		plusOne()
+		//super.processStartElement() // don't write :)
+	}
+
+	public void process() {
+		File tmp = File.createTempFile("sdfs", "sdfsdf")
+		process(tmp)
+		tmp.delete();
+	}
+
+	private void plusOne() {
+		if (!paths.containsKey(path)) {
+			paths.put(path, 1)
+		} else {
+			paths.put(path, paths.get(path)+1)
+		}
+	}
+
+	public void printCounts(boolean sortByPath) {
+		def keys = new ArrayList(paths.keySet());
+		if (sortByPath) {
+			Collator collator = Collator.getInstance(new Locale("en", "US"));
+			collator.setStrength(Collator.TERTIARY);
+			Collections.sort(keys, collator)
+		} else {
+			keys.sort() { key -> -paths.get(key) }
+		}
+		for (String key : keys) {
+			println "$key\t"+paths.get(key)
+		}
+	}
+
+	public static void processDirectory(File srcdir, File outfile, String ext, boolean usePaths, boolean useAttributes, boolean useAttributeValues) {
+
+		def results = [:]
+		def allkeys = new HashSet<String>()
+
+		// counts
+		for (File infile : srcdir.listFiles()) {
+			if (!infile.getName().endsWith(".xml")) continue
+
+				XMLStatistics diag = new XMLStatistics(infile.toURI().toURL())
+			diag.setUsePaths(usePaths)
+			diag.setUseAttributes(useAttributes)
+			diag.setUseAttributeValues(useAttributeValues)
+			diag.process();
+			results.put(infile.getName(), diag.getPaths())
+			allkeys.addAll(diag.getPaths().keySet())
+			println ""
+		}
+
+		allkeys = new ArrayList(allkeys)
+		def table = [:]
+		def files = new ArrayList<String>(results.keySet())
+
+		// create table header
+		def header = []
+		header.addAll(files)
+		header << "TOTAL"
+		table["", header]
+
+		// Create empty cells
+		for (int i = 0 ; i < allkeys.size() ; i++) {
+			table[allkeys.get(i)] = new ArrayList(files.size())
+		}
+
+		// fill cells
+		for (int i = 0 ; i < allkeys.size() ; i++) {
+			String key = allkeys.get(i)
+			for (int j = 0 ; j < files.size() ; j++) {
+				def c = results.get(files.get(j)).get(key)
+				if (c != null) {
+					table[key][j] = c
+				} else {
+					table[key][j] = 0
+				}
+			}
+		}
+
+		// compute column margins
+		def colmargins = [];
+		allkeys << "TOTAL"
+		table["TOTAL"] = colmargins
+		for (int i = 0 ; i < files.size() ; i++) {
+			colmargins[i] = 0;
+			for (String key : allkeys) {
+				colmargins[i] += table[key].get(i)
+			}
+		}
+
+		// compute line margins
+		for (int i = 0 ; i < allkeys.size() ; i++) {
+			String key = allkeys.get(i)
+			int total = 0;
+			for (int j = 0 ; j < files.size() ; j++) {
+				total += table[key][j]
+			}
+			table[key][files.size()] = total
+		}
+
+		// sort keys
+		allkeys.sort() { it -> -table[it][files.size()] }
+		allkeys.remove(0)
+		allkeys << "TOTAL"
+
+		// print
+		outfile.withWriter("UTF-8") { writer ->
+			for(def h : header)
+				writer.print "\t"+h
+			writer.println ""
+			for (int i = 0 ; i < allkeys.size() ; i++) {
+				writer.print allkeys[i]
+				for (def c : table.get(allkeys[i])) {
+					if (c == 0) { writer.print "\t " }
+					else { writer.print "\t"+c }
+				}
+				writer.println ""
+			}
+		}
+	}
+	public static void main(String[] args) {
+		File srcdir = new File("/home/mdecorde/xml/temoignagesnice/corpus Matrice - fichiers xmlisés/");
+		File outdir = new File("/home/mdecorde/xml/temoignagesnice/corpus Matrice - fichiers xmlisés/stats");
+
+		// Process a directory
+		outdir.deleteDir()
+		outdir.mkdir()
+		processDirectory(srcdir, new File(outdir,"FFF.csv"), ".xml", false, false, false);
+		processDirectory(srcdir, new File(outdir,"FTF.csv"), ".xml", false, true, false);
+		processDirectory(srcdir, new File(outdir,"FTT.csv"), ".xml", false, true, true);
+		processDirectory(srcdir, new File(outdir,"TFF.csv"), ".xml", true, false, false);
+		processDirectory(srcdir, new File(outdir,"TTF.csv"), ".xml", true, true, false);
+		processDirectory(srcdir, new File(outdir,"TTT.csv"), ".xml", true, true, true);
+
+//		// Process one file
+//		File xmlfile = new File("/home/mdecorde/Bureau/matrice/témoignages/CONVERSIONS/ratz/odt-ratz.xml");
+//		XmlStatistics diag = new XmlStatistics(xmlfile.toURI().toURL());
+//		diag.setUsePaths(true)
+//		diag.setUseAttributes(false)
+//		diag.setUseAttributeValues(false)
+//		diag.process();
+//		diag.printCounts(true); 
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testPrintenv.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testPrintenv.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testPrintenv.groovy	(revision 321)
@@ -0,0 +1,84 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.clix;
+import java.io.File;
+import java.util.Properties;
+
+import org.txm.importer.filters.Filter;
+import org.txm.utils.ProcessBuilderBuilder;
+import groovy.lang.GroovyClassLoader;
+
+import org.txm.Toolbox
+/**
+ * test Printenv wrapper
+ */
+//ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/printenv-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/Printenv.groovy"));
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+GroovyClassLoader gcl = new GroovyClassLoader();
+
+def aScript
+
+try
+{
+	gcl.addClasspath(".");
+	Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/Printenv.groovy"));
+	aScript = clazz.newInstance("C:/unx/usr/local/wbin/");
+	
+}
+catch(Exception e)
+{	System.err.println(e);
+	return null;
+}
+
+aScript.printenvexe()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/diff-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/diff-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/diff-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<application name="Diff" version="0.0.0" desc="Diff src and target">
+	<progs>
+		<prog exec="diff" version="0.0.0" desc="Diff">
+			<args>
+				<arg prefix="-" name="nb" type="int" state="optional" desc="number of context lines"/>
+				<arg prefix="-" name="a" type="none" state="optional" desc="process files as text files"/>
+				<arg prefix="-" name="b" type="none" state="optional" desc="don't check whitespaces"/>
+				<arg prefix="-" name="B" type="none" state="optional" desc="don't check empty lines"/>
+				<arg prefix="-" name="c" type="none" state="optional" desc="Utiliser le format de sortie contextuel."/>
+				<arg prefix="-" name="C" type="int" state="optional" desc="number of line displayed."/>
+				<arg prefix="-" name="d" type="none" state="optional" desc="makes diff slower but change the algorithm to find more small differences"/>
+				<arg prefix="-" name="D" type="String" state="optional" desc="outfile of the merged texts"/>
+				<arg prefix="-" name="e" type="none" state="optional" desc="valid output for ed."/>
+				<arg prefix="" name="source" type="String" state="must" desc="the source file"/>
+				<arg prefix="" name="target" type="String" state="must" desc="the target file"/>
+			</args>
+		</prog>
+	</progs>
+</application>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/diff-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/treetagger-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/treetagger-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/treetagger-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,48 @@
+<?xml version="1.0">
+<application name="TreeTagger" version="0.0.0" desc="Tag files">
+	<progs>
+		<prog exec="tree-tagger" version="0.4.6" desc="Tag">
+			<args>
+				<arg state="optional" type="none" name="token" desc="Print the token"/>
+				<arg state="optional" type="none" name="lemma" desc="Print the lemma"/>
+				<arg state="optional" type="none" name="sgml" desc="Don't tag SGML annotations"/>
+				<arg state="optional" type="float" name="threshold" desc="Print all tags of a word with a probability higher than p times the largest probability"/>
+				<arg state="optional" type="none" name="prob" desc="Print tag probabilities"/>
+				<arg state="optional" type="none" name="ignore-prefix" desc=" Ignore prefix when guessing pos for unknown words"/>
+				<arg state="optional" type="none" name="no-unknown" desc="Print the token rather than [unknown] for unknown lemmas"/>
+				<arg state="optional" type="none" name="cap-heuristics" desc="Look up unknown capitalized words in the list of lower-case words"/>
+				<arg state="optional" type="none" name="hyphen-heuristics" desc="Turn on the heuristics fur guessing the parts of speech of unknown hyphenated words"/>
+				<arg state="optional" type="none" name="quiet" desc="quiet mode"/>
+				<arg state="optional" type="none" name="pt-with-prob" desc="pretagging with probabilities"/>
+				<arg state="optional" type="none" name="pt-with-lemma" desc="pretagging with lemmata"/>
+				<arg state="optional" type="none" name="proto" desc="Print lexical information for each word"/>
+				<arg state="optional" type="none" name="base" desc="Use only lexical probabilities for tagging"/>
+				<arg state="optional" type="float" name="eps" desc="Set minimal tag frequency to [epsilon]"/>
+				<arg state="optional" type="String" name="lex" desc="Read auxiliary lexicon entries from file "/>
+				<arg state="optional" type="String" name="wc" desc="Read a word-class automaton from file"/>
+				<arg state="optional" type="String" name="eos-tag" desc="The SGML tag [tag] signals the end of a sentence. this function implies the -sgml option"/>
+				<arg state="must" type="String" name="modelfile" desc="The model file"/>
+				<arg state="must" type="String" name="infile" desc="The In file"/>
+				<arg state="must" type="String" name="outfile" desc="The Out file"/>
+			</args>
+		</prog>
+
+		<prog exec="train-tree-tagger" version="0.0.0" desc="Train">
+			<args>
+				<arg state="optional" type="int" name="cl" desc="number of preceding words"/>
+				<arg state="optional" type="float" name="dtg" desc="min. decision tree gain"/>
+				<arg state="optional" type="float" name="sw" desc="A smoothing weight parameter"/>
+				<arg state="optional" type="float" name="atg" desc="affix tree gain"/>
+				<arg state="optional" type="String" name="st" desc="the end-of-sentence part-of-speech tag"/>
+				<arg state="optional" type="float" name="ecw" desc="weight of the equivalence class based probability estimates"/>
+				<arg state="optional" type="float" name="lt" desc="probability for lexical entries. default 0.001"/>
+				<arg state="optional" type="none" name="utf8" desc="assume that the data is encoded with UTF8"/>
+				<arg state="optional" type="none" name="quiet" desc="quiet mode"/>
+				<arg state="must" type="String" name="lexiconfile" desc="The lexicon file"/>
+				<arg state="must" type="String" name="openclassfile" desc="The open class file"/>
+				<arg state="must" type="String" name="infile" desc="The In file"/>
+				<arg state="must" type="String" name="outfile" desc="The Out file"/>
+			</args>
+		</prog>
+	</progs>
+</application>	

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/treetagger-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/sh-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/sh-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/sh-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,15 @@
+<?xml version="1.0">
+<application name="Sh" version="0.0.0" desc="Shell">
+	<progs>
+		<prog exec="sh.exe" version="0.0.0" desc="Sh">
+			<args>
+				<arg state="optional" type="String" name="c" desc="execute command line in argument"/>
+			</args>
+		</prog>
+		<prog exec="sh" version="0.0.0" desc="Sh">
+			<args>
+				<arg state="optional" type="String" name="c" desc="execute command line in argument"/>
+			</args>
+		</prog>
+	</progs>
+</application>	

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/sh-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains various Groovy scripts generated with the CLIX module</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testSh.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testSh.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testSh.groovy	(revision 321)
@@ -0,0 +1,93 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.clix;
+import java.io.File;
+import java.util.Properties;
+
+import org.txm.importer.filters.Filter;
+import org.txm.utils.ProcessBuilderBuilder;
+import groovy.lang.GroovyClassLoader;
+
+import org.txm.Toolbox
+
+/**
+ * test sh wrapper
+ */
+
+//ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/sh-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/Sh.groovy"));
+// TODO: Auto-generated Javadoc
+
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+ProcessBuilderBuilder.build(new File("groovy/org/txm/scripts/sh-wrapper-definition.xml"), new File("groovy/org/textometrie/scripts/Sh.groovy"));
+
+GroovyClassLoader gcl = new GroovyClassLoader();
+
+def aScript
+
+try
+{
+	gcl.addClasspath(".");
+	//Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/Sh.groovy"));
+	Class clazz = gcl.parseClass(new File("groovy/org/txm/scripts/Sh.groovy"));
+	aScript = clazz.newInstance("/bin/");
+
+}
+catch(Exception e)
+{	System.err.println(e);
+	return null;
+}
+String regpath ="~/.txm/registry";
+//String regpath ="C:/Documents and Settings/sheiden/.txm/registry";
+String execpath = "~/Bureau/trunkCWB/cwb-3.0/cqp/";
+//String execpath ="C:/Projets/Textom�trie/Logiciel/Toolbox/0.4.5/toolbox/src/main/C/cwb-3.0/utils/cwb-decode.exe";
+aScript.setc("\""+execpath+" -C -s 0 -e 5 -r '"+regpath+"' DISCOURS -P word\"")
+//aScript.shexe()
+aScript.sh()
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-decode-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-decode-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-decode-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<application name="CwbDecode" version="0.0.0" desc="Decode CWB indexes">
+	<progs>
+		<prog exec="cwb-decode.exe" version="0.0.0" desc="Decode">
+			<args>
+				<arg state="optional" type="none" name="L" desc="Lisp output mode"/>
+				<arg state="optional" type="none" name="H" desc="concordance line ('horizonta'l) output mode"/>
+				<arg state="optional" type="none" name="C" desc="compact output mode (suitable for encode)"/>
+				<arg state="optional" type="none" name="Cx" desc="XML-compatible compact output (for {encode -x ...}))"/>
+				<arg state="optional" type="none" name="X" desc="XML output mode"/>
+				<arg state="optional" type="none" name="n" desc="show corpus position ('numbers')"/>
+				<arg state="optional" type="int" name="s" desc="first token to print (at corpus position {n})"/>
+				<arg state="optional" type="int" name="e" desc="last token to print (at corpus position {n})"/>
+				<arg state="optional" type="String" name="r" desc="set registry directory"/>
+				<arg state="optional" type="none" name="p" desc="matchlist mode (input from stdin)"/>
+				<arg state="optional" type="String" name="f" desc="matchlist mode (input from {file})"/>
+				<arg state="must" type="String" name="corpus" desc="corpus name"/>
+				<arg state="optional" type="none" name="h" desc="this help page"/>
+				<arg state="multiple" type="String" name="P" desc="print p-attribute {att}"/>
+				<arg state="multiple" type="String" name="S" desc="print s-attribute {att} (possibly including annotations)"/>
+				<arg state="multiple" type="String" name="V" desc="show s-attribute annotation for each range in matchlist mode"/>
+				<arg state="multiple" type="String" name="A" desc="print alignment attribute {att}"/>
+				<arg state="optional" type="none" name="ALL" desc="print all p-attributes and s-attributes"/>
+				<arg state="optional" type="String" name="c" desc="expand ranges to full {att} region (matchlist mode)"/>
+			</args>
+		</prog>
+		<prog exec="cwb-decode" version="0.0.0" desc="Decode">
+			<args>
+				<arg state="optional" type="none" name="L" desc="Lisp output mode"/>
+				<arg state="optional" type="none" name="H" desc="concordance line ('horizonta'l) output mode"/>
+				<arg state="optional" type="none" name="C" desc="compact output mode (suitable for encode)"/>
+				<arg state="optional" type="none" name="Cx" desc="XML-compatible compact output (for {encode -x ...}))"/>
+				<arg state="optional" type="none" name="X" desc="XML output mode"/>
+				<arg state="optional" type="none" name="n" desc="show corpus position ('numbers')"/>
+				<arg state="optional" type="int" name="s" desc="first token to print (at corpus position {n})"/>
+				<arg state="optional" type="int" name="e" desc="last token to print (at corpus position {n})"/>
+				<arg state="optional" type="String" name="r" desc="set registry directory"/>
+				<arg state="optional" type="none" name="p" desc="matchlist mode (input from stdin)"/>
+				<arg state="optional" type="String" name="f" desc="matchlist mode (input from {file})"/>
+				<arg state="must" type="String" name="corpus" desc="corpus name"/>
+				<arg state="optional" type="none" name="h" desc="this help page"/>
+				<arg state="multiple" type="String" name="P" desc="print p-attribute {att}"/>
+				<arg state="multiple" type="String" name="S" desc="print s-attribute {att} (possibly including annotations)"/>
+				<arg state="multiple" type="String" name="V" desc="show s-attribute annotation for each range in matchlist mode"/>
+				<arg state="multiple" type="String" name="A" desc="print alignment attribute {att}"/>
+				<arg state="optional" type="none" name="ALL" desc="print all p-attributes and s-attributes"/>
+				<arg state="optional" type="String" name="c" desc="expand ranges to full {att} region (matchlist mode)"/>
+			</args>
+		</prog>
+	</progs>
+</application>	

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-decode-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/ls-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/ls-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/ls-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<application name="Ls" version="0.0.0" desc="List directory">
+	<progs>
+		<prog exec="ls" version="0.0.0" desc="Ls">
+			<args>
+				<arg prefix="-" name="l" type="none" state="optional" desc="use a long listing format"/>
+				<arg prefix="-" name="a" type="none" state="optional" desc="do not ignore entries starting with '.'"/>
+				<arg prefix="-" name="d" type="String" state="optional" desc="list directory entries instead of contents"/>
+				
+			</args>
+		</prog>
+	</progs>
+</application>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/ls-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/Ls.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/Ls.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/Ls.groovy	(revision 321)
@@ -0,0 +1,127 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.clix;
+
+import java.util.Date;
+import java.text.DateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+// List directory
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class Ls.
+ */
+class Ls {
+	
+	/** The binpath. */
+	String binpath = "";
+
+	/**
+	 * Instantiates a new ls.
+	 *
+	 * @param binpath the binpath
+	 */
+	public Ls(String binpath) {
+		this.binpath = binpath;
+	}
+
+	/** The version. */
+	String version = "0.0.0";
+	
+	/** The desc. */
+	String desc = "List directory";
+
+	// use a long listing format
+	
+	/** The isl. */
+	private Boolean isl = false;
+
+	/**
+	 * Sets the isl.
+	 */
+	public void setIsl() {
+		this.isl = true;
+	}
+
+	/**
+	 * Unset isl.
+	 */
+	public void unsetIsl() {
+		this.isl = false;
+	}
+
+	// Ls
+	
+	/**
+	 * Lsexe.
+	 */
+	public void lsexe() throws IOException {
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "ls.exe");
+		if (isl)
+			args.add("-l");
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = process.waitFor();
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.LONG).format(
+							new Date()));
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		Ls tt = new Ls("");
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testLs.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testLs.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/testLs.groovy	(revision 321)
@@ -0,0 +1,85 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.clix;
+import java.io.File;
+import java.util.Properties;
+
+import org.txm.importer.filters.Filter;
+import org.txm.utils.ProcessBuilderBuilder;
+import groovy.lang.GroovyClassLoader;
+
+import org.txm.Toolbox
+// TODO: Auto-generated Javadoc
+
+/**
+ * test Ls wrapper.
+ *
+ * @return the java.lang. object
+ */
+ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/ls-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/Ls.groovy"));
+
+GroovyClassLoader gcl = new GroovyClassLoader();
+
+def aScript
+
+try
+{
+	gcl.addClasspath(".");
+	Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/Ls.groovy"));
+	aScript = clazz.newInstance("C:/Program Files/UnxUtils/usr/local/wbin/");
+	
+}
+catch(Exception e)
+{	System.err.println(e);
+	return null;
+}
+
+aScript.setl()
+aScript.lsexe()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/xmlstarlet-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/xmlstarlet-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/xmlstarlet-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,26 @@
+<?xml version="1.0">
+<!-- TO BE FINISHED, WORK IN PROGRESS : SLH 2010-04-10 -->
+<application name="XMLStarlet" version="1.0.1" desc="XMLStarlet Toolkit: Command line utilities for XML">
+	<progs>
+		<prog exec="xml.exe ed" version="1.0.1" desc="XMLStarlet Toolkit: Edit XML document(s), Usage: xml ed <global-options> {<action>} [ <xml-file-or-uri> ... ], where <global-options>  - global options for editing, <xml-file-or-uri> - input XML document file name/uri (stdin otherwise)">
+			<args>
+				<arg state="optional" type="none" name="P" desc="-P (or --pf)        - preserve original formatting"/>
+				<arg state="optional" type="none" name="S" desc="-S (or --ps)        - preserve non-significant spaces"/>
+				<arg state="optional" type="none" name="O" desc="-O (or --omit-decl) - omit XML declaration (&lt;?xml ...?>)"/>
+				<arg state="optional" type="String" name="N" desc="-N &lt;name>=&lt;value>   - predefine namespaces (name without 'xmlns:'), ex: xsql=urn:oracle-xsql, Multiple -N options are allowed., -N options must be last global options."/>
+				<arg state="optional" type="none" name="h" desc="--help or -h        - display help"/>
+				<arg state="optional" type="String" name="d" desc="-d or --delete &lt;xpath>"/>
+				<arg state="optional" type="String" name="i" desc="-i or --insert &lt;xpath> -t (--type) elem|text|attr -n &lt;name> -v (--value) &lt;value>"/>
+				<arg state="optional" type="String" name="t" desc="-t (--type) elem|text|attr"/>
+				<arg state="optional" type="String" name="n" desc="-n &lt;name>"/>
+				<arg state="optional" type="String" name="v" desc="-v (--value) &lt;value>"/>
+				<arg state="optional" type="String" name="a" desc="-a or --append &lt;xpath> -t (--type) elem|text|attr -n &lt;name> -v (--value) &lt;value>"/>
+				<arg state="optional" type="String" name="s" desc="-s or --subnode &lt;xpath> -t (--type) elem|text|attr -n &lt;name> -v (--value) &lt;value>"/>
+				<arg state="optional" type="String" name="m" desc="-m or --move &lt;xpath1> &lt;xpath2>"/>
+				<arg state="optional" type="String" name="r" desc="-r or --rename &lt;xpath1> -v &lt;new-name>"/>
+				<arg state="optional" type="String" name="u" desc="-u or --update &lt;xpath> -v (--value) &lt;value>"/>
+				<arg state="optional" type="String" name="x" desc="-x (--expr) &lt;xpath> (-x is not implemented yet)"/>
+			</args>
+		</prog>
+	</progs>
+</application>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/xmlstarlet-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-align-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-align-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-align-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<application name="CwbAlign" version="3.2" desc="Align a corpus with an other">
+	<progs>
+		<prog exec="cwb-align" version="3.2" desc="create align file">
+			<args>
+				<arg state="optional" type="String" name="P" desc="use positional attribute [word]"/>
+				<arg state="optional" type="String" name="S" desc="ordered prealignment"/>
+				<arg state="optional" type="String" name="V" desc="ordered prealignment with annotations, ex : seg_id"/>
+				<arg state="optional" type="String" name="o" desc="outfile file [out.align]"/>
+				<arg state="optional" type="String" name="s" desc="set 2:2 alignment split factor [1.2]"/>
+				<arg state="optional" type="String" name="w" desc="use best path search beam of width [50]"/>
+				<arg state="optional" type="String" name="r" desc="use registry directory"/>
+				<arg state="optional" type="none" name="v" desc="verbose mode"/>
+				<arg state="optional" type="none" name="h" desc="show help"/>
+				<arg state="must" type="String" name="source" desc="source corpus identifier"/>
+				<arg state="must" type="String" name="target" desc="target corpus identifier"/>
+				<arg state="must" type="String" name="sattrib" desc="the aligned structure, ex : seg"/>
+			</args>
+		</prog>
+		<prog exec="cwb-align-encode" version="3.2" desc="create corpus align data">
+			<args>
+				<arg state="optional" type="String" name="d" desc="write data file to directory"/>
+				<arg state="optional" type="none" name="D" desc="write data files to corpus data directory"/>
+				<arg state="optional" type="none" name="C" desc="compatibility mode (creates .alg file)"/>
+				<arg state="optional" type="String" name="r" desc="set registry directory"/>
+				<arg state="optional" type="none" name="v" desc="verbose mode"/>
+				<arg state="optional" type="none" name="h" desc="show help"/>
+				<arg state="must" type="String" name="alignmentfile" desc="the align file created by cwb-align"/>
+			</args>
+		</prog>
+	</progs>
+</application>	

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/cwb-align-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/printenv-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/printenv-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/printenv-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<application name="Printenv" version="0.0.0" desc="Print Environment">
+	<progs>
+		<prog exec="printenv.exe" version="0.0.0" desc="Printenv">
+			<args>
+			</args>
+		</prog>
+	</progs>
+</application>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/printenv-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/tnt-wrapper-definition.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/tnt-wrapper-definition.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/tnt-wrapper-definition.xml	(revision 321)
@@ -0,0 +1,59 @@
+<?xml version="1.0">
+<application name="TnT" version="0.0.0" desc="Trigrams'n'Tags : part-of-speach tagger">
+	<progs>
+	
+		<prog exec="tnt" version="0.0.0" desc="tagging. the language model is loaded from model.lex and model.123. if model.map exists, it is used for output mapping compressed files are recognized by suffix .gz, .bz2, or .Z">
+			<args>
+				<arg state="optional" type="int" name="a" desc="use suffix trie with max. suffix length = len; default = 10"/>
+				<arg state="optional" type="File" name="b" desc="use fil as backup lexicon"/>
+				<arg state="optional" type="int" name="B" desc="backup mode : 0=main only, 1=mix, 2=backup only (default: 1)"/>
+				<arg state="optional" type="String" name="d" desc="sparse data mode"/>
+				<arg state="optional" type="none" name="H" desc="copy HTML tags to output (without tagging)"/>
+				<arg state="optional" type="none" name="m" desc="mark unknown words in output with an asterisk (*)"/>
+				<arg state="optional" type="int" name="n" desc="use num-grams, num = 1, 2, 3, default = 3"/>
+				<arg state="optional" type="int" name="u" desc="unknown word count, default = 3"/>
+				<arg state="optional" type="int" name="v" desc="set verbosity level num: (default = 3)"/>
+				<arg state="optional" type="int" name="z" desc="output tag if prob is in beam num, default =0 (means infinity)"/>
+				<arg state="optional" type="int" name="Z" desc="cut off path if prob is not in beam num (default 1000)"/>
+				<arg state="must" type="String" name="model" desc="model name, will be searched in current dir or in the path specified in the env var TNT_MODELS"/>
+				<arg state="must" type="File" name="corpus" desc="corpus file ?"/>
+			</args>
+		</prog>
+		
+		<prog exec="tnt-diff" version="0.0.0" desc="counting differences">
+			<args>
+				<arg state="optional" type="none" name="a" desc="print accuracy vs. ambiguity"/>
+				<arg state="optional" type="int" name="f" desc="print accuracy vs. frequency, max freq = max"/>
+				<arg state="optional" type="none" name="i" desc="ignore upper/lower case of tokens"/>
+				<arg state="optional" type="File" name="l" desc="lexicon to account for known/unknown words"/>
+				<arg state="optional" type="File" name="m" desc="use tagset mapping in file"/>
+				<arg state="must" type="File" name="originalfile" desc="model file"/>
+				<arg state="must" type="File" name="resultfile" desc="corpus file ?"/>
+			</args>
+		</prog>
+		
+		<prog exec="tnt-para" version="0.0.0" desc="parameters generation">
+			<args>
+				<arg state="optional" type="none" name="c" desc="encode capitalization in tag"/>
+				<arg state="optional" type="none" name="H" desc="ignore HTML tags for parameter generation"/>
+				<arg state="optional" type="none" name="i" desc="ignore case"/>
+				<arg state="optional" type="none" name="l" desc="generate lexicon only"/>
+				<arg state="optional" type="none" name="n" desc="generate ngrams only"/>
+				<arg state="optional" type="String" name="o" desc="base name for output files, default=basename of corpus"/>
+				<arg state="optional" type="none" name="v" desc="generate verbose ngrams"/>
+				<arg state="must" type="File" name="corpus" desc="corpus file ?"/>
+			</args>
+		</prog>
+		
+		<prog exec="tnt-wc" version="0.0.0" desc="counting tokens and types">
+			<args>
+				<arg state="optional" type="none" name="H" desc="ignore HTML tags for parameter generation"/>
+				<arg state="optional" type="none" name="i" desc="ignore case"/>
+				<arg state="optional" type="none" name="l" desc="count word types"/>
+				<arg state="optional" type="none" name="t" desc="count tags"/>
+				<arg state="optional" type="none" name="w" desc="count words tokens"/>
+				<arg state="must" type="File" name="corpus" desc="corpus file ?"/>
+			</args>
+		</prog>
+	</progs>
+</application>

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/clix/tnt-wrapper-definition.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FixPortalProfiles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FixPortalProfiles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FixPortalProfiles.groovy	(revision 321)
@@ -0,0 +1,125 @@
+package org.txm.scripts
+
+import groovy.xml.StreamingMarkupBuilder
+
+def importxml = new File("/home/mdecorde/TEMP/import.xml")
+def usersdir = new File("/home/mdecorde/TEMP/users")
+def usersdir2 = new File("/home/mdecorde/TEMP/users2")
+usersdir2.deleteDir()
+usersdir2.mkdir()
+
+// 1- get subcorpus & partitions per user
+def entities = [:];
+corpusInfo = [:];
+def root = new XmlSlurper().parse(importxml)
+
+def corpusNode = null
+for (def tmp : root.corpora.corpus) corpusNode = tmp
+def prefix = "/"+corpusNode.@name.toString()
+
+for (def preBuild : corpusNode.preBuild) {
+	for (def child : preBuild.children()) {
+		def tmp_entities = getEntities(prefix, child)
+		def user = "";
+		for (def entity : tmp_entities) {
+			String name = entity;
+			user = entity.substring(prefix.length()+1, entity.indexOf(":"))
+			//println user
+			break;
+		}
+
+		if (!entities.keySet().contains(user)) entities[user] = new HashSet<String>();
+		for (def entity : tmp_entities) {
+			entities[user] << entity
+		}
+	}
+}
+println entities
+println corpusInfo
+
+// 2- fix user_xxxx.xml
+for (def file : usersdir.listFiles()) {
+	if (!file.getName().endsWith(".xml")) continue;
+
+	def rootuser = new XmlParser().parse(file);
+	String login = rootuser.@login
+	if (entities.keySet().contains(login)) {
+		def user_entities = entities[login]
+
+		def existingEntities = new HashSet();
+		for (def entityNode : rootuser.permissions.entity) {
+			existingEntities << entityNode.@path.toString()
+		}
+		println "FIXING $file with $user_entities existing: $existingEntities"
+		for (def newentity : user_entities) {
+			if (!existingEntities.contains(newentity)) {
+				def permissionNode = null
+				for( def tmp : rootuser.permissions) permissionNode = tmp
+				// new
+				println " append $newentity "
+				def newnode = new Node(null, "entity");
+				newnode.@path = newentity
+				addPermissions(newnode, corpusInfo[newentity])
+				permissionNode.children().add(newnode)
+			}
+		}
+
+		new File(usersdir2, file.getName()).withWriter("UTF-8") { writer ->
+			new XmlNodePrinter(new PrintWriter(writer)).print(rootuser)
+		}
+	}
+}
+
+
+
+
+def addPermissions(def node, def type) {
+	
+	def subcorpusPerm = ["EditionPermission", "SpecificitiesPermission","IndexPermission","DictionnaryPermission",
+		"TextSelectionPermission", "CreateSubcorpusPermission", "DeletePermission", "DimensionsPermission",
+		"CreatePartitionPermission","DisplayPermission"]
+	
+	def partitionPerm = ["SpecificitiesPermission","IndexPermission","DictionnaryPermission",
+		"AFCPermission", "DeletePermission", "DimensionsPermission","DisplayPermission"]
+	
+	def children = node.children();
+
+	if (type == "partition") {
+		for( def perm : partitionPerm) {
+			def newnode = new Node(null, "permission");
+			newnode.@anti = "false"
+			newnode.@value = perm
+			children.add(newnode);
+		}
+	} else if (type == "subcorpus") {
+		def concnode = new Node(null, "permission");
+		concnode.@anti = "false"
+		concnode.@value = "ConcordancePermission"
+		concnode.@leftContext = 9999
+		concnode.@rightContext = 9999
+		children.add(concnode);
+		for( def perm : subcorpusPerm) {
+			def newnode = new Node(null, "permission");
+			newnode.@anti = "false"
+			newnode.@value = perm
+			children.add(newnode);
+		}
+	}
+}
+
+def getEntities(String prefix, def node) {
+	def list = []
+
+	String name = node.@name
+	String tmp = prefix +"/"+name
+	list << tmp
+
+	corpusInfo[tmp] = node.name()
+
+	for (def child : node.children()) {
+		if (child.name().toString() == "partition" | child.name().toString() == "subcorpus") {
+			list.addAll(getEntities(tmp,child))
+		}
+	}
+	return list
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/SubCorpusToSimpleCorpus.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/SubCorpusToSimpleCorpus.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/SubCorpusToSimpleCorpus.groovy	(revision 321)
@@ -0,0 +1,146 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.scripts.tigersearch;
+
+import org.txm.sw.ReplaceXmlDomNode;
+
+import java.io.File;
+import java.util.ArrayList;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class SubCorpusToSimpleCorpus.
+ */
+class SubCorpusToSimpleCorpus
+{
+	
+	/** The srcdir. */
+	File srcdir;
+	
+	/** The outdir. */
+	File outdir;
+	
+	/** The files. */
+	ArrayList<File> files = new ArrayList<File>();
+	
+	/** The master. */
+	File master;
+	
+	/**
+	 * Instantiates a new sub corpus to simple corpus.
+	 *
+	 * @param srcdir the srcdir
+	 * @param outdir the outdir
+	 */
+	public SubCorpusToSimpleCorpus(File srcdir, File outdir)
+	{
+		this.srcdir = srcdir;
+		this.outdir = outdir;
+	}
+	
+	/**
+	 * Scan.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean scan()
+	{
+		if(!srcdir.exists())
+		{
+			System.out.println("Cant find srcdir: "+srcdir);			
+		}
+		
+		outdir.mkdir();
+		if(!outdir.exists())
+		{
+			System.out.println("Cant find outdir: "+srcdir);			
+		}	
+		
+		master = new File(srcdir, "master.xml");
+		def root = new XmlParser().parse(master);
+		for(def subcorpus : root.body.subcorpus)
+		{
+			String filename = subcorpus.@external
+			filename = filename.substring(5);
+			File f = new File(srcdir, filename);
+			if(f.exists())
+				files.add(f);
+			else
+				println("Missing file : "+f);
+		}
+		return true;
+	}
+	
+	/**
+	 * Fusion.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean fusion()
+	{
+		for(File f : files)
+		{
+			println "replace "+f.getName();
+			String filename = f.getName();
+			File target = master;
+			
+			String xpathsrc = "//subcorpus"
+				
+			String xpathtarget = "//subcorpus[@external=\"file:"+filename+"\"]"
+			
+			new ReplaceXmlDomNode(f, target, xpathsrc, xpathtarget)
+		}
+	}
+	
+	/**
+	 * Gets the files.
+	 *
+	 * @return the files
+	 */
+	public ArrayList<File> getFiles()
+	{
+		return files;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File srcdir = new File("/home/mdecorde/xml/beroul_bp")
+		File outdir = new File("/home/mdecorde/xml/beroul_bp/merged") // not used
+		def fusioner = new SubCorpusToSimpleCorpus(srcdir, outdir);
+		if(fusioner.scan())
+		{
+			println "subcorpus files : "+fusioner.getFiles()
+			fusioner.fusion();
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/BuildTTFile.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/BuildTTFile.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/BuildTTFile.groovy	(revision 321)
@@ -0,0 +1,193 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.tigersearch;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class BuildTTFile.
+ *
+ * @author mdecorde
+ * 
+ * build the TT source for tigerSearch
+ */
+
+public class BuildTTFile {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The output. */
+	private def output;
+
+	/** The solotags. */
+	ArrayList<String> solotags;
+
+	/**
+	 * Instantiates a new builds the tt file.
+	 *
+	 * @param url the url
+	 * @param solotags the solotags
+	 */
+	public BuildTTFile(URL url, ArrayList<String> solotags) {
+		try {
+			this.url = url;
+			this.solotags = solotags;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			File f = outfile;
+			output = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+	 * @param targetbalise the targetbalise
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile, String targetbalise) {
+		if (createOutput(outfile)) {
+
+			String lastopenlocalname = "";
+			String localname = "";
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser
+						.next()) {
+
+					switch (event) {
+
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+						switch (localname) {
+						case targetbalise:
+							String word = parser
+									.getAttributeValue(null, "word");
+							output.write(word + "\n");
+							break;
+						case "s":
+							output.write("<s>\n");
+							break;
+						}
+
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName();
+						switch (localname) {
+
+						case targetbalise:
+
+							break;
+						case "s":
+							output.write("</s>\n");
+							break;
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						// output.write(parser.getText().trim());
+						break;
+					}
+				}
+				output.close();
+				parser.close();
+			} catch (XMLStreamException ex) {
+				System.out.println(ex);
+			} catch (IOException ex) {
+				System.out.println("IOException while parsing " + inputData);
+			}
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/beroul/";
+		// new File(rootDir+"/identity/").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();// the tags who
+		// you want them
+		// to stay
+		// milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		milestones.add("catRef");
+
+		File srcfile = new File(rootDir, "beroul.xml");
+		File resultfile = new File(rootDir, "beroul.tt");
+		println("identity file : " + srcfile + " to : " + resultfile);
+
+		def builder = new BuildTTFile(srcfile.toURL(), milestones);
+		builder.process(resultfile, "t");
+
+		return;
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/InjectAnnotations.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/InjectAnnotations.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/InjectAnnotations.groovy	(revision 321)
@@ -0,0 +1,255 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.tigersearch;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class InjectAnnotations.
+ *
+ * @author mdecorde
+ * 
+ * inject annotations into ONE file
+ */
+
+public class InjectAnnotations {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The reader. */
+	private Reader reader;
+
+	/** The output. */
+	private def output;
+
+	/** The solotags. */
+	ArrayList<String> solotags;
+	
+	/** The lespos. */
+	HashSet<String> lespos = new HashSet<String>();
+
+	/**
+	 * Instantiates a new inject annotations.
+	 *
+	 * @param url the url
+	 * @param annotations the annotations
+	 * @param solotags the solotags
+	 */
+	public InjectAnnotations(URL url, File annotations,
+			ArrayList<String> solotags) {
+		try {
+			this.url = url;
+			this.solotags = solotags;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+
+			reader = new FileReader(annotations);
+
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			File f = outfile;
+			output = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Gets the next annotation.
+	 *
+	 * @return the next annotation
+	 */
+	private String getNextAnnotation() {
+		String line = reader.readLine();
+		while (line.startsWith("<"))
+			line = reader.readLine();
+		lespos.add(line.split("\t")[1]);
+		return line = line.split("\t")[1];
+	}
+
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile) {
+		if (createOutput(outfile)) {
+
+			String lastopenlocalname = "";
+			String localname = "";
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser
+						.next()) {
+
+					switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+						String prefix = parser.getPrefix();
+						if (prefix == null || prefix == "")
+							prefix = "";
+						else
+							prefix += ":";
+
+						lastopenlocalname = localname;
+						output.write("\n<" + prefix + localname);
+
+						for (int i = 0; i < parser.getAttributeCount(); i++)
+							output.write(" " + parser.getAttributeLocalName(i)
+									+ "=\"" + parser.getAttributeValue(i)
+									+ "\"");
+
+						// get annotation
+						if (localname.equals("t"))
+							output.write(" pos=\"" + getNextAnnotation()
+											+ "\"");
+
+						if (solotags.contains(localname))
+							output.write("/>");
+						else
+							output.write(">");
+						break;
+
+					case XMLStreamConstants.END_ELEMENT:
+
+						localname = parser.getLocalName();
+						String prefix = parser.getPrefix();
+						if (prefix == null || prefix == "")
+							prefix = "";
+						else
+							prefix += ":";
+
+						switch (localname) {
+
+						default:
+							if (!solotags.contains(localname))
+								if (lastopenlocalname.equals(localname))
+									output.write("</" + prefix + localname
+											+ ">");
+								else
+									output.write("\n</" + prefix + localname
+											+ ">");
+						}
+						break;
+
+					case XMLStreamConstants.CHARACTERS:
+						output.write(parser.getText().trim());
+						break;
+					}
+				}
+				output.close();
+				parser.close();
+			} catch (XMLStreamException ex) {
+				System.out.println(ex);
+			} catch (IOException ex) {
+				System.out.println("IOException while parsing " + inputData);
+			}
+		}
+	}
+
+	/**
+	 * Gets the feature.
+	 *
+	 * @param f the f
+	 * @return the feature
+	 */
+	public void getFeature(File f)
+	{
+		Writer writer = new OutputStreamWriter(new FileOutputStream(f) , "UTF-8");
+		writer.write("<feature name=\"pos\" domain=\"T\">\n")
+		for(String pos : lespos)
+			writer.write("<value name=\""+pos+"\"></value>\n");
+		writer.write("</feature>\n")
+		writer.close();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/beroul/";
+		new File(rootDir + "/identity/").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();// the tags who
+		// you want them
+		// to stay
+		// milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		milestones.add("catRef");
+
+		File srcfile = new File(rootDir, "beroul.xml");
+		File annotationsfiles = new File(rootDir, "result.tt");
+		File resultfile = new File(rootDir, "beroul-result.xml");
+		println("identity file : " + srcfile + " to : " + resultfile);
+
+		def builder = new InjectAnnotations(srcfile.toURL(), annotationsfiles,
+				milestones);
+		builder.process(resultfile);
+
+		return;
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Manage tigerSearch source file</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/InsertAnnotationsBeroul.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/InsertAnnotationsBeroul.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/tigersearch/InsertAnnotationsBeroul.groovy	(revision 321)
@@ -0,0 +1,110 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.tigersearch;
+
+import org.txm.utils.treetagger.TreeTagger;
+import org.txm.scripts.teitxm.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * script to insert annotation into beroul file.
+ *
+ * @return the java.lang. object
+ */
+
+String home = System.getProperty("user.home")
+File rootDir = new File(home, "xml/fullberoul/")
+
+ArrayList<String> milestones = new ArrayList<String>();//the tags who you want them to stay milestones
+milestones.add("tagUsage");
+milestones.add("pb");
+milestones.add("lb");
+milestones.add("catRef");
+
+//transform xml tiger >> TTsrc
+File srcfile = new File(rootDir,"beroul.xml");
+File resultfile = new File(rootDir,"beroul.tt");
+println("xml>>TT from : "+srcfile+" to : "+resultfile );
+
+def builder = new BuildTTFile(srcfile.toURL(), milestones);
+builder.process(resultfile, "t");
+
+//tag TT
+String infile = resultfile;
+String modelfile = home+"/treetagger/models/fro.par";
+String outfile = rootDir.getAbsolutePath()+"/result.tt";
+
+println("proj "+modelfile+ " on " +resultfile +" >> "+outfile);
+
+TreeTagger tt = new TreeTagger(home+"/treetagger/bin/");
+tt.settoken();
+tt.setquiet();
+tt.setsgml();
+tt.seteostag("<s>");
+tt.treetagger( modelfile, infile, outfile)
+
+//inject new TTattributes
+//File srcfile = new File(rootDir,"beroul.xml");
+File annotationsfiles = new File(rootDir,"result.tt");
+File lastresultfile = new File(rootDir,"beroul-result.xml");
+println("insert TT annotations : "+srcfile+" to : "+resultfile );
+
+builder = new InjectAnnotations(srcfile.toURL(),annotationsfiles, milestones);
+builder.process(lastresultfile);
+
+builder.getFeature(new File(rootDir,"feature.xml"));
+/*
+//TAG with TnT
+//need to replace <s> by nothing and </s> by \n
+String encoding = "UTF-8"
+for(String text : texts)
+{
+	//patch src files
+	File f = new File(textsDir,text+".t");
+	File temp = new File("tempFileCVScleaner")
+	println("patch texts files "+f+": rmv <s> and replace </s>");
+	Reader reader = new InputStreamReader(new FileInputStream(f),encoding);
+	Writer writer = new FileWriter(temp);
+	reader.eachLine 
+			{
+				if(it.trim().startsWith("</s"))
+					writer.write("\n")
+				else if(it.trim().startsWith("<s"))
+					writer.write("")
+				else
+					writer.write(it+"\n")
+			}
+	reader.close();
+	writer.close();
+	if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+}*/
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/CwbDecode.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/CwbDecode.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/CwbDecode.groovy	(revision 321)
@@ -0,0 +1,691 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import java.util.Date;
+import java.util.Locale;
+import java.text.DateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+// Decode CWB indexes
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class CwbDecode.
+ */
+class CwbDecode {
+	
+	/** The binpath. */
+	String binpath = "";
+
+	/**
+	 * Instantiates a new cwb decode.
+	 *
+	 * @param binpath the binpath
+	 */
+	public CwbDecode(String binpath) {
+		this.binpath = binpath;
+	}
+
+	/** The version. */
+	String version = "0.0.0";
+	
+	/** The desc. */
+	String desc = "Decode CWB indexes";
+	
+	/** The debug. */
+	boolean debug = false;
+
+	/**
+	 * Debug.
+	 *
+	 * @param b the b
+	 */
+	public void debug(boolean b) {
+		debug = b;
+	};
+
+	// Lisp output mode
+	
+	/** The is l. */
+	private Boolean isL = false;
+
+	/**
+	 * Sets the l.
+	 */
+	public void setL() {
+		this.isL = true;
+	}
+
+	/**
+	 * Unset l.
+	 */
+	public void unsetL() {
+		this.isL = false;
+	}
+
+	// concordance line ('horizonta'l) output mode
+	
+	/** The is h. */
+	private Boolean isH = false;
+
+	/**
+	 * Sets the h.
+	 */
+	public void setH() {
+		this.isH = true;
+	}
+
+	/**
+	 * Unset h.
+	 */
+	public void unsetH() {
+		this.isH = false;
+	}
+
+	// compact output mode (suitable for encode)
+	
+	/** The is c. */
+	private Boolean isC = false;
+
+	/**
+	 * Sets the c.
+	 */
+	public void setC() {
+		this.isC = true;
+	}
+
+	/**
+	 * Unset c.
+	 */
+	public void unsetC() {
+		this.isC = false;
+	}
+
+	// XML-compatible compact output (for {encode -x ...}))
+	
+	/** The is cx. */
+	private Boolean isCx = false;
+
+	/**
+	 * Sets the cx.
+	 */
+	public void setCx() {
+		this.isCx = true;
+	}
+
+	/**
+	 * Unset cx.
+	 */
+	public void unsetCx() {
+		this.isCx = false;
+	}
+
+	// XML output mode
+	
+	/** The is x. */
+	private Boolean isX = false;
+
+	/**
+	 * Sets the x.
+	 */
+	public void setX() {
+		this.isX = true;
+	}
+
+	/**
+	 * Unset x.
+	 */
+	public void unsetX() {
+		this.isX = false;
+	}
+
+	// show corpus position ('numbers')
+	
+	/** The isn. */
+	private Boolean isn = false;
+
+	/**
+	 * Setn.
+	 */
+	public void setn() {
+		this.isn = true;
+	}
+
+	/**
+	 * Unsetn.
+	 */
+	public void unsetn() {
+		this.isn = false;
+	}
+
+	// first token to print (at corpus position {n})
+	
+	/** The iss. */
+	private Boolean iss = false;
+	
+	/** The s. */
+	private int s;
+
+	/**
+	 * Sets the s.
+	 *
+	 * @param arg the new s
+	 */
+	public void sets(int arg) {
+		this.iss = true;
+		this.s = arg;
+	}
+
+	/**
+	 * Unsets.
+	 */
+	public void unsets() {
+		this.iss = false;
+	}
+
+	// last token to print (at corpus position {n})
+	
+	/** The ise. */
+	private Boolean ise = false;
+	
+	/** The e. */
+	private int e;
+
+	/**
+	 * Sets the e.
+	 *
+	 * @param arg the new e
+	 */
+	public void sete(int arg) {
+		this.ise = true;
+		this.e = arg;
+	}
+
+	/**
+	 * Unsete.
+	 */
+	public void unsete() {
+		this.ise = false;
+	}
+
+	// set registry directory
+	
+	/** The isr. */
+	private Boolean isr = false;
+	
+	/** The r. */
+	private String r;
+
+	/**
+	 * Sets the r.
+	 *
+	 * @param arg the new r
+	 */
+	public void setr(String arg) {
+		this.isr = true;
+		this.r = arg;
+	}
+
+	/**
+	 * Unsetr.
+	 */
+	public void unsetr() {
+		this.isr = false;
+	}
+
+	// matchlist mode (input from stdin)
+	
+	/** The isp. */
+	private Boolean isp = false;
+
+	/**
+	 * Setp.
+	 */
+	public void setp() {
+		this.isp = true;
+	}
+
+	/**
+	 * Unsetp.
+	 */
+	public void unsetp() {
+		this.isp = false;
+	}
+
+	// matchlist mode (input from {file})
+	
+	/** The isf. */
+	private Boolean isf = false;
+	
+	/** The f. */
+	private String f;
+
+	/**
+	 * Sets the f.
+	 *
+	 * @param arg the new f
+	 */
+	public void setf(String arg) {
+		this.isf = true;
+		this.f = arg;
+	}
+
+	/**
+	 * Unsetf.
+	 */
+	public void unsetf() {
+		this.isf = false;
+	}
+
+	// this help page
+	
+	/** The ish. */
+	private Boolean ish = false;
+
+	/**
+	 * Seth.
+	 */
+	public void seth() {
+		this.ish = true;
+	}
+
+	/**
+	 * Unseth.
+	 */
+	public void unseth() {
+		this.ish = false;
+	}
+
+	// print p-attribute {att}
+	
+	/** The is p. */
+	private Boolean isP = false;
+	
+	/** The P. */
+	private List<String> P;
+
+	/**
+	 * Sets the p.
+	 *
+	 * @param arg the new p
+	 */
+	public void setP(List<String> arg) {
+		this.isP = true;
+		this.P = arg;
+	}
+
+	/**
+	 * Unset p.
+	 */
+	public void unsetP() {
+		this.isP = false;
+	}
+
+	// print s-attribute {att} (possibly including annotations)
+	
+	/** The is s. */
+	private Boolean isS = false;
+	
+	/** The S. */
+	private List<String> S;
+
+	/**
+	 * Sets the s.
+	 *
+	 * @param arg the new s
+	 */
+	public void setS(List<String> arg) {
+		this.isS = true;
+		this.S = arg;
+	}
+
+	/**
+	 * Unset s.
+	 */
+	public void unsetS() {
+		this.isS = false;
+	}
+
+	// show s-attribute annotation for each range in matchlist mode
+	
+	/** The is v. */
+	private Boolean isV = false;
+	
+	/** The V. */
+	private List<String> V;
+
+	/**
+	 * Sets the v.
+	 *
+	 * @param arg the new v
+	 */
+	public void setV(List<String> arg) {
+		this.isV = true;
+		this.V = arg;
+	}
+
+	/**
+	 * Unset v.
+	 */
+	public void unsetV() {
+		this.isV = false;
+	}
+
+	// print alignment attribute {att}
+	
+	/** The is a. */
+	private Boolean isA = false;
+	
+	/** The A. */
+	private List<String> A;
+
+	/**
+	 * Sets the a.
+	 *
+	 * @param arg the new a
+	 */
+	public void setA(List<String> arg) {
+		this.isA = true;
+		this.A = arg;
+	}
+
+	/**
+	 * Unset a.
+	 */
+	public void unsetA() {
+		this.isA = false;
+	}
+
+	// print all p-attributes and s-attributes
+	
+	/** The is all. */
+	private Boolean isALL = false;
+
+	/**
+	 * Sets the all.
+	 */
+	public void setALL() {
+		this.isALL = true;
+	}
+
+	/**
+	 * Unset all.
+	 */
+	public void unsetALL() {
+		this.isALL = false;
+	}
+
+	// expand ranges to full {att} region (matchlist mode)
+	
+	/** The isc. */
+	private Boolean isc = false;
+	
+	/** The c. */
+	private String c;
+
+	/**
+	 * Sets the c.
+	 *
+	 * @param arg the new c
+	 */
+	public void setc(String arg) {
+		this.isc = true;
+		this.c = arg;
+	}
+
+	/**
+	 * Unsetc.
+	 */
+	public void unsetc() {
+		this.isc = false;
+	}
+
+	// Decode
+	
+	/**
+	 * Cwbdecodeexe.
+	 *
+	 * @param corpus the corpus
+	 */
+	public void cwbdecodeexe(String corpus) throws IOException
+	// arg : corpus name
+	{
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "cwb-decode.exe");
+		if (isL)
+			args.add("-L");
+		if (isH)
+			args.add("-H");
+		if (isC)
+			args.add("-C");
+		if (isCx)
+			args.add("-Cx");
+		if (isX)
+			args.add("-X");
+		if (isn)
+			args.add("-n");
+		if (iss) {
+			args.add("-s");
+			args.add("" + s);
+		}
+		if (ise) {
+			args.add("-e");
+			args.add("" + e);
+		}
+		if (isr) {
+			args.add("-r");
+			args.add("" + r);
+		}
+		if (isp)
+			args.add("-p");
+		if (isf) {
+			args.add("-f");
+			args.add("" + f);
+		}
+		args.add("" + corpus);
+		if (ish)
+			args.add("-h");
+		if (isP)
+			for (int c = 0; c < P.size(); c++) {
+				args.add("-P");
+				args.add("" + P.get(c));
+			}
+		if (isS)
+			for (int c = 0; c < S.size(); c++) {
+				args.add("-S");
+				args.add("" + S.get(c));
+			}
+		if (isV)
+			for (int c = 0; c < V.size(); c++) {
+				args.add("-V");
+				args.add("" + V.get(c));
+			}
+		if (isA)
+			for (int c = 0; c < A.size(); c++) {
+				args.add("-A");
+				args.add("" + A.get(c));
+			}
+		if (isALL)
+			args.add("-ALL");
+		if (isc) {
+			args.add("-c");
+			args.add("" + c);
+		}
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+
+			for (int c = 0; c < args.size(); c++)
+				System.out.print("" + args.get(c) + " ");
+			System.out.println();
+		}
+	}
+
+	// Decode
+	
+	/**
+	 * Cwbdecode.
+	 *
+	 * @param corpus the corpus
+	 */
+	public void cwbdecode(String corpus) throws IOException
+	// arg : corpus name
+	{
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "cwb-decode");
+		if (isL)
+			args.add("-L");
+		if (isH)
+			args.add("-H");
+		if (isC)
+			args.add("-C");
+		if (isCx)
+			args.add("-Cx");
+		if (isX)
+			args.add("-X");
+		if (isn)
+			args.add("-n");
+		if (iss) {
+			args.add("-s");
+			args.add("" + s);
+		}
+		if (ise) {
+			args.add("-e");
+			args.add("" + e);
+		}
+		if (isr) {
+			args.add("-r");
+			args.add("" + r);
+		}
+		if (isp)
+			args.add("-p");
+		if (isf) {
+			args.add("-f");
+			args.add("" + f);
+		}
+		args.add("" + corpus);
+		if (ish)
+			args.add("-h");
+		if (isP)
+			for (int c = 0; c < P.size(); c++) {
+				args.add("-P");
+				args.add("" + P.get(c));
+			}
+		if (isS)
+			for (int c = 0; c < S.size(); c++) {
+				args.add("-S");
+				args.add("" + S.get(c));
+			}
+		if (isV)
+			for (int c = 0; c < V.size(); c++) {
+				args.add("-V");
+				args.add("" + V.get(c));
+			}
+		if (isA)
+			for (int c = 0; c < A.size(); c++) {
+				args.add("-A");
+				args.add("" + A.get(c));
+			}
+		if (isALL)
+			args.add("-ALL");
+		if (isc) {
+			args.add("-c");
+			args.add("" + c);
+		}
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+
+			for (int c = 0; c < args.size(); c++)
+				System.out.print("" + args.get(c) + " ");
+			System.out.println();
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		CwbDecode tt = new CwbDecode("");
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/EmptyFiles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/EmptyFiles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/EmptyFiles.groovy	(revision 321)
@@ -0,0 +1,63 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.scripts;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+File orig = new File ("/home/mdecorde/TXM/corpora/temoignages-bin/txm/TEMOIGNAGES/")
+/*File dest = new File ("chemin2")
+dest.mkdir()*/
+for (File f:orig.listFiles()) {
+	//File copy = new File (dest,f.getName())
+	//copy.createNewFile()
+	f.delete()
+	f.createNewFile()
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/CountNumberOfChar.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/CountNumberOfChar.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/CountNumberOfChar.groovy	(revision 321)
@@ -0,0 +1,104 @@
+package org.txm.scripts
+
+
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.io.File;
+import javax.xml.stream.*;
+import java.net.URL;
+
+import org.txm.importer.*;
+
+/**
+ * Counts the number of chars in tokens in a specific element
+ * NB: Tokens are separated by whitespaces
+ * 
+ * Can set attribute filter
+ * @author mdecorde
+ */
+class CountNumberOfChar {
+	int total = 0;
+	int nbWords = 0;
+	String elem = "w";
+	String attribute = null;
+	String regex = null;
+
+	/**
+	 * @param dir a directory
+	 * @param elem the element to focus on
+	 */
+	public CountNumberOfChar(File dir, String elem, String attribute, String regex) {
+		this.elem = elem;
+		this.attribute  = attribute;
+		this.regex = regex;
+
+		for (File f : dir.listFiles())
+			if (ValidateXml.test(f))
+				countInXML(f)
+	}
+
+	protected void countInXML(File f)
+	{
+		println "count: "+f
+		boolean startCount = false
+		try {
+			URL url = f.toURI().toURL();
+			def inputData = url.openStream();
+			def factory = XMLInputFactory.newInstance();
+			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+			{
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+
+						if (parser.getLocalName() == elem) {
+							if (attribute == null || regex == null) {
+								startCount = true;
+								nbWords++
+							} else {
+								String attrvalue = parser.getAttributeValue(null, attribute)
+								if (attrvalue != null && attrvalue.matches(regex)) {
+									startCount = true;
+									nbWords++
+								}
+							}
+						}
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						if (parser.getLocalName() == elem) {
+							startCount = false
+						}
+						break;
+					case XMLStreamConstants.CHARACTERS:
+						if (startCount) {
+							for (String tok : parser.getText().trim()) {
+								total += tok.length()
+							}
+							
+						}
+				}
+			}
+
+		}
+		catch(Exception e){e.printStackTrace();}
+	}
+
+	public float getMean() {
+		return ((float)total)/(float)nbWords;
+	}
+
+	public int getNbWords() {
+		return nbWords;
+	}
+
+	public int getTotal() {
+		return total;
+	}
+
+	public static void main(def args)
+	{
+		CountNumberOfChar c = new CountNumberOfChar(new File(System.getProperty("user.home")+"/TXM/corpora/bfm1/txm"), "form", null, null)
+		//CountNumberOfChar c = new CountNumberOfChar(new File(System.getProperty("user.home")+"/TXM/corpora/bfm1/txm"), "form", "type", "pon")
+		System.out.println("Moyenne: "+c.getMean());
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/trainTT.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/trainTT.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/trainTT.groovy	(revision 321)
@@ -0,0 +1,57 @@
+package org.txm.scripts;
+
+import java.util.ArrayList;
+import org.txm.utils.treetagger.TreeTagger;
+import org.txm.utils.*;
+import org.txm.importer.*;
+import org.txm.importer.RGAQCJ.*;
+import java.io.File;
+import java.text.DateFormat;
+import java.util.Date;
+import java.text.DecimalFormat;
+/**
+ * test TT wrapper
+ * add NLP annotation
+ * build result matrix
+ * 
+ */
+rootDir = "C:/Documents and Settings/alavrent/Mes documents/Projets/Morpho MultiDim/bfm3_2011";
+new File(rootDir+"/models/").mkdir();
+new File(rootDir+"/lexicon/").mkdir();
+//new File(rootDir+"/proj/").mkdir();
+new File(rootDir+"/ttlearn/").mkdir();
+//new File(rootDir+"/ttsrc/").mkdir();
+//new File(rootDir+"/pos/").mkdir();
+//new File(rootDir+"/injection/").mkdir();
+//new File(rootDir+"/anainline/").mkdir();
+//new File(rootDir+"/xml-txm/").mkdir();
+
+
+//train
+def ttlearnfiles = ["bfm3_2011-learn.tt"];
+def modelfiles = ["bfm3_2011.par"];
+
+def lexfiles = ["bfm3_2011-lex-fus.txt"];
+def openclassfiles = ["cattex2009_open_class.txt"];
+//def lexiconfile = rootDir+"/lexicon/"+"lexfra-rgaqcj.txt";
+
+
+for(int i=0; i < ttlearnfiles.size();i++)
+{
+	String infile = rootDir+"/ttlearn/"+ttlearnfiles[i];
+	String outfile = rootDir+"/models/"+modelfiles[i];
+	String lexiconfile = rootDir+"/lexicon/"+lexfiles[i];
+	String openclassfile = rootDir+"/lexicon/"+openclassfiles[i];
+	TreeTagger tt = new TreeTagger("C:/Program Files/treetagger/bin/");
+	println("learn "+ttlearnfiles[i] +" >> "+modelfiles[i]);
+	tt.settoken();
+	tt.setquiet();
+	//tt.setlemma();
+	tt.setsgml();
+	tt.setst("PONfrt");
+	tt.setproto();
+	tt.setutf8();
+	tt.traintreetagger( lexiconfile, openclassfile, infile, outfile)
+}
+
+//
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationInjectionFromTTOutput.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationInjectionFromTTOutput.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationInjectionFromTTOutput.groovy	(revision 321)
@@ -0,0 +1,358 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-03-02 14:17:06 +0100 (mer., 02 mars 2016) $
+// $LastChangedRevision: 3134 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm;
+import javax.xml.stream.*
+
+import org.txm.importer.StaxIdentityParser
+import org.txm.importer.filters.*
+import org.txm.importer.graal.PersonalNamespaceContext
+// TODO: Auto-generated Javadoc
+/**
+ * The Class AnnotationInjection.
+ *
+ * @author mdecorde
+ * 
+ * inject annotation from a stand-off file into a xml-tei-txm
+ * file
+ */
+
+public class AnnotationInjectionFromTTOutput extends StaxIdentityParser {
+
+	public static String TXMNS = "http://textometrie.org/1.0"
+
+	/** The xml reader factory. */
+	private def factory;
+
+	/** The links. */
+	private List<String> links;
+
+	/** The linkparsers. key=type*/
+	private HashMap<String, XMLStreamReader> linkparsers;
+
+	/** The anaurl. */
+	private def anaurl;
+
+	/** The anainput data. */
+	private def anainputData;
+
+	/** The anafactory. */
+	private XMLInputFactory anafactory = XMLInputFactory.newInstance();
+
+	/** The anaparser. */
+	private XMLStreamReader anaparser;
+	private XMLStreamReader headerparser;
+
+	/** The resp stmt id. */
+	String respStmtID = "";
+
+	/** The present taxonomies. */
+	ArrayList<String> presentTaxonomies = new ArrayList();
+
+	/**
+	 * Instantiates a new annotation injection.
+	 *
+	 * @param url the xml-tei-txm file
+	 * @param anaurl the stand-off file
+	 */
+	public AnnotationInjectionFromTTOutput(URL url, URL anaurl) {
+		super(url); // init reader and writer
+		try {
+			this.anaurl = anaurl;
+			factory = XMLInputFactory.newInstance();
+			this.buildLinkParsers();// build a parser per linkgroup
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	private void getHeaderInfos(String containertag, boolean captureTheTag)
+	{
+		anainputData = new BufferedInputStream(anaurl.openStream());
+		headerparser = anafactory.createXMLStreamReader(anainputData);
+		boolean start = false;
+		String localname;
+		for (int event = headerparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = headerparser.next())
+		{
+			//String prefix = headerparser.getNamespaceURI();
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				localname = headerparser.getLocalName();
+				if (captureTheTag && localname == containertag) // start copy  after the tag
+					start = true;
+				if (start) { // copy header
+					String prefix = headerparser.getPrefix();
+					if (prefix.length() > 0)
+						writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+					else
+						writer.writeStartElement(localname);
+					for (int i = 0 ; i < headerparser.getNamespaceCount(); i++)
+						writer.writeNamespace(headerparser.getNamespacePrefix(i), headerparser.getNamespaceURI(i));
+					for (int i = 0 ; i < headerparser.getAttributeCount(); i++)
+						writer.writeAttribute(headerparser.getAttributeLocalName(i), headerparser.getAttributeValue(i));
+				}
+				if (!captureTheTag && localname == containertag) // start copy  after the tag
+					start = true;
+			} else if (event == XMLStreamConstants.END_ELEMENT) {
+				localname = headerparser.getLocalName();
+				if (!captureTheTag && localname == containertag)
+					break;// stop looping
+
+				if (start)
+					writer.writeEndElement();
+
+				if (captureTheTag && localname == containertag)
+					break;// stop looping
+			} else if (event == XMLStreamConstants.CHARACTERS) {
+				if (start)
+					writer.writeCharacters(headerparser.getText());
+			} else if (event == XMLStreamConstants.COMMENT) {
+				if (start)
+					writer.writeComment(headerparser.getText());
+			}
+		}
+		headerparser.close();
+	}
+
+	/**
+	 * find all refs.
+	 *
+	 * @return the list of link parser
+	 */
+	private List<String> findGrpLink()
+	{
+		ArrayList<String> links = new ArrayList<String>();
+		anainputData = anaurl.openStream();
+		anaparser = anafactory.createXMLStreamReader(anainputData);
+
+		for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if (anaparser.getLocalName().matches("linkGrp")) {
+					String targetsvalue = anaparser.getAttributeValue(0)
+
+					if (links.contains(targetsvalue)) {
+						System.err.println("Warning: Multiple group declaration : "+targetsvalue+" has already been added, the first one will be used")
+					} else {
+						links.add(targetsvalue); // add the taxonomy type
+					}
+				} else if (anaparser.getLocalName().matches("respStmt")) {
+					respStmtID = anaparser.getAttributeValue(0); // one attribute (id) only
+				}
+			}
+		}
+		anaparser.close();
+		return links;
+	}
+
+	/**
+	 * Builds the link parsers.
+	 * I need to know what groups exists to build a parser per taxonomy and go to the first link element
+	 */
+	private void buildLinkParsers()
+	{
+		// link group of the standoff file
+		links = findGrpLink();
+		linkparsers = new HashMap<String, XMLStreamReader>();
+
+		// build one parser per link group
+		for (String link : links) { // build a parser per group
+			anainputData = new BufferedInputStream(anaurl.openStream());
+			linkparsers.put(link, anafactory.createXMLStreamReader(anainputData));
+		}
+
+		//for each parser
+		for (String link : links) {
+			anaparser = linkparsers.get(link);
+			for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {
+				if (event == XMLStreamConstants.START_ELEMENT) {
+					if (anaparser.getLocalName().equals("linkGrp")) { // position the parser to the right group
+						String targetsvalue = anaparser.getAttributeValue(0)
+						if (targetsvalue.equals(link))
+							break; // next element is a link start tag
+					}
+				}
+			}
+		}
+	}
+
+	/**
+	 * get the next tei:link value of a tei:LinkGrp.
+	 *
+	 * @param link the link
+	 * @return the next ana
+	 */
+	private String getNextAnaValue(String link, String wordId)
+	{
+		anaparser = linkparsers.get(link);
+		def m;
+		for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if (anaparser.getLocalName().matches("link")) {
+					String targetsvalue = anaparser.getAttributeValue(0)
+					if ((m = targetsvalue =~ /#(.*) #(.*)/)) { // balise externe
+						def g1 = m[0][1];
+						def g2 = m[0][2];
+
+						String anavalue = g2;
+						anavalue = anavalue.replace("&lt;", "<")
+						return anavalue;
+					} else {
+						System.err.println("Error: getNextAna(): link target is not well formed:  = "+anaparser.getAttributeValue(0));
+					}
+				}
+			}
+		}
+		return "";
+	}
+
+	/**
+	 * build the ana tags of a word.
+	 *
+	 * @param wordId the word id
+	 * @return the ana tag
+	 */
+	private void writeAnaTag(String wordId)
+	{
+		String anabalises ="\n";
+		for (String link : links) {
+			writer.writeStartElement(TXMNS, "ana");
+			writer.writeAttribute("resp", "#"+respStmtID);
+			writer.writeAttribute("type", "#"+link);
+			writer.writeCharacters(getNextAnaValue(link, wordId));
+			writer.writeEndElement(); // txm:ana
+		}
+	}
+
+	String wordId;
+	boolean flagSourceDesc = false;
+	protected void processStartElement()
+	{
+		if (parser.getLocalName().matches("taxonomy")) {
+			String taxo = parser.getAttributeValue(0) // taxonomy type
+			presentTaxonomies.add(taxo);
+		}
+
+		for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {
+			if (parser.getAttributeLocalName(i) == "id") {
+				wordId = parser.getAttributeValue(i);
+				break
+			}
+		}
+		super.processStartElement();
+	}
+
+	boolean applicationWritten = false;
+	boolean taxonomiesWritten = false;
+	protected void processEndElement() {
+		switch (parser.getLocalName()) {
+			case "w":
+				writeAnaTag(wordId);
+				break;
+
+			case "appInfo":
+				applicationWritten = true;
+				getHeaderInfos("appInfo", false);
+				break;
+
+			case "classDecl":
+				taxonomiesWritten = true;
+				getHeaderInfos("classDecl", false);
+				break;
+
+			case "encodingDesc":
+				if (!applicationWritten) {
+					writer.writeStartElement("appInfo");
+					getHeaderInfos("appInfo", false);
+					writer.writeEndElement(); // appInfo
+				}
+				if (!taxonomiesWritten) {
+					writer.writeStartElement("classDecl");
+					getHeaderInfos("classDecl", false);
+					writer.writeEndElement(); // classDecl
+				}
+				break;
+				
+			case "titleStmt":
+				if (flagSourceDesc) {
+					//output.write(this.respStmt+"\n")
+					getHeaderInfos("respStmt", true);
+					flagSourceDesc = false;
+					break;
+				}
+				break;
+		}
+		super.processEndElement();
+	}
+
+	/** The declarenamespace. */
+	boolean declarenamespace = false;
+
+	/**
+	 * Declare namespace.
+	 *
+	 * @return the java.lang. object
+	 */
+	private declareNamespace() {
+		if (!declarenamespace) {
+			writer.writeDefaultNamespace("http://www.tei-c.org/ns/1.0");
+			writer.writeNamespace("txm", TXMNS);
+			declarenamespace = true;
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir + "/injection/").mkdir();
+
+		def milestones = ["tagUsage", "pb", "lb","catRef"]// the tags who
+
+		File srcfile = new File(rootDir, "/anainline/", "roland.xml");
+		File pos1file = new File(rootDir, "/pos/", "rolandTT1-w-ana.xml");
+
+		File src2file = new File(rootDir, "/injection/", "roland.xml");
+		File pos2file = new File(rootDir, "/pos/", "rolandTT2-w-ana.xml");
+		
+		println("process file : " + srcfile + " with : " + pos1file);
+		def builder = new AnnotationInjectionFromTTOutput(srcfile.toURI().toURL(),
+				pos1file.toURI().toURL(), milestones);
+		builder.transfomFile(new File(rootDir + "/injection/", "roland.xml"));
+
+		println("process file : " + src2file + " with : " + pos1file);
+		builder = new AnnotationInjectionFromTTOutput(src2file.toURI().toURL(), pos2file.toURI().toURL(),
+				milestones);
+		builder.transfomFile(rootDir + "/injection/", "roland-FINAL.xml");
+
+		return;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Manages xml-tei-txm files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationExtraction.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationExtraction.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationExtraction.groovy	(revision 321)
@@ -0,0 +1,358 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class AnnotationExtraction.
+ *
+ * @author mdecorde
+ * 
+ * Extract ana tags from a xml-tei-txm file
+ * saved into a stand-off file
+ */
+public class AnnotationExtraction
+{
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The writer. */
+	private Writer writer;
+
+	/** The url. */
+	private def url;
+
+	/** The posfile. */
+	private def posfile;
+
+	/** The anafile. */
+	private def anafile;
+
+	/** The encoding. */
+	private def encoding;
+
+	/** The extract types. */
+	private List<String> extractTypes;
+
+	/** The resp stmt. */
+	private String respStmt;
+
+	/** The solotags. */
+	ArrayList<String> solotags;
+
+	/**
+	 * initialize.
+	 *
+	 * @param anafile the path to the xml-tei-txm file
+	 * @param posfile the output stand-off file
+	 * @param encoding the encoding of the xml-tei-txm file
+	 * @param milestones milestones tags (temporary)
+	 */
+	public AnnotationExtraction(File anafile, File posfile,String encoding,milestones) {
+		this.url = anafile.toURI().toURL();
+		this.posfile = posfile;
+		this.encoding = encoding;
+		this.solotags = milestones;
+
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+	}
+
+	/**
+	 * process !!!.
+	 */
+	public void process() {
+		assert(extractTypes != null);
+		if (this.createOutput(posfile,encoding)) {
+			this.writeHead();
+			this.writeBody( posfile, encoding);
+			this.writeTail();
+		}
+		writer.close();
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @param encoding the encoding
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile, String encoding){
+		try {
+			writer = new OutputStreamWriter(new FileOutputStream(outfile) , "UTF-8");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * set the respStmt tag of the stand-off file by reading the respstmt of the xml-tei-txm file.
+	 */
+	private void setResp()
+	{
+		this.respStmt = "";
+		boolean begincapture = false;
+		String lastopenlocalname= "";
+
+		inputData = url.openStream();
+		parser = factory.createXMLStreamReader(inputData);
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			String prefix = parser.getPrefix();
+			if (prefix == null || prefix == "")
+				prefix = "";
+			else
+				prefix +=":";
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if (parser.getLocalName().matches("resp")) {
+					String id = parser.getAttributeValue("xml","id")
+					if (id == null)
+						id = parser.getAttributeValue(null,"id")
+					if (extractTypes.contains(id)) {
+						this.respStmt = "<respStmt>\n";
+						begincapture= true;
+					}
+				}
+				if (begincapture) {
+					lastopenlocalname = parser.getLocalName();
+					respStmt += ("\n<"+prefix+parser.getLocalName());
+					for (int i= 0 ; i < parser.getAttributeCount() ;i++ )
+						respStmt += (" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"" );
+					if (solotags.contains(lastopenlocalname))
+						respStmt += ("/>");
+					else
+						respStmt += (">");
+				}
+			}
+			else if (event == XMLStreamConstants.END_ELEMENT) {
+				if (parser.getLocalName().matches("respStmt")) {
+					begincapture= false;
+					this.respStmt += "\n</respStmt>"
+					System.out.println(this.respStmt);
+					parser.close();
+					return;
+				} else {
+					if (!solotags.contains(parser.getLocalName()))
+						if (lastopenlocalname.equals(parser.getLocalName()))
+							respStmt += ("</"+prefix+parser.getLocalName()+">");
+						else
+							respStmt += ("\n</"+prefix+parser.getLocalName()+">");
+				}
+			} else if (event == XMLStreamConstants.CHARACTERS) {
+				String txt = parser.getText().trim();
+				respStmt += txt;
+			}
+		}
+	}
+
+	/**
+	 * write the header of the stand-off ile.
+	 */
+	private void writeHead() {
+		this.setResp();
+
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+		writer.write("<!DOCTYPE TEI SYSTEM \"tei_bfm_v1.dtd\">\n");
+		writer.write("<TEI xmlns:txm=\"http://textometrie.ens-lyon.fr/1.0\">\n");
+		writer.write("<teiHeader xml:lang=\"eng\">\n");
+		writer.write("<fileDesc>\n");
+		writer.write("<titleStmt>\n");
+		for (int i=0;i < extractTypes.size(); i++)
+			writer.write("<title>"+extractTypes[i]+"</title>\n");
+		writer.write(this.respStmt+"\n");
+		writer.write("</titleStmt>\n");
+		writer.write("<publicationStmt>\n");
+		writer.write("<distributor>BFM project - http://bfm.ens-lsh.fr</distributor>\n");
+		writer.write("<availability>\n");
+		writer.write("<p>(c) 2010 Projet BFM - CNRS/ENS-LSH.\n");
+		writer.write("<hi>Conditions d'utilisation</hi> : \n");
+		writer.write("Sous licence <ref target=\"http://creativecommons.org/licenses/by-sa/2.0/fr/\">Creative Commons</ref>.\n");
+		writer.write("</p>\n");
+		writer.write("</availability>\n");
+		writer.write("</publicationStmt>\n");
+		writer.write("<sourceDesc>\n");
+		writer.write("<p>born digital : TXM project - http://textometrie.org</p>\n");
+		writer.write("</sourceDesc>\n");
+		writer.write("</fileDesc>\n");
+		writer.write("</teiHeader>\n");
+		writer.write("<text xml:lang=\"fr\" type=\"standoff\">\n");
+		writer.write("<body>\n");
+		writer.write("<div>\n");
+	}
+
+	/**
+	 * write the group of ana tags with the attribute type = type.
+	 *
+	 * @param type the type
+	 */
+	private void writeGroup(String type)
+	{
+		System.out.println("write group : "+type);
+		inputData = url.openStream();
+		factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+
+		boolean flagAna = false;
+		boolean flagForm = false;
+		boolean flagW = false;
+		String wordid="";
+		String vAna="";
+		String currentType;
+
+		writer.write("<linkGrp type=\""+type+"\">\n")
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					switch (parser.getLocalName()) {
+						case "w":
+						wordid = parser.getAttributeValue(null,"id");
+						flagW = true;
+						break;
+						case "ana":
+						if (flagW) {
+							currentType = (parser.getAttributeValue(null,"type"));
+							if (currentType.matches(type)) {
+								flagAna = true;
+								vAna ="";
+							}
+						}
+						break;
+					}
+					break;
+
+				case XMLStreamConstants.END_ELEMENT:
+					switch (parser.getLocalName()) {
+						case "w":
+						flagW = false;
+						break;
+
+						case "ana":
+						if (flagW) {
+							writer.write("<link targets=\"#"+wordid+" #"+vAna+"\"/>\n")
+							flagAna = false;
+						}
+						break;
+					}
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					if (flagAna)
+						vAna += parser.getText().trim();//catch interp text
+					break;
+			}
+		}
+		writer.write("</linkGrp>\n");
+		parser.close();
+		inputData.close();
+	}
+
+	/**
+	 * write a group per ana type to extract from the anafile.
+	 *
+	 * @param TTrez the t trez
+	 * @param encoding the encoding
+	 * @return the java.lang. object
+	 */
+	private writeBody(File TTrez, String encoding)
+	{
+		for (String type : extractTypes) {
+			writeGroup(type);
+		}
+	}
+
+	/**
+	 * write the tail of the stand-off file = close body, text and TEI tags.
+	 */
+	private void writeTail()
+	{
+		writer.write("</div>\n");
+		writer.write("</body>\n");
+		writer.write("</text>\n");
+		writer.write("</TEI>\n");
+	}
+
+	/**
+	 * define the types to extract.
+	 *
+	 * @param types the new types to extract
+	 */
+	public void setTypesToExtract(List<String> types)
+	{
+		extractTypes = types;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/"
+		new File(rootDir+"/extract/").mkdir()
+
+		File anafile = new File(rootDir+"/injection/","roland-ana.xml")
+		File posfile =  new File(rootDir+"/extract/","roland-pos.xml")
+		String encoding = "UTF-8"
+		//extrait les txm:ana dont la ref est dans cette liste
+		List<String> types = ["#t1","#POS"]
+
+		ArrayList<String> milestones = new ArrayList<String>()
+		//the tags you want to keep as milestones
+		milestones.add("tagUsage")
+		milestones.add("pb")
+		milestones.add("lb")
+		milestones.add("catRef")
+
+		AnnotationExtraction builder = new AnnotationExtraction(anafile,posfile,encoding,milestones)
+		builder.setTypesToExtract(types)
+		builder.process()
+		return
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationInjection.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationInjection.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotationInjection.groovy	(revision 321)
@@ -0,0 +1,414 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-03-29 09:51:35 +0200 (Tue, 29 Mar 2016) $
+// $LastChangedRevision: 3185 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm;
+import javax.xml.stream.*
+
+import org.txm.importer.StaxIdentityParser
+import org.txm.importer.filters.*
+import org.txm.importer.graal.PersonalNamespaceContext
+// TODO: Auto-generated Javadoc
+/**
+ * The Class AnnotationInjection.
+ *
+ * @author mdecorde
+ * 
+ * inject annotation from a stand-off file into a xml-tei-txm
+ * file
+ */
+
+public class AnnotationInjection extends StaxIdentityParser {
+
+	public static String TXMNS = "http://textometrie.org/1.0"
+
+	/** The xml reader factory. */
+	private def factory;
+
+	/** The links. */
+	private LinkedHashSet<String> links;
+	boolean replace
+
+	/** The linkparsers. key=type*/
+	private LinkedHashMap<String, XMLStreamReader> linkparsers;
+
+	/** The anaurl. */
+	private def anaurl;
+
+	/** The anainput data. */
+	private def anainputData;
+
+	/** The anafactory. */
+	private XMLInputFactory anafactory = XMLInputFactory.newInstance();
+
+	/** The anaparser. */
+	private XMLStreamReader anaparser;
+	private XMLStreamReader headerparser;
+
+	/** The resp stmt id. */
+	String respStmtID = "";
+
+	/** The present taxonomies. */
+	ArrayList<String> presentTaxonomies = new ArrayList();
+
+	/**
+	 * Instantiates a new annotation injection.
+	 *
+	 * @param url the xml-tei-txm file
+	 * @param anaurl the stand-off file
+	 */
+	public AnnotationInjection(URL url, URL anaurl) {
+		this(url, anaurl, false)
+	}
+
+	/**
+	 * Instantiates a new annotation injection.
+	 *
+	 * @param url the xml-tei-txm file
+	 * @param anaurl the stand-off file
+	 */
+	public AnnotationInjection(URL url, URL anaurl, boolean replace) {
+		super(url); // init reader and writer
+		try {
+			this.anaurl = anaurl;
+			this.replace = replace
+			factory = XMLInputFactory.newInstance();
+			this.buildLinkParsers();// build a parser per linkgroup
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	private void getHeaderInfos(String containertag, boolean captureTheTag)
+	{
+		anainputData = new BufferedInputStream(anaurl.openStream());
+		headerparser = anafactory.createXMLStreamReader(anainputData);
+		boolean start = false;
+		String localname;
+		for (int event = headerparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = headerparser.next())
+		{
+			//String prefix = headerparser.getNamespaceURI();
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				localname = headerparser.getLocalName();
+				if (captureTheTag && localname == containertag) // start copy  after the tag
+					start = true;
+				if (start) { // copy header
+					String prefix = headerparser.getPrefix();
+					if (prefix.length() > 0)
+						writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)
+					else
+						writer.writeStartElement(localname);
+					for (int i = 0 ; i < headerparser.getNamespaceCount(); i++)
+						writer.writeNamespace(headerparser.getNamespacePrefix(i), headerparser.getNamespaceURI(i));
+					for (int i = 0 ; i < headerparser.getAttributeCount(); i++)
+						writer.writeAttribute(headerparser.getAttributeLocalName(i), headerparser.getAttributeValue(i));
+				}
+				if (!captureTheTag && localname == containertag) // start copy  after the tag
+					start = true;
+			} else if (event == XMLStreamConstants.END_ELEMENT) {
+				localname = headerparser.getLocalName();
+				if (!captureTheTag && localname == containertag)
+					break;// stop looping
+
+				if (start)
+					writer.writeEndElement();
+
+				if (captureTheTag && localname == containertag)
+					break;// stop looping
+			} else if (event == XMLStreamConstants.CHARACTERS) {
+				if (start)
+					writer.writeCharacters(headerparser.getText());
+			} else if (event == XMLStreamConstants.COMMENT) {
+				if (start)
+					writer.writeComment(headerparser.getText());
+			}
+		}
+		headerparser.close();
+	}
+
+	/**
+	 * find all refs.
+	 *
+	 * @return the list of link parser
+	 */
+	private LinkedHashSet<String> findGrpLink()
+	{
+		LinkedHashSet<String> links = new LinkedHashSet<String>();
+		anainputData = anaurl.openStream();
+		anaparser = anafactory.createXMLStreamReader(anainputData);
+
+		for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if (anaparser.getLocalName().equals("linkGrp")) {
+					String targetsvalue = anaparser.getAttributeValue(0)
+
+					if (links.contains(targetsvalue)) {
+						System.err.println("Warning: Multiple group declaration : "+targetsvalue+" has already been added, the first one will be used")
+					} else {
+						links.add(targetsvalue); // add the taxonomy type
+					}
+				} else if (anaparser.getLocalName().equals("respStmt")) {
+					respStmtID = anaparser.getAttributeValue(0); // one attribute (id) only
+				}
+			}
+		}
+		anaparser.close();
+		return links;
+	}
+
+	/**
+	 * Builds the link parsers.
+	 * I need to know what groups exists to build a parser per taxonomy and go to the first link element
+	 */
+	private void buildLinkParsers()
+	{
+		// link group of the standoff file
+		links = findGrpLink();
+		linkparsers = new LinkedHashMap<String, XMLStreamReader>();
+
+		// build one parser per link group
+		for (String link : links) { // build a parser per group
+			anainputData = new BufferedInputStream(anaurl.openStream());
+			linkparsers.put(link, anafactory.createXMLStreamReader(anainputData));
+		}
+
+		//for each parser
+		for (String link : links) {
+			anaparser = linkparsers.get(link);
+			for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {
+				if (event == XMLStreamConstants.START_ELEMENT) {
+					if (anaparser.getLocalName().equals("linkGrp")) { // position the parser to the right group
+						String targetsvalue = anaparser.getAttributeValue(0)
+						if (targetsvalue.equals(link))
+							break; // next element is a link start tag
+					}
+				}
+			}
+		}
+	}
+
+	/**
+	 * get the next tei:link value of a tei:LinkGrp.
+	 *
+	 * @param link the link
+	 * @return the next ana
+	 */
+	private String getNextAnaValue(String link, String wordId)
+	{
+		anaparser = linkparsers.get(link);
+		def m;
+		for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				if (anaparser.getLocalName().equals("link")) {
+					String targetsvalue = anaparser.getAttributeValue(0)
+					if ((m = targetsvalue =~ /#(.*) #(.*)/)) { // balise externe
+						def g1 = m[0][1];
+						def g2 = m[0][2];
+
+						String anavalue = g2;
+						anavalue = anavalue.replace("&lt;", "<")
+						return anavalue;
+					} else {
+						System.err.println("Error: getNextAna(): link target is not well formed:  = "+anaparser.getAttributeValue(0));
+					}
+				}
+			}
+		}
+		return "";
+	}
+
+	/**
+	 * build the ana tags of a word.
+	 *
+	 * @param wordId the word id
+	 * @return the ana tag
+	 */
+	private void writeAnaTag(String wordId)
+	{
+		String anabalises ="\n";
+		for (String link : links) {
+			writer.writeStartElement(TXMNS, "ana");
+			writer.writeAttribute("resp", "#"+respStmtID);
+			writer.writeAttribute("type", "#"+link);
+			if (replace) {
+				if (linkparsers.containsKey(link)) { // order is important
+					writer.writeCharacters(getNextAnaValue(link, wordId));
+				} else {
+					writer.writeCharacters(anaValues.get(link));
+				}
+			} else { // ok no problem
+				writer.writeCharacters(getNextAnaValue(link, wordId));
+			}
+			
+			writer.writeEndElement(); // txm:ana
+		}
+	}
+
+	String wordId;
+	HashMap<String, String> anaValues = new HashMap<String, String>();
+	boolean flagSourceDesc = false, flagW = false, flagAna = false;
+	String type = null, resp = null, anaValue= "";
+	protected void processStartElement()
+	{
+		if (localname.equals("taxonomy")) {
+			String taxo = parser.getAttributeValue(0) // taxonomy type
+			presentTaxonomies.add(taxo);
+		} else if (flagW && replace && localname.equals("ana")) {
+			flagAna = true
+			anaValue= ""
+			type = null
+			resp = null
+			for (int i= 0 ; i < parser.getAttributeCount() ; i++ ) {
+				if (parser.getAttributeLocalName(i) == "resp") {
+					resp = parser.getAttributeValue(i);
+				} else if (parser.getAttributeLocalName(i) == "type") {
+					type = parser.getAttributeValue(i);
+				}
+			}
+			if (type != null) type = type.substring(1);
+			if (resp != null) resp = resp.substring(1);
+			return; // don't write the "ana" start element
+		} else if (localname.equals("w")) {
+			for (int i= 0 ; i < parser.getAttributeCount() ; i++ ) {
+				if (parser.getAttributeLocalName(i) == "id") {
+					wordId = parser.getAttributeValue(i);
+					break
+				}
+			}
+			flagW = true
+			anaValues.clear()
+		}
+
+		super.processStartElement();
+	}
+
+	protected void processCharacters() {
+		if (flagAna) anaValue += parser.getText();
+		else super.processCharacters(); // FORM CONTENT LOST !!!!!!!!!!!!!
+	};
+
+	boolean applicationWritten = false;
+	boolean taxonomiesWritten = false;
+	protected void processEndElement() {
+		switch (parser.getLocalName()) {
+			case "w":
+				writeAnaTag(wordId);
+				flagW = false
+				break;
+			case "ana":
+				if (flagAna && replace && type != null && resp != null && anaValue != null) {
+					anaValues.put(type, anaValue)
+					links.add(type)
+					flagAna = false
+					return; // don't write the "ana" end element
+				}
+				flagAna = false
+				break;
+
+			case "appInfo":
+				applicationWritten = true;
+				getHeaderInfos("appInfo", false);
+				break;
+
+			case "classDecl":
+				taxonomiesWritten = true;
+				getHeaderInfos("classDecl", false);
+				break;
+
+			case "encodingDesc":
+				if (!applicationWritten) {
+					writer.writeStartElement("appInfo");
+					getHeaderInfos("appInfo", false);
+					writer.writeEndElement(); // appInfo
+				}
+				if (!taxonomiesWritten) {
+					writer.writeStartElement("classDecl");
+					getHeaderInfos("classDecl", false);
+					writer.writeEndElement(); // classDecl
+				}
+				break;
+
+			case "titleStmt":
+				if (flagSourceDesc) {
+					//output.write(this.respStmt+"\n")
+					getHeaderInfos("respStmt", true);
+					flagSourceDesc = false;
+					break;
+				}
+				break;
+		}
+		super.processEndElement();
+	}
+
+	/** The declarenamespace. */
+	boolean declarenamespace = false;
+
+	/**
+	 * Declare namespace.
+	 *
+	 * @return the java.lang. object
+	 */
+	private declareNamespace() {
+		if (!declarenamespace) {
+			writer.writeDefaultNamespace("http://www.tei-c.org/ns/1.0");
+			writer.writeNamespace("txm", TXMNS);
+			declarenamespace = true;
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir + "/injection/").mkdir();
+
+		def milestones = ["tagUsage", "pb", "lb","catRef"]// the tags who
+
+		File srcfile = new File(rootDir, "/anainline/", "roland.xml");
+		File pos1file = new File(rootDir, "/pos/", "rolandTT1-w-ana.xml");
+
+		File src2file = new File(rootDir, "/injection/", "roland.xml");
+		File pos2file = new File(rootDir, "/pos/", "rolandTT2-w-ana.xml");
+
+		println("process file : " + srcfile + " with : " + pos1file);
+		def builder = new AnnotationInjection(srcfile.toURI().toURL(),
+				pos1file.toURI().toURL(), milestones);
+		builder.transfomFile(new File(rootDir + "/injection/", "roland.xml"));
+
+		println("process file : " + src2file + " with : " + pos1file);
+		builder = new AnnotationInjection(src2file.toURI().toURL(), pos2file.toURI().toURL(),
+				milestones);
+		builder.transfomFile(rootDir + "/injection/", "roland-FINAL.xml");
+
+		return;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/BuildTTSrc.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/BuildTTSrc.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/BuildTTSrc.groovy	(revision 321)
@@ -0,0 +1,217 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.teitxm;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import javax.xml.stream.*;
+import java.net.URL;
+
+import org.txm.Toolbox;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class BuildTTSrc.
+ *
+ * @author mdecorde
+ * build the TT source for tigerSearch
+ */
+
+public class BuildTTSrc {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+	
+	/** The output. */
+	private BufferedWriter output;
+	
+	/**
+	 * Instantiates a new builds the tt src.
+	 * uses XML-TXM V2
+	 *
+	 * @param url the url of the file to process
+	 */
+	public BuildTTSrc(URL url) {
+		try {
+			this.url = url;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+			
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+	
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			File f = outfile;
+			output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile),
+					"UTF-8"));
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+	
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+     * @param formtype, if multiple form, use this param to choose the correct one, if null takes the first form found
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile, String formtype) {
+		if (!createOutput(outfile))
+			return false;
+		
+		boolean flagform = false; // to catch the content of the form tag
+		boolean firstform = false; // to know if its the first form of the w element
+		String form = ""; // the content of the form tag
+		String lastopenlocalname = "";
+		String localname = "";
+		StringBuffer buffer = new StringBuffer();
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				switch (event) {
+					case XMLStreamConstants.START_ELEMENT:
+						localname = parser.getLocalName();
+						switch (localname) {
+							case "w":
+								//firstform = true;
+								break;
+							case "form":
+//								if (firstform) {
+//									if (formtype != null) {
+//										if(parser.getAttributeCount() > 0 
+//											&& parser.getAttributeValue(0).equals(formtype)) // only one attribute in form, type
+//											flagform = true;
+//									}
+//									else
+									flagform = true;
+									form = "";
+									firstform = false;
+								//}
+								break;
+							case "s": // TreeTagger can use s tags
+								buffer.append("<s>\n");
+								break;
+						}
+						break;
+					case XMLStreamConstants.END_ELEMENT:
+						localname = parser.getLocalName();
+						switch (localname) {
+							case "form":
+								flagform = false;
+								form = form.trim()
+								if (form.length() == 0) buffer.append("__EMPTY__\n");
+								else buffer.append(form.replace("\n", "").replace("<", "&lt;")+ "\n");
+								//buffer.append(form+ "\n"); // its a txt file no need to use entities
+								break;
+							
+							case "s":
+								buffer.append("</s>\n");
+								break;
+						}
+						break;
+					
+					case XMLStreamConstants.CHARACTERS:
+						if (flagform) {
+							if (parser.getText().length() > 0)
+								form += parser.getText();
+						}
+						break;
+				}
+			}
+
+			String str = buffer.toString()
+			if ("false".equals(Toolbox.getParam(Toolbox.TREETAGGER_APOSTROPHE))) {
+				str = str.replace("’", "'").replace("‘", "'");
+			}
+			output.write(str)
+			output.close();
+			parser.close();
+			inputData.close();
+		} catch (Exception ex) {
+			System.out.println(ex);
+			return false;
+		} 
+		
+		return true;
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		
+		String rootDir = "~/xml/rgaqcj/";
+		// new File(rootDir+"/identity/").mkdir();
+		
+		ArrayList<String> milestones = new ArrayList<String>();// the tags who
+		// you want them
+		// to stay
+		// milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		milestones.add("catRef");
+		
+		File srcfile = new File(rootDir + "anainline/", "roland-p5.xml");
+		File resultfile = new File(rootDir + "ttsrc/", "roland-p5.tt");
+		println("build ttsrc file : " + srcfile + " to : " + resultfile);
+		
+		def builder = new BuildTTSrc(srcfile.toURL(), milestones);
+		builder.process(resultfile);
+		
+		return;
+	}
+	
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/Xml2Ana.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/Xml2Ana.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/Xml2Ana.groovy	(revision 321)
@@ -0,0 +1,567 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-06-14 10:33:10 +0200 (Tue, 14 Jun 2016) $
+// $LastChangedRevision: 3234 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm;
+
+import org.txm.importer.HasElement;
+import org.txm.importer.StaxIdentityParser;
+import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class Xml2Ana.
+ *
+ * @author mdecorde
+ * transform : pre xml-tei file >> xml-tei-txm file
+ * The pre xml-tei file must contains a minimal teiHeader with classDecl, encodingDesc and titleStmt
+ * 
+ * you must specify the correspondance between word attributs and ana types&respStmtIDs
+ * then the attributes of w tags will be transformed into interp tag
+ */
+public class Xml2Ana extends StaxIdentityParser
+{
+	/** The dir. */
+	private def dir;
+
+	/** The convert all attributes. */
+	private boolean convertAllAttributes = false;
+
+	/** The corresp type. */
+	HashMap<String,String> correspType;
+
+	/** The corresp ref. */
+	HashMap<String,String> correspRef;
+
+	/** The check tags. */
+	HashMap<String,Boolean> checkTags;
+
+	/** The resp id. */
+	def respId = [];
+
+	/** The applications. */
+	HashMap<String,File> applications;
+
+	/** The taxonomies. */
+	HashMap<String,String[]> taxonomies;
+
+	/** The resps. */
+	HashMap<String,String[]> resps;
+
+	/** The items. */
+	HashMap<String,HashMap<String,String>> items;
+
+	/** The XML headeradded. */
+	boolean XMLHeaderadded = false;
+	String textname;
+	String wtag = "w";
+	
+	public static final String TEXT = "text"
+	public static final String ID = "id"
+
+	/**
+	 * Instantiates a new xml2 ana.
+	 *
+	 * @param url the url
+	 * @param wordprefix the wordprefix
+	 */
+	public Xml2Ana(File file) {
+		super(file.toURI().toURL());
+		//File file = new File(url.getFile()).getAbsoluteFile()
+		textname = file.getName();
+		int idx = textname.indexOf(".");
+		if (idx > 0)
+			textname = textname.substring(0, idx)
+
+		checkTags = new HashMap<String,Boolean>();
+		checkTags.put("respStmt",false);
+		checkTags.put("titleStmt",false);
+		checkTags.put("appInfo",false);
+
+		hasText = new HasElement(file, TEXT).process();
+	}
+
+	/**
+	 * Sets the convert all atrtibutes.
+	 *
+	 * @param value the value
+	 * @return the java.lang. object
+	 */
+	public setConvertAllAtrtibutes(boolean value) {
+		convertAllAttributes = value;
+	}
+	
+	/**
+	 * Sets the convert all atrtibutes.
+	 *
+	 * @param value the value
+	 * @return the java.lang. object
+	 */
+	public setWordTag(String wtag) {
+		this.wtag = wtag
+	}
+
+	int idcount = 0;
+	boolean flagWord = false;
+	int firstElement = 0;
+	boolean teiElementAdded = false;
+	boolean teiHeaderElementAdded = false;
+	boolean hasText = false;
+	boolean textElementAdded = false;
+	def anabalises = [];
+	protected void processStartElement()
+	{
+		//String localname = parser.getLocalName();
+		firstElement++;
+		if (checkTags.containsKey(parser.getLocalName()))
+			checkTags.put(parser.getLocalName(), true);
+
+		switch (parser.getLocalName()) {
+			case wtag:
+				if (!hasText) {
+					writer.writeStartElement(TEXT);
+					writer.writeAttribute(ID, textname);
+					textElementAdded = true;
+					hasText = true;
+				}
+				idcount++; // increment word counter
+				anabalises.clear();
+
+				writer.writeStartElement(parser.getLocalName()); // write w
+
+				for (int i = 0 ; i < parser.getNamespaceCount() ; i++) // write namespaces
+					writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+
+				for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) { // transform attributes
+					String type = parser.getAttributeLocalName(i);
+					String value = parser.getAttributeValue(i);
+					if (correspType.containsKey(type)) { // check if txm:ana
+						String corresptype = correspType.get(type);
+						String ref = correspRef.get(type);
+						anabalises.add(["#"+ref, "#"+corresptype, value]);
+					} else if (type == ID) {// keep attribute id
+						String wordid = value
+						if (wordid.startsWith("w")) {
+							if (!wordid.startsWith("w_"))
+								wordid = "w_"+wordid.substring(1)
+						} 
+//						else {
+//							wordid = "w_"+textname+"_"+wordid;
+//						}
+
+						writer.writeAttribute(type, wordid);
+					} else { // add attributes that was in the original <w>
+						if (convertAllAttributes)
+							anabalises.add(["none","#"+type, value])
+						else
+							writer.writeAttribute(type, value);
+					}
+				}
+
+				flagWord = true; // start to capture the form
+				writer.writeStartElement(TXMNS, "form");
+				break;
+
+			case "TEI":
+				super.processStartElement();
+				boolean hasTeiNS = false;
+				boolean hasTXMNs = false;
+				for (int i = 0 ; i < parser.getNamespaceCount() ; i++) {
+					if (parser.getNamespaceURI(i) == TXMNS)
+						hasTXMNs = true;
+					else if (parser.getNamespaceURI(i) == TEINS)
+						hasTeiNS = true;
+				}
+				if (!hasTeiNS) {
+					writer.writeDefaultNamespace(TEINS);
+				}
+				if (!hasTXMNs)
+					writer.writeNamespace(TXM, TXMNS);
+				break;
+
+			default:
+
+				if (TEXT.equals(localname)) {
+					hasText = true;
+				}
+
+				if (firstElement == 1) { // test if first element is TEI
+					//println "first tag: "+parser.getLocalName()
+					if (localname != "TEI") { // "TEI" is missing
+						teiElementAdded = true;
+						addTEIElement();
+					} else if (!hasText) {
+						writer.writeStartElement(TEXT);
+						writer.writeAttribute(ID, textname);
+						textElementAdded = true;
+						hasText = true;
+					}
+				}
+				if (firstElement == 2 && teiElementAdded != true) {
+					//println "second tag: "+parser.getLocalName()
+					if (localname != "teiHeader") { // teiHeader is missing
+						writeTeiHeader();
+						hasTeiHeader = true
+						teiHeaderElementAdded = true
+					}
+				} else if (!hasText & (teiElementAdded | teiHeaderElementAdded)) {
+					writer.writeStartElement(TEXT);
+					writer.writeAttribute(ID, textname);
+					textElementAdded = true;
+					hasText = true;
+				}
+
+				super.processStartElement();
+				if (TEXT.equals(localname)) {
+					if (!parser.getAttributeValue(null, ID)) {
+						writer.writeAttribute(ID, textname);
+					}
+				}
+		}
+	}
+
+	protected void after()
+	{
+		if (textElementAdded) {
+			writer.writeEndElement(); // text
+		}
+		if (teiElementAdded) {
+			writer.writeEndElement(); // TEI
+		}
+		super.after(); // close writer, parser, etc
+	}
+
+	protected void addTEIElement()
+	{
+		writer.writeStartElement("TEI");
+		writer.writeDefaultNamespace(TEINS);
+		writer.writeNamespace(TXM, TXMNS);
+		writer.writeNamespace(TEI, TEINS);
+		writeTeiHeader();
+	}
+
+	protected void processCharacters()
+	{
+		if (flagWord) {
+			writer.writeCharacters(parser.getText().trim()); // keep form in 1 line
+		} else {
+			super.processCharacters();
+		}
+	}
+	
+	boolean hasClassDecl = false;
+	boolean hasFileDesc = false;
+	boolean hasEncodingDesc = false;
+	boolean hasTeiHeader = false;
+	boolean hasTEI = false;
+	public static String ANA = "ana"
+	public static String RESP = "resp"
+	public static String TYPE = "type"
+	protected void processEndElement()
+	{
+		switch (parser.getLocalName()) {
+			case wtag:
+				writer.writeEndElement(); // txm:form
+				for (def values : anabalises)
+				{// <txm:ana resp=ref type=corresptype>value</txm:ana>
+					writer.writeStartElement(TXMNS, ANA);
+					writer.writeAttribute(RESP, values[0]);
+					writer.writeAttribute(TYPE, values[1]);
+					writer.writeCharacters(values[2]);
+					writer.writeEndElement(); // txm:ana
+				}
+
+				flagWord = false;
+				break;
+
+			case "fileDesc":
+				hasFileDesc = true;
+				this.writeTXMResps();
+				break;
+
+			case "classDecl":
+				hasClassDecl=true;
+				this.writeTXMTaxonomies();
+				break;
+			case "encodingDesc":
+				hasEncodingDesc = true;
+				writeContentOfEncodingDesc();
+				break;
+
+			case "teiHeader":
+				hasTeiHeader = true
+				if (!hasEncodingDesc) {
+					writer.writeStartElement("encodingDesc");
+					writeContentOfEncodingDesc();
+					writer.writeEndElement();
+				}
+
+				break;
+			case "TEI":
+				hasTEI = true;
+				if (!hasTeiHeader) {
+					writeTeiHeader();
+				}
+				break;
+		}
+
+		super.processEndElement();
+	}
+
+	protected void writeTeiHeader()
+	{
+		writer.writeStartElement("teiHeader");
+		writer.writeStartElement("fileDesc")
+		this.writeTXMResps();
+		writer.writeStartElement("titleStmt")
+		writer.writeStartElement("title")
+		writer.writeEndElement(); // title
+		writer.writeEndElement(); // titleStmt
+		writer.writeStartElement("publicationStmt")
+		writer.writeEndElement(); // publicationStmt
+		writer.writeStartElement("sourceDesc")
+		writer.writeEndElement(); // sourceDesc
+		writer.writeEndElement(); // fileDesc
+		writer.writeStartElement("encodingDesc");
+		writeContentOfEncodingDesc();
+		writer.writeEndElement(); // encodingDesc
+		writer.writeEndElement(); // teiHeader
+	}
+
+	protected void writeContentOfEncodingDesc()
+	{
+		writer.writeStartElement("appInfo")
+		this.writeTXMApps();
+		writer.writeEndElement(); // appInfo
+		if (!hasClassDecl) {
+			writer.writeStartElement("classDecl");
+			this.writeTXMTaxonomies();
+			writer.writeEndElement(); // classDecl
+		}
+	}
+
+	/**
+	 * Check resp.
+	 *
+	 * @return the string
+	 */
+	public String checkResp()
+	{
+		String rez ="found tags : \n";
+		for (String key : checkTags)
+			rez += "\t"+key+"\n";
+		return rez;
+	}
+
+	/**
+	 * Sets the correspondances.
+	 *
+	 * @param correspRef the corresp ref
+	 * @param correspType the corresp type
+	 */
+	public void setCorrespondances(correspRef, correspType)
+	{
+		this.correspRef = correspRef;
+		this.correspType = correspType;
+	}
+
+	/**
+	 * Sets the header infos.
+	 *
+	 * @param respId the resp id
+	 * @param resps the resps
+	 * @param applications the applications
+	 * @param taxonomies the taxonomies
+	 * @param items the items
+	 */
+	public void setHeaderInfos(respId,resps, applications, taxonomies, items)
+	{
+		this.respId = respId
+		this.resps = resps
+		this.applications = applications
+		this.taxonomies = taxonomies;
+		this.items = items;
+	}
+
+	/**
+	 * Write txm resps.
+	 */
+	public void writeTXMResps()
+	{
+		for (String ref : respId) {
+			String[] infos = resps.get(ref);
+			writer.writeStartElement("respStmt");
+			writer.writeStartElement(RESP);
+			writer.writeAttribute(ID,ref);
+			writer.writeCharacters(infos[0]);
+			writer.writeStartElement("date");
+			writer.writeAttribute("when",infos[2]);
+			writer.writeCharacters(infos[3]);
+			writer.writeEndElement(); // date
+			writer.writeEndElement(); //resp
+			writer.writeStartElement("name");
+			writer.writeAttribute(TYPE, "person");
+			writer.writeCharacters(infos[1])
+			writer.writeEndElement(); // name
+			writer.writeEndElement(); //respStmt
+		}
+	}
+
+	/**
+	 * Write txm apps.
+	 */
+	public void writeTXMApps()
+	{
+		for (String ref : respId) {
+			List<String> list= applications.get(ref);
+			String ident = list.get(0);
+			String version = list.get(1);
+			File report = list.get(2);
+
+			writer.writeStartElement(TXMNS, "application");
+			writer.writeAttribute("ident", ident);
+			writer.writeAttribute("version", version);
+			writer.writeAttribute(RESP, ref);
+
+			//get txm:commandLine from GeneratedReport
+			if (report != null) {
+				writer.writeCharacters("");writer.flush();
+				Reader reader = new FileReader(report);
+				String line = reader.readLine();
+				while (line != null) {
+					if (line.length() != 0)
+						output.write(line+"\n");
+					line = reader.readLine();
+				}
+				reader.close();
+			}
+
+			writer.writeStartElement("ab");
+			writer.writeAttribute(TYPE, "annotation");
+			for (String item : taxonomies.get(ref)) {
+				writer.writeStartElement("list");
+				writer.writeEmptyElement("ref");
+				writer.writeAttribute(TYPE, "tagset");
+				writer.writeAttribute("target", item);
+				writer.writeEndElement(); // list
+			}
+			writer.writeEndElement(); // ab
+			writer.writeEndElement(); // txm:application
+		}
+	}
+
+	/**
+	 * Write txm taxonomies.
+	 */
+	public void writeTXMTaxonomies()
+	{
+		for (String tax : items.keySet()) {
+			writer.writeStartElement("taxonomy");
+			writer.writeAttribute(ID, tax);
+
+			writer.writeStartElement("bibl");
+			writer.writeAttribute(TYPE, "tagset");
+			writer.writeStartElement("title");
+			writer.writeCharacters(tax);
+			writer.writeEndElement(); // title
+
+			for (String type : items.get(tax).keySet()) {
+				writer.writeEmptyElement("ref");
+				writer.writeAttribute(TYPE, type);
+				writer.writeAttribute("target", items.get(tax).get(type));
+			}
+			writer.writeEndElement(); // bibl
+			writer.writeEndElement(); // taxonomy
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir+"anainline/").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();
+
+		String file = "roland-p5.xml";
+		String anafile = "roland-p5.xml";
+
+		def correspType = new HashMap<String,String>()
+		// correspType(attribut word wlx, attribut type de la propriété ana du w txm)
+		correspType.put("p2","CATTEX2009");
+
+		def correspRef = new HashMap<String,String>()
+		// correspRef (attribut word wlx, attribut ref de la propriété ana du w txm. ref pointe vers l'identifiant du respStmt du TEIheader)
+		correspRef.put("p2","ctx1");
+
+		//il faut lister les id de tous les respStmt
+		def respId = ["ctx1"];//,"TT1", "TnT1"];
+
+		//fait la correspondance entre le respId et le rapport d'execution de l'outil
+		def applications = new HashMap<String,HashMap<String,String>>();
+		applications.put("ctx1",new ArrayList<String>());
+		applications.get("ctx1").add("Oxygen");//app ident
+		applications.get("ctx1").add("9.3");//app version
+		applications.get("ctx1").add(null);//app report file path
+
+		//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm
+		//pour construire les ref vers les taxonomies
+		def taxonomiesUtilisees = new HashMap<String,String[]>();
+		taxonomiesUtilisees.put("ctx1",["CATTEX2009"]);//,"lemma","lasla","grace"]);
+
+		//associe un id d'item avec sa description et son URI
+		def itemsURI = new HashMap<String,HashMap<String,String>>();
+		itemsURI.put("CATTEX2009",new HashMap<String,String>());
+		itemsURI.get("CATTEX2009").put("tagset","http://bfm.ens-lsh.fr/IMG/xml/cattex2009.xml");
+		itemsURI.get("CATTEX2009").put("website","http://bfm.ens-lsh.fr/article.php3?id_article=176");
+
+		//informations de respStmt
+		//resps (respId <voir ci-dessus>, [description, person, date])
+		def resps = new HashMap<String,String[]>();
+		resps.put("ctx1", ["initial tagging","alavrentiev","2010-03-02","Tue Mar  2 21:02:55 Paris, Madrid 2010"])
+
+		//lance le traitement
+		def builder = new Xml2Ana(new File(rootDir+"/src/",file));
+		builder.setCorrespondances(correspRef, correspType);
+		builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)
+		//dossier de sortie + nom fichier sortie
+		builder.process(anafile);
+
+		return
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/CSV2W_ANA.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/CSV2W_ANA.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/CSV2W_ANA.groovy	(revision 321)
@@ -0,0 +1,491 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-03-29 09:51:35 +0200 (Tue, 29 Mar 2016) $
+// $LastChangedRevision: 3185 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm;
+
+import java.io.File;
+import java.io.Writer;
+
+import org.txm.importer.graal.PersonalNamespaceContext;
+import org.txm.utils.treetagger.TreeTagger;
+import java.util.Iterator;
+
+import javax.xml.namespace.NamespaceContext;
+
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+
+import javax.xml.stream.*;
+// TODO: Auto-generated Javadoc
+/**  
+ * @author mdecorde  
+ * 
+ * Take a CVS file with columns [word, pos1, pos2, ...] Build xml-pos With given infos : - setAnnotationTypes( types, idform); - setResp( person, software, cmdLine);  types : list of type [pos1, pos2, ...] idform : the ___ of the words' id person : the resp software : the software which creates annotations cmdline : the used cmd line 
+ * 
+ */
+class CSV2W_ANA {
+	String encoding;
+
+	/** The target. */
+	String target = "" // ana file targeted
+
+	/** The distributor. */
+	String distributor= "";
+
+	/** The publi chat. */
+	String publiChat= "";
+
+	/** The source chat. */
+	String sourceChat= "";
+
+	/** The reportfile. */
+	File reportfile;
+
+	/** The resp person. */
+	String respPerson; // used for respStmt/resp/name(person)
+
+	/** The resp id. */
+	String respId;
+
+	/** The resp chat. */
+	String respChat;
+
+	/** The resp date. */
+	String respDate;
+
+	/** The resp when. */
+	String respWhen;
+
+	/** The app ident. */
+	String appIdent;
+
+	/** The app version. */
+	String appVersion;
+
+	/** The types. */
+	def types ;// used for linkGrp type
+
+	/** The types desc. */
+	def typesDesc ;// used for types' description
+
+	/** The types uri. */
+	def typesURI; // used for types' URI
+
+	/** The types web. */
+	def typesWEB;//used for types siteweb
+
+	/** The idform. */
+	String idform ="";
+
+	/** The writer. */
+	XMLOutputFactory factory = XMLOutputFactory.newInstance();
+	BufferedOutputStream output;
+	XMLStreamWriter writer;
+
+	public static String TXMNS = "http://textometrie.org/1.0";
+	public static String TEINS = "http://www.tei-c.org/ns/1.0";
+
+	/**
+	 * Process.
+	 *
+	 * @param TTrez the tt result file
+	 * @param xmlform the xmlform the standoff file created
+	 * @param encoding the encoding of the result file
+	 * @return true, if successful
+	 */
+	public boolean process(File TTrez,File xmlform, String encoding )
+	{
+		try {
+			this.encoding = encoding;
+			output = new BufferedOutputStream(new FileOutputStream(xmlform));
+			writer = factory.createXMLStreamWriter(output, "UTF-8"); // create a new file
+			writer.setNamespaceContext(new PersonalNamespaceContext());
+			writeHead();
+			writeBody(TTrez,encoding)
+			writeTail();
+			writer.close();
+			output.close();
+		} catch(Exception e) {
+			e.printStackTrace();
+			writer.close();
+			output.close();
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Write head.
+	 */
+	private void writeHead()
+	{
+		Charset UTF8 = Charset.forName("UTF8");
+		writer.writeStartDocument(encoding, "1.0");
+		writer.writeStartElement("TEI");
+		writer.writeDefaultNamespace(TEINS);
+		writer.writeNamespace("txm", TXMNS);
+		writer.writeStartElement("teiHeader");
+		writer.writeStartElement("fileDesc");
+		writer.writeStartElement("titleStmt");
+		writer.writeStartElement("title");
+		writer.writeCharacters("Standoff annotation file for "+target);
+		writer.writeEndElement(); // title
+		writer.writeStartElement("respStmt");
+		writer.writeAttribute("xml:id", respId);
+		writer.writeStartElement("resp");
+		writer.writeCharacters(respChat)
+		writer.writeStartElement("date");
+		writer.writeAttribute("when", respWhen);
+		writer.writeCharacters(respDate);
+		writer.writeEndElement(); // date
+		writer.writeEndElement(); // resp
+		writer.writeStartElement("name");
+		writer.writeAttribute("type", "person");
+		writer.writeCharacters(respPerson+"\n");
+		writer.writeEndElement(); // name
+		writer.writeEndElement(); // respStmt
+		writer.writeEndElement(); // titleStmt
+		writer.writeStartElement("publicationStmt")
+		writer.writeCharacters("\n");
+		writer.writeStartElement("distributor");
+		writer.writeCharacters("\n");
+		writer.flush();
+		output.write(distributor.getBytes(UTF8))
+		output.flush()
+		writer.writeEndElement(); // distributor
+		writer.writeCharacters("\n");
+		writer.flush();
+		output.write(publiChat.getBytes(UTF8));
+		output.flush()
+		writer.writeEndElement(); // publicationStmt
+		writer.writeStartElement("sourceDesc");
+		writer.writeCharacters("\n");
+		writer.flush();
+		output.write(sourceChat.getBytes(UTF8));
+		output.flush()
+		writer.writeEndElement(); // sourceDesc
+		writer.writeEndElement(); // fileDesc
+		writer.writeStartElement("encodingDesc");
+		writer.writeStartElement("appInfo");
+		writer.writeStartElement(TXMNS,"application");
+		writer.writeAttribute("ident",appIdent);
+		writer.writeAttribute("version", appVersion);
+		writer.writeAttribute("resp", respId);
+		writer.writeCharacters("\n");
+		writer.flush();
+		def report = this.reportfile;
+		if (report != null) {
+			//System.out.println("read report : "+report);
+			Reader reader = new FileReader(report);
+			String line = reader.readLine();
+			while (line != null) {
+				if (line.length() != 0)
+					output.write((line.replace("&","&amp;")+"\n").getBytes(UTF8));
+				line = reader.readLine();
+			}
+			reader.close();
+			output.flush()
+		} else {
+			writer.writeEmptyElement(TXMNS, "commandLine");
+			writer.writeCharacters("\n");
+		}
+
+		writer.writeStartElement("ab");
+		writer.writeAttribute("type", "annotation");
+		writer.writeCharacters("\n");
+		writer.writeStartElement("list");
+		writer.writeCharacters("\n");
+		for (int i = 0; i < types.size(); i++) {
+			writer.writeStartElement("item");
+			writer.writeEmptyElement("ref");
+			writer.writeAttribute("type", "tagset");
+			writer.writeAttribute("target","#"+types[i])
+			writer.writeEndElement(); // item
+			writer.writeCharacters("\n");
+		}
+		writer.writeEndElement(); // list
+		writer.writeEndElement(); // ab
+		writer.writeEndElement(); // txm:application
+		writer.writeEndElement(); // appInfo
+		writer.writeStartElement("classDecl");
+		for (int i = 0; i < types.size(); i++) {
+			writer.writeStartElement("taxonomy");
+			writer.writeAttribute("xml:id", types[i]);
+			writer.writeStartElement("bibl");
+			writer.writeStartElement("title");
+			writer.writeCharacters(typesDesc[i]);
+			writer.writeEndElement(); // title
+			if (!typesURI[i].equals("")) {
+				writer.writeEmptyElement("ref");
+				writer.writeAttribute("type", "tagset");
+				writer.writeAttribute("target", typesURI[i]);
+				writer.writeCharacters("\n");
+			}
+			if (!typesWEB[i].equals("")) {
+				writer.writeEmptyElement("ref");
+				writer.writeAttribute("type", "siteweb");
+				writer.writeAttribute("target", typesWEB[i]);
+				writer.writeCharacters("\n");
+			}
+			writer.writeEndElement(); // bibl
+			writer.writeCharacters("\n");
+			writer.writeEndElement(); // taxonomy
+			writer.writeCharacters("\n");
+		}
+		writer.writeEndElement(); // classDecl
+		writer.writeCharacters("\n");
+		writer.writeEndElement(); // encodingDesc
+		writer.writeCharacters("\n");
+		writer.writeEndElement(); // teiHeader
+
+		writer.writeStartElement("text");
+		writer.writeAttribute("type", "standoff");
+		writer.writeStartElement("body");
+		writer.writeCharacters("\n");
+		writer.writeStartElement("div");
+		writer.writeCharacters("");
+	}
+
+	/**
+	 * Write body.
+	 *
+	 * @param TTrez the tt result file
+	 * @param encoding the encoding
+	 * @return the java.lang. object
+	 */
+	private writeBody(File TTrez, String encoding) {
+		String tmp = "";
+		String targets= idform+"";
+
+		//def content = TTrez.getText(encoding)
+		//content = content.replace("&lt;", "<")
+
+		def separator= "\t"
+		def maxsize = types.size()+1;
+		for (int i = 0 ; i< types.size() ; i++) {
+			int id = 1;
+			//output.write("<linkGrp type=\""+types[i]+"\">\n");
+			writer.writeStartElement("linkGrp");
+			writer.writeAttribute("type", types[i]);
+			TTrez.eachLine("UTF-8") { line ->
+				line = line.replaceAll("&lt;", "<")
+				def fields = line.split(separator);
+				//content.splitEachLine(separator) {fields ->
+					if (fields.size() == maxsize)
+						if (!(fields[0].startsWith("<s>") || fields[0].startsWith("</s>")))// don't use the sentence tags
+					{
+						writer.writeEmptyElement("link");
+						writer.writeAttribute("targets", "#"+targets+id+" #"+ fields[i+1]);
+						id++;
+						writer.writeCharacters("\n");
+					}
+				//}
+			}
+			writer.writeEndElement(); // linkGrp
+		}
+	}
+
+	/**
+	 * Write tail.
+	 */
+	private void writeTail() {
+		writer.writeEndElement(); // div
+		writer.writeEndElement(); // body
+		writer.writeEndElement(); // text
+		writer.writeEndElement(); // TEI
+	}
+
+	/**
+	 * Sets the annotation types.
+	 *
+	 * @param types the types
+	 * @param typesDesc the types desc
+	 * @param typesURI the types uri
+	 * @param typesWEB the types web
+	 * @param idform the idform
+	 */
+	public void setAnnotationTypes(def types,def typesDesc,def typesURI,def typesWEB, String idform) {
+		this.types = types;
+		this.typesDesc = typesDesc;
+		this.typesURI = typesURI;
+		this.typesWEB = typesWEB;
+		this.idform = idform;
+	}
+
+	/**
+	 * Sets the resp.
+	 *
+	 * @param respId the resp id
+	 * @param respChat the resp chat
+	 * @param respDate the resp date
+	 * @param respPerson the resp person
+	 * @param respWhen the resp when
+	 */
+	public void setResp(respId, respChat,respDate, respPerson,respWhen) {
+		this.respId = respId;
+		this.respChat = respChat;
+		this.respDate = respDate
+		this.respPerson = respPerson;
+		this.respWhen = respWhen;
+	}
+
+	/**
+	 * Sets the target.
+	 *
+	 * @param target the target
+	 * @param reportFile the report file
+	 */
+	public void setTarget(target, reportFile) {
+		this.target = target;
+		this.reportfile = reportFile;
+	}
+
+	/**
+	 * Sets the app.
+	 *
+	 * @param appIdent the app ident
+	 * @param appVersion the app version
+	 */
+	public void setApp(String appIdent, String appVersion) {
+		this.appIdent = appIdent;
+		this.appVersion = appVersion;
+	}
+
+	/**
+	 * Sets the infos.
+	 *
+	 * @param distributor the distributor
+	 * @param publiChat the publi chat
+	 * @param sourceChat the source chat
+	 */
+	public void setInfos(distributor,  publiChat, sourceChat) {
+		this.distributor = distributor;
+		this.publiChat = publiChat;
+		this.sourceChat = sourceChat;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir,"proj").mkdir();
+
+		//targeted file for annotations
+		String target = "roland.xml"
+		//contains txm:application/txm:commandLine
+		File reportFile = new File("~/xml/rgaqcj/tt1_REPORT.xml");
+
+		String respPerson ="slh";
+		String respId = "TT1";
+		String respDesc = "NLP annotation tool";
+		String respDate = "Tue Mar  11 1:02:55 Paris, Madrid 2010";
+		String respWhen="2010-03-11"
+
+		String appIdent = "TreeTagger";
+		String appVersion = "3.2";
+
+		String distributor = "BFM project - http://bfm.ens-lsh.fr";
+		String publiStmt = """<availability>
+			  <p>(c) 2010 Projet BFM - CNRS/ENS-LSH.
+			  <hi>Conditions d'utilisation</hi> : 
+			  Sous licence <ref target="http://creativecommons.org/licenses/by-sa/2.0/fr/">Creative Commons</ref>.
+			  </p>
+			</availability>""";
+
+		String sourceStmt = """<p>Born digital: TXM project - http://textometrie.org</p>
+			<p>Automatically generated on <date when=" """+respWhen+"""">"""+respDate+"""</date>
+			from <ref target="#"""+respId+"""">Tree Tagger</ref> applied on the text roland-tt.
+			</p>""";
+
+		def types = ["CATTEX2009"];
+		def typesTITLE = ["CATTEX2009"];
+		def typesDesc = ["CATTEX2009 POS description"]
+		def typesTAGSET = ["http://bfm.ens-lsh.fr/IMG/xml/cattex2009.xml"]
+		def typesWEB = ["http://bfm.ens-lsh.fr/article.php3?id_article=176"]
+		String idform ="w_fro_";
+
+		File ttfile = new File(rootDir+"/proj/","rol_rga.tt");
+		File posfile = new File(rootDir+"/pos/","rolandTT1-w-ana.xml");
+		String encoding ="UTF-8";
+		println("Process file : "+ttfile+" to : "+posfile );
+
+		def transfo = new CSV2W_ANA();
+		transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+		transfo.setResp(respId, respDesc,respDate, respPerson, respWhen);
+		transfo.setApp(appIdent, appVersion);
+		transfo.setTarget(target, reportFile);
+		transfo.setInfos(distributor,  publiStmt, sourceStmt);
+		transfo.process( ttfile, posfile, encoding );
+
+		//create annotation file for afrpar on roalnd.xml
+		String infile = rootDir+"/ttsrc/roland-learn.tt";
+		String modelfile =  rootDir+"/models/afr.par";
+		String outfile = rootDir+"/proj/afr_rol.tt";
+
+		println("proj "+modelfile+ " on " +infile +" >> "+outfile);
+
+		TreeTagger tt = new TreeTagger("~/Bureau/treetagger/bin/");
+		tt.settoken();
+		tt.setquiet();
+		tt.setsgml();
+		tt.seteostag("<s>");
+		tt.treetagger( modelfile, infile, outfile)
+
+		reportFile = new File("~/xml/rgaqcj/tt2_REPORT.xml");
+		respPerson ="slh";
+		respId = "TT2";
+		respDesc = "NLP annotation tool";
+		respDate = "Tue Mar  15 1:02:55 Paris, Madrid 2010";
+		respWhen = "2010-03-15"
+
+		types = ["AFRPOS"];
+		typesDesc = ["AFRPOS description"]
+		typesWEB = ["http://www.uni-stuttgart.de/lingrom/stein/forschung/resource.html"]
+		typesTAGSET = [""]
+		idform ="w_fro_";
+
+		ttfile = new File(rootDir+"/proj/","afr_rol.tt");
+		posfile = new File(rootDir+"/pos/","rolandTT2-w-ana.xml");
+		encoding ="UTF-8";
+		println("Process file : "+ttfile+" to : "+posfile );
+
+		transfo = new CSV2W_ANA();
+		transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+		transfo.setResp(respId, respDesc,respDate, respPerson,respWhen);
+		transfo.setTarget(target, reportFile);
+		transfo.setApp(appIdent, appVersion);
+		transfo.setInfos(distributor,  publiStmt, sourceStmt);
+		transfo.process( ttfile, posfile, encoding );
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotateWTC.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotateWTC.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/AnnotateWTC.groovy	(revision 321)
@@ -0,0 +1,172 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2013-12-06 16:57:35 +0100 (Fri, 06 Dec 2013) $
+// $LastChangedRevision: 2583 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm
+
+import org.txm.Toolbox
+import org.txm.importer.*
+import org.txm.importer.cwb.*
+import org.txm.objects.*
+import org.txm.stat.utils.ConsoleProgressBar
+import org.txm.utils.treetagger.TreeTagger
+
+// TODO: Auto-generated Javadoc
+/**
+ * Annotate and replace the TEI-TXM files of the folder $rootDirFile/txm with TreeTagger.
+ * creates $rootDirFile/interp and $rootDirFile/treetagger
+ *
+ */
+class AnnotateWTC {
+
+	/** The debug. */
+	boolean debug = true;
+
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug() { debug=true; }
+
+	String id;
+
+	/**
+	 * Apply tt.
+	 *
+	 * @param ttsrcfile the ttsrcfile
+	 * @param ttoutfile the ttoutfile
+	 * @param modelfile the modelfile
+	 * @return true, if successful
+	 */
+	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile)
+	{
+		try {
+			File infile = ttsrcfile;
+			File outfile = ttoutfile;
+
+			// TODO: why org.txm.utils.treetagger.TreeTagger tt = ..., throw IllegalAccessError ???
+			String opt = Toolbox.getParam(Toolbox.TREETAGGER_OPTIONS);
+			if (opt == null) opt = "";
+			String[] options = opt.split("  ");
+			def tt = new TreeTagger(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/", options);
+			tt.settoken();
+			tt.setlemma();
+			tt.setquiet();
+			tt.setsgml();
+			tt.setnounknown();
+			tt.seteostag("<s>");
+			tt.treetagger(modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())
+			//infile.delete();
+		}
+		catch(Exception e)
+		{
+			e.printStackTrace();
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Run.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param modelfilename the modelfilename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir,  String modelfilename)
+	{
+		//test if modelfile exists
+		if(debug) {
+			println "rootDirFile "+binDir
+			println "txmDir "+txmDir
+			println "TREETAGGER INSTALL PATH : "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)
+			println "TREETAGGER MODELS PATH : "+Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH)
+		}
+
+		//test if the Toolbox know TreeTagger
+		if (!new File(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/").exists())
+		{
+			println("Could not find TreeTagger binaries in "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/")
+			return false;
+		}
+
+		//cleaning
+		File annotDir = new File(binDir, "annotations")
+		annotDir.mkdir();
+
+		//BUILD TT FILE READY TO BE TAGGED
+		//println "Re "+Toolbox.getParam(Toolbox.TREETAGGER_APOSTROPHE)
+		List<File> files = txmDir.listFiles()
+
+		// get model file and check it
+		File modelfile = new File(Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH), modelfilename);
+		if (debug)
+			println "model file : "+modelfile;
+
+		if (!modelfile.exists()) {
+			println "Skipping ANNOTATE: Incorrect modelfile path: "+modelfile;
+			if(System.getProperty("os.name").startsWith("Windows"))
+				println "Windows users: Windows might be hiding files extension. To see them, in the explorer parameters."
+			return false;
+		}
+
+		//APPLY TREETAGGER
+		println("Applying $modelfilename TreeTagger model on dir: "+txmDir+ " ("+files.size()+" files)")
+		if (files == null || files.size() == 0)
+			return false;
+
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size());
+		for (File f : files) {
+			cpb.tick();
+			
+			File infile = f;
+			File outfile = new File(new File(binDir, "annotations"), f.getName());
+			if (outfile.exists() && // outfile exists
+				outfile.lastModified() >= infile.lastModified() && // outfile is more recent 
+				outfile.length() > infile.length()) { // outfile is bigger
+				// skip
+			} else {
+				if (!applyTT(infile, outfile, modelfile)) {
+					System.out.println("Failed to apply treetagger on file "+f);
+					return false;
+				}
+			}
+		}
+		println("")
+
+//		if (DeleteDir.deleteDirectory(txmDir)) {
+//			if (annotDir.renameTo(txmDir)) {
+//
+//			} else {
+//				println "Could not rename  'annotations' directory to 'txm' directory"
+//				return false
+//			}
+//		} else {
+//			println "Could not delete txmDir: $txmDir (and could not copy 'annotations' directory"
+//		}
+
+		return true;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/WLX2TXMTEI.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/WLX2TXMTEI.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/WLX2TXMTEI.groovy	(revision 321)
@@ -0,0 +1,286 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm;
+
+import java.util.ArrayList;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class AnnotationInliner.
+ *
+ * @author mdecorde build a xml-tei-txm from a weblex xml file
+ * @deprecated
+ */
+
+public class WLX2TXMTEI {
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private def parser;
+
+	/** The dir. */
+	private def dir;
+
+	/** The output. */
+	private def output;
+
+	/** The url. */
+	private def url;
+
+	/** The solotags. */
+	ArrayList<String> solotags;
+
+	/**
+	 * initialize.
+	 *
+	 * @param url the url to the file to transform
+	 * @param solotags temporary list of milestones tags
+	 */
+	public WLX2TXMTEI(URL url, ArrayList<String> solotags) {
+		try {
+			this.url = url;
+			this.solotags = solotags;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			parser = factory.createXMLStreamReader(inputData);
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * create output FileWriter.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	private boolean createOutput(String dirPathName, String fileName){
+		try {
+			dir = new File(dirPathName)
+			File f = new File(dir, fileName);
+			output = new java.io.FileWriter(f)
+
+			return true;
+		} catch (Exception e) {
+			System.out.println("la?"+e);
+			return false;
+		}
+	}
+
+	/**
+	 * process !!!.
+	 *
+	 * @param dirPathName the dir path name
+	 * @param fileName the file name
+	 * @return true, if successful
+	 */
+	public boolean transfomFile(String dirPathName, String fileName){
+		if(createOutput(dirPathName, fileName)){
+
+			int idPb = 0;
+			def idLb = "";
+			String idLinesuiv;
+			int charcount=0;
+			String lastopenlocalname= "";
+
+			boolean flagWord = false;
+			boolean flagNorm = false;
+			String vWordNorm = "";
+			String anabalises = "";
+			String vP2 = "";
+			int idcount=1;
+			String targets= "w_fro_"
+
+			try {
+				File inputfile = new File(this.url.getFile());
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+					switch (event) {
+						case XMLStreamConstants.START_ELEMENT:
+							switch (parser.getLocalName()) {
+								case "w":
+								output.write("\n<w xml:id=\""+targets+idcount+"\"");
+								idcount++;
+								anabalises = "";
+								for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+								{
+									switch(parser.getAttributeLocalName(i))
+									{
+										case "p2":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#t1\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p3":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#lemma\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p4":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#lasla\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p5":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#grace\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p6":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#t5\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p7":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p8":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p9":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p10":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p11":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p12":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										case "p13":
+											anabalises += "<txm:ana ref=\"#TT1\" type=\"#tt\">" + parser.getAttributeValue(i)+"</txm:ana>\n"
+											break
+										default:
+											output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"" );
+									}
+								}
+
+								output.write(">\n");
+								flagWord = true;
+								break;
+
+								case "TEI":
+								output.write("\n<TEI");
+								for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+									output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"" );
+								output.write(" xmlns:txm=\"http://textometrie.ens-lyon.fr/1.0\"" );
+								output.write(">\n");
+								break;
+								default:
+								lastopenlocalname = parser.getLocalName();
+								output.write("\n<"+parser.getLocalName());
+								for(int i= 0 ; i < parser.getAttributeCount() ;i++ )
+									output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i)+"\"" );
+								if(solotags.contains(lastopenlocalname))
+									output.write("/>");
+								else
+									output.write(">");
+								charcount=0;
+							}
+							break;
+
+						case XMLStreamConstants.END_ELEMENT:
+							switch (parser.getLocalName()) {
+								case "w":
+								output.write("<txm:form>"+vWordNorm+"</txm:form>")
+								output.write("\n"+anabalises);
+								output.write("</w>");
+								vWordNorm = "";
+								anabalises ="";
+								flagWord = false;
+								break;
+								default:
+								//if(charcount > 0)
+								//output.write("\n");
+								if(!solotags.contains(parser.getLocalName()))
+									if(lastopenlocalname.equals(parser.getLocalName()))
+										output.write("</"+parser.getLocalName()+">");
+									else
+										output.write("\n</"+parser.getLocalName()+">");
+							}
+							break;
+
+						case XMLStreamConstants.CHARACTERS:
+							if (flagWord) {
+
+								vWordNorm += parser.getText().trim();
+
+							} else
+							{
+								String txt = parser.getText().trim();
+								output.write(txt);
+								charcount += txt.length();
+							}
+							break;
+					}
+				}
+
+				output.close();
+				parser.close();
+			}
+			catch (XMLStreamException ex) {
+				System.out.println(ex);
+			}
+			catch (IOException ex) {
+				System.out.println("IOException while parsing " + inputData);
+			}
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir+"/anainline/").mkdir();
+
+		def files = ["roland.xml","qgraal_cm.xml","artu.xml","qjm.xml","commyn1.xml","jehpar.xml"];
+		def anafiles = ["roland-ana.xml","qgraal_cm-ana.xml","artu-ana.xml","qjm-ana.xml","commyn1-ana.xml","jehpar-ana.xml"];
+		ArrayList<String> solotags = new ArrayList<String>();
+		solotags.add("tagUsage");
+		solotags.add("pb");
+		solotags.add("lb");
+		solotags.add("catRef");
+
+		for(int i=0; i < files.size();i++)
+		{
+			String file = files[i];
+			String anafile = anafiles[i];
+			def builder = new WLX2TXMTEI(new File(rootDir+"/src/",file).toURL(),solotags);
+			builder.transfomFile(rootDir+"/anainline/",anafile);
+		}
+
+
+		return
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/InlineCleaner.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/InlineCleaner.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/InlineCleaner.groovy	(revision 321)
@@ -0,0 +1,205 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-03-29 09:51:35 +0200 (Tue, 29 Mar 2016) $
+// $LastChangedRevision: 3185 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.teitxm;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class InlineCleaner.
+ *
+ * @author mdecorde
+ * remove all ana tags of a xml-tei-txm file it supose you have
+ * the stand-off version
+ */
+
+public class InlineCleaner {
+	
+	/** The url. */
+	private def url;
+	
+	/** The input data. */
+	private def inputData;
+	
+	/** The factory. */
+	private def factory;
+	
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The output. */
+	private def output;
+	
+	/** The solotags. */
+	ArrayList<String> solotags;
+
+	/**
+	 * Instantiates a new inline cleaner.
+	 *
+	 * @param url the url
+	 * @param outfile the outfile
+	 * @param solotags the solotags
+	 */
+	public InlineCleaner(URL url, File outfile, ArrayList<String> solotags) {
+		try {
+			this.url = url;
+			this.solotags = solotags;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+
+			parser = factory.createXMLStreamReader(inputData);
+
+			this.transformFile(outfile);
+
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		} catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		output = new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8");
+		return true;
+	}
+
+	/**
+	 * Transform file.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	public boolean transformFile(File outfile) {
+		boolean shouldwrite = true;
+		boolean isW = false;
+		String lastopenlocalname = "";
+
+		if (!createOutput(outfile))
+			return false;
+
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser
+				.next()) {
+			String prefix = parser.getPrefix();
+			if (prefix == null)
+				prefix = "";
+			else
+				prefix += ":";
+
+			switch (event) {
+			case XMLStreamConstants.START_ELEMENT:
+				
+				if (parser.getLocalName().equals("w")) {
+					isW = true;
+				} 
+				
+				if (parser.getLocalName().equals("ana") && isW) {
+					shouldwrite = false;
+				} else {
+					lastopenlocalname = parser.getLocalName();
+					output.write("\n<" + prefix + parser.getLocalName());
+					if (parser.getLocalName().equals("TEI"))
+						output
+								.write(" xmlns:txm=\"http://textometrie.ens-lyon.fr/1.0\"");
+					for (int i = 0; i < parser.getAttributeCount(); i++) {
+						String attname = parser.getAttributeLocalName(i);
+						output.write(" " + attname + "=\""
+								+ parser.getAttributeValue(i) + "\"");
+					}
+					if (solotags.contains(lastopenlocalname))
+						output.write("/>");
+					else
+						output.write(">");
+				}
+				break;
+			case XMLStreamConstants.END_ELEMENT:
+				switch (parser.getLocalName()) {
+				case "ana":
+					if(isW)
+						shouldwrite = true;
+					break;
+				case "w":
+					isW = false;
+					//do default
+				default:
+					if (!solotags.contains(parser.getLocalName()))
+						if (lastopenlocalname.equals(parser.getLocalName()))
+							output.write("</" + prefix + parser.getLocalName()+ ">");
+						else
+							output.write("\n</" + prefix
+									+ parser.getLocalName() + ">");
+				}
+				break;
+
+			case XMLStreamConstants.CHARACTERS:
+				if (shouldwrite) {
+					String txt = parser.getText().trim();
+					output.write(txt);
+				}
+				break;
+			}
+		}
+		output.close();
+		parser.close();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir + "/cleaner/").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();// the tags who stay milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+		milestones.add("catRef");
+
+		File srcfile = new File(rootDir + "/anainline/", "roland-ana.xml");
+		File cleanfile = new File(rootDir + "/cleaner/", "roland-off.xml");
+
+		System.out.println("clean file " + srcfile);
+		def builder = new InlineCleaner(srcfile.toURL(), cleanfile, milestones);
+		return;
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/Annotate.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/Annotate.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/teitxm/Annotate.groovy	(revision 321)
@@ -0,0 +1,550 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-10-03 15:30:36 +0200 (Mon, 03 Oct 2016) $
+// $LastChangedRevision: 3313 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.scripts.teitxm
+
+import java.text.DateFormat
+import java.util.concurrent.*
+
+import org.txm.Toolbox
+import org.txm.importer.*
+import org.txm.importer.cwb.*
+import org.txm.objects.*
+import org.txm.stat.utils.ConsoleProgressBar
+import org.txm.utils.LangDetector
+import org.txm.utils.logger.Log
+import org.txm.utils.treetagger.TreeTagger
+
+// TODO: Auto-generated Javadoc
+/**
+ * Annotate and replace the TEI-TXM files of the folder $rootDirFile/txm with TreeTagger.
+ * creates $rootDirFile/interp and $rootDirFile/treetagger
+ *
+ */
+class Annotate
+{
+	boolean cancelNow = false;
+
+	/** The report file. */
+	File reportFile;//contains the txm:application tag content
+
+	/** The resp person. */
+	String respPerson;
+
+	/** The resp id. */
+	String respId;
+
+	/** The resp desc. */
+	String respDesc;
+
+	/** The resp date. */
+	String respDate;
+
+	/** The resp when. */
+	String respWhen;
+
+	/** The app ident. */
+	String appIdent;
+
+	/** The app version. */
+	String appVersion;
+
+	/** The distributor. */
+	String distributor;
+
+	/** The publi stmt. */
+	String publiStmt;
+
+	/** The source stmt. */
+	String sourceStmt;
+
+	/** The types. */
+	def types;
+
+	/** The types title. */
+	def typesTITLE;
+
+	/** The types desc. */
+	def typesDesc;
+
+	/** The types tagset. */
+	def typesTAGSET;
+
+	/** The types web. */
+	def typesWEB;
+
+	/** The idform. */
+	String idform;
+
+	/** The debug. */
+	boolean debug = false;
+
+	File modelsDirectory;
+	
+	public Annotate() {
+		modelsDirectory = new File(Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH)); // default models directory is set in the Toolbox
+	}
+		
+	/**
+	 * Sets the debug.
+	 */
+	public void setDebug() {
+		debug = true;
+	}
+
+	String id;
+	/**
+	 * Inits the tt outfile infos.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param modelfile the modelfile
+	 */
+	public void initTTOutfileInfos(File rootDirFile, File modelfile, String modelfilename)
+	{
+		initTTOutfileInfos(rootDirFile, modelfile, modelfilename, null);
+	}
+	
+	/**
+	 * Inits the tt outfile infos.
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param modelfile the modelfile
+	 * @param properties : 2 element array that contains the word properties to create. It can be null (the modelfilename will be used)
+	 */
+	public void initTTOutfileInfos(File rootDirFile, File modelfile, String modelfilename, String[] properties)
+	{
+		id = modelfilename;
+		String[] split = id.split("\\.");
+		if (split.length > 0) id = split[0];
+		if (id.equals("??")) id = "xx"
+
+		reportFile = new File(rootDirFile,"NLPToolsParameters.xml");
+
+		respPerson = System.getProperty("user.name");
+		respId = "txm";
+		respDesc = "NLP annotation tool";
+		respDate = DateFormat.getDateInstance(DateFormat.SHORT, Locale.UK).format(new Date());
+		respWhen = DateFormat.getDateInstance(DateFormat.FULL, Locale.UK).format(new Date());
+
+		appIdent = "TreeTagger";
+		appVersion = "3.2";
+
+		distributor = "";
+		publiStmt = """""";
+		sourceStmt = """""";
+
+		if (properties != null && properties.length == 2) {
+			types = [properties[0],properties[1]];
+			typesTITLE = [properties[0],properties[1]];
+		} else {
+			types = [id+"pos",id+"lemma"];
+			typesTITLE = [id+"pos",id+"lemma"];
+		}
+		
+		//TODO: the tagset, website and description should be referenced in the model catalog
+		if(modelfile.getName() == "rgaqcj.par") {
+			typesDesc = ["CATTEX pos tagset built with BFM texts","fr lemma of the model "+modelfile+" - "]
+			typesTAGSET = ["http://bfm.ens-lyon.fr/IMG/pdf/Cattex2009_Manuel.pdf",""]
+			typesWEB = ["http://bfm.ens-lyon.fr/",""]
+		} else {
+			typesDesc = ["pos tagset built from model "+modelfile,id+" lemma of the model "+modelfile+" - "]
+			typesTAGSET = ["",""]
+			typesWEB = ["",""]
+		}
+		
+		idform ="w";
+	}
+
+	/**
+	 * Apply tt.
+	 *
+	 * @param ttsrcfile the ttsrcfile
+	 * @param ttoutfile the ttoutfile
+	 * @param modelfile the modelfile
+	 * @return true, if successful
+	 */
+	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile)
+	{
+		applyTT(ttsrcfile, ttoutfile, modelfile, null)
+	}
+	
+	/**
+	 * Apply tt.
+	 *
+	 * @param ttsrcfile the ttsrcfile
+	 * @param ttoutfile the ttoutfile
+	 * @param modelfile the modelfile
+	 * @param options, if null use value set in Toolbox preferences
+	 * @return true, if successful
+	 */
+	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile, String[] options)
+	{
+		try {
+			File infile = ttsrcfile;
+			File outfile = ttoutfile;
+
+			// TODO: why org.txm.utils.treetagger.TreeTagger tt = ..., throw IllegalAccessError ???
+			String opt = Toolbox.getParam(Toolbox.TREETAGGER_OPTIONS);
+			if (options == null) {
+				if (opt == null) opt = "";
+				options = opt.split("  ");
+			}
+			
+			def tt = new TreeTagger(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/", options);
+			tt.settoken();
+			tt.setlemma();
+			tt.setquiet();
+			tt.setsgml();
+			tt.setnounknown();
+			tt.seteostag("<s>");
+			tt.debug(debug);
+			tt.treetagger(modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())
+			infile.delete();
+		} catch(Exception e) {
+			Log.printStackTrace(e);
+			System.out.println("Failed to apply TreeTagger on $f input file with the $modelfile model file.");
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Write standoff file.
+	 *
+	 * @param ttoutfile the ttoutfile
+	 * @param posfile the posfile
+	 * @return true, if successful
+	 */
+	public boolean writeStandoffFile(File ttoutfile, File posfile)
+	{
+		def encoding ="UTF-8";
+		def transfo = new CSV2W_ANA();
+		//println("build w-interp "+ttfile.getName()+ ">>"+posfile.getName())
+		transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);
+		transfo.setResp(respId, respDesc, respDate, respPerson, respWhen);
+		transfo.setApp(appIdent, appVersion);
+		transfo.setTarget(ttoutfile.getAbsolutePath(), reportFile);
+		transfo.setInfos(distributor,  publiStmt, sourceStmt);
+		return transfo.process( ttoutfile, posfile, encoding );
+	}
+
+	/**
+	 * Run step by step : build TT src files, run TT, build xml-standoff files, inject standoff annotations
+	 *
+	 * @param rootDirFile the root dir file
+	 * @param modelfilename the modelfilename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir,  String modelfilename)
+	{
+		//test if modelfile exists
+		if (debug) {
+			println "rootDirFile "+binDir
+			println "txmDir "+txmDir
+			println "TREETAGGER INSTALL PATH : "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)
+			println "TREETAGGER MODELS PATH : "+Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH)
+		}
+
+		//test if the Toolbox know TreeTagger
+		if (!new File(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/").exists()) {
+			println("Could not find TreeTagger binaries in "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/")
+			return false;
+		}
+		String langAll = null
+		String lang;
+		if (modelfilename.startsWith("??")) {
+			langAll = new LangDetector(binDir).getLang();
+			println "General lang $langAll"
+		}
+
+		//cleaning
+		new File(binDir, "annotations").deleteDir();
+		new File(binDir, "annotations").mkdir();
+		new File(binDir, "treetagger").deleteDir();
+		new File(binDir, "treetagger").mkdir();
+
+		ArrayList<String> milestones = [];
+
+		//BUILD TT FILE READY TO BE TAGGED
+		//println "Re "+Toolbox.getParam(Toolbox.TREETAGGER_APOSTROPHE)
+		List<File> files = txmDir.listFiles()
+
+		println("Building TT source files ("+files.size()+") from directory "+txmDir)
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File f : files) {
+			cpb.tick()
+			File srcfile = f;
+			File resultfile = new File(binDir, "treetagger/"+f.getName()+".tt");
+			if(debug)
+				println "build tt src : "+srcfile+" >> "+resultfile
+			def ttsrcbuilder = new BuildTTSrc(srcfile.toURI().toURL())
+			if (!ttsrcbuilder.process(resultfile, null))
+				System.out.println("Failed to build tt src file of "+srcfile);
+		}
+
+		if (cancelNow) return;
+
+		File modelDirectory = new File(Toolbox.getParam(Toolbox.TREETAGGER_MODELS_PATH));
+		if (!modelDirectory.exists()) {
+			println "Skipping ANNOTATE: TreeTagger language model file directory not found: "+modelDirectory.getAbsolutePath();
+			return false;
+		} else 	if (!modelDirectory.canRead()) {
+			println "Skipping ANNOTATE: impossible to access the TreeTagger language model file directory: "+modelDirectory.getAbsolutePath();
+			return false;
+		}
+		println("")
+		//Convert encoding if needed
+
+		//APPLY TREETAGGER
+		files = new File(binDir, "treetagger").listFiles()
+		println("Applying $modelfilename TreeTagger model on dir: "+new File(binDir, "treetagger")+ " ("+files.size()+" files)")
+		if (files == null || files.size() == 0)
+			return false;
+		File modelfile;
+		cpb = new ConsoleProgressBar(files.size())
+		for (File f : files) {
+			String tmpModelFileName = modelfilename
+			if (modelfilename.startsWith("??")) {
+				lang = langAll;
+				if (f.length() > LangDetector.MINIMALSIZE) {
+					lang = new LangDetector(f).getLang();
+					//println "guessing lang $f : $lang"
+				}
+				tmpModelFileName = lang+".par"
+			}
+			modelfile = new File(modelsDirectory, tmpModelFileName);
+			if (debug)
+				println "model file : "+modelfile;
+
+			File
+
+			if (!modelfile.exists()) {
+				println "Skipping ANNOTATE: '$modelfile' TreeTagger language model file not found."
+				if(System.getProperty("os.name").startsWith("Windows") || System.getProperty("os.name").startsWith("Mac"))
+					println "Windows&Mac users: the operating system might be hiding file extensions. Use your file explorer to check the file name."
+				return false;
+			} else if (!modelfile.canRead()) {
+				println "Skipping ANNOTATE: impossible to access the '$modelfile' TreeTagger language model file."
+				return false;
+			}
+
+			//			if (modelfile.getName().equals("sp.par")) {//UTF >> Latin1
+			//				if(debug)
+			//					println "fix encoding for model "+modelfile
+			//				new EncodingConverter(f, "UTF-8", "ISO-8859-1")
+			//			}
+
+			cpb.tick()
+			File infile = f;
+			File outfile = new File(f.getParent(),f.getName()+"-out.tt");
+			if (!applyTT(infile, outfile, modelfile)) {
+				return false;
+			}
+
+			//			//Reconvert encoding if needed
+			//			if (modelfile.getName().equals("sp.par")) {
+			//				if(debug)
+			//					println "convert "+f+" latin1 >> UTF-8"
+			//				new EncodingConverter(f, "ISO-8859-1", "UTF-8")
+			//			}
+		}
+		println("")
+
+		if (cancelNow) return;
+
+		//BUILD STAND-OFF FILES
+		if (modelfile == null) {
+			println "no model applied"
+			return false;
+		}
+
+		initTTOutfileInfos(binDir, modelfile, modelfilename);
+		files = new File(binDir, "treetagger").listFiles()// now contains the result files of TT
+		println "Building stdoff files ("+files.size()+") from dir:"+new File(binDir, "treetagger")+" to "+new File(binDir, "annotations");
+		if (files == null || files.size() == 0)
+			return false;
+		cpb = new ConsoleProgressBar(files.size())
+		for (File ttrezfile : files) {
+			cpb.tick()
+			File annotfile = new File(binDir, "annotations/"+ttrezfile.getName()+"-STOFF.xml");
+			if (!writeStandoffFile(ttrezfile, annotfile))
+				println("Failed to build standoff file of "+ttrezfile);
+		}
+		println("")
+
+		if (cancelNow) return;
+
+		//INJECT ANNOTATIONS
+		List<File> interpfiles = new File(binDir, "annotations").listFiles();
+		List<File> txmfiles = txmDir.listFiles();
+		if (txmfiles == null) {
+			println "No file to annotate in "+txmDir.getAbsolutePath()
+			return false;
+		}
+		interpfiles.sort(); // same order
+		txmfiles.sort(); //same order
+		println "Injecting stdoff files ("+interpfiles.size()+") data from "+new File(binDir, "annotations")+ " to xml-txm files of "+txmDir;
+		if (interpfiles == null || interpfiles.size() == 0)
+			return false;
+		cpb = new ConsoleProgressBar(interpfiles.size())
+		for (int i = 0 ; i < interpfiles.size() ; i++) {
+			cpb.tick()
+			File srcfile = txmfiles.get(i);
+			File pos1file = interpfiles.get(i);
+			File temp = File.createTempFile("Annotate", "temp", srcfile.getParentFile());
+			def builder = new AnnotationInjection(srcfile.toURI().toURL(), pos1file.toURI().toURL());
+			if (!builder.process(temp)) {
+				return false;
+			}
+			builder = null;
+
+			//println "renaming files..."
+			if (!(srcfile.delete() && temp.renameTo(srcfile)))
+				println "Warning can't rename file "+temp+" to "+srcfile
+		}
+		println("")
+		return true;
+	}
+	
+	public void setModelsDirectory(File modelsDirectory) {
+		this.modelsDirectory = modelsDirectory;
+	}
+
+	/**
+	 * Run file by file. Allow to have one different lang per file. Default behavior add new word properties
+	 *
+	 * @param binDir
+	 * @param txmDir
+	 * @param lang associate a file name with a model filename
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, HashMap<String, String> langs)
+	{
+		return run(binDir, txmDir, langs, false, new String[0], new String[0]);
+	}
+	
+	/**
+	 * Run file by file. Allow to have one different lang per file
+	 *
+	 * @param binDir 
+	 * @param txmDir
+	 * @param lang associate a file name with a model filename
+	 * @param replace, replace or create a word property
+	 * @return true, if successful
+	 */
+	public boolean run(File binDir, File txmDir, HashMap<String, String> langs, boolean replace, String[] properties, String[] options)
+	{
+		if (!new File(Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/").exists()) {
+			println("Path to TreeTagger is wrong "+Toolbox.getParam(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/")
+			return true;
+		}
+
+		List<File> listfiles = txmDir.listFiles();
+
+		//cleaning
+		File annotDir = new File(binDir,"annotations");
+		annotDir.deleteDir();
+		annotDir.mkdir();
+		File ptreetaggerDir = new File(binDir,"ptreetagger");
+		ptreetaggerDir.deleteDir();
+		ptreetaggerDir.mkdir();
+		File treetaggerDir = new File(binDir,"treetagger");
+		treetaggerDir.deleteDir();
+		treetaggerDir.mkdir();
+
+		int cores = Runtime.getRuntime().availableProcessors()
+		int coresToUse = Math.max(1.0, cores * 0.7)
+		ExecutorService pool = Executors.newFixedThreadPool(coresToUse)
+
+		def files = txmDir.listFiles()
+		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())
+		for (File teiFile : files) {
+			int counter = 1;
+			ThreadFile t = new ThreadFile("TT_"+counter++, teiFile) {
+
+						public void run() {
+
+							if (cancelNow) return;
+
+							if (langs.get(f.getName()) == null) {
+								println "Error: no lang defined for file $f"
+								return;
+							}
+
+							String model = langs.get(f.getName());
+							File modelfile = new File(modelsDirectory, model+".par");
+							if (!modelfile.exists()) {
+								println "Error: No Modelfile available for lang "+modelfile+". Continue import process ";
+								return;
+							}
+							File annotfile = new File(annotDir, f.getName()+"-STDOFF.xml");
+							File ttsrcfile = new File(ptreetaggerDir, f.getName()+"-src.tt");
+							File ttrezfile = new File(treetaggerDir, f.getName()+"-out.tt");
+							//println ("TT with $model "+f+"+"+annotfile+" > "+ttsrcfile+" > "+ttrezfile);
+
+							//BUILD TT FILE READY TO BE TAGGED
+							def builder = new BuildTTSrc(f.toURL());
+							builder.process(ttsrcfile, null);
+
+							//Apply TT
+							applyTT(ttsrcfile, ttrezfile, modelfile, options);
+
+							//CREATE STANDOFF FILES
+							initTTOutfileInfos(binDir, modelfile, model, properties);
+							writeStandoffFile(ttrezfile, annotfile)
+
+							//INJECT ANNOTATIONS
+							File tmpFile = new File(txmDir, "temp_"+f.getName())
+							builder = new AnnotationInjection(f.toURL(), annotfile.toURL(), replace);
+							builder.process(tmpFile);
+							if (!(f.delete() && tmpFile.renameTo(f))) println "Warning can't rename file "+tmpFile+" to "+f
+
+							cpb.tick();
+						}
+					};
+
+			pool.execute(t)
+		}
+
+		pool.shutdown()
+		pool.awaitTermination(10, TimeUnit.HOURS)
+		println ""
+		return true;
+	}
+
+	public void setCancelNow() {
+		cancelNow = true;
+	}
+
+	public class ThreadFile extends Thread {
+		File f;
+		public ThreadFile(String name, File f) {
+			this.setName(name)
+			this.f = f;
+		}
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/testTT.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/testTT.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/testTT.groovy	(revision 321)
@@ -0,0 +1,549 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import java.util.ArrayList;
+import org.txm.utils.treetagger.TreeTagger;
+import org.txm.utils.*;
+import org.txm.importer.*;
+import org.txm.importer.RGAQCJ.*;
+import java.io.File;
+import java.text.DateFormat;
+import java.util.Date;
+import java.text.DecimalFormat;
+// TODO: Auto-generated Javadoc
+
+/**
+ * test TT wrapper
+ * add NLP annotation
+ * build result matrix.
+ *
+ * @return the java.lang. object
+ */
+String rootDir = "~/xml/rgaqcj/";
+new File(rootDir+"/models/").mkdir();
+new File(rootDir+"/lexicon/").mkdir();
+new File(rootDir+"/proj/").mkdir();
+new File(rootDir+"/ttlearn/").mkdir();
+new File(rootDir+"/ttsrc/").mkdir();
+new File(rootDir+"/pos/").mkdir();
+new File(rootDir+"/injection/").mkdir();
+new File(rootDir+"/anainline/").mkdir();
+new File(rootDir+"/xml-txm/").mkdir();
+
+//create srcana.xml from src.xml
+def srcfiles = ["roland.xml","artu.xml","qjm.xml","commyn1.xml","jehpar.xml"];
+def anafiles = ["roland-ana.xml","artu-ana.xml","qjm-ana.xml","commyn1-ana.xml","jehpar-ana.xml"];
+def xslfiles = ["bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl"];
+//bfmgraal2txm-w.xsl
+for(int i=0 ; i < xslfiles.size() ; i++)
+{
+	String xslfile = rootDir+"xsl/"+xslfiles[i];
+	String infile = rootDir+"src/"+srcfiles[i];
+	String outfile = rootDir+"anainline/"+anafiles[i];
+	println("create xml-txm from "+srcfiles[i]);
+	ApplyXsl a = new ApplyXsl(xslfile);
+	a.process(infile,outfile);
+}
+
+//Import to CWB
+def cwbLoc ="~/Bureau/textometrie/CWB/cwb-3.0/utils/";//chemin vers executable cqp
+BuildXmlRGAQCJ.process( anafiles,  rootDir, cwbLoc)
+println("build wtc RGAQCJ");
+println("encode + makeall");
+
+//create TT TRAIN files
+def ttlearnfiles = ["roland-learn.tt","artu-learn.tt","qjm-learn.tt","commyn1-learn.tt","jehpar-learn.tt","rgaqcj-learn.tt"];
+String initiales = "RAQCJZ"
+
+//ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/cwb-decode-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/CwbDecode.groovy"));
+GroovyClassLoader gcl = new GroovyClassLoader();
+gcl.addClasspath(".");
+
+String registryPath = rootDir+"/registry";
+String cwbdecodeexecDir = "~/Bureau/trunkCWB/cwb-3.0/utils/"
+
+Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/CwbDecode.groovy"));
+def aScript = clazz.newInstance(cwbdecodeexecDir);
+
+aScript.setC()
+def pAttributesTrain = ["word","cat"];
+def pAttributesLex = ["word","cat","LEMMA"];
+def pAttributesTag = ["word"];
+def sAttributes = ["s"];
+aScript.setP(pAttributesTag)
+aScript.setS(sAttributes)
+aScript.setr(registryPath)
+aScript.debug(true);
+int[] nbmots = [0,35306,134170,173526,198929,228149]
+
+for(int i=0 ; i < initiales.length() ; i++)
+{
+	println("creation TTsrc "+ttlearnfiles[i]);
+	FileOutputStream fos = new FileOutputStream(rootDir+"/ttsrc/"+ttlearnfiles[i]);
+	PrintStream ps = new PrintStream(fos);
+	def out = System.out;
+	if(!(initiales[i]+"").matches("Z"))
+	{
+		if(i > 0)
+			aScript.sets(nbmots[i]+1)
+		else
+			aScript.sets(nbmots[i])
+		aScript.sete(nbmots[i+1])
+	}
+	else
+	{
+		aScript.unsets();
+		aScript.unsete();
+	}
+	System.setOut(ps);
+	
+	if(System.getProperty("os.name").contains("Windows"))
+		aScript.cwbdecodeexe("RGAQCJ")
+	else
+		aScript.cwbdecode("RGAQCJ")
+	
+	System.setOut(out);
+}
+
+
+aScript.unsetS()
+aScript.setP(pAttributesTrain)
+for(int i=0 ; i < initiales.length() ; i++)
+{
+	println("creation TTlearn "+ttlearnfiles[i]);
+	FileOutputStream fos = new FileOutputStream(rootDir+"/ttlearn/"+ttlearnfiles[i]);
+	PrintStream ps = new PrintStream(fos);
+	def out = System.out;
+	if(!(initiales[i]+"").matches("Z"))
+	{
+		if(i > 0)
+			aScript.sets(nbmots[i]+1)
+		else
+			aScript.sets(nbmots[i])
+		aScript.sete(nbmots[i+1])
+	}
+	else
+	{
+		aScript.unsets();
+		aScript.unsete();
+	}
+	System.setOut(ps);
+	
+	if(System.getProperty("os.name").contains("Windows"))
+		aScript.cwbdecodeexe("RGAQCJ")
+	else
+		aScript.cwbdecode("RGAQCJ")
+	
+	System.setOut(out);
+}
+aScript.setP(pAttributesLex)
+HSQLFunctions.clearAll();
+HashMap<String,Integer> nbFormLexique = new HashMap<String,Integer>()
+HashMap<String,Integer> nbOccurencesTxt = new HashMap<String,Integer>()
+for(int i=0 ; i < initiales.length() ; i++)
+{
+	println("creation lex brut lex_"+initiales[i]);
+	FileOutputStream fos = new FileOutputStream(rootDir+"lexicon/lex_"+initiales[i]);
+	PrintStream ps = new PrintStream(fos);
+	def out = System.out;
+	if(!(initiales[i]+"").matches("Z"))
+	{
+		if(i > 0)
+			aScript.sets(nbmots[i]+1)
+		else
+			aScript.sets(nbmots[i])
+		aScript.sete(nbmots[i+1])
+	}
+	else
+	{
+		aScript.unsets();
+		aScript.unsete();
+	}
+	System.setOut(ps);
+	
+	if(System.getProperty("os.name").contains("Windows"))
+		aScript.cwbdecodeexe("RGAQCJ")
+	else
+		aScript.cwbdecode("RGAQCJ")
+	
+	System.setOut(out);
+	def argsname = ["form","cat","lemme"];
+	def types = ["VARCHAR(30)","VARCHAR(30)","VARCHAR(30)"];
+	File lexfile = new File(rootDir+"lexicon/lex_"+initiales[i]);
+	int nbMot = HSQLFunctions.ImportOrderedCSVTable("lexbrut_"+initiales[i], argsname,types,lexfile,"\t","UTF-8");
+	nbOccurencesTxt.put("lexbrut_"+initiales[i],nbMot);
+	//HSQLFunctions.ImportCSVTable("lex_"+initiales[i], argsname,types,lexfile,"\t","ISO-8859-1");
+	
+	argsname = ["cat","lemme"];
+	int nbForm = HSQLFunctions.toRefHierarchicalFile( "lexbrut_"+initiales[i], lexfile.getAbsolutePath(),"form",argsname,"UTF-8");
+	nbFormLexique.put("lex_"+initiales[i],nbForm);
+}
+aScript.unsets();
+aScript.unsete();
+println("creation lex brut RGAQCJ");
+FileOutputStream fos = new FileOutputStream(rootDir+"/lexicon/rgaqcj-brut");
+PrintStream ps = new PrintStream(fos);
+def out = System.out;
+System.setOut(ps);
+
+if(System.getProperty("os.name").contains("Windows"))
+	aScript.cwbdecodeexe("RGAQCJ")
+else
+	aScript.cwbdecode("RGAQCJ")
+
+System.setOut(out);
+
+println("create lexiq from csv");
+
+//train
+//def ttlearnfiles = ["roland-learn.tt","qgraal_cm-learn.tt","artu-learn.tt","qjm-learn.tt","commyn1-learn.tt","jehpar-learn.tt","rgaqcj-learn.tt"];
+def modelfiles = ["roland.par","artu.par","qjm.par","commyn1.par","jehpar.par","rgaqcj.par"];
+
+def lexfiles = ["lex_R","lex_A","lex_Q","lex_C","lex_J","lex_Z"];
+def openclassfiles = ["open_class_R.txt","open_class_A.txt","open_class_Q.txt","open_class_C.txt","open_class_J.txt","open_class_Z.txt"];
+//def lexiconfile = rootDir+"/lexicon/"+"lexfra-rgaqcj.txt";
+
+
+for(int i=0; i < ttlearnfiles.size();i++)
+{
+	String infile = rootDir+"/ttlearn/"+ttlearnfiles[i];
+	String outfile = rootDir+"/models/"+modelfiles[i];
+	String lexiconfile = rootDir+"/lexicon/"+lexfiles[i];
+	String openclassfile = rootDir+"/lexicon/"+openclassfiles[i];
+	TreeTagger tt = new TreeTagger("~/Bureau/treetagger/bin/");
+	println("learn "+ttlearnfiles[i] +" >> "+modelfiles[i]);
+	tt.settoken();
+	tt.setquiet();
+	//tt.setlemma();
+	tt.setsgml();
+	tt.setst("PONfrt");
+	tt.setproto();
+	tt.setutf8();
+	tt.traintreetagger( lexiconfile, openclassfile, infile, outfile)
+}
+
+//projection
+def ttsrcfiles = ["roland-learn.tt","artu-learn.tt","qjm-learn.tt","commyn1-learn.tt","jehpar-learn.tt","rgaqcj-learn.tt"];
+def ttrezfiles = [];
+for(int i=0; i < ttsrcfiles.size();i++)
+{
+	for(int j=0; j < modelfiles.size();j++)
+	{
+		String infile = rootDir+"/ttsrc/"+ttsrcfiles[i];
+		String modelfile =  rootDir+"/models/"+modelfiles[j];
+		String outfile = ttsrcfiles[i].substring(0,3)+"_"+modelfiles[j].substring(0,3)+".tt";
+		ttrezfiles.add(outfile);
+		println("proj "+modelfiles[j]+ " on " +ttsrcfiles[i] +" >> "+outfile);
+		
+		TreeTagger tt = new TreeTagger("~/Bureau/treetagger/bin/");
+		tt.settoken();
+		tt.setquiet();
+		tt.setsgml();
+		tt.seteostag("<s>");
+		tt.treetagger( modelfile, infile, rootDir+"proj/"+outfile)
+	}
+}
+
+
+
+/*
+ //create ana-pos.xml files
+ def posfiles = ["roland-pos","artu-pos","qjm-pos","commyn1-pos","jehpar-pos","rgaqcj-pos"];
+ String encoding ="ISO-8859-1";
+ String person ="Serge Heiden"; 
+ String software = "tree-tagger cmd line with options ..."; 
+ String cmdLine = "tree-tagger -token ...";
+ String title ="POS annotation";
+ def types = ["POS","lemma"];
+ String idform ="fro";
+ for(int i=0; i < ttsrcfiles.size();i++)
+ {
+ for(int j = 0; j< ttsrcfiles.size();j++)
+ {
+ int c = j +i*ttsrcfiles.size(); // id ttrez
+ File ttfile = new File(rootDir+"/tt/",ttrezfiles[c]);
+ File posfile = new File(rootDir+"/pos/",posfiles[i]+"-"+modelfiles[j].substring(0,3)+".xml");
+ println("TTrez > ana-pos : "+ttfile+" to : "+posfile );
+ c++;
+ def transfo = new TTrezToXml();
+ transfo.setAnnotationTypes( types, idform);
+ transfo.setResp( person, software, cmdLine);
+ transfo.process( ttfile, posfile, encoding );
+ }
+ }	
+ //injection annotations
+ String respPerson ="sheiden"
+ String respId = "";
+ String respDate = DateFormat.getDateInstance(DateFormat.FULL, Locale.UK).format(new Date());
+ String respCDATA = "";
+ for(int i=0; i < anafiles.size();i++)
+ {
+ for(int j = 0; j < 6 ; j++)
+ {
+ File srcfile = new File(rootDir+"/anainline/",anafiles[i]);
+ File posfile = new File(rootDir+"/pos/",posfiles[i]+"-"+modelfiles[j].substring(0,3)+".xml");
+ println("annote file : "+srcfile+" with : "+posfile );
+ def builder = new AnnotationInjection(srcfile.toURL(), posfile.toURL(), solotags);
+ builder.setResp(respId,respPerson, respDate, respCDATA);
+ builder.transfomFile(rootDir+"/injection/",anafiles[i]);
+ }
+ }	
+ */
+//remove <s> from ttrezfiles
+String encoding = "UTF-8"
+for(int i=0; i < ttrezfiles.size();i++)
+{
+	String csvfile = rootDir+"proj/"+ttrezfiles[i];
+	File f = new File(csvfile);
+	File temp = new File("tempFileCVScleaner")
+	println("fix <s> "+f);
+	Reader reader = new InputStreamReader(new FileInputStream(csvfile),encoding);
+	Writer writer = new FileWriter(temp);
+	reader.eachLine 
+			{
+				if(!it.startsWith("<s") && !it.startsWith("</s"))
+					writer.write(it+"\n")
+			}
+	reader.close();
+	writer.close();
+	if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+}
+
+//Build process infos
+//import table into hsql
+for(int i=0; i < ttrezfiles.size();i++)
+{
+	String csvfile = rootDir+"proj/"+ttrezfiles[i];
+	def argsname = ["form","cat"];
+	def types = ["VARCHAR(30)","VARCHAR(30)"];
+	HSQLFunctions.ImportOrderedCSVTable(ttrezfiles[i].replace(".",""), argsname,types,new File(csvfile),"\t","UTF-8");
+	println("create Table "+ttrezfiles[i].replace(".",""));
+}
+
+//calc richesses lexicales
+LinkedHashMap<String,ArrayList<String>> richesses = new LinkedHashMap<String,ArrayList<String>>();
+richesses.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+
+LinkedHashMap<String,ArrayList<String>> richessesCat = new LinkedHashMap<String,ArrayList<String>>();
+richessesCat.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+
+LinkedHashMap<String,ArrayList<String>> richessesocc = new LinkedHashMap<String,ArrayList<String>>();
+richesses.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+
+LinkedHashMap<String,ArrayList<String>> richessesoccCat = new LinkedHashMap<String,ArrayList<String>>();
+richessesCat.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+for(int i=0;i < initiales.size();i++)
+{
+	richesses.put(""+initiales[i], new ArrayList<String>());
+	richessesCat.put(""+initiales[i], new ArrayList<String>());
+	richessesocc.put(""+initiales[i], new ArrayList<String>());
+	richessesoccCat.put(""+initiales[i], new ArrayList<String>());
+}
+for(int i=0; i < initiales.size();i++)
+{
+	for(int j=0; j < initiales.size();j++)
+	{
+		String query = "SELECT count(*) FROM ((SELECT DISTINCT form FROM lexbrut_"+initiales[i]+" ) MINUS (SELECT DISTINCT form FROM lexbrut_"+initiales[j]+"))"
+		println(query);
+		String query2 = "SELECT count(*) FROM ((SELECT DISTINCT cat FROM lexbrut_"+initiales[i]+" ) MINUS (SELECT DISTINCT cat FROM lexbrut_"+initiales[j]+"))"
+		println(query2);
+		String query3 = "SELECT count(form) FROM lexbrut_"+initiales[i]+" WHERE form NOT IN (SELECT form FROM lexbrut_"+initiales[j]+")"
+		println(query3);
+		String query4 = "SELECT count(cat) FROM lexbrut_"+initiales[i]+" WHERE cat NOT IN (SELECT cat FROM lexbrut_"+initiales[j]+")"
+		println(query4);
+		HSQLFunctions.getGroovySql().eachRow(query) {
+			def rich = it.getAt(0);
+			println("rich form :"+ rich);
+			richesses.get(""+initiales[i]).add( ""+rich ) ;	
+		}
+		
+		HSQLFunctions.getGroovySql().eachRow(query2) {
+			def rich = it.getAt(0);
+			println("rich cat :"+ rich);
+			richessesCat.get(""+initiales[i]).add( ""+rich ) ;	
+		}
+		
+		
+		HSQLFunctions.getGroovySql().eachRow(query3) {
+			def rich = it.getAt(0);
+			println("richocc form :"+ rich);
+			richessesocc.get(""+initiales[i]).add( ""+rich ) ;	
+		}
+		
+		HSQLFunctions.getGroovySql().eachRow(query4) {
+			def rich = it.getAt(0);
+			println("richocc cat :"+ rich);
+			richessesoccCat.get(""+initiales[i]).add( ""+rich ) ;	
+		}
+	}
+}
+
+LinkedHashMap<String,ArrayList<String>> matrix = new LinkedHashMap<String,ArrayList<String>>();
+
+matrix.put("Text", ["T    ","Vf    ","Vc    ","TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+for(int i=0;i < initiales.size();i++)
+{
+	matrix.put(""+initiales[i], new ArrayList<String>());
+}
+
+ArrayList<Integer> countOccForm = new ArrayList<Integer>();
+ArrayList<Integer> countOccCat = new ArrayList<Integer>();
+ArrayList<Integer> countLexForm = new ArrayList<Integer>();
+ArrayList<Integer> countLexCat = new ArrayList<Integer>();
+
+for(int i=0;i < initiales.size() ; i++ )
+{
+	String query1 = "SELECT count(*) FROM (SELECT form from lexbrut_"+initiales[i]+")";
+	String query2 = "SELECT count(*) FROM (SELECT cat from lexbrut_"+initiales[i]+")";
+	String query3 = "SELECT count(*) FROM (SELECT DISTINCT form from lexbrut_"+initiales[i]+")";
+	String query4 = "SELECT count(*) FROM (SELECT DISTINCT cat from lexbrut_"+initiales[i]+")";
+	
+	HSQLFunctions.getGroovySql().eachRow(query1) {
+		def rich = it.getAt(0);
+		println("count occ form :"+ rich);
+		countOccForm.add( ""+rich ) ;	
+	}
+	HSQLFunctions.getGroovySql().eachRow(query2) {
+		def rich = it.getAt(0);
+		println("coutn occ cat :"+ rich);
+		countOccCat.add( ""+rich ) ;	
+	}
+	HSQLFunctions.getGroovySql().eachRow(query3) {
+		def rich = it.getAt(0);
+		println("count lex form :"+ rich);
+		countLexForm.add( ""+rich ) ;	
+	}
+	HSQLFunctions.getGroovySql().eachRow(query4) {
+		def rich = it.getAt(0);
+		println("count lex cat :"+ rich);
+		countLexCat.add( ""+rich ) ;	
+	}
+	println("col1 et 2 : "+nbOccurencesTxt.get("lexbrut_"+initiales[i])+" "+nbFormLexique.get("lex_"+initiales[i]))
+	matrix.get(""+initiales[i]).add( ""+nbOccurencesTxt.get("lexbrut_"+initiales[i]) ) ;
+	matrix.get(""+initiales[i]).add( ""+nbFormLexique.get("lex_"+initiales[i]) ) ;
+	matrix.get(""+initiales[i]).add( "0" ) ;
+}
+for(int i=0; i < ttrezfiles.size();i++)
+{
+	//get occurences ordonnées de la gold
+		//get occurences ordonnées de la projection
+		//compare
+	
+	int id = i/ttsrcfiles.size();
+	
+	println("comp "+ttrezfiles[i]+" VS lexbrut_"+initiales[id]);
+	String query = "SELECT count(*) FROM ((SELECT n,form,cat FROM lexbrut_"+initiales[id]+" ) MINUS (SELECT n,form,cat FROM "+ttrezfiles[i].replace(".","")+"))"
+	//HSQLFunctions.executeQuery(query);
+	//println(query)
+	int ttotal=0;
+	int total ;
+	HSQLFunctions.getGroovySql().eachRow(query) {
+		def dif = it.getAt(0);
+		if((""+initiales[id]) != "Z")
+		{
+			total = nbmots[id+1] - nbmots[id];
+			ttotal += total;
+		}
+		else
+			total= ttotal;
+		
+		total = nbOccurencesTxt.get("lexbrut_"+initiales[id])
+		println("dif "+dif+"/ tot "+total+" = "+((float)dif/(float)total));
+		Float perf = ((float)dif/(float)total)*100f;			
+		matrix.get(""+initiales[id]).add( ""+perf ) ;
+	}
+	
+	/*	HSQLFunctions.getGroovySql().eachRow("SELECT count(*) from "+table+" WHERE form NOT IN (SELECT form FROM "+ttrezfiles[i].replace(".","")+")") {
+	 def dif = it.getAt(0);
+	 int total = nbmots[i+1] - nbmots[i];
+	 richesses.put(""+initiales[i],""+(dif/total)) ;
+	 }*/
+}
+println("Matrice d'erreur : ");
+for(String k : matrix.keySet())
+{
+	print(k)
+	for(String f : matrix.get(k))
+		print("\t"+f)
+	println()
+}
+println("richesse lexiques en Form : ");
+for(String k : richesses.keySet())
+{
+	print(k)
+	for(String f : richesses.get(k))
+		print("\t"+f)
+	println()
+}
+println("richesse lexiques en Cat : ");
+for(String k : richessesCat.keySet())
+{
+	print(k)
+	for(String f : richessesCat.get(k))
+		print("\t"+f)
+	println()
+}
+
+println("richesse occurance en Form : ");
+for(String k : richessesocc.keySet())
+{
+	print(k)
+	for(String f : richessesocc.get(k))
+		print("\t"+f)
+	println()
+}
+println("richesse occurance en Cat : ");
+for(String k : richessesoccCat.keySet())
+{
+	print(k)
+	for(String f : richessesoccCat.get(k))
+		print("\t"+f)
+	println()
+}
+
+println("RichBrutes	roland	artu	qjm	comm	jehpar	rgaqcj");
+print("Focc")
+for(int i=0;i < countOccForm.size() ; i++)
+	print("\t"+countOccForm[i]); 
+print("\nCocc")
+for(int i=0;i < countOccCat.size() ; i++)
+	print("\t"+countOccCat[i]); 
+print("\nFvoc")
+for(int i=0;i < countLexForm.size() ; i++)
+	print("\t"+countLexForm[i]); 
+print("\nCvoc")
+for(int i=0;i < countLexCat.size() ; i++)
+	print("\t"+countLexCat[i]); 
+println()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/ParseAndInject.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/ParseAndInject.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/ParseAndInject.groovy	(revision 321)
@@ -0,0 +1,292 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2015-12-17 12:11:39 +0100 (Thu, 17 Dec 2015) $
+// $LastChangedRevision: 3087 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.StreamTokenizer;
+import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import javax.xml.stream.XMLStreamWriter;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Read a list of word from a txt file and inject them in w elements
+ *
+ * @author mdecorde
+ */
+class ParseAndInject {
+	
+	/** The writer. */
+	XMLStreamWriter writer;
+	def output;
+	
+	/** The lang. */
+	String lang;
+	
+	/** The type. */
+	String type;
+	
+	/** The brutreader. */
+	BufferedReader brutreader;
+	
+	/** The line. */
+	String line;
+	
+	/** The current. */
+	int current;
+	
+	/** The split. */
+	String[] split;
+	
+	/** The token. */
+	String token;
+	
+	/** The is ex. */
+	boolean isEx;
+	
+	/** The is w. */
+	boolean isW;
+	
+	/**
+	 * start the processing.
+	 *
+	 * @param brutfile the brutfile
+	 * @param xmlfile the xmlfile
+	 * @param outfile the outfile
+	 */
+	public void run(File brutfile, File xmlfile,  File outfile) throws Exception
+	{
+		String localname;
+		
+		// create XML writer
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		output = new FileOutputStream(outfile);
+		writer = factory.createXMLStreamWriter(output, "UTF-8");
+		
+		// create XML reader
+		InputStream inputData = xmlfile.toURI().toURL().openStream();
+		XMLInputFactory inputfactory = XMLInputFactory.newInstance();
+		XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);
+		
+		//create Exbrut whitespace tokenizer
+		brutreader = new BufferedReader(new FileReader(brutfile));
+		
+		writer.writeStartDocument("UTF-8","1.0");
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())
+		{
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+				
+					if (localname == "w") {
+						isW=true;
+					}
+					writer.writeStartElement(localname); // create text tag in each xml file
+					for (int i= 0 ; i < parser.getAttributeCount() ; i++ ) {
+						writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+					}
+					break;
+				
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					if (localname == "w") {
+						isW=false;
+					}
+					writer.writeEndElement();
+				
+					break;
+				
+				case XMLStreamConstants.CHARACTERS:
+					if (isW) {
+						token = this.getNextToken();
+						//System.err.println("next token "+token);
+						if (token == null) {
+							System.err.println("error getting next word");
+							return;
+						}
+						String text = parser.getText();
+						int dist = Distance.LD(token, text);// check integrity
+						int max = 0;
+						if (token.contains("(("))
+						 max += 2
+						if (token.contains("))"))
+							max += 2
+						if (dist > max) {
+							writer.writeComment("Warning !! Distance($token, $text) > 4")
+						}
+						
+						processToken();
+						//writer.writeCharacters(token);
+					} else {
+						writer.writeCharacters(parser.getText());
+					}
+					break;
+			}
+		}
+		
+		writer.close();
+		output.close();
+		parser.close();
+		inputData.close();
+	}
+	
+	/**
+	 * Process token.
+	 */
+	public void processToken()
+	{
+		//System.err.println("process $token");
+		if (token.length() == 0)
+			return;
+		int idxopen = token.indexOf("((")
+		int idxclose = token.indexOf("))")
+		if (isEx) {
+			if (idxopen != -1)
+			if (idxopen < idxclose) {
+				System.err.println("Error: isEx $isEx token: $token : $idxopen < $idxclose");
+				return;
+			}
+			if (idxclose == -1) {
+				writer.writeStartElement("ex");
+				writer.writeCharacters(token);
+				writer.writeEndElement();
+				return;
+			}
+			
+			String ok = token.substring(0, idxclose);
+			
+			writer.writeStartElement("ex");
+			writer.writeCharacters(ok);
+			writer.writeEndElement();
+			isEx = !isEx;
+			
+			token = token.substring(idxclose+2);
+			processToken();
+		} else {
+			if (idxclose != -1)
+			if (idxclose < idxopen) {
+				System.err.println("Error: isEx $isEx token: $token : $idxclose < $idxopen");
+				return;
+			}
+			if (idxopen == -1) {
+				//System.out.println("write "+token);
+				writer.writeCharacters(token);
+				return;
+			}
+			
+			String ok = token.substring(0, idxopen);
+			writer.writeCharacters(ok);
+			isEx = !isEx;
+			token = token.substring(idxopen+2);
+			processToken();
+		}		
+	}
+	
+	/**
+	 * Gets the next token.
+	 *
+	 * @return the next token
+	 */
+	public String getNextToken() throws IOException
+	{
+		if (split == null) {
+			line = brutreader.readLine();
+			if (line == null)
+				return null;
+			
+			split = line.split(" ");
+			if (split == null || split.length == 1) {
+				split = null;
+				return line;
+			} else {
+				current = 0;
+				return split[current++];
+			}
+		} else {
+			if (current < split.length) {
+				return split[current++];
+			} else {
+				split = null;
+				return getNextToken();
+			}
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File xmlfile = new File(System.getProperty("user.home"),"xml/injectEx/test.xml");
+		File brutfile = new File(System.getProperty("user.home"),"xml/injectEx/brut.txt");
+		File outfile = new File(System.getProperty("user.home"),"xml/injectEx/test-out.xml");
+		try {
+			new ParseAndInject().run(brutfile, xmlfile, outfile);
+			System.out.println("done");
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+}
+
+
+
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Distance.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Distance.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Distance.groovy	(revision 321)
@@ -0,0 +1,124 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//  
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.scripts;
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class Distance.
+ */
+public class Distance {
+	
+	//****************************
+	// Get minimum of three values
+	//****************************
+	
+	/**
+	 * Minimum.
+	 *
+	 * @param a the a
+	 * @param b the b
+	 * @param c the c
+	 * @return the int
+	 */
+	private static int Minimum (int a, int b, int c) {
+		int mi;
+		
+		mi = a;
+		if (b < mi) {
+			mi = b;
+		}
+		if (c < mi) {
+			mi = c;
+		}
+		return mi;
+		
+	}
+	
+	//*****************************
+	// Compute Levenshtein distance
+	//*****************************
+	
+	/**
+	 * LD.
+	 *
+	 * @param s the s
+	 * @param t the t
+	 * @return the int
+	 */
+	public static int LD (String s, String t) 
+	{
+		int[][] d; // matrix
+		int n; // length of s
+		int m; // length of t
+		int i; // iterates through s
+		int j; // iterates through t
+		char s_i; // ith character of s
+		char t_j; // jth character of t
+		int cost; // cost
+		
+		// Step 1
+		n = s.length ();
+		m = t.length ();
+		if (n == 0) {
+			return m;
+		}
+		if (m == 0) {
+			return n;
+		}
+		d = new int[n+1][m+1];
+		
+		// Step 2
+		for (i = 0; i <= n; i++) {
+			d[i][0] = i;
+		}
+		for (j = 0; j <= m; j++) {
+			d[0][j] = j;
+		}
+		
+		// Step 3
+		for (i = 1; i <= n; i++) {	
+			s_i = s.charAt (i - 1);
+			// Step 4
+			for (j = 1; j <= m; j++) {
+				t_j = t.charAt (j - 1);
+				
+				// Step 5
+				if (s_i == t_j) {
+					cost = 0;
+				}
+				else {
+					cost = 1;
+				}	
+				
+				// Step 6
+				d[i][j] = Minimum (d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1] + cost);				
+			}
+		}
+		// Step 7	
+		return d[n][m];
+	}	
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizationFilesUpdater.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizationFilesUpdater.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizationFilesUpdater.groovy	(revision 321)
@@ -0,0 +1,312 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2013-07-25 16:24:13 +0200 (jeu., 25 juil. 2013) $
+// $LastChangedRevision: 2490 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts.i18n;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.util.LinkedList;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Tool to update the key of externalization files xxxx_fr.properties
+ * adds _NA to missing keys
+ * 
+ * @author mdecorde
+ * 
+ */
+class ExternalizationFilesUpdater {
+
+	/** The dirfiles. */
+	LinkedList<File> dirfiles = new LinkedList<File>();
+	
+	/** The propertyfiles. */
+	HashMap<String, File> propertyfiles = new HashMap<String, File>();
+	
+	/** The fileentries. */
+	HashMap<File, List<String>> fileentries = new HashMap<String, List<String>>();
+	
+	/** The fileentriesvalues. */
+	HashMap<File, HashMap<String, String>> fileentriesvalues = new HashMap<String, HashMap<String, String>>();
+
+	/**
+	 * Gets the entries.
+	 *
+	 * @param file the file
+	 * @return the entries
+	 */
+	public List<String> getEntries(File file)
+	{
+		List<String> entries = [];
+		BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file) , "iso-8859-1"));
+		String line = reader.readLine();
+		while (line != null) {
+			String[] split = line.split("=", 2);
+			if (split.length > 0) {
+				String key = split[0];
+				entries.add(key);
+			}
+			line = reader.readLine();
+		}
+		return entries;
+	}
+
+	/**
+	 * Gets the values.
+	 *
+	 * @param file the file
+	 * @return the values
+	 */
+	public HashMap<String, String> getValues(File file) {
+		HashMap<String, String> values = new HashMap<String, String>();
+		BufferedReader reader = new BufferedReader(new InputStreamReader(
+				new FileInputStream(file), "iso-8859-1"));
+		String line = reader.readLine();
+		while (line != null) {
+			String[] split = line.split("=");
+			if (split.length > 0) {
+				String key = split[0];
+				String concat = "";
+				for (int i = 1; i < split.length; i++)
+					concat += split[i];
+				values.put(key, concat);
+			}
+			line = reader.readLine();
+		}
+		return values;
+	}
+
+	/**
+	 * Process found directories.
+	 */
+	public void processFoundDirectories()
+	{
+		for (String dirfile : dirfiles) {
+			println "DIRECTORY : "+dirfile
+			List<File> files = propertyfiles.get(dirfile)
+			for (File f : files) {
+				this.fileentries.put(f, getEntries(f));
+				this.fileentriesvalues.put(f, getValues(f));
+			}
+			File reference = null;
+			if (files.get(0).getName().startsWith("m")) {
+				reference = new File(dirfile,"messages.properties");
+			}
+			else if (files.get(0).getName().startsWith("b"))
+				reference = new File(dirfile,"bundle.properties");
+			
+			if (reference != null && reference.exists()) {
+				for (File f : files) {
+					if (f != reference) {
+						List<String> refentries = this.fileentries.get(reference).clone();
+						List<String> tmp1 = this.fileentries.get(f).clone();
+						tmp1.removeAll(refentries);
+						
+						refentries = this.fileentries.get(reference).clone();
+						List<String> tmp = this.fileentries.get(f);
+						refentries.removeAll(tmp);
+						for (String missing : refentries) {
+							this.fileentriesvalues.get(f).put(missing,"N/A_"+this.fileentriesvalues.get(reference).get(missing)); // put entry's value
+						}
+						
+						this.fileentries.put(f, this.fileentries.get(reference)); // update file entries
+						
+						if (tmp1.size() > 0 || refentries.size() > 0)
+							println " "+f.getName()
+						if (tmp1.size() > 0)
+							println "  Removed keys : "+tmp1;
+						if (refentries.size() > 0)
+							println "  Added keys : "+refentries;
+					}
+				}
+			}
+		}
+	}
+
+	/**
+	 * Update files.
+	 */
+	public void updateFiles()
+	{
+		for(String dirfile : dirfiles)
+		{
+			List<File> files = propertyfiles.get(dirfile);
+			for(File f : files)
+			{
+				List<String> entries =  this.fileentries.get(f);
+				HashMap<String, String> values = this.fileentriesvalues.get(f);
+				
+				Writer writer = new OutputStreamWriter(new FileOutputStream(f) , "iso-8859-1");
+				for(String entry : entries)
+				{
+					writer.write(entry+"="+values.get(entry)+"\n");
+				}
+				writer.close()
+			}
+		}
+	}
+
+	/**
+	 * Creates the missing files.
+	 *
+	 * @param suffix the suffix
+	 */
+	public void createMissingFiles(String suffix)
+	{
+		println "Looking for missing messages files "+ suffix;
+		for(String dirfile : dirfiles)
+		{
+			//println "DIRECTORY : "+dirfile
+			File reference = null;
+			String lookingname = "";
+			List<File> files = propertyfiles.get(dirfile)
+			if(files.get(0).getName().startsWith("m"))
+			{
+				reference = new File(dirfile,"messages.properties");
+				lookingname = "messages_"+suffix+".properties"
+			}
+			else if (files.get(0).getName().startsWith("b"))
+			{
+				reference = new File(dirfile,"bundle.properties");
+				lookingname = "bundle_"+suffix+".properties"
+			}
+			
+			boolean create = true;
+			if(reference != null && reference.exists())
+			{
+				for(File f : files)
+				{
+					if(f.getName() == lookingname)
+						create = false;
+				}
+			}
+			if(create)
+			{
+				new File(dirfile,lookingname).createNewFile();
+				println "Create file " +new File(dirfile,lookingname)
+			}
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	public String toString()
+	{
+		String rez = "";
+		for(String dirfile : dirfiles)
+		{
+			rez += dirfile+"\n";
+			for(File f : propertyfiles.get(dirfile))
+				rez += "  "+f.getName()+"\n"
+		}
+		return rez;
+	}
+
+	/**
+	 * Scan directory.
+	 *
+	 * @param directory the directory
+	 */
+	public void scanDirectory(File directory)
+	{
+		if(!directory.exists())
+		{
+			println "directory '$directory' does not exists"
+			return;
+		}
+		
+		println "scan directory : "+directory.getAbsolutePath();
+		LinkedList<File> files = new LinkedList<File>();
+		files.add(directory);
+		
+		while(!files.isEmpty())
+		{
+			File current = files.removeFirst();
+			if(current.isDirectory())
+			{
+				List<String> currentpfiles = [];
+				for(File sfile : current.listFiles())
+				{
+					if(sfile.isDirectory())
+						files.add(sfile);
+					else if(sfile.getName().endsWith(".properties") && ( sfile.getName().startsWith("messages") || sfile.getName().startsWith("bundle")) )
+						currentpfiles.add(sfile)
+				}
+				if(currentpfiles.size() > 0)
+				{
+					dirfiles.add(current.getAbsolutePath());
+					propertyfiles.put(current.getAbsolutePath(), currentpfiles);
+				}
+			}
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String userdir = System.getProperty("user.home");
+
+		println "\nRCP\n";
+		
+		ExternalizationFilesUpdater scanner = new ExternalizationFilesUpdater();
+//		scanner.scanDirectory(new File(userdir, "workspace37/org.txm.rcp")); // find directories with a messages.properties file
+//		scanner.createMissingFiles("fr"); // create messages_fr.properties files when a messages.properties is found
+//		scanner.processFoundDirectories(); // find missing and obsolets keys
+//		scanner.updateFiles(); // update messages files content
+		
+		println "\nTOOLBOX\n";
+		
+		scanner = new ExternalizationFilesUpdater();
+		scanner.scanDirectory(new File(userdir, "workspace43/org.txm.rcp/src/main/java"));
+		scanner.createMissingFiles("ru");
+		scanner.processFoundDirectories();
+		scanner.updateFiles();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/I18nDiff.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/I18nDiff.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/I18nDiff.groovy	(revision 321)
@@ -0,0 +1,84 @@
+package org.txm.scripts.i18n
+
+import java.io.File;
+import java.util.HashMap;
+/**
+ * Display a diff of 2 message files
+ * 
+ * @author mdecorde
+ *
+ */
+class I18nDiff {
+	Properties values1, values2;
+	File file1;
+	File file2;
+
+	def notInValues1, notInValues2
+	I18nDiff(File file1, File file2) {
+		this.file1 = file1
+		this.file2 = file2
+		values1 = getValues(file1)
+		values2 = getValues(file2)
+
+		def inter = values1.keySet().intersect(values2.keySet());
+		notInValues1 = values2.keySet().minus(values1.keySet());
+		notInValues2 = values1.keySet().minus(values2.keySet());
+
+		println "not in $file1"
+		for(String key : notInValues1)
+			if (key.trim().length() > 0)
+				println key+" = "+values2.get(key)
+
+		println "not in $file2"
+		for(String key : notInValues2)
+			if (key.trim().length() > 0)
+				println key+" = "+values1.get(key)
+
+		//		println "values diff"
+		//
+		//		for(String key : inter ) {
+		//			if(values1.get(key) != values2.get(key))
+		//				println key+" = "+values1.get(key)+" != "+values2.get(key)
+		//		}
+	}
+
+	public void synchronizeFromF1toF2() {
+		
+		for(String key : notInValues2) {
+			if (key.trim().length() > 0)
+				values2.setProperty(key, "NA_"+values1.get(key));
+		}
+		
+		for(String key : notInValues1)
+			if (key.trim().length() > 0)
+				values2.remove(key)
+				
+		file2.withWriter("iso-8859-1") { writer ->
+			values2.store(writer, "")
+		}
+	}
+
+	/**
+	 * Gets the values.
+	 *
+	 * @param file the file
+	 * @return the values
+	 */
+	public Properties getValues(File file) {
+		Properties props = new Properties();
+		file.withReader("iso-8859-1") { input->
+			props.load(input);
+			input.close()
+		}
+		return props;
+	}
+
+	public static void main(String[] args) {
+		String userhome = System.getProperty("user.home");
+		File f1 = new File(userhome, "workspace43/org.txm.rcp/OSGI-INF/l10n/bundle.properties")
+		File f2 = new File(userhome, "workspace43/org.txm.rcp/OSGI-INF/l10n/bundle_fr.properties")
+		
+		I18nDiff diff = new I18nDiff(f1, f2);
+		diff.synchronizeFromF1toF2()
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/MergeExternalizedStringFiles.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/MergeExternalizedStringFiles.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/MergeExternalizedStringFiles.groovy	(revision 321)
@@ -0,0 +1,293 @@
+package org.txm.scripts.i18n
+
+import java.io.File;
+import org.eclipse.osgi.util.NLS;
+import org.txm.utils.DeleteDir;
+
+/**
+ * Scan for *.properties files and merge them into one
+ * @author mdecorde
+ *
+ */
+class MergeExternalizedStringFiles {
+	
+	ArrayList<File> alldirs;
+	LinkedHashMap<File, ArrayList> data = [:];
+	
+	public MergeExternalizedStringFiles(File projectDir)
+	{
+		alldirs = scanDirectory(projectDir);
+		alldirs.sort{it.getAbsolutePath();}
+		//System.out.println("All dirs size: "+alldirs.size);
+		findMessageClasses();
+		mergeAll()
+	}
+	
+	public findMessageClasses()
+	{
+		File messageClass;
+		ArrayList<File> externalizeMessages;
+		for(int i = 0 ; i < alldirs.size() ; i++) {
+			File dir = alldirs.get(i)
+			if (!dir.isDirectory())
+				continue;
+			//System.out.println("Dir: "+dir);
+			externalizeMessages = new ArrayList<File>();
+			for (File f : dir.listFiles()) {
+				if (f.getName() == "Messages.java")
+					messageClass = f
+				if (f.getName() =~ "messages(_..)?\\.properties")
+					externalizeMessages << f
+			}
+			
+			if (messageClass != null) {
+				String bundle = findBundle(messageClass);
+				data.put(dir, [messageClass, bundle, externalizeMessages])
+			} else {
+				alldirs.remove(i);
+				i--;
+			}
+		}
+		println "all dirs size: "+alldirs.size()
+		//		for(def f : alldirs)
+		//			println f
+	}
+	
+	public String findBundle(File message)
+	{
+		String ret;
+		message.eachLine("iso-8859-1") {
+			if (it.contains("private static final String BUNDLE_NAME = ")) {
+				int start = it.indexOf("\"")
+				int end = it.lastIndexOf("\"")
+				ret = it.substring(start+1, end)
+			}
+		}
+		return ret;
+	}
+	
+	/**
+	 * Scan directory.
+	 *
+	 * @param directory the directory
+	 */
+	public ArrayList<File> scanDirectory(File directory)
+	{
+		ArrayList<File> dirfiles = [];
+		
+		if (!directory.exists()) {
+			println "directory '$directory' does not exists"
+			return;
+		}
+		
+		println "scan directory : "+directory.getAbsolutePath();
+		LinkedList<File> files = new LinkedList<File>();
+		files.add(directory);
+		
+		while (!files.isEmpty()) {
+			File current = files.removeFirst();
+			if (current.isDirectory() && !current.getName().startsWith(".")) {
+				//println "add dir: "+current
+				List<String> currentpfiles = [];
+				for (File sfile : current.listFiles()) {
+					if (sfile.isDirectory())
+						files.add(sfile);
+				}
+				
+				dirfiles.add(current);
+			}
+		}
+		return dirfiles;
+	}
+	
+	public void mergeAll()
+	{
+		def files = new ArrayList<File>(data.keySet());
+		println files
+		if (files.size() < 1)
+			return;
+		
+		File refDir = files.get(0);
+		File refClass = data.get(refDir)[0];
+		String refBundle = data.get(refDir)[1];
+		def externalized = data.get(refDir)[2];
+		
+		System.out.println("ref dir: "+refDir);
+		System.out.println("ref class: "+refClass);
+		System.out.println("ref bundle: "+refBundle);
+		System.out.println("ref files: "+externalized);
+		
+		// get externalized ref strings
+		//      Lang            Key     externalized string 
+		HashMap<String, HashMap<String, String>> externalizedLines = [:];
+		for (File source : externalized) {
+			def lines = new HashMap<String, String>();
+			externalizedLines.put(source.getName(), lines);
+			source.eachLine("iso-8859-1"){
+				if (it =~ ".+=.+") {
+					int idx = it.indexOf("=");
+					lines.put(it.substring(0, idx), it.substring(idx+1))
+				} else if(it =~ ".+=\$") {
+					int idx = it.indexOf("=")
+					String key = it.substring(0, idx)
+					String value = ""
+					lines.put(key, value)
+				}
+			}
+		}
+		
+		// get references keys
+		HashSet<String> refKeys = new HashSet<String>();
+		refClass.eachLine("iso-8859-1") {
+			if (it =~ ".+public static String .+")
+				refKeys << it.trim()
+		}
+		
+		println "BEFORE"
+		println "refKeys size: "+refKeys.size();
+		println "fr externalizedLines size: "+externalizedLines.get("messages_fr.properties").size();
+		println "en externalizedLines size: "+externalizedLines.get("messages.properties").size();
+		
+		// for all sub dir
+		for (int i = 1 ; i < files.size() ; i++) {
+			File srcDir = files.get(i);
+			File srcClass = data.get(srcDir)[0];
+			String srcBundle = data.get(srcDir)[1];
+			
+			def srcExternalized = data.get(srcDir)[2];
+			// merge externalized strings
+			for (File source : srcExternalized) {
+				def lines = externalizedLines.get(source.getName());
+				source.eachLine("iso-8859-1") {
+					if (it =~ ".+=.+") {
+						int idx = it.indexOf("=")
+						String key = it.substring(0, idx)
+						String value = it.substring(idx+1)
+						if (lines.containsKey(key) && lines.get(key) != value)
+							println "merged message of "+source.getName()+": "+it+ " >> "+lines.get(key)
+						lines.put(key, value)
+					} else if(it =~ ".+=\$") {
+						int idx = it.indexOf("=")
+						String key = it.substring(0, idx)
+						String value = ""
+						lines.put(key, value)
+					}
+				}
+				source.delete(); // bye bye *.properties files
+			}
+						
+			// merge static String
+			if(!srcClass.exists())
+			{
+				println "file removed: "+srcClass
+				println  srcDir
+				println  srcClass
+				println  srcBundle
+				println  srcExternalized
+				continue;
+			}
+			srcClass.eachLine("iso-8859-1"){
+				if(it =~ ".+public static String .+")
+				{
+//					if(refKeys.contains(it.trim()))
+//						println "Externalized key merged: "+it
+					refKeys << it.trim()
+				}
+			}
+			srcClass.delete(); // bye bye Messages.java
+		}
+		
+		println "AFTER"
+		println "refKeys size: "+refKeys.size();
+		println "fr externalizedLines size: "+externalizedLines.get("messages_fr.properties").size();
+		println "en externalizedLines size: "+externalizedLines.get("messages.properties").size();
+		
+		//check unused externalized strings
+		def keysDeclared = refKeys.collect{it.substring(21, it.length() -1) };
+		System.out.println("number of keys declared in Message.java: "+keysDeclared.size());
+		def missingDeclarations = new HashMap<String, HashSet<String>>();
+		def missingExternalizations = new HashMap<String, HashSet<String>>();
+		for(def lang : externalizedLines.keySet())
+		{
+			def keys = externalizedLines.get(lang).keySet(); // keys  in propeties files
+			missingExternalizations.put(lang, new HashSet<String>());
+			missingDeclarations.put(lang, new HashSet<String>());
+			for(String key : keys)
+				if(!keysDeclared.contains(key))
+					missingDeclarations.get(lang) << key
+					
+			for(String key : keysDeclared)
+				if(!keys.contains(key))
+					missingExternalizations.get(lang) << key
+		}
+		
+		println "fix missing key declaration and externalization"
+		for(String lang: missingExternalizations.keySet())
+		{
+			println "Lang: "+lang
+			println " missingDeclarations: "+missingDeclarations.get(lang)
+			for(String key : missingDeclarations.get(lang))
+				refKeys << "public static String "+key+";"
+			println " missingExternalizations: "+missingExternalizations.get(lang)
+			for(String key : missingExternalizations.get(lang))
+				externalizedLines.get(lang).put(key, "N/A")
+		}
+		
+		println "AFTER FIX"
+		println "refKeys size: "+refKeys.size();
+		println "fr externalizedLines size: "+externalizedLines.get("messages_fr.properties").size();
+		println "en externalizedLines size: "+externalizedLines.get("messages.properties").size();
+		
+	
+		// write externalized lines
+		for(File source : externalized)
+		{
+			source.withWriter("iso-8859-1") { out ->
+				def hash = externalizedLines.get(source.getName())
+				def keys = hash.keySet();
+				keys = keys.sort()
+				keys.each() { 
+					out.writeLine(it+"="+hash.get(it))
+				}
+			}
+		}
+
+		// write classRef File
+		refClass.withWriter("iso-8859-1") { out ->			
+			// write start
+			out.writeLine('''
+package org.txm.;
+
+import org.eclipse.osgi.util.NLS;
+
+public class Messages extends NLS {
+
+	// The Constant BUNDLE_NAME. 
+	private static final String BUNDLE_NAME = "org.txm.messages"; //$NON-NLS-1$
+	
+				''')
+			
+			//write keys
+			for(String key : refKeys.sort())
+				out.writeLine("\t"+key);
+			
+			// write end
+			out.writeLine('''
+	static {
+		// initialize resource bundle
+		NLS.initializeMessages(BUNDLE_NAME, Messages.class);
+	}
+
+	private Messages() {
+	}
+}
+				''')
+		}
+	}
+	
+	public static void main(String[] args)
+	{
+		File projectdir = new File("/home/mdecorde/workspace37/org.txm.toolbox/src/java")
+		def rez = new MergeExternalizedStringFiles(projectdir);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/IsStringUsedInPlugin.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/IsStringUsedInPlugin.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/IsStringUsedInPlugin.groovy	(revision 321)
@@ -0,0 +1,63 @@
+package org.txm.scripts.i18n
+
+/**
+ * Display the String and key not used in the plugin messages
+ * @author mdecorde
+ *
+ */
+class IsStringUsedInPlugin {
+	Properties props;
+
+	public boolean scan(File propertyFile, File pluginFile) {
+		if(!(propertyFile.exists() && propertyFile.canRead() && propertyFile.canWrite() && propertyFile.isFile()))
+		{
+			println "error file : "+propertyFile
+			return false;
+		}
+
+		props = new Properties();
+		propertyFile.withReader("iso-8859-1") { input->
+			props.load(input);
+			input.close()
+		}
+
+		String content = pluginFile.getText();
+		def toRemove = [];
+		for(def key : props.keySet()) {
+			if (!content.contains("\"%"+key+"\"")) {
+				println "not used $key "+props.get(key);
+				toRemove << key
+			}
+		}
+
+		for(String key : toRemove) {
+			props.remove(key);
+		}
+
+
+		if (toRemove == 0) {
+			println "nothing to do"
+			return false;
+		}
+		return true
+	}
+
+	public boolean writeFixes(File propertyFile) {
+		if (props == null) return false;
+		propertyFile.withWriter("iso-8859-1") { output ->
+			props.store(output, "")
+		}
+		return true;
+	}
+
+	public static void main(String[] args) {
+		String userhome = System.getProperty("user.home")
+		File propertyFile = new File(userhome, "workspace43/org.txm.rcp/OSGI-INF/l10n/bundle.properties");
+		File pluginFile = new File(userhome, "workspace43/org.txm.rcp/plugin.xml");
+		IsStringUsedInPlugin fixer = new IsStringUsedInPlugin();
+		//uncomment to remove unused Strings
+		if (fixer.scan(propertyFile, pluginFile)) {
+			fixer.writeFixes(propertyFile)
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizerUI.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizerUI.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizerUI.groovy	(revision 321)
@@ -0,0 +1,145 @@
+package org.txm.scripts.i18n;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.table.*
+import java.awt.GridLayout
+import java.awt.event.*
+
+import org.txm.utils.LS;
+/**
+ * This replace the eclipse externalisation interface
+ * Scan the java files for not translated Strings and update the messages.properties file
+ * 
+ * uses ExternalizeStringFile
+ * @author mdecorde
+ *
+ */
+class ExternalizeUI extends JPanel {
+	ExternalizeStringFile externalizer;
+
+	ExternalizeUI() {
+		super(new FlowLayout());
+		File userDir = new File(System.getProperty("user.home"))
+		File srcDir = new File(userDir, "workspace43/org.txm.rcp/src/main/java")
+		File propFile = new File(srcDir, "org/txm/rcpapplication/messages.properties")
+		File messageFile = new File(srcDir, "org/txm/rcpapplication/Messages.java")
+
+		println userDir
+		println srcDir
+		println propFile
+		println messageFile
+
+		String[] columnNames = ["Files in "+srcDir.getPath(), "Nb of changes"];
+
+
+		def files = LS.list(srcDir, true, false);
+		files = files.findAll { file -> file.getName().endsWith(".java") }
+		files = files.sort()
+		println files
+
+		externalizer = new ExternalizeStringFile(propFile, messageFile);
+		println "externalizer ready"
+		def data = []
+		def selectedFiles = []
+		for (File file : files) {
+			int count = externalizer.getNbOfChanges(file)
+			if (count != 0)	{
+				selectedFiles << file
+				data << [file.getPath().substring(srcDir.getPath().length()), count]
+			}
+		}
+		Object[][] arrayData = data;
+		JTable table = new JTable(arrayData, columnNames);
+		table.getColumnModel().getColumn(0).setPreferredWidth(600);
+		table.getColumnModel().getColumn(1).setPreferredWidth(100);
+
+		table.addMouseListener(new MouseAdapter() {
+					@Override
+					public void mouseClicked(MouseEvent e) {
+						if (SwingUtilities.isLeftMouseButton(e) && e.getClickCount() == 1) {
+							int selectedRowIndex = table.getSelectedRow();
+							int c = externalizer.process(selectedFiles[selectedRowIndex])
+							println ""+c
+							int after = table.getValueAt(table.getSelectedRow(), 1);
+							table.setValueAt(after - c, table.getSelectedRow(), 1);
+							//show your dialog with the selected row's contents
+						}
+					}
+				});
+
+		JScrollPane scrollPane = new JScrollPane(table);
+		table.setFillsViewportHeight(true);
+
+		add(scrollPane)
+
+		JButton cancelButton = new JButton("CANCEL");
+		JButton okButton = new JButton("OK + SAVE");
+		okButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e)
+			{
+				ExternalizeUI.this.doSave();
+				System.exit(0);
+			}
+		});
+		
+		cancelButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e)
+			{
+				println "Changes not saved: "
+				System.exit(0);
+			}
+		});
+		cancelButton.setSize(100, 20);
+		okButton.setSize(100, 20);
+		add(cancelButton)
+		add(okButton)
+
+		println "end of constructor"
+	}
+
+	public void doSave() {
+		if (externalizer != null && externalizer.saveChanges()) {
+			JOptionPane.showMessageDialog(this, "Changes saved");
+		}
+	}
+
+	/**
+	 * Create the GUI and show it.  For thread safety,
+	 * this method should be invoked from the
+	 * event-dispatching thread.
+	 */
+	private static void createAndShowGUI() {
+		//Create and set up the window.
+		JFrame frame = new JFrame("ExternalizeUI");
+		frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+
+		//Create and set up the content pane.
+		ExternalizeUI newContentPane = new ExternalizeUI();
+		newContentPane.setOpaque(true); //content panes must be opaque
+
+		//		frame.addWindowListener(new java.awt.event.WindowAdapter() {
+		//			public void windowClosing(WindowEvent winEvt) {
+		//				newContentPane.doSave();
+		//				System.exit(0);
+		//			}
+		//		});
+		frame.setContentPane(newContentPane);
+
+		println "display"
+		//Display the window.
+		frame.pack();
+		frame.setVisible(true);
+	}
+
+
+	public static void main(String[] args) {
+		//Schedule a job for the event-dispatching thread:
+		//creating and showing this application's GUI.
+		javax.swing.SwingUtilities.invokeLater(new Runnable() {
+					public void run() {
+						createAndShowGUI();
+					}
+				});
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/FixExternalizedDuplicatedStrings.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/FixExternalizedDuplicatedStrings.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/FixExternalizedDuplicatedStrings.groovy	(revision 321)
@@ -0,0 +1,135 @@
+package org.txm.scripts.i18n
+
+import org.txm.utils.DeleteDir;
+
+/**
+ * Browse messages files and look for duplicated values
+ * @author mdecorde
+ *
+ */
+class FixExternalizedDuplicatedStrings {
+	
+	static main(args) {
+//		File propertyFile = new File("/home/mdecorde/Bureau/Experiences/rcpapplication/messages.properties");
+//		File messageClass = new File("/home/mdecorde/Bureau/Experiences/rcpapplication/Messages.java");
+//		File srcdirectory = new File("/home/mdecorde/Bureau/Experiences/rcpapplication");
+		File propertyFile = new File("/home/mdecorde/workspace37/org.txm.rcp/src/main/java/org/txm/rcpapplication/messages.properties");
+		File messageClass = new File("/home/mdecorde/workspace37/org.txm.rcp/src/main/java/org/txm/rcpapplication/Messages.java");
+		File srcdirectory = new File("/home/mdecorde/workspace37/org.txm.rcp/src/main/java/org/txm/rcpapplication");
+		if (!(propertyFile.exists() && propertyFile.canRead() && propertyFile.canWrite() && propertyFile.isFile())) {
+			println "error file : "+propertyFile
+			return
+		}
+		
+		// build hash of externalised Strings
+		//						  key	  externalized string
+		def entries = new HashMap<String, String>()
+		propertyFile.eachLine("iso-8859-1"){
+			if(it =~ ".+=.+")
+			{
+				int idx = it.indexOf("=")
+				String key = it.substring(0, idx)
+				String value = it.substring(idx+1)
+				entries.put(key, value)
+			}
+			else if(it =~ ".+=\$")
+			{
+				int idx = it.indexOf("=")
+				String key = it.substring(0, idx)
+				String value = ""
+				entries.put(key, value)
+			}
+		}
+		
+		// invert hash
+		def hash = entries
+		def reverse = [:]
+		hash.each{
+			if(reverse[it.value] == null)
+				reverse[it.value] = []
+			reverse[it.value] << it.key
+		};
+		
+		for(def key : reverse.keySet())
+			if(reverse[key].size() > 1)
+				println "doublons of $key: "+reverse[key]
+		
+		// translate dictionnary
+		def dictionnary = [:]
+		for(def value : reverse.keySet())
+		{
+			def keys = reverse[value];
+			def first = keys[0];
+			for(int i = 1 ; i < keys.size() ; i++) // the first key will not be removed
+			{
+				def key = keys.get(i)
+				if(key != first)
+				{
+					dictionnary.put(key , first)
+					println key+ " >> "+first
+				}
+			}
+		}
+		
+		// patch source files : iterates over all java files
+		
+		/*def files = DeleteDir.scanDirectory(srcdirectory, true);
+		for(File f : files)
+		{
+			if(f.getName().endsWith(".java") && f.getName() != "Messages.java")
+			{
+				def lines = f.readLines("UTF-8")
+				f.withWriter("UTF-8"){ writer ->
+					
+					for(String line : lines)
+					{
+						for(String key : dictionnary.keySet())
+						{
+							String old = line;
+							//line = line.replaceAll(, "Messages."+dictionnary[key]);
+							line = (line =~ /Messages\.$key([^0-9]|$)/).replaceAll("Messages."+dictionnary[key]+'$1')
+
+							if(old != line)
+								System.out.println(f.getAbsolutePath()+":"+old+": replaced "+"Messages\\."+key+"[^0-9] Messages."+dictionnary[key]);
+						}
+						writer.writeLine(line);
+					}	
+				}
+			}
+		}
+		
+		// remove key declaration from Messages.java
+		String txt = messageClass.getText("UTF-8")
+		String[] split = txt.split("\n");
+		messageClass.withWriter "UTF-8", { writer -> 
+			for(String line : split)
+			{
+				if(line.contains("public static String "))
+				{
+					for(String key : dictionnary.keySet())
+					{
+						if(line.contains("public static String "+key+";"))
+						{
+							line = line.trim()
+							line = null
+							println "remove key: "+key
+							break;
+						}
+					}
+					if(line != null)
+						writer.writeLine(line);
+				}
+				else
+					writer.writeLine(line);
+			}
+		}*/
+		
+		// rewrite messages.properties
+		def keytoRemove = dictionnary.keySet();
+		propertyFile.withWriter("UTF-8") { writer ->
+			for(String key : entries.keySet().sort())
+				if(!keytoRemove.contains(key))
+					writer.writeLine(key+"="+entries[key])
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/Merge2PropertiesFile.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/Merge2PropertiesFile.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/Merge2PropertiesFile.groovy	(revision 321)
@@ -0,0 +1,20 @@
+package org.txm.scripts.i18n
+
+File propFile1 = new File("/home/mdecorde/workspace43/org.eclipse.equinox.p2.ui.nl_fr/org/eclipse/equinox/internal/p2/ui/messages_fr.properties")
+File propFile2 = new File("/home/mdecorde/workspace43/org.txm.rcp.p2.ui/src/org/eclipse/equinox/internal/p2/ui/messages_fr.properties")
+
+Properties props1 = new Properties()
+props1.load(propFile1.newReader("iso-8859-1"))
+Properties props2 = new Properties()
+props2.load(propFile2.newReader("iso-8859-1"))
+
+println "props1.size = "+props1.size()
+println "props2.size = "+props2.size()
+
+for( def key : props1.keySet()) {
+	println "update: "+key
+	props2.put(key, props1.get(key))
+}
+
+println "props2.size AFTER = "+props2.size()
+props2.store(propFile2.newWriter("iso-8859-1"), "$propFile1 + $propFile2")
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ReverseI18nDict.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ReverseI18nDict.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ReverseI18nDict.groovy	(revision 321)
@@ -0,0 +1,125 @@
+package org.txm.scripts.i18n
+
+import org.txm.utils.BiHashMap
+
+/**
+ * Bihashmap 
+ * Check any missing key in both of messages.properties and Message.java files
+ * @author mdecorde
+ *
+ */
+class ReverseI18nDict extends BiHashMap<String, String> {
+	File propFile;
+	File messageFile;
+	
+	def messageKeys = new HashSet<String>()
+	
+	public ReverseI18nDict(File propFile) {
+		this(propFile, null)
+	}
+	
+	public ReverseI18nDict(File propFile, File messageFile) {
+		this.propFile = propFile
+		this.messageFile = messageFile
+		BufferedReader reader = new BufferedReader(new InputStreamReader(
+				new FileInputStream(propFile), "iso-8859-1"));
+		String line = reader.readLine();
+		while (line != null) {
+			if (!line.startsWith("#")) {
+				String[] split = line.split("=", 2);
+				if (split.length == 2) {
+					this.put(split[0], split[1]);
+				}
+			}
+			line = reader.readLine();
+		}
+		
+		if (messageFile != null) {
+			BufferedReader reader2 = new BufferedReader(new InputStreamReader(new FileInputStream(messageFile), "iso-8859-1"));
+			String line2 = reader2.readLine();
+			while (line2 != null) {
+				line2 = line2.trim();
+				if (line2.startsWith("public static String")) {
+					line2 = line2.substring(21, line2.length() -1);
+					messageKeys << line2
+				}
+				line2 = reader2.readLine();
+			}
+		}
+	}
+	
+	public def getMissingsMessageKeys() {
+		def missing = []
+		for (String pKey : this.getKeys())
+			if (!messageKeys.contains(pKey))
+				missing << pKey
+				
+		return missing
+	}
+	
+	public getMissingPropKeys() {
+		def missing = []
+		def pKeys = this.getKeys()
+		for (String mKey : messageKeys)
+			if (!pKeys.contains(mKey))
+				missing << mKey
+				
+		return missing
+	}
+	
+	public void saveChanges() {
+
+		//Write prop File
+		File oldFile = new File(propFile.getParentFile(), propFile.getName()+".cpy");
+		propFile.renameTo(oldFile);
+		propFile.withWriter("iso-8859-1") { out ->
+			for(String key : this.getKeys().sort()) {
+				out.println(key+"="+this.get(key))
+			}
+		}
+		
+		// write message File
+		if (messageFile == null) return;
+		
+		File oldFile2 = new File(messageFile.getParentFile(), messageFile.getName()+".cpy");
+		messageFile.renameTo(oldFile2); // back up
+		messageFile.withWriter("iso-8859-1") { out ->
+			// write start
+			out.println('''package org.txm;
+
+import org.eclipse.osgi.util.NLS;
+
+public class Messages extends NLS {
+	
+	// The Constant BUNDLE_NAME. 
+	private static final String BUNDLE_NAME = "org.txm.messages"; //$NON-NLS-1$
+				''')
+			
+			//write keys
+			for(String key : this.getKeys().sort())
+				out.println("\tpublic static String $key;");
+			
+			// write end
+			out.println('''
+	static {
+		// initialize resource bundle
+		NLS.initializeMessages(BUNDLE_NAME, Messages.class);
+	}
+
+	private Messages() {
+	}
+}
+				''')
+		}
+	}
+	
+	public static void main(String[] args) {
+		File propFile = new File("/home/mdecorde/workspace43/org.txm.toolbox/src/java/org/txm/messages.properties")
+		File messageFile = new File("/home/mdecorde/workspace43/org.txm.toolbox/src/java/org/txm/Messages.java")
+		ReverseI18nDict dict = new ReverseI18nDict(propFile, messageFile);
+		println dict.getMissingsMessageKeys()
+		println dict.getMissingPropKeys();
+		println dict.toString();
+		//dict.saveChanges();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizeStringFile.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizeStringFile.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/ExternalizeStringFile.groovy	(revision 321)
@@ -0,0 +1,263 @@
+package org.txm.scripts.i18n
+
+import javax.swing.*;
+
+/**
+ * Scan a java file to find unexternalized Strings
+ * create the Messages.java entries and the *.properties entries too
+ * replace externalized Strings in the java file
+ * 
+ * uses a ReverseI18nDict
+ * 
+ * @author mdecorde
+ */
+class ExternalizeStringFile {
+
+	File propFile;
+	File messageFile
+
+	ReverseI18nDict dict
+
+	String prefix
+	int iprefix = 0;
+	
+	boolean mustSaveChanges = false;
+
+	public ExternalizeStringFile(File propFile, File messageFile) {
+		dict = new ReverseI18nDict(propFile, messageFile);		
+	}
+	
+	private ArrayList<String> getLineStrings(String l) {
+		def strings = []
+		boolean inS = false;
+		int start = 0;
+		int comment = 0;
+		for(int i = 0 ; i < l.length() ; i++) {
+			def c = l[i]
+			if (inS) {
+				comment = 0
+				if (c == "\\") {
+					i++;
+				} else if (c == "\"") {
+					inS = false;
+					strings << l.substring(start, i+1);
+				}
+			} else {
+				if (c == "\"") {
+					inS = true;
+					start = i;
+					comment = 0
+				} else if (c == "/") {
+					comment++;
+					if (comment == 2) break;
+				} else {
+					comment = 0
+				}
+			}
+		}
+		return strings;
+	}
+	
+	BufferedReader reader;
+	String line;
+	String previousLine;
+	boolean ignore = false;
+	int nline = 1;
+
+	def nlserrors = []
+	def linesToWrite = []
+
+	int nTotalChanges = 0;
+	
+	def strings;
+	def nonnls;
+	def toExternalize = []
+	def toNLSize = []
+	
+	private int processSourceFile(File sourceFile, boolean processMode) {
+		
+		prefix = sourceFile.getName();
+		prefix = prefix.substring(0, prefix.indexOf("."))+"_" ;
+		iprefix = 0;
+		
+		reader = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile), "UTF-8"));
+		line = reader.readLine();
+		previousLine;
+		ignore = false;
+		nline = 1;
+
+		nlserrors = []
+		linesToWrite = []
+
+		nTotalChanges = 0;
+		
+		while (line != null) {
+			String l = line.trim()
+			if (ignore) {
+				if (l.startsWith("*/") || l.endsWith("*/")) {
+					ignore = false
+				} else if (l.startsWith("*")) {
+
+				} else {
+//					println "ERROR LINE $nline : not a comment ? "+l
+//					return -1;
+				}
+			} else {
+				if (l.startsWith("//")) {
+					// ignore
+				} else if (l.startsWith("@")) {
+					// ignore
+				} else if (l.startsWith("/*")) {
+					ignore = true;
+					if (l.endsWith("*/")) ignore=false
+				} else {
+					strings = getLineStrings(l);
+					nonnls = getNONNLS(l);
+					toExternalize = []
+					toNLSize = []
+
+					// fix NON-NLS
+					if (nonnls.size() > 0 && nonnls.size() != strings.size()) {
+						println "NLS error line $nline: "+l
+						nlserrors << nline
+					}
+
+					if (nlserrors.size() == 0) {
+						// quick pre process
+						for (int i = 0 ; i < strings.size() ; i++) {
+							if (nonnls.contains(i)) continue; // ignore
+
+							if (strings[i] == "\"\"" || strings[i] == "\" \"" || strings[i] == "\"\\n\"") {
+								toNLSize << i;
+							} else {
+								toExternalize << i
+							}
+						}
+
+						if (processMode) {
+							nTotalChanges += processLine();
+						} else {
+							nTotalChanges += (toExternalize.size() + toNLSize.size());
+						}
+					}
+				}
+			}
+
+			linesToWrite << line
+			previousLine = line;
+			line = reader.readLine();
+			nline++;
+		}
+
+		if (nlserrors.size() > 0) {
+			println "NLS ERROR "+sourceFile.getName()+" lines $nlserrors"
+			return -1;
+		} else if (processMode && nTotalChanges > 0) {
+			mustSaveChanges = true;
+			File oldFile = new File(sourceFile.getParentFile(), sourceFile.getName()+".cpy");
+			if (!sourceFile.renameTo(oldFile)) {
+				println "Failed to backup old source file $sourceFile to $oldFile"
+				return 0;
+			} else {
+				sourceFile.withWriter("UTF-8") { writer ->
+					for(String l : linesToWrite) {
+						writer.println l
+					}
+				}
+			}
+		}
+		return nTotalChanges
+	}
+	
+	private int processLine() {
+		int nChanges = 0;
+		int ichar = 0;
+		for (int i = 0 ; i < strings.size() ; i++) {
+			ichar = line.indexOf(strings[i], ichar);
+			if (toExternalize.contains(i)) {
+				println "Line $nline : $strings $nonnls $toNLSize $toExternalize"
+				nChanges++;
+				String value = strings[i].substring(1,strings[i].length() -1)
+				if (dict.containsValue(value)) {
+					String str = "Messages."+dict.getKey(value)
+					println " automatic replace "+strings[i]+" with Messages."+dict.getKey(value);
+					String p1 = line.substring(0, ichar);
+					String p2 = line.substring(ichar+strings[i].length(), line.length())
+					println " old: $line"
+					line = p1+str+p2
+					println " new: $line"
+				} else {
+					String key = getNextKey();
+					String ok = JOptionPane.showInputDialog(null, "Key for value=$value\n\n$previousLine\n$line\n", key);
+					if(ok == null) {
+						println " add new NON-NLS "+(i+1)
+						println " old: $line"
+						line += " //\$NON-NLS-"+(i+1)+"\$"
+						println " new: $line"
+					} else {
+						println " add new key: $key"
+						dict.put(key, value);
+
+						String p1 = line.substring(0, ichar);
+						String p2 = line.substring(ichar+strings[i].length(), line.length())
+						String str = "Messages."+key
+						println " old: $line"
+						line = p1+str+p2
+						println " new: $line"
+					}
+				}
+			} else if (toNLSize.contains(i)) {
+				println "Line $nline : $strings $nonnls $toNLSize $toExternalize"
+				nChanges++;
+				println " add automatic new NON-NLS "+(i+1)
+				println " old: $line"
+				line += " //\$NON-NLS-"+(i+1)+"\$"
+				println " new: $line"
+			}
+		}
+		return nChanges;
+	}
+
+	private def getNONNLS(String l) {
+		def idx = []
+		def m = l =~ /\/\/.NON-NLS-[0-9]\$/
+		m.each { idx << Integer.parseInt(it[11])-1 }
+		return idx.sort()
+	}
+
+	private def getNextKey() {
+		String tmp = prefix+(iprefix++);
+		while (dict.containsKey(tmp)) {
+			tmp = prefix+(iprefix++);
+		}
+		return tmp;
+	}
+	
+	int getNbOfChanges(File sourceFile) {
+		return processSourceFile(sourceFile, false);
+	}
+	
+	int process(File sourceFile) {
+		return processSourceFile(sourceFile, true);
+	}
+	
+	public boolean saveChanges() {
+		if (mustSaveChanges ) {
+			mustSaveChanges = false;
+			dict.saveChanges();
+			return true;
+		}
+		return false;
+	}
+
+	public static void main(String[] args) {
+		File sourceFile = new File("/home/mdecorde/workspace37/org.txm.toolbox/src/java/org/txm/functions/cooccurrences/Cooccurrence.java");
+		File propFile = new File("/home/mdecorde/workspace37/org.txm.toolbox/src/java/org/txm/messages.properties")
+		File messageFile = new File("/home/mdecorde/workspace37/org.txm.toolbox/src/java/org/txm/Messages.java")
+
+		ExternalizeStringFile externalizer = new ExternalizeStringFile(propFile, messageFile)
+		println externalizer.getNbOfChanges(sourceFile);
+		println externalizer.process(sourceFile);
+		externalizer.saveChanges();
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/IsStringMissingInPlugin.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/IsStringMissingInPlugin.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/i18n/IsStringMissingInPlugin.groovy	(revision 321)
@@ -0,0 +1,54 @@
+package org.txm.scripts.i18n
+
+/**
+ * Display the missing i18n keys of the plugin.xml file 
+ * 
+ * @author mdecorde
+ *
+ */
+class IsStringMissingInPlugin {
+	
+	public static void main(String[] args) {
+		String userhome = System.getProperty("user.home")
+		File propertyFile = new File(userhome, "workspace43/org.txm.rcp/OSGI-INF/l10n/bundle_fr.properties");
+		File copyFile = new File(userhome, "workspace43/org.txm.rcp/OSGI-INF/l10n/bundle_copy.properties");
+		File pluginFile = new File(userhome, "workspace43/org.txm.rcp/plugin.xml");
+		if (!(propertyFile.exists() && propertyFile.canRead() && propertyFile.canWrite() && propertyFile.isFile()))
+		{
+			println "error file : "+propertyFile
+			return
+		}
+
+		println "Get properties of "+propertyFile
+		Properties props = new Properties();
+		propertyFile.withReader("iso-8859-1") { input->
+			props.load(input);
+			input.close()
+		}
+
+		
+		String pluginXMLcontent = pluginFile.getText();
+		def pluginKeys = [];
+		def matcher = pluginXMLcontent =~ /"%([.A-Za-z0-9]+)+"/ // %key.of.the.field
+		println "Comparing keys "+matcher.pattern() +" of "+pluginFile+" with those declared"
+		matcher.each { // iterates of %key in the plugin.xml content
+			def key = it[1]
+			pluginKeys << key
+			if (!props.containsKey(key)) {
+				println "Prop file does not contains the key: $key"
+				//println " adding default value."
+				props.setProperty(key, "NA")
+			}
+		} 
+		
+		for (String key : props.keySet()) {
+			if (!pluginKeys.contains(key)) {
+				println "plugin.xml does not use the key: "+key
+			}
+		}
+		
+		propertyFile.withWriter("iso-8859-1") { output ->
+			props.store(output, "")
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TTrezToXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TTrezToXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TTrezToXml.groovy	(revision 321)
@@ -0,0 +1,212 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+// TODO: Auto-generated Javadoc
+/** The person. @deprecated @author mdecorde tranform Ttrez into a stand-off file */
+class TTrezToXml {
+	String person = ""; // used for respStmt/resp/name(person)
+	
+	/** The software. */
+	String software = ""; // used for respStmt/resp/name(software)
+	
+	/** The cmd line. */
+	String cmdLine = "";// used for respStmt/resp/name(software)/p
+	
+	/** The title. */
+	String title = "";// used for <title>
+	
+	/** The types. */
+	def types;// used for linkGrp type
+	
+	/** The idform. */
+	String idform = "";
+	
+	/** The comments. */
+	String comments = "...";// used for linkGrp type comments
+
+	/** The writer. */
+	Writer writer;
+
+	/**
+	 * Process.
+	 *
+	 * @param TTrez the t trez
+	 * @param xmlform the xmlform
+	 * @param encoding the encoding
+	 */
+	public void process(File TTrez,File xmlform, String encoding )
+	{
+		writer = new FileWriter(xmlform);
+		writeHead();	
+		writeBody(TTrez,encoding)
+		writeTail();
+		writer.close();
+	}
+
+	/**
+	 * Write head.
+	 */
+	private void writeHead() {
+		writer.write("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>\n");
+		writer.write("<!DOCTYPE TEI SYSTEM \"tei_bfm_v1.dtd\">\n");
+		writer.write("<TEI xmlns:txm=\"http://textometrie.ens-lyon.fr/1.0\">\n");
+		writer.write("<teiHeader xml:lang=\"eng\">\n");
+		writer.write("<fileDesc>\n");
+		writer.write("<titleStmt>\n");
+		for (int i = 0; i < types.size(); i++)
+			writer.write("<title>" + types[i] + "</title>\n");
+		writer.write("<respStmt>\n");
+		writer.write("<resp>produced by</resp>\n");
+		writer.write("<name type=\"person\">" + person + "</name>\n");
+		writer.write("<name type=\"software\">" + software + "</name>\n");
+		writer.write("<p>" + cmdLine + "</p>\n");
+		writer.write("</respStmt>\n");
+		writer.write("</titleStmt>\n");
+		writer.write("<publicationStmt>\n");
+		writer
+				.write("<distributor>BFM project - http://bfm.ens-lsh.fr</distributor>\n");
+		writer.write("<availability>\n");
+		writer.write("<p>(c) 2010 Projet BFM - CNRS/ENS-LSH.\n");
+		writer.write("<hi>Conditions d'utilisation</hi> : \n");
+		writer
+				.write("Sous licence <ref target=\"http://creativecommons.org/licenses/by-sa/2.0/fr/\">Creative Commons</ref>.\n");
+		writer.write("</p>\n");
+		writer.write("</availability>\n");
+		writer.write("</publicationStmt>\n");
+		writer.write("<sourceDesc>\n");
+		writer
+				.write("<p>born digital : TXM project - http://textometrie.org</p>\n");
+		writer.write("</sourceDesc>\n");
+		writer.write("</fileDesc>\n");
+		writer.write("</teiHeader>\n");
+		writer.write("<text xml:lang=\"fr\" type=\"standoff\">\n");
+		writer.write("<body>\n");
+		writer.write("<div>\n");
+	}
+
+	/**
+	 * Write body.
+	 *
+	 * @param TTrez the t trez
+	 * @param encoding the encoding
+	 * @return the java.lang. object
+	 */
+	private writeBody(File TTrez, String encoding)
+	{
+		
+		String targets= "w_"+idform+"_";
+		
+		def content = TTrez.getText(encoding) 
+		def separator= "\t"
+		
+		for(int i=0 ; i< types.size() ; i++)
+		{
+			int id=1;
+			writer.write("<linkGrp type=\""+types[i]+"\">\n");
+			content.splitEachLine(separator) {fields ->
+				if(! ((String)fields[0]).substring( 0,1).matches("<"))
+				{
+					writer.write("<link targets=\"#"+targets+id+" #"+ fields[i+1]+"\"/>\n");
+					id++;
+				}
+			}
+			writer.write("</linkGrp>\n");
+		}
+	}
+
+	/**
+	 * Write tail.
+	 */
+	private void writeTail() {
+
+		writer.write("</div>\n");
+		writer.write(" </body>\n");
+		writer.write("</text>\n");
+		writer.write("</TEI>\n");
+	}
+
+	/**
+	 * Sets the annotation types.
+	 *
+	 * @param types the types
+	 * @param idform the idform
+	 */
+	public void setAnnotationTypes(def types, String idform) {
+		this.types = types;
+		this.idform = idform;
+	}
+
+	/**
+	 * Sets the resp.
+	 *
+	 * @param person the person
+	 * @param software the software
+	 * @param cmdLine the cmd line
+	 */
+	public void setResp(String person, String software, String cmdLine) {
+		this.person = person;
+		this.software = software;
+		this.cmdLine = cmdLine;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String rootDir = "~/xml/rgaqcj/";
+		new File(rootDir+"/pos/").mkdir();
+		
+		
+		def ttfiles = ["roland-tt.txt","artu-tt.txt","qjm-tt.txt","commyn1-tt.txt","jehpar-tt.txt"];
+		def posfiles = ["roland-pos.xml","artu-pos.xml","qjm-pos.xml","commyn1-pos.xml","jehpar-pos.xml"];
+		
+		String encoding ="ISO-8859-1";
+		String person ="Serge Heiden"; 
+		String software = "tree-tagger cmd line with options ..."; 
+		String cmdLine = "tree-tagger -token ...";
+		String title ="POS annotation";
+		def types = ["POS","lemma"];
+		String idform ="fro";
+		
+		for(int i=0; i < ttfiles.size();i++)
+		{
+			File ttfile = new File(rootDir+"/tt/",ttfiles[i]);
+			File posfile = new File(rootDir+"/pos/",posfiles[i]);
+			println("Process file : "+ttfile+" to : "+posfile );
+			
+			def transfo = new TTrezToXml();
+			transfo.setAnnotationTypes( types, idform);
+			transfo.setResp( person, software, cmdLine);
+			transfo.process( ttfile, posfile, encoding );
+		}	
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Grep.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Grep.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Grep.groovy	(revision 321)
@@ -0,0 +1,86 @@
+package org.txm.scripts
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2012-02-01 17:58:45 +0100 (mer., 01 févr. 2012) $
+// $LastChangedRevision: 2122 $
+// $LastChangedBy: mdecorde $
+//
+
+// PARAMETERS //
+outfile = new File("/home/mdecorde/xml/test/grep")
+dir = new File("/home/mdecorde/xml/test/")			// **change this parameter**
+ext = "\\.xml"                                      // **change this parameter**
+find = /(<...+)/           							// **change this parameter**
+replaceWith = 'gp1=$1'             				// **change this parameter**
+
+// MAIN //
+
+grep(dir, outfile, ext, find, replaceWith) // process a directory
+//grep(dir, outfile, null, find, replaceWith) // process a directory with no constraint on files
+//grep(dir, outfile, ext, find, null) // directory + don't replace
+
+//grep(infile, outfile, find, replaceWith) // process a file
+//grep(dir, outfile, find, null) // file + don't replace
+
+// PUBLIC METHODS //
+
+// to process a directory
+def grep(File dir, File outfile, def ext, def regex, def replace) {
+	if (dir.isDirectory()) {
+		outfile.createNewFile()
+		if (ext == null) ext = /.+/
+		outfile.write('')                            // create empty file
+		outfile.withWriter { writer ->
+			dir.eachFileMatch(~/.*$ext/){ file ->    // for each file matching extension
+				_grep(file, writer, regex, replace)
+			}
+		}
+		return true;
+	} else {
+		println "$outfile is not a directory"
+		return false;
+	}
+}
+
+// to process a file with outfile
+def grep(File file, File outfile, def regex, def replace) {
+	if (file.isDirectory()) return false;
+	outfile.withWriter { writer ->
+		_grep(file, writer, regex, replace)
+	}
+	return true;
+}
+
+// PRIVATE METHODS //
+
+// to process a file with writer
+private def _grep(def file, def writer, def regex, def replace) {
+	file.eachLine { line ->
+		_testLine(line, writer, regex, replace)
+	}
+}
+
+private def _testLine(String line, def writer, def regex, def replace) {
+	if (replace == null) {
+		if (line.matches(find))
+			writer.println line
+	} else {
+		if (line.matches(find))
+			writer.println line.replaceAll(regex, replace)
+	}	
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FixEditionPageIds.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FixEditionPageIds.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FixEditionPageIds.groovy	(revision 321)
@@ -0,0 +1,63 @@
+package org.txm.scripts
+
+
+String corpus = "qgraal"
+File PARAMS = new File("/home/mdecorde/TXM/corpora/$corpus/import.xml")
+File PARAMS2 = new File("/home/mdecorde/TXM/corpora/$corpus/params2.xml")
+File HTMLs = new File("/home/mdecorde/TXM/corpora/$corpus/HTML/QGRAALFRO")
+
+def doc = DomUtils.load(PARAMS)
+def root = doc.getDocumentElement();
+
+def corporaList = root.getElementsByTagName("corpora");
+def corporaElem = corporaList.item(0)
+
+def corpusList = corporaElem.getElementsByTagName("corpus");
+def corpusElem = corpusList.item(0)
+
+def textsList = corpusElem.getElementsByTagName("texts")
+def textsElem = textsList.item(0);
+
+def textList = textsElem.getElementsByTagName("text")
+for(int i = 0 ; i < textList.getLength() ; i++)
+{
+	def textElem = textList.item(i)
+	String textname = textElem.getAttribute("name");
+	println "process text: "+textname
+	def editionsList = textElem.getElementsByTagName("editions")
+	def editionsElem = editionsList.item(0)
+
+	def editionList = editionsElem.getElementsByTagName("edition")
+
+	for(int j = 0 ; j < editionList.getLength() ; j++)
+	{
+		def editionElem = editionList.item(j)
+		String editionName = editionElem.getAttribute("name");
+
+		def pages = new File(HTMLs, editionName).listFiles();
+		pages.sort();
+
+		def pageList = editionElem.getElementsByTagName("page")
+		println pageList.getLength()
+		println pages.size()
+		for(int k = 0 ; k < pageList.getLength() ; k++)
+		{
+			def pageElem = pageList.item(k);
+			String id = pages[k].getName();
+			int idx1 = id.lastIndexOf("_");
+			int idx2 = id.lastIndexOf(".html");
+			id = id.substring(idx1 + 1, idx2)
+			pageElem.setAttribute("id", id);
+		}
+
+		if(pageList.getLength() != pages.size())
+		{
+			println "Error size pageList.getLength() != pages.size()"+pageList.getLength()+", "+pages.size()
+		}
+	}
+}
+
+DomUtils.save(doc, PARAMS2);
+
+if (!(PARAMS.delete() && PARAMS2.renameTo(PARAMS))) println "Warning can't rename file "+PARAMS2+" to "+PARAMS
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/JoinCaesura.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/JoinCaesura.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/JoinCaesura.java	(revision 321)
@@ -0,0 +1,226 @@
+package org.txm.scripts;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.txm.utils.xml.DomUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * Process tokenized text to find caesura and merge words
+ */
+class JoinCaesura{
+	String caesuraTag = "lb|br"; // caesura
+	String caesuraString = "-|$";
+	String wordTag = "w";
+	String intraWordTag = "hi";
+	String extraWordTag = "p|s";
+
+	public boolean process(Element root)
+	{
+		// find caesuraTags
+		ArrayList<Element> currentNodes = new ArrayList<Element>();
+		currentNodes.add(root);
+		Element currentNode;
+		while(currentNodes.size() > 0)
+		{
+			System.out.println(currentNodes);
+			currentNode = currentNodes.remove(0);
+			NodeList children = currentNode.getChildNodes();
+
+			if(currentNode.getNodeName().matches(caesuraTag))
+			{
+				processCaesuraTag(currentNode);
+			}
+
+			//System.out.println("children: "+children.getLength());
+			int c = 0;
+			for(int i = 0 ; i < children.getLength() ; i++)
+			{
+				if(children.item(i).getNodeType() == Node.ELEMENT_NODE)
+				{
+					Element child = (Element)children.item(i);
+					//System.out.println("child: "+child.getNodeName());
+					if(!child.getNodeName().matches(wordTag)) // don't look in the words
+					{
+						currentNodes.add(c++, child);
+					}
+				}
+			}
+
+		}
+		return true;
+	}
+
+	protected Node getPreviousWord(Element node)
+	{
+		int count = 0;
+		Node previous = node.getPreviousSibling();
+		while(previous != null)
+		{
+			
+			if(previous.getNodeType() == Node.ELEMENT_NODE && previous.getNodeName().matches(wordTag))
+			{
+				//System.out.print("W:"+previous);
+				return previous;
+			}
+			else if(previous.getNodeType() == Node.ELEMENT_NODE && previous.getNodeName().matches(intraWordTag))
+			{
+				//System.out.print("P:"+previous);
+				if(isLastChildrenWord(previous))
+				{
+					//System.out.println("P: "+previous+" has a word");
+					return previous;
+				}
+				else
+				{
+					//System.out.println("P: "+previous+" has no word");
+				}
+			}
+			else if(previous.getNodeType() == Node.ELEMENT_NODE && previous.getNodeName().matches(extraWordTag))
+			{
+				return null;
+			}
+			previous = previous.getPreviousSibling();
+			if(count++ >= 10)
+				return null;
+		}
+		return null;
+	}
+
+	protected boolean isLastChildrenWord(Node node)
+	{
+		//System.out.println("test "+node);
+		NodeList children = node.getChildNodes();
+		for(int i = children.getLength() -1 ; i >= 0 ; i--) // begin by the last child
+		{
+			Node child = children.item(i);
+			//System.out.println(" child "+child);
+			if(child.getNodeType() == Node.ELEMENT_NODE && child.getNodeName().matches(wordTag))
+				return true; // return the parent node
+			else if(child.getNodeType() == Node.ELEMENT_NODE && child.getNodeName().matches(intraWordTag))
+				return isLastChildrenWord(child);
+		}
+		return false;
+	}
+	
+	protected boolean isFirstChildrenWord(Node node)
+	{
+		NodeList children = node.getChildNodes();
+		for(int i = 0 ; i < children.getLength() ; i++) // begin by the last child
+		{
+			Node child = children.item(i);
+			if(child.getNodeType() == Node.ELEMENT_NODE && child.getNodeName().matches(wordTag))
+				return true; // return the parent node
+			else if(child.getNodeType() == Node.ELEMENT_NODE && child.getNodeName().matches(intraWordTag))
+				return isLastChildrenWord(child);
+		}
+		return false;
+	}
+	
+	protected Node getNextWord(Element node)
+	{
+		int count = 0;
+		Node next = node.getNextSibling();
+		while(next != null)
+		{
+			if(next.getNodeType() == Node.ELEMENT_NODE && next.getNodeName().matches(wordTag))
+				return next;
+			else if(next.getNodeType() == Node.ELEMENT_NODE)
+			{
+				//System.out.print("P:"+previous);
+				if(isFirstChildrenWord(next))
+				{
+					System.out.println("N: "+next+" has a word");
+					return next;
+				}
+				else
+				{
+					System.out.println("N: "+next+" has no word");
+				}
+			}
+			else if(next.getNodeType() == Node.ELEMENT_NODE && next.getNodeName().matches(extraWordTag))
+			{
+				return null;
+			}
+			next = next.getNextSibling();
+			if(count++ >= 10)
+				return null;
+		}
+		return null;
+	}
+
+	protected boolean processCaesuraTag(Element caesura)
+	{
+		Node previous = getPreviousWord(caesura);
+		
+		if(previous != null)
+		{
+			Node next = getNextWord(caesura);
+			print(previous);
+			print(caesura);
+			print(next);
+			if(next != null)
+			{
+				// process !!
+			}
+		}
+
+		System.out.println("\n");
+		return true;
+	}
+
+	protected void print(Node n)
+	{
+		if(n != null)
+			switch(n.getNodeType())
+			{
+			case Node.ELEMENT_NODE:
+				if(n.getNodeName().matches(wordTag))
+					System.out.print("<"+n.getNodeName()+">"+n.getTextContent()+"</"+n.getNodeName()+">");
+				else
+					System.out.print("<"+n.getNodeName()+">");//+n.getTextContent()+"</"+n.getNodeName()+">");
+				break;
+			case Node.TEXT_NODE:
+				n.getNodeValue();
+				break;
+			default:
+			}
+	}
+
+	public static void main(String[] args)
+	{
+		File infile = new File(System.getProperty("user.home"), "xml/caesura/test.xml");
+		File outfile = new File(System.getProperty("user.home"), "xml/caesura/test-c.xml");
+		try {
+			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+			DocumentBuilder constructeur;
+
+			constructeur = factory.newDocumentBuilder();
+
+			Document document = constructeur.parse(infile);
+			Element root = document.getDocumentElement();
+			//process the text
+			JoinCaesura builder = new JoinCaesura();
+			if(builder.process(root))
+			{
+				System.out.println("done");
+			}
+			else
+			{
+				System.out.println("error");
+			}
+
+			DomUtils.save(document, outfile);
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/JoinCaesura.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/HSQLFunctions.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/HSQLFunctions.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/HSQLFunctions.groovy	(revision 321)
@@ -0,0 +1,740 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-07-04 16:11:30 +0200 (Mon, 04 Jul 2016) $
+// $LastChangedRevision: 3257 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import java.util.LinkedList;
+
+import groovy.sql.Sql;
+
+import java.sql.*;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.logging.Level;
+
+import org.txm.utils.logger.Log;
+
+// TODO: Auto-generated Javadoc
+/**
+ * set of Sql function for HSQL.
+ *
+ * @return the connection
+ * @author mdecorde
+ */
+class HSQLFunctions {
+	
+	Connection c
+	String path;
+	
+	public HSQLFunctions(String path) {
+		this.path = path;
+		
+		String urlProperty = "jdbc:hsqldb:file:"+path+"/db;shutdown=true;hsqldb.write_delay=false;"
+		if (Log.getLevel().intValue() < Level.WARNING.intValue()) {
+			urlProperty += "";
+		} else {
+			urlProperty += "hsqldb.applog=0;hsqldb.sqllog=0";
+		}
+		
+		this.c = DriverManager.getConnection(urlProperty, "SA", "")
+	}
+
+	public Connection getConnection() {
+		return c;
+	}
+	
+	/**
+	 * Creates the table.
+	 *
+	 * @param name the name
+	 * @param argsname the argsname
+	 * @param types the types
+	 */
+	public void CreateTable(String name)
+	{
+		Statement statement = c.createStatement();
+		String query ="CREATE TABLE "+name+";";
+		
+		println("query : "+query);
+		statement.executeUpdate(query);
+		
+		statement.close();
+		c.commit();
+	}
+	
+	/**
+	 * Creates the table.
+	 *
+	 * @param name the name
+	 * @param argsname the argsname
+	 * @param types the types
+	 */
+	public void CreateTable(String name, String idColumn)
+	{
+		Statement statement = c.createStatement();
+		String query = "CREATE TABLE \""+name+"\" (\""+idColumn+"\" INTEGER IDENTITY PRIMARY KEY);";
+	
+		println("query : "+query);
+		statement.executeUpdate(query);
+		
+		statement.close();
+		c.commit();
+	}
+	
+	public boolean IsTableExisting(String name) {
+		boolean e = false;
+		
+		DatabaseMetaData dbm = c.getMetaData();
+		ResultSet tables = dbm.getTables(null, null, name, null);
+		e = tables.next();
+		tables.close()
+		
+		return e;		
+	}
+	
+	
+	/**
+	 * Creates the table.
+	 *
+	 * @param name the name
+	 * @param argsname the argsname
+	 * @param types the types
+	 */
+	public void CreateTable(String name, def argsname, def types)
+	{
+		assert(argsname.size() == types.size());
+		
+		Statement statement = c.createStatement();
+		String query ="CREATE TABLE "+name+" ( ";
+		for (int i=0; i < types.size() ; i++) {
+			query += argsname[i]+" "+types[i]
+			if (i < types.size()-1)	{query += ",";}
+		}
+		query +=")";
+		
+		println("query : "+query);
+		statement.executeUpdate(query);
+		
+		statement.close();
+		c.commit();
+	}
+	
+	
+	public void DeleteTable(String name) {
+		Statement statement = c.createStatement();
+		statement.executeUpdate("DROP TABLE entries;");
+		statement.close();
+		c.commit();
+	}
+
+	/**
+	 * Insert csv col.
+	 *
+	 * @param TableName the table name
+	 * @param argsnames the argsnames
+	 * @param types the types
+	 * @param csvFile the csv file
+	 * @param separator the separator
+	 * @param encoding the encoding
+	 */
+	public void InsertCsvCol(String TableName, def argsnames, def types, File csvFile, String separator, String encoding)
+	{
+		Statement statement = c.createStatement();
+		statement.setEscapeProcessing(true);
+		Reader reader = new InputStreamReader(new FileInputStream(csvFile),encoding);
+		reader.splitEachLine(separator) {fields ->
+			//println("fields size "+fields.size())
+			String query ="INSERT INTO "+TableName+" (";
+			for(int i=0;i< argsname.size; i++)
+			{
+				query+= argsname[i];
+				if(i < argsnames.size-1)	{query += ",";}
+			}
+			query +=") VALUES (";
+			for(int i=0;i< fields.size(); i++)
+			{
+				if(fields[i].contains("'"))
+					query+= "'"+fields[i].replace("'","''")+"'";
+				else
+					query+= "'"+fields[i]+"'";
+				
+				if(i < argsnames.size-1)	{query += ",";}
+			}
+			query +=")";
+			
+			//println("query : "+query);
+			statement.executeUpdate(query);
+		}
+		statement.close();
+		c.commit();
+	}
+
+	/**
+	 * Import ordered csv table.
+	 *
+	 * @param TableName the table name
+	 * @param argsname the argsname
+	 * @param types the types
+	 * @param csvFile the csv file
+	 * @param separator the separator
+	 * @param encoding the encoding
+	 * @return the int
+	 */
+	public int ImportOrderedCSVTable(String TableName, List<String> argsname, List<String> types, File csvFile, String separator, String encoding)
+	{
+		argsname.add("n");
+		types.add("INT");
+		HSQLFunctions.CreateTable( TableName, argsname, types);
+		argsname.remove("n");
+		types.remove("INT");
+		Statement statement = c.createStatement();
+		statement.setEscapeProcessing(true);
+		Reader reader = new InputStreamReader(new FileInputStream(csvFile),encoding);
+		int n=0;
+		reader.splitEachLine(separator) {fields ->
+			n++;
+			//println("fields size "+fields.size())
+			String query ="INSERT INTO "+TableName+" (n,";
+			for(int i=0;i< argsname.size(); i++)
+			{
+				query+= argsname[i];
+				if(i < argsname.size()-1)	{query += ",";}
+			}
+			query +=") VALUES ('"+n+"',";
+			for(int i=0;i< argsname.size(); i++)
+			{
+				if( i >= fields.size())
+				{
+					query+= "''";
+				}
+				else
+				{
+					if(fields[i].contains("'"))
+						query+= "'"+fields[i].replace("'","''")+"'";
+					else
+						query+= "'"+fields[i]+"'";
+				}
+				if(i < argsname.size()-1)	{query += ",";}
+			}
+			query +=")";
+			
+			//println("query : "+query);
+			statement.executeUpdate(query);
+		}
+		statement.close();
+		c.commit();
+		return n;
+	}
+
+	/**
+	 * Import csv table.
+	 *
+	 * @param TableName the table name
+	 * @param argsname the argsname
+	 * @param types the types
+	 * @param csvFile the csv file
+	 * @param separator the separator
+	 * @param encoding the encoding
+	 */
+	public void ImportCSVTable(String TableName, def argsname, def types, File csvFile, String separator, String encoding)
+	{
+		HSQLFunctions.CreateTable( TableName, argsname, types);
+		
+		Statement statement = c.createStatement();
+		statement.setEscapeProcessing(true);
+		Reader reader = new InputStreamReader(new FileInputStream(csvFile),encoding);
+		reader.splitEachLine(separator) {fields ->
+			//println("fields size "+fields.size())
+			String query ="INSERT INTO "+TableName+" (";
+			for(int i=0;i< argsname.size; i++)
+			{
+				query+= argsname[i];
+				if(i < argsname.size-1)	{query += ",";}
+			}
+			query +=") VALUES (";
+			for(int i=0;i< argsname.size; i++)
+			{
+				if( i >= fields.size())
+				{
+					query+= "''";
+				}
+				else
+				{
+					if(fields[i].contains("'"))
+						query+= "'"+fields[i].replace("'","''")+"'";
+					else
+						query+= "'"+fields[i]+"'";
+				}
+				if(i < argsname.size-1)	{query += ",";}
+			}
+			query +=")";
+			
+			//println("query : "+query);
+			statement.executeUpdate(query);
+		}
+		statement.close();
+		c.commit();
+	}
+
+	// word column val column val2 column val3
+	
+	/**
+	 * Import ref table.
+	 *
+	 * @param TableName the table name
+	 * @param argsname the argsname
+	 * @param types the types
+	 * @param csvFile the csv file
+	 * @param separator the separator
+	 * @param encoding the encoding
+	 */
+	public void ImportRefTable(String TableName, def argsname, def types, File csvFile, String separator, String encoding)
+	{
+		HSQLFunctions.CreateTable( TableName, argsname, types);
+		
+		Statement statement = c.createStatement();
+		statement.setEscapeProcessing(true);
+		Reader reader = new InputStreamReader(new FileInputStream(csvFile),encoding);
+		reader.splitEachLine(separator) 
+				{fields ->
+					String ref;
+					if(fields[0].contains("'"))
+						ref= "'"+fields[0].replace("'","''")+"'";
+					else
+						ref= "'"+fields[0]+"'";
+					
+					for(int j=1 ; j < fields.size() ; j +=argsname.size -1)
+					{
+						String query ="INSERT INTO "+TableName+" (";
+						for(int i=0;i< argsname.size; i++)
+						{
+							query+= argsname[i];
+							if(i < types.size-1)	{query += ",";}
+						}
+						query +=") VALUES (";
+						query+= ref+",";
+						for(int i=j;i < j+argsname.size -1; i++)
+						{
+							if(fields[i].contains("'"))
+								query+= "'"+fields[i].replace("'","''")+"'";
+							else
+								query+= "'"+fields[i]+"'";
+							
+							if(i < j+argsname.size -2)	{query += ",";}
+						}
+						query +=")";
+						
+						//println("query : "+query);
+						statement.executeUpdate(query);
+					}
+				}
+		statement.close();
+		c.commit();
+	}
+
+	/**
+	 * Creates the simple table.
+	 *
+	 * @param TableName the table name
+	 * @param argsname the argsname
+	 * @param type the type
+	 * @param values the values
+	 */
+	public void CreateSimpleTable(String TableName, String argsname , String type, def values)
+	{
+		def argsnames = [argsname];
+		def types = [type]
+		HSQLFunctions.CreateTable( TableName, argsnames, types);
+		
+		Statement statement = c.createStatement();
+		for(int i=0; i < values.size; i++)
+		{
+			String query ="INSERT INTO "+TableName+" (";
+			query+= argsname;
+			query +=") VALUES ('";
+			query+= values[i];
+			query +="')";
+			
+			println("query : "+query);
+			statement.executeUpdate(query);
+		}
+		statement.close();
+		c.commit();
+	}
+
+	/**
+	 * Test.
+	 *
+	 * @param table the table
+	 * @param file the file
+	 * @return the java.lang. object
+	 */
+	public test(String table, String file) {
+		Statement statement = c.createStatement();
+
+		statement.execute("SET TABLE " + table + " SOURCE \"" + file
+				+ ";ignore_first=false;all_quoted=false\"");
+		statement.close();
+		c.commit();
+	}
+
+	// fonction temporaire, j'ai rien trouvé de mieux -_-
+	
+	/**
+	 * Gets the column number.
+	 *
+	 * @param TableName the table name
+	 * @return the column number
+	 */
+	public int getColumnNumber(String TableName)
+	{
+		Sql sql = Sql.newInstance(c)
+		int i =0;
+		try {
+			sql.eachRow("select * from "+TableName) {
+				for(i=0;;i++)
+					it.getAt(i);
+			}
+		}
+		catch(Exception e)
+		{
+			return i;
+		}
+		
+		c.close();
+		return 0;
+	}
+
+	/**
+	 * Prints the table.
+	 *
+	 * @param TableName the table name
+	 */
+	public void printTable(String TableName)
+	{
+		Sql sql = Sql.newInstance(c)
+		int colnum = getColumnNumber(TableName);
+		sql.eachRow("select * from "+TableName) {
+			for(int i=0;i< colnum; i++)
+			{
+				print(it.getAt(i));
+				if(i < colnum -1)print("\t");
+			}
+			println();
+		}
+		
+		c.commit();
+	}
+
+	/**
+	 * Execute query.
+	 *
+	 * @param query the query
+	 */
+	public void executeQuery(String query) {
+		Statement statement = c.createStatement();
+		statement.setEscapeProcessing(true);
+		println("query : " + query);
+		statement.execute(query);
+		statement.close();
+		c.commit();
+	}
+
+	/**
+	 * Gets the groovy sql.
+	 *
+	 * @return the groovy sql
+	 */
+	public Sql getGroovySql() {
+		return Sql.newInstance(c)
+	}
+
+	/**
+	 * To cvs file.
+	 *
+	 * @param TableName the table name
+	 * @param outfile the outfile
+	 */
+	public void toCVSFile(String TableName, String outfile) {
+		def sql = Sql.newInstance(c)
+		
+		Writer writer = new FileWriter(outfile);
+		
+		int colnum = getColumnNumber(TableName);
+		sql.eachRow("select * from "+TableName) {
+			for(int i=0;i< colnum; i++)
+			{
+				//println('get at '+i)
+				writer.write(it.getAt(i));
+				if(i < colnum -1)writer.write("\t");
+			}
+			writer.write("\n");
+		}
+		
+		writer.close();
+		c.commit();
+	}
+
+	// only for sql result with 3 columns
+	
+	/**
+	 * To ref file.
+	 *
+	 * @param TableName the table name
+	 * @param outfile the outfile
+	 * @param groupcolumn the groupcolumn
+	 * @param argname the argname
+	 * @param encoding the encoding
+	 * @return the int
+	 */
+	public int toRefFile(String TableName, String outfile, String groupcolumn,def argname, String encoding)
+	{
+		assert(argname.size() == 2);
+		def sql = Sql.newInstance(c)
+		LinkedList<String> linkset;
+		File f = new File(outfile);
+		Writer writer = new BufferedWriter(new OutputStreamWriter(new PrintStream(new FileOutputStream(f),true,encoding)));
+		def content = new File(outfile).getText(encoding) 
+		int colnum = getColumnNumber(TableName);
+		String ref = "";
+		String line ="";
+		boolean firstline = true;
+		HashMap<String,LinkedList<String>> reftable = new HashMap<String,LinkedList<String>>();
+		HashMap<String,LinkedList<String>> lemmetable = new HashMap<String,LinkedList<String>>();
+		
+		String query = "SELECT "+groupcolumn;
+		for (String s : argname)
+			query +=","+s
+		//System.out.println(query+" FROM "+TableName)
+		sql.eachRow(query+" FROM "+TableName){
+			String currentref = it.getAt(0);
+			if (!reftable.containsKey(currentref)) {
+				linkset = new LinkedHashSet<String>();
+				reftable.put(currentref,linkset)
+				linkset = new LinkedHashSet<String>();
+				lemmetable.put(currentref,linkset)
+			}
+			
+			linkset = reftable.get(currentref)
+			if (!linkset.contains(it.getAt(1))) {
+				linkset.add(""+it.getAt(1))
+				linkset = lemmetable.get(currentref)
+				linkset.add(""+it.getAt(2))
+			}
+		}
+		//println("ref : "+reftable)
+		for (String key : reftable.keySet()) {
+			writer.write (key)
+			for (int i = 0 ; i < reftable.get(key).size(); i++) {
+				String s = (reftable.get(key)).get(i);
+				String l = (lemmetable.get(key)).get(i);
+				writer.write ("\t"+s+"\t"+l);
+			}
+			writer.write ("\n")
+		}
+		
+		writer.close();
+		c.commit();
+		return reftable.size();
+	}
+
+	/**
+	 * To ref hierarchical file.
+	 *
+	 * @param TableName the table name
+	 * @param outfile the outfile
+	 * @param groupcolumn the groupcolumn
+	 * @param argname the argname
+	 * @param encoding the encoding
+	 * @return the int
+	 */
+	public int toRefHierarchicalFile(String TableName, String outfile, String groupcolumn,def argname, String encoding)
+	{
+		assert(argname.size() == 2);
+		def sql = Sql.newInstance(c);
+		LinkedList<String> linkset;
+		File f = new File(outfile);
+		Writer writer = new BufferedWriter(new OutputStreamWriter(new PrintStream(new FileOutputStream(f),true,encoding)));
+		def content = new File(outfile).getText(encoding);
+		int colnum = getColumnNumber(TableName);
+		String ref = "";
+		String line ="";
+		boolean firstline = true;
+		HashMap<String,LinkedList<String>> reftable = new HashMap<String,LinkedList<String>>();
+		//HashMap<String,HashMap<String,Integer>> counttable = new HashMap<String,HashMap<String,Integer>>();
+		HashMap<String,LinkedList<Integer>> counttable = new HashMap<String,LinkedList<Integer>>();
+		HashMap<String,LinkedList<String>> lemmetable = new HashMap<String,LinkedList<String>>();
+		
+		String query = "SELECT "+groupcolumn;
+		for(String s : argname)
+			query +=","+s;
+		//System.out.println(query+" FROM "+TableName)
+		sql.eachRow(query+" FROM "+TableName){
+			String currentref = it.getAt(0);
+			if (!reftable.containsKey(currentref)) {
+				linkset = new LinkedHashSet<String>();
+				reftable.put(currentref,linkset);
+				linkset = new LinkedHashSet<String>();
+				lemmetable.put(currentref,linkset);
+			}
+			
+			linkset = reftable.get(currentref);
+			if (!linkset.contains(it.getAt(1))) {
+				linkset.add(""+it.getAt(1));
+				linkset = lemmetable.get(currentref);
+				linkset.add(""+it.getAt(2));
+			}
+			
+			
+			//if(it.getAt(2) != "-")
+			//{
+			if(!counttable.containsKey(currentref))
+				counttable.put(currentref, new HashMap<String,Integer>());
+			
+			HashMap<String,Integer> ht = counttable.get(currentref);
+			if(!ht.containsKey(it.getAt(1)))
+				ht.put(it.getAt(1),0);
+			
+			ht.put(it.getAt(1),ht.get(it.getAt(1)) +1)	
+			
+			//}
+		}
+		//println("countable : "+counttable)
+		for (String key : counttable.keySet()) {
+			HashMap<String,Integer> ht = counttable.get(key);
+			LinkedList<String> indexorder = new LinkedList<Integer>();
+			
+			for(String cat : counttable.get(key).keySet())
+			{
+				if(indexorder.size() == 0)
+				{
+					indexorder.add(cat)
+				}
+				else
+				{
+					boolean inserted= false;
+					for(int j = 0 ; j < indexorder.size();j++)
+					{
+						String comp = indexorder.get(j);
+						if(ht.get(cat) > ht.get(comp))
+						{
+							indexorder.add(j, cat);
+							inserted=true;
+							break;
+						}
+					}
+					if(!inserted)
+						indexorder.addLast(cat);
+				}
+			}
+			writer.write(key);
+			for (String cat : indexorder) {
+				
+				int i=0;
+				for(int r=0; r < reftable.get(key).size();r++)
+					if((reftable.get(key)).get(r) == cat)
+					{
+						i=r;
+						break;
+					}
+				String s = (reftable.get(key)).get(i);
+				String l = (lemmetable.get(key)).get(i);
+				writer.write ("\t"+s+"\t"+l);
+			}
+			writer.write ("\n")
+		}
+		
+		writer.close();
+		c.commit();
+		return reftable.size();
+	}
+
+	/**
+	 * Clear all.
+	 */
+	public void clearAll() {
+		new File(path, "db.script").delete();
+		new File(path, "db.properties").delete();
+		new File(path, "db.log").delete();
+		new File(path, "db.tmp").deleteDir();
+	}
+	
+	/**
+	 * Clear all.
+	 */
+	public void close() {
+		c.close();
+	}
+
+	/**
+	 * Main.
+	 *
+	 * @param args the args
+	 */
+	public static void main(String[] args)
+	{
+		HSQLFunctions hsql = new HSQLFunctions("/home/mdecorde/TEMP")
+		//hsql.clearAll();
+		
+	//	hsql.executeQuery("SELECT col1 from test where col1 LIKE '@'' ESCAPE '@' ")
+		
+		def argsname = ["n","col1","col2"];
+		def types = ["INT","VARCHAR(30)","VARCHAR(30)"];
+		
+		hsql.CreateTable( "test", argsname, types);
+		hsql.CreateTable( "test2", argsname, types);
+		hsql.executeQuery("INSERT INTO test VALUES (1,'poeut',1)");
+		hsql.executeQuery("INSERT INTO test VALUES (2,'poeut',2)");
+		hsql.executeQuery("INSERT INTO test VALUES (3,'poeut',3)");
+		hsql.executeQuery("INSERT INTO test VALUES (4,'poeut',4)");
+		hsql.executeQuery("INSERT INTO test VALUES (5,'poeut',5)");
+		hsql.executeQuery("INSERT INTO test VALUES (5,'poeut',1)");
+		
+		hsql.executeQuery("INSERT INTO test2 VALUES (11,'poeut',52)");
+		hsql.executeQuery("INSERT INTO test2 VALUES (22,'poeut',42)");
+		hsql.executeQuery("INSERT INTO test2 VALUES (33,'poeut',32)");
+		hsql.executeQuery("INSERT INTO test2 VALUES (44,'poeut',22)");
+		hsql.executeQuery("INSERT INTO test2 VALUES (45,'poeut',22)");
+		hsql.executeQuery("INSERT INTO test2 VALUES (55,'poeut',12)");
+		
+		hsql.printTable("test")
+		hsql.printTable("test2")
+
+		hsql.close();
+		
+		//
+		//hsql.CreateTable( "test", argsname, types);
+		//	hsql.ImportOrderedCSVTable( "test", argsname, types, new File("~/xml/rgaqcj/lexicon/lex_R"),"\t","UTF-8");
+		//hsql.test("test","test.csv");
+		//def forms=["hello","pouet","truc"]
+		//hsql.CreateSimpleTable("test","form","VARCHAR(20)",forms)
+		//hsql.printTable("test");
+		//hsql.executeQuery("SELECT col1 from test where col1 LIKE '@'' ESCAPE '@' ");
+		
+
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/PatchAfrlex.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/PatchAfrlex.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/PatchAfrlex.groovy	(revision 321)
@@ -0,0 +1,85 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Patch.
+ *
+ * @param correspondanceFile the correspondance file
+ * @param afrlex the afrlex
+ * @param afrlexfixed the afrlexfixed
+ * @param encoding the encoding
+ * @deprecated
+ * @author mdecorde patch a cvs file replace POS name with the
+ * correspondanceFile
+ */
+class PatchAfrlex {
+	public static void patch(File correspondanceFile,File afrlex,File afrlexfixed, String encoding )
+	{
+		HashMap<String,String> correspondances = new HashMap<String,String>();
+		
+		def content = correspondanceFile.getText(encoding) 
+		
+		String separator= "\t"
+		content.splitEachLine(separator) {fields ->
+			correspondances.put( fields[0], fields[1]);
+		}
+		println(correspondances);
+		
+		
+		Writer writer = new FileWriter(afrlexfixed);
+		
+		content = afrlex.getText(encoding) 
+		separator= "\t"
+		content.splitEachLine(separator) {fields ->
+			writer.write(fields[0]);
+			for(int i = 1 ; i< fields.size(); i = i+2)
+			{
+				writer.write("\t"+ correspondances.get(fields[i])+"\t"+fields[i+1]); 				
+			}
+			writer.write("\n")
+		}
+		writer.close();
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File afrlex = new File('afrlex.txt');
+		File afrlexfixed = new File('afrlex-patched.txt');
+		File correspondanceFile = new File('TTnca_2_CTX9.txt');
+		String encoding ="ISO-8859-1"
+		
+		PatchAfrlex.patch( correspondanceFile, afrlex, afrlexfixed, encoding );
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TxmToHSQL.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TxmToHSQL.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TxmToHSQL.groovy	(revision 321)
@@ -0,0 +1,77 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import org.txm.lexicon.core.corpusengine.cqp.Lexicon;
+import org.txm.searchengine.cqp.corpus.Corpus
+import org.txm.searchengine.cqp.corpus.CorpusManager
+import org.txm.searchengine.cqp.corpus.Property
+// TODO: Auto-generated Javadoc
+
+/**
+ * temporary class.
+ *
+ * @author mdecorde
+ */
+class TxmToSQL 
+{
+	
+	/**
+	 * Lexicon to sql.
+	 *
+	 * @param corpus the corpus
+	 * @param lexproperties the lexproperties
+	 * @return the string
+	 */
+	static String LexiconToSQL(Corpus corpus, Property lexproperties)
+	{
+		
+		Lexicon lex = corpus.getLexicon(lexproperties);
+		
+		String[] forms = lex.getForms();
+		
+		HSQLFunctions.CreateSimpleTable(lex.getName(),"form","VARCHAR(50)",forms)
+		return lex.getName();
+	}
+	
+	/**
+	 * Main.
+	 *
+	 * @param args the args
+	 */
+	static main(args) 
+	{
+		//lexicon example
+		CorpusManager cm = CorpusManager.getCorpusManager();
+		Corpus corpus = cm.getCorpus("RGAQCJ");
+		Property p = corpus.getProperty("word");
+		
+		String table = TxmToSQL.LexiconToSQL(corpus,p);
+		HSQLFunctions.printTable(table);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/testTnT.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/testTnT.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/testTnT.groovy	(revision 321)
@@ -0,0 +1,507 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-03-29 09:51:35 +0200 (Tue, 29 Mar 2016) $
+// $LastChangedRevision: 3185 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+/**
+ * test tnt Wrapper :
+ *  add NLP annotations
+ *  build result matrix
+ *  
+ */
+
+import org.txm.utils.*;
+import org.txm.importer.*;
+import org.txm.importer.RGAQCJ.*;
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+String rootDir = "~/xml/rgaqcj/TnT/";
+String modelsDir = rootDir+"models/";
+String textsDir = rootDir+"texts/";
+String projsDir = rootDir+"proj/";
+String srcDir = rootDir+"src/"
+String xslDir = rootDir+"xsl/"
+String anaDir = rootDir+"anainline/"
+new File(modelsDir).mkdir();
+new File(textsDir).mkdir();
+new File(projsDir).mkdir();
+
+def texts =["roland","artu","qjm","commyn1","jehpar","rgaqcj"]
+//def texts =["commyn1","jehpar"]
+String initiales = "RAQCJZ"
+
+//import src
+def srcfiles = ["roland.xml","artu.xml","qjm.xml","commyn1.xml","jehpar.xml"];
+def anafiles = ["roland-ana.xml","artu-ana.xml","qjm-ana.xml","commyn1-ana.xml","jehpar-ana.xml"];
+def xslfiles = ["bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl","bfm2txm-w.xsl"];
+/*
+ for(int i=0 ; i < xslfiles.size() ; i++)
+ {
+ String xslfile = xslDir+xslfiles[i];
+ String infile = srcDir+srcfiles[i];
+ String outfile = anaDir+anafiles[i];
+ println("create xml-txm from "+srcfiles[i]);
+ ApplyXsl a = new ApplyXsl(xslfile);
+ a.process(infile,outfile);
+ }
+ //Import to CWB
+ def cwbLoc ="~/Bureau/textometrie/CWB/cwb-3.0/utils/";//chemin vers executable cqp
+ BuildXmlRGAQCJ.process( anafiles,  rootDir, cwbLoc)
+ */
+//build CWBDecode wrapper
+ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/cwb-decode-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/CwbDecode.groovy"));
+GroovyClassLoader gcl = new GroovyClassLoader();
+gcl.addClasspath(".");
+
+String registryPath = rootDir+"/registry";
+String cwbdecodeexecDir = "~/Bureau/trunkCWB/cwb-3.0/utils/"
+
+Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/CwbDecode.groovy"));
+def cwbdecodewrapper = clazz.newInstance(cwbdecodeexecDir);
+def pAttributesTrain = ["word","cat"];
+def pAttributesTag = ["word"];
+def sAttributes = ["s"];
+cwbdecodewrapper.setC();
+cwbdecodewrapper.setP(pAttributesTag);
+cwbdecodewrapper.setS(sAttributes);
+cwbdecodewrapper.setr(registryPath);
+cwbdecodewrapper.debug(true);
+int[] nbmots = [0,35306,134170,173526,198929,228149];
+
+//create file to proj on
+for(int i = 0 ; i < texts.size() ; i++)
+{
+	String text = texts[i];
+	println("creation TTsrc "+text);
+	
+	//build src files
+	cwbdecodewrapper.setP(pAttributesTag);
+	
+	if(!(initiales[i]+"").equals("Z"))
+	{
+		if(i > 0)
+			cwbdecodewrapper.sets(nbmots[i]+1)
+		else
+			cwbdecodewrapper.sets(nbmots[i])
+		cwbdecodewrapper.sete(nbmots[i+1])
+	}
+	else
+	{
+		cwbdecodewrapper.unsets();
+		cwbdecodewrapper.unsete();
+	}
+	FileOutputStream fos = new FileOutputStream(textsDir+texts[i]+".t");
+	PrintStream ps = new PrintStream(fos);
+	def out = System.out;
+	System.setOut(ps);
+	
+	if(System.getProperty("os.name").contains("Windows"))
+		cwbdecodewrapper.cwbdecodeexe("RGAQCJ")
+	else
+		cwbdecodewrapper.cwbdecode("RGAQCJ")
+	System.setOut(out);
+	
+	//build train files
+	cwbdecodewrapper.setP(pAttributesTrain);
+	
+	fos = new FileOutputStream(modelsDir+texts[i]+".t");
+	ps = new PrintStream(fos);
+	System.setOut(ps);
+	
+	if(System.getProperty("os.name").contains("Windows"))
+		cwbdecodewrapper.cwbdecodeexe("RGAQCJ")
+	else
+		cwbdecodewrapper.cwbdecode("RGAQCJ")
+	System.setOut(out);
+	
+}
+
+//need to replace <s> by nothing and </s> by \n
+String encoding = "UTF-8"
+for(String text : texts)
+{
+	//patch src files
+	File f = new File(textsDir,text+".t");
+	File temp = new File("tempFileCVScleaner")
+	println("patch texts files "+f+": rmv <s> and replace </s>");
+	Reader reader = new InputStreamReader(new FileInputStream(f),encoding);
+	Writer writer = new FileWriter(temp);
+	reader.eachLine 
+			{
+				if(it.trim().startsWith("</s"))
+					writer.write("\n")
+				else if(it.trim().startsWith("<s"))
+					writer.write("")
+				else
+					writer.write(it+"\n")
+			}
+	reader.close();
+	writer.close();
+	if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+}
+//need to replace <s> by nothing and </s> by \n
+for(String text : texts)
+{
+	//patch training files
+	File f = new File(modelsDir,text+".t");
+	File temp = new File("tempFileCVScleaner")
+	println("patch models files "+f+": rmv <s> and replace </s>");
+	Reader reader = new InputStreamReader(new FileInputStream(f),encoding);
+	Writer writer = new FileWriter(temp);
+	reader.eachLine 
+			{
+				if(it.trim().startsWith("</s"))
+					writer.write("\n")
+				else if(it.trim().startsWith("<s"))
+					writer.write("")
+				else
+					writer.write(it+"\n")
+			}
+	reader.close();
+	writer.close();
+	if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+}
+
+//build Tnt Wrapper
+//ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/tnt-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/TnT.groovy"));
+gcl = new GroovyClassLoader();
+gcl.addClasspath(".");
+String tntexecDir = "~/Bureau/tnt/"
+clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/TnT.groovy"));
+def tntwrapper = clazz.newInstance(tntexecDir);
+
+def ttrezfiles = [];
+for(String text : texts)
+{
+	println "Apprentissage de "+text+"..."
+	tntwrapper.tntpara(new File(modelsDir,text+".t"));
+	//new File(text+".123").renameTo(new File(modelsDir,text+".123"));
+	//new File(text+".lex").renameTo(new File(modelsDir,text+".lex"));
+	for(String target : texts)
+	{
+		println "Projection de "+target+" sur "+target+"..."
+		for(int mode=3 ; mode <=3 ; mode++)
+		{
+			println("Mode "+mode+"...")
+			tntwrapper.setu(2);
+			
+			FileOutputStream fos = new FileOutputStream(projsDir+"model_"+text+"_target_"+target+"_mode"+mode+".t");
+			ttrezfiles.add("model_"+text+"_target_"+target+"_mode"+mode+".t")
+			PrintStream ps = new PrintStream(fos);
+			def out = System.out;
+			System.setOut(ps);
+			
+			tntwrapper.tnt(text,new File(textsDir,target+".t"));
+			
+			System.setOut(out);	
+		}
+	}
+}
+
+//def ttrezfiles = ["model_roland_targetroland_mode2.t", "model_roland_targetroland_mode3.t", "model_roland_targetartu_mode2.t", "model_roland_targetartu_mode3.t", "model_roland_targetqjm_mode2.t", "model_roland_targetqjm_mode3.t", "model_roland_targetcommyn1_mode2.t", "model_roland_targetcommyn1_mode3.t", "model_roland_targetjehpar_mode2.t", "model_roland_targetjehpar_mode3.t", "model_roland_targetrgaqcj_mode2.t", "model_roland_targetrgaqcj_mode3.t", "model_artu_targetroland_mode2.t","model_artu_targetroland_mode3.t", "model_artu_targetartu_mode2.t", "model_artu_targetartu_mode3.t", "model_artu_targetqjm_mode2.t", "model_artu_targetqjm_mode3.t", "model_artu_targetcommyn1_mode2.t", "model_artu_targetcommyn1_mode3.t", "model_artu_targetjehpar_mode2.t", "model_artu_targetjehpar_mode3.t", "model_artu_targetrgaqcj_mode2.t", "model_artu_targetrgaqcj_mode3.t", "model_qjm_targetroland_mode2.t, model_qjm_targetroland_mode3.t, model_qjm_targetartu_mode2.t, model_qjm_targetartu_mode3.t, model_qjm_targetqjm_mode2.t, model_qjm_targetqjm_mode3.t, model_qjm_targetcommyn1_mode2.t, model_qjm_targetcommyn1_mode3.t, model_qjm_targetjehpar_mode2.t, model_qjm_targetjehpar_mode3.t, model_qjm_targetrgaqcj_mode2.t, model_qjm_targetrgaqcj_mode3.t, model_commyn1_targetroland_mode2.t, model_commyn1_targetroland_mode3.t, model_commyn1_targetartu_mode2.t, model_commyn1_targetartu_mode3.t, model_commyn1_targetqjm_mode2.t, model_commyn1_targetqjm_mode3.t, model_commyn1_targetcommyn1_mode2.t, model_commyn1_targetcommyn1_mode3.t, model_commyn1_targetjehpar_mode2.t, model_commyn1_targetjehpar_mode3.t, model_commyn1_targetrgaqcj_mode2.t, model_commyn1_targetrgaqcj_mode3.t, model_jehpar_targetroland_mode2.t, model_jehpar_targetroland_mode3.t, model_jehpar_targetartu_mode2.t, model_jehpar_targetartu_mode3.t, model_jehpar_targetqjm_mode2.t, model_jehpar_targetqjm_mode3.t, model_jehpar_targetcommyn1_mode2.t, model_jehpar_targetcommyn1_mode3.t, model_jehpar_targetjehpar_mode2.t, model_jehpar_targetjehpar_mode3.t, model_jehpar_targetrgaqcj_mode2.t, model_jehpar_targetrgaqcj_mode3.t, model_rgaqcj_targetroland_mode2.t, model_rgaqcj_targetroland_mode3.t, model_rgaqcj_targetartu_mode2.t, model_rgaqcj_targetartu_mode3.t, model_rgaqcj_targetqjm_mode2.t, model_rgaqcj_targetqjm_mode3.t, model_rgaqcj_targetcommyn1_mode2.t, model_rgaqcj_targetcommyn1_mode3.t", "model_rgaqcj_targetjehpar_mode2.t", "model_rgaqcj_targetjehpar_mode3.t", "model_rgaqcj_targetrgaqcj_mode2.t", "model_rgaqcj_targetrgaqcj_mode3.t"];
+println ttrezfiles;
+//Build process infos
+//remove lines which starts with %%
+for(String text : ttrezfiles)
+{
+	//def encoding ="UTF-8"
+	println "patch proj file"+text+" : remove %% lines and blank lines AND replace n\t by only one \t";
+	File f = new File(projsDir,text);
+	File temp = new File("tempFileCVScleaner")
+	Reader reader = new InputStreamReader(new FileInputStream(f),encoding);
+	Writer writer = new FileWriter(temp);
+	reader.eachLine 
+			{
+				if(it.trim().startsWith("%%") || it.length() == 0)
+					writer.write("")
+				else
+					writer.write(it.replaceAll("(\t)+","\t")+"\n")
+			}
+	reader.close();
+	writer.close();
+	if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+}
+
+for(String text : texts)
+{
+	//def encoding ="UTF-8"
+	println "patch model file"+text+" : remove blank lines";
+	File f = new File(modelsDir,text+".t");
+	File temp = new File("tempFileCVScleaner")
+	Reader reader = new InputStreamReader(new FileInputStream(f),encoding);
+	Writer writer = new FileWriter(temp);
+	reader.eachLine 
+			{
+				if(it.length() == 0)
+					writer.write("")
+				else
+					writer.write(it+"\n")
+			}
+	reader.close();
+	writer.close();
+	if (!(f.delete() && temp.renameTo(f))) println "Warning can't rename file "+temp+" to "+f
+}
+
+HSQLFunctions.clearAll();
+//import proj table into hsql
+for(int i=0; i < ttrezfiles.size();i++)
+{
+	String csvfile = rootDir+"proj/"+ttrezfiles[i];
+	def argsname = ["form","cat"];
+	def types = ["VARCHAR(30)","VARCHAR(30)"];
+	int linenumber = HSQLFunctions.ImportOrderedCSVTable(ttrezfiles[i].replace(".",""), argsname,types,new File(csvfile),"\t","UTF-8");
+	println("create Table "+ttrezfiles[i].replace(".","")+" : "+linenumber+" lines");
+}
+
+for(String text : texts)
+{
+	File f = new File(modelsDir,text+".t");
+	def argsname = ["form","cat"];
+	def types = ["VARCHAR(30)","VARCHAR(30)"];
+	int linenumber = HSQLFunctions.ImportOrderedCSVTable("lexbrut_"+text.replace(".",""), argsname,types,f,"\t","UTF-8");
+	println("create Table lexbrut_"+text.replace(".","")+" : "+linenumber+" lines");
+}
+
+//calc richesses lexicales
+LinkedHashMap<String,ArrayList<String>> richesses = new LinkedHashMap<String,ArrayList<String>>();
+richesses.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+
+LinkedHashMap<String,ArrayList<String>> richessesCat = new LinkedHashMap<String,ArrayList<String>>();
+richessesCat.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+
+LinkedHashMap<String,ArrayList<String>> richessesocc = new LinkedHashMap<String,ArrayList<String>>();
+richessesocc.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+
+LinkedHashMap<String,ArrayList<String>> richessesoccCat = new LinkedHashMap<String,ArrayList<String>>();
+richessesoccCat.put("Text", ["TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+for(String text : texts)
+{
+	richesses.put(text, new ArrayList<String>());
+	richessesCat.put(text, new ArrayList<String>());
+	richessesocc.put(text, new ArrayList<String>());
+	richessesoccCat.put(text, new ArrayList<String>());
+}
+
+for(String text : texts)
+{
+	for(String model : texts)
+	{
+		if(text.matches(model))
+		{
+			richesses.get(text).add( "0" ) ;
+			richessesCat.get(text).add( "0") ;	
+			richessesocc.get(text).add( "0" ) ;	
+			richessesoccCat.get(text).add( "0" ) ;	
+		}
+		else
+		{
+			String query = "SELECT count(*) FROM ((SELECT DISTINCT form FROM lexbrut_"+text+" ) MINUS (SELECT DISTINCT form FROM lexbrut_"+model+"))"
+			println(query);
+			String query2 = "SELECT count(*) FROM ((SELECT DISTINCT cat FROM lexbrut_"+text+" ) MINUS (SELECT DISTINCT cat FROM lexbrut_"+model+"))"
+			println(query2);
+			String query3 = "SELECT count(form) FROM lexbrut_"+text+" WHERE form NOT IN (SELECT form FROM lexbrut_"+model+")"
+			println(query3);
+			String query4 = "SELECT count(cat) FROM lexbrut_"+text+" WHERE cat NOT IN (SELECT cat FROM lexbrut_"+model+")"
+			println(query4);
+			HSQLFunctions.getGroovySql().eachRow(query) {
+				def rich = it.getAt(0);
+				println("rich form :"+ rich);
+				richesses.get(text).add( ""+rich ) ;	
+			}
+			
+			HSQLFunctions.getGroovySql().eachRow(query2) {
+				def rich = it.getAt(0);
+				println("rich cat :"+ rich);
+				richessesCat.get(text).add( ""+rich ) ;	
+			}
+			
+			
+			HSQLFunctions.getGroovySql().eachRow(query3) {
+				def rich = it.getAt(0);
+				println("richocc form :"+ rich);
+				richessesocc.get(text).add( ""+rich ) ;	
+			}
+			
+			HSQLFunctions.getGroovySql().eachRow(query4) {
+				def rich = it.getAt(0);
+				println("richocc cat :"+ rich);
+				richessesoccCat.get(text).add( ""+rich ) ;	
+			}
+		}
+	}
+}
+
+LinkedHashMap<String,ArrayList<String>> matrix = new LinkedHashMap<String,ArrayList<String>>();
+
+matrix.put("Text", ["T    ","Vf    ","Vc    ","TTrola","TTartu","TTqjm","TTcomm","TTjehpar","TTrgaqcj"]);
+for(String text : texts)
+{
+	matrix.put(text, new ArrayList<String>());
+}
+
+ArrayList<Integer> countOccForm = new ArrayList<Integer>();
+ArrayList<Integer> countOccCat = new ArrayList<Integer>();
+ArrayList<Integer> countLexForm = new ArrayList<Integer>();
+ArrayList<Integer> countLexCat = new ArrayList<Integer>();
+
+for(String text : texts)
+{
+	println("count occ and voc : lexbrut_"+text)
+	String query1 = "SELECT count(*) FROM (SELECT form from lexbrut_"+text+")";
+	String query2 = "SELECT count(*) FROM (SELECT cat from lexbrut_"+text+")";
+	String query3 = "SELECT count(*) FROM (SELECT DISTINCT form from lexbrut_"+text+")";
+	String query4 = "SELECT count(*) FROM (SELECT DISTINCT cat from lexbrut_"+text+")";
+	
+	HSQLFunctions.getGroovySql().eachRow(query1) {
+		def rich = it.getAt(0);
+		println("count occ form :"+ rich);
+		countOccForm.add( ""+rich ) ;	
+	}
+	HSQLFunctions.getGroovySql().eachRow(query2) {
+		def rich = it.getAt(0);
+		println("coutn occ cat :"+ rich);
+		countOccCat.add( ""+rich ) ;	
+	}
+	HSQLFunctions.getGroovySql().eachRow(query3) {
+		def rich = it.getAt(0);
+		println("count lex form :"+ rich);
+		countLexForm.add( ""+rich ) ;	
+	}
+	HSQLFunctions.getGroovySql().eachRow(query4) {
+		def rich = it.getAt(0);
+		println("count lex cat :"+ rich);
+		countLexCat.add( ""+rich ) ;	
+	}
+	//println("col1 et 2 : "+nbOccurencesTxt.get("lexbrut_"+text)+" "+nbFormLexique.get("lex_"+text))
+	matrix.get(text).add( "0") ;
+	matrix.get(text).add( "0") ;
+	matrix.get(text).add( "0") ;
+}
+
+for(String text : texts)
+{
+	int textindex = 0;
+	for(String target : texts)
+	{
+		//get occurences ordonnées de la gold
+			//get occurences ordonnées de la projection
+			//compar
+		String proj = "model_"+text+"_target_"+target+"_mode"+3+"t"
+		println("comp "+proj+" VS lexbrut_"+target);
+		String query = "SELECT count(*) FROM ((SELECT n,form,cat FROM lexbrut_"+target+" ) MINUS (SELECT n,form,cat FROM "+proj+"))"
+		//HSQLFunctions.executeQuery(query);
+		//println(query)
+		int ttotal=0;
+		int total ;
+		HSQLFunctions.getGroovySql().eachRow(query) {
+			def dif = it.getAt(0);
+			/*if((""+initiales[id]) != "Z")
+			 {
+			 total = nbmots[id+1] - nbmots[id];
+			 ttotal += total;
+			 }
+			 else
+			 total= ttotal;
+			 */
+			
+			//total = nbOccurencesTxt.get("lexbrut_"+initiales[id])
+			println("textindex : "+textindex);
+			total = Integer.parseInt(countOccForm[textindex]);
+			println("dif "+dif+"/ tot "+total+" = "+((float)dif/(float)total));
+			Float perf = ((float)dif/(float)total)*100f;			
+			matrix.get(target).add( ""+perf ) ;
+		}
+		textindex++;
+	}
+	
+}
+
+println("Matrice d'erreur : ");
+for(String k : matrix.keySet())
+{
+	print(k)
+	for(String f : matrix.get(k))
+		print("\t"+f)
+	println()
+}
+println("richesse lexiques en Form : ");
+for(String k : richesses.keySet())
+{
+	print(k)
+	for(String f : richesses.get(k))
+		print("\t"+f)
+	println()
+}
+println("richesse lexiques en Cat : ");
+for(String k : richessesCat.keySet())
+{
+	print(k)
+	for(String f : richessesCat.get(k))
+		print("\t"+f)
+	println()
+}
+
+println("richesse occurance en Form : ");
+for(String k : richessesocc.keySet())
+{
+	print(k)
+	for(String f : richessesocc.get(k))
+		print("\t"+f)
+	println()
+}
+println("richesse occurance en Cat : ");
+for(String k : richessesoccCat.keySet())
+{
+	print(k)
+	for(String f : richessesoccCat.get(k))
+		print("\t"+f)
+	println()
+}
+
+println("RichBrutes	roland	artu	qjm	comm	jehpar	rgaqcj");
+print("Focc")
+for(int i=0;i < countOccForm.size() ; i++)
+	print("\t"+countOccForm[i]); 
+print("\nCocc")
+for(int i=0;i < countOccCat.size() ; i++)
+	print("\t"+countOccCat[i]); 
+print("\nFvoc")
+for(int i=0;i < countLexForm.size() ; i++)
+	print("\t"+countLexForm[i]); 
+print("\nCvoc")
+for(int i=0;i < countLexCat.size() ; i++)
+	print("\t"+countLexCat[i]); 
+println()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TnT.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TnT.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/TnT.groovy	(revision 321)
@@ -0,0 +1,778 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import java.util.Date;
+import java.util.Locale;
+import java.text.DateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+// Trigrams'n'Tags : part-of-speach tagger
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class TnT.
+ */
+class TnT {
+	
+	/** The binpath. */
+	String binpath = "";
+
+	/**
+	 * Instantiates a new tn t.
+	 *
+	 * @param binpath the binpath
+	 */
+	public TnT(String binpath) {
+		this.binpath = binpath;
+	}
+
+	/** The version. */
+	String version = "0.0.0";
+	
+	/** The desc. */
+	String desc = "Trigrams'n'Tags : part-of-speach tagger";
+	
+	/** The debug. */
+	boolean debug = false;
+
+	/**
+	 * Debug.
+	 *
+	 * @param b the b
+	 */
+	public void debug(boolean b) {
+		debug = b;
+	};
+
+	// use suffix trie with max. suffix length = len; default = 10
+	
+	/** The isa. */
+	private Boolean isa = false;
+	
+	/** The a. */
+	private int a;
+
+	/**
+	 * Sets the a.
+	 *
+	 * @param arg the new a
+	 */
+	public void seta(int arg) {
+		this.isa = true;
+		this.a = arg;
+	}
+
+	/**
+	 * Unseta.
+	 */
+	public void unseta() {
+		this.isa = false;
+	}
+
+	// use fil as backup lexicon
+	
+	/** The isb. */
+	private Boolean isb = false;
+	
+	/** The b. */
+	private File b;
+
+	/**
+	 * Sets the b.
+	 *
+	 * @param arg the new b
+	 */
+	public void setb(File arg) {
+		this.isb = true;
+		this.b = arg;
+	}
+
+	/**
+	 * Unsetb.
+	 */
+	public void unsetb() {
+		this.isb = false;
+	}
+
+	// backup mode : 0=main only, 1=mix, 2=backup only (default: 1)
+	
+	/** The is b. */
+	private Boolean isB = false;
+	
+	/** The B. */
+	private int B;
+
+	/**
+	 * Sets the b.
+	 *
+	 * @param arg the new b
+	 */
+	public void setB(int arg) {
+		this.isB = true;
+		this.B = arg;
+	}
+
+	/**
+	 * Unset b.
+	 */
+	public void unsetB() {
+		this.isB = false;
+	}
+
+	// sparse data mode
+	
+	/** The isd. */
+	private Boolean isd = false;
+	
+	/** The d. */
+	private String d;
+
+	/**
+	 * Sets the d.
+	 *
+	 * @param arg the new d
+	 */
+	public void setd(String arg) {
+		this.isd = true;
+		this.d = arg;
+	}
+
+	/**
+	 * Unsetd.
+	 */
+	public void unsetd() {
+		this.isd = false;
+	}
+
+	// copy HTML tags to output (without tagging)
+	
+	/** The is h. */
+	private Boolean isH = false;
+
+	/**
+	 * Sets the h.
+	 */
+	public void setH() {
+		this.isH = true;
+	}
+
+	/**
+	 * Unset h.
+	 */
+	public void unsetH() {
+		this.isH = false;
+	}
+
+	// mark unknown words in output with an asterisk (*)
+	
+	/** The ism. */
+	private Boolean ism = false;
+
+	/**
+	 * Setm.
+	 */
+	public void setm() {
+		this.ism = true;
+	}
+
+	/**
+	 * Unsetm.
+	 */
+	public void unsetm() {
+		this.ism = false;
+	}
+
+	// use num-grams, num = 1, 2, 3, default = 3
+	
+	/** The isn. */
+	private Boolean isn = false;
+	
+	/** The n. */
+	private int n;
+
+	/**
+	 * Sets the n.
+	 *
+	 * @param arg the new n
+	 */
+	public void setn(int arg) {
+		this.isn = true;
+		this.n = arg;
+	}
+
+	/**
+	 * Unsetn.
+	 */
+	public void unsetn() {
+		this.isn = false;
+	}
+
+	// unknown word count, default = 3
+	
+	/** The isu. */
+	private Boolean isu = false;
+	
+	/** The u. */
+	private int u;
+
+	/**
+	 * Sets the u.
+	 *
+	 * @param arg the new u
+	 */
+	public void setu(int arg) {
+		this.isu = true;
+		this.u = arg;
+	}
+
+	/**
+	 * Unsetu.
+	 */
+	public void unsetu() {
+		this.isu = false;
+	}
+
+	// set verbosity level num: (default = 3)
+	
+	/** The isv. */
+	private Boolean isv = false;
+	
+	/** The v. */
+	private int v;
+
+	/**
+	 * Sets the v.
+	 *
+	 * @param arg the new v
+	 */
+	public void setv(int arg) {
+		this.isv = true;
+		this.v = arg;
+	}
+
+	/**
+	 * Unsetv.
+	 */
+	public void unsetv() {
+		this.isv = false;
+	}
+
+	// output tag if prob is in beam num, default =0 (means infinity)
+	
+	/** The isz. */
+	private Boolean isz = false;
+	
+	/** The z. */
+	private int z;
+
+	/**
+	 * Sets the z.
+	 *
+	 * @param arg the new z
+	 */
+	public void setz(int arg) {
+		this.isz = true;
+		this.z = arg;
+	}
+
+	/**
+	 * Unsetz.
+	 */
+	public void unsetz() {
+		this.isz = false;
+	}
+
+	// cut off path if prob is not in beam num (default 1000)
+	
+	/** The is z. */
+	private Boolean isZ = false;
+	
+	/** The Z. */
+	private int Z;
+
+	/**
+	 * Sets the z.
+	 *
+	 * @param arg the new z
+	 */
+	public void setZ(int arg) {
+		this.isZ = true;
+		this.Z = arg;
+	}
+
+	/**
+	 * Unset z.
+	 */
+	public void unsetZ() {
+		this.isZ = false;
+	}
+
+	// tagging. the language model is loaded from model.lex and model.123. if
+	// model.map exists, it is used for output mapping compressed files are
+	// recognized by suffix .gz, .bz2, or .Z
+	
+	/**
+	 * Tnt.
+	 *
+	 * @param model the model
+	 * @param corpus the corpus
+	 */
+	public void tnt(String model, File corpus) throws IOException
+	// arg : model name, will be searched in current dir or in the path
+	// specified in the env var TNT_MODELS
+	// arg : corpus file ?
+	{
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "tnt");
+		if (isa) {
+			args.add("-a");
+			args.add("" + a);
+		}
+		if (isb) {
+			args.add("-b");
+			args.add("" + b);
+		}
+		if (isB) {
+			args.add("-B");
+			args.add("" + B);
+		}
+		if (isd) {
+			args.add("-d");
+			args.add("" + d);
+		}
+		if (isH)
+			args.add("-H");
+		if (ism)
+			args.add("-m");
+		if (isn) {
+			args.add("-n");
+			args.add("" + n);
+		}
+		if (isu) {
+			args.add("-u");
+			args.add("" + u);
+		}
+		if (isv) {
+			args.add("-v");
+			args.add("" + v);
+		}
+		if (isz) {
+			args.add("-z");
+			args.add("" + z);
+		}
+		if (isZ) {
+			args.add("-Z");
+			args.add("" + Z);
+		}
+		args.add("" + model);
+		args.add("" + corpus);
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		// pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+
+			for (int c = 0; c < args.size(); c++)
+				System.out.print("" + args.get(c) + " ");
+			System.out.println();
+		}
+	}
+
+	// print accuracy vs. frequency, max freq = max
+	
+	/** The isf. */
+	private Boolean isf = false;
+	
+	/** The f. */
+	private int f;
+
+	/**
+	 * Sets the f.
+	 *
+	 * @param arg the new f
+	 */
+	public void setf(int arg) {
+		this.isf = true;
+		this.f = arg;
+	}
+
+	/**
+	 * Unsetf.
+	 */
+	public void unsetf() {
+		this.isf = false;
+	}
+
+	// ignore upper/lower case of tokens
+	
+	/** The isi. */
+	private Boolean isi = false;
+
+	/**
+	 * Seti.
+	 */
+	public void seti() {
+		this.isi = true;
+	}
+
+	/**
+	 * Unseti.
+	 */
+	public void unseti() {
+		this.isi = false;
+	}
+
+	// lexicon to account for known/unknown words
+	
+	/** The isl. */
+	private Boolean isl = false;
+	
+	/** The l. */
+	private File l;
+
+	/**
+	 * Sets the l.
+	 *
+	 * @param arg the new l
+	 */
+	public void setl(File arg) {
+		this.isl = true;
+		this.l = arg;
+	}
+
+	/**
+	 * Unsetl.
+	 */
+	public void unsetl() {
+		this.isl = false;
+	}
+
+	// counting differences
+	
+	/**
+	 * Tntdiff.
+	 *
+	 * @param originalfile the originalfile
+	 * @param resultfile the resultfile
+	 */
+	public void tntdiff(File originalfile, File resultfile) throws IOException
+	// arg : model file
+	// arg : corpus file ?
+	{
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "tnt-diff");
+		if (isa)
+			args.add("-a");
+		if (isf) {
+			args.add("-f");
+			args.add("" + f);
+		}
+		if (isi)
+			args.add("-i");
+		if (isl) {
+			args.add("-l");
+			args.add("" + l);
+		}
+		if (ism) {
+			args.add("-m");
+			args.add("" + m);
+		}
+		args.add("" + originalfile);
+		args.add("" + resultfile);
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+
+			for (int c = 0; c < args.size(); c++)
+				System.out.print("" + args.get(c) + " ");
+			System.out.println();
+		}
+	}
+
+	// encode capitalization in tag
+	
+	/** The isc. */
+	private Boolean isc = false;
+
+	/**
+	 * Setc.
+	 */
+	public void setc() {
+		this.isc = true;
+	}
+
+	/**
+	 * Unsetc.
+	 */
+	public void unsetc() {
+		this.isc = false;
+	}
+
+	// base name for output files, default=basename of corpus
+	
+	/** The iso. */
+	private Boolean iso = false;
+	
+	/** The o. */
+	private String o;
+
+	/**
+	 * Sets the o.
+	 *
+	 * @param arg the new o
+	 */
+	public void seto(String arg) {
+		this.iso = true;
+		this.o = arg;
+	}
+
+	/**
+	 * Unseto.
+	 */
+	public void unseto() {
+		this.iso = false;
+	}
+
+	// parameters generation
+	
+	/**
+	 * Tntpara.
+	 *
+	 * @param corpus the corpus
+	 */
+	public void tntpara(File corpus) throws IOException
+	// arg : corpus file ?
+	{
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "tnt-para");
+		if (isc)
+			args.add("-c");
+		if (isH)
+			args.add("-H");
+		if (isi)
+			args.add("-i");
+		if (isl)
+			args.add("-l");
+		if (isn)
+			args.add("-n");
+		if (iso) {
+			args.add("-o");
+			args.add("" + o);
+		}
+		if (isv)
+			args.add("-v");
+		args.add("" + corpus);
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+
+			for (int c = 0; c < args.size(); c++)
+				System.out.print("" + args.get(c) + " ");
+			System.out.println();
+		}
+	}
+
+	// count tags
+	
+	/** The ist. */
+	private Boolean ist = false;
+
+	/**
+	 * Sett.
+	 */
+	public void sett() {
+		this.ist = true;
+	}
+
+	/**
+	 * Unsett.
+	 */
+	public void unsett() {
+		this.ist = false;
+	}
+
+	// count words tokens
+	
+	/** The isw. */
+	private Boolean isw = false;
+
+	/**
+	 * Setw.
+	 */
+	public void setw() {
+		this.isw = true;
+	}
+
+	/**
+	 * Unsetw.
+	 */
+	public void unsetw() {
+		this.isw = false;
+	}
+
+	// counting tokens and types
+	
+	/**
+	 * Tntwc.
+	 *
+	 * @param corpus the corpus
+	 */
+	public void tntwc(File corpus) throws IOException
+	// arg : corpus file ?
+	{
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "tnt-wc");
+		if (isH)
+			args.add("-H");
+		if (isi)
+			args.add("-i");
+		if (isl)
+			args.add("-l");
+		if (ist)
+			args.add("-t");
+		if (isw)
+			args.add("-w");
+		args.add("" + corpus);
+
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (Exception err) {
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+
+			for (int c = 0; c < args.size(); c++)
+				System.out.print("" + args.get(c) + " ");
+			System.out.println();
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		TnT tt = new TnT("");
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Sh.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Sh.java	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Sh.java	(revision 321)
@@ -0,0 +1,194 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-09-05 09:27:13 +0200 (jeu., 05 sept. 2013) $
+// $LastChangedRevision: 2529 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import java.util.Date;
+import java.util.Locale;
+import java.text.DateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+import org.txm.utils.logger.Log;
+
+// TODO: Auto-generated Javadoc
+// Shell
+/**
+ * The Class Sh.
+ */
+public class Sh {
+	
+	/** The binpath. */
+	String binpath = "";
+
+	/** The version. */
+	String version = "0.0.0";
+	
+	/** The desc. */
+	String desc = "Shell";
+
+	/**
+	 * Instantiates a new sh.
+	 *
+	 * @param binpath the binpath
+	 */
+	public Sh(String binpath) {
+		// System.out.println("set binpath "+binpath);
+		this.binpath = binpath;
+	}
+
+	// execute command line in argument
+	/** The isc. */
+	private Boolean isc = false;
+	
+	/** The c. */
+	private String c;
+
+	/**
+	 * Sets the isc.
+	 *
+	 * @param arg the new isc
+	 */
+	public void setIsc(String arg) {
+		this.isc = true;
+		this.c = arg;
+	}
+
+	/**
+	 * Unset isc.
+	 */
+	public void unsetIsc() {
+		this.isc = false;
+	}
+
+	// Sh
+	/**
+	 * Shexe.
+	 *
+	 * @throws IOException Signals that an I/O exception has occurred.
+	 */
+	public void shexe() throws IOException {
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "sh.exe");
+		if (isc)
+			args.add("-c " + c);
+		System.out.println(args);
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			System.err.println(e);
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			System.out.println(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (InterruptedException e1) {
+			// TODO Auto-generated catch block
+			e1.printStackTrace();
+		}
+		if (e != 0) {
+			System.err.println("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+		}
+	}
+
+	// Sh
+	/**
+	 * Sh.
+	 *
+	 * @throws IOException Signals that an I/O exception has occurred.
+	 */
+	public void sh() throws IOException {
+		ArrayList<String> args = new ArrayList<String>();
+		args.add(binpath + "sh");
+		if (isc) {
+			args.add("-c");
+			args.add(c);
+		}
+		// System.out.println(args);
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.redirectErrorStream(true);
+		Process process = null;
+		try {
+			process = pb.start();
+		} catch (IOException e) {
+			Log.info(Log.toString(e));
+		}
+		InputStream is = process.getInputStream();
+		InputStreamReader isr = new InputStreamReader(is);
+		BufferedReader br = new BufferedReader(isr);
+		String line;
+		while ((line = br.readLine()) != null) {
+			Log.info(line);
+		}
+		int e = 0;
+		try {
+			e = process.waitFor();
+		} catch (InterruptedException e1) {
+			// TODO Auto-generated catch block
+			e1.printStackTrace();
+		}
+		if (e != 0) {
+			Log.info("Process exited abnormally with code "
+					+ e
+					+ " at "
+					+ DateFormat.getDateInstance(DateFormat.FULL, Locale.UK)
+							.format(new Date()));
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		// Sh tt = new Sh("");
+		// tt.setIsc("kill `ps aux | grep cqpserver | awk '/-P 4877/ {print \\$2}'`");//
+		// try {
+		// tt.sh();
+		// } catch (IOException e) {
+		// // TODO Auto-generated catch block
+		// e.printStackTrace();
+		// }
+	}
+}
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/Sh.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FindUnclosedTag.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FindUnclosedTag.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/FindUnclosedTag.groovy	(revision 321)
@@ -0,0 +1,63 @@
+package org.txm.scripts
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+
+File xmlFile = new File("/home/mdecorde/xml/errorxml/error.xml")
+
+def inputurl = xmlFile.toURI().toURL();
+def inputData = inputurl.openStream();
+def factory = XMLInputFactory.newInstance();
+def parser = factory.createXMLStreamReader(inputData);
+
+opened = []
+locations = []
+structPath = "/"
+
+try {
+for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+	switch (event) {
+		case XMLStreamConstants.START_ELEMENT:
+		String localname = parser.getLocalName();
+			startElement(localname, parser.getLocation().getLineNumber());
+		break;
+		case XMLStreamConstants.END_ELEMENT:
+			String localname = parser.getLocalName();
+			endElement(localname);
+		break
+	}
+}
+} catch(Exception e) {
+	println "** Error while parsing: "+e
+	println "** Error occured at : "+structPath+" line "+parser.getLocation().getLineNumber()
+	println "** Opened tags: "+opened
+	println "** at lines: "+locations
+}
+
+public boolean startElement(String localname, def loc) {
+	println structPath
+	structPath += localname+"/"
+	opened << localname
+	locations << loc
+	return true;
+}
+
+public boolean endElement(String localname) {
+	if (structPath.length() > 1) {
+		def lastTag = opened[-1]
+		if (localname != lastTag) {
+			println "ERROR: closing tag '$localname' does not match last opened tag '$lastTag'"
+			println opened
+			return false;
+		}
+	
+		int idx = structPath.substring(0,structPath.length()-1).lastIndexOf("/");
+		if (idx > 0)
+			structPath = structPath.substring(0, idx+1)
+		//
+		opened.pop()
+		locations.pop()
+			
+		return true
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains various Groovy scripts not yet classified</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/BuildAfrLexicon.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/BuildAfrLexicon.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/BuildAfrLexicon.groovy	(revision 321)
@@ -0,0 +1,140 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts;
+
+import groovy.lang.GroovyClassLoader
+
+import java.io.File
+
+import org.txm.utils.ProcessBuilderBuilder
+// TODO: Auto-generated Javadoc
+
+/**
+ * build an afr lexicon in a SQL table.
+ *
+ * @author mdecorde
+ */
+class BuildAfrLexicon 
+{
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	static void main(String[] args)
+	{
+		HSQLFunctions.clearAll();
+		
+		//get lexicon from TXM RGAQCJ
+		ProcessBuilderBuilder.build(new File("src/groovy/org/txm/scripts/cwb-decode-wrapper-definition.xml"), new File("src/groovy/org/textometrie/scripts/CwbDecode.groovy"));
+
+		GroovyClassLoader gcl = new GroovyClassLoader();
+		gcl.addClasspath(".");
+
+		String registryPath = "~/Bureau/trunkCWB/corpora/registry";
+		//String registryPath = "C:/Documents and Settings/sheiden/.txm/registry"
+		 String cwbdecodeexecDir = "~/Bureau/trunkCWB/cwb-3.0/utils/"
+		//String cwbdecodeexecDir = "C:/Projets/Textom�trie/Logiciel/Toolbox/0.4.5/toolbox/src/main/C/cwb-3.0/utils/"
+
+		Class clazz = gcl.parseClass(new File("src/groovy/org/txm/scripts/CwbDecode.groovy"));
+		def aScript = clazz.newInstance(cwbdecodeexecDir);
+
+		aScript.setC()
+		def pAttributes = ["word","CATTEX2009","LEMMA"];
+		def sAttributes = ["s"];
+		aScript.setP(pAttributes)
+		//aScript.setS(sAttributes)
+		aScript.setr(registryPath)
+		
+String rootdir = "~/xml/rgaqcj/lexicon";
+File lexRGAQCJFile = new File(rootdir, "lexRGAQCJ.txt");
+FileOutputStream fos = new FileOutputStream(lexRGAQCJFile);
+PrintStream ps = new PrintStream(fos);
+def out = System.out;
+System.setOut(ps);
+
+		if(System.getProperty("os.name").contains("Windows"))
+			aScript.cwbdecodeexe("RGAQCJ")
+		else
+			aScript.cwbdecode("RGAQCJ")
+System.setOut(out);	
+println("fin")
+		//Load lexRGAQCJ.txt >> lexRGAQCJ(form,cat,val)
+		String[] argsname = ["form","cat","val"];
+		String[] types = ["VARCHAR(30)","VARCHAR(30)","VARCHAR(100)"];
+
+		HSQLFunctions.ImportRefTable("lexRGAQCJ", argsname,types,lexRGAQCJFile,"\t","ISO-8859-1");
+		
+		//load afrlex.txt >> afrlex(form,cat,val)
+		File afrlexFile = new File(rootdir, "afrlex.txt");
+		HSQLFunctions.ImportRefTable("afrlex", argsname,types,afrlexFile,"\t","ISO-8859-1")
+
+		//Load TTcna_2_CTX9.txt >> TTcna_2_CTX9(catcna,catctx9)
+		File correspFile = new File(rootdir, "TTnca_2_CTX9.txt");
+		argsname = ["catcna","catctx9"];
+		types = ["VARCHAR(20)","VARCHAR(20)"];
+		HSQLFunctions.ImportCSVTable("TTnca_2_CTX9", argsname,types,correspFile,"\t","ISO-8859-1")
+
+		//Union(lexRGAQCJ, afrlex) >> lextotal(form,cat,val)
+		//HSQLFunctions.CreateTable( "lextotal", argsname, types);
+		String query = 	"INSERT INTO afrlex " +
+						"SELECT * from lexRGAQCJ "+
+						"WHERE " +
+						"form NOT IN (SELECT form FROM afrlex) " +
+						"AND cat NOT IN (SELECT cat FROM afrlex) ";  
+		HSQLFunctions.executeQuery( query );
+
+		
+		//check for missing cat correspondence
+		String query2 = "SELECT cat from afrlex WHERE cat NOT IN (SELECT catcna FROM TTnca_2_CTX9);"
+		int numberOfCatNotSpecif = 0
+		def addedCat = [];
+		HSQLFunctions.getGroovySql().eachRow(query2) {
+			if(!addedCat.contains(it.getAt(0)))
+			{
+				addedCat.add(it.getAt(0));
+				println("!! No correspondance for "+ it.getAt(0));
+				HSQLFunctions.executeQuery("INSERT INTO TTnca_2_CTX9 VALUES ('"+it.getAt(0)+"','"+it.getAt(0)+"ERROR')");
+				numberOfCatNotSpecif++;
+			}
+		}
+		
+		//Substitution des cat >> lexRGAQCJ(form,cat,val)
+		String query3 = "UPDATE afrlex " +
+				"SET cat = (SELECT catctx9 FROM TTnca_2_CTX9 WHERE cat=catcna)";				
+		HSQLFunctions.executeQuery( query3 );
+		
+		HSQLFunctions.printTable "afrlex";
+	
+		//export lexRGAQCJ >> lexRGAQCJ.txt
+		File rezlexRGAQCJFile = new File(rootdir, "lexfra-rgaqcj.txt");
+		HSQLFunctions.toRefFile( "afrlex", rezlexRGAQCJFile.getAbsolutePath(),"form");
+		
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/RegexCounter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/RegexCounter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/scripts/RegexCounter.groovy	(revision 321)
@@ -0,0 +1,49 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.scripts
+// TODO: Auto-generated Javadoc
+
+/**
+ * TODO.
+ *
+ * @param sourcedir the sourcedir
+ * @param regex the regex
+ * @author mdecorde
+ */
+class RegexCounter {
+	public RegexCounter(File sourcedir, String encoding, String regex)
+	{
+		String text = sourcedir.getText(encoding);
+		java.util.regex.Matcher m = text =~ regex;
+		println m.count;
+	}
+	
+	public static void main(String[] args) {
+		new RegexCounter(new File("/home/mdecorde/xml/discours/01_DeGaulle.cnr"), "UTF-8", "je")
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Prototype scripts of the "Mesures" functionnality</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Proportion.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Proportion.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Proportion.groovy	(revision 321)
@@ -0,0 +1,200 @@
+package org.txm.functions.mesures;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.txm.searchengine.cqp.clientExceptions.CqiClientException;
+import org.txm.searchengine.cqp.corpus.Corpus;
+import org.txm.searchengine.cqp.corpus.QueryResult;
+import org.txm.searchengine.cqp.corpus.query.Match;
+import org.txm.searchengine.cqp.corpus.query.Query;
+
+public class Proportion extends Mesure {
+
+	private static final long serialVersionUID = -3160461437451974147L;
+
+	public static List<Synthese> SUPORTED = Arrays.asList(Synthese.COUNTMATCHES, 
+			Synthese.SUM);
+
+	protected String all;
+	protected List<String> cas;
+
+	public Proportion(String domaine, String unit, def syntheses, String all, List<String> cas) {
+		this.domaine = domaine;
+		this.unit= unit;
+		this.syntheses = syntheses;
+
+		this.all = all;
+		this.cas = cas;
+	}
+
+	@Override
+	public boolean compute(ArrayList<Corpus> arg0) {
+		this.corpora = arg0;
+		//		switch (synthese) {
+		//		case COUNTMATCHES:
+		//			return computeMatches();
+		//		case SUM:
+		//			return computeSum();
+		//		default:
+		//			System.out.println("UNSUPPORTED SYNTHESIS: "+synthese);
+		//			return false;
+		//		}
+		return computeSynthese();
+	}
+
+	HashMap<Synthese, ArrayList<Object[]>> rezsPerSynt = new HashMap<Synthese, ArrayList<Object[]>>();
+	private boolean computeSynthese() {
+
+		// initialize list of results 
+		for (Synthese synt : syntheses) {
+			ArrayList<Object[]> rezs = new ArrayList<Object[]>();
+			for (String c : cas) {
+				rezs.add(new Number[corpora.size()]);
+			}
+			rezsPerSynt.put(synt, rezs);
+		}
+
+		for (int icorpus = 0 ; icorpus < corpora.size() ; icorpus++) {
+			computeForCorpus(icorpus, corpora.get(icorpus)); // one cell per cas
+			//			Object[] r = compute(icorpus, corpora.get(icorpus)); // one cell per cas
+			//			for( int i = 0 ; i < cas.size() ; i++)
+			//				rezs.get(i)[icorpus] = r[i];
+		}
+
+		// register results
+		for (Synthese synt : syntheses) {
+			ArrayList<Object[]> rezs = rezsPerSynt.get(synt);
+			for (int i = 0 ; i < cas.size() ; i++) {
+				Object[] rez = rezs.get(i);
+				this.add(new MesureResult(this, rez, "Proportion synt="+synt+" domaine="+domaine+" cas="+cas.get(i)+" all="+all+")"));
+			}
+		}
+		return true;
+	}
+
+	private Integer[] countMatchesInMatches(List<Match> domaine, List<Match> cas) {
+		//		System.out.println("count dom.size "+domaine.size()+" cas.size "+cas.size());
+		//		System.out.println(domaine);
+		//		System.out.println(cas);
+
+		Integer[] rez = new Integer[domaine.size()];
+		int iCas = 0;
+		int iDom = 0;
+		rez[0] = 0;
+
+		int firsterror = 0;
+		while(iCas < cas.size() & iDom < domaine.size()) {
+			Match mDom = domaine.get(iDom);
+			Match mCas = cas.get(iCas);
+
+			if (mDom.getStart() <= mCas.getStart() & mCas.getEnd() <= mDom.getEnd()) {
+				rez[iDom] = rez[iDom] + 1; 
+				iCas++;
+			} else if ( mCas.getStart() < mDom.getStart()) {
+				iCas++;
+			} else if ( mDom.getEnd() < mCas.getEnd()) {
+				iDom++;
+				rez[iDom] = 0;
+			} else {
+				if (firsterror++ == 0)
+					System.out.println("ERROR MATCH COUNT: dom="+domaine+" cas="+cas);
+				System.out.println("ERROR idom="+iDom+" icas="+iCas);
+			}
+		}
+		//		for(int i = 0 ; i < rez.length ; i++) {
+		//			Match m = domaine.get(i);
+		//			while (currentMatchCas != null) {
+		//				if (m.contains(currentMatchCas)) {
+		//					rez[i] = rez[i]+1;
+		//				} else {
+		//					break; // next domaine match
+		//				}
+		//				// next cas match
+		//				if (iCas == cas.size()) break;
+		//				else currentMatchCas = cas.get(iCas++);
+		//			}
+		//		}
+
+		// complete counts
+		for (int i = 0 ; i < rez.length ; i++)
+			if (rez[i] == null) rez[i] = 0;
+
+		return rez;
+	}
+
+	private void computeForCorpus(int icorpus, Corpus corpus) {
+		HashMap<Synthese, Number> allSyntheses = new HashMap<Synthese, Number>();
+		HashMap<Synthese, Number[]> casSyntheses = new HashMap<Synthese, Number[]>();
+		for (Synthese synt : syntheses) {
+			casSyntheses.put(synt, new Number[cas.size()]);
+		}
+		//Object[] rez = new Object[cas.size()];
+		try {
+			Corpus domaineSub = corpus;
+			if (domaine != null && domaine.length() > 0)
+				domaineSub = corpus.createSubcorpus(new Query(domaine), "domaine");
+			else 
+				domaine = corpus.getName();
+			//System.out.println("domaineSub.name "+ domaineSub.getName());
+			//System.out.println("domaine "+ domaine);
+			List<Match> domaineMatches = domaineSub.getMatches();
+			//System.out.println("domaineMatches.size"+ domaineMatches.size());
+			// first process the all case
+
+			QueryResult allSub = domaineSub.query(new Query(all), "all", false);
+			Integer[] allCounts = countMatchesInMatches(domaineMatches, allSub.getMatches());
+			//System.out.println("allCounts.length "+allCounts.length+" count : "+Arrays.toString(allCounts));
+			builder = new SyntheseBuilder(allCounts);
+			for( Synthese synt : syntheses) {
+				Synthese s = synt;
+				if (s.equals(Synthese.NONE)) s = Synthese.SUM;
+				allSyntheses.put(synt, builder.doSynthese(s));
+			}
+
+			// then process each cas
+			for (int iCas = 0 ; iCas < cas.size() ; iCas++) {
+				String c = cas.get(iCas);
+				QueryResult result = domaineSub.query(new Query(c), "TMP", false);
+				Integer[] counts = countMatchesInMatches(domaineMatches, result.getMatches());
+
+				Float[] rapports = new Float[counts.length];
+				for (int i = 0 ; i < rapports.length ; i++) {
+					if (allCounts[i] == 0) rapports[i] = 0.0f;
+					else rapports[i] = (float)counts[i] / (float)allCounts[i];
+				}
+				
+				for (Synthese synt : syntheses) {
+					
+					if (synt.equals(Synthese.NONE)) {
+						int allTotal = 0;
+						for( int i : allCounts) allTotal += i;
+						int casTotal = 0;
+						for( int i : counts) casTotal += i;
+								
+						ArrayList<Object[]> rez = rezsPerSynt.get(synt);
+						rez.get(iCas)[icorpus] = (float)casTotal / (float)allTotal;
+					} else {
+						
+						builder = new SyntheseBuilder(rapports);
+
+						Synthese s = synt;
+						if (s.equals(Synthese.NONE)) s = Synthese.SUM;
+
+						ArrayList<Object[]> rez = rezsPerSynt.get(synt);
+						rez.get(iCas)[icorpus] = builder.doSynthese(s);
+					}
+				}
+			}
+
+			if (!domaineSub.equals(corpus)) { // cleaning
+				domaineSub.delete();
+			}
+		} catch (CqiClientException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Magnitude.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Magnitude.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Magnitude.groovy	(revision 321)
@@ -0,0 +1,127 @@
+package org.txm.functions.mesures;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.txm.searchengine.cqp.clientExceptions.CqiClientException;
+import org.txm.searchengine.cqp.corpus.Corpus;
+import org.txm.searchengine.cqp.corpus.QueryResult;
+import org.txm.searchengine.cqp.corpus.query.Match;
+import org.txm.searchengine.cqp.corpus.query.Query;
+
+/**
+ * 
+ * Magnitude synonymes
+extent	Magnitude, importance, extension, superficie, longueur
+magnitude	Magnitude, grandeur, magnitude
+amplitude	amplitude, Magnitude, largeur
+fullness	plénitude, Magnitude, abondance
+width	largeur, Magnitude
+fulness	plénitude, Magnitude, abondance
+richness	richesse, abondance, Magnitude, luxe, fertilité
+ampleness	Magnitude
+thoroughness	minutie, Magnitude
+roundness	rondeur, circularité, Magnitude, franchise
+
+ * @author mdecorde
+ *
+ */
+public class Magnitude extends Mesure {
+
+	private static final long serialVersionUID = 3317066047569716511L;
+
+	private static String TMP = "TMP";
+	
+	public Magnitude(String domaine, String unit, Synthese[] syntheses) {
+		this.domaine = domaine;
+		this.unit= unit;
+		this.syntheses = syntheses;
+
+		//if (synthese.equals(Synthese.NONE)) this.synthese = Synthese.SUM; // NO synthese will sum the matches size
+	}
+
+	@Override
+	public boolean compute(ArrayList<Corpus> corpora) {
+		this.corpora = corpora;
+
+		//		switch (synthese) {
+		//		case COUNTMATCHES:
+		//			return computeCounts();
+		//		case MAX:
+		//		case MIN:
+		//		case MEAN:
+		//		case QUARTILE1:
+		//		case QUARTILE3:
+		//		case MEDIAN:
+		//		case NONE:
+		//		case SUM:
+		return computeSynthese();
+		//		default:
+		//			System.out.println("UNSUPPORTED SYNTHESIS: "+synthese);
+		//			return false;
+		//		}
+	}
+
+	//	private boolean computeCounts() {
+	//		Integer[] rez = new Integer[corpora.size()];
+	//		for (int i = 0 ; i < corpora.size() ; i++) {
+	//			Corpus corpus = corpora.get(i);
+	//			try {
+	//				rez[i] = computeCount(corpus);
+	//			} catch (CqiClientException e) {
+	//				e.printStackTrace();
+	//				return false;
+	//			}
+	//		}
+	//		this.add(new MesureResult(this, rez, "Magnitude synt="+synthese+" unit="+unit+" dom="+domaine));
+	//		return true;
+	//	}
+
+	//	private int computeCount(Corpus corpus) throws CqiClientException {
+	//		QueryResult result = corpus.query(new Query(domaine), TMP);
+	//		return result.getNMatch();
+	//	}
+	HashMap<Synthese, Object[]> rezPerSynthese = new HashMap<Synthese, Object[]>();
+	private boolean computeSynthese() {
+
+		for (Synthese synt : syntheses) {
+			rezPerSynthese.put(synt, new Object[corpora.size()]);
+		}
+
+		for (int i = 0 ; i < corpora.size() ; i++) {
+			Corpus corpus = corpora.get(i);
+			try {
+				computeSynthese(i, corpus);
+			} catch (CqiClientException e) {
+				e.printStackTrace();
+				return false;
+			}
+		}
+		for ( Synthese synt : syntheses) {
+			this.add(new MesureResult(this, rezPerSynthese.get(synt), "Magnitude synt="+synt+" unit="+unit+" dom="+domaine));
+		}
+		return true;
+	}
+
+	private void computeSynthese(int icorpus, Corpus corpus) throws CqiClientException {
+        QueryResult result = corpus.query(new Query(domaine), TMP, false);		
+		List<Match> matches = result.getMatches();
+		Integer[] sizes = new Integer[matches.size()];
+		for (int i = 0 ; i < matches.size() ; i++) {
+			Match m  = matches.get(i);
+			sizes[i] = m.getEnd() - m.getStart() + 1;
+		}
+		
+		builder = new SyntheseBuilder(sizes);
+		for (Synthese synt : syntheses) {
+			Object[] rez = rezPerSynthese.get(synt);
+			if (synt.equals(Synthese.NONE)) {
+				rez[icorpus] = builder.doSynthese(Synthese.SUM);
+			} else {
+				rez[icorpus] = builder.doSynthese(synt);
+			}
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/MesureResult.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/MesureResult.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/MesureResult.groovy	(revision 321)
@@ -0,0 +1,13 @@
+package org.txm.functions.mesures;
+
+public class MesureResult {
+	public Mesure from;
+	public Object[] data;
+	public String name;
+	
+	public MesureResult(Mesure from, Object[] data, String name) {
+		this.from = from;
+		this.data = data;
+		this.name = name;
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Mesure.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Mesure.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Mesure.groovy	(revision 321)
@@ -0,0 +1,19 @@
+package org.txm.functions.mesures;
+
+import java.util.ArrayList;
+
+import org.txm.searchengine.cqp.corpus.Corpus;
+
+public abstract class Mesure extends ArrayList<MesureResult> {
+	
+	private static final long serialVersionUID = 4804073482551008502L;
+	
+	protected String domaine;
+	protected String unit;
+	protected Synthese[] syntheses;
+	protected SyntheseBuilder builder;
+
+	protected ArrayList<Corpus> corpora;
+	
+	public abstract boolean compute(ArrayList<Corpus> subcorpora);
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/PresenceRate.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/PresenceRate.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/PresenceRate.groovy	(revision 321)
@@ -0,0 +1,10 @@
+package org.txm.functions.mesures;
+import java.util.Arrays;
+
+public class PresenceRate extends Proportion {
+	public PresenceRate(String unit, def syntheses, String all, String cas) {
+		super(all, unit, syntheses, all, Arrays.asList(cas));
+//remplace momentanément la ligne suivante, pour pouvoir tourner sur une liste de corpus
+//		super(null, unit, syntheses, all, Arrays.asList(cas));
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Diversity.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Diversity.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Diversity.groovy	(revision 321)
@@ -0,0 +1,60 @@
+package org.txm.functions.mesures;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import org.txm.Toolbox;
+import org.txm.searchengine.cqp.clientExceptions.CqiClientException;
+import org.txm.searchengine.cqp.corpus.Corpus;
+import org.txm.searchengine.cqp.corpus.Property;
+import org.txm.searchengine.cqp.corpus.QueryResult;
+import org.txm.searchengine.cqp.corpus.query.Query;
+import org.txm.searchengine.cqp.serverException.CqiServerError;
+
+public class Diversity extends Mesure {
+
+	private static final long serialVersionUID = -375293299211271669L;
+	
+	public static List<Synthese> SUPORTED = Arrays.asList(Synthese.SUM);
+	private String query;
+	private String property;
+
+	public Diversity(String domaine, String query, String property) {
+		this.domaine = domaine;
+		this.query = query;
+		this.property = property;
+	}
+
+	@Override
+	public boolean compute(ArrayList<Corpus> corpora) {
+		this.corpora = corpora;
+		Integer[] rez = new Integer[corpora.size()];
+		for (int icorpus = 0 ; icorpus < corpora.size() ; icorpus++) {
+			try {
+				Corpus corpus = corpora.get(icorpus);
+				rez[icorpus] = computeForCorpus(corpus);
+			} catch (Exception e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+				return false;
+			}	
+		}
+		this.add(new MesureResult(this, rez, "Diversite: "+query+" "+property));
+		return true;
+	}
+	
+	private Integer computeForCorpus(Corpus corpus) throws CqiClientException, IOException, CqiServerError {
+		if (domaine != null) corpus = corpus.createSubcorpus(new Query(domaine), "DOMAINE");
+		
+		Property p = corpus.getProperty(property);
+		QueryResult result = corpus.query(new Query(query), "TMP", false);
+		int[] starts = result.getStarts();
+		int[] ids = Toolbox.getCqiClient().cpos2Id(p.getQualifiedName(), starts);
+		HashSet<Integer> set = new HashSet<Integer>();
+		for (int i : ids) set.add(i); // no double
+		return set.size();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Synthese.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Synthese.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Synthese.groovy	(revision 321)
@@ -0,0 +1,17 @@
+package org.txm.functions.mesures;
+
+public enum Synthese {
+	
+	NONE,
+	COUNTMATCHES,
+	SUM,
+	MEAN,
+	MIN,
+	QUARTILE1,
+	MEDIAN,
+	QUARTILE3,
+	MAX,
+	FIRST,
+	LAST,
+	MIDDLE,
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Mesures.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Mesures.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/Mesures.groovy	(revision 321)
@@ -0,0 +1,164 @@
+package org.txm.functions.mesures;
+
+import java.text.DecimalFormat
+
+import org.txm.searchengine.cqp.corpus.Corpus
+import org.txm.searchengine.cqp.corpus.Partition
+
+/**
+ * Container of mesures
+ * 
+ * @author mdecorde
+ *
+ */
+public class Mesures extends ArrayList<Mesure>{
+
+	private static final long serialVersionUID = 324346022230541761L;
+
+	ArrayList<Corpus> data = new ArrayList<Corpus>();
+
+	public Mesures(Corpus corpus) {
+		data = new ArrayList<Corpus>();
+		data.add(corpus);
+	}
+
+	public Mesures(ArrayList<Corpus> corpora) {
+		data = corpora;
+		println "hello TXM"
+	}
+
+	public Mesures(Partition partition) {
+		data.addAll(partition.getParts());
+	}
+
+	public boolean add(Mesure mesure) {
+		if (!mesure.compute(data)) {
+			return false;
+		}
+
+		return super.add(mesure);
+	}
+
+	public boolean export(Writer writer, String encoding, String colseparator, String txtseparator) throws IOException {
+		String line = "Mesures";
+		for (Corpus c : data) {
+			line += colseparator+txtseparator+c.getName().replaceAll(txtseparator,txtseparator+txtseparator)+txtseparator;
+		}
+		writer.write(line+"\n");
+
+		for (Mesure m : this) {
+			for (MesureResult r : m) {
+				line = r.name;
+				for (int i = 0 ; i < r.data.length ; i++)
+					line += colseparator+r.data[i];
+				writer.write(line+"\n");
+			}
+		}
+
+		return true;
+	}
+
+	public boolean export(File outfile, String encoding, String colseparator, String txtseparator) throws IOException {
+		Writer writer = new OutputStreamWriter(
+				new FileOutputStream(outfile), encoding); //$NON-NLS-1$
+		boolean rez = export(writer, encoding, colseparator, txtseparator);
+		writer.close();
+		return rez;
+	}
+
+	DecimalFormat floatFormatter = new DecimalFormat("###,###.##");
+	DecimalFormat intFormatter = new DecimalFormat("###,###,###");
+	public void prettyPrint() {	
+		ArrayList<String[]> lines = new ArrayList<String[]>();
+		int[] maxx = new int[data.size()+1];
+		int[] maxxDecimales = new int[data.size()+1];
+
+
+		String[] line = new String[data.size() + 1];
+		line[0] = "Mesures";
+		maxx[0] = line[0].length();
+		for (int i = 1 ; i <= data.size() ; i++) {
+			line[i] = data.get(i-1).getName();
+			maxx[i] = line[i].length();
+		}
+		lines.add(line);
+
+		// fill lines
+		for (Mesure m : this) {
+			for(MesureResult r : m) {
+				line = new String[data.size()+1];
+				line[0] = r.name;
+				if (maxx[0] < r.name.length())
+					maxx[0] = r.name.length();
+
+				int i = 1;
+				for (Object o : r.data) {
+					String str = ""+o;
+					if (o instanceof Integer) str = intFormatter.format(o);
+					else if (o instanceof Float) str = floatFormatter.format(o);
+
+					line[i] = str;
+
+//					if (maxx[i] < str.length())
+//						maxx[i] = str.length();
+
+					int idxDecimal = str.indexOf(",");
+					if (idxDecimal > 0) {
+						idxDecimal = str.length() - idxDecimal;
+						if (maxxDecimales[i] < idxDecimal)
+							maxxDecimales[i] = idxDecimal;
+					}
+					i++;
+				}
+				lines.add(line);
+			}
+		}
+
+//		System.out.println("maxx: "+Arrays.toString(maxx));
+//		System.out.println("maxxDecimales: "+Arrays.toString(maxxDecimales));
+
+		// fix cells right lengths (decimals)
+		boolean notfirst = false;
+		for (String[] l : lines) {
+			for (int i = 1 ; i < data.size() +1; i++) {
+				String str = l[i];
+				int nbDecimalToAdd = 0;
+
+				if (str.indexOf(",") < 0) {
+					nbDecimalToAdd = maxxDecimales[i];
+				} else {
+					nbDecimalToAdd = maxxDecimales[i] - (str.length() - str.indexOf(","));
+				}
+
+				if (notfirst)
+				for (int j = 0 ; j < nbDecimalToAdd ; j++)
+					str += " ";
+
+				if (maxx[i] < str.length())
+					maxx[i] = str.length();
+				
+				l[i] = str;
+			}
+			notfirst = true;
+		}
+		
+		// fix cells left lengths (alignment)
+		for (String[] l : lines) {
+			for (int i = 0 ; i < data.size() +1; i++) {
+				while (l[i].length() < maxx[i]) {
+					if (i == 0)	l[i] = ""+l[i]+" ";
+					else l[i] = " "+l[i];
+				}
+			}
+		}
+
+		// print cells
+		for (String[] l : lines) {
+			for (int i = 0 ; i < data.size() +1; i++) {
+				if (i == 0) System.out.print(l[i]);
+				else  System.out.print(" "+l[i]);
+			}
+			System.out.println("");
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/SyntheseBuilder.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/SyntheseBuilder.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/functions/mesures/SyntheseBuilder.groovy	(revision 321)
@@ -0,0 +1,157 @@
+package org.txm.functions.mesures;
+
+import java.util.Arrays;
+
+import org.apache.tools.ant.types.resources.selectors.InstanceOf;
+
+public class SyntheseBuilder {
+	private Integer[] isrc;
+	private Integer[] isorted;
+	private Float[] fsrc;
+	private Float[] fsorted;
+
+	public SyntheseBuilder(Integer[] rez) {
+		assert(rez.length > 0);
+		isrc = rez;
+		isorted = new Integer[rez.length];
+		System.arraycopy(rez, 0, isorted, 0, rez.length);
+		Arrays.sort(isorted);
+	}
+
+	public SyntheseBuilder(Float[] rez) {
+		assert(rez.length > 0);
+		fsrc = rez;
+		fsorted = new Float[rez.length];
+		System.arraycopy(rez, 0, fsorted, 0, rez.length);
+		Arrays.sort(fsorted);
+	}
+
+	public Number doSynthese(Synthese synt) {
+		if (isorted != null) {
+			return doSynthese(synt, isrc);
+		} else {
+			return doSynthese(synt, fsrc);
+		}
+	}
+
+	private Integer[] getSorted(Integer[] rez) {
+		return isorted;
+	}
+
+	private Float[] getSorted(Float[] rez) {
+		return fsorted;
+	}
+
+	private Number[] getSorted(Number[] rez) {
+		if (isorted != null) return isorted;
+		return fsorted;
+	}
+
+	public Number doSynthese(Synthese synt, Number[] rez) {
+		if (rez instanceof Integer[])
+			return doISynthese(synt, rez);
+		else if  (rez instanceof Float[])
+			return doFSynthese(synt, rez);
+
+		println "Error: doSynthese unsuported array type: "+rez.getClass();
+		return -1;
+	}
+
+	public Number doISynthese(Synthese synt, Integer[] rez) {
+		switch (synt) {
+			case Synthese.MIN: return min(rez);
+			case Synthese.MAX: return max(rez);
+			case Synthese.MEAN: return mean(rez);
+			case Synthese.QUARTILE1: return quartile1(rez);
+			case Synthese.MEDIAN: return median(rez);
+			case Synthese.QUARTILE3: return quartile3(rez);
+			case Synthese.SUM: return sum(rez);
+			case Synthese.COUNTMATCHES: return rez.length;
+			case Synthese.FIRST:
+				if (rez.length > 0) return rez[0];
+				else return -1;
+			case Synthese.LAST:
+				if (rez.length > 0) return rez[(int)rez.length -1];
+				else return -1;
+			case Synthese.MIDDLE:
+				if (rez.length > 0) return rez[(int)rez.length/2];
+				else return -1;
+		}
+		return null;
+	}
+
+	public Number doFSynthese(Synthese synt, Float[] rez) {
+		switch (synt) {
+			case Synthese.MIN: return min(rez);
+			case Synthese.MAX: return max(rez);
+			case Synthese.MEAN: return mean(rez);
+			case Synthese.QUARTILE1: return quartile1(rez);
+			case Synthese.MEDIAN: return median(rez);
+			case Synthese.QUARTILE3: return quartile3(rez);
+			case Synthese.SUM: return sum(rez);
+			case Synthese.COUNTMATCHES: return rez.length;
+			case Synthese.FIRST:
+				if (rez.length > 0) return rez[0];
+				else return -1;
+			case Synthese.LAST:
+				if (rez.length > 0) return rez[(int)rez.length -1];
+				else return -1;
+			case Synthese.MIDDLE:
+				if (rez.length > 0) return rez[(int)rez.length/2];
+				else return -1;
+		}
+		return null;
+	}
+
+	public Number min(Number[] obj) {
+		obj = getSorted(obj);
+		return obj[0];
+	}
+
+	public Number max(Number[] obj) {
+		obj = getSorted(obj);
+		return obj[obj.length - 1];
+	}
+
+	public Number sum(Float[] obj) {
+		Float sum = 0.0f;
+		for (int i = 0 ; i < obj.length ; i++)
+			sum += obj[i];
+		return sum;
+	}
+
+	public Number sum(Integer[] obj) {
+		Integer sum = 0;
+		for (int i = 0 ; i < obj.length ; i++)
+			sum += obj[i];
+		return sum;
+	}
+
+	public Number mean(Integer[] obj) {
+		Integer i = (Integer) sum(obj);
+		return (float) i / (float)obj.length;
+	}
+
+	public Number mean(Float[] obj) {
+		Float f = (Float) sum(obj);
+		return (float) f / obj.length;
+	}
+
+	public Number quartile1(Number[] obj) {
+		obj = getSorted(obj);
+		int len = obj.length;
+		return obj[(int)len/4];
+	}
+
+	public Number quartile3(Number[] obj) {
+		obj = getSorted(obj);
+		int len = obj.length;
+		return obj[(int)len/4 + len/2];
+	}
+
+	public Number median(Number[] obj) {
+		obj = getSorted(obj);
+		int len = obj.length;
+		return obj[(int)len/2];
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SplitFileRegExp.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SplitFileRegExp.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SplitFileRegExp.groovy	(revision 321)
@@ -0,0 +1,58 @@
+package org.txm.sw
+
+// PARAMETERS //
+
+infile = new File("/home/mdecorde/xml/test/grep")			// the file to process
+outdir = new File("/home/mdecorde/xml/test/result")			// the directory which received result files
+regex = /(<\?(.+))/           						// matching line test
+prefix = "prefix"							// result files prefix
+replace = '$2'								// use the matching line to build the filename
+ext = ".xml"                                     			// result files extension
+includeSeparatorLine = true						// should include matching line in result files
+writeAtStartup = false							// should create files only if a matching line is found
+encoding = "UTF-8"
+// CALL //
+splitByRegExp(infile, outdir, 
+//	regex, prefix, replace, ext,
+	regex, prefix, null, ext, 
+	includeSeparatorLine, writeAtStartup)
+
+
+// PUBLIC METHODS //
+def splitByRegExp(def infile, def outdir, def regex, def prefix, def replace, def ext, def includeSeparatorLine, def writeAtStartup) {
+	int count = 1;
+	outdir.deleteDir()
+	outdir.mkdirs()
+	if (!outdir.exists()) return false
+	if (!infile.isFile()) return false
+
+	File outfile = null;
+	def writer = null
+
+	if (writeAtStartup) {
+		if (replace == null) {
+			outfile = new File(outdir, prefix+(count++)+ext)
+		}
+		writer = outfile.newWriter(encoding)
+	}
+	infile.eachLine(encoding) { line ->
+		if (line ==~ regex) { // new file
+			if (replace == null) {
+				outfile = new File(outdir, prefix+(count++)+ext)
+			} else {
+				String name = prefix+line.replaceAll(regex, replace)+ext
+				outfile = new File(outdir, name)
+			}
+			if (writer != null)	writer.close() // close previous writer
+			writer = outfile.newWriter(encoding)
+			println "new file $outfile"
+			if (includeSeparatorLine) {
+				writer.println line
+			}
+		} else {
+			if (writer != null)
+				writer.println line
+		}
+	}
+	if (writer != null)	writer.close()
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RemoveTag.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RemoveTag.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RemoveTag.groovy	(revision 321)
@@ -0,0 +1,101 @@
+package org.txm.sw
+
+import javax.xml.parsers.*
+import javax.xml.transform.*
+import javax.xml.xpath.*
+
+import org.txm.importer.graal.PersonalNamespaceContext
+import org.txm.utils.xml.DomUtils
+import org.w3c.dom.Document
+import org.w3c.dom.Element
+
+class RemoveTag {
+
+	public static boolean xpath(File xmlfile, String xpath)
+	{
+		if (!xmlfile.exists()) {
+			println "Error: $xmlfile does not exists"
+		}
+
+		if (!(xmlfile.canRead() && xmlfile.canWrite())) {
+			println "Error: $xmlfile is not readable or writable"
+		}
+
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		Document doc = builder.parse(xmlfile);
+
+		XPathFactory xpathfactory = XPathFactory.newInstance();
+		def _xpath = xpathfactory.newXPath();
+		_xpath.setNamespaceContext(new PersonalNamespaceContext());
+		def expr = _xpath.compile(xpath);
+		def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+
+		for(Element node : nodes) {
+			def parent = node.getParentNode();
+			if (parent != null)
+				parent.removeChild(node)
+		}
+		return DomUtils.save(doc, xmlfile)
+	}
+
+	public static boolean xpath(File xmlfile, def xpaths)
+	{
+		if (!xmlfile.exists()) {
+			println "Error: $xmlfile does not exists"
+		}
+
+		if (!(xmlfile.canRead() && xmlfile.canWrite())) {
+			println "Error: $xmlfile is not readable or writable"
+		}
+
+		DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+		domFactory.setNamespaceAware(true); // never forget this!
+		DocumentBuilder builder = domFactory.newDocumentBuilder();
+		Document doc = builder.parse(xmlfile);
+		XPathFactory xpathfactory = XPathFactory.newInstance();
+
+		for (String xpath : xpaths) {
+			def _xpath = xpathfactory.newXPath();
+			_xpath.setNamespaceContext(new PersonalNamespaceContext());
+			def expr = _xpath.compile(xpath);
+			def nodes = expr.evaluate(doc, XPathConstants.NODESET);
+
+			for (Element node : nodes) {
+				def parent = node.getParentNode();
+				if (parent != null)
+					parent.removeChild(node)
+			}
+		}
+		return DomUtils.save(doc, xmlfile)
+
+	}
+	public static void main(String[] args) {
+		File dir = new File("/home/mdecorde/Bureau/matrice/témoignages")
+		File infile = new File(dir, "tei2.xml")
+		File outfile = new File(dir, "tei2-out.xml")
+
+		def doc = DomUtils.load(infile);
+		def addList = doc.getDocumentElement().getElementsByTagName("add")
+		println "nb <add> to delete: "+addList.getLength()
+		for(int i = 0 ; i < addList.getLength() ; i++) {
+			Element node = addList.item(i);
+			String text = node.getTextContent().trim();
+			Element parent = node.getParentNode();
+			parent.insertBefore(doc.createTextNode(text), node)
+			parent.removeChild(node);
+			i--
+		}
+
+		def delList = doc.getDocumentElement().getElementsByTagName("del")
+		println "nb <del> to delete: "+delList.getLength()
+		for(int i = 0 ; i < delList.getLength() ; i++) {
+			Element node = delList.item(i);
+			node.getParentNode().removeChild(node);
+			i--
+		}
+
+		DomUtils.save(doc, outfile);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByXpathElementContentTrace.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByXpathElementContentTrace.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByXpathElementContentTrace.groovy	(revision 321)
@@ -0,0 +1,38 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 10:05:14 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2377 $
+// $LastChangedBy: sheiden $
+//
+
+import org.txm.importer.*
+
+// parameters
+													// **change this parameter**
+dir = new File(System.getProperty("user.home"), "TXM/sources/Factiva")
+													// **change this parameter**
+xpath = "/article/@date"
+
+print "Renaming"
+// main body
+dir.eachFileMatch(~/.*.xml/){ file ->               // for each file
+	file.renameTo(new File(file.getParent(), XPathResult.getXpathResponse(file, xpath)+"_"+file.name));
+	print "."
+}
+println "Done."
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Contains Groovy scripts that will be included in the Source Workbench</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByXpathElementContent.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByXpathElementContent.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByXpathElementContent.groovy	(revision 321)
@@ -0,0 +1,36 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 10:05:14 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2377 $
+// $LastChangedBy: sheiden $
+//
+import org.txm.importer.*
+
+// parameters
+
+// **change this parameter**
+dir = new File(System.getProperty("user.home"), "TXM/sources/Factiva")
+
+// **change this parameter**
+xpath = "/article/@date"
+
+// main body
+dir.eachFileMatch(~/.*.xml/){ file -> // for each file
+	file.renameTo(new File(file.getParent(), XPathResult.getXpathResponse(file, xpath)+"_"+file.name));
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/ChangeEncoding.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/ChangeEncoding.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/ChangeEncoding.groovy	(revision 321)
@@ -0,0 +1,47 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2012-02-01 17:58:45 +0100 (mer., 01 févr. 2012) $
+// $LastChangedRevision: 2122 $
+// $LastChangedBy: mdecorde $
+//
+
+// PARAMETERS
+
+File dir = new File("/home/mdecorde/TEMP/atelier") // the directory containing the files
+String fromEncoding = "UTF-8" // the sources encoding
+String toEncoding = "windows-1252" // the final encoding
+
+// END OF PARAMETERS
+
+def processDirectory(File dir, String fromEncoding, String toEncoding) {
+	dir.eachFileMatch(~/.*.txt/){ file -> // for each file with the "txt" extension
+		processFile(file, fromEncoding, toEncoding)
+	}
+}
+
+def processFile(File file, String fromEncoding, String toEncoding) {
+	println "Convert file: $file"
+	String text = file.getText(fromEncoding);
+	file.withWriter(toEncoding) {writer -> // read with the "fromEncoding"
+		writer.write(text) // write with the "toEncoding"
+	}
+}
+
+processDirectory(dir, fromEncoding, toEncoding)
+println "Done"
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByRegExp.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByRegExp.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/RenameFilesInDirectoryByRegExp.groovy	(revision 321)
@@ -0,0 +1,39 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2012-02-01 17:58:45 +0100 (mer., 01 févr. 2012) $
+// $LastChangedRevision: 2122 $
+// $LastChangedBy: mdecorde $
+//
+import org.txm.importer.*
+
+// PARAMETERS
+
+File dir = new File("/home/mdecorde/TEMP/atelier")
+String searchRegexp = "(.+).txt"
+String replaceRegexp = 't$1.txt' // WARNING: Use simple quotes only
+
+// END OF PARAMETERS
+
+// main body
+dir.eachFileMatch(~/.*.txt/){ file -> // for each file with extension "txt"
+	String filename = file.getName()
+	filename = filename.replaceAll(searchRegexp, replaceRegexp)
+	println "Rename file '$file' to '$filename'"
+	file.renameTo(new File(file.getParent(), filename))
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceInDirectoryTrace.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceInDirectoryTrace.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceInDirectoryTrace.groovy	(revision 321)
@@ -0,0 +1,56 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate$
+// $LastChangedRevision$
+// $LastChangedBy$
+//
+
+// parameters
+
+dir = new File("/home/sheiden/TXM/sources/Factiva") // **change this parameter**
+ext = "\\.xml"                                      // **change this parameter**
+find = /date="([0-9]+)-([0-9]+-[0-9]+)"/            // **change this parameter**
+replaceWith = 'date="$1-$2" year="$1"'              // **change this parameter**
+encoding = "cp1252"
+test = false // XML Validation
+
+// main body
+println "SearchReplaceInDirectory: processing directory '$dir'..."
+dir.eachFileMatch(~/.*$ext/) { file ->               // for each file matching extension
+    println "SearchReplaceInDirectory: processing file '$file'..."
+    println "XML syntax initial check"
+    if (org.txm.importer.ValidateXml.test(file)) {
+        println "Search&Replace"
+        def tmp = File.createTempFile("SearchReplaceInDirectoryTemp", ".xml", dir) // create temporary file
+        tmp.withWriter(encoding) { writer ->
+            file.eachLine(encoding) { line ->                  // for each line
+                writer.println line.replaceAll(find, replaceWith) // find&replace and print
+            }
+        }
+        file.delete()
+        tmp.renameTo(file)                           // save results
+        
+        println "XML syntax final check"
+        if (test && !org.txm.importer.ValidateXml.test(file)) {
+            println "** Warning: bad XML syntax for result file '$file'"
+        }
+    } else {
+        println "Skipping file '$file'"
+    }
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceInDirectory.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceInDirectory.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceInDirectory.groovy	(revision 321)
@@ -0,0 +1,73 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author sheiden
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 11:28:42 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2378 $
+// $LastChangedBy: sheiden $
+//
+
+// parameters
+
+// **change this parameter**
+dir = new File(System.getProperty("user.home"), "Bureau/voeux")
+
+// **change this parameter**
+ext = "\\.txt"
+
+// **change this parameter**
+find = "’"
+
+// **change this parameter**
+replaceWith = "'"
+
+// **change this parameter**
+encoding = "utf-8"
+
+// **change this parameter**
+// 'true' = only display matching lines, 'false' = replace in matching files
+showMatchingFilesOnly = false
+
+
+//find = /date="([0-9]+)-([0-9]+-[0-9]+)"/
+		      // **change this parameter**
+//replaceWith = 'date="$1-$2" year="$1"'
+		      // **change this parameter** (warning: '$1', '$2'... can be interpreted by Groovy in "..." strings)
+
+// main body
+println "in $dir ... "+/.*$ext/
+dir.eachFileMatch(~/.*$ext/) { file ->               // for each file matching extension
+	println file.getName()
+	
+if (showMatchingFilesOnly) {
+	
+       file.eachLine(encoding) { line ->                  // for each line
+           if (line.matches(".*$find.*")) { println " *match* : "+line }
+	   }
+} else {
+       def tmp = File.createTempFile("SearchReplaceInDirectoryTemp", ".tmp", dir) // create temporary file
+       tmp.write('')                                // create empty file
+       tmp.withWriter(encoding) { writer ->
+           file.eachLine(encoding) { line ->                  // for each line
+           	writer.println line.replaceAll(find, replaceWith) // find&replace and print
+           }
+       }
+       file.delete()
+       tmp.renameTo(file)                           // save results
+ }
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceXMLInDirectoryTrace.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceXMLInDirectoryTrace.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceXMLInDirectoryTrace.groovy	(revision 321)
@@ -0,0 +1,53 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2013-05-02 10:05:14 +0200 (jeu., 02 mai 2013) $
+// $LastChangedRevision: 2377 $
+// $LastChangedBy: sheiden $
+//
+
+// parameters
+
+dir = new File("/home/sheiden/TXM/sources/Factiva") // **change this parameter**
+ext = "\\.xml"                                      // **change this parameter**
+find = /date="([0-9]+)-([0-9]+-[0-9]+)"/            // **change this parameter**
+replaceWith = 'date="$1-$2" year="$1"'              // **change this parameter**
+
+// main body
+println "SearchReplaceInDirectory: processing directory '$dir'..."
+dir.eachFileMatch(~/.*$ext/){ file ->               // for each file matching extension
+    println "SearchReplaceInDirectory: processing file '$file'..."
+    println "XML syntax initial check"
+    if (org.txm.importer.ValidateXml.test(file)) {
+        println "Search&Replace"
+        def tmp = File.createTempFile("SearchReplaceInDirectoryTemp", ".xml", dir) // create temporary file
+        tmp.write('')                                // create empty file
+        tmp.withWriter { writer ->
+            file.eachLine { line ->                  // for each line
+                writer.println line.replaceAll(find, replaceWith) // find&replace and print
+            }
+        }
+        tmp.renameTo(file)                           // save results
+        println "XML syntax final check"
+        if (!org.txm.importer.ValidateXml.test(file)) {
+            println "** Warning: bad XML syntax for result file '$file'"
+        }
+    } else {
+        println "Skipping file '$file'"
+    }
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/AddHeadTailInDirectory.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/AddHeadTailInDirectory.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/AddHeadTailInDirectory.groovy	(revision 321)
@@ -0,0 +1,68 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author sheiden
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2012-02-01 17:58:45 +0100 (mer., 01 févr. 2012) $
+// $LastChangedRevision: 2122 $
+// $LastChangedBy: mdecorde $
+//
+
+import static groovy.io.FileType.*
+
+// PARAMETERS
+
+// dir
+def dir = new File("/home/mdecorde/TEMP/atelier")
+
+// inputEncoding
+def inputEncoding = "UTF-8"
+
+// outputEncoding
+def outputEncoding = "UTF-8"
+
+// head
+def head = "<?xml version=\"1.0\" encoding=\"${outputEncoding}\"?>\n<discours>\n" // can be empty
+
+// tail
+def tail = "</discours>\n"; // can be empty
+
+// fileExtension
+def fileExtension = "txt"
+
+//END OF PARAMETERS
+
+// filePattern
+def filePattern = ".*\\.$fileExtension\$"
+
+// main body
+dir.eachFileMatch(~/$filePattern/) { inputFile ->				// for each file matching extension
+    def outputTmp = File.createTempFile("AddHeadInDirectoryTemp", '.tmp', dir) // create temporary file
+    outputTmp.withWriter(outputEncoding) { writer ->
+    if (head != null && head.length() > 0)
+    	writer.print(head)
+	inputFile.withReader(inputEncoding) { reader ->
+			reader.eachLine { inputLine ->               		// for each line
+				writer.println inputLine
+			}
+		}
+	if (tail != null && tail.length() > 0)
+		writer.print(tail)
+	writer.close()
+    }
+	assert outputTmp.renameTo(inputFile)						// save results
+}		
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceFromList.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceFromList.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/SearchReplaceFromList.groovy	(revision 321)
@@ -0,0 +1,57 @@
+package org.txm.sw
+// Copyright © - ENS de Lyon - http://textometrie.ens-lyon.fr
+// @author mdecorde
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+// $LastChangedDate: 2012-10-01 08:24:58 +0200 (lun., 01 oct. 2012) $
+// $LastChangedRevision: 2269 $
+// $LastChangedBy: sheiden $
+//
+
+// parameters
+
+file = new File("/home/mdecorde/xml/apollinaire/apollinaire.xml")
+out = new File("/home/mdecorde/xml/apollinaire/apo/apollinaire.xml")
+def changesFind = [ "</recueil>",	"<paragraphe>", "</paragraphe>", "<racine>",	"</racine>", "<interp",	"</interp>", "<TEI xmlns=\"http://www.tei-c.org/ns/1.0\" xmlns:txm=\"http://textometrie.org/1.0\">"	]
+def changesReplace = [ "</recueil><pb/>", "<p>", "</p><lb/>", '<text id ="apollinaire">', '</text',	'<txm:ana',	'</txm:ana>', """<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:txm="http://textometrie.org/1.0">
+<teiHeader type="text">
+<fileDesc>
+<titleStmt>
+<title>null</title>
+<respStmt>
+<resp id="cordial">initial tagging</resp></respStmt>
+</titleStmt>
+</fileDesc>
+<encodingDesc>
+<classDecl>
+<taxonomy id="pos"><bibl type="tagset"/></taxonomy>
+<taxonomy id="func"><bibl type="tagset"/></taxonomy>
+<taxonomy id="lemma"><bibl type="tagset"/></taxonomy>
+</classDecl>
+</encodingDesc>
+</teiHeader>""" ]
+
+// main body
+out.write('')                                // create empty file
+out.withWriter { writer ->
+	String text = file.getText("UTF-8");
+	for (int i = 0 ; i < changesFind.size() ; i++) {
+		println "loop $i: "+changesFind[i] +" >> "+ changesReplace[i]
+		text = text.replaceAll(changesFind[i], changesReplace[i])
+	}
+	out.write(text)
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/sw/ReplaceXmlDomNode.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/sw/ReplaceXmlDomNode.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/sw/ReplaceXmlDomNode.groovy	(revision 321)
@@ -0,0 +1,161 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.sw;
+
+import java.io.File;
+
+import javax.xml.namespace.NamespaceContext;
+import org.w3c.dom.*;
+import javax.xml.parsers.*;
+import org.xml.sax.SAXException;
+import javax.xml.stream.*;
+import javax.xml.xpath.*;
+
+import org.txm.sw.ReplaceXmlDomNode;
+import org.txm.importer.graal.PersonalNamespaceContext;
+import javax.xml.namespace.NamespaceContext;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ReplaceXmlDomNode.
+ */
+class ReplaceXmlDomNode
+{
+	
+	/** The ns context. */
+	NamespaceContext nsContext = new PersonalNamespaceContext();
+	
+	/**
+	 * Instantiates a new replace xml dom node.
+	 *
+	 * @param src the src
+	 * @param target the target
+	 * @param xpathsrc the xpathsrc
+	 * @param xpathtarget the xpathtarget
+	 */
+	public ReplaceXmlDomNode(File src, File target, String xpathsrc, String xpathtarget)
+	{
+		def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+		
+		def docsrc = builder.parse(src);
+		def doctarget = builder.parse(target);
+		
+		Node srcnode = null;
+		Node targetnode = null;
+		
+		XPathFactory factory = XPathFactory.newInstance();
+		XPath XpathObj = factory.newXPath();
+		XpathObj.setNamespaceContext(nsContext);
+		
+		XPathExpression expr = XpathObj.compile(xpathsrc);
+		for(Node node : (NodeList) expr.evaluate(docsrc, XPathConstants.NODESET))
+		{
+			srcnode = node;
+			break;
+		}
+		
+		XPathExpression expr2 = XpathObj.compile(xpathtarget);
+		for(Node node : (NodeList) expr2.evaluate(doctarget, XPathConstants.NODESET))
+		{
+			targetnode = node;
+			break;
+		}
+		
+		//println "replace : "+targetnode
+		//println "by : "+srcnode
+		if(targetnode != null && srcnode != null)
+		{
+			Node parent = targetnode.getParentNode();
+			parent.removeChild(targetnode);
+			parent.appendChild(doctarget.importNode(srcnode, true));
+			
+			save(doctarget, target);
+		}
+		else
+		{
+			println "Error: "
+			println "target node = "+targetnode
+			println "src node = "+srcnode
+		}
+		
+	}
+	
+	/**
+	 * Save.
+	 *
+	 * @param doc the doc
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean save(def doc, File outfile)
+	{
+		try {
+			// Création de la source DOM
+				Source source = new DOMSource(doc);
+			
+			// Création du fichier de sortie
+				Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8")); 
+			Result resultat = new StreamResult(writer);
+			
+			// Configuration du transformer
+			TransformerFactory fabrique = new net.sf.saxon.TransformerFactoryImpl();
+			Transformer transformer = fabrique.newTransformer();
+			transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
+			
+			// Transformation
+			transformer.transform(source, resultat);
+			writer.close();
+			doc = null;
+			return true;
+		} catch (Exception e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		String home = System.getProperty("user.home")
+		File src = new File(home, "xml/subberoul/subcorpus0000.xml")
+		File target = new File(home, "xml/subberoul/master.xml")
+		String xpathsrc = "//subcorpus"
+		String xpathtarget = "//subcorpus[@name=\"subcorpus0000\"]"
+		
+		new ReplaceXmlDomNode(src, target, xpathsrc, xpathtarget)
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/TTest.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/TTest.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/TTest.groovy	(revision 321)
@@ -0,0 +1,39 @@
+package org.txm.tokenizer
+
+public class TTest {
+	def regex;
+	def type = "w";
+	def before=1, hit=2, after=3;
+	public TTest(def regex, String type, int before, int hit, int after) {
+		this.regex = regex
+		this.type = type
+		this.before = before
+		this.hit = hit
+		this.after = after
+	}
+
+	public TTest(def regex, int before, int hit, int after) {
+		this.regex = regex
+		this.before = before
+		this.hit = hit
+		this.after = after
+	}
+
+	public TTest(def regex, String type) {
+		this.regex = regex
+		this.type= type
+	}
+
+	public TTest(def regex) {
+		this.regex = regex
+	}
+
+	/* (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	@Override
+	public String toString() {
+		return "[regex=" + regex + ", type=" + type + ", before="+ before + ", hit=" + hit + ", after=" + after + "]";
+	}
+}
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/TokenizerClasses.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/TokenizerClasses.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/TokenizerClasses.groovy	(revision 321)
@@ -0,0 +1,620 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2016-10-03 15:30:36 +0200 (Mon, 03 Oct 2016) $
+// $LastChangedRevision: 3313 $
+// $LastChangedBy: mdecorde $
+//
+package org.txm.tokenizer
+
+import java.util.regex.Pattern
+
+import javax.xml.stream.*
+
+import org.txm.utils.xml.DomUtils
+import org.w3c.dom.Document
+import org.w3c.dom.Element
+import org.w3c.dom.Node
+import org.w3c.dom.NodeList
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class TokenizerClasses.
+ */
+public class TokenizerClasses {
+
+	//// ROOTS ? ////
+	public static boolean debug = false;
+	/** The tag_all. */
+	public static String tag_all = "<[A-Za-z][^>]+>";
+	
+	/** The Weblex enclitics. */
+	public static String enclitics = "je|m[eo]i|tu|t[eo]i|il|lui|luy|ilz|ils|no?u?s|vo?u?s|on|ce|ci|là|elles?"; // FRO
+	public static String encliticsFR = "je|tu|il|elle|on|nous|vous|ils|elles|toi|moi|en|y|t|leur|lui|le|la|les"; // FR
+
+	/** The TT enclitics. */
+	public static String FClitic_en = "'(s|re|ve|d|m|em|ll)|n['‘’]t";
+	public static String PClitic_fr = '[dcjlmnstDCJLNMST][\'‘’]|[Qq]u[\'‘’]|[Jj]usqu[\'‘’]|[Ll]orsqu[\'‘’]';
+	public static String FClitic_fr = '-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mmes?|-m[\'‘’]|-moi|-nous|-on|-toi|-tu|-t[\'‘’]|-vous|-en|-y|-ci|-là';
+	public static String PClitic_it = '[dD][ae]ll[\'‘’]|[nN]ell[\'‘’]|[Aa]ll[\'‘’]|[lLDd][\'‘’]|[Ss]ull[\'‘’]|[Qq]uest[\'‘’]|[Uu]n[\'‘’]|[Ss]enz[\'‘’]|[Tt]utt[\'‘’]';
+	public static String FClitic_gl = '-la|-las|-lo|-los|-nos';
+	
+	/** The div_tags. */
+	public static String div_tags = "TEI|text|front|body|div|div1|div2|div3|div4|div5|div6|back|head|trailer|p|ab|sp|speaker|list|notice|bibl|opener|dateline";
+
+	/** The q_tags. */
+	public static String q_tags = "q|quote|said|item|stage|cit|label|heraldry";
+
+	/** The extraword_tags. */
+	public static String extraword1_tags = "expan|pb|lb|milestone|gap|note|s|locus|title|ref|hi|witDetail";
+
+	/** The corr_tags_no_seg. */
+	public static String corr_tags_no_seg = "expan|unclear|choice|corr|sic|reg|orig|foreign|hi|title|name|supplied|subst|add|del|damage|date|idno|surplus";
+
+	/** The word_tags. */
+	public static String word_tags = "w|abbr|num";
+
+	/** The intraword_tags. */
+	public static String intraword_tags = "c|ex|caesura";
+
+	/** The punct_quotes. */
+	public static String punct_quotes = "'‘’’"
+
+	/** The punct_strong1. */
+	public static String punct_strong1 = ".!?";
+
+	/** The punct_strong2. */
+	public static String punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
+
+	/** The punct_paren_open1. */
+	public static String punct_paren_open1 = "«";
+
+	/** The punct_paren_open2. */
+	public static String punct_paren_open2 = "``|\\(|\\[|\\{";
+
+	/** The punct_paren_close1. */
+	public static String punct_paren_close1 = "»";
+
+	/** The punct_paren_close2. */
+	public static String punct_paren_close2 = "''|‘‘|’’|\\)|\\]|\\}";
+
+	/** The punct_weak. */
+	public static String punct_weak = "\\-,;∼ˆ·*:\"“”\\+±=/\\≤≥<>\\—ν√μ•@≈→αγ∞≡σ&%|#¼½Θĩ†φθΓ§ẽə∈";
+
+	public static String entity = "&[^;]+;";
+
+	public static String seg_tags = "seg"
+
+	//// COMBINAISONS ////
+
+	/** The corr_tags. */
+	public static String corr_tags = "$corr_tags_no_seg|$seg_tags";
+	/** The extraword_tags. */
+	public static String extraword_tags = "$div_tags|$q_tags|$extraword1_tags";
+
+	/** The punct_strong. */
+	public static String punct_strong = "[$punct_strong1]+|$punct_strong2";
+
+	/** The punct_paren_open. */
+	public static String punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
+
+	/** The punct_paren_close. */
+	public static String punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
+
+	/** The punct_paren. */
+	public static String punct_paren = "$punct_paren_open|$punct_paren_close";
+
+	/** The punct_all. */
+	public static String punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
+
+	/** The word_chars. */
+	public static String word_chars = "[^ $punct_quotes$punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|$entity";
+
+	public static def tests = [
+		new TTest(/\A(.*-)(je|m[eo]i|tu|t[eo]i|lui|luy|ilz|ils?|no?u?s|vo?u?s|on|leurs|ce|ci|là|elles?)(.*)\Z/),
+		new TTest(/\A(.*)(-)()\Z/, "pon"),
+		//new TTest(/\A(.*)(Bande dessinée|eau de vie|machine à écrire|Moyen Âge|petit pois|poule d'eau|Avoir l'air|se rendre compte|faire semblant|prendre froid|s'en aller|Comme il faut|bon marché|bon enfant|en retard|en colère|à la mode|de bonne humeur|hors de propos|de travers|en vacances|de standing|à l'abandon|sans défense|pieds nus|Tout à fait|d'ores et déjà|de temps en temps|tout de suite|de gré ou de force|ne pas|ne jamais|ne plus|sans doute|on ne peut plus|Quelque chose|ce dernier|n'importe quoi|N'importe quel|beaucoup de|plus d'un|peu de|un tas de|un groupe de|je ne sais quel|De sorte que|bien que|parce que|c'est pourquoi|de même que|depuis que|Quant à|au-dessus de|par rapport à|à côté de|grâce à|hors de|le long de|Par ma barbe !|Mon dieu !|Eh bien !|Bonne nuit !|Pas de panique !)(.*)\Z/),
+		new TTest(/\A([Aa]ujourd'hui)(.*)/,0,1,2),
+		
+		// BFM ONLY
+//		new TTest(/\A()([.·][^ .·]+[.·])(.*)\Z/, "num"), 
+
+		// TXM REFMAN ONLY
+//		new TTest(/\A(.*)($tag_all)(.*)\Z/, "tag"),
+//		new TTest(/\A(.*)([0-9][0-9][\/][0-9][0-9][\/][0-9][0-9])(.*)\Z/, "date", 1,2,3),
+//		new TTest(/\A(.*)([0-9][0-9]:[0-9][0-9]:[0-9][0-9])(.*)\Z/, "time", 1,2,3),
+//		new TTest(/\A([^0-9]*)([0-9]+(\.[0-9]+[a-z]+))(.*)\Z/, "version", 1,2,4),
+//		new TTest(/\A([^0-9]*)([0-9]+(\.[0-9]+)(\.[0-9]+)+)(\.[0-9]+[a-z]+)?(.*)\Z/, "version", 1,2,6),
+//		new TTest(/\A()([a-z]++:\/\/)(.*)\Z/, "protocol", 1,2,3),
+//		new TTest(/\A()([0-9]+([.,][0-9]+)?)()\Z/, "num", 1,2,4),
+//		new TTest(/\A()(([A-Z]:)?([\/\\][^\/\\]+)+)()\Z/, "path", 1,2,5),
+		
+		new TTest(/\A([\p{Ps}|\p{Pe}|\p{Pi}|\p{Pf}|\p{Po}|\p{S}]*)(\p{L}++['‘’])(.*)\Z/, "w", 1, 2, 3),
+		new TTest(/\A(.*)(\.\.\.)(.*)\Z/, "pon"),
+		new TTest(/\A(.*)(\p{Ps}|\p{Pe}|\p{Pi}|\p{Pf}|\p{Po}|\p{S})(.*)\Z/, "pon")
+		
+	];
+	//tag_all = "<[^>]+>";
+	//enclitics = "je|m[eo]i|tu|t[eo]i|il|lui|luy|ilz|ils|no?u?s|vo?u?s|on|ce|ci|là|elles?"; // FRO
+	//encliticsFR = "je|tu|il|elle|on|nous|vous|ils|elles|toi|moi|en|y|t|leur|lui|le|la|les"; // FR
+	//div_tags = "TEI|text|front|body|div|div1|div2|div3|div4|div5|div6|back|head|trailer|p|ab|sp|speaker|list|notice|bibl";
+	//q_tags = "q|quote|item|stage";
+	//extraword_tags = "$div_tags|$q_tags|expan|pb|lb|milestone|gap|note|s|locus|title|ref|hi|witDetail";
+	//corr_tags_no_seg = "expan|unclear|choice|corr|sic|reg|orig|foreign|hi|title|name|supplied|subst|add|del|damage|date|idno";
+	//corr_tags = "$corr_tags_no_seg|seg";
+	//word_tags = "w|abbr|num";
+	//intraword_tags = "c|ex";
+
+	public static String whitespaces = "[\\p{Z}\\p{C}]+";
+	public static String regElision = "['‘’]" // obsolete
+	public static String regPunct = "[\\p{Ps}\\p{Pe}\\p{Pi}\\p{Pf}\\p{Po}\\p{S}]"
+
+	//punct_strong1 = ".!?";
+	//punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
+	//punct_strong = "[$punct_strong1]|$punct_strong2";
+	//punct_paren_open1 = "«";
+	//punct_paren_open2 = "<<|``|\\(|\\[|\\{";
+	//punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
+	//punct_paren_close1 = "»";
+	//punct_paren_close2 = ">>|''|‘‘|’’|\\)|\\]|\\}";
+	//punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
+	//punct_paren = "$punct_paren_open|$punct_paren_close";
+	//punct_weak = ",;∼ˆ·*:\"`'“”\\+±=/\\-≤≥<>\\—_ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
+	//punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
+	//word_chars = "[^ $punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|&[^;]+;";
+
+	public static reset() {
+		tag_all = "<[^>]+>";
+		enclitics = "je|m[eo]i|tu|t[eo]i|il|lui|luy|ilz|ils|no?u?s|vo?u?s|on|ce|ci|là|elles?"; // FRO
+		encliticsFR = "je|tu|il|elle|on|nous|vous|ils|elles|toi|moi|en|y|t|leur|lui|le|la|les"; // FR
+		div_tags = "TEI|text|front|body|div|div1|div2|div3|div4|div5|div6|back|head|trailer|p|ab|sp|speaker|list|notice|bibl";
+		q_tags = "q|quote|item|stage|cit";
+		extraword1_tags = "expan|pb|lb|milestone|gap|note|s|locus|title|ref|hi|witDetail";
+		corr_tags_no_seg = "expan|unclear|choice|corr|sic|reg|orig|foreign|hi|title|name|supplied|subst|add|del|damage|date|idno|surplus";
+		word_tags = "w|abbr|num";
+		intraword_tags = "c|ex|caesura";
+		punct_quotes = "'‘’’"
+		punct_strong1 = ".!?";
+		punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
+		punct_paren_open1 = "«";
+		punct_paren_open2 = "``|\\(|\\[|\\{";
+		punct_paren_close1 = "»";
+		punct_paren_close2 = "''|‘‘|’’|\\)|\\]|\\}";
+		punct_weak = "\\-,;∼ˆ·*:\"“”\\+±=/\\-≤≥<>\\—ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
+		entity = "&[^;]+;";
+		seg_tags = "seg"
+		
+		whitespaces = "[\\p{Z}\\p{C}]+";
+		regElision = "['‘’]"
+		regPunct = "[\\p{Ps}\\p{Pe}\\p{Pi}\\p{Pf}\\p{Po}\\p{S}]"
+
+		recombine();
+	}
+
+	public static recombine() {
+		corr_tags = "$corr_tags_no_seg|$seg_tags";
+		extraword_tags = "$div_tags|$q_tags|$extraword1_tags";
+		punct_strong = "[$punct_strong1]|$punct_strong2";
+		punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
+		punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
+		punct_paren = "$punct_paren_open|$punct_paren_close";
+		punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
+		word_chars = "[^ $punct_quotes$punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|$entity";
+	}
+
+	/**
+	 * To dom.
+	 *
+	 * @param doc the doc
+	 * @param parent the parent
+	 * @return the java.lang. object
+	 */
+	public static toDom(Document doc, Node parent) {
+		Element tokenizer = doc.createElement("tokenizer");
+		tokenizer.setAttribute("onlyThoseTests", "false")
+		parent.appendChild(tokenizer)
+		
+		//String tag_all = "<[^>]+>";
+		Element p = doc.createElement("param");
+		p.setAttribute("key","tag_all");
+		p.setTextContent(tag_all);
+		tokenizer.appendChild(p)
+		//String enclitics = "je|m[eo]i|tu|t[eo]i|il|lui|luy|ilz|ils|no?u?s|vo?u?s|on|ce|ci|là|elles?"; // FRO
+		p = doc.createElement("param");
+		p.setAttribute("key","enclitics");
+		p.setTextContent(enclitics);
+		tokenizer.appendChild(p)
+		//String encliticsFR = "je|tu|il|elle|on|nous|vous|ils|elles|toi|moi|en|y|t|leur|lui|le|la|les"; // FR
+		p = doc.createElement("param");
+		p.setAttribute("key","encliticsFR");
+		p.setTextContent(encliticsFR);
+		tokenizer.appendChild(p)
+		//String div_tags = "TEI|text|front|body|div|div1|div2|div3|div4|div5|div6|back|head|trailer|p|ab|sp|speaker|list|notice|bibl";
+		p = doc.createElement("param");
+		p.setAttribute("key","div_tags");
+		p.setTextContent(div_tags);
+		tokenizer.appendChild(p)
+		//String q_tags = "q|quote|item|stage";
+		p = doc.createElement("param");
+		p.setAttribute("key","q_tags");
+		p.setTextContent(q_tags);
+		tokenizer.appendChild(p)
+		//String extraword_tags = "$div_tags|$q_tags|expan|pb|lb|milestone|gap|note|s|locus|title|ref|hi|witDetail";
+		p = doc.createElement("param");
+		p.setAttribute("key","extraword_tags");
+		p.setTextContent(extraword_tags);
+		tokenizer.appendChild(p)
+		//String corr_tags_no_seg = "expan|unclear|choice|corr|sic|reg|orig|foreign|hi|title|name|supplied|subst|add|del|damage|date|idno";
+		p = doc.createElement("param");
+		p.setAttribute("key","enclitics");
+		p.setTextContent(enclitics);
+		tokenizer.appendChild(p)
+		//String corr_tags = "$corr_tags_no_seg|seg";
+		p = doc.createElement("param");
+		p.setAttribute("key","corr_tags");
+		p.setTextContent(corr_tags);
+		tokenizer.appendChild(p)
+		//String word_tags = "w|abbr|num";
+		p = doc.createElement("param");
+		p.setAttribute("key","word_tags");
+		p.setTextContent(word_tags);
+		tokenizer.appendChild(p)
+		//String intraword_tags = "c|ex";
+		p = doc.createElement("param");
+		p.setAttribute("key","intraword_tags");
+		p.setTextContent(intraword_tags);
+		tokenizer.appendChild(p)
+
+		//String punct_strong1 = ".!?";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_strong1");
+		p.setTextContent(punct_strong1);
+		tokenizer.appendChild(p)
+		//String punct_strong2 = "\\.\\.|\\.\\.\\.|…|\\|";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_strong2");
+		p.setTextContent(punct_strong2);
+		tokenizer.appendChild(p)
+		//String punct_strong = "[$punct_strong1]|$punct_strong2";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_strong");
+		p.setTextContent(punct_strong);
+		tokenizer.appendChild(p)
+		//String punct_paren_open1 = "«";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren_open1");
+		p.setTextContent(punct_paren_open1);
+		tokenizer.appendChild(p)
+		//String punct_paren_open2 = "<<|``|\\(|\\[|\\{";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren_open2");
+		p.setTextContent(punct_paren_open2);
+		tokenizer.appendChild(p)
+		//String punct_paren_open = "$punct_paren_open1|$punct_paren_open2";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren_open");
+		p.setTextContent(punct_paren_open);
+		tokenizer.appendChild(p)
+		//String punct_paren_close1 = "»";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren_close1");
+		p.setTextContent(punct_paren_close1);
+		tokenizer.appendChild(p)
+		//String punct_paren_close2 = ">>|''|‘‘|’’|\\)|\\]|\\}";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren_close2");
+		p.setTextContent(punct_paren_close2);
+		tokenizer.appendChild(p)
+		//String punct_paren_close = "$punct_paren_close1|$punct_paren_close2";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren_close");
+		p.setTextContent(punct_paren_close);
+		tokenizer.appendChild(p)
+		//String punct_paren = "$punct_paren_open|$punct_paren_close";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_paren");
+		p.setTextContent(punct_paren);
+		tokenizer.appendChild(p)
+		//String punct_weak = ",;∼ˆ·*:\"`'“”\\+±=/\\-≤≥<>\\—_ν√μ•@≈→αγ∞≡σ&¼½Θĩ†φθΓ§ẽə∈";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_weak");
+		p.setTextContent(punct_weak);
+		tokenizer.appendChild(p)
+		//String punct_all = "$punct_strong|$punct_paren|[$punct_weak]";
+		p = doc.createElement("param");
+		p.setAttribute("key","punct_all");
+		p.setTextContent(punct_all);
+		tokenizer.appendChild(p)
+		//String word_chars = "[^ $punct_strong1$punct_paren_open1$punct_paren_close1$punct_weak]|&[^;]+;";
+		p = doc.createElement("param");
+		p.setAttribute("key","word_chars");
+		p.setTextContent(word_chars);
+		tokenizer.appendChild(p)
+		//whitespaces = "[\\p{Z}\\p{C}]+";
+		p = doc.createElement("param");
+		p.setAttribute("key","whitespaces");
+		p.setTextContent(whitespaces);
+		tokenizer.appendChild(p)
+		//regElision = "['‘’]"
+		p = doc.createElement("param");
+		p.setAttribute("key","regElision");
+		p.setTextContent(regElision);
+		tokenizer.appendChild(p)
+		//regPunct = "[\\p{Ps}\\p{Pe}\\p{Pi}\\p{Pf}\\p{Po}\\p{S}]"
+		p = doc.createElement("param");
+		p.setAttribute("key","regPunct");
+		p.setTextContent(regPunct);
+		tokenizer.appendChild(p)
+	}
+
+	/**
+	 * Dump.
+	 */
+	public static void dump() {
+		println "BRUT"
+		println "tag_all = $tag_all"
+		println "enclitics = $enclitics"
+		println "encliticsFR = $encliticsFR"
+		println "div_tags = $div_tags"
+		println "q_tags = $q_tags"
+		println "extraword1_tags = $extraword1_tags"
+		println "corr_tags_no_seg = $corr_tags_no_seg"
+		println "word_tags = $word_tags"
+		println "intraword_tags = $intraword_tags"
+		println "punct_quotes = $punct_quotes"
+		println "punct_strong1 = $punct_strong1"
+		println "punct_strong2 = $punct_strong2"
+		println "punct_paren_open1 = $punct_paren_open1"
+		println "punct_paren_open2 = $punct_paren_open2"
+		println "punct_paren_close1 = $punct_paren_close1"
+		println "punct_paren_close2 = $punct_paren_close2"
+		println "punct_weak = $punct_weak"
+		println "entity = $entity"
+		println "seg_tags = $seg_tags"
+		println "COMBINED"
+		println "corr_tags = $corr_tags"
+		println "extraword_tags = $extraword_tags"
+		println "punct_strong = $punct_strong"
+		println "punct_paren_open = $punct_paren_open"
+		println "punct_paren_close = $punct_paren_close"
+		println "punct_paren = $punct_paren"
+		println "punct_all = $punct_all"
+		println "word_chars = $word_chars"
+		println "whitespaces = $whitespaces"
+		println "regElision = $regElision"
+		println "regPunct = $regPunct"
+		println "TESTS"
+		for(def test : tests)
+			println " $test"
+	}
+
+	public static boolean isValid() {
+		try { Pattern.compile(tag_all) }
+		catch (Exception e) { println "tag_all=${tag_all}: "+e;return false; }
+
+		try { Pattern.compile(enclitics) }
+		catch (Exception e) { println "enclitics=${enclitics}: "+e;return false; }
+
+		try { Pattern.compile(encliticsFR) }
+		catch (Exception e) { println "encliticsFR=${encliticsFR}: "+e;return false; }
+
+		try { Pattern.compile(div_tags) }
+		catch (Exception e) { println "div_tags=${div_tags}: "+e;return false; }
+
+		try { Pattern.compile(q_tags) }
+		catch (Exception e) { println "q_tags=${q_tags}: "+e;return false; }
+
+		try { Pattern.compile(extraword1_tags) }
+		catch (Exception e) { println "extraword1_tags=${extraword1_tags}: "+e;return false; }
+
+		try { Pattern.compile(corr_tags_no_seg) }
+		catch (Exception e) { println "corr_tags_no_seg=${corr_tags_no_seg}: "+e;return false; }
+
+		try { Pattern.compile(word_tags) }
+		catch (Exception e) { println "word_tags=${word_tags}: "+e;return false; }
+
+		try { Pattern.compile(intraword_tags) }
+		catch (Exception e) { println "intraword_tags=${intraword_tags}: "+e;return false; }
+
+		try { Pattern.compile(punct_quotes) }
+		catch (Exception e) { println "punct_quotes=${punct_quotes}: "+e;return false; }
+
+		try { Pattern.compile(punct_strong1) }
+		catch (Exception e) { println "punct_strong1=${punct_strong1}: "+e;return false; }
+
+		try { Pattern.compile(punct_strong2) }
+		catch (Exception e) { println "punct_strong2=${punct_strong2}: "+e;return false; }
+
+		try { Pattern.compile(punct_paren_open1) }
+		catch (Exception e) { println "punct_paren_open1=${punct_paren_open1}: "+e;return false; }
+
+		try { Pattern.compile(punct_paren_open2) }
+		catch (Exception e) { println "punct_paren_open2=${punct_paren_open2}: "+e;return false; }
+
+		try { Pattern.compile(punct_paren_close1) }
+		catch (Exception e) { println "punct_paren_close1=${punct_paren_close1}: "+e;return false; }
+
+		try { Pattern.compile(punct_paren_close2) }
+		catch (Exception e) { println "punct_paren_close2=${punct_paren_close2}: "+e;return false; }
+
+		try { Pattern.compile(punct_weak) }
+		catch (Exception e) { println "punct_weak=${punct_weak}: "+e;return false; }
+
+		try { Pattern.compile(entity) }
+		catch (Exception e) { println "entity=${entity}: "+e;return false; }
+
+		try { Pattern.compile(seg_tags) }
+		catch (Exception e) { println "seg_tags=${seg_tags}: "+e;return false; }
+
+		try { Pattern.compile(corr_tags) }
+		catch (Exception e) { println "corr_tags: "+e;return false; }
+
+		try { 	Pattern.compile(extraword_tags) }
+		catch (Exception e) { println "extraword_tags: "+e;return false; }
+
+		if (punct_strong != null)
+		try { 	Pattern.compile(punct_strong) }
+		catch (Exception e) { println "punct_strong: "+e;return false; }
+
+		try { Pattern.compile(punct_paren_open) }
+		catch (Exception e) { println "punct_paren_open: "+e;return false; }
+
+		try { Pattern.compile(punct_paren_close) }
+		catch (Exception e) { println "punct_paren_close: "+e;return false; }
+
+		try { Pattern.compile(punct_paren) }
+		catch (Exception e) { println "punct_paren: "+e;return false; }
+
+		try { Pattern.compile(punct_all) }
+		catch (Exception e) { println "punct_all: "+e;return false; }
+
+		try { Pattern.compile(word_chars) }
+		catch (Exception e) { println "word_chars: "+e;return false; }
+		
+		if (regPunct != null)
+		try { Pattern.compile(regPunct) }
+		catch (Exception e) { println "regPunct=${regPunct}: "+e;return false; }
+
+		if (regElision != null)
+		try { Pattern.compile(regElision) }
+		catch (Exception e) { println "regElision=${regElision}: "+e;return false; }
+
+		if (whitespaces != null)
+		try { Pattern.compile(whitespaces) }
+		catch (Exception e) { println "whitespaces=${whitespaces}: "+e;return false; }
+
+		for(TTest test : tests) {
+			try { Pattern.compile(test.regex) }
+			catch (Exception e) { println test.regex+": "+e;return false; }
+		}
+		return true;
+	}
+
+	public static boolean loadFromNode(Element tokenizerElement) {
+		//load params
+		String tmp_strong_punct;
+		NodeList params = tokenizerElement.getElementsByTagName("param");
+		for (int i = 0 ; i < params.getLength() ; i++) {
+			Element param = params.item(i);
+			String key = param.getAttribute("name")
+			String value = param.getAttribute("value")
+			if (value == null || value.length() == 0) value = param.getTextContent()
+			if (value.length() == 0) value = null;
+			
+			if (debug) println (" Tokenizer parametrized with $key=$value.")
+			if (key == "tag_all")
+				tag_all = value
+			else if (key == "enclitics")
+				enclitics = value
+			else if (key == "encliticsFR")
+				encliticsFR = value
+			else if (key == "div_tags")
+				div_tags = value
+			else if (key == "q_tags")
+				q_tags = value
+			else if (key == "extraword1_tags")
+				extraword1_tags = value
+			else if (key == "corr_tags_no_seg")
+				corr_tags_no_seg = value
+			else if (key == "word_tags")
+				word_tags = value
+			else if (key == "intraword_tags")
+				intraword_tags = value
+			else if (key == "punct_quotes")
+				punct_quotes = value
+			else if (key == "punct_strong1")
+				punct_strong1 = value
+			else if (key == "punct_strong2")
+				punct_strong2 = value
+			else if (key == "punct_paren_open1")
+				punct_paren_open1 = value
+			else if (key == "punct_paren_open2")
+				punct_paren_open2 = value
+			else if (key == "punct_paren_close1")
+				punct_paren_close1 = value
+			else if (key == "punct_paren_close2")
+				punct_paren_close2 = value
+			else if (key == "punct_weak")
+				punct_weak = value
+			else if (key == "entity")
+				entity = value
+			else if (key == "seg_tags")
+				seg_tags = value
+			else if (key == "regPunct")
+				regPunct = value
+			else if (key == "regElision")
+				regElision = value
+			else if (key == "whitespaces")
+				whitespaces = value
+			else if (key == "punct_strong") // this is temporary
+				tmp_strong_punct = value // this is temporary
+			else
+				println "MISSING TOKENIZER KEY: "+key
+		}
+		//recombine
+		recombine();
+		if (tmp_strong_punct != null) punct_strong = tmp_strong_punct;  // this is temporary
+		
+		String shouldResetTests = tokenizerElement.getAttribute("onlyThoseTests")
+		if ("true".equals(shouldResetTests)) {
+			println "Warning: tokenizer only using import parameters tests"
+			tests = []
+		}
+			
+		NodeList testsList = tokenizerElement.getElementsByTagName("test");
+		if (testsList.getLength() > 0) {
+			//println "Add "+testsList.getLength()+" tests to the tokenizer"
+			for (int i = 0 ; i < testsList.getLength() ; i++) {
+				Element test = testsList.item(i);
+				tests.add(i, new TTest(test.getTextContent(), test.getAttribute("type"),
+						Integer.parseInt(test.getAttribute("before")),
+						Integer.parseInt(test.getAttribute("hit")),
+						Integer.parseInt(test.getAttribute("after")))
+						)
+			}
+			//println "Tests: "+tests
+		}
+
+		if (isValid())
+			return true
+		else {
+			reset();
+			recombine();
+			return false;
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		Document doc = DomUtils.load(new File("/home/mdecorde/xml/discours/import.xml"));
+		Node corpora = doc.getDocumentElement().getElementsByTagName("corpora").item(0);
+		Node corpus = ((Element)corpora).getElementsByTagName("corpus").item(0)
+		Node tokenizer = ((Element)corpus).getElementsByTagName("tokenizer").item(0)
+		println "OK? "+ TokenizerClasses.loadFromNode(tokenizer)
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>All necessary classes to tokenize XML-TEI</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/tokenizer/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ReleaseFunctionsValidation.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ReleaseFunctionsValidation.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ReleaseFunctionsValidation.groovy	(revision 321)
@@ -0,0 +1,159 @@
+//package org.txm.test
+//import org.txm.*
+//import org.txm.functions.*
+//import org.txm.functions.ca.*
+//import org.txm.functions.classification.*
+//import org.txm.functions.concordances.*
+//import org.txm.functions.concordances.comparators.*
+//import org.txm.functions.cooccurrences.Cooccurrence
+//import org.txm.functions.diagnostic.*
+//import org.txm.functions.index.*
+//import org.txm.functions.referencer.Referencer
+//import org.txm.functions.specificities.*
+//import org.txm.searchengine.cqp.corpus.*
+//import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.stat.data.LexicalTable
+//import org.txm.utils.DeleteDir
+//
+//File exporttestdir = new File(System.getProperty("user.home"), "TXM/testrelease");
+//DeleteDir.deleteDirectory exporttestdir;
+//exporttestdir.mkdir()
+//println "Results are saved in dir: "+ exporttestdir
+//
+////diag corpus
+//def corpora = CorpusManager.getCorpusManager().getCorpora()
+//MainCorpus discours = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+//println "CORPUS: "+discours
+//
+//// word properties
+//def word_property = discours.getProperty("word")
+//def pos_property = discours.getProperty("pos")
+//def lemma_property = discours.getProperty("lemma")
+//def func_property = discours.getProperty("func")
+//println "props: $word_property, $pos_property, $lemma_property and $func_property"
+//
+//// structure properties
+//StructuralUnit text_su = discours.getStructuralUnit("text")
+//StructuralUnit s_su = discours.getStructuralUnit("s")
+//println "structs: $text_su and $s_su"
+//
+//Property text_id_property = text_su.getProperty("id")
+//Property text_type_property = text_su.getProperty("type")
+//Property text_date_property = text_su.getProperty("date")
+//Property text_loc_property = text_su.getProperty("loc")
+//println "struct props: $text_id_property, $text_type_property, $text_date_property and $text_loc_property"
+//ReferencePattern referencePattern = new ReferencePattern().addProperty(text_id_property)
+//
+//long time;
+//
+//// INFORMATIONS
+//time = System.currentTimeMillis();
+//Diagnostic diag = new Diagnostic(discours, 20)
+//diag.stepGeneralInfos();
+//diag.stepLexicalProperties();
+//diag.stepStructuralUnits();
+//diag.toHTML(new File(exporttestdir, "discours_diag"))
+//println("Diag: "+(System.currentTimeMillis()-time)/1000)
+//
+//// LEXICON
+//time = System.currentTimeMillis();
+//discours.getLexicon(lemma_property).toTxt(new File(exporttestdir, "discours_lexpos"), "UTF-8", "\t", "");
+//println("Lex: "+(System.currentTimeMillis()-time)/1000)
+//
+//// INDEX
+//time = System.currentTimeMillis();
+//Query query = new Query(Query.fixQuery("j.*"))
+//IndexSample index = new IndexSample(discours, query, [lemma_property, func_property])
+////index.sortLines(LineComparator.SortMode.FREQUNIT, true)
+////index.filterLines(2, 3000)
+//index.toTxt(new File(exporttestdir, "discours_indexlemmafuncj"), "UTF-8", "\t", "")
+//println("Ind: "+(System.currentTimeMillis()-time)/1000)
+//
+//// REFERENCER
+//time = System.currentTimeMillis();
+//Referencer referencer = new Referencer(discours, query, lemma_property, [text_type_property], true);
+//referencer.getQueryMatches()
+//referencer.getQueryindexes()
+//referencer.groupPositionsbyId()
+//referencer.toTxt(new File(exporttestdir, "discours_referencer"), "UTF-8")
+//println("Ref: "+(System.currentTimeMillis()-time)/1000)
+//
+//// CONCORDANCE
+//time = System.currentTimeMillis();
+//Concordance concordance = new Concordance(discours, query, word_property, [word_property, pos_property], referencePattern, referencePattern, 15, 15)
+//LexicographicLeftContextComparator comparator = new LexicographicLeftContextComparator()
+//comparator.initialize(discours)
+//concordance.sort(comparator)
+//concordance.setViewProperties([word_property])
+//concordance.toTxt(new File(exporttestdir,"discours_concj"), Concordance.Format.CONCORDANCE)
+//println("Conc: "+(System.currentTimeMillis()-time)/1000)
+//
+//// COOCCURRENCE WORD WINDOW
+//time = System.currentTimeMillis();
+//Cooccurrence cooc = new Cooccurrence(discours, query, [word_property], null, 21, 1, 1, 11, 2, 3, 1, false);
+//cooc.process();
+//cooc.toTxt(new File(exporttestdir, "discours_cooc_wordwindow"), "UTF-8")
+//println("Cooc1: "+(System.currentTimeMillis()-time)/1000)
+//
+//// COOCCURRENCE SENTENCE WINDOW
+//time = System.currentTimeMillis();
+//Cooccurrence cooc2 = new Cooccurrence(discours, query, [word_property], s_su,2, 1, 1, 1, 2, 3,1, false);
+//cooc2.process();
+//cooc2.toTxt(new File(exporttestdir, "discours_cooc_swindow"), "UTF-8")
+//println("Cooc2: "+(System.currentTimeMillis()-time)/1000)
+//
+//// SUBCORPORA
+//time = System.currentTimeMillis();
+//Corpus DGcorpus = discours.createSubcorpus(text_su, text_loc_property, "de Gaulle", "dgsubcorpus")
+//println("sub: "+(System.currentTimeMillis()-time)/1000)
+//
+//// PARTITIONS
+//time = System.currentTimeMillis();
+//Partition discours_types = discours.createPartition(text_su, text_type_property)
+//Partition discours_dates = discours.createPartition(text_su, text_date_property)
+//println("partitions: "+(System.currentTimeMillis()-time)/1000)
+//
+//// LEXICAL TABLE
+//time = System.currentTimeMillis();
+//LexicalTable table = discours_types.getLexicalTable(word_property, 2);
+//table.exportData(new File(exporttestdir, "discours_type_LT"), "\t", "");
+//println("lex table: "+(System.currentTimeMillis()-time)/1000)
+//
+////// SPECIF PARTITION
+////time = System.currentTimeMillis();
+////SpecificitesResult specifresult = org.txm.functions.specificities.Specificites.specificites(discours_types, word_property, [], [], 1);
+////specifresult.toTxt(new File(exporttestdir,"discours_speciftype"), "UTF-8", "\t", "")
+////println("sepcif part: "+(System.currentTimeMillis()-time)/1000)
+////
+////// SPECIF SUBCORPUS
+////time = System.currentTimeMillis();
+////SpecificitesResult specifresult2 = org.txm.functions.specificities.Specificites.specificites(DGcorpus.getParent(), DGcorpus, word_property)
+////specifresult2.toTxt(new File(exporttestdir,"discours_dgsub_specifloc"), "UTF-8", "\t", "")
+////println("specif sub: "+(System.currentTimeMillis()-time)/1000)
+////
+////// SPECIF LEXICAL TABLE
+////time = System.currentTimeMillis();
+////SpecificitesResult specifresult3 = org.txm.functions.specificities.Specificites.specificites(table);
+////specifresult3.toTxt(new File(exporttestdir,"discours_speciftype"), "UTF-8", "\t", "")
+////println("specif LT: "+(System.currentTimeMillis()-time)/1000)
+////
+////// AFC PARTITION
+////time = System.currentTimeMillis();
+////CA ca = new CA(discours_dates, lemma_property)
+////ca.toSVGFactorialMap(new File(exporttestdir,"discours_cadates"), true, true)
+////ca.toSVGSingularValues(new File(exporttestdir,"discours_cadates_singularvalues"))
+////println("CA part: "+(System.currentTimeMillis()-time)/1000)
+////
+////// AFC LEXICAL TABLE
+////time = System.currentTimeMillis();
+////CA ca2 = new CA(table);
+////ca2.toSVGFactorialMap(new File(exporttestdir,"discours_cadates"), true, true)
+////ca2.toSVGSingularValues(new File(exporttestdir,"discours_cadates_singularvalues"))
+////println("CA LT: "+(System.currentTimeMillis()-time)/1000)
+////
+////// CAH
+////time = System.currentTimeMillis();
+////CAH cah = new CAH(ca, true, CAH.getDefaultMethod(), CAH.getDefaultMetric(), 4, false)
+////cah.stepCompute()
+////cah.toSVG(new File(exporttestdir, "discours_dates_cah"),__RDevice.SVG);
+////println("CAH: "+(System.currentTimeMillis()-time)/1000)
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/OpenAnEditor.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/OpenAnEditor.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/OpenAnEditor.groovy	(revision 321)
@@ -0,0 +1,18 @@
+package org.txm.test
+
+//import org.eclipse.ui.*;
+//import org.txm.rcpapplication.*;
+//
+//import org.txm.rcpapplication.editors.input.*;
+//import org.txm.rcpapplication.editors.queryindex.*;
+
+final def sel = corpusViewSelection
+
+monitor.syncExec(new Runnable() {
+	public void run() {
+//		def page = TXMWindows.getActiveWindow().getActivePage();
+//		def editorInput = new QueryIndexEditorInput(sel);
+//		def editor = (QueryIndexEditor) page.openEditor(editorInput, QueryIndexEditor.ID); //$NON-NLS-1$
+//		editor.initializeFields();
+	}
+})
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ShowSpecifHistogram.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ShowSpecifHistogram.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ShowSpecifHistogram.groovy	(revision 321)
@@ -0,0 +1,81 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2011-05-09 14:58:28 +0200 (lun., 09 mai 2011) $
+// $LastChangedRevision: 1839 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.test
+
+// imports des fonctions que l'on va utiliser
+import org.txm.functions.specificities.*
+import org.txm.searchengine.cqp.corpus.*
+
+def corpusname = "DISCOURS"
+def struct = "text"
+def structprop = "date"
+
+MainCorpus discours = CorpusManager.getCorpusManager().getCorpus(corpusname)
+println "Building partition..."
+Partition discours_dates = discours.createPartition( discours.getStructuralUnit(struct), discours.getStructuralUnit(struct).getProperty(structprop))
+	
+// on récupère ses propriétés
+Property pos = discours.getProperty("pos")
+Property lemme = discours.getProperty("lemma")
+Property word = discours.getProperty("word")
+
+def Fmin = 2
+def Vmax = 10
+
+println "Building lexical table..."
+def table = discours_dates.getLexicalTable(lemme, Fmin)
+//for(int i = 0 ; i table.get)
+table.cut(Vmax)
+
+println "Computing specificities..." 
+def	result = Specificites.specificites(table)
+
+File outfile = new File("specif.txt")
+result.toTxt(outfile, "UTF-8")
+println "Saved in file "+outfile.getAbsolutePath()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/InjectMetadataCSV.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/InjectMetadataCSV.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/InjectMetadataCSV.groovy	(revision 321)
@@ -0,0 +1,29 @@
+package org.txm.test;
+import org.txm.metadatas.*;
+
+def txmfilesdir = new File("C:\\Documents and Settings\\alavrent\\xml\\metadata")
+def outdir = new File("C:\\Documents and Settings\\alavrent\\xml\\metadata\\meta")
+outdir.mkdir()
+def allmetadatasfile = new File("C:\\Documents and Settings\\alavrent\\xml\\metadata\\metadata.csv");
+println "-- INJECT METADATA - from csv file: "+allmetadatasfile+" in directory: "+txmfilesdir
+def metadatas = new Metadatas(allmetadatasfile, "UTF-8", ",","\"", 1)
+if(metadatas != null)
+{
+	println("Injecting metadata: "+metadatas.getHeadersList()+" in texts of directory "+txmfilesdir)
+	for (File infile : txmfilesdir.listFiles()) {
+		if(!infile.isFile() || infile.getName().endsWith(".csv.xml") || !infile.getName().endsWith(".xml"))
+			continue;
+		
+		print "."
+		File outfile = new File(outdir, infile.getName());
+		String tag = "text";
+		
+		if(!metadatas.injectMetadatasInXmlTXM(infile, outfile))
+		{
+			outfile.delete();
+			println "Error: Injection failed, stop import"
+			return;
+		}
+	}
+	println ""
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/AFC.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/AFC.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/AFC.groovy	(revision 321)
@@ -0,0 +1,181 @@
+///**
+// * Main.
+// *
+// * @param args the args
+// */
+//// Copyright © 2010-2013 ENS de Lyon.
+//// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+//// Lyon 2, University of Franche-Comté, University of Nice
+//// Sophia Antipolis, University of Paris 3.
+//// 
+//// The TXM platform is free software: you can redistribute it
+//// and/or modify it under the terms of the GNU General Public
+//// License as published by the Free Software Foundation,
+//// either version 2 of the License, or (at your option) any
+//// later version.
+//// 
+//// The TXM platform is distributed in the hope that it will be
+//// useful, but WITHOUT ANY WARRANTY; without even the implied
+//// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//// PURPOSE. See the GNU General Public License for more
+//// details.
+//// 
+//// You should have received a copy of the GNU General
+//// Public License along with the TXM platform. If not, see
+//// http://www.gnu.org/licenses.
+//// 
+//// 
+//// 
+//// $LastChangedDate:$
+//// $LastChangedRevision:$
+//// $LastChangedBy:$ 
+////
+//package org.txm.test
+//
+//import org.txm.Toolbox
+//import org.txm.functions.ca.*
+//import org.txm.searchengine.cqp.clientExceptions.CqiClientException
+//import org.txm.searchengine.cqp.clientExceptions.InvalidCqpIdException
+//import org.txm.searchengine.cqp.corpus.*
+//import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.stat.StatException
+//import org.txm.stat.data.LexicalTable
+//import org.apache.commons.lang.time.StopWatch
+//
+//// TODO: Auto-generated Javadoc
+///* (non-Javadoc)
+// * @see groovy.lang.Script#run()
+// */
+//def timer = new StopWatch()
+//
+//		// on récupère le corpus VOEUX
+//		//EN: let's get the VOEUX corpus
+//		def voeux = CorpusManager.getCorpusManager().getCorpus("VOEUX")
+//
+//timer.start()
+//
+//		// on crée une partition sur les textes avec comme propriété discriminante l'id du texte
+//		//EN: let create a partition on texts with their ID's
+//		def text_su = voeux.getStructuralUnit("text")
+//		def text_id = text_su.getProperty("id")
+//		def textes = voeux.createPartition(text_su, text_id)
+//		
+//timer.stop()
+//println "Création de la partition : "+timer.toString()	
+//timer.reset()
+//
+//		// choisir la propriété lexicale pour construire la table lexicale
+//		//EN: choose the lexical property to build the lexical table
+//		def prop = voeux.getProperty("lemme") 
+//
+//timer.start()
+//
+//		// on crée une table lexicale sans les hapax
+//		//EN: let create a lexical table without hapax legomenon
+//		def table = textes.getLexicalTable(prop, 2)
+//		
+//timer.stop()
+//println "Création de la table lexicale des lemmes : "+timer.toString()
+//timer.reset()		
+//	
+//timer.start()
+//		// on fait une afc
+//		//EN: compute the CA
+//		def ca = new CA(table)
+//		ca.processFromTable();
+//
+//timer.stop()
+//println "calcul de l'AFC 1 : "+timer.toString()		
+//timer.reset()
+//
+//timer.start()
+//
+//		// les valeurs propres
+//		//EN: singular values
+//		ca.getSingularValues()
+//		
+//timer.stop()
+//println "récupération des valeurs propres SV1 : "+timer.toString()	
+//timer.reset()	
+//
+//timer.start()
+//
+//		// les données de colonnes produites par le package CA de R
+//		//EN: column data produced by the CA R package
+//		ca.getColsCoords() 
+//		ca.getColsDist()
+//		ca.getColsInertia()
+//		ca.getColsMass()
+//		
+//timer.stop()
+//println "récupération de toutes les infos de colonnes1 : "+timer.toString()
+//timer.reset()
+//
+//timer.start()
+//
+//		// les données de lignes produites par le package CA de R
+//		//EN: row data produced by the CA R package
+//		ca.getRowsCoords()
+//		ca.getRowsDist()
+//		ca.getRowsInertia()
+//		ca.getRowsMass()
+//		
+//timer.stop()
+//println "récupération de toutes les infos de lignes1 : "+timer.toString()
+//timer.reset()
+//
+//timer.start()
+//
+//// on joue un peu avec la table EN: let's play with the table
+//table.cut(3000) 						// pas plus de 3000 valeurs, EN: not more than 3000 values
+//table.filter(table.getFreqs(), 2000, 5) // coupe a 2000 valeurs et Fmin=5, EN: cut to 2000 values and Fmin=5
+//
+//timer.stop()
+//println "On filtre la table lexicale en fréquence et en nombre de lignes : "+timer.toString()
+//timer.reset()
+//
+//timer.start()
+//
+//// on fait une afc
+////EN: we compute a CA
+//def ca2 = new CA(table)
+//ca2.processFromTable();
+//timer.stop()
+//println "calcul de la deuxième AFC : "+timer.toString()		
+//timer.reset()
+//
+//timer.start()
+//
+//// les valeurs propres
+////EN: singular values
+//ca2.getSingularValues()
+//
+//timer.stop()
+//println "récupération des valeurs propres SV2 : "+timer.toString()	
+//timer.reset()	
+//
+//timer.start()
+//
+//// les données de colonnes produites par le package CA de R
+////EN: column data produced by the CA R package
+//ca2.getColsCoords() 
+//ca2.getColsDist()
+//ca2.getColsInertia()
+//ca2.getColsMass()
+//
+//timer.stop()
+//println "récupération de toutes les infos de colonnes2 : "+timer.toString()
+//timer.reset()
+//
+//timer.start()
+//
+//// les données de lignes produites par le package CA de R
+////EN: row data produced by the CA R package
+//ca2.getRowsCoords()
+//ca2.getRowsDist()
+//ca2.getRowsInertia()
+//ca2.getRowsMass()
+//
+//timer.stop()
+//println "récupération de toutes les infos de lignes2 : "+timer.toString()
+//timer.reset()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/textselection.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/textselection.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/textselection.groovy	(revision 321)
@@ -0,0 +1,13 @@
+package org.txm.test
+
+
+//import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.selection.*
+
+MainCorpus corpus = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+
+Selection sel = new Selection(corpus);
+sel.fullDump()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/HelloWorldR.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/HelloWorldR.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/HelloWorldR.groovy	(revision 321)
@@ -0,0 +1,47 @@
+package org.txm.test;
+
+/**
+ * 
+Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+def r = RWorkspace.getRWorkspaceInstance() 	// get the R connection
+
+r.setLog(true)								// start R logging
+
+r.eval("print(\"Hello World!\")")			// do it
+
+r.setLog(false) 							// stop R log
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/Cooccurrences.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/Cooccurrences.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/Cooccurrences.groovy	(revision 321)
@@ -0,0 +1,75 @@
+///**
+// * Main.
+// *
+// * @param args the args
+// */
+//// Copyright © 2010-2013 ENS de Lyon.
+//// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+//// Lyon 2, University of Franche-Comté, University of Nice
+//// Sophia Antipolis, University of Paris 3.
+//// 
+//// The TXM platform is free software: you can redistribute it
+//// and/or modify it under the terms of the GNU General Public
+//// License as published by the Free Software Foundation,
+//// either version 2 of the License, or (at your option) any
+//// later version.
+//// 
+//// The TXM platform is distributed in the hope that it will be
+//// useful, but WITHOUT ANY WARRANTY; without even the implied
+//// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//// PURPOSE. See the GNU General Public License for more
+//// details.
+//// 
+//// You should have received a copy of the GNU General
+//// Public License along with the TXM platform. If not, see
+//// http://www.gnu.org/licenses.
+//// 
+//// 
+//// 
+//// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+//// $LastChangedRevision: 2386 $
+//// $LastChangedBy: mdecorde $ 
+////
+//package org.txm.test
+//
+//// we import the packages containing the functions we are going to use
+//import org.txm.Toolbox
+//import org.txm.searchengine.cqp.corpus.*
+//import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.functions.cooccurrences.*
+//
+//// TODO: Auto-generated Javadoc
+///* (non-Javadoc)
+// * @see groovy.lang.Script#run()
+// */
+//def discours = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+//
+//def word = discours.getProperty("word")
+//def pos = discours.getProperty("pos")
+//
+//def limit = discours.getStructuralUnit("s")
+//def minleft = 1
+//def maxleft = 2
+//def minright = 1
+//def maxright = 2
+//
+//def minf = 0
+//def mincof = 0
+//def minscore = 0
+//
+//def query = new Query(Query.fixQuery("je"))
+//
+////long time = System.currentTimeMillis()
+//
+//def cooccurrence = new Cooccurrence(discours, query, [word], limit, minleft, maxleft, minright, maxright, minf, mincof, minscore)
+//cooccurrence.process();
+////cooccurrence.compute()
+//
+////println("Compute time : "+(System.currentTimeMillis()-time)+" ms")
+//
+//
+//def file = new File("cooc.txt")
+//cooccurrence.toTxt(file)
+//
+//println "printed cooccurrents in "+file.getAbsolutePath()
+//
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/testSummary.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/testSummary.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/testSummary.groovy	(revision 321)
@@ -0,0 +1,12 @@
+package org.txm.test
+
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.functions.summary.*
+
+def discours = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+Summary summary = new Summary(discours);
+if(summary.init())
+	println "success";
+else
+	pritnln "fail";
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/CQPBenchmark.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/CQPBenchmark.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/CQPBenchmark.groovy	(revision 321)
@@ -0,0 +1,324 @@
+//package org.txm.test
+//
+//import org.txm.*
+//import org.txm.functions.*
+////import org.txm.functions.ca.*
+////import org.txm.functions.classification.CAH
+//import org.txm.functions.concordances.*
+//import org.txm.functions.concordances.comparators.*
+//import org.txm.functions.cooccurrences.*
+//import org.txm.functions.diagnostic.*
+//import org.txm.functions.index.*
+//import org.txm.functions.referencer.Referencer
+//import org.txm.functions.specificities.*
+//import org.txm.searchengine.cqp.corpus.*
+//import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.stat.data.LexicalTable
+//import org.txm.utils.DeleteDir
+//
+//
+//testDir = new File(System.getProperty("user.home"), "TXM/testrelease");
+//DeleteDir.deleteDirectory testDir;
+//testDir.mkdir()
+//
+////get corpus
+//csv = [];
+//firstExecution = 0;
+//
+//def process(String CORPUSNAME, String QUERY1, String QUERY2) {
+//	
+//	String ENCODING = "UTF-8"
+//	String COLSEPARATOR = "\t"
+//	String TXTSEPARATOR = ""
+//	int i = 0;
+//	def corpora = CorpusManager.getCorpusManager().getCorpora()
+//	MainCorpus corpus = CorpusManager.getCorpusManager().getCorpus(CORPUSNAME)
+//	//println "CORPUS: "+corpus
+//
+//	Query query = new Query(QUERY1) //"\"..............*\"", "\"..........*\""
+//	Query query2 = new Query(QUERY2)
+//
+//	File exporttestdir = new File(testDir, corpus.getName());
+//	File reportFile = new File(exporttestdir, "report.csv")
+//	DeleteDir.deleteDirectory exporttestdir;
+//	exporttestdir.mkdir()
+//	//println "Results are saved in dir: "+ exporttestdir
+//
+//	// word properties
+//	def word_property = corpus.getProperty("word")
+//
+//	// structure properties
+//	StructuralUnit text_su = corpus.getStructuralUnit("text")
+//	StructuralUnit s_su = corpus.getStructuralUnit("s")
+//	Property text_id_property = text_su.getProperty("id")
+//	ReferencePattern referencePattern = new ReferencePattern().addProperty(text_id_property)
+//
+//	long time;
+//	// START START START START
+//	if (firstExecution == 0)
+//		csv << ["object", "size", "nPart", "command", "query", "query freq", "$CORPUSNAME $QUERY1 $QUERY2 mode "+Toolbox.getParam(Toolbox.CQI_NETWORK_MODE)]
+//	else
+//		csv[i++] << "$CORPUSNAME $QUERY1 $QUERY2 mode "+Toolbox.getParam(Toolbox.CQI_NETWORK_MODE)
+//	
+//	// INFORMATIONS
+//	print " INFO"
+//	time = System.currentTimeMillis();
+//	Diagnostic diag = new Diagnostic(corpus, 20)
+//	diag.stepGeneralInfos();
+//	diag.stepLexicalProperties();
+//	diag.stepStructuralUnits();
+//	diag.toHTML(new File(exporttestdir, "diag"))
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Informations", "no query", "no freq", (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// LEXICON
+//	print " LEX"
+//	time = System.currentTimeMillis();
+//	corpus.getLexicon(word_property).toTxt(new File(exporttestdir, "lexpos"), ENCODING, COLSEPARATOR, TXTSEPARATOR);
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Lexicon", "no query", "no freq", (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//		
+//	// INDEX
+//	print " INDEX"
+//	time = System.currentTimeMillis();
+//	IndexSample index = new IndexSample(corpus, query, [word_property])
+//	index.toTxt(new File(exporttestdir, "indexlemmafuncj"), ENCODING, COLSEPARATOR, TXTSEPARATOR)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Index", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// REFERENCER
+//	print " REF"
+//	time = System.currentTimeMillis();
+//	Referencer referencer = new Referencer(corpus, query, word_property, [text_id_property], true);
+//	referencer.getQueryMatches()
+//	referencer.getQueryindexes()
+//	referencer.groupPositionsbyId()
+//	referencer.toTxt(new File(exporttestdir, "referencer"), ENCODING)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Referencer", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// CONCORDANCE
+//	print " CONC"
+//	time = System.currentTimeMillis();
+//	Concordance concordance = new Concordance(corpus, query, word_property, [word_property, word_property], referencePattern, referencePattern, 15, 15)
+//	concordance.toTxt(new File(exporttestdir,"concj"), Concordance.Format.CONCORDANCE)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Concordances", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// COOCCURRENCE WORD WINDOW
+//	print " COOC"
+//	time = System.currentTimeMillis();
+//	Cooccurrence cooc = new Cooccurrence(corpus, query, [word_property], null, 21, 1, 1, 11, 2, 3, 1, false);
+//	cooc.process();
+//	cooc.toTxt(new File(exporttestdir, "cooc_wordwindow"), ENCODING)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences words", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// COOCCURRENCE SENTENCE WINDOW
+//	print " COOC"
+//	time = System.currentTimeMillis();
+//	Cooccurrence cooc2 = new Cooccurrence(corpus, query, [word_property], s_su,2, 1, 1, 1, 2, 3,1, false);
+//	cooc2.process();
+//	cooc2.toTxt(new File(exporttestdir, "cooc_swindow"), ENCODING)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences structures", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// INDEX
+//	print " INDEX"
+//	time = System.currentTimeMillis();
+//	index = new IndexSample(corpus, query2, [word_property])
+//	index.toTxt(new File(exporttestdir, "indexlemmafuncj"), ENCODING, COLSEPARATOR, TXTSEPARATOR)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Index", query2, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// REFERENCER
+//	print " REF"
+//	time = System.currentTimeMillis();
+//	referencer = new Referencer(corpus, query2, word_property, [text_id_property], true);
+//	referencer.getQueryMatches()
+//	referencer.getQueryindexes()
+//	referencer.groupPositionsbyId()
+//	referencer.toTxt(new File(exporttestdir, "referencer"), ENCODING)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Referencer", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// CONCORDANCE
+//	print " CONC"
+//	time = System.currentTimeMillis();
+//	concordance = new Concordance(corpus, query2, word_property, [word_property, word_property], referencePattern, referencePattern, 15, 15)
+//	concordance.toTxt(new File(exporttestdir,"concj"), Concordance.Format.CONCORDANCE)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Concordances", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// COOCCURRENCE WORD WINDOW
+//	print " COOC"
+//	time = System.currentTimeMillis();
+//	cooc = new Cooccurrence(corpus, query2, [word_property], null, 21, 1, 1, 11, 2, 3, 1, false);
+//	cooc.process();
+//	cooc.toTxt(new File(exporttestdir, "cooc_wordwindow"), ENCODING)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences words", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// COOCCURRENCE SENTENCE WINDOW
+//	print " COOC"
+//	time = System.currentTimeMillis();
+//	cooc2 = new Cooccurrence(corpus, query2, [word_property], s_su,2, 1, 1, 1, 2, 3,1, false);
+//	cooc2.process();
+//	cooc2.toTxt(new File(exporttestdir, "cooc_swindow"), ENCODING)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Cooccurrences structures", query, index.getT(), (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// SUBCORPORA
+//	print " SUBCORPUS"
+//	time = System.currentTimeMillis();
+//	Corpus DGcorpus = corpus.createSubcorpus(text_su, text_id_property, "01_DeGaulle", "dgsubcorpus")
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Subcorpus", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// PARTITIONS
+//	print " PARTITIONS"
+//	time = System.currentTimeMillis();
+//	Partition discours_types = corpus.createPartition(text_su, text_id_property)
+//	Partition discours_dates = corpus.createPartition(text_su, text_id_property)
+//	if (firstExecution == 0)
+//		csv << [corpus.getName(), corpus.getSize(), 1, "Partition 2x", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+//	// LEXICAL TABLE
+//	print " LT"
+//	time = System.currentTimeMillis();
+//	LexicalTable table = discours_types.getLexicalTable(word_property, 2);
+//	table.exportData(new File(exporttestdir, "type_LT"), COLSEPARATOR, TXTSEPARATOR);
+//	if (firstExecution == 0)
+//		csv << [discours_types.getName(), corpus.getSize(), discours_types.getNPart(), "LT part", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+//	else
+//		csv[i++] << (System.currentTimeMillis()-time)/1000
+//
+////	// SPECIF SUBCORPUS
+////	time = System.currentTimeMillis();
+////	print " SPECIF"
+////	SpecificitesResult specifresult2 = org.txm.functions.specificities.Specificites.specificites(DGcorpus.getParent(), DGcorpus, word_property)
+////	specifresult2.toTxt(new File(exporttestdir,"dgsub_specifloc"), ENCODING, COLSEPARATOR, TXTSEPARATOR)
+////	if (firstExecution == 0)
+////		csv << [DGcorpus.getName(), corpus.getSize(), 1, "Specif sub", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+////	else
+////		csv[i++] << (System.currentTimeMillis()-time)/1000
+////
+////	// SPECIF LEXICAL TABLE
+////	print " SPECIF"
+////	time = System.currentTimeMillis();
+////	SpecificitesResult specifresult3 = org.txm.functions.specificities.Specificites.specificites(table);
+////	specifresult3.toTxt(new File(exporttestdir,"speciftype"), ENCODING, COLSEPARATOR, TXTSEPARATOR)
+////	if (firstExecution == 0)
+////		csv << [table.getName(), corpus.getSize(), table.getNColumns(), "specif LT", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+////	else
+////		csv[i++] << (System.currentTimeMillis()-time)/1000
+////
+////	// AFC PARTITION
+////	print " AFC"
+////	time = System.currentTimeMillis();
+////	CA ca = new CA(discours_dates, word_property, 0 ,9999999)
+////	ca.stepLexicalTable();
+////	ca.stepSortTableLexical();
+////	ca.stepCompute()
+////	ca.toSVGFactorialMap(new File(exporttestdir,"cadates"), true, true)
+////	ca.toSVGSingularValues(new File(exporttestdir,"cadates_singularvalues"))
+////	if (firstExecution == 0)
+////		csv << [discours_dates.getName(), corpus.getSize(), discours_dates.getNPart(), "AFC part", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+////	else
+////		csv[i++] << (System.currentTimeMillis()-time)/1000
+////
+////	// AFC LEXICAL TABLE
+////	print " AFC"
+////	time = System.currentTimeMillis();
+////	CA ca2 = new CA(table);
+////	ca2.stepCompute()
+////	ca2.toSVGFactorialMap(new File(exporttestdir,"cadates"), true, true)
+////	ca2.toSVGSingularValues(new File(exporttestdir,"cadates_singularvalues"))
+////	if (firstExecution == 0)
+////		csv << [table.getName(), corpus.getSize(), table.getNColumns(), "specif LT", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+////	else
+////		csv[i++] << (System.currentTimeMillis()-time)/1000
+////
+////	// CAH
+////	print " CAH"
+////	time = System.currentTimeMillis();
+////	CAH cah = new CAH(ca, true, CAH.getDefaultMethod(), CAH.getDefaultMetric(), 4, false)
+////	cah.stepCompute()
+////	cah.toSVG(new File(exporttestdir, "dates_cah"),__RDevice.SVG);
+////	if (firstExecution == 0)
+////		csv << [discours_dates.getName(), corpus.getSize(), discours_dates.getNPart(), "CAH ca table", "no query", "no freqs", (System.currentTimeMillis()-time)/1000]
+////	else
+////		csv[i++] << (System.currentTimeMillis()-time)/1000
+////
+////	firstExecution++;
+////	println ""
+//
+//}
+//
+//
+//
+//println "restarting TBX..."
+//Toolbox.setParam(Toolbox.CQI_NETWORK_MODE, "false")
+//println "MEMORY MODE: "+Toolbox.restart();
+//process("DISCOURS", "\"..............*\"", "\"..........*\"") // first time
+//println "MEMORY MODE: "+Toolbox.restart();
+//process("DISCOURS", "\"..............*\"", "\"..........*\"")
+//
+//println "restarting TBX..."
+//Toolbox.setParam(Toolbox.CQI_NETWORK_MODE, "true")
+//println "NETWORK MODE: "+Toolbox.restart();
+//process("DISCOURS", "\"..............*\"", "\"..........*\"")
+//println "NETWORK MODE: "+Toolbox.restart();
+//process("DISCOURS", "\"..............*\"", "\"..........*\"")
+//
+//
+//
+//
+//// add TOTAL line
+//int[] totaux = ["","","","","","",""]
+//for (int i = 7 ; i < csv.size() ; i++) {
+//	int total = 0;
+//	for (def line : csv) {
+//		total += line[i]
+//	}
+//	totaux << total
+//}
+//csv << totaux;
+//
+//// WRITE ALL RESULTS
+//for (def line : csv) {
+//	for (int i = 0 ; i < firstExecution ; i++) {
+//		def item = line[i]
+//		if (i > 0) print "\t"+item
+//		else print item
+//	}
+//	println ""
+//}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ClixTest.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ClixTest.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ClixTest.groovy	(revision 321)
@@ -0,0 +1,86 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.test
+
+/**
+ * How to use ProcessBuilder to wrap an external program
+ * 
+ * 
+ * ======= Definition file =======
+<?xml version="1.0" encoding="UTF-8"?>
+<application name="Ls" version="0.0.0" desc="List directory">
+	<progs>
+		<prog exec="ls" version="0.0.0" desc="Ls">
+			<args>
+				<arg state="optional" type="none" name="l" desc="use a long listing format"/>
+				<arg state="optional" type="none" name="a" desc="do not ignore entries starting with '.'"/>
+				<arg state="optional" type="String" name="d" desc="list directory entries instead of contents"/>
+			</args>
+		</prog>
+	</progs>
+</application>
+ * 
+ */
+
+/**
+ * the file containing the declaration of the Ls program : ls-wrapper-definition.xml
+ *
+ */
+def definitionFile = new File("~/workspace/org.txm.toolbox/src/groovy/org/textometrie/scripts/ls-wrapper-definition.xml")
+
+/**
+ * The file which will contain the code : Ls.groovy
+ */
+def classFile = new File("~/Ls.groovy")
+
+/**
+ * the directory containing the binaries which will be called: /bin/
+ */
+def binariesDirectory = new File("/bin/")
+
+/**
+ * build wrapping code
+ */
+def aProgram = org.txm.utils.processbuilder.ProcessBuilder.wrapProgram(definitionFile, classFile, binariesDirectory)
+
+/**
+ * set call parameters
+ */
+aProgram.setl()
+aProgram.seta()
+
+/**
+ * call the ls external program
+ */
+aProgram.ls()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/Concordances.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/Concordances.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/Concordances.groovy	(revision 321)
@@ -0,0 +1,83 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2015-10-22 10:13:24 +0200 (Thu, 22 Oct 2015) $
+// $LastChangedRevision: 3041 $
+// $LastChangedBy: finnishgazelle $ 
+//
+
+package org.txm.test
+
+// import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.concordances.*
+import org.txm.functions.concordances.comparators.*
+import org.txm.functions.ReferencePattern
+
+	// get the DISCOURS corpus
+	// TODO: Auto-generated Javadoc
+
+	/* (non-Javadoc)
+	 * @see groovy.lang.Script#run()
+	 */
+	def discours = CorpusManager.getCorpusManager().getCorpus("VOEUX")
+	
+	// get some properties
+	def pos = discours.getProperty("pos")
+	def word = discours.getProperty("word")
+	def text = discours.getStructuralUnit("text")
+	def text_id = text.getProperty("id")
+
+	// create a query. Here the concordance keyword will be the French word "Je" (the "I" pronoun in English)
+	def query = new Query(Query.fixQuery("je"))
+	
+	// define the references pattern for each concordance line
+	def referencePattern = new ReferencePattern().addProperty(text_id)
+	def sortReferencePattern = new ReferencePattern().addProperty(text_id)
+
+	// compute the concordance with contexts of 15 words on each side of the keyword
+	def concordance = new Concordance(discours, query, word, [word, pos], referencePattern,sortReferencePattern, 15, 15)
+
+	// get a builtin sort function
+	def comparator = new LexicographicLeftContextComparator()
+	comparator.initialize(discours)
+
+	
+	// sort the concordance
+	concordance.sort(comparator)
+	
+	// define which occurrence properties will be displayed
+	concordance.setViewProperties([word])
+	
+	// write all the concordance in a text file
+	def file = new File("conc.txt")
+	concordance.toTxt(file, "UTF-8")
+	
+	println("Concordance written in file: "+ file.getAbsolutePath())
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/mesuresBP.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/mesuresBP.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/mesuresBP.groovy	(revision 321)
@@ -0,0 +1,39 @@
+package org.txm.test;
+import org.txm.functions.mesures.*;
+import org.txm.searchengine.cqp.corpus.Corpus;
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+
+def corpus1 = CorpusManager.getCorpusManager().getCorpus("MESURES")
+
+Mesures mesures = new Mesures([corpus1]);
+// Taille en mot des textes
+mesures.add(new Magnitude('<text> [] expand to text', "word", false, Synthese.SUM));
+// Qurtiles des discours direct
+mesures.add(new Magnitude('<q> [_.q_type="DD"] expand to q', "word", true, Synthese.QUARTILE));
+// Quartiles des paragraphes (ici div1, j'ai pas de paragraphes -_-)
+mesures.add(new Magnitude('<div1> [] expand to div1', "word", true, Synthese.QUARTILE));
+// moyenne de la taille en mot des paragraphes
+//mesures.add(new Magnitude('<p> [] expand to p', "word", true, Synthese.MOYENNE));
+
+// proportion des ! et ? dans le discours direct
+mesures.add(new Proportion('[_.q_type="DD"]', "word", true, Synthese.COUNTMATCHES, '[word=".*[.!?].*"]', ['[word=".*\\?*"]', '[word=".*!.*"]']));
+// proportion de discours direct avec tiret ou guillemet
+mesures.add(new Proportion(null, "word", true, Synthese.COUNTMATCHES, '[_.q_type="DD"] expand to q', ['[_.q_type="DD" & _.q_rend="tiret"] expand to q', '[_.q_type="DD" & _.q_rend="guillemet"] expand to q']));
+// Proportion de discours direct avec incise de dire / verbe introducteur
+mesures.add(new Proportion(null, "word", true, Synthese.COUNTMATCHES, '[_.q_type="DD"] expand to q', ['[_.q_type="DD" & _.seg_ana="incise_di"] expand to q', '[_.seg_ana="int_DD"] </seg> []{0,10} <q> [_.q_type="DD"]']));
+//mesures.add(new Proportion('"j.*"%c expand to text', "word", true, Synthese.SUM, '"j.*" expand to text', ['"j.....*"%c expand to text', '"j........*"%c expand to text']));
+
+// Taux de présence linéaire de ponctuation forte dans le DD
+mesures.add(new PresenceRate('[_.q_type="DD"] expand to q', "word", true, Synthese.SUM, '[word=".*[;!?].*"]'));
+// Taux de présence linéaire de DD
+mesures.add(new PresenceRate(null, "word", true, Synthese.SUM, '[_.q_type="DD"]'));
+// Taux de présence Linéaire de PR 
+mesures.add(new PresenceRate(null, "word", true, Synthese.SUM, '<pr> [] expand to pr'));
+
+// Diversité (en lemmes) des verbes introducteurs
+mesures.add(new Diversity(null, "word", true, Synthese.SUM, '[_.seg_ana="int.*" & frpos="V.*"]', "frlemma"));
+//mesures.add(new Diversity('[_.q_type="DD"] expand to q', "word", true, Synthese.SUM, '"j.*"', "frpos"));
+
+mesures.prettyPrint();
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/WGet.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/WGet.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/WGet.groovy	(revision 321)
@@ -0,0 +1,20 @@
+URL url = new URL("http://docs.codehaus.org/display/GROOVY/Simple+file+download+from+URL");
+File outFile = new File("test.out")
+
+static def download(def file, def url) {
+		url.withInputStream() {is->
+			 file.withOutputStream() {os->
+				 def bs = new BufferedOutputStream( os )
+				 bs << is
+			 }
+		 }
+}
+
+// if proxy , uncomment this and set up properties
+//System.properties.putAll( ["http.proxyHost":"proxy-host",
+//							 "http.proxyPort":"proxy-port",
+//							 "http.proxyUserName":"user-name",
+//							 "http.proxyPassword":"proxy-passwd"] )
+
+WGet.download(outFile, url);
+println outFile.getText()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/Informations.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/Informations.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/Informations.groovy	(revision 321)
@@ -0,0 +1,64 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.test
+
+
+// imports des fonctions que l'on va utiliser
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.functions.diagnostic.Diagnostic
+import org.txm.objects.*
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+def manualInit = false
+if (!org.txm.Toolbox.isInitialized())
+{
+	manualInit = true
+	org.txm.Toolbox.initialize(new File(System.getProperty("user.home")+"/TXM/install.prefs"))
+}
+
+def corpus = CorpusManager.getCorpusManager().getCorpus("QUETE")
+
+try
+{
+	def d = new Diagnostic(corpus, 20)
+	d.process();
+	println d.htmldump()
+} catch(Exception e){	e.printStackTrace() }
+
+if(manualInit)
+	org.txm.Toolbox.shutdown()
+
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/CooccurencesAndReferenceCorpus.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/CooccurencesAndReferenceCorpus.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/CooccurencesAndReferenceCorpus.groovy	(revision 321)
@@ -0,0 +1,73 @@
+//package org.txm.test
+//import org.txm.Toolbox
+//import org.txm.searchengine.cqp.corpus.*
+//import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.functions.cooccurrences.*
+//import org.txm.stat.engine.r.RWorkspace;
+//import org.txm.stat.engine.r.data.*;
+//import org.txm.functions.cooccurrences.comparators.*;
+//
+///////////////////PARAMS/////////////////////
+//
+//def corpusname = "DANS16FRANTEXT"
+//def PROP = "word"
+//def QUERY ="""
+//[type="expFound"]
+//"""
+//
+//def minleft = 0
+//def maxleft = 10
+//def minright = 0
+//def maxright = 10
+//
+//def minf = 1
+//def mincof = 1
+//def minscore = 0
+//
+//File refFile = new File("/home/mdecorde/TEMP/lexiqueFrantext16.txt");
+//def file = new File("/home/mdecorde/TEMP/cooc_dans_ref.tsv")
+//
+/////////////////////////////////////////////
+//
+//minleft++
+//maxleft++
+//minright++
+//maxright++
+//if (!refFile.exists()) { println "ref file does not exists: $refFile"; return;}
+//QUERY = QUERY.trim();
+//MainCorpus corpus = CorpusManager.getCorpusManager().getCorpus(corpusname)
+//Property property = corpus.getProperty(PROP)
+//
+//// Build the reference corpus
+//println "building reference corpus."
+//def ref_lt = LexicalTableImpl.createLexicalTableImpl(refFile);
+//
+//// Build the cooccurrences
+//def limit = null
+//def query = new Query(Query.fixQuery(QUERY))
+//print "building cooc"
+//def cooccurrence = new Cooccurrence(corpus, query, [property], limit, maxleft, minleft, minright, maxright, minf, mincof, minscore, false)
+//System.out.println("cooc: "+corpus+" "+query+" "+[property]+" "+null+" "+maxleft+" "+minleft+" "+minright+" "+maxright+" "+minf+" "+mincof+" "+minscore+" "+false);
+//print "."
+//if (refFile != null)
+//	cooccurrence.setReferenceCorpus(ref_lt.getSymbol()) // SET THE REFERENCE CORPUS
+//print "."
+//cooccurrence.stepQueryLimits();
+//print "."
+//cooccurrence.stepGetMatches();
+//print "."
+//cooccurrence.stepBuildSignatures();
+//print "."
+//cooccurrence.stepCount();
+//print "."
+//cooccurrence.stepBuildLexicalTable();
+//print "."
+//cooccurrence.stepGetScores();
+//print "."
+//cooccurrence.getLines();
+//cooccurrence.sort(new OccComparator())
+//println ""
+//
+//println "exporting."
+//cooccurrence.toTxt(file, "UTF-8", "\t", "")
+//println "printed cooccurrents in "+file.getAbsolutePath()
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ExecuteRscript.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ExecuteRscript.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ExecuteRscript.groovy	(revision 321)
@@ -0,0 +1,66 @@
+package org.txm.test;
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+// import org.txm.rcpapplication.commands.*
+// uncomment for test
+
+// get a connection to R
+// TODO: Auto-generated Javadoc
+
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+def r = RWorkspace.getRWorkspaceInstance()
+
+// start logging R output in the console
+r.setLog(true)
+
+// a sample plotting script
+def script = new File(System.getProperty("user.home")+"/TXM/scripts/samples/R/plot100.R")
+// we need this for Windows
+def scriptpath = script.getAbsolutePath().replace("\\","\\\\")
+// use a temporary file to transfert the graphic
+def f = File.createTempFile("txm", ".svg", script.getParentFile())
+
+// generate the graphic
+r.plot(f, "source(\""+scriptpath+"\")")
+
+// draw the graphic in a new window of TXM
+monitor.syncExec(new Runnable() {
+	@Override
+	public void run() {	OpenSVGGraph.OpenSVGFile(f.getAbsolutePath(), "sample plot") }
+});
+
+
+// stop logging R output
+r.setLog(false)
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/tables.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/tables.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/tables.groovy	(revision 321)
@@ -0,0 +1,52 @@
+//package org.txm.test
+//
+//import org.txm.stat.engine.r.*
+//import org.txm.stat.engine.r.data.*
+//
+//def r = RWorkspace.getRWorkspaceInstance()
+//r.setLog(true)
+//
+//int[][] data = [[1,2,3,4,0], [78,64,18,3,0], [2,84,7,168,0], [5,7,6,0,0], [12,4,88,1668,0], [0,0,0,0,1]]
+//String[] lineNames = ["L1", "L2", "L3", "L4", "L5", "L6"]
+//String[] colNames = ["C1", "C2", "C3", "C4", "C5"]
+//
+//MatrixImpl matrix = new MatrixImpl(data, lineNames, colNames)
+//matrix.print()
+//
+////println "remove rows 1"
+////matrix.removeRows([1], true)
+////matrix.print()
+//
+////println "remove cols 4"
+////matrix.removeCols([4], true)
+////matrix.print()
+//
+////println "merge rows 0 1"
+////int[] rows = [0,1]
+////matrix.mergeRows(rows);
+////matrix.print()
+//
+////println "merge cols 0 1"
+////int[] cols = [0,1]
+////matrix.mergeCols(cols);
+////matrix.print()
+//
+////println "sortByFreqs"
+////matrix.sortByFreqs();
+////matrix.print()
+//
+////println "filter fmin=100 nlines=4"
+////matrix.filter(5,100);
+////matrix.print()
+//
+//println "save data in test.txt"
+//matrix.exportData(new File("test.txt"), ";", "\"")
+//println "do something: delete col 4"
+//matrix.removeCols([4], true)
+//matrix.print()
+//
+//println "reload data from test.txt"
+//matrix.importData(new File("test.txt"))
+//matrix.print()
+//
+//r.setLog(false)
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Scripts to test the platform. All theses scripts can be executed in TXM too</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/org/txm/test/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/HelloWorld.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/HelloWorld.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/HelloWorld.groovy	(revision 321)
@@ -0,0 +1,48 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.test
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+println "Hello World !"
+for(File f : new File("/home/mdecorde/Bureau/bioling").listFiles())
+{
+	if(f.getName().endsWith(" corpus.txt"))
+	{
+		String name = f.getName();
+		name = name.substring(0, name.length()- 11)+".txt"
+		f.renameTo(new File(f.getParent(), name))
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ParaContexts.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ParaContexts.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ParaContexts.groovy	(revision 321)
@@ -0,0 +1,24 @@
+package org.txm.test
+
+import java.util.ArrayList;
+import org.txm.functions.parallelcontexts.ParallelContexts
+import org.txm.searchengine.cqp.corpus.Corpus;
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+
+
+ArrayList<String> queries = ["je", "je"]
+String struct = "text_id"
+ArrayList<Corpus> corpora = ["QGRAAL", "QGRAAL"]
+ArrayList<String> props = ["word", "frlemma"]
+ArrayList<String> refs = ["text_id", "text_titre"]
+ArrayList<Integer> CGs = [30, 10]
+ArrayList<Integer> CDs = [10, 1]
+ArrayList<Boolean> participates = [true, true]
+
+ParallelContexts ctx = new ParallelContexts();
+
+println "starts-ends: "+ctx.getKeywordsStartsEnds()
+println "keywords: "+ctx.getKeywordsStrings()
+println "seg keys; "+ctx.getSegKeys()
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ProgressionSample.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ProgressionSample.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ProgressionSample.groovy	(revision 321)
@@ -0,0 +1,57 @@
+//package org.txm.test;
+//import org.txm.functions.progression.*
+//import org.txm.searchengine.cqp.corpus.*
+//import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.stat.engine.r.*
+//
+//def discours = CorpusManager.getCorpusManager().getCorpus("DISCOURS");
+//
+//def q1 = new Query(Query.fixQuery("je|j'"))
+//def q2 = new Query(Query.fixQuery("tu"))
+//def q3 = new Query(Query.fixQuery("il"))
+//def q4 = new Query(Query.fixQuery("elle"))
+//def q5 = new Query(Query.fixQuery("on"))
+//def q6 = new Query(Query.fixQuery("nous"))
+//def q7 = new Query(Query.fixQuery("vous"))
+//def q8 = new Query(Query.fixQuery("ils"))
+//def q9 = new Query(Query.fixQuery("elles"))
+//def q10 = new Query(Query.fixQuery("eux"))
+//def q11 = new Query(Query.fixQuery("ceux"))
+//def q12 = new Query(Query.fixQuery("autre|autres"))
+//def q13 = new Query(Query.fixQuery("monde"))
+//def q14 = new Query(Query.fixQuery("France"))
+//def q15 = new Query(Query.fixQuery("Algérie"))
+//def q16 = new Query(Query.fixQuery("Europe"))
+//
+////Progression(Corpus corpus, List<Query> queries,
+////			StructuralUnit structure, StructuralUnitProperty property, String propertyregex,
+////			boolean doCumulative, boolean monochrome, boolean monostyle, 
+////			int linewidth, boolean repeat, float bandemultiplier) {
+//		
+//int linewidth = 2;
+//float bandewidth = 1.0f
+//Progression progression5 = new Progression(discours, [q1, q2, q3, q4, q5],
+//	null, null,	"", 
+//	true, false, false, 
+//	linewidth, false, bandewidth);
+//___Progression progression10 = new Progression(discours, [q1, q2, q3, q4, q5, q6, q7, q8, q9, q10],
+//	null, null,	"", 
+//	true, false, false, 
+//	linewidth, false, bandewidth);
+//___Progression progression16 = new Progression(discours, [q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, q15, q16],
+//	null, null,	"", 
+//	true, false, false, 
+//	linewidth, false, bandewidth);
+//
+//File outputdir = new File("/home/mdecorde/Bureau")
+//File svgFile5 = new File(outputdir, "progression5");
+//File svgFile10 = new File(outputdir, "progression10");
+//File svgFile16 = new File(outputdir, "progression16");
+//
+//progression5.toSvg(svgFile5, __RDevice.SVG);
+//progression10.toSvg(svgFile10, __RDevice.SVG);
+//progression16.toSvg(svgFile16, __RDevice.SVG);
+//
+//OpenSVGGraph.OpenSVGFile(svgFile5.getAbsolutePath(), "progression5")
+//OpenSVGGraph.OpenSVGFile(svgFile10.getAbsolutePath(), "progression10")
+//OpenSVGGraph.OpenSVGFile(svgFile16.getAbsolutePath(), "progression16")
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/ControlUI.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/ControlUI.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/ControlUI.groovy	(revision 321)
@@ -0,0 +1,44 @@
+package org.txm.test
+
+//import org.txm.rcpapplication.views.*
+
+// Bindings
+println "The current job monitor: "+monitor
+println "The current editor: "+editor
+println "The current selection: "+selection
+
+// Interact with SWT Objects
+monitor.syncExec(new Runnable() {
+	public void run() {
+		//CorporaView.reload()
+		println "corpora view refreshed !"
+	}
+})
+
+// Move the progress bar and cancel if necessary
+println "waiting..."
+Thread.sleep(1000)
+monitor.worked(20)
+if (monitor.isCanceled()) {monitor.done(); return}
+
+println "waiting..."
+Thread.sleep(1000)
+monitor.worked(20)
+if (monitor.isCanceled()) {monitor.done(); return}
+
+println "waiting..."
+Thread.sleep(1000)
+monitor.worked(20)
+if (monitor.isCanceled()) {monitor.done(); return}
+
+println "waiting..."
+Thread.sleep(1000)
+monitor.worked(20)
+if (monitor.isCanceled()) {monitor.done(); return}
+
+println "waiting..."
+Thread.sleep(1000)
+monitor.worked(20)
+if (monitor.isCanceled()) {monitor.done(); return}
+
+println "Done."
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/groovy4editor.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/groovy4editor.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/groovy4editor.groovy	(revision 321)
@@ -0,0 +1,7 @@
+package org.txm.test;
+
+File currentFile = editor.getEditedFile()
+String text = currentFile.getText("UTF-8")
+currentFile.withWriter("UTF-8") { writer ->
+	writer.write(text.replaceAll("e", "E"));
+}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/SomeRCode.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/SomeRCode.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/SomeRCode.groovy	(revision 321)
@@ -0,0 +1,75 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package org.txm.test
+
+import org.txm.statsengine.r.core.RWorkspace
+
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+RWorkspace rw = RWorkspace.getRWorkspaceInstance();//get the RWorkspace instance
+rw.setLog(true);//start R log
+
+//execute a simple line of code
+rw.eval("print(2+2)")
+println "-"
+
+//execute a file
+/*
+ * script.R:
+a <- 2+2
+print(a)
+b <- 4+4
+print(b)
+c <- 6+6
+print(c)
+ */
+File scriptR = new File("script.R")// get a File
+String filepath = scriptR.getAbsolutePath().replace("\\","\\\\")//We need to to it for Windows OS
+rw.eval("source(\""+filepath+"\")")
+println "-"
+
+//execute a file line per line
+Reader reader = new FileReader(scriptR);// create a file reader
+String line = reader.readLine();// read next line
+while(line != null)//continue until the end of the file
+{
+	rw.eval(line)//evaluate line in R
+	line = reader.readLine();//read next line
+}
+println "-"
+
+println "Last log line : "+rw.getLastLogLine();// get the last log
+rw.setLog(false);//stop R log
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/Quotes.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/Quotes.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/Quotes.groovy	(revision 321)
@@ -0,0 +1,37 @@
+ package org.txm.test
+
+println "Pre-balisage du discours direct"
+
+List<String> containers = ["p","ab"]; // elements qui contiennent des <s>
+
+File infile = new File(System.getProperty("user.home"), "TXM-SRC/quote-orig/perceval2.xml")
+File outfile = new File(System.getProperty("user.home"), "TXM-SRC/quote-orig/perceval2-q.xml")
+
+def doc = new XmlParser().parse (infile)
+List<Node> nodesToInspect = doc.text.body.div
+// nodesToInspect << doc.text.body.div // on en rajoute
+
+/*************************/
+new org.txm.importer.EncodeTEIQuotes(nodesToInspect, containers, "\"", "pon");
+
+//copy the doc in "outfile" File  
+String encoding = "UTF-8"
+OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(outfile) , encoding);
+//writer.write("<?xml version=\"1.0\" encoding=\""+encoding+"\"?>");
+//writer.write ("<?xml-stylesheet type=\"text/css\" href=\"tei-graal.css\"?>\n")
+def pwriter = new PrintWriter(writer, true)
+XmlNodePrinter xmlwriter = new XmlNodePrinter(pwriter)
+xmlwriter.setPreserveWhitespace(false)
+xmlwriter.print(doc)
+pwriter.close()
+writer.close()
+xmlwriter = null
+//println "write output file "+outfile
+
+//update counts
+if(outfile.exists())
+{
+	String txtid = org.txm.importer.WordCounter.findTextId(infile, "s"); // retrouve l'id du text qui a été concaténé aux id des S originels (ex : s19_12 >> 19)
+	new org.txm.importer.WordCounter(outfile, "s", txtid);
+	new org.txm.importer.WordCounter(outfile, "q", txtid);
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/References.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/References.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/References.groovy	(revision 321)
@@ -0,0 +1,88 @@
+/**
+ * Main.
+ *
+ * @param args the args
+ */
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate: 2013-05-06 17:38:43 +0200 (lun., 06 mai 2013) $
+// $LastChangedRevision: 2386 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.test
+
+//we import the packages containing the functions we are going to use
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+import org.txm.functions.referencer.*
+import org.txm.functions.ReferencePattern
+
+// TODO: Auto-generated Javadoc
+/* (non-Javadoc)
+ * @see groovy.lang.Script#run()
+ */
+Corpus discours = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+
+// we get some properties
+Property word = discours.getProperty("word")
+StructuralUnit text = discours.getStructuralUnit("text")
+StructuralUnitProperty text_id = text.getProperty("id")
+
+// we create a query to find the positions
+Query query = new Query(Query.fixQuery("j.*"))
+
+// what the index must show
+ReferencePattern referencePattern = new ReferencePattern().addProperty(text_id)
+
+//define the referencer
+// We want the references of the words who matche the query "je" in the discours corpus
+long time = System.currentTimeMillis()
+Referencer ref = new Referencer(discours, query, word, referencePattern.getProperties())
+ref.compute()
+println "time get positions : "+(System.currentTimeMillis()-time)
+
+time = System.currentTimeMillis()
+def lines =ref.getLines()
+println "time get "+lines.size()+" lines : "+(System.currentTimeMillis()-time)
+
+File file = new File("ref.txt")
+ref.toTxt(file, "UTF-8")
+println "writen in : "+file.getAbsolutePath()
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/test/mesures.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/test/mesures.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/test/mesures.groovy	(revision 321)
@@ -0,0 +1,32 @@
+package org.txm.test
+
+import org.txm.functions.mesures.*;
+import org.txm.searchengine.cqp.corpus.Corpus;
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.searchengine.cqp.corpus.query.*
+//import org.txm.rcpapplication.commands.*;
+
+def corpus1 = CorpusManager.getCorpusManager().getCorpus("ALCESTE")
+def corpus2 = CorpusManager.getCorpusManager().getCorpus("DISCOURS")
+def corpus3 = CorpusManager.getCorpusManager().getCorpus("RENOUEE")
+Mesures mesures = new Mesures([corpus1, corpus2, corpus3]);
+mesures.add(new Magnitude('"je"%c expand to text', "word", false, Synthese.COUNTMATCHES));
+mesures.add(new Magnitude('"je"%c expand to text', "word", false, Synthese.SUM));
+mesures.add(new Magnitude('"je"%c expand to text', "word", true, Synthese.MEAN));
+mesures.add(new Magnitude('"je"%c expand to text', "word", true, Synthese.QUARTILE));
+mesures.add(new Proportion('"j.*"%c expand to text', "word", true, Synthese.COUNTMATCHES, '"j.*" expand to text', ['"j.....*"%c expand to text', '"j........*"%c expand to text']));
+mesures.add(new Proportion('"j.*"%c expand to text', "word", true, Synthese.SUM, '"j.*" expand to text', ['"j.....*"%c expand to text', '"j........*"%c expand to text']));
+mesures.add(new PresenceRate('"j.*" expand to text', "word", true, Synthese.SUM, '"je"'));
+mesures.add(new PresenceRate('"j.*" expand to text', "word", true, Synthese.MIN, '"je"'));
+
+File csvFile = new File("mesures.txt");
+mesures.export(csvFile, "UTF-8", "\t", "");
+
+monitor.syncExec(new Runnable() {
+	public void run() {
+	//	EditFile.openfile(csvFile);
+	}
+})
+
+mesures.prettyPrint();
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to remove attributes of XML-TEI files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/eliminateNAttributes.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/eliminateNAttributes.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/eliminateNAttributes/eliminateNAttributes.groovy	(revision 321)
@@ -0,0 +1,85 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.eliminateNAttributes;
+
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class eliminatingNAttributes.
+ */
+class eliminatingNAttributes extends Filter {
+	
+	/** The counter. */
+	int counter;
+	
+	/** The old. */
+	def old;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		counter = 0;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		println "Deleted $counter PART='N' attributes\n";
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment;
+		// Write your code here, but don't forget to write in the output
+		// ex : output.write("TheStringToWrite " + line );
+		// in the var line is the current line
+		old = line;
+		line = (line =~ / part=["']N['"]/).replaceAll("");
+		
+		if( old != line)
+			counter++;
+		
+		output.write(line+lineSeparator);
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Sample Filter</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/MinimalFilter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/MinimalFilter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/MinimalFilter/MinimalFilter.groovy	(revision 321)
@@ -0,0 +1,81 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.MinimalFilter;
+
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class MinimalFilter.
+ */
+class MinimalFilter extends Filter {
+	
+	/** The linecounter. */
+	int linecounter = 0;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		System.out.println("strat Minimal " + this);
+		return true;
+	}
+
+	/**
+	 * Sets the used param.
+	 *
+	 * @param args the args
+	 */
+	void SetUsedParam(Object[] args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+		System.out.println("readed lines : " + linecounter);
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter() {
+		linecounter++;
+		output.write(line + lineSeparator);
+		output.flush();
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to prepare the BFM tokenizer files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/OneTagPerLine.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/OneTagPerLine.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/OneTagPerLine/OneTagPerLine.groovy	(revision 321)
@@ -0,0 +1,104 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.OneTagPerLine;
+
+//Pre-processing extra-word tags (1)
+import org.txm.importer.filters.*;
+import java.util.regex.*;
+import org.txm.tokenizer.TokenizerClasses;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class OneTagPerLine.
+ */
+class OneTagPerLine extends Filter {
+	
+	/** The tag_all. */
+	String tag_all = TokenizerClasses.tag_all;
+	
+	/** The counterreg1. */
+	int counterreg1 = 0;
+	
+	/** The counterreg2. */
+	int counterreg2 = 0;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		boolean firstTest = true;
+
+		while(!(line ==~ /\A\s*\Z/))
+		{
+			if (( m = line =~ /^([^<]+)(.*)$/) )// trucs(<balise>
+			{
+				if(firstTest)
+				{
+					output.write(lineSeparator);
+					firstTest=false;
+				}
+				output.write(m[0][1]);
+				line = m[0][2];
+			}
+			else if ((m = line =~ /^($tag_all)(.*)$/)) {//
+				firstTest=false;
+				output.write("\n"+m[0][1]);
+				line = m[0][2];
+			}
+			else
+			{
+				println "Error in "+ line;
+				line = " ";
+			}
+		}
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Main filter of the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/test.xml
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/test.xml	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/test.xml	(revision 321)
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<text><!--Fichier XML de test du tokenizer XML de TXM. Chaque div contient un cas particulier que le tokenizer doit gérer-->
+    <div id="ignore element"></div>
+    <div id="delete element"></div>    
+    <div id="delete element and its content"></div>
+    <div id="replace newlines with spaces"></div>
+    <div id="delete Ctrl (C class) characters"></div>
+    <div id="split using whitespaces regexp"></div>
+    <div id="fclitics"></div>
+    <div id="elision"></div>
+    <div id="..."></div>
+    <div id="punct"></div>
+    <div id="other things"></div>
+</text>

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/test.xml
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TokenizerXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TokenizerXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TokenizerXml.groovy	(revision 321)
@@ -0,0 +1,398 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.Tokeniser;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.txm.objects.Base;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.tokenizer.TokenizerClasses;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class TokenizerXml.
+ */
+public class TokenizerXml {
+	
+	/** The word_tags. */
+	String word_tags = TokenizerClasses.word_tags;
+	
+	/** The intraword_tags. */
+	String intraword_tags = TokenizerClasses.intraword_tags;
+	
+	/** The punct_strong. */
+	def punct_strong = TokenizerClasses.punct_strong;
+	
+	/** The punct_all. */
+	def punct_all = TokenizerClasses.punct_all;
+	
+	/** The word_chars. */
+	def word_chars = TokenizerClasses.word_chars;
+	
+	/** The ignorable_tags. */
+	String ignorable_tags = "note"
+	
+	/** The outfile. */
+	File outfile;
+	
+	/** The infile. */
+	File infile;
+	
+	/** The writer. */
+	XMLStreamWriter writer;
+	FileOutputStream output;
+	
+	/** The parser. */
+	XMLStreamReader parser
+	
+	/** The localname. */
+	String localname;
+	
+	/** The prefix. */
+	String prefix;
+	
+	/**
+	 * Instantiates a new tokenizer xml.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public TokenizerXml(File infile, File outfile)
+	{
+		this.outfile = outfile;
+		this.infile = infile;
+	}
+	
+	/**
+	 * Fill infos.
+	 *
+	 * @param event the event
+	 * @return the java.lang. object
+	 */
+	public fillInfos(int event)
+	{
+		if (event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT) 
+		{
+			localname = parser.getLocalName();
+			prefix = parser.getPrefix();
+		}
+	}
+	
+	/**
+	 * Donothing.
+	 *
+	 * @param event the event
+	 * @return the java.lang. object
+	 */
+	public donothing(int event)
+	{
+		if (event == XMLStreamConstants.START_ELEMENT ) {
+			if(prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname);
+			else
+				writer.writeStartElement(localname);
+			String attrprefix;
+			for(int i = 0 ; i < parser.getAttributeCount() ; i++)
+			{
+				attrprefix = parser.getAttributePrefix(i);
+				if(attrprefix != null && attrprefix.length() > 0)
+					writer.writeAttribute(attrprefix+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+				else
+					writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+			}
+		}
+		else if(event == XMLStreamConstants.END_ELEMENT)
+		{
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.CHARACTERS)
+		{
+			writer.writeCharacters(parser.getText());
+		}
+	}
+	
+	/**
+	 * Process.
+	 */
+	public void process()
+	{
+		boolean ignorecontent = true;//tokenize a partir de <body>
+		int wordcount = 0;
+		
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		output = new FileOutputStream(outfile)
+		writer = factory.createXMLStreamWriter(output, "UTF-8")
+		
+		def inputData = infile.toURI().toURL().openStream();
+		def inputfactory = XMLInputFactory.newInstance();
+		parser = inputfactory.createXMLStreamReader(inputData);
+		
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 
+		{
+			fillInfos(event);
+			if (event == XMLStreamConstants.START_ELEMENT) 
+			{
+				//println "Open: "+localname;
+				if(localname == "text")
+				{
+					ignorecontent = false;
+					donothing(event);
+				}
+				if(localname.matches(word_tags))
+				{
+					//println "Found pretagged word";
+					donothing(event);
+					wordcount++;
+					if(parser.getAttributeValue(null, "id") == null)
+						writer.writeAttribute("id", "w_"+wordcount);
+					if(parser.getAttributeValue(null, "type") == null)
+						writer.writeAttribute("type","w");
+					ignorecontent = true;
+				}
+				else if(localname.matches(ignorable_tags))
+				{
+					//println "Found note";
+					donothing(event);
+					ignorecontent = true;
+				}
+				else
+				{
+					donothing(event);
+				}
+			}
+			else if(event == XMLStreamConstants.END_ELEMENT) 
+			{
+				//println "Close: "+localname;
+				if(localname == "w")
+				{
+					ignorecontent = false;
+					writer.writeEndElement();
+					writer.writeCharacters("\n");
+				}
+				else if(localname == "note")
+				{
+					ignorecontent = false;
+					writer.writeEndElement();
+					writer.writeCharacters("\n");
+				}
+				else
+				{
+					donothing(event);
+				}
+			}
+			else if(event == XMLStreamConstants.CHARACTERS) 
+			{
+				if(ignorecontent)
+				{	
+					//println " ignore chars: "+parser.getText().trim();
+					donothing(event);
+				}
+				else
+				{
+					//println " chars: "+parser.getText().trim();
+					String text = parser.getText().trim().replace("\t"," ");
+					text = text.replaceAll("\\p{C}", "");
+					for(String s : tokenize(text))
+					{
+						if (s.matches(/\.[^ .]+\./)) {
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeAttribute("type","num");
+							writer.writeCharacters(s);
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						} else if(s.matches(/$punct_all|['‘’]/))
+										{
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeAttribute("type","pon");
+							writer.writeCharacters(s);
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						}
+						else if(s.matches(/[\[\(\{].*[\]\)\}]/))
+						{
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeStartElement("supplied");
+							writer.writeAttribute("rend","crochets");
+							writer.writeCharacters(s);
+							writer.writeEndElement();
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						}
+						else if(s.contains("'") || s.contains("’"))
+						{
+							int sep = s.indexOf("'");
+							if(sep < 0)
+								sep = s.indexOf("’");
+
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeCharacters(s.substring(0,sep+1));
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+							
+							if(s.substring(sep+1).length() > 0)
+							{
+								wordcount++;	
+								writer.writeStartElement("w");
+								writer.writeAttribute("id","w_"+wordcount);
+								writer.writeCharacters(s.substring(sep+1));
+								writer.writeEndElement();
+								writer.writeCharacters("\n");
+							}
+						}
+						else if(s.matches(/($punct_all)(.*)/))
+						{
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeAttribute("type","pon");
+							writer.writeCharacters(s.substring(0, 1));
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+							
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeCharacters(s.substring(1));
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						}
+						else if(s.matches(/(.*)($punct_all)/))
+						{
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeCharacters(s.substring(0, s.length()-1));
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+							
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeAttribute("type","pon");
+							writer.writeCharacters(s.substring(s.length()-1, s.length()));
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						} else {
+							wordcount++;	
+							writer.writeStartElement("w");
+							writer.writeAttribute("id","w_"+wordcount);
+							writer.writeCharacters(s);
+							writer.writeEndElement();
+							writer.writeCharacters("\n");
+						}
+					}
+				}
+			}
+			else if(event == XMLStreamConstants.COMMENT) 
+			{
+				writer.writeComment(parser.getText())
+			}
+			else if(event == XMLStreamConstants.DTD) 
+			{
+				//println "DTD!";
+			}
+			else
+			{
+				
+			}
+		}
+		
+		parser.close()
+		writer.close();
+		output.close();
+		inputData.close();
+	}
+	
+	/**
+	 * Tokenize.
+	 *
+	 * @param str the str
+	 * @return the list
+	 */
+	public List<String> tokenize(String str)
+	{
+		return str.tokenize()//cut by whitespace
+	}
+	
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File infile = new File("~/xml/quete","qgraal_cm.xml");
+		File outfile = new File("~/xml/quete","qgraal_cm-out.xml");
+		TokenizerXml tokenizer = new TokenizerXml(infile, outfile)
+		tokenizer.process();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/OneTagPerLine.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/OneTagPerLine.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/OneTagPerLine.groovy	(revision 321)
@@ -0,0 +1,295 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $
+// $LastChangedRevision: 3400 $
+// $LastChangedBy: mdecorde $
+//
+package filters.Tokeniser;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.ArrayList;
+import javax.xml.stream.*;
+import java.net.URL;
+import org.txm.importer.filters.*;
+import java.net.URLEncoder;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Used by the tokenizer. put one tag per line
+ * 
+ * @author mdecorde
+ */
+
+public class OneTagPerLine {
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The factory. */
+	private def factory;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The output. */
+	private def output;
+
+	/** The solotags. */
+	ArrayList<String> solotags;
+
+	/**
+	 * Instantiates a new one tag per line.
+	 *
+	 * @param url the url
+	 * @param solotags the solotags
+	 */
+	public OneTagPerLine(URL url, ArrayList<String> solotags){
+		try {
+			this.url = url;
+			this.solotags = solotags;
+			inputData = url.openStream();
+			factory = XMLInputFactory.newInstance();
+			factory.setProperty(XMLInputFactory.IS_VALIDATING,false)
+			parser = factory.createXMLStreamReader(inputData);
+
+		} catch (XMLStreamException ex) {
+			System.out.println(ex);
+		}catch (IOException ex) {
+			System.out.println("IOException while parsing ");
+		}
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile) {
+		try {
+			File f = outfile;
+			output = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/**
+	 * Read next.
+	 *
+	 * @param n the n
+	 * @return the java.lang. object
+	 */
+	public def readNext(int n)
+	{
+		try
+		{
+			return parser.next()
+		}catch(Exception e){
+			if(n > 10)
+				return parser.next();
+			else
+				readNext(n+1)
+		}
+	}
+	boolean wordtag = false;
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile) {
+
+		StringBuffer chars = new StringBuffer();
+		if (createOutput(outfile)) {
+
+			String lastopenlocalname= "";
+			String localname = "";
+			boolean word_part = false;
+			output.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
+			try {
+				for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = readNext(0)) {
+
+					String prefix = parser.getPrefix();
+					if (prefix == null || prefix == "")
+						prefix = "";
+					else
+						prefix +=":";
+
+					switch (event) {
+						case XMLStreamConstants.START_ELEMENT:
+							if (chars.length() > 0) {
+								output.write(chars.substring(0, chars.length()-1));
+								chars = new StringBuffer();
+							}
+							localname = parser.getLocalName();
+							lastopenlocalname = localname;
+							word_part = false;
+							for (int i= 0 ; i < parser.getAttributeCount() ;i++ )
+								if (parser.getAttributeValue(i).contains("word_part"))
+									word_part = true;
+
+						/*if(word_part == true)
+						 output.write("<"+prefix+localname);
+						 else*/
+						//						if(word_part)
+						//							output.write("<"+prefix+localname);
+						//						else
+							if (wordtag)
+								output.write("<"+prefix+localname);
+							else
+								output.write("\n<"+prefix+localname);
+							for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {
+								String ns = parser.getAttributePrefix(i);
+								if (ns == null || ns == "")
+									output.write(" "+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i).replace("&","&amp; ").replace("<","&lt;")+"\"" );
+								else
+									output.write(" "+ns+":"+parser.getAttributeLocalName(i)+"=\""+parser.getAttributeValue(i).replace("&","&amp; ").replace("<","&lt;")+"\"" );
+
+							}
+
+							if (solotags.contains(localname))
+								output.write("/>");
+							else
+								output.write(">");
+
+						//						if(!word_part)
+							if (localname == "w") {
+								wordtag=true;
+							} else if(!wordtag) {
+								output.write("\n");
+							}
+							break;
+
+						case XMLStreamConstants.END_ELEMENT:
+							if (chars.length() > 0) {
+								output.write(chars.substring(0, chars.length()-1));
+								chars = new StringBuffer();
+							}
+
+							localname = parser.getLocalName();
+							switch (localname) {
+								default:
+									if(!solotags.contains(localname))
+									/*if(word_part) // see filter ReunitBrokenWords
+							 output.write("</"+prefix+localname+">");
+							 else*/
+									if (localname == "w") {
+										wordtag = false
+										output.write("</"+prefix+localname+">\n");
+									} else if (wordtag) {
+										output.write("</"+prefix+localname+">");
+									} else {
+										output.write("\n</"+prefix+localname+">\n");
+									}
+							}
+							break;
+
+						case XMLStreamConstants.CHARACTERS:
+							if (parser.getText().length() > 0) {
+								int start=0;
+								while (parser.getText().charAt(start) == "\n" || parser.getText().charAt(start) == " ")
+									if (start < parser.getText().length()-1) {
+										start++;
+									} else {
+										start++;
+										break;
+									}
+								//replace & by &amp; ; < by &lt; ; insecable space by space
+								chars.append(parser.getText().substring(start).replace("\t", " ").replace("&","&amp;").replace("<","&lt;").replace(" "," ").trim()+ " ");
+								//String text = URLEncoder.encode(parser.getText().substring(start))
+								//output.write(text);
+							}
+							break;
+					}
+				}
+				output.close();
+				parser.close();
+
+				Reader input = new InputStreamReader(new FileInputStream(outfile) , "UTF-8");
+				File temp = new File(outfile.getParent(),"temp");
+				output = new OutputStreamWriter(new FileOutputStream(temp) , "UTF-8");
+				String line = input.readLine();
+				while (line != null) {
+					if (line.length() > 0)
+						output.write(line+"\n");
+					line = input.readLine();
+				}
+				input.close();
+				output.close();
+
+				if (!(outfile.delete() && temp.renameTo(outfile))) println "Warning can't rename file "+temp+" to "+outfile
+			} catch (XMLStreamException ex) {
+				input.close();
+				output.close();
+				System.out.println(ex);
+				return false;
+			} catch (IOException ex) {
+				input.close();
+				output.close();
+				System.out.println("IOException while parsing " + inputData);
+				return false;
+			}
+		}
+		return true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+
+		String rootDir = "~/xml/Hyperprince/";
+		new File(rootDir + "/identity/").mkdir();
+
+		ArrayList<String> milestones = new ArrayList<String>();// the tags who
+		// you want them
+		// to stay
+		// milestones
+		milestones.add("tagUsage");
+		milestones.add("pb");
+		milestones.add("lb");
+
+		File srcfile = new File(rootDir, "Corpus-Hyperprince_2009-06-10.xml");
+		File resultfile = new File(rootDir,
+				"Corpus-Hyperprince_2009-06-10-T.xml");
+		println("prepare tokenizer file : " + srcfile + " to : " + resultfile);
+
+		def builder = new OneTagPerLine(srcfile.toURL(), milestones);
+		builder.process(resultfile);
+
+		return;
+	}
+
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TEITokenizer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TEITokenizer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TEITokenizer.groovy	(revision 321)
@@ -0,0 +1,264 @@
+package filters.Tokeniser;
+
+import java.net.URL;
+import java.util.regex.Pattern;
+
+import org.txm.importer.StaxIdentityParser;
+import org.txm.tokenizer.*;
+
+public class TEITokenizer extends StaxIdentityParser {
+
+	int DEBUG = 0;
+
+	enum STATES { WORD, TEXT, IGNORE, NOTE };
+	STATES state = STATES.IGNORE;
+	String STARTTAG = ".*"
+	String NOTETAG = ""
+	//String WORDTAG = TokenizerClasses.word_tags;
+
+	static String PONTEST = "[.;:,!?]";
+	static String TYPE = "type";
+	static String ID = "id"
+	static String PON = "pon"
+	static String NUM = "num"
+	static String ENDOFLINE = System.getProperty("line.separator");
+	def tests = TokenizerClasses.tests;
+
+	int firstElement = 1;
+	String filename = "";
+	int wordcount = 0;
+	String previousElem;
+
+	def word_tags = TokenizerClasses.word_tags;
+	def intraword_tags = TokenizerClasses.intraword_tags;
+	def punct_strong = TokenizerClasses.punct_strong;
+	def punct_all = TokenizerClasses.punct_all;
+	def word_chars = TokenizerClasses.word_chars;
+	def punct_quotes = TokenizerClasses.punct_quotes;
+
+	StringBuffer buffer = new StringBuffer();
+
+	public TEITokenizer(URL inputurl) {
+		super(inputurl);
+
+		this.filename = new File(inputurl.getFile()).getName();
+		int index = filename.indexOf(".");
+		if (index > 0)
+			filename = filename.substring(0, index);
+	}
+
+	protected void processBuffer() {
+		if (buffer.size() > 0) {
+			tokenize(buffer.toString());
+			buffer = new StringBuffer();
+		}
+	}
+
+	protected void processStartElement() {
+		//println "OPEN "+localname+" = "+buffer.toString()
+		if ((firstElement++) == 1) {
+			if (localname == "TEI") {
+				STARTTAG = "text"
+				NOTETAG = "note"
+			} else 
+				STARTTAG = localname
+		}
+		processBuffer();
+		switch (state) {
+			case STATES.IGNORE:
+				super.processStartElement();
+				if (localname.matches(STARTTAG)) {
+					//println "ENTER $STARTTAG"
+					state = STATES.TEXT;
+				}
+				break;
+			case STATES.TEXT:
+				if (localname.matches(NOTETAG)) {
+					super.processStartElement();
+					//println "ENTER $NOTETAG"
+					state = STATES.NOTE;
+				} else if (localname.matches(word_tags)) {
+					//println "ENTER $WORDTAG"
+					state = STATES.WORD;
+					processStartWordElement();
+				} else {
+					super.processStartElement();
+					writer.writeCharacters(ENDOFLINE);
+				}
+				break;
+			default: // note, comment
+				super.processStartElement();
+		}
+		previousElem = localname;
+	}
+
+	protected void processEndElement() {
+		//println "CLOSE "+localname+" = "+buffer.toString()
+		processBuffer();
+		switch (state) {
+			case STATES.NOTE:
+				super.processEndElement();
+				if (localname.matches(NOTETAG)) {
+					writer.writeCharacters(ENDOFLINE);
+					//println "EXIT $NOTETAG"
+					state = STATES.TEXT;
+				}
+				break;
+			case STATES.WORD:
+				super.processEndElement();
+				writer.writeCharacters(ENDOFLINE);
+				if (localname.matches(word_tags)) {
+					//println "EXIT $WORDTAG"
+					state = STATES.TEXT;
+				}
+				break;
+			case STATES.TEXT:
+				super.processEndElement();
+				writer.writeCharacters(ENDOFLINE);
+				if (localname.matches(STARTTAG)) {
+					//println "EXIT $STARTTAG"
+					state = STATES.IGNORE;
+				}
+				break;
+			default: // note, comment
+				super.processEndElement();
+		}
+		previousElem = localname;
+	}
+
+	protected void processStartWordElement() {
+		super.processStartElement();
+		wordcount++;
+	}
+
+	protected void tokenize(String text) {
+		if (DEBUG > 0) {println "buffer="+text}
+		text = text.replaceAll("\n", " ");
+		text = text.replaceAll("\\p{C}", "");						// remove ctrl characters
+		for (String s : text.split("[\\p{Z}\\p{C}]+") ) {
+			s = s.trim();
+			iterate(s);
+		}
+	}
+
+	/**
+	 * Iterate.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected iterate(String s)
+	{
+		while (s != null && s.length() > 0) {
+			if (DEBUG > 0){println "  > $s"}
+			s = standardChecks(s);
+		}
+	}
+
+	/**
+	 * Standard checks.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected standardChecks(String s)
+	{
+		if (DEBUG > 0){println "checks : "+s}
+		def m;
+
+		for (TTest test : tests) {
+			if ((m = s =~ test.regex)) {
+				if (DEBUG > 0){println "test : "+test.regex}
+				if (test.before > 0)
+					iterate(m[0][test.before])
+
+				if (test.hit > 0) {
+					DEBUG--
+					wordcount++;
+					writer.writeStartElement("w");
+					writeWordAttributes();// id
+					writer.writeAttribute("type",test.type);
+					writer.writeCharacters(m[0][test.hit]);
+					writer.writeEndElement();
+					writer.writeCharacters("\n");
+				}
+				if( test.after > 0)
+					return m[0][test.after];
+			}
+		}
+
+		if (DEBUG > 0) {println "Other : "+s}
+		//		def o = s.split("[\\p{Z}\\p{C}]+")
+		//		for (String sub : o) {
+		//			if (sub.length() == 0) continue
+		wordcount++;
+		writer.writeStartElement("w");
+		writeWordAttributes();// id
+		if (s.matches(/\p{P}/))
+			writer.writeAttribute("type","pon");
+		else
+			writer.writeAttribute("type","w");
+		writer.writeCharacters(s);
+		writer.writeEndElement();
+		writer.writeCharacters("\n");
+		//		}
+		return "";
+
+
+	}
+
+	/**
+	 * Write word attributes.
+	 *
+	 * @return the java.lang. object
+	 */
+	protected writeWordAttributes()
+	{
+		writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+		writer.writeAttribute("n",""+wordcount);
+	}
+
+	protected void processCharacters()
+	{
+		//println "TEXT: "+parser.getText()
+		switch (state) {
+			case STATES.TEXT:
+				buffer.append(parser.getText())
+			//tokenize();
+				break
+			default:
+				super.processCharacters();
+		}
+	}
+
+	public static void main(String[] args) {
+		long start = System.currentTimeMillis();
+		File dir = new File("/home/mdecorde/xml/txmrefman")
+		File infile = new File(dir, "TEI/refman.xml");
+		File outfile = new File(dir, "tokenization/refman-t3.xml");
+
+		TEITokenizer tokenizer = new TEITokenizer(infile.toURI().toURL());
+
+		if (tokenizer.process(outfile))
+			println "Done - OK"
+		else
+			println "Done - FAIL"
+
+		println "time: "+(System.currentTimeMillis()-start)/1000
+
+		for (File f : new File("/home/mdecorde/xml/txmrefman/tokenization").listFiles()) {
+			String text = f.getText("UTF-8")
+			f.withWriter("UTF-8") { writer ->
+				text = text.replaceAll(/ id="w_refman_.+" n="[^"]+"/, "")
+				text = text.replaceAll("xml:id=", "id=")
+				text = text.replaceAll("<w>", "<w type=\"w\">")
+//				text = text.replaceAll("</?(p|body|item|text|table|list|ref|anchor|cell|row|head|div|term|hi|index|figure|graphic|lb|note).+", "")
+//				text = text.replaceAll("\n[^<>\n]+\n", "\n")
+//				text = text.replaceAll("\n[^<>\n]+\n", "\n")
+//				text = text.replaceAll("\n[^<>\n]+\n", "\n")
+//				text = text.replaceAll("\n++", "\n")
+				writer.write(text)
+			}
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TTTokenizer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TTTokenizer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/TTTokenizer.groovy	(revision 321)
@@ -0,0 +1,290 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+
+
+package filters.Tokeniser
+import java.io.File;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+// TODO: Auto-generated Javadoc
+/**
+########################################################################
+#                                                                      #
+#  tokenization script for tagger preprocessing                        #
+#  Author: Helmut Schmid, IMS, University of Stuttgart                 #
+#          Serge Sharoff, University of Leeds                          #
+#  Groovy Translation: mdecorde                                        #
+#  Description:                                                        #
+#  - splits input text into tokens (one token per line)                #
+#  - cuts off punctuation, parentheses etc.                            #
+#  - disambiguates periods                                             #
+#  - preserves SGML markup                                             #
+#                                                                      #
+########################################################################
+*/
+class TTTokenizer {
+	
+	// characters which have to be cut off at the beginning of a word
+	
+	/** The P char. */
+	def PChar="\\[{(\\`\"‚„†‡‹‘’“”•–—›";
+	// characters which have to be cut off at the end of a word
+	
+	/** The F char. */
+	def FChar="\\]}'`\"),;:!?%‚„…†‡‰‹‘’“”•–—›";
+	// character sequences which have to be cut off at the beginning of a word
+	
+	/** The P clitic. */
+	def PClitic = "";
+	// character sequences which have to be cut off at the end of a word
+	
+	/** The F clitic. */
+	def FClitic = "";
+		
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File input = new File("/home/mdecorde/xml/tokenizer/test.xml");
+		File output = new File("/home/mdecorde/xml/tokenizer/test-t.xml");
+		
+		def tokenizer = new TTTokenizer("", null);
+		tokenizer.run(input, output)
+	}
+	
+	
+	/** The Token. */
+	def Token = [:];
+	
+	/**
+	 * Instantiates a new tT tokenizer.
+	 *
+	 * @param lang the lang
+	 * @param abbr the abbr
+	 */
+	public TTTokenizer(String lang, File abbr) {
+		switch(lang) {
+		case "en": // English
+			  FClitic = "'(s|re|ve|d|m|em|ll)|n\'t";
+			  break;
+		case "it": // Italian
+			  PClitic = '[dD][ae]ll\'|[nN]ell\'|[Aa]ll\'|[lLDd]\'|[Ss]ull\'|[Qq]uest\'|[Uu]n\'|[Ss]enz\'|[Tt]utt\''
+			  break;
+		case "fr": // French
+			  PClitic = '[dcjlmnstDCJLNMST]\'|[Qq]u\'|[Jj]usqu\'|[Ll]orsqu\'';
+			  FClitic = '-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mmes?|-m\'|-moi|-nous|-on|-toi|-tu|-t\'|-vous|-en|-y|-ci|-l';
+			  break;
+		}
+		
+		if(abbr != null && abbr.canRead()) {
+			Reader reader = new FileReader(abbr);
+			String line = reader.readLine();
+			while(line != null) {
+				line = line.replaceAll("^[ \t\r\n]+", "");
+				line = line.replaceAll("^[ \t\r\n]+\$", "");
+				if(!(line.matches("^#") || line.matches("\\s\$"))) // ignore comments
+				{
+					Token.put(line, 1);
+				}
+				line = reader.readLine();
+			}
+			reader.close();
+		}
+	}
+	
+	/** The m. */
+	def m;
+	
+	/**
+	 * Run.
+	 *
+	 * @param input the input
+	 * @param output the output
+	 * @return true, if successful
+	 */
+	public boolean run(File input, File output) {
+		if(!input.canRead()){
+			println "Can't read input file: "+input
+			return false;
+		}
+		if(!output.getParentFile().canWrite()){
+			println "Can't write out file: "+input
+			return false;
+		}
+		
+		Reader reader = new FileReader(input);
+		Writer writer = new FileWriter(output);
+		String line = reader.readLine();
+		while(line != null) 
+		{
+			println "line: "+line
+			line = line.replaceAll("\t"," "); // replace newlines and tab characters with blanks
+			line = line.replaceAll("\n"," "); // replace newlines and tab characters with blanks
+			
+			String replace = line.replaceAll(/(<[^<> ]*) ([^<>]*>)/, '$1ÿ$2');
+			while(replace != line) // replace blanks within SGML tags
+			{
+				line = replace;
+				replace = replace.replaceAll(/(<[^<> ]*) ([^<>]*>)/, '$1ÿ$2');
+			}
+			line = replace;
+			
+			line = line.replaceAll(/ /, "þ"); // replace whitespace with a special character
+			// tr/\377\376/ \377/; // 377=ÿ , 376=þ
+			line = line.replaceAll(/ÿ/, " "); // restore SGML tags
+			line = line.replaceAll(/þ/, " ÿ"); // restore SGML tags
+			//line = line.replaceAll(/ÿ/," ");
+			
+			// prepare SGML-Tags for tokenization
+			line = line.replaceAll(/(<[^<>]*>)/, 'ÿ$1ÿ');
+			line = line.replaceAll(/^ÿ/,"");
+			line = line.replaceAll(/ÿ$/,"");
+			line = line.replaceAll(/ÿÿÿ*/,"ÿ");
+			
+			String[] split1 = line.split("ÿ");
+			boolean finish = true;
+			String suffix = "";
+			for(String s : split1)
+			{
+				if(s =~ /^<.*>$/) // SGML tag
+				{
+					writer.write("$s\n");
+				}
+				else { // add a blank at the beginning and the end of each segment
+					s = " $s ";
+					// insert missing blanks after punctuation
+					s = s.replaceAll(/(\.\.\.)/, " ... ");
+					s = s.replaceAll(/([;\!\?])([^ ])/, '$1 $2');
+					s = s.replaceAll(/([.,:])([^ 0-9.])/, '$1 $2');
+
+					for(String s2 : s.split(" "))
+					{
+						
+						finish = false;
+						suffix = "";
+						// separate punctuation and parentheses from words
+						while(!finish) 
+						{
+							//println "suffix: "+suffix
+							finish = true;
+						// 	cut off preceding punctuation
+							if((m = (s2 =~ /^([$PChar])(.*)$/)))
+							{
+								s2 = m[0][2]
+								writer.write(m[0][1]+"\n")
+								finish = false;
+							}
+							
+						// 	cut off trailing punctuation
+							if((m = (s2 =~ /^(.*)([$FChar])$/)))
+							{
+								s2 = m[0][1];
+								suffix = m[0][2] + "\n$suffix";
+								finish = false;
+								
+								if(s2.length() == 0)
+									writer.write(suffix)
+							}
+						
+							// cut off trailing periods if punctuation precedes
+							if((m = (s2 =~ /([$FChar])\.$/))) {
+								suffix = ".\n$suffix";
+								if(s2.length() == 0)
+								{
+									 s2 = m[0][1];
+								}
+								else
+								{
+									suffix = m[0][1]+"\n$suffix"
+									s2 = "";
+								}
+								finish = false;
+							}
+						}// end while
+						
+						// handle explicitly listed tokens
+						if(Token.containsKey(s2))
+						{
+							writer.write("$s2\n$suffix");
+							continue;
+						}
+						
+						// abbreviations of the form A. or U.S.A.
+						if(s2 =~ /^([A-Za-z-]\.)+$/)	
+						{
+							writer.write("$s2\n$suffix");
+							continue;
+						}
+						
+						// disambiguate periods
+						if((m = (s2 =~ /^(..*)\.$/)) && 
+								(s2 !="...") && 
+								!( s ==~ /^[0-9]+\.$/))
+						{
+							s2 = m[0][1];
+							suffix = ".\n$suffix";
+							if(Token.containsKey(s2))
+							{
+								writer.write("$s2\n$suffix");
+								continue;
+							}
+						}
+						
+						// cut off clitics
+						if(PClitic.length() > 0)
+						{
+							while((m = (s2 =~ /^($PClitic)(.*)/)))
+							{
+								s2 = m[0][2];
+								writer.write(m[0][1] + "\n")
+							}
+						}
+
+						if(FClitic.length() > 0)
+						{
+							while((m = (s2 =~ /(.*)($FClitic)$/)))
+							{
+								s2 = m[0][1];
+								writer.write(m[0][2] + "\n")
+							}
+						}
+						if(s2.length() > 0)
+							writer.write("$s2\n$suffix");
+					}
+				}	
+			}
+			line = reader.readLine();
+		}
+		writer.close()
+		reader.close()
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/FastSimpleTokenizerXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/FastSimpleTokenizerXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/FastSimpleTokenizerXml.groovy	(revision 321)
@@ -0,0 +1,634 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package filters.Tokeniser;
+
+import static groovy.transform.TypeCheckingMode.SKIP
+import groovy.transform.CompileStatic
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+import javax.xml.stream.*
+
+import org.txm.importer.graal.PersonalNamespaceContext
+import org.txm.tokenizer.TokenizerClasses
+@CompileStatic
+public class FastSimpleTokenizerXml {
+
+	/** The word_tags. */
+	String word_tags = TokenizerClasses.word_tags;
+
+	/** The intraword_tags. */
+	String intraword_tags = TokenizerClasses.intraword_tags;
+
+	/** The punct_strong. */
+	def punct_strong = TokenizerClasses.punct_strong;
+
+	/** The punct_all. */
+	def punct_all = TokenizerClasses.punct_all;
+
+	/** The word_chars. */
+	def word_chars = TokenizerClasses.word_chars;
+	
+	def fclitics = null; // default behavior don't manage clitics
+	def pclitics = null; // default behavior don't manage clitics
+
+	/** The ignorable_tags. */
+	String ignorable_tags = "";
+	String startTag = null
+
+	/** The DEBUG. */
+	boolean DEBUG = false;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The infile. */
+	File infile;
+	
+	String lang;
+
+	/** The buffer. */
+	StringBuffer buffer;
+
+	/** The writer. */
+	XMLStreamWriter writer;
+	BufferedOutputStream output;
+	
+	/** The parser. */
+	XMLStreamReader parser
+
+	/** The localname. */
+	String localname;
+
+	/** The prefix. */
+	String prefix;
+	String filename;
+	
+	def regElision;
+	def reg3pts;
+	def regPunct;
+	def regFClitics = null ;
+	String whitespaces;
+	Pattern regLN;
+	Pattern regCTRL;
+	Pattern regSplitWhiteSpaces;
+	
+	public FastSimpleTokenizerXml(File infile, File outfile)
+	{
+		this(infile, outfile, "");
+	}
+	
+	/**
+	 * Instantiates a new simple tokenizer xml.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public FastSimpleTokenizerXml(File infile, File outfile, String lang)
+	{
+		this.lang = lang;
+		if (lang != null)
+		if (lang.startsWith("en")) {
+			fclitics = TokenizerClasses.FClitic_en
+		} else if (lang.startsWith("fr")) {
+			fclitics = TokenizerClasses.FClitic_fr
+			pclitics = TokenizerClasses.PClitic_fr
+		} else if (lang.startsWith("gl")) {
+			fclitics = TokenizerClasses.FClitic_gl
+		} else if (lang.startsWith("it")) {
+			pclitics = TokenizerClasses.PClitic_it
+		}
+		
+		this.outfile = outfile;
+		this.infile = infile;
+		this.filename = infile.getName();
+		int index = filename.indexOf(".");
+		if (index > 0) filename = filename.substring(0, index);
+		
+		String strRegElision = TokenizerClasses.regElision;
+		String strRegPunct = TokenizerClasses.regPunct;
+		regElision = ~/\A(.*?)(\p{L}++$strRegElision)(.*)\Z/
+		reg3pts = ~/\A(.*)(\.\.\.)(.*)\Z/
+		regPunct = ~/\A(.*)($strRegPunct)(.*)\Z/
+		if (fclitics != null)
+			regFClitics = ~/(.+)($fclitics)/
+		whitespaces = ~TokenizerClasses.whitespaces;
+		
+		regSplitWhiteSpaces = Pattern.compile(TokenizerClasses.whitespaces);
+		regLN = Pattern.compile("/\n/");
+		regCTRL = Pattern.compile("/\\p{C}/");
+		
+	}
+
+	/**
+	 * Fill infos.
+	 *
+	 * @param event the event
+	 * @return the java.lang. object
+	 */
+	public fillInfos(int event)
+	{
+		if (event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT)
+		{
+			localname = parser.getLocalName();
+			prefix = parser.getPrefix();
+		}
+	}
+
+	/**
+	 * Donothing.
+	 *
+	 * @param event the event
+	 * @param wordid the wordid
+	 * @return the java.lang. object
+	 */
+	public donothing(int event, Integer wordid)
+	{
+		if (event == XMLStreamConstants.START_ELEMENT ) {
+			String namespace = parser.getNamespaceURI();
+
+			localname = parser.getLocalName();
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname);
+			else
+			//				if(namespace != null)
+			//					writer.writeStartElement(namespace, localname);
+			//				else
+			writer.writeStartElement(localname);
+
+			//			if(parser.getNamespaceCount() > 0)
+			//				writer.writeDefaultNamespace(parser.getNamespaceURI(0))
+			//			for(int i = 1 ; i < parser.getNamespaceCount() ; i++)
+			//				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+
+			String namespace_prefix;
+			for (int i = 0 ; i<   parser.getNamespaceCount() ; i++) {
+				namespace_prefix = parser.getNamespacePrefix(i);
+				if ((namespace_prefix != null)&&   (namespace_prefix.length()>   0)) {
+					writer.writeNamespace(namespace_prefix, parser.getNamespaceURI(i));
+				} else {
+					writer.writeDefaultNamespace(parser.getNamespaceURI(i));
+				}
+			}
+
+			String attrprefix, attname;
+			boolean hasId = false;
+			boolean hasType = false
+			boolean hasN = false
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				attname = parser.getAttributeLocalName(i);
+				attrprefix = parser.getAttributePrefix(i);
+				if ("id".equals(attname)) hasId = true;
+				if ("type".equals(attname)) hasType = true;
+				if ("n".equals(attname)) hasN = true;
+
+				if(attrprefix != null && attrprefix.length() > 0)
+					writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))
+				else
+					writer.writeAttribute(attname, parser.getAttributeValue(i))
+			}
+
+			if (wordid != null && !hasId && localname == "w")
+				writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+
+			if (!hasType && localname == "w")
+				writer.writeAttribute("type", "w");
+				
+			if (!hasN && localname == "w")
+				writer.writeAttribute("n", ""+wordcount);
+
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.END_ELEMENT)
+		{
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.CHARACTERS)
+		{
+			//println parser.getText();
+			//writer.writeCharacters("𦟛");
+			
+			
+			// checks if the token starts with an high surrogate
+//			if(isHighSurrogate(parser.getText().charAt(0)))	{
+//				println "warning: invalid UTF-8 XML range, token " + parser.getText() + " has been replaced.";
+//				writer.writeCharacters("__invalidXMLChar__")
+//				//writer.writeCharacters("𦟛");
+//				println "high surrogate: " + Integer.toHexString((int)parser.getText().charAt(0)); 
+//				println "low surrogate: " + Integer.toHexString((int)parser.getText().charAt(1));
+//				int charSum = parser.getText().charAt(0) + parser.getText().charAt(1);
+//				println "char sum: " + charSum;
+//				println "test " + surrogatesPairToScalar(parser.getText().charAt(0), parser.getText().charAt(1));
+//				int scalar = surrogatesPairToScalar(parser.getText().charAt(0), parser.getText().charAt(1));
+//				//writer.writeCharacters(String.valueOf((char)112692));
+//				writer.writeCharacters("&#" + scalar + ";");
+//			}
+//			else
+			
+			
+			if (insideword) { // ensure there is not \t or \n in the word form value
+				writer.writeCharacters(parser.getText().trim().replace("\n", " ").replace("\t", " "));
+			} else {
+				writer.writeCharacters(parser.getText());
+			}
+			
+			
+			
+		}
+	}
+	
+	
+	/**
+	 * Converts the specified surrogates pair to scalar.
+	 * @param highSurrogate
+	 * @param lowSurrogate
+	 * @return
+	 */
+	public int surrogatesPairToScalar(char highSurrogate, char lowSurrogate)	{
+		return ((highSurrogate - 0xD800) * 0x400) + (lowSurrogate - 0xDC00) + 0x10000;
+	}
+	
+	/**
+	 * Checks if the specified character is an high/leading surrogate.
+	 * @param character
+	 * @return
+	 */
+	public boolean isHighSurrogate(char character)	{
+		return (character >= 0xD800 && character <= 0xDBFF);
+	}
+	
+	
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The ignorecontent. */
+	boolean ignorecontent = true;//tokenize a partir de <body>
+	boolean insideword = false;
+	/**
+	 * Process.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean process()
+	{
+		if (!infile.exists()) {
+			println "$infile does not exists"
+			return false;
+		}
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		output = new BufferedOutputStream(new FileOutputStream(outfile));
+		
+//		output = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(outfile), 2048 * 1024), "UTF-8");
+		
+		writer = factory.createXMLStreamWriter(output, "UTF-8")
+		writer.setNamespaceContext(new PersonalNamespaceContext());
+
+		def inputData = infile.toURI().toURL().openStream();
+		def inputfactory = XMLInputFactory.newInstance();
+		//inputfactory.setProperty("http://apache.org/xml/properties/input-buffer-size", new Integer(2048));
+		//inputfactory.setExpandEntityReferences(false);
+		parser = inputfactory.createXMLStreamReader(inputData, "UTF-8");
+		//println "PARSER: "+parser.getClass()
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeCharacters("\n");
+
+		int previousEvent = 0;
+		boolean startProcess = false;
+		if (startTag == null) // if no startTag specified we process from the start
+			startProcess = true;
+		buffer = new StringBuffer();
+		//println "process - start start tag: "+startTag+" startProcess: $startProcess"
+		ignorecontent = !startProcess;
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				if (!startProcess) {
+					if (event == XMLStreamConstants.START_ELEMENT) {
+						if (parser.getLocalName().matches(startTag)) {
+
+							startProcess = true
+							ignorecontent = false;
+						}
+					}
+					if (!startProcess) {
+						donothing(event, null);
+						continue;
+					}
+				}
+
+				if (previousEvent == XMLStreamConstants.CHARACTERS && previousEvent != event) {
+					processWord(); // tokenize
+					buffer = new StringBuffer();
+				}
+				fillInfos(event);//get localname and prefix
+				if (event == XMLStreamConstants.START_ELEMENT) {
+					//println "Open: "+localname;
+
+					if (localname.matches(word_tags)) // ignore the content of the word but keep counting
+					{
+						//println "Found pretagged word";
+						wordcount++;
+						donothing(event, wordcount);
+						ignorecontent = true;
+						insideword = true;
+					} else if (localname.matches(ignorable_tags)) // ignore the content of the file
+					{
+						donothing(event, null);
+						ignorecontent = true;
+					} else {
+						donothing(event, null);
+					}
+				} else if(event == XMLStreamConstants.END_ELEMENT) {
+					//println "Close: "+localname;
+					if (localname.matches(word_tags)) {
+						ignorecontent = false;
+						insideword = false;
+						writer.writeEndElement();
+						writer.writeCharacters("\n");
+					} else if (localname.matches(ignorable_tags)) // ignore the content of the file
+					{
+						ignorecontent = false;
+						donothing(event, null);
+
+					} else {
+						donothing(event, null);
+					}
+				} else if (event == XMLStreamConstants.CHARACTERS) {
+					if (ignorecontent) {
+						//println " ignore chars: "+parser.getText().trim();
+						donothing(event, null);
+					} else {
+						//println " process chars: "+parser.getText().trim();
+						buffer.append(parser.getText());
+						if (buffer.length() >= 128 && buffer.charAt(buffer.length()-1) == " ") {
+							processWord();
+							buffer = new StringBuffer();
+						}
+					}
+				} else if (event == XMLStreamConstants.COMMENT) {
+					writer.writeComment(parser.getText())
+				} else if (event == XMLStreamConstants.DTD) {
+					//println "DTD!";
+				} else {
+					if (DEBUG) println "Warning in $infile: ignore XML event at location "+parser.getLocation()
+				}
+				previousEvent = event;
+			}
+
+			parser.close()
+			writer.close();
+			output.close();
+			inputData.close();
+		} catch (Exception e) {
+			System.err.println("Error : "+infile);
+			e.printStackTrace();
+			return false;
+		}
+		return true;
+	}
+
+
+	/**
+	 * Process word.
+	 */
+	protected void processWord()
+	{
+		String text = buffer.toString();//parser.getText().trim().replace("\t", " ");
+		if (DEBUG) println "-- chars: "+text+"--";
+		
+		text = regLN.matcher(text).replaceAll(" ");
+		text = regCTRL.matcher(text).replaceAll("");						// remove ctrl characters
+		//for (String s : text.split(whitespaces) )			// separate with unicode white spaces
+		for (String s : regSplitWhiteSpaces.split(text))			// separate with unicode white spaces
+		{
+			if (DEBUG){println "process $s"}
+			iterate(s);
+		}
+	}
+
+	/**
+	 * Iterate.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected iterate(String s)
+	{
+		while (s != null && s.length() > 0) {
+			if (DEBUG){println "  > $s"}
+			s = standardChecks(s);
+		}
+	}
+
+
+	/**
+	 * Standard checks.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+//	@CompileStatic(SKIP)
+	protected String standardChecks(String s)
+	{
+		
+		Matcher m;
+		if (fclitics != null && (m = s =~ regFClitics) ) {
+//			println "CLITIC found: $s ->"+ m
+			if (m.group(1) != null && m.group(1).length() > 0) standardChecks(m.group(1));// process first part of the string
+			
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type", "w");
+			writer.writeCharacters(m.group(2));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+		}
+		else if ((m = s =~ regElision) )
+		{
+			iterate(m.group(1))
+
+			int sep = s.indexOf("'");
+			if (sep < 0)
+				sep = s.indexOf("’");
+			if (sep < 0)
+				sep = s.indexOf("‘");
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type", "w");
+			writer.writeCharacters(m.group(2));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m.group(3);
+		}
+		else if ((m = s =~ reg3pts) )
+		{
+			iterate(m.group(1))
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type","pon");
+			writer.writeCharacters("...");
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m.group(3);
+		}
+		else if ((m = s =~ regPunct) )
+		//else if((m = s =~ /\A(.*)($punct_all)(.*)\Z/) )
+		{
+			iterate(m.group(1));
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type","pon");
+			writer.writeCharacters(m.group(2));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m.group(3);
+		}
+		else
+		{
+			if(DEBUG){println "Other : "+s}
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			if(s.matches(/\p{P}/))
+				writer.writeAttribute("type","pon");
+			else
+				writer.writeAttribute("type","w");
+			writer.writeCharacters(s);
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return "";
+		}
+	}
+
+	/**
+	 * Write word attributes.
+	 *
+	 * @return the java.lang. object
+	 */
+	protected writeWordAttributes()
+	{
+		writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+		writer.writeAttribute("n",""+wordcount);
+	}
+
+	public void setStartTag(String tag)
+	{
+		this.startTag = tag;
+	}
+
+	/**
+	 * Tokenize.
+	 *
+	 * @param str the str
+	 * @return the list
+	 */
+	public List<String> tokenize(String str)
+	{
+		return str.tokenize()	// cut by whitespace
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		
+		
+		
+		// FIXME: tests UTF-8 XML ranges
+		
+		
+		
+		File inputDir = new File("C:/Tools/Textometrie/___corpus/xml/test_bug_import_chinois/")
+		File inputFile = new File(inputDir, "problem2.xml");
+
+		
+//		File inputDir = new File("/home/mdecorde/xml/SMS88Mil/out")
+//		File inputFile = new File(inputDir, "sms-sample.xml");
+		for (String lang : ["none", "en", "fr", null]) {
+			print "."
+			File outputFile = new File(inputDir, "test-${lang}.xml")
+			FastSimpleTokenizerXml tokenizer = new FastSimpleTokenizerXml(inputFile, outputFile, lang)
+			tokenizer.process();
+		}
+		
+		println ""
+		println "Done"
+		
+//		String lang = "en"
+//		File inDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant")
+//		File outDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant-t")
+//		outDir.deleteDir()
+//		outDir.mkdir()
+//		
+//		println "processing "+inDir.listFiles().size()+" files."
+//		for (def infile : inDir.listFiles()) {			
+//			if (!infile.getName().endsWith(".xml")) continue;
+//			
+//			print "."
+//			File outfile = new File(outDir, infile.getName())
+//			FastSimpleTokenizerXml tokenizer = new FastSimpleTokenizerXml(infile, outfile, lang)
+//			tokenizer.setDEBUG false
+//			tokenizer.process();
+//		}
+//		println ""
+//		println "Done"
+
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/Tokeniser.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/Tokeniser.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/Tokeniser.groovy	(revision 321)
@@ -0,0 +1,469 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package filters.Tokeniser;
+
+import org.txm.importer.filters.*;
+import org.txm.tokenizer.TokenizerClasses;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class Tokeniser.
+ */
+class Tokeniser extends Filter {
+
+	public Tokeniser(File text)
+	{
+		this.filename = text.getName();
+		int index = filename.indexOf(".");
+		if(index > 0)
+			filename = filename.substring(0, index);
+	}
+
+	static public int DEBUG = 0;
+	/** The charerrors. */
+	HashSet<String> charerrors = new HashSet<String>();
+	/*
+	 * Elements processed by the tokenizer:
+	 * 
+	 * DEFS: - w|abbr|num (word_tags) - c|ex|caes|choice|corr|sic|reg
+	 * (intraword_tags) - <[^>]+> (tag_all)
+	 * 
+	 * INPUT: OUTPUT: MODE: COMMENTS:
+	 * 
+	 * - &apos; ' BEGIN - <\/($word_tags)> </w>+NL word - --> -->+NL comment -
+	 * <\/($head_name)> </($head_name)>+NL head - <\/note> </note>-->+NL note -
+	 * <(head|p) [^>]*lang=['"]fr['"] ibid text Editor's title - <w(
+	 * [^>]*)?>\s*)($tag_all\s*) ibid+NL if not </w> text - <note[^>]*>
+	 * NL+<!--$note text Note found - <\/note> </note>-->+NL text - $tag_all
+	 * ibid+NL text Line with tags only - <!-- NL+<!-- text XML comment found -
+	 * --> -->+NL - <($word_tags)([^>]*)> <w type=\"$tag_name\"
+	 * $tag_atts>$word</w>+NL text Tagged words, punctuation marks,
+	 * abbreviations and numbers - $tag_all ibid+NL text Separated tags - \.[^
+	 * .]+\. <w type=\"num\">$word</w>+NL text Numbers and abbreviations
+	 * surrounded with dots - $punct_all|['‘’] <w type=\"pon\">$word</w>+NL text
+	 * Punctuation marks - ($word_chars)*[\(\[]($word_cha <w >$word</w> text
+	 * Words with brackets rs)+[\)\]]($word_chars)+|($wor
+	 * d_chars)+[\(\[]($word_chars)+[ \)\]])(\s+|$punct_all) -
+	 * (((<[^>]+>)*($word_chars)+(<[^ <w >$word</w> text Other word patterns
+	 * without elision >]+>)*)+)(\s+|$punct_all) -
+	 * (((<[^>]+>)*($word_chars)+(<[^ <w >$word</w> text Other word patterns
+	 * with  >]+>)*)+)['’]
+	 */
+
+	/** The counter. */
+	int counter = 1;
+
+	/** The m. */
+	def m;
+
+	/** The segment. */
+	def segment;
+
+	/** The note. */
+	def note;
+
+	/** The tag. */
+	def tag;
+
+	/** The tag_name. */
+	def tag_name;
+
+	/** The tag_atts. */
+	def tag_atts;
+
+	/** The starttag. */
+	def starttag;
+
+	/** The endtag. */
+	def endtag;
+
+	/** The word. */
+	def word;
+
+	/** The linetype. */
+	def linetype = "text"; // input processing state, can be : text, word, comment, head, note
+
+	/** The head_name. */
+	def head_name;
+
+	/** The tag_all. */
+	def tag_all = "<[^>]+>";
+
+	/** The word_tags. */
+	def word_tags = TokenizerClasses.word_tags;
+
+	/** The intraword_tags. */
+	def intraword_tags = TokenizerClasses.intraword_tags;
+
+	/** The punct_strong. */
+	def punct_strong = TokenizerClasses.punct_strong;
+
+	/** The punct_all. */
+	def punct_all = TokenizerClasses.punct_all;
+
+	/** The word_chars. */
+	def word_chars = TokenizerClasses.word_chars;
+	def punct_quotes = TokenizerClasses.punct_quotes;
+	String filename;
+	int wordcount = 1;
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args)
+	{
+		try {
+			word_tags = args.get("word_tags");
+			intraword_tags = args.get("intraword_tags");
+			punct_strong = args.get("punct_strong");
+			punct_all = args.get("punct_all");
+			word_chars = args.get("word_chars");
+		} catch(Exception e) {
+			System.err.println(e)
+			System.err.println("tokenizer needs an \"args\" map with the following elements :\n word_tags, intraword_tags, punct_strong, " +
+					"punct_all, word_chars")
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		// System.out.println("beginning of tokenizer processing");
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+		if (charerrors.size() > 0) {
+			System.out.print("Tokenizer unknown word chars in $filename: ");
+			println charerrors;
+			
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		String word;
+		// Write your code here, but don't forget to write to the output
+		// Example : output.print("TheStringToWrite " + line);
+		// The line variable contains the current line
+		segment = line+" ";
+		segment = segment.replace("&amp;", "&").replace("&apos;", "'").replace("&quot;","'");
+
+		if(DEBUG-- > 0)
+			println("Line: "+line);
+		while (true) {
+			if (DEBUG-- > 0)
+				println " segment: "+segment+"\n linetype: "+linetype;
+
+			if (linetype == "word") {
+				if(	(m = segment =~ /\A\s*(.*?)<\/($word_tags)>(.*)\Z/) )
+				{
+					segment = m[0][3];
+					linetype = "text";
+					output.print( m[0][1]+"</w>"+lineSeparator);
+					if (segment ==~ /\S/)
+					{
+						continue;
+					}
+				}
+				else
+				{
+					segment = (segment =~ /\A\s*/).replaceAll("");
+					segment = segment.trim();
+					output.print(segment);
+				}
+			}
+			else if (linetype == "comment")
+			{
+				if(	(m = segment =~ /\A\s*(.*?)-->(.*)\Z/))
+				{
+					linetype = "text";
+					output.print(m[0][1]+"-->"+lineSeparator);
+					//println(m[0][1]+"-->\n");
+					segment = m[0][2];
+					if(! (segment ==~ /^\s*$/))
+					{
+						continue;
+					}
+				}
+				else
+				{
+					segment = (segment =~ /\A\s*/).replaceAll("");
+					output.print(segment);
+					//println(segment);
+				}
+			}
+			else if (linetype == "head")
+			{
+				if(	(m = segment =~ /\A\s*(.*?)<\/($head_name)>\s*\Z/))
+				{
+					linetype = "text";
+					output.print(m[0][1]+"</$head_name>"+lineSeparator);
+					head_name = "";
+				}
+				else
+				{
+					output.print(segment);
+					//print "Editor's title: $segment\n";
+					//println("Editor's title: $segment\n");
+				}
+			}
+			else if (linetype == "note")
+			{
+				if(	(m = segment =~ /\A\s*(.*?)<\/note>/))
+				{
+					linetype = "text";
+					output.print(m[0][1]+"</note>-->"+lineSeparator);
+					//println(m[0][1]+"</note>\n");
+				}
+				else
+				{
+					segment = (segment =~ /\A\s*/).replaceAll("");
+					output.print(segment);
+					//println(segment);
+				}
+			}
+			else if (linetype == "text")
+			{
+				if(	(m = segment =~ /\A\s*<(head|p) [^>]*lang=['"]fr['"]/) )
+				{
+					if(DEBUG-- > 0)
+						println "  tag head|p: m01"+head_name;
+					head_name = m[0][1];
+					output.print(segment);
+					linetype = "head";
+				}
+				else if( (m = segment =~ /\A(\s*<w( [^>]*)?>\s*)($tag_all\s*)*\Z/))
+				{
+					String wordtag = m[0][1];
+					String tail = m[0][3].trim();
+					if(!wordtag.contains("id=\""))
+						wordtag ="<w id=\"w_"+filename+"_"+wordcount+"\""+ wordtag.substring(2);
+					wordcount++;
+					if(DEBUG-- > 0)
+						println "  tag w: m01"+head_name;
+					segment = tail;
+					if (segment == "</w>") {
+						println("\nEmpty word tag found, adding [???] content!\nsegment: '$wordtag[???]$tail'");
+						output.print(wordtag.trim()+"[???]"+tail.trim()+lineSeparator);
+					} else if( !(segment ==~ /<\/w>/)) {
+						linetype = "word";
+						output.print(wordtag.trim()+tail.trim());
+					}
+					else
+						output.print(wordtag.trim()+tail.trim()+lineSeparator);
+				}
+				else if( (m = segment =~ /\A\s*(<note[^>]*>)(.*)\Z/)){
+					if(DEBUG-- > 0)
+						println "  tag note: new seg="+m[0][2];
+					segment = m[0][2];
+					def note = m[0][1];
+					//println("Note found: $note");
+					output.print(lineSeparator+"<!--$note");
+					if( (m = segment =~ /\A(.*?)<\/note>(.*)\Z/)){
+						segment = m[0][2];
+						//println(m[0][1]+"</note>\n");
+						output.print(m[0][1]+"</note>-->"+lineSeparator);
+					}
+					else {
+						linetype = "note";
+						//println("$segment ");
+						output.print("$segment ");
+						segment = " ";
+					}
+				}
+				else if (segment ==~ /\A\s*($tag_all\s*)+\Z/)  {
+					//print "Line with tags only : $line\n";
+					if(DEBUG-- > 0)
+						println "  tag all: ";
+
+					output.print(segment+lineSeparator);
+				}
+				else if( (m = segment =~ /\A\s*(.*?)\s*\Z/)){
+
+
+					segment = m[0][1]+" ";
+					while(! (segment ==~ /\A\s*\Z/) )
+					{
+						if(DEBUG-- > 0)
+							println "  other: new seg="+segment;
+						if( (m = segment =~ /\A\s*<!--(.*)\Z/))
+						{
+							segment = m[0][1]+" ";
+							if(DEBUG-- > 0)
+								println("   comment: new seg="+segment);
+							output.print(lineSeparator+"<!--");
+							if( (m = segment =~ /\A(.*?)-->(.*)\Z/)) {
+								segment = m[0][2];
+								//println(m[0][1]+"-->"+lineSeparator);
+								output.print(m[0][1]+"-->"+lineSeparator);
+							}
+							else  {
+								linetype = "comment";
+								//println("$segment ");
+								output.print("$segment ");
+								segment = " ";
+							}
+						}
+						else if( (m = segment =~ /\A\s*<($word_tags)([^>]*)>(.*)\Z/)) {
+							if(DEBUG-- > 0)
+								println("   tag word: 1w="+m[0][1]+" 2="+m[0][2]+" 3seg="+m[0][3]);
+							segment = m[0][3];
+							tag_name = m[0][1];
+							tag_atts = m[0][2];
+							//print "Tagged words, punctuation marks, abbreviations and numbers : <$tag_name$tag_atts>\n";
+							output.print("<w");
+							if(!tag_atts.contains("id=\""))
+							{
+								tag_atts = " id=\"w_"+filename+"_"+wordcount+"\" ";
+							}
+							wordcount++;
+							if (tag_name ==~ /abbr|num/) {
+								output.print(" type=\"$tag_name\"");
+							}
+							output.print("$tag_atts>");
+							if( (m = segment =~ /\A(.*?)<\/($tag_name)>(.*)\Z/)) {
+								word = m[0][1];
+								segment = m[0][3];
+								word = (word =~ /^\s*(.*)\s*$/).replaceAll("\$1");
+								output.print("$word</w>"+lineSeparator);
+							}
+							else  {
+								linetype = "word";
+								output.print(segment);
+								segment = " ";
+							}
+						}
+						else if( (m = segment =~ /\A\s*($tag_all)\s+(.*)\Z/)) {
+							if(DEBUG-- > 0)
+								println "   Separated tags: 1w="+m[0][1]+" 2seg="+m[0][2];
+							segment = m[0][2];
+							tag = m[0][1];
+							output.print("$tag"+lineSeparator);
+						}
+						else if( (m = segment =~ /\A\s*([.·][^ .·]+[.·])(.*)\Z/))
+						{
+							if(DEBUG-- > 0)
+								println "   Numbers and abbreviations: 1w="+m[0][1]+" 2seg="+m[0][2];
+							segment = m[0][2];
+							word = m[0][1];
+							word = (word =~ /^\s*(.*)\s*$/).replaceAll("\$1");
+							output.print("<w id=\"w_"+filename+"_$wordcount\" type=\"num\">$word</w>"+lineSeparator);
+							wordcount++;
+							//push @numbers, $word;
+							//print "Number found: <w type=\"num\">$word</w>\n";
+						}
+						else if( (m = segment =~ /\A\s*($punct_all|[$punct_quotes])(.*)\Z/))
+						{
+							if(DEBUG-- > 0)
+								println "   Punctuation marks: 1w="+m[0][1]+" 2seg="+m[0][2]
+							segment = m[0][2];
+							word = m[0][1];
+							word = (word =~ /^\s*(.*)\s*$/).replaceAll("\$1");
+							word = word.replace("<", "&lt;");
+							output.print("<w id=\"w_"+filename+"_$wordcount\" type=\"pon\">"+word+"</w>"+lineSeparator);
+							wordcount++;
+						}
+						else if( (m = segment =~ /\A\s*(($word_chars)*[{\(\[]($word_chars)+[\)\]}]($word_chars)+|($word_chars)+[{\(\[]($word_chars)+[\)\]}])(\s+|$punct_all)(.*?)\Z/))						{
+							if(DEBUG-- > 0)
+								println "   Words with brackets: 1w="+m[0][1]+" 2="+m[0][2]+" 3="+m[0][3]+" 4="+m[0][4]+" 5="+m[0][5]+" 6="+m[0][6]+" 7seg="+m[0][7]+" 8seg="+m[0][8];
+							segment = m[0][7]+m[0][8];//"$7$8";
+							word = m[0][1];
+							word = (word =~ /^\s*(.*)\s*$/).replaceAll("\$1");
+							//push @w_brackets, $word;
+							def word_clean = word;
+							word_clean = (word_clean =~ /($tag_all)/).replaceAll("");
+							if (word_clean ==~ /[^0-9A-Za-zœÀ-ÿ'’\(\)\[\]–\-]/)
+							{	//System.err.println("Line $linecounter Error : " +word);
+								charerrors.add(word);
+							}
+							output.print("<w id=\"w_"+filename+"_$wordcount\">"+word+"</w>"+lineSeparator);
+							wordcount++;
+						}
+						else if( (m = segment =~ /\A\s*(((<[^>]+>)*($word_chars+)(<[^>]+>)*)+)(\s+|$punct_all)(.*?)\Z/))
+						{
+							segment = m[0][6]+m[0][7];//"$6$7";
+							word = m[0][1];
+
+							if(DEBUG-- > 0)
+							{
+								println "Other word without elision: 1w="+m[0][1]+" 2="+m[0][2]+" 3="+m[0][3]+" 4="+m[0][4]+" 5="+m[0][5]+" 6seg="+m[0][6]+" 7seg="+m[0][7];
+							}
+							def word_clean = word;
+							word_clean = (word_clean =~ /($tag_all)/).replaceAll("");
+							if (word_clean ==~ /[^0-9A-Za-zœÀ-ÿ'’()[\\]–\\-]/)
+							{
+								//System.err.println("Line $linecounter Error : " +word);
+								charerrors.add(word);
+							}
+							String towrite = "<w id=\"w_"+filename+"_$wordcount\">"+word.toString()+"</w>"+lineSeparator;
+							wordcount++;
+							output.print(towrite);
+						}
+						else if( (m = segment =~ /\A\s*(((<[^>]+>)*($word_chars)+(<[^>]+>)*)+)[$punct_quotes](.*?)\Z/))
+						{
+							if(DEBUG-- > 0)
+								println "   Other word patterns with elision: 1w="+m[0][1]+" 2="+m[0][2]+" 3="+m[0][3]+" 4="+m[0][4]+" 5="+m[0][5]+" 6seg="+m[0][6];
+							segment = m[0][6];
+							word = m[0][1]+"'";
+							def word_clean = word;
+							word_clean = (word_clean =~ /($tag_all)/).replaceAll("");
+							if (word_clean ==~ /[^0-9A-Za-zœÀ-ÿ'’\(\)\[\]–\-]/)                    			{
+								//System.err.println("Line $linecounter Error : " +word);
+								charerrors.add(word);
+							}
+							output.print("<w id=\"w_"+filename+"_$wordcount\">"+word+"</w>"+lineSeparator);
+							wordcount++;
+						}
+						else{
+							System.err.println("   Unknown word pattern $segment");
+							output.print("<note>Error! Unknown word pattern: $segment</note>"+lineSeparator);
+							segment = " ";
+						}
+					}
+				}
+				else
+				{
+					println(" line (?) : $segment \n");
+					System.exit(-1)
+				}
+			}
+			else
+			{
+				output.print(segment);
+			}
+			break;
+		}
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/OldSimpleTokenizerXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/OldSimpleTokenizerXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/OldSimpleTokenizerXml.groovy	(revision 321)
@@ -0,0 +1,617 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package filters.Tokeniser;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.txm.importer.graal.PersonalNamespaceContext;
+import org.txm.objects.Base;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+import javax.xml.stream.*;
+
+import java.net.URL;
+
+import org.txm.tokenizer.TokenizerClasses;
+
+public class OldSimpleTokenizerXml {
+
+	/** The word_tags. */
+	String word_tags = TokenizerClasses.word_tags;
+
+	/** The intraword_tags. */
+	String intraword_tags = TokenizerClasses.intraword_tags;
+
+	/** The punct_strong. */
+	def punct_strong = TokenizerClasses.punct_strong;
+
+	/** The punct_all. */
+	def punct_all = TokenizerClasses.punct_all;
+
+	/** The word_chars. */
+	def word_chars = TokenizerClasses.word_chars;
+
+	def fclitics = null; // default behavior don't manage clitics
+	def pclitics = null; // default behavior don't manage clitics
+
+	/** The ignorable_tags. */
+	String ignorable_tags
+	String outside_text_tags_keep_content
+	String outside_text_tags
+	String startTag = null
+
+	/** The DEBUG. */
+	public boolean DEBUG = false;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The infile. */
+	File infile;
+
+	String lang;
+
+	/** The buffer. */
+	StringBuffer buffer;
+
+	/** The writer. */
+	XMLStreamWriter writer;
+	BufferedOutputStream output;
+
+	/** The parser. */
+	XMLStreamReader parser
+
+	/** The localname. */
+	String localname;
+
+	/** The prefix. */
+	String prefix;
+	String filename;
+
+	def regElision;
+	def reg3pts;
+	def regPunct;
+	def regFClitics = null ;
+	def regPClitics = null ;
+	String whitespaces;
+
+	public OldSimpleTokenizerXml(File infile, File outfile)
+	{
+		this(infile, outfile, "");
+	}
+
+	/**
+	 * Instantiates a new simple tokenizer xml.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public OldSimpleTokenizerXml(File infile, File outfile, String lang)
+	{
+		this.lang = lang;
+		if (lang != null)
+			if (lang.startsWith("en")) {
+				fclitics = TokenizerClasses.FClitic_en
+			} else if (lang.startsWith("fr")) {
+				fclitics = TokenizerClasses.FClitic_fr
+				pclitics = TokenizerClasses.PClitic_fr
+			} else if (lang.startsWith("gl")) {
+				fclitics = TokenizerClasses.FClitic_gl
+			} else if (lang.startsWith("it")) {
+				pclitics = TokenizerClasses.PClitic_it
+			}
+
+		this.outfile = outfile;
+		this.infile = infile;
+		this.filename = infile.getName();
+		int index = filename.indexOf(".");
+		if (index > 0) filename = filename.substring(0, index);
+
+		String strRegElision = TokenizerClasses.regElision;
+		String strRegPunct = TokenizerClasses.regPunct;
+		regElision = /\A(.*?)(\p{L}++$strRegElision)(.*)\Z/
+		reg3pts = /\A(.*)(\.\.\.)(.*)\Z/
+		regPunct = /\A(.*)($strRegPunct)(.*)\Z/
+		if (fclitics != null)
+			regFClitics = /(.+)($fclitics)(.*)/
+		if (pclitics != null)
+			regPClitics = /(.+)(pclitics)(.*)/
+		whitespaces = TokenizerClasses.whitespaces;
+	}
+
+	/**
+	 * Fill infos.
+	 *
+	 * @param event the event
+	 * @return the java.lang. object
+	 */
+	public fillInfos(int event)
+	{
+		if (event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT)
+		{
+			localname = parser.getLocalName();
+			prefix = parser.getPrefix();
+		}
+	}
+
+	/**
+	 * Donothing.
+	 *
+	 * @param event the event
+	 * @param wordid the wordid
+	 * @return the java.lang. object
+	 */
+	public donothing(int event, Integer wordid)
+	{
+		if (event == XMLStreamConstants.START_ELEMENT ) {
+			String namespace = parser.getNamespaceURI();
+
+			localname = parser.getLocalName();
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname);
+			else
+				//				if(namespace != null)
+				//					writer.writeStartElement(namespace, localname);
+				//				else
+				writer.writeStartElement(localname);
+
+			//			if(parser.getNamespaceCount() > 0)
+			//				writer.writeDefaultNamespace(parser.getNamespaceURI(0))
+			//			for(int i = 1 ; i < parser.getNamespaceCount() ; i++)
+			//				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+
+			String namespace_prefix;
+			for (int i = 0 ; i<   parser.getNamespaceCount() ; i++) {
+				namespace_prefix = parser.getNamespacePrefix(i);
+				if ((namespace_prefix != null)&&   (namespace_prefix.length()>   0)) {
+					writer.writeNamespace(namespace_prefix, parser.getNamespaceURI(i));
+				} else {
+					writer.writeDefaultNamespace(parser.getNamespaceURI(i));
+				}
+			}
+
+			String attrprefix, attname;
+			boolean hasId = false;
+			boolean hasType = false
+			boolean hasN = false
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				attname = parser.getAttributeLocalName(i);
+				attrprefix = parser.getAttributePrefix(i);
+				if ("id".equals(attname)) hasId = true;
+				if ("type".equals(attname)) hasType = true;
+				if ("n".equals(attname)) hasN = true;
+
+				if(attrprefix != null && attrprefix.length() > 0)
+					writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))
+				else
+					writer.writeAttribute(attname, parser.getAttributeValue(i))
+			}
+
+			if (wordid != null && !hasId && localname == "w")
+				writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+
+			if (!hasType && localname == "w")
+				writer.writeAttribute("type", "w");
+
+			if (!hasN && localname == "w")
+				writer.writeAttribute("n", ""+wordcount);
+
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.END_ELEMENT)
+		{
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.CHARACTERS)
+		{
+			if (insideword) { // ensure there is not \t or \n in the word form value
+				writer.writeCharacters(parser.getText().trim().replace("\n", " ").replace("\t", " "));
+			} else {
+				writer.writeCharacters(parser.getText());
+			}
+		}
+	}
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The ignorecontent. */
+	boolean ignorecontent = true;//tokenize a partir de <body>
+	boolean insideword = false;
+	/**
+	 * Process.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean process()
+	{
+		if (!infile.exists()) {
+			println "$infile does not exists"
+			return false;
+		}
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		output = new BufferedOutputStream(new FileOutputStream(outfile))
+		writer = factory.createXMLStreamWriter(output, "UTF-8")
+		writer.setNamespaceContext(new PersonalNamespaceContext());
+
+		def inputData = infile.toURI().toURL().openStream();
+		def inputfactory = XMLInputFactory.newInstance();
+		//inputfactory.setProperty("http://apache.org/xml/properties/input-buffer-size", new Integer(2048));
+		//inputfactory.setExpandEntityReferences(false);
+		parser = inputfactory.createXMLStreamReader(inputData, "UTF-8");
+		//println "PARSER: "+parser.getClass()
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeCharacters("\n");
+
+		int previousEvent = 0;
+		boolean startProcess = false;
+		if (startTag == null) // if no startTag specified we process from the start
+			startProcess = true;
+		buffer = new StringBuffer();
+		//println "process - start start tag: "+startTag+" startProcess: $startProcess"
+		ignorecontent = !startProcess;
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				if (!startProcess) {
+					if (event == XMLStreamConstants.START_ELEMENT) {
+						if (parser.getLocalName().matches(startTag)) {
+
+							startProcess = true
+							ignorecontent = false;
+						}
+					}
+					if (!startProcess) {
+						donothing(event, null);
+						continue;
+					}
+				}
+
+				if (previousEvent == XMLStreamConstants.CHARACTERS && previousEvent != event) {
+					processWord(); // tokenize
+					buffer = new StringBuffer();
+				}
+				fillInfos(event);//get localname and prefix
+				if (event == XMLStreamConstants.START_ELEMENT) {
+					//println "Open: "+localname;
+
+					if (localname.matches(word_tags)) // ignore the content of the word but keep counting
+					{
+						//println "Found pretagged word";
+						wordcount++;
+						donothing(event, wordcount);
+						ignorecontent = true;
+						insideword = true;
+					} else if (outside_text_tags_keep_content != null && localname.matches(outside_text_tags_keep_content)) { // ignore the content of the tag ONLY
+						//just ignore the tag
+					} else if (ignorable_tags != null && localname.matches(ignorable_tags)) { // ignore the content of the tag ONLY
+						donothing(event, null);
+						ignorecontent = true;
+					} else if (outside_text_tags != null && localname.matches(outside_text_tags)) { // ignore the tag and its content of the tag
+						goToEndOfElement(localname);
+					} else {
+						donothing(event, null);
+					}
+				} else if(event == XMLStreamConstants.END_ELEMENT) {
+					//println "Close: "+localname;
+					if (localname.matches(word_tags)) {
+						ignorecontent = false;
+						insideword = false;
+						writer.writeEndElement();
+						writer.writeCharacters("\n");
+					} else if (ignorable_tags != null && localname.matches(ignorable_tags)) { // ignore the content of the tag
+						ignorecontent = false;
+						donothing(event, null);
+					} else if (outside_text_tags_keep_content != null && localname.matches(outside_text_tags_keep_content)) { // ignore the content of the tag ONLY
+						//just ignore the tag
+					} else {
+						donothing(event, null);
+					}
+				} else if (event == XMLStreamConstants.CHARACTERS) {
+					if (ignorecontent) {
+						//println " ignore chars: "+parser.getText().trim();
+						donothing(event, null);
+					} else {
+						//println " process chars: "+parser.getText().trim();
+						buffer.append(parser.getText());
+						if (buffer.length() >= 128 && buffer.charAt(buffer.length()-1) == " ") {
+							processWord();
+							buffer = new StringBuffer();
+						}
+					}
+				} else if (event == XMLStreamConstants.COMMENT) {
+					writer.writeComment(parser.getText())
+				} else if (event == XMLStreamConstants.DTD) {
+					//println "DTD!";
+				} else {
+					if (DEBUG) println "Warning in $infile: ignore XML event at location "+parser.getLocation()
+				}
+				previousEvent = event;
+			}
+
+			parser.close()
+			writer.close();
+			output.close();
+			inputData.close();
+		} catch (Exception e) {
+			System.err.println("Error : "+infile);
+			e.printStackTrace();
+			return false;
+		}
+		return true;
+	}
+
+	public void setOutSideElements(String regexp) {
+		this.outside_text_tags = regexp;
+	}
+
+	public void setOutSideTextTags(String regexp) {
+		this.outside_text_tags_keep_content = regexp;
+	}
+
+	protected void goToEndOfElement(String name) {
+		//println "START ignoring tag and content of $name"
+		def openedTags = []
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				openedTags << parser.getLocalName()
+				//println "append "+openedTags
+			} else if (event == XMLStreamConstants.END_ELEMENT) {
+				if (openedTags.size() == 0 && name == parser.getLocalName()) {
+					//println "END ignoring tag and content of $name"
+					return;
+				}
+				openedTags.pop()
+				//println "pop $openedTags"
+			}
+		}
+	}
+
+	/**
+	 * Process word.
+	 */
+	protected void processWord()
+	{
+		String text = buffer.toString();//parser.getText().trim().replace("\t", " ");
+		if (DEBUG) println "-- chars: "+text+"--";
+		text = text.replaceAll("\n", " ");
+		text = text.replaceAll("\\p{C}", "");						// remove ctrl characters
+		for (String s : text.split(whitespaces) )			// separate with unicode white spaces
+		{
+			if (DEBUG){println "process $s"}
+			iterate(s);
+		}
+	}
+
+	/**
+	 * Iterate. a String, should be called when a word is found in a String
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected iterate(String s)
+	{
+		while (s != null && s.length() > 0) {
+			if (DEBUG){println "  > $s"}
+			s = standardChecks(s);
+		}
+	}
+
+
+	/**
+	 * Standard checks.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected standardChecks(String s)
+	{
+		def m;
+		if (fclitics != null && (m = s =~ regFClitics) ) {
+			if (DEBUG) println "CLITIC found: $s ->"+ m[0][1]+" + "+m[0][2]+" + "+m[0][3]
+			if (m[0][1] != null && m[0][1].length() > 0) iterate(m[0][1]);// process first part of the string
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type", "w");
+			writer.writeCharacters(m[0][2]);
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m[0][3];
+		} else if ((m = s =~ regElision)) {
+			if (DEBUG) println "Elision found: $s ->"+ m[0][1]+" + "+m[0][2]+" + "+m[0][3]
+			iterate(m[0][1])
+
+			int sep = s.indexOf("'");
+			if (sep < 0)
+				sep = s.indexOf("’");
+			if (sep < 0)
+				sep = s.indexOf("‘");
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type", "w");
+			writer.writeCharacters(m[0][2]);
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m[0][3];
+		}
+		else if ((m = s =~ reg3pts) )
+		{
+			if(DEBUG){println "REG '...' found: $s -> "+m[0][1]+" + "+m[0][2]+" + "+m[0][3]}
+			iterate(m[0][1])
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type","pon");
+			writer.writeCharacters("...");
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m[0][3];
+		}
+		else if ((m = s =~ regPunct) )
+			//else if((m = s =~ /\A(.*)($punct_all)(.*)\Z/) )
+		{
+			if(DEBUG){println "PUNCT found: $s ->"+m[0][1]+" + "+m[0][2]+" + "+m[0][3]}
+			iterate(m[0][1]);
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			writer.writeAttribute("type","pon");
+			writer.writeCharacters(m[0][2]);
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m[0][3];
+		}
+		else
+		{
+			if(DEBUG){println "Other found: "+s}
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			if (s.matches(/\p{P}/))
+				writer.writeAttribute("type","pon");
+			else
+				writer.writeAttribute("type","w");
+			writer.writeCharacters(s);
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return "";
+		}
+	}
+
+	/**
+	 * Write word attributes.
+	 *
+	 * @return the java.lang. object
+	 */
+	protected writeWordAttributes()
+	{
+		writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+		writer.writeAttribute("n",""+wordcount);
+	}
+
+	public void setStartTag(String tag)
+	{
+		this.startTag = tag;
+	}
+
+	/**
+	 * Tokenize.
+	 *
+	 * @param str the str
+	 * @return the list
+	 */
+	public List<String> tokenize(String str)
+	{
+		return str.tokenize()	// cut by whitespace
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File inputDir = new File("/home/mdecorde/xml/annotation/")
+		File inputFile = new File(inputDir, "test.xml");
+		for (String lang : ["fr", null]) {
+			print "."
+			File outputFile = new File(inputDir, "test-${lang}.xml")
+
+			OldSimpleTokenizerXml tokenizer = new OldSimpleTokenizerXml(inputFile, outputFile, lang)
+			tokenizer.DEBUG = true
+			tokenizer.process();
+		}
+
+		println ""
+		println "Done"
+
+		//		String lang = "en"
+		//		File inDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant")
+		//		File outDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant-t")
+		//		outDir.deleteDir()
+		//		outDir.mkdir()
+		//
+		//		println "processing "+inDir.listFiles().size()+" files."
+		//		for (def infile : inDir.listFiles()) {
+		//			if (!infile.getName().endsWith(".xml")) continue;
+		//
+		//			print "."
+		//			File outfile = new File(outDir, infile.getName())
+		//			OldSimpleTokenizerXml tokenizer = new OldSimpleTokenizerXml(infile, outfile, lang)
+		//			tokenizer.setDEBUG false
+		//			tokenizer.process();
+		//		}
+		//		println ""
+		//		println "Done"
+
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/SimpleTokenizerXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/SimpleTokenizerXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Tokeniser/SimpleTokenizerXml.groovy	(revision 321)
@@ -0,0 +1,748 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+//
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+//
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+//
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package filters.Tokeniser;
+
+import static groovy.transform.TypeCheckingMode.SKIP
+import groovy.transform.CompileStatic
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+import javax.xml.stream.*
+
+import org.txm.importer.graal.PersonalNamespaceContext
+import org.txm.tokenizer.TokenizerClasses
+
+@CompileStatic
+public class SimpleTokenizerXml {
+
+	/** The word_tags. */
+	String word_tags = TokenizerClasses.word_tags;
+	Pattern reg_word_tags;
+	/** The intraword_tags. */
+	String intraword_tags = TokenizerClasses.intraword_tags;
+
+	/** The punct_strong. */
+	String punct_strong = TokenizerClasses.punct_strong;
+
+	/** The punct_all. */
+	String punct_all = TokenizerClasses.punct_all;
+
+	/** The word_chars. */
+	String word_chars = TokenizerClasses.word_chars;
+
+	String fclitics = null; // default behavior don't manage clitics
+	String pclitics = null; // default behavior don't manage clitics
+
+	/** The outside_text_tags_ignore_content. */
+	String note_content = null;
+	String outside_text_tags_keep_content = null // tag and content NOT removed but not tokenized
+	String outside_text_tags = null // tag and content removed
+	String startTag = null
+	Pattern reg_note_content;
+	Pattern reg_outside_text_tags_keep_content; 
+	Pattern reg_outside_text_tags;
+	Pattern reg_startTag;
+	
+	Pattern reg_punct_other = Pattern.compile("\\p{P}")
+
+	/** The DEBUG. */
+	public boolean DEBUG = false;
+
+	/** The outfile. */
+	File outfile;
+
+	/** The infile. */
+	File infile;
+
+	String lang;
+
+	/** The buffer. */
+	StringBuffer buffer;
+
+	/** The writer. */
+	XMLStreamWriter writer;
+	BufferedOutputStream output;
+
+	/** The parser. */
+	XMLStreamReader parser
+
+	/** The localname. */
+	String localname;
+
+	/** The prefix. */
+	String prefix;
+	String filename;
+
+	def regElision = null;
+	def reg3pts = null;
+	def regPunct;
+	def regFClitics = null ;
+	def regPClitics = null ;
+	String whitespaces;
+	Pattern regWhitespaces;
+	Pattern regLN;
+	Pattern regCTRL;
+	Pattern regSplitWhiteSpaces;
+
+	public SimpleTokenizerXml(File infile, File outfile) {
+		this(infile, outfile, "");
+	}
+
+	/**
+	 * Instantiates a new simple tokenizer xml.
+	 *
+	 * @param infile the infile
+	 * @param outfile the outfile
+	 */
+	public SimpleTokenizerXml(File infile, File outfile, String lang) {
+		this.lang = lang;
+		if (lang != null)
+			if (lang.startsWith("en")) {
+				fclitics = TokenizerClasses.FClitic_en
+			} else if (lang.startsWith("fr")) {
+				fclitics = TokenizerClasses.FClitic_fr
+				pclitics = TokenizerClasses.PClitic_fr
+			} else if (lang.startsWith("gl")) {
+				fclitics = TokenizerClasses.FClitic_gl
+			} else if (lang.startsWith("it")) {
+				pclitics = TokenizerClasses.PClitic_it
+			}
+
+		word_tags = TokenizerClasses.word_tags;
+		reg_word_tags = Pattern.compile(word_tags);
+		intraword_tags = TokenizerClasses.intraword_tags;
+		punct_strong = TokenizerClasses.punct_strong;
+		punct_all = TokenizerClasses.punct_all;
+		word_chars = TokenizerClasses.word_chars;
+
+		this.outfile = outfile;
+		this.infile = infile;
+		this.filename = infile.getName();
+		int index = filename.indexOf(".");
+		if (index > 0) filename = filename.substring(0, index);
+
+		String strRegElision = TokenizerClasses.regElision;
+		if (strRegElision != null && strRegElision.length() > 0)
+			regElision = ~/^([\p{L}-]++${strRegElision}[\p{L}-]++)(.*)$/ // the test must start with the start of string  ^
+		reg3pts = ~/^(.*?)(\.\.\.)(.*)$/
+		
+		String strRegPunct = TokenizerClasses.regPunct;
+		if (strRegPunct != null && strRegPunct.length() > 0)
+			regPunct = ~/^(.*?)($strRegPunct)(.*)$/
+		
+		if (fclitics != null && fclitics.length() > 0)
+			regFClitics = ~/(.+)($fclitics)$/ // the test must end with the end of string $
+			
+		if (pclitics != null && pclitics.length() > 0)
+			regPClitics = /^($pclitics)(.*)/ // the test must start with the start of string  ^
+			
+		if (TokenizerClasses.whitespaces != null && TokenizerClasses.whitespaces.length() > 0)
+			regWhitespaces = ~TokenizerClasses.whitespaces;
+
+		if (TokenizerClasses.whitespaces != null && TokenizerClasses.whitespaces.length() > 0)
+			regSplitWhiteSpaces = Pattern.compile(TokenizerClasses.whitespaces);
+			
+		regLN = Pattern.compile("/\n/");
+		regCTRL = Pattern.compile("/\\p{C}/");
+	}
+
+	/**
+	 * Fill infos.
+	 *
+	 * @param event the event
+	 * @return the java.lang. object
+	 */
+	public fillInfos(int event) {
+		if (event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT) {
+			localname = parser.getLocalName();
+			prefix = parser.getPrefix();
+		}
+	}
+
+	/**
+	 * Donothing: just write what is read
+	 * 
+	 * special processing for words: prefix id attributes with "w_" and remove special characters like \n \t etc.
+	 *
+	 * @param event the event
+	 * @param wordid the wordid
+	 * @return the java.lang. object
+	 */
+	public donothing(int event, Integer wordid) {
+		if (event == XMLStreamConstants.START_ELEMENT ) {
+			localname = parser.getLocalName();
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname);
+			else
+				//				if(namespace != null)
+				//					writer.writeStartElement(namespace, localname);
+				//				else
+				writer.writeStartElement(localname);
+
+			//			if(parser.getNamespaceCount() > 0)
+			//				writer.writeDefaultNamespace(parser.getNamespaceURI(0))
+			//			for(int i = 1 ; i < parser.getNamespaceCount() ; i++)
+			//				writer.writeNamespace(parser.getNamespacePrefix(i), parser.getNamespaceURI(i));
+
+			String namespace_prefix;
+			for (int i = 0 ; i<   parser.getNamespaceCount() ; i++) {
+				namespace_prefix = parser.getNamespacePrefix(i);
+				if ((namespace_prefix != null)&&   (namespace_prefix.length()>   0)) {
+					writer.writeNamespace(namespace_prefix, parser.getNamespaceURI(i));
+				} else {
+					writer.writeDefaultNamespace(parser.getNamespaceURI(i));
+				}
+			}
+
+			String attrprefix, attname;
+			boolean hasId = false;
+			//boolean hasType = false
+			boolean hasN = false
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				attname = parser.getAttributeLocalName(i);
+				attrprefix = parser.getAttributePrefix(i);
+				if ("id".equals(attname)) hasId = true;
+				//if ("type".equals(attname)) hasType = true;
+				if ("n".equals(attname)) hasN = true;
+
+				if (attrprefix != null && attrprefix.length() > 0)
+					writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))
+				else
+					writer.writeAttribute(attname, parser.getAttributeValue(i))
+			}
+
+			if (wordid != null && !hasId && localname == "w")
+				writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+
+			//if (!hasType && localname == "w") writer.writeAttribute("type", "w");
+
+			if (!hasN && localname == "w")
+				writer.writeAttribute("n", ""+wordcount);
+
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.END_ELEMENT)
+		{
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+		}
+		else if(event == XMLStreamConstants.CHARACTERS)
+		{
+			//println parser.getText();
+			//writer.writeCharacters("𦟛");
+
+
+			// checks if the token starts with an high surrogate
+			//			if(isHighSurrogate(parser.getText().charAt(0)))	{
+			//				println "warning: invalid UTF-8 XML range, token " + parser.getText() + " has been replaced.";
+			//				writer.writeCharacters("__invalidXMLChar__")
+			//				//writer.writeCharacters("𦟛");
+			//				println "high surrogate: " + Integer.toHexString((int)parser.getText().charAt(0));
+			//				println "low surrogate: " + Integer.toHexString((int)parser.getText().charAt(1));
+			//				int charSum = parser.getText().charAt(0) + parser.getText().charAt(1);
+			//				println "char sum: " + charSum;
+			//				println "test " + surrogatesPairToScalar(parser.getText().charAt(0), parser.getText().charAt(1));
+			//				int scalar = surrogatesPairToScalar(parser.getText().charAt(0), parser.getText().charAt(1));
+			//				//writer.writeCharacters(String.valueOf((char)112692));
+			//				writer.writeCharacters("&#" + scalar + ";");
+			//			}
+			//			else
+
+
+			if (insideword) { // ensure there is not \t or \n in the word form value
+				writer.writeCharacters(parser.getText().trim().replace("\n", " ").replace("\t", " "));
+			} else {
+				writer.writeCharacters(parser.getText());
+			}
+		}
+	}
+
+
+	/**
+	 * Converts the specified surrogates pair to scalar.
+	 * @param highSurrogate
+	 * @param lowSurrogate
+	 * @return
+	 */
+	public int surrogatesPairToScalar(char highSurrogate, char lowSurrogate)	{
+		return ((highSurrogate - 0xD800) * 0x400) + (lowSurrogate - 0xDC00) + 0x10000;
+	}
+
+	/**
+	 * Checks if the specified character is an high/leading surrogate.
+	 * @param character
+	 * @return
+	 */
+	public boolean isHighSurrogate(char character)	{
+		return (character >= 0xD800 && character <= 0xDBFF);
+	}
+
+
+
+	/** The wordcount. */
+	int wordcount = 0;
+
+	/** The ignorecontent. */
+	boolean ignorecontent = true; // tokenization starts with element name matching $reg_startTag
+	boolean insideword = false;
+	/**
+	 * Process.
+	 *
+	 * @return true, if successful
+	 */
+	public boolean process()
+	{
+		if (!infile.exists()) {
+			println "$infile does not exists"
+			return false;
+		}
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		output = new BufferedOutputStream(new FileOutputStream(outfile))
+		writer = factory.createXMLStreamWriter(output, "UTF-8")
+		writer.setNamespaceContext(new PersonalNamespaceContext());
+
+		def inputData = infile.toURI().toURL().openStream();
+		def inputfactory = XMLInputFactory.newInstance();
+		//inputfactory.setProperty("http://apache.org/xml/properties/input-buffer-size", new Integer(2048));
+		//inputfactory.setExpandEntityReferences(false);
+		parser = inputfactory.createXMLStreamReader(inputData, "UTF-8");
+		//println "PARSER: "+parser.getClass()
+		writer.writeStartDocument("UTF-8","1.0");
+		writer.writeCharacters("\n");
+
+		int previousEvent = 0;
+		boolean startProcess = false;
+		if (startTag == null) // if no startTag specified we process from the start
+			startProcess = true;
+		ignorecontent = !startProcess;
+		
+		buffer = new StringBuffer();
+		//println "process - start start tag: "+startTag+" startProcess: $startProcess"
+//		println "reg_outside_text_tags_keep_content=$reg_outside_text_tags_keep_content"
+//		println "reg_outside_text_tags=$reg_outside_text_tags"
+//		println "reg_note_content=$reg_note_content"
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				if (!startProcess) {
+					if (event == XMLStreamConstants.START_ELEMENT) {
+						if (reg_startTag.matcher(parser.getLocalName()).matches()) {
+							startProcess = true
+							ignorecontent = false;
+						}
+					}
+					if (!startProcess) {
+						donothing(event, null);
+						continue;
+					}
+				}
+
+				if (previousEvent == XMLStreamConstants.CHARACTERS && previousEvent != event) {
+					processWord(); // tokenize now!
+					buffer.setLength(0);
+				}
+				fillInfos(event);//get localname and prefix
+				if (event == XMLStreamConstants.START_ELEMENT) {
+					//println "Open: "+localname;
+					localname = parser.getLocalName()
+					if (reg_word_tags.matcher(localname).matches()) { // ignore the content of the word but keep counting
+						//println "Found pretagged word";
+						wordcount++;
+						donothing(event, wordcount);
+						ignorecontent = true;
+						insideword = true;
+					} else if (reg_note_content != null && reg_note_content.matcher(localname).matches()) { // ignore the content of the tag ONLY
+						// ignore the tag only
+						donothing(event, null); // write the tag
+						//println "IGNORING NOTE CONTENT OF "+localname
+						ignorecontent = true;
+					} else if (reg_outside_text_tags_keep_content != null && reg_outside_text_tags_keep_content.matcher(localname).matches()) { // ignore the content of the tag ONLY
+						// 	ignore the content only
+						donothing(event, null); // write the tag
+						//println "IGNORING CONTENT OF "+localname
+						ignorecontent = true;
+					} else if (reg_outside_text_tags != null && reg_outside_text_tags.matcher(localname).matches()) { // ignore the tag and its content of the tag
+						goToEndOfElement(localname); // parse until the end of the element is passed
+					} else {
+						donothing(event, null);
+					}
+				} else if(event == XMLStreamConstants.END_ELEMENT) {
+					//println "Close: "+localname;
+					localname = parser.getLocalName()
+					if (reg_word_tags.matcher(localname).matches()) {
+						//ignorecontent = false;
+						insideword = false;
+						writer.writeEndElement();
+						writer.writeCharacters("\n");
+					} else if (reg_outside_text_tags_keep_content != null && reg_outside_text_tags_keep_content.matcher(localname).matches()) { // ignore the content of the tag
+						ignorecontent = false;
+						donothing(event, null);
+					} else if (reg_note_content != null && reg_note_content.matcher(localname).matches()) { // ignore the content of the tag ONLY
+						ignorecontent = false;
+						donothing(event, null);
+					} else {
+						donothing(event, null);
+					}
+				} else if (event == XMLStreamConstants.CHARACTERS) {
+					if (ignorecontent) {
+						//println " dont tokenize chars: "+parser.getText().trim();
+						donothing(event, null);
+					} else {
+						//println " process chars: "+parser.getText().trim();
+						buffer.append(parser.getText());
+						if (buffer.length() >= 128 && buffer.charAt(buffer.length()-1) == " ") {
+							processWord();
+							buffer = new StringBuffer();
+						}
+					}
+				} else if (event == XMLStreamConstants.COMMENT) {
+					writer.writeComment(parser.getText())
+				} else if (event == XMLStreamConstants.DTD) {
+					//println "DTD!";
+				} else {
+					if (DEBUG) println "Warning in $infile: ignore XML event at location "+parser.getLocation()
+				}
+				previousEvent = event;
+			}
+
+			parser.close()
+			writer.close();
+			output.close();
+			inputData.close();
+		} catch (Exception e) {
+			System.err.println("Error : "+infile);
+			e.printStackTrace();
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Set the element and content to ignore
+	 * 
+	 * @param regexp
+	 */
+	public void setOutSideTextTags(String regexp) {
+		this.outside_text_tags = regexp;
+		this.reg_outside_text_tags = Pattern.compile(outside_text_tags);
+	}
+	
+	/**
+	 * Set element content to NOT tokenize
+	 *
+	 * @param regexp
+	 */
+	public void setNote(String regexp) {
+		this.note_content = regexp;
+		this.reg_note_content = Pattern.compile(note_content);
+	}
+	
+	/**
+	 * Set the element to ignore but not their content
+	 *
+	 * @param regexp
+	 */
+	public void setOutSideTextTagsAndKeepContent(String regexp) {
+		this.outside_text_tags_keep_content = regexp;
+		this.reg_outside_text_tags_keep_content = Pattern.compile(outside_text_tags_keep_content);
+	}
+	
+	protected void goToEndOfElement(String name) {
+		//println "START ignoring tag and content of $name"
+		def openedTags = []
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			if (event == XMLStreamConstants.START_ELEMENT) {
+				openedTags << parser.getLocalName()
+				//println "append "+openedTags
+			} else if (event == XMLStreamConstants.END_ELEMENT) {
+				if (openedTags.size() == 0 && name == parser.getLocalName()) {
+					//println "END ignoring tag and content of $name"
+					return;
+				}
+				openedTags.pop()
+				//println "pop $openedTags"
+			}
+		}
+	}
+	
+	public final static String WHITESPACE = " ";
+	public final static String EMPTY = "";
+	/**
+	 * Process word.
+	 */
+	protected void processWord() {
+		String text = buffer.toString();//parser.getText().trim().replace("\t", " ");
+		//if (DEBUG) println "-- chars: "+text+"--";
+		text = regLN.matcher(text).replaceAll(WHITESPACE);
+		text = regCTRL.matcher(text).replaceAll(EMPTY);						// remove ctrl characters
+		if (regSplitWhiteSpaces != null) {
+			for (String s : regSplitWhiteSpaces.split(text)) {		// separate with unicode white spaces
+				//	if (DEBUG){println "process $s"}
+					iterate(s);
+				}
+		} else {
+			iterate(text);
+		}
+	}
+
+	/**
+	 * Iterate. a String, should be called when a word is found in a String
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	protected iterate(String s) {
+		while (s != null && s.length() > 0) {
+		//	if (DEBUG){println "  > $s"}
+			s = standardChecks(s);
+		}
+	}
+
+	/**
+	 * Standard checks.
+	 *
+	 * @param s the s
+	 * @return the java.lang. object
+	 */
+	//	@CompileStatic(SKIP)
+	public String standardChecks(String s) {
+		Matcher m;
+		if (fclitics != null && (m = s =~ regFClitics) ) {
+		//	if (DEBUG) println "CLITIC found: $s ->"+ m
+			iterate(m.group(1))
+			
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			//writer.writeAttribute("type", "w");
+			writer.writeCharacters(m.group(2));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+			
+			return "";
+		} else if (pclitics != null && (m = s =~ regPClitics) ) {
+			if (DEBUG) println "PCLITIC found: $s ->"+ m
+			
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			//writer.writeAttribute("type", "w");
+			writer.writeCharacters(m.group(1));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+			
+			iterate(m.group(2))
+			
+			return "";
+		} else if (regElision != null && (m = s =~ regElision) ) {
+		//	if (DEBUG) println "Elision found: $s ->"+ m.group(1)+" + "+m.group(2)+" + "+m.group(3)
+			//iterate(m.group(1))
+
+//			int sep = s.indexOf("'");
+//			if (sep < 0)
+//				sep = s.indexOf("’");
+//			if (sep < 0)
+//				sep = s.indexOf("‘");
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			//writer.writeAttribute("type", "w");
+			writer.writeCharacters(m.group(1));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			iterate(m.group(2))
+			
+			return "";
+		} else if (reg3pts != null && (m = s =~ reg3pts) )	{
+		//	if(DEBUG){println "REG '...' found: $s -> "+m.group(1)+" + "+m.group(2)+" + "+m.group(3)}
+			iterate(m.group(1))
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			//writer.writeAttribute("type","pon");
+			writer.writeCharacters("...");
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m.group(3);
+		} else if (regPunct != null && (m = s =~ regPunct) ) {
+			if(DEBUG){println "PUNCT found: $s ->"+m.group(1)+" + "+m.group(2)+" + "+m.group(3)}
+			iterate(m.group(1));
+
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+			//writer.writeAttribute("type","pon");
+			writer.writeCharacters(m.group(2));
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return m.group(3);
+		} else {
+	//		if(DEBUG){println "Other found: "+s}
+			wordcount++;
+			writer.writeStartElement("w");
+			writeWordAttributes();// id
+//			if (reg_punct_other.matcher(s).matches())
+//				writer.writeAttribute("type","pon");
+//			else
+//				writer.writeAttribute("type","w");
+			writer.writeCharacters(s);
+			writer.writeEndElement();
+			writer.writeCharacters("\n");
+
+			return "";
+		}
+	}
+
+	/**
+	 * Write word attributes.
+	 *
+	 * @return the java.lang. object
+	 */
+	protected writeWordAttributes() {
+		writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+		writer.writeAttribute("n",""+wordcount);
+	}
+
+	public void setStartTag(String tag)
+	{
+		this.startTag = tag;
+		this.reg_startTag = Pattern.compile(startTag);
+	}
+
+	/**
+	 * Tokenize.
+	 *
+	 * @param str the str
+	 * @return the list
+	 */
+	public List<String> tokenize(String str)
+	{
+		return str.tokenize()	// cut by whitespace
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File SVNDIR = new File(System.getProperty("user.home"), "SVN/TXMSVN/trunk/corpora/tokenizer");
+		Runtime.getRuntime().exec("svn", "update", SVNDIR.getAbsolutePath());
+		File inputFile = new File(SVNDIR, "test1.xml")
+		File outputFile = new File(SVNDIR, "test1-t.xml")
+		SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inputFile, outputFile, "fr")
+		tokenizer.DEBUG = true
+		tokenizer.setNote("note")
+		tokenizer.setOutSideTextTags("outside")
+		tokenizer.setOutSideTextTagsAndKeepContent("outsideToEdit")
+		println outputFile.toString() + " : "+tokenizer.process();
+		// FIXME: tests UTF-8 XML ranges
+		//		File inputDir = new File("C:/Tools/Textometrie/___corpus/xml/test_bug_import_chinois/")
+		//	File inputFile = new File(inputDir, "problem2.xml");
+
+		/*
+		File inputDir = new File("/home/mdecorde/xml/bvhepistemon2016")
+		File outputDir = new File("/home/mdecorde/xml/bvhepistemon2016/out-fast")
+		File outputDir2 = new File("/home/mdecorde/xml/bvhepistemon2016/out-slow")
+		File outputDir3 = new File("/home/mdecorde/xml/bvhepistemon2016/out-fast2")
+		File inputFile = new File(inputDir, "1542_RabelaisGrgt.xml");
+		outputDir.deleteDir()
+		outputDir2.deleteDir()
+		outputDir3.deleteDir()
+		outputDir.mkdir()
+		outputDir2.mkdir()
+		outputDir3.mkdir()
+		long time;
+		for (String lang : ["none"]) {
+			println "LANG: $lang"
+
+			time = System.currentTimeMillis();
+			File outputFile = new File(outputDir, "${lang}.xml")
+			SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inputFile, outputFile, lang)
+			tokenizer.process();
+			println "FAST $lang: "+(System.currentTimeMillis()-time)
+
+//			time = System.currentTimeMillis();
+//			File outputFile3 = new File(outputDir3, "${lang}.xml")
+//			FastSimpleTokenizerXml tokenizer3 = new FastSimpleTokenizerXml(inputFile, outputFile3, lang)
+//			tokenizer3.process();
+//			println "FAST2 $lang: "+(System.currentTimeMillis()-time)
+//			
+//			time = System.currentTimeMillis();
+//			File outputFile2 = new File(outputDir2, "${lang}.xml")
+//			OldSimpleTokenizerXml tokenizer2 = new OldSimpleTokenizerXml(inputFile, outputFile2, lang)
+//			tokenizer2.process();
+//			println "SLOW $lang: "+(System.currentTimeMillis()-time)
+		}
+*/
+		println ""
+		println "Done"
+
+		//		String lang = "en"
+		//		File inDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant")
+		//		File outDir = new File("/home/mdecorde/xml/ny911/cycle1/Data_Modif_Pour_Import_XML_Cycle1/xml_files/1xml_par_participant-t")
+		//		outDir.deleteDir()
+		//		outDir.mkdir()
+		//
+		//		println "processing "+inDir.listFiles().size()+" files."
+		//		for (def infile : inDir.listFiles()) {
+		//			if (!infile.getName().endsWith(".xml")) continue;
+		//
+		//			print "."
+		//			File outfile = new File(outDir, infile.getName())
+		//			SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)
+		//			tokenizer.setDEBUG false
+		//			tokenizer.process();
+		//		}
+		//		println ""
+		//		println "Done"
+
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/CutHeader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/CutHeader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/CutHeader.groovy	(revision 321)
@@ -0,0 +1,107 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.CutHeader;
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+import org.txm.importer.filters.Filter;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class CutHeader.
+ */
+class CutHeader extends Filter
+{
+	
+	/** The xml header file. */
+	String xmlHeaderFile = ""; 
+	
+	/** The header. */
+	boolean header = true;
+	
+	/** The HEADER. */
+	OutputStreamWriter HEADER;
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+		if (args.size() == 1) {
+			xmlHeaderFile = args[0].toString();
+		} else {
+			System.err.println("CutHeader needs 1 args\nString:xmlHEADERFilePath");
+			System.exit(-1);
+		}
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		HEADER.close();
+		//println("AFTER CUTHEADER");
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	protected void filter()
+	{
+		if(header) {
+			//if(line ==~ /\s*<text>\s*/ )
+			if (line.startsWith("<text ") || line.startsWith("<text>")) {
+				header = false;
+				HEADER.close();
+				output.print(line+lineSeparator);
+			} else {
+				//print(line+lineSeparator);
+				HEADER.print(line+lineSeparator);
+			}
+		} else {
+			output.write(line+lineSeparator);	
+		}
+		//System.out.println( "!"+linecounter);
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		if (xmlHeaderFile == "") {
+			println("need to define xmlHeaderFile before lauching this filter");
+			return false;
+		}
+		File outputFile = new File(xmlHeaderFile)
+		//HEADER = new PrintWriter(new BufferedWriter(new FileWriter(outputFile)));
+		HEADER = new OutputStreamWriter(new FileOutputStream(outputFile) , this.encodageSortie);
+		//println("BEFORE CUTHEADER");
+		return true;
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to cut TEIHeader of XML-TEI files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/CutHeader/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to eliminate whitespace in the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/EliminateWhiteSpaces.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/EliminateWhiteSpaces.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/EliminateWhiteSpaces/EliminateWhiteSpaces.groovy	(revision 321)
@@ -0,0 +1,86 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.EliminateWhiteSpaces;
+
+import org.txm.importer.filters.*;
+import java.util.regex.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class EliminatingWhiteSpaces.
+ */
+class EliminatingWhiteSpaces extends Filter {
+	
+	/** The counter. */
+	int counter;
+	
+	/** The old. */
+	def old;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		counter = 0;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		println "Deleted $counter whitespaces \n";
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment;
+		// Write your code here, but don't forget to write in the output
+		// ex : output.write("TheStringToWrite " + line );
+		// in the var line is the current line
+		old = line;
+		line = (line =~ /\s+<\/w>/).replaceAll("<\\/w>");
+
+		if( old != line)
+			counter++;
+		
+		output.write(line+lineSeparator);
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/FusionXmlHeaderBody.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/FusionXmlHeaderBody.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/FusionXmlHeaderBody.groovy	(revision 321)
@@ -0,0 +1,133 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.FusionXmlHeaderBody;
+
+import org.txm.importer.filters.*;
+import org.xml.sax.InputSource
+import org.xml.sax.helpers.ParserAdapter
+import org.xml.sax.helpers.*;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.SAXParser;
+import java.io.Reader;
+import groovy.xml.XmlUtil
+
+import javax.xml.transform.*;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.xml.sax.helpers.DefaultHandler
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class FusionXmlHeaderBody.
+ */
+class FusionXmlHeaderBody extends Filter
+{
+	
+	/** The body file. */
+	String bodyFile = "";
+	
+	/** The header file. */
+	String headerFile = "";
+	
+	/** The Lastinput. */
+	Reader Lastinput;
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args)
+	{
+		
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before()
+	{
+		if(bodyFile == "" || headerFile == "")
+		{
+			System.out.println("FusionXmlHeaderBody need 2 args : \nString:XmlBodyFile\nString:XmlHeaderFile");
+		}
+		System.out.println("start fusionxmlheaderbody "+this);
+		Lastinput = this.input;
+		this.input = new BufferedReader(new java.io.StringReader("") );
+	}
+	
+	/**
+	 * Sets the used param.
+	 *
+	 * @param args the args
+	 */
+	void SetUsedParam(Object[] args)
+	{
+		if(args.size() == 2)
+		{
+			bodyFile = args[0];//needed to get back its ID
+			headerFile = args[1];
+		}
+		else
+		{
+			System.out.println("FusionXmlHeaderBody need 2 args : \nString:XmlBodyFile\nString:XmlHeaderFile");
+		}
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		this.input = this.Lastinput;
+		//System.out.println("input ready "+this.input.ready());
+		def body = new XmlSlurper().parse(this.input);
+		
+		def headers = new XmlSlurper().parse(headerFile)
+		
+		def id = bodyFile.split('\\.')[0]
+		def header = headers.discours.find{it.@file == id}
+		
+		this.output.withWriter{writer ->
+			def xmlBuilder = new groovy.xml.StreamingMarkupBuilder();
+			def xml = xmlBuilder.bind{
+				mkp.xmlDeclaration();
+				top (header <<body.children());	
+			};
+		
+			writer <<XmlUtil.serialize(xml);
+		}		
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+	}
+	
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/ReaderInputStream.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/ReaderInputStream.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/ReaderInputStream.groovy	(revision 321)
@@ -0,0 +1,231 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.FusionXmlHeaderBody;
+
+/*
+ * Copyright 2004-2005 The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+
+// TODO: Auto-generated Javadoc
+/**
+ * Adapts a <code>Reader</code> as an <code>InputStream</code>. Adapted from
+ * <CODE>StringInputStream</CODE>.
+ */
+public class ReaderInputStream extends InputStream {
+	Reader input;
+
+	/** The encoding. */
+	String encoding = System.getProperty("file.encoding");
+
+	/** The slack. */
+	byte[] slack;
+
+	/** The begin. */
+	int begin;
+
+	/**
+	 * Construct a <CODE>ReaderInputStream</CODE> for the specified
+	 * <CODE>Reader</CODE>.
+	 * 
+	 * @param reader
+	 *            <CODE>Reader</CODE>. Must not be <code>null</code>.
+	 */
+	public ReaderInputStream(Reader reader) {
+		input = reader;
+	}
+
+	/**
+	 * Construct a <CODE>ReaderInputStream</CODE> for the specified
+	 * <CODE>Reader</CODE>, with the specified encoding.
+	 * 
+	 * @param reader
+	 *            non-null <CODE>Reader</CODE>.
+	 * @param encoding
+	 *            non-null <CODE>String</CODE> encoding.
+	 */
+	public ReaderInputStream(Reader reader, String encoding) {
+		this(reader);
+		if (encoding == null) {
+			throw new IllegalArgumentException("encoding must not be null");
+		} else {
+			this.encoding = encoding;
+		}
+	}
+
+	/**
+	 * Reads from the <CODE>Reader</CODE>, returning the same value.
+	 *
+	 * @return the value of the next character in the <CODE>Reader</CODE>.
+	 */
+	public synchronized int read() throws IOException {
+		if (input == null) {
+			throw new IOException("Stream Closed");
+		}
+
+		byte result;
+		if (slack != null && begin < slack.length) {
+			result = slack[begin];
+			if (++begin == slack.length) {
+				slack = null;
+			}
+		} else {
+			byte[] buf = new byte[1];
+			if (read(buf, 0, 1) <= 0) {
+				result = -1;
+			}
+			result = buf[0];
+		}
+
+		if (result < -1) {
+			result += 256;
+		}
+
+		return result;
+	}
+
+	/**
+	 * Reads from the <code>Reader</code> into a byte array.
+	 *
+	 * @param b the byte array to read into
+	 * @param off the offset in the byte array
+	 * @param len the length in the byte array to fill
+	 * @return the actual number read into the byte array, -1 at the end of the
+	 * stream
+	 */
+	public synchronized int read(byte[] b, int off, int len) throws IOException {
+		if (input == null) {
+			throw new IOException("Stream Closed");
+		}
+
+		while (slack == null) {
+			char[] buf = new char[len]; // might read too much
+			int n = input.read(buf);
+			if (n == -1) {
+				return -1;
+			}
+			if (n > 0) {
+				slack = new String(buf, 0, n).getBytes(encoding);
+				begin = 0;
+			}
+		}
+
+		if (len > slack.length - begin) {
+			len = slack.length - begin;
+		}
+
+		System.arraycopy(slack, begin, b, off, len);
+
+		if ((begin += len) >= slack.length) {
+			slack = null;
+		}
+
+		return len;
+	}
+
+	/**
+	 * Marks the read limit of the StringReader.
+	 * 
+	 * @param limit
+	 *            the maximum limit of bytes that can be read before the mark
+	 *            position becomes invalid
+	 */
+	public synchronized void mark(final int limit) {
+		try {
+			input.mark(limit);
+		} catch (IOException ioe) {
+			throw new RuntimeException(ioe.getMessage());
+		}
+	}
+
+	/**
+	 * Available.
+	 *
+	 * @return the current number of bytes ready for reading
+	 */
+	public synchronized int available() throws IOException {
+		if (input == null) {
+			throw new IOException("Stream Closed");
+		}
+		if (slack != null) {
+			return slack.length - begin;
+		}
+		if (input.ready()) {
+			return 1;
+		} else {
+			return 0;
+		}
+	}
+
+	/**
+	 * Mark supported.
+	 *
+	 * @return false - mark is not supported
+	 */
+	public boolean markSupported() {
+		return false; // would be imprecise
+	}
+
+	/**
+	 * Resets the StringReader.
+	 *
+	 */
+	public synchronized void reset() throws IOException {
+		if (input == null) {
+			throw new IOException("Stream Closed");
+		}
+		slack = null;
+		input.reset();
+	}
+
+	/**
+	 * Closes the Stringreader.
+	 *
+	 */
+	public synchronized void close() throws IOException {
+		if (input != null) {
+			input.close();
+			slack = null;
+			input = null;
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to restore the XML-TEI header of XML-TEI files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/FusionXmlHeaderBody/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to restore the XML-TEI header of XML-TEI files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/FusionHeader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/FusionHeader.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/FusionHeader/FusionHeader.groovy	(revision 321)
@@ -0,0 +1,125 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.FusionHeader;
+
+import java.io.File;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class FusionHeader.
+ */
+class FusionHeader extends Filter {
+	
+	/** The xml header file. */
+	String xmlHeaderFile = "";
+	
+	/** The first loop. */
+	boolean firstLoop = true;
+	
+	/** The HEADER. */
+	BufferedReader HEADER;
+	
+	/** The cut headerfilter. */
+	Thread cutHeaderfilter;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+		if (args.size() == 2) {
+			xmlHeaderFile = args[0].toString();
+			cutHeaderfilter = (Thread) args[1];
+		} else {
+			System.err
+					.println("CutHeader needs 2 args\nString:xmlHEADERFilePath\nThread:cutHeaderFilter");
+			System.exit(-1);
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+		try {
+			HEADER.close();
+			HEADER= null;
+		} catch (Exception e) {
+			System.err
+					.println("Can't close Header file, one cause might be that the header file is not correctly built");
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		if (xmlHeaderFile == "" || cutHeaderfilter == null) {
+			System.err
+					.println("CutHeader needs 2 args\nString:xmlHEADERFilePath\nThread:cutHeaderFilter");
+		}
+		// System.out.println("start FUSIONHEADER");
+
+		return true;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	protected void filter()
+	{
+		if (firstLoop) {
+			//println("BEGIN COPYING HEADER");
+		
+			String templine;
+			File inputFile = new File(xmlHeaderFile)
+			if ( !inputFile.exists()) {
+				System.err.println("ERROR : DOES NOT EXISTS "+xmlHeaderFile);
+			}
+			HEADER = new BufferedReader(new InputStreamReader(new FileInputStream(inputFile), encodage));
+			while (!HEADER.ready()) {
+				Thread.sleep(10);
+				//system.out.println("waiting for cutHeader to finish");
+			}
+			//HEADER = new FileReader("/home/ayepdieu/xml/header (copie).xml");
+			//System.out.println("ready ? "+HEADER.ready());
+			while ((templine = this.HEADER.readLine()) != null) {
+				//println(templine+lineSeparator);
+				output.print(templine+lineSeparator);
+			}
+			//println("FINISH COPYING HEADER");
+			firstLoop=false;
+			HEADER.close();
+		}
+		output.print(line+lineSeparator);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to process enclitics in the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/ProcessEnclitics.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/ProcessEnclitics.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ProcessEnclitics/ProcessEnclitics.groovy	(revision 321)
@@ -0,0 +1,103 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.ProcessEnclitics;
+
+import org.txm.importer.filters.*;
+import java.util.regex.*;
+import org.txm.tokenizer.TokenizerClasses;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ProcessEnclitics.
+ */
+class ProcessEnclitics extends Filter {
+	
+	/** The counter. */
+	int counter;
+	
+	/** The enclitics. */
+	String enclitics = TokenizerClasses.enclitics;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args)
+	{
+		try
+		{
+			enclitics = args.get("enclitics");
+
+		}
+		catch(Exception e)
+		{
+			System.err.println(e);
+			System.err.println("Processenclitics needs 1 Map with arg  :\n enclitics")
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		counter = 0;
+		System.out.println("begin enclitics");
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		print "$counter enclitics with dashes found\n";
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment;
+		// Write your code here, but don't forget to write in the output
+		// ex : output.write("TheStringToWrite " + line );
+		// in the var line is the current line
+		if( (m = line =~ /\A\s*(<w[^>]*>)(.*)-($enclitics)<\/w>\Z/))
+		{
+			counter++;
+			def word1_tag = (m[0][1]);
+			def word1 = (m[0][2]);
+			def word2 = (m[0][3]);
+			output.write("$word1_tag$word1-</w>\n<w>$word2</w>"+lineSeparator);
+		}
+		else 
+		{
+			output.write(line+lineSeparator);
+		}
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to prepare the BFM tokenizer files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/OneOpenTagPerLine.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/OneOpenTagPerLine.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/OneOpenTagPerLine/OneOpenTagPerLine.groovy	(revision 321)
@@ -0,0 +1,94 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.OneOpenTagPerLine;
+
+//Pre-processing extra-word tags (1)
+import org.txm.importer.filters.Filter;
+import org.txm.tokenizer.TokenizerClasses;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class OneOpenTagPerLine.
+ */
+class OneOpenTagPerLine extends Filter {
+	
+	/** The tag_all. */
+	String tag_all = TokenizerClasses.tag_all;
+	
+	/** The counterreg1. */
+	int counterreg1 = 0;
+	
+	/** The counterreg2. */
+	int counterreg2 = 0;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		println "reg1 : $counterreg1";
+		println "reg2 : $counterreg2";
+	}
+
+	/** The segment. */
+	def segment;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		segment = line.trim()
+		if(segment[0] == "<")
+			output.write(lineSeparator+line);
+		else
+		output.write(" "+segment)
+		/*if ( line ==~ /([^<])+>$/)
+		{
+			output.write(" "+line.trim())
+		}
+		else
+		{
+			output.write(lineSeparator+line);
+		}*/
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to process quotes in the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/ProcessQuotes.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/ProcessQuotes.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ProcessQuotes/ProcessQuotes.groovy	(revision 321)
@@ -0,0 +1,84 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.ProcessQuotes;
+
+//Pre-processing extra-word tags (1)
+import org.txm.importer.filters.*;
+import java.util.regex.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ProcessQuotes.
+ */
+class ProcessQuotes extends Filter {
+	
+	/** The counter. */
+	int counter;
+	
+	/** The old. */
+	def old;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		counter = 0;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		println "Replaced $counter quotes\n";
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment;
+
+		old = line;
+		line = (line =~ /&(quot|ldquo|rdquo);/).replaceAll("\"");
+		
+		if( old != line)
+			counter++;
+		
+		output.write(line+lineSeparator);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to concat 2 files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/Concatenator.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/Concatenator.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/Concatenator/Concatenator.groovy	(revision 321)
@@ -0,0 +1,103 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.Concatenator;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+
+import org.txm.importer.filters.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class Concatenator.
+ */
+class Concatenator extends Filter {
+	
+	/** The destination. */
+	String destination;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		if (destination == "") {
+			println("You need to define destination file before launching this filter");
+			return false;
+		}
+		def f = new File(destination);
+		this.output = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+		return true;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+		if (args.size() == 1) {
+			destination = args[0];
+		} else {
+			System.out
+					.println("Concatenator need 1 args : \nString:destinatione");
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+		System.out.println("Concatenator readed lines : " + linecounter);
+		this.output.close();
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter() {
+		output.write(line + lineSeparator);
+		output.flush();
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to select lines with a regular expression</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/RegexFilter.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/RegexFilter.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/RegexFilter/RegexFilter.groovy	(revision 321)
@@ -0,0 +1,79 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.RegexFilter;
+
+import org.txm.importer.filters.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.txm.importer.filters.Filter;
+
+// TODO: Auto-generated Javadoc
+/** Filters out lines not matching a regexp. @author jmague */
+public class RegexFilter extends Filter {
+	private Pattern pattern;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+		if (args.size() == 1) {
+			pattern = Pattern.compile(args[0]);
+		} else {
+			System.out
+					.println("Regexfilter needs 1 args :\nString:patternRegex");
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		if (pattern == null) {
+			System.out
+					.println("Regex filter needs 1 args :\nString:patternRegex");
+			return false;
+		}
+		return true;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter() {
+		Matcher matcher = pattern.matcher(line);
+		if (matcher.matches())
+			output.write(line + lineSeparator);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/WordInternalElement.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/WordInternalElement.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/WordInternalElement.groovy	(revision 321)
@@ -0,0 +1,102 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.WordInternalElement;
+
+import org.txm.importer.filters.*;
+import java.util.regex.*;
+import org.txm.tokenizer.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class WordInternalElement.
+ */
+class WordInternalElement extends Filter {
+	
+	/** The counter. */
+	int counter;
+	
+	/** The old. */
+	def old;
+	
+	/** The corr_tags_no_seg. */
+	String corr_tags_no_seg = TokenizerClasses.corr_tags_no_seg;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args)
+	{
+		try
+		{
+			corr_tags_no_seg = args.get("corr_tags_no_seg");
+
+		}
+		catch(Exception e)
+		{
+			System.err.println(e);
+			System.err.println("wordinternal needs 1 Map with arg  :\n corr_tags_no_seg")
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before()
+	{
+		counter = 0;
+		println "begin wordinternal \n";
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		println "Deleted $counter wordinternalspaces \n";
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment;
+		// Write your code here, but don't forget to write in the output
+		// ex : output.write("TheStringToWrite " + line );
+		// in the variable "line" is the current line value
+		old = line;
+		line = (line =~ /(<(corr_tags_no_seg) [^>]*word_part[^>]*>)\s*([^<]*?)\s*(<\/\2>)/).replaceAll("<seg type=\"word_part\">\$1\$3\$4<\\/seg>");
+		
+		if( old != line)
+			counter++;
+		
+		output.write(line+lineSeparator);
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter of the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/WordInternalElement/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TEISentencer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TEISentencer.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TEISentencer.groovy	(revision 321)
@@ -0,0 +1,426 @@
+package filters.TagSentences
+
+import org.txm.importer.StaxIdentityParser;
+import org.txm.importer.ValidateXml;
+
+import java.net.URL;
+import filters.Tokeniser.TEITokenizer.STATES;
+
+class TEISentencer extends StaxIdentityParser {
+
+	boolean DEBUG = false;
+	String filename;
+	
+	enum STATES { IGNORE, NOTE, WORK, W };
+	enum SENT { IN, OUT };
+	STATES state = STATES.IGNORE;
+	SENT sentence = SENT.OUT;
+	
+	String STARTTAGS = "text"
+	String NOTETAGS = "note"
+	String WORDTAGS = "w"
+	String DIVTAGS = "div"
+	String QTAGS = "q"
+	String EDTAGS = "supplied"
+	def openTags = [];
+	
+	public TEISentencer(URL inputurl) {
+		super(inputurl);
+
+		this.filename = new File(inputurl.getFile()).getName();
+		int index = filename.indexOf(".");
+		if (index > 0)
+			filename = filename.substring(0, index);
+	}
+
+	protected void processStartElement() {
+
+		if (state == STATES.IGNORE) {
+			sentence = SENT.OUT;
+			if (localname == STARTTAGS) {
+				state = STATES.WORK;
+			}
+			super.processStartElement();
+		} else {
+			if (localname == NOTETAGS) {
+				state = STATES.NOTE;
+				super.processStartElement();
+				goToEnd(localname)
+			} else {
+				switch(sentence) {
+					case SENT.IN:
+						processStartElementIN();
+						break;
+					case SENT.OUT:
+						processStartElementOUT();
+						break;
+				}
+			}
+		}
+	}
+	
+	@Override
+	protected void processEndElement() {
+		if (state == STATES.IGNORE) {
+			super.processEndElement();
+		} else {
+			if (localname == STARTTAGS) {
+				state = STATES.IGNORE;
+				super.processEndElement();
+			} else {
+				switch(sentence) {
+					case SENT.IN:
+						processEndElementIN();
+						break;
+					case SENT.OUT:
+						processEndElementOUT();
+						break;
+				}
+			}
+		}
+	}
+	
+	protected void cutSentence() {
+		for( int i = 0 ; i < openTags.size() ; i++) {
+			writer.writeEndElement();
+		}
+		endSentence() // close sent
+		super.processStartElement();
+		startSentence()
+		for( Tag tag : openTags) {
+			tag.writeStart(writer);
+		}
+	}
+	
+	protected void cutSentenceAndClose() {
+		for( int i = 0 ; i < openTags.size() ; i++) {
+			writer.writeEndElement();
+		}
+		endSentence() // close sent
+		super.processEndElement();
+		startSentence()
+		for( Tag tag : openTags) {
+			tag.writeStart(writer);
+		}
+	}
+	
+	protected void closeRemainingSentence() {
+		def tags = []
+		while( openTags.size() > 0) {
+			writer.writeEndElement();
+			tags << openTags.pop()
+		}
+		writer.writeEndElement(); // close sent
+		sentence = SENT.OUT;
+	}
+	
+	protected void processStartElementIN() {
+		if (localname.matches(DIVTAGS)) {
+			if (openTags.size() == 0) {
+				endSentence();
+				super.processStartElement();
+				println "WARNING: start DIVTAG in sentence at "+getLocation()
+			} else {
+				cutSentence();
+				println "WARNING: DIVTAG opepened inside EDTAG at "+getLocation()
+			}
+		} else if (localname.matches(QTAGS)) {
+			if (openTags.size() == 0) {
+				endSentence();
+				super.processStartElement();
+			} else {
+				cutSentence();
+				println "WARNING: QTAG opepened inside EDTAG at "+getLocation()
+			}
+		} else if (localname.matches(EDTAGS)) {
+			if (openTags.size() == 0) {
+				if (hasMULTIS()) {
+					endSentence();
+				} else {
+					if (openTags.size() > 0 && openTags[0].localname == localname) {
+						println "STOP: openning same EDTAG $localname at "+getLocation()
+						closeForError();
+					}
+					openTags << new Tag(parser)
+				}
+				super.processStartElement();
+			} else {
+				if (hasMULTIS()) {
+					cutSentence();
+					println "WARNING: EDTAG opened  inside EDTAG"
+				} else {
+					super.processStartElement();
+					if (openTags.size() > 0 && openTags[0].localname == localname) {
+						println "STOP: openning same EDTAG $localname at "+getLocation()
+						closeForError();
+					}
+					openTags << new Tag(parser)
+				}
+			}
+		} else if (localname.matches(WORDTAGS)) {
+			boolean shouldClose = isWEndPunct()
+			if (openTags.size() == 0) {
+				super.processStartElement();
+				goToEnd(localname);
+				if (shouldClose) {
+					endSentence();
+				}
+			} else {
+				super.processStartElement();
+				goToEnd(localname);
+				if (shouldClose) {
+					println "WARNING: Strong ponctuation found inside EDTAG at "+getLocation()
+				}
+			}
+		} else { // OTHER TAGS
+			if (openTags.size() == 0) {
+				if (hasMULTIS()) {
+					endSentence();
+				} else {
+					if (openTags.size() > 0 && openTags[0].localname == localname) {
+						println "STOP: openning same EDTAG $localname at "+getLocation()
+						closeForError();
+					}
+					openTags << new Tag(parser)
+				}
+				super.processStartElement();
+			} else {
+				if (hasMULTIS()) {
+					cutSentence();
+					println "WARNING: multi_s tag opens inside EDTAG"
+				} else {
+					super.processStartElement();
+					if (openTags.size() > 0 && openTags[0].localname == localname) {
+						println "STOP: openning same TAG $localname at "+getLocation()
+						closeForError();
+					}
+					openTags << new Tag(parser);
+				}
+			}
+		}
+	}
+
+	protected void processEndElementIN() {
+		if (localname.matches(DIVTAGS)) {
+			if (openTags.size() == 0) {
+				endSentence();
+				super.processEndElement();
+			} else {
+				cutSentenceAndClose();
+				println "WARNING: DIVTAG closed  inside EDTAG"
+			}
+		} else if (localname.matches(QTAGS)) {
+			if (openTags.size() == 0) {
+				endSentence();
+				super.processEndElement();
+			} else {
+				cutSentenceAndClose();
+				println "WARNING: QTAG closed  inside EDTAG"
+			}
+		} else if (localname.matches(EDTAGS)) {
+			if (openTags.size() == 0) { // it was a multi_s EDTAG
+				endSentence();
+				super.processEndElement();
+			} else {
+				if (openTags[-1].localname == localname) {
+					super.processEndElement();
+					openTags.pop();
+				} else {
+					println "STOP: closing EDTAG does not match last opened EDTAG "+localname+" != "+openTags[-1].localname+" at "+getLocation()
+					closeForError()
+				}
+			}
+		} else if (localname.matches(WORDTAGS)) {
+			println "ERROR: processEndElementIN: </w> at "+getLocation();
+		} else {
+			if (openTags.size() == 0) {
+				super.processEndElement();
+			} else {
+				if (openTags[-1].localname == localname) {
+					super.processEndElement();
+					openTags.pop();
+				} else {
+					println "STOP: closing tag does not match last opened EDTAG "+localname+" != "+openTags[-1].localname+" at "+getLocation()
+					closeForError()
+				}
+			}
+		}
+	}
+
+	protected void processStartElementOUT() {
+		if (localname.matches(DIVTAGS)) {
+			super.processStartElement();
+		} else if (localname.matches(QTAGS)) {
+			super.processStartElement();
+		} else if (localname.matches(EDTAGS)) {
+			if (hasMULTIS()) {
+				super.processStartElement();
+			} else {
+				startSentence();
+				super.processStartElement();
+				if (openTags.size() > 0 && openTags[0].localname == localname) {
+					println "STOP: openning same EDTAG name at "+getLocation()
+					closeForError();
+				}
+				openTags << new Tag(parser);
+			}
+		} else if (localname.matches(WORDTAGS)) {
+			if (isWEndPunct()) {
+				startSentence();
+				super.processStartElement();
+				goToEnd(localname);
+				endSentence();
+				println "WARNING: empty sentence at "+getLocation();
+			} else {
+				startSentence();
+				super.processStartElement();
+				goToEnd(localname);
+			}
+
+		} else {
+
+			if (hasMULTIS()) {
+				super.processStartElement();
+			} else {
+				startSentence();
+				super.processStartElement();
+				if (openTags.size() > 0 && openTags[0].localname == localname) {
+					println "STOP: openning same TAG $localname name at "+getLocation()
+					closeForError();
+				}
+				openTags << new Tag(parser)
+				println "WARNING: unclassified tag: "+localname+" at "+getLocation();
+			}
+		}
+	}
+
+	protected void processEndElementOUT() {
+		if (localname.matches(DIVTAGS)) {
+			super.processEndElement();
+		} else if (localname.matches(QTAGS)) {
+			super.processEndElement();
+		} else if (localname.matches(EDTAGS)) {
+			super.processEndElement();
+		} else if (localname.matches(WORDTAGS)) {
+			println "ERROR: processEndElementOUT: </w> at "+getLocation();
+		} else {
+			super.processEndElement();
+		}
+	}
+
+	private static String STAG = "s"
+	protected void startSentence() {
+		sentence = SENT.IN;
+		writer.writeStartElement(STAG);
+	}
+
+	protected void endSentence() {
+		sentence = SENT.OUT;
+		writer.writeEndElement();
+	}
+
+	private static String ENDOFSENTENCE = "PONfrt"
+	protected boolean isWEndPunct() {
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			if (parser.getAttributeValue(i) == ENDOFSENTENCE) return true;
+		}
+		return false;
+	}
+
+	private static String MULTI_S = "multi_s"
+	protected boolean hasMULTIS() {
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+			if (parser.getAttributeValue(i) == MULTI_S) return true;
+		}
+		return false;
+	}
+
+	public static void main(String[] args) {
+		File dir = new File ("/home/mdecorde/TXM/corpora/alceste/tokenized")
+		File outdir = new File ("/home/mdecorde/TXM/corpora/alceste/tokenized/s")
+		outdir.mkdir()
+		for (File infile : dir.listFiles()) {
+			long start = System.currentTimeMillis()
+			println infile
+			File outfile = new File(outdir, infile.getName())
+	
+			TEISentencer sentencer = new TEISentencer(infile.toURI().toURL());
+			if (sentencer.process(outfile)) {
+				println "DONE - OK: "+ValidateXml.test(outfile);
+			} else {
+				println "DONE - FAIL"
+			}
+			println "time: "+(System.currentTimeMillis()-start)/1000
+		}
+	}
+
+	public class Tag {
+		public String[] attnames, attvalues, attprefix;
+		public String localname, prefix
+		int count;
+		ArrayList<String> contents = [""];
+		public ArrayList<Tag> children = [];
+		boolean inChild = false;
+
+		public Tag(def parser) {
+			prefix = parser.getPrefix()
+			localname = parser.getLocalName();
+			count = parser.getAttributeCount()
+			attnames =  new String[count]
+			attvalues = new String[count]
+			attprefix = new String[count]
+			for (int i = 0 ; i < count ; i++) {
+				attnames[i] = parser.getAttributeLocalName(i)
+				attprefix[i] = parser.getAttributePrefix(i).toString()
+				attvalues[i] = parser.getAttributeValue(i).toString()
+			}
+		}
+
+		public void appendContent(String content) {
+			if (inChild) {
+				children[-1].appendContent(content);
+			} else {
+				this.contents[-1] += content;
+			}
+		}
+
+		public void endOfChild() {
+			contents << "";
+			inChild = false;
+		}
+
+		public void addChild(Tag child) {
+			if (inChild) {
+				children[-1].addChild(child);
+			} else {
+				children.add(child)
+				inChild = true;
+			}
+		}
+
+		public writeStart(def writer) {
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname)
+			else
+				writer.writeStartElement(localname)
+
+			for (int i = 0 ; i < count ; i++) {
+				if (attprefix[i] != null && attprefix[i].length() > 0) {
+					writer.writeAttribute(attprefix[i]+":"+attnames[i], attvalues[i])
+				} else {
+					writer.writeAttribute(attnames[i], attvalues[i])
+				}
+			}
+
+			//			for ( Tag child: children) {
+			//				child.write(writer)
+			//			}
+
+			//			writer.writeEndElement();
+		}
+
+		public String getContent() {
+			return contents.toString();
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package:  BFM sentencer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TagSentences.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TagSentences.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TagSentences.groovy	(revision 321)
@@ -0,0 +1,481 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.TagSentences;
+
+import java.util.regex.*
+
+import org.txm.importer.filters.*
+import org.txm.tokenizer.*
+
+import filters.CutHeader.*
+import filters.FusionHeader.*
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class TagSentences.
+ */
+class TagSentences extends Filter {
+	
+	/** The counter. */
+	int counter;
+	
+	/** The m. */
+	def m;
+	
+	/** The segment. */
+	def segment;
+	
+	/** The linetype. */
+	def linetype = "out";
+	
+	/** The open_corr_tags. */
+	def open_corr_tags = new LinkedList<String>();
+	
+	/** The LAS topen_corr_tags. */
+	def LASTopen_corr_tags = "";
+	
+	/** The open_div_tags. */
+	def open_div_tags = new LinkedList<String>();
+	
+	/** The LAST open_div_tags. */
+	def LASTopen_div_tags = "";
+	
+	/** The corr. */
+	def corr = "no";
+	
+	/** The corr_name. */
+	def corr_name;
+	
+	/** The corr_tag. */
+	def corr_tag;
+	
+	/** The pending. */
+	def pending = "no";
+	
+	/** The scounter. */
+	def scounter = 0;
+	
+	/** The _before. */
+	def _before;
+	
+	/** The _after. */
+	def _after;
+	
+	/** The div_tags. */
+	def div_tags = TokenizerClasses.div_tags;
+	
+	/** The q_tags. */
+	def q_tags = TokenizerClasses.q_tags;
+	
+	/** The corr_tags_no_seg. */
+	def corr_tags_no_seg = TokenizerClasses.corr_tags_no_seg;
+	
+	/** The corr_tags. */
+	def corr_tags = TokenizerClasses.corr_tags;
+	
+	def strongPunct = TokenizerClasses.punct_strong;
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+		try {
+			div_tags = args.get("div_tags");
+			q_tags = args.get("q_tags");
+			corr_tags_no_seg = args.get("corr_tags_no_seg");
+			corr_tags = args.get("corr_tags");
+		}
+		catch(Exception e) {
+			System.err.println(e);
+			System.err.println("TagginSentences needs 1 Map with args  :\n div_tags, q_tags, extraword_tags, " +
+					"corr_tags_no_seg, corr_tags, word_tags, intraword_tags")
+		}
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		// System.out.println("begin sentences");
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after() {
+		// System.out.println("end sentences");
+	}
+	
+	/**
+	 * Test before after.
+	 *
+	 * @return true, if successful
+	 */
+	boolean testBeforeAfter() {
+		if (_before ==~ /\S/ || _after ==~ /\S/) {
+			print "TagSentences : ERROR: $line";
+			return false;
+		}
+		return true;
+	}
+	
+	/**
+	 * Affect13.
+	 *
+	 * @param m the m
+	 */
+	void affect13(def m) {
+		_before = m[0][1];
+		_after = m[0][3];
+		testBeforeAfter();
+	}
+	
+	/**
+	 * Affect12.
+	 *
+	 * @param m the m
+	 */
+	void affect12(def m) {
+		_before = m[0][1];
+		_after = m[0][2];
+		testBeforeAfter();
+	}
+	
+	/** The MINPRINT. */
+	int MINPRINT = 0
+	
+	/** The MAXPRINT. */
+	int MAXPRINT = 0
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter() {
+		String tag;
+		// Write your code here, but don't forget to write in the output
+		// ex : output.print("TheStringToWrite " + line );
+		// in the var line is the current line
+		//line = line.trim()
+		
+		if( scounter > MINPRINT && scounter < MAXPRINT)
+			println linetype+"LINE : "+line;
+		
+		if (line ==~ /\A\s*<!--.*-->\s*\Z/) {		
+			output.print(line);
+		}
+		else if (linetype ==~ /out/) { 
+			//on est en dehors d'une phrase
+			if ( (m = line =~ /^(.*)<s( [^>]*)?>(.*)$/) ) { 
+				//on trouve une balise de phrase => on met à jour le numéro, on est dans une phrase
+				affect13(m);
+				if (pending == "yes") {
+					output.write("</s>\n")
+					pending = "no";
+				}
+				
+				scounter++;
+				line = (line =~ /<s( [^>]*)?>/).replaceAll("<s n=\""+scounter+"\">");
+				output.print(line+"\n");	
+				linetype = "in";
+			}
+			else if( (m = line =~ /^(.*)<\/s>(.*)$/) ) { ////on trouve une balise de phrase fermante
+				//println "found </s> "+line
+				affect12(m);
+				if (pending == "yes") {
+					linetype = "out";
+					pending = "no";
+					//pending = "no";
+					output.print(line);
+				}
+				else
+				{
+					//output.print(line);
+					System.err.println("Found </s>, but pending = no and linetype= out : "+line+" ; scount $scounter");
+				}
+			}
+			else if ( (	m = line =~ /^(.*)<w type="pon"[^>]*>.*<\/w>(.*)$/)  && pending ==~ /yes/) { 
+				// on trouve une ponctuation
+				// et la balise de la phrase précédente n'est pas fermée => on ne fait rien
+							//println "found w type pon"
+				affect12(m);
+				output.print(line);
+			}
+			
+			else if( (m = line =~ /^(.*)<w .*<\/w>(.*)/) ) //on rencontre un mot...
+			{ 	
+				//println "found <w> open a sentence"
+				affect12(m);
+				if(pending == "yes")
+				{
+					output.write("</s>\n")
+					pending = "no";
+				}
+				scounter++;
+				output.print("<s n=\""+scounter+"\">\n"+line);
+				linetype = "in";
+			}
+			else if( (m = line =~ /^\s*(<($corr_tags_no_seg)( [^>\/]*)?>)\s*$/) 
+			|| line.matches("<($corr_tags_no_seg)>"))
+			{ 
+				if(pending == "yes")
+				{
+					output.write("</s>\n")
+					pending = "no";
+				}
+				//System.out.println("OUT: found open corr "+line);
+				//on trouve une balise de correction ouvrante
+				def tag_name = m[0][2];
+				def tag_end = m[0][3];
+				
+				if (tag_end != null && tag_end.contains("multi_s")) // traite le corr comme un div tag
+				{ 
+					open_div_tags.push(tag_name);
+					LASTopen_div_tags = tag_name;
+				} else  { 
+					open_corr_tags.push(tag_name);
+
+					scounter++;
+					output.print("<s n=\""+scounter+"\">\n"); //on ouvre une <s>
+					linetype = "in";
+				}
+				if (scounter > MINPRINT && scounter < MAXPRINT) {
+					println "stacks "+open_corr_tags+" ; "+open_div_tags
+				}
+				output.print(line);
+			} else if( ( m = line =~ /^(.*)<\/($corr_tags_no_seg)>(.*)$/) ) 
+			{ 
+				//on trouve la balise fermante correspondante à la dernière correction ouverte	
+						affect13(m);
+				if (scounter > MINPRINT && scounter < MAXPRINT)
+					println "closing corr tag "+line
+				
+				if (open_corr_tags.size() == 0 )//il n'y a plus de simple balise de corr
+				{	
+					open_div_tags.pop();
+					if(open_div_tags.size() == 0)
+					{
+						if(pending == "yes")
+						{
+							output.print("</s>\n"+line);// comme une div tag
+							pending = "no";
+						}
+						else
+						{
+							output.print(line);// comme une div tag
+						}
+					}
+					else
+					{
+						output.print(line);// comme une div tag
+					}
+				}
+				else
+				{	
+					open_corr_tags.pop();
+					
+					if(pending == "yes")
+					{
+						if(open_corr_tags.size() == 0)
+						{
+							output.print("</s>\n");
+							output.print(line);
+							linetype = "out";
+							pending = "no";
+						}
+						else
+						{
+							output.print(line);
+						}
+					}
+					else
+					{
+						output.print(line);
+					}
+				}
+				if( scounter > MINPRINT && scounter < MAXPRINT)
+				{
+					println "stacks "+open_corr_tags+" ; "+open_div_tags
+				}
+			}
+			else if( (	m = line =~ /^(.*)<\/($div_tags|$q_tags)>(.*)$/) ||
+			line.matches("<($div_tags|$q_tags)>")) { ////on trouve une balise de citation ou division fermante
+				//println "found closing div|quote tag "+line
+				//affect13(m);
+				
+				if (pending == "yes") 
+				{	//println "a sentence was closed"
+					output.print("</s>\n$line");
+					pending = "no";
+				} else {
+					//println "was NOT in sentence"
+					output.print(line);
+				}
+			}
+			else if( (m = line =~ /^(.*)<($div_tags|$q_tags)>(.*)$/) ||
+			(m = line =~ /^(.*)<($div_tags|$q_tags)( [^>]*)?>(.*)$/) ||
+			line.matches("<($div_tags|$q_tags)( [^>]*)?>")) { //balise div ouvrante
+				
+				//pending = "no";
+				//push @s_errors, scounter;
+				if(pending == "yes")
+				{
+					output.print("</s>\n");
+					pending = "no";
+				}
+				
+				output.print("$line");
+			}		
+			else {	//println "ELSE de 'out'"
+				output.print(line);
+			}
+		}
+		else if (linetype ==~ /in/) //on est à l'intérieur d'une phrase
+				{ 	//println "in sentence"
+			
+				if( ( m = line =~ /^(.*)<w [^>]*>$strongPunct<\/w>(.*)/) ) { ////on trouve une ponctuation forte ==> on est à l'extérieur d'une phrase
+						//println "found word .!? "+line
+				affect12(m);
+				
+				if(open_corr_tags.size() == 0) // il n'y a pas de correction en cours
+				{
+					linetype = "out";
+					pending ="yes"
+					output.print(line);
+				}
+				else //on ne ferme pas desuite la phrase mais apres les corr
+				{
+					pending = "yes"
+					output.print(line);
+				}
+			}else if( (m = line =~ /^(.*)<w .*<\/w>(.*)/) || (m = line =~ /^(.*)<w( [^>]*)?>.*<\/w>(.*)/) ) //on rencontre un mot...
+			{ 	
+				//println "found <w> open a sentence"
+				affect12(m);
+				output.print(line);
+				
+			}
+			else if( (m = line =~ /^\s*(<($corr_tags_no_seg)( [^>\/]*)?>)\s*$/) 
+			|| line.matches("<($corr_tags_no_seg)>"))
+			{ 
+				if(scounter > MINPRINT && scounter < MAXPRINT)
+					System.out.println("open corr "+line);
+				//on trouve une balise de correction ouvrante
+				def tag_name = m[0][2];
+				def tag_end = m[0][3];
+				
+				if (tag_end != null && tag_end.contains("multi_s")) // traite le corr comme un div tag
+				{ 
+					open_div_tags.push(tag_name);
+					LASTopen_div_tags = tag_name;
+					output.print("</s>\n");
+					linetype="out"
+				}
+				else 
+				{ 
+					open_corr_tags.push(tag_name);
+				}
+				if(scounter > MINPRINT && scounter < MAXPRINT)
+				{
+					println "stacks "+open_corr_tags+" ; "+open_div_tags
+				}
+				output.print(line);
+			}
+			else if( ( m = line =~ /^(.*)<\/($corr_tags_no_seg)>(.*)$/) ) 
+			{ 
+				//on trouve la balise fermante correspondante à la dernière correction ouverte	
+				if(scounter > MINPRINT && scounter < MAXPRINT)
+					System.out.println("closing corr "+line);
+				affect13(m);
+				
+				if (open_corr_tags.size() == 0 )//il n'y a plus de simple balise de corr
+				{	
+					if (open_div_tags.size() > 1) {
+						output.print("</s>\n"+line);// comme une div tag
+						linetype = "out";
+						open_div_tags.pop();
+					} else {
+						println("error : found a non classified corr : "+line);
+					}
+				} else {	
+					open_corr_tags.pop();
+					output.print(line);
+					if (pending == "yes" && open_corr_tags.size() == 0) // c'était le dernier corr
+					{
+						output.print("</s>\n");
+						linetype = "out";
+						pending = "no";
+					}
+				}
+				if(scounter > MINPRINT && scounter < MAXPRINT)
+				{
+					println "stacks "+open_corr_tags+" ; "+open_div_tags
+				}
+			}
+			else if( (m = line =~ /^(.*)<\/s>(.*)$/) ) { ////on trouve une balise de phrase fermante
+				//println "found </s> "+line
+				affect12(m);
+				
+				linetype = "out";
+				//pending = "no";
+				output.print(line);
+			}
+			else if( (m = line =~ /^(.*)<\/($div_tags|$q_tags)>(.*)$/) ||
+			line.matches("</($div_tags|$q_tags)>")) { ////on trouve une balise de division ou de citation fermante ==> on ferme une </s>
+				//println "found div or quote closing tag "+line
+				//affect13(m);
+				
+				linetype = "out";
+				//pending = "no";
+				//push @s_errors, scounter;
+				output.print("</s>\n$line");
+			}
+			else if( (m = line =~ /^(.*)<($div_tags|$q_tags)>(.*)$/) ||
+			(m = line =~ /^(.*)<($div_tags|$q_tags)( [^>]*)?>(.*)$/) ||
+			line.matches("<($div_tags|$q_tags)( [^>]*)?>")) { ////on trouve une balise de division ou de citation fermante ==> on ferme une </s>
+				//println "found div or quote closing tag "+line
+				//affect13(m);
+
+				//push @s_errors, scounter;
+				output.print("</s>\n")
+				output.print("$line");
+				output.print("<s n=\""+scounter+"\">\n"); //on ouvre une <s>
+				
+			}		
+			else {
+				if(scounter > MINPRINT && scounter < MAXPRINT)
+					println "ELSE "+line
+				output.print(line);
+			}
+		}
+		else {
+			print "Error in sentence tagging : $line";
+			System.err.println("Error in sentence tagging : $line");
+		}
+		// End
+	}
+}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TagSentencesXml.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TagSentencesXml.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/TagSentences/TagSentencesXml.groovy	(revision 321)
@@ -0,0 +1,503 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+
+//
+// This file is part of the TXM platform.
+//
+// The TXM platform is free software: you can redistribute it and/or modif y
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The TXM platform is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.
+//
+//
+//
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$
+//
+package filters.TagSentences;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.txm.importer.ValidateXml;
+import org.txm.tokenizer.TokenizerClasses;
+import org.txm.utils.AsciiUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+
+import javax.xml.stream.*;
+import java.net.URL;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class TagSentencesXml.
+ */
+class TagSentencesXml {
+
+	/** The url. */
+	private def url;
+
+	/** The input data. */
+	private def inputData;
+
+	/** The parser. */
+	private XMLStreamReader parser;
+
+	/** The factory. */
+	XMLOutputFactory factory = XMLOutputFactory.newInstance();
+
+	/** The output. */
+	FileOutputStream output;
+
+	/** The writer. */
+	XMLStreamWriter writer;
+
+	/** The IN. */
+	boolean IN = false;
+	boolean WORD = false;
+
+	/** The pending. */
+	boolean pending = false;
+
+	/** The div_tags. */
+	List<String> div_tags = ["front","body","div","head","trailer","p","ab","sp","speaker","list","item"];
+
+	/** The flow_tags. */
+	List<String> flow_tags = ["seg","foreign","hi","title","name","supplied","subst","add","damage", "unclear", "corr", "sic"];
+
+	String startTag = "text";
+	String rendMulti = "MULTI_S";
+	boolean multi = false;
+	boolean startProcess = false;
+	String strongPunct = TokenizerClasses.punct_strong;
+
+	/** The localname. */
+	String localname;
+	boolean DEBUG = false;
+
+	/**
+	 * Instantiates a new tag sentences xml.
+	 *
+	 * @param discoursxml the discoursxml
+	 */
+	public TagSentencesXml(File discoursxml)
+	{
+		this.url = discoursxml.toURI().toURL();
+		//corr_tags.addAll(corr_tags_no_seg);
+
+		inputData = discoursxml.toURI().toURL().openStream();
+		XMLInputFactory factory = XMLInputFactory.newInstance();
+		parser = factory.createXMLStreamReader(inputData);
+	}
+
+	/**
+	 * Creates the output.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	private boolean createOutput(File outfile){
+		try {
+			output = new BufferedOutputStream(new FileOutputStream(outfile))
+			writer = factory.createXMLStreamWriter(output, "UTF-8");//create a new file
+			writer.writeStartDocument("UTF-8", "1.0");
+			writer.writeCharacters("\n");
+			return true;
+		} catch (Exception e) {
+			System.out.println(e.getLocalizedMessage());
+			return false;
+		}
+	}
+
+	/** The s_id. */
+	int s_id = 1;
+
+	/**
+	 * Process.
+	 *
+	 * @param outfile the outfile
+	 * @return true, if successful
+	 */
+	public boolean process(File outfile)
+	{
+		s_id = 1;
+		if (!createOutput(outfile))
+			return false;
+		String localname = "";
+
+		try {
+			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+				if (startProcess) {
+					if (IN) {
+						processIn(event);
+					} else {
+						processOut(event);
+					}
+				} else {
+					writeEvent(event);
+				}
+
+				//detect start tag
+				if (!startProcess)
+					if (event == XMLStreamConstants.START_ELEMENT)
+						if (parser.getLocalName() == startTag)
+							startProcess = true;
+			}
+			
+			writer.close()
+			output.close()
+			parser.close()
+		} catch (Exception ex) {
+			System.err.println(parser.getLocation().toString());
+			ex.printStackTrace();
+			
+			writer.close()
+			output.close()
+			parser.close()
+			return false;
+		}
+
+		return true;
+	}
+
+	/**
+	 * ignore content until the existing s is closed
+	 * this sentence may contains others sentence
+	 */
+	public void processContentExistingSentence() // we're IN
+	{
+		ArrayList<String> openedtags = [];
+		if(DEBUG) writer.writeComment("SENT+");
+		assert(IN == true);
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					openedtags.add(localname); // stack the openened tag
+					writeEvent(event);
+					break;
+
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+					writeEvent(event);
+
+					if (localname == "s") {
+						//	println "end of word: "+parser.getLocation().getLineNumber();
+						if(openedtags.size() == 0) // it's the end of the existing sentence
+						{
+							IN = false; // we're now OUT
+							return; // we've finish to write the word
+						}
+					}
+					openedtags.pop(); // pop the tag stack
+					break;
+
+				case XMLStreamConstants.CHARACTERS:
+					writeEvent(event);
+					break;
+				case XMLStreamConstants.COMMENT:
+					writer.writeComment(parser.getText());
+					break;
+			}
+		}
+		if(DEBUG) writer.writeComment("+SENT");
+	}
+
+	/**
+	 * process the content of a word. start element must have been writen before calling this method
+	 */
+	public void processWordContent() // we're IN
+	{
+		if(DEBUG) writer.writeComment("WORD");
+		assert(IN == true);
+		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+			switch (event) {
+				case XMLStreamConstants.START_ELEMENT:
+					localname = parser.getLocalName();
+					if(localname == "w")
+						println "Error: found a word in a word: "+parser.getLocation();
+					writeEvent(event);
+					break;
+
+				case XMLStreamConstants.END_ELEMENT:
+					localname = parser.getLocalName();
+
+					if (localname == "w" && pending) // we've found a strong punct in a word
+					{
+						IN = false;
+						pending = false; // reset pending
+						writeEvent(event);
+						writer.writeEndElement();
+					} else {
+						writeEvent(event);
+					}
+
+					if (localname == "w") {
+						//	println "end of word: "+parser.getLocation().getLineNumber();
+						return; // we've finish to write the word
+					}
+
+					break;
+
+				case XMLStreamConstants.CHARACTERS: // detect end of sentence
+					String txt = parser.getText();
+					if (txt.matches(strongPunct)) {
+						pending = true;
+					}
+					writeEvent(event);
+					break;
+				case XMLStreamConstants.COMMENT:
+					writer.writeComment(parser.getText());
+					break;
+			}
+		}
+	}
+
+	/**
+	 * Process out.
+	 *
+	 * @param event the event
+	 */
+	public void processOut(int event)
+	{
+		if (DEBUG) writer.writeComment("OUT");
+		switch (event) {
+			case XMLStreamConstants.START_ELEMENT:
+				localname = parser.getLocalName();
+				if (localname == "s") {
+					IN = true;
+					writeEvent(event);
+					processContentExistingSentence();
+				} else if (localname == "w") {
+					IN = true;
+					writer.writeStartElement("s");
+					writer.writeAttribute("id", "s_"+s_id++)
+					writeEvent(event);
+					processWordContent();
+				} else if (flow_tags.contains(localname)) {
+					String rend = getRendAttribute();
+					if (!rend.contains(rendMulti)) {
+						IN = true;
+						writer.writeStartElement("s");
+						writer.writeAttribute("id", "s_"+s_id++)
+					} else {
+						multi = true;
+					}
+					writeEvent(event);
+				} else if (div_tags.contains(localname)) // found a tag out of sentences, we do nothing
+				{
+					writeEvent(event);
+				} else {
+					writeEvent(event);
+				}
+				break;
+
+			case XMLStreamConstants.END_ELEMENT:
+				localname = parser.getLocalName();
+				if (localname == "s") {
+					System.out.println("Error: found a closing s but we are OUT");
+				}
+				writeEvent(event);
+				break;
+
+			case XMLStreamConstants.CHARACTERS:
+				writeEvent(event);
+				break;
+			case XMLStreamConstants.COMMENT:
+				writer.writeComment(parser.getText());
+				break
+		}
+	}
+
+	public String getRendAttribute()
+	{
+		for (int i = 0 ; i < parser.getAttributeCount() ; i++)
+			if (parser.getAttributeLocalName(i) == "rend")
+				return parser.getAttributeValue(i);
+		return "";
+	}
+
+	/**
+	 * Process xml in a created sentence.
+	 *
+	 * @param event the event
+	 */
+	public void processIn(int event)
+	{
+		if(DEBUG) writer.writeComment("IN")
+		switch (event) {
+			case XMLStreamConstants.START_ELEMENT:
+				localname = parser.getLocalName();
+				if (localname == "s") // found opening s, we close ours and stay IN
+				{
+					writer.writeEndElement(); // close our sentence
+					writeEvent(event); // write the start element
+					processContentExistingSentence(); // process its content
+					//println "ERROR "+parser.getLocation();
+				} else if (localname == "w") // can't close s from here, see END_ELEMENT
+				{
+					writeEvent(event);
+					processWordContent();
+				} else if (flow_tags.contains(localname)) // flow tag = we stay in the sentence
+				{
+					String rend = getRendAttribute();
+					if (rend.contains(rendMulti)) // but this one must be seens as a div tag
+					{
+						writer.writeEndElement(); // close s
+						IN = false; // get out of the sentence
+						multi = true;
+					}
+					writeEvent(event);
+				} else if (div_tags.contains(localname)) // div tag = we must get out the sentence
+				{
+					writer.writeEndElement(); // close s
+					IN = false; // get out of the sentence
+					writeEvent(event);
+				} else {
+					writeEvent(event);
+				}
+				break;
+			case XMLStreamConstants.END_ELEMENT:
+				localname = parser.getLocalName();
+				if (localname == "s") // the sentence seems to already exists, we write the event and get OUT
+				{
+					IN = false;
+					writeEvent(event);
+				} else if (flow_tags.contains(localname)) // closing flow tag = we do nothing
+				{
+					if (multi)
+						println "end of multi?"
+					writeEvent(event);
+				} else if (div_tags.contains(localname)) // closing div tag = we close the current sentence
+				{
+					//println "closing s because of "+localname+" "+parser.getLocation();
+					writer.writeEndElement(); // close s
+					IN = false;
+					writeEvent(event);
+				} else {
+					writeEvent(event);
+				}
+				break;
+
+			case XMLStreamConstants.CHARACTERS: // detect end of sentence
+				writeEvent(event);
+				break;
+			case XMLStreamConstants.COMMENT:
+				writer.writeComment(parser.getText());
+				break
+		}
+	}
+
+	/** The prefix. */
+	String prefix;
+
+	/**
+	 * write the current event.
+	 *
+	 * @param event the stax event
+	 */
+	private void writeEvent(int event)
+	{
+		prefix = parser.getPrefix();
+		if (event == XMLStreamConstants.START_ELEMENT )
+		{
+			localname = parser.getLocalName();
+			if (prefix != null && prefix.length() > 0)
+				writer.writeStartElement(prefix+":"+localname);
+			else
+				writer.writeStartElement(localname);
+			String attrprefix;
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++)
+			{
+				attrprefix = parser.getAttributePrefix(i);
+				if (attrprefix != null && attrprefix.length() > 0)
+					writer.writeAttribute(attrprefix+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+				else
+					writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i))
+			}
+			//writer.writeCharacters("\n");
+		} else if (event == XMLStreamConstants.END_ELEMENT)
+		{
+			writer.writeEndElement();
+			//writer.writeCharacters("\n");
+		} else if (event == XMLStreamConstants.CHARACTERS)
+		{
+			writer.writeCharacters(parser.getText());
+		} else if (event == XMLStreamConstants.COMMENT)
+		{
+			writer.writeComment(parser.getText());
+		}
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args)
+	{
+		File dir = new File ("/home/mdecorde/TXM/corpora/alceste/tokenized")
+		File outdir = new File ("/home/mdecorde/TXM/corpora/alceste/tokenized/s")
+
+		def files = dir.listFiles();//scan directory split
+
+		println("Sentencing "+files.size()+" files")
+		for (File infile : files) {
+			print ".";
+			File result = new File(outdir, infile.getName())
+			TagSentencesXml sentencer = new TagSentencesXml(infile);
+			if (!sentencer.process(result)) {
+				println "error with: "+infile
+				return;
+			}
+
+			//validation
+			if (!ValidateXml.test(result)) {
+				println "validation failed with: "+infile
+				return;
+			}
+		}
+		println "done"
+
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter to transform the "Liturgie" XML files</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/BuildXmlLiturgie.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/BuildXmlLiturgie.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/BuildXmlLiturgie/BuildXmlLiturgie.groovy	(revision 321)
@@ -0,0 +1,136 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.BuildXmlLiturgie
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+
+import org.txm.importer.filters.*;
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class BuildXmlLiturgie.
+ *
+ * @author ayepdieu
+ */
+
+public class BuildXmlLiturgie extends Filter
+{
+	/*
+	 * ville : msDescription>msIdentifier>settlement
+	 * cote : msDescription>msIdentifier>idno
+	 * institution : msDescription>msIdentifier>institution
+	 * usage : msDescription>msItem>respStmt>name
+	 * post : msDescription>history>origin>origDate , notBefore
+	 * ante : msDescription>history>origin>origDate , notAfter
+	 */
+	
+	/** The dir path name. */
+	String dirPathName;;
+	
+	/** The file name. */
+	String fileName;
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	public void SetUsedParam(Object args)
+	{
+		
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	public void filter()
+	{
+		
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	public boolean before()
+	{
+		this.readInput = false;
+		return true;
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	public void after() {
+		println("start transformPage");
+
+		def body = new XmlSlurper().parse(this.input);
+		def nodes = body.text.body.listBibl.msDescription;
+		def idNotice=1;
+		def idp=1;
+
+		def xmlBuilder = new groovy.xml.StreamingMarkupBuilder();
+		xmlBuilder.encoding = this.encodage;
+
+		def xml = xmlBuilder.bind{
+			mkp.declareNamespace("":"http://www.tei-c.org/ns/1.0") 
+
+				nodes.each {
+					//msDescription
+					def ville = it.msIdentifier.settlement.text();
+					def institution = it.msIdentifier.institution.text();
+					def cote = it.msIdentifier.idno.text();
+					def usage = it.msContents.msItem.respStmt.name.text();
+					def post = it.history.origin.origDate.'@notBefore';
+					def ante = it.history.origin.origDate.'@notAfter';
+					
+					def msitems = it.msContents.msItem.msItem;
+					//println msitems.size();
+					notice(n:idNotice, ville:ville, institution:institution, cote:cote, usage:usage,post:post,ante:ante )
+							{
+								idp=1;
+								msitems.each {
+									def pp = (it.p);
+									
+									pp.each {
+										it.attributes().put("n",idp) 
+										mkp.yield(it)
+										
+										idp++
+									}
+								}
+							}
+					idNotice++;
+				}
+		}
+		this.output.write(xml);
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/ReunitBrokenTags.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/ReunitBrokenTags.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/ReunitBrokenTags.groovy	(revision 321)
@@ -0,0 +1,113 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.ReunitBrokenTags;
+
+//Pre-processing extra-word tags (1)
+import org.txm.importer.filters.*;
+import java.util.regex.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ReunitBrokenTags.
+ */
+class ReunitBrokenTags extends Filter {
+	
+	/** The counter. */
+	int counter;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args) {
+
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before() {
+		counter = 0;
+		System.out.println("begin reunitbroken");
+	}
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{
+		println "$counter Reunited tags \n";
+	}
+
+	/** The printcounter. */
+	int printcounter = 0;
+
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment = (line =~ / /).replaceAll(" ");
+		segment = (line =~ /&#160;|&nbsp;|&#xa0;/).replaceAll(" ");
+		while(! (segment ==~ /\A\s*\Z/)) 
+		{
+			if (segment ==~ /^[^<>]*$/) {
+				segment = (segment =~ /^\s+/).replaceFirst(" ");
+				//if(printcounter < 15)
+					//println(segment);
+				printcounter++;
+				output.write(segment+lineSeparator);
+				segment = " ";
+			}
+			else if (segment ==~ /^\s*([^<]*<[^>]+)$/) {
+				segment = (segment =~ /^\s+/).replaceFirst(" ");
+				segment = (segment =~ /\s+$/).replaceFirst(" ");
+				output.write(segment);
+				segment = " ";
+				counter++;
+			}
+			else if( (m = (segment =~ /^\s+([^<]*<[^>]+>)(.*)$/)) )	{
+				segment = m[0][2];
+				output.write(" "+m[0][1]);
+			}
+			else if( (m = (segment =~ /^([^< ]*<[^>]+>)(.*)$/)) ) {
+				segment = m[0][2];
+				output.write(m[0][1]);
+			}
+			else if( (m = (segment =~ /^\s*([^>]*)>(.*)$/)) ) {
+				segment = m[0][2];
+				output.write(" "+m[0][1]+">");
+			}
+			else {
+				output.write(segment+lineSeparator);
+				println "ERROR in tag restitution segment\n";
+				segment = " ";
+			}
+		}
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter of the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenTags/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/package.html
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/package.html	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/package.html	(revision 321)
@@ -0,0 +1,5 @@
+<html>
+<body>
+<p>Deprecated package: Filter of the BFM tokenizer</p>
+</body>
+</html>
\ No newline at end of file

Property changes on: tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/package.html
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/ReunitBrokenWords.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/ReunitBrokenWords.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/filters/ReunitBrokenWords/ReunitBrokenWords.groovy	(revision 321)
@@ -0,0 +1,241 @@
+// Copyright © 2010-2013 ENS de Lyon.
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate:$
+// $LastChangedRevision:$
+// $LastChangedBy:$ 
+//
+package filters.ReunitBrokenWords;
+//Pre-processing extra-word tags (2)
+import org.txm.importer.filters.*;
+import org.txm.tokenizer.*;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ReunitBrokenWords.
+ */
+class ReunitBrokenWords extends Filter
+{
+	
+	/** The tag_all. */
+	def tag_all = TokenizerClasses.tag_all;
+	
+	/** The div_tags. */
+	def div_tags = TokenizerClasses.div_tags;
+	
+	/** The q_tags. */
+	def q_tags = TokenizerClasses.q_tags;
+	
+	/** The extraword_tags. */
+	def extraword_tags = TokenizerClasses.extraword_tags;
+	
+	/** The corr_tags_no_seg. */
+	def corr_tags_no_seg = TokenizerClasses.corr_tags_no_seg;
+	
+	/** The corr_tags. */
+	def corr_tags = TokenizerClasses.corr_tags;
+	
+	/** The word_tags. */
+	def word_tags = TokenizerClasses.word_tags;
+	
+	/** The intraword_tags. */
+	def intraword_tags = TokenizerClasses.intraword_tags;
+	
+	/** The tag. */
+	def tag;
+	
+	/** The tag_end. */
+	def tag_end;
+	
+	/** The tag_name. */
+	def tag_name;
+	
+	/** The word_part_tags. */
+	List<String> word_part_tags = [];
+	
+	/** The unknowned. */
+	Set<String> unknowned = new HashSet<String>();
+	
+	/** The level. */
+	private String level = "extraword";
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#SetUsedParam(java.lang.Object)
+	 */
+	void SetUsedParam(Object args)
+	{
+		try
+		{
+			div_tags = args.get("div_tags");
+			q_tags = args.get("q_tags");
+			extraword_tags = args.get("extraword_tags");
+			corr_tags_no_seg = args.get("corr_tags_no_seg");
+			corr_tags = args.get("corr_tags");
+			word_tags = args.get("word_tags");
+			intraword_tags = args.get("intraword_tags");
+		}
+		catch(Exception e)
+		{
+			System.err.println(e);
+			System.err.println("PreProcess2 needs 1 Map with args  :\n div_tags, q_tags, extraword_tags, " +
+					"corr_tags_no_seg, corr_tags, word_tags, intraword_tags")
+		}
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#before()
+	 */
+	boolean before(){	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#after()
+	 */
+	void after()
+	{ 
+		if(unknowned.size() > 0)
+			println "Unknown tags: "+unknowned;
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.txm.importer.filters.Filter#filter()
+	 */
+	void filter()
+	{
+		def m;
+		def segment;
+		boolean firstTest = true;
+
+		while(!(line ==~ /\A\s*\Z/))// tant que la ligne contient des charracters autres que des blancs
+		{
+			if ((m = line =~ /^([^<]*)$/)) 	 // si la ligne NE commence PAS par une balise
+			{
+				if(firstTest)
+				{
+					//output.print(lineSeparator);
+					firstTest=false;
+				}
+				output.print(line);//on ecrit la ligne
+				line = " "
+			}
+			else if ((m = line =~ /\A\s*(<\/?($extraword_tags)( [^>]*|\/)?>)(.*)\Z/)) // balise externe 	
+			{	
+				tag = m[0][1]; // le nom du tag
+				line = m[0][4];
+				//println "balise ext : "+tag
+				level = "extraword"; //di qu'on est ds un tag extern
+				output.print("\n$tag \n");
+			}
+			else if ((m = line =~ /\A\s*(<($corr_tags)( [^>]*|\/)?>)(.*)\Z/)) //balise corr ouvrante		
+			{   
+				tag = m[0][1];
+				String tag_end = m[0][3]; // ( [^>]*|\/)?>)
+				line = m[0][4];
+				//println "balise corr : "+tag+" end : "+tag_end
+				if(tag_end != null && tag_end.find("word_part")) //on trouve un word_part	
+				{
+					//println "set level : intra :"+tag
+					word_part_tags.add(m[0][2]);
+					level = "intraword";
+					output.print(tag);
+				}
+				else {
+					level = "extraword";
+					output.print("\n$tag\n");
+				}
+			} 
+			else if ((m = line =~ /^\s*(<($word_tags)( [^>]*|\/)?>)(.*)$/)) 				
+			{
+				tag = m[0][1];
+				line = m[0][4];
+				//println "balise word : "+tag
+				//output.print("\n$tag");
+				output.write(line)
+				level = "intraword";
+			}
+			else if ((m = line =~ /^\s*(<($intraword_tags)( [^>]*|\/)?>)(.*)$/))			
+			{
+				tag = m[0][1];
+				line = m[0][4];
+				//println "balise intra : "+tag
+				output.print(tag);
+				level = "intraword";
+			}
+			else if ((m = line =~ /^\s*(<\/($corr_tags|$word_tags|$intraword_tags)>)\s*(.*)$/))
+			{
+				tag = m[0][1];
+				tag_name = m[0][2];
+				line = m[0][3];
+				//println "balise tous : "+tag+" level : "+level
+				if (level ==~ /extraword/) 
+				{//println "write extra"
+					output.print("\n");
+				}//else{println "dont write intra"}
+				
+				output.print(tag);
+				if (tag_name ==~ /^($corr_tags|$word_tags)$/) 				
+				{
+					//println("> corrwordtags "+tag_name+" level "+level+" Word_part_tag "+word_part_tags)
+					if (level ==~ /extraword/) 
+					{
+						output.print("\n");
+					}
+					//else{println "dont write intra corr"}
+					if(word_part_tags.size() > 0)
+					{
+						//String word_part_tag = word_part_tags[-1];
+						//if(tag_name == word_part_tag)
+						//{
+							word_part_tags.remove(word_part_tags.size()-1)
+							if(word_part_tags.size() == 0 )
+							{
+								level = "extraword";
+								//output.print("\n");
+							}
+						//}
+					}
+					else
+						level = "extraword"
+				}
+			}
+			else if ((m = line =~ /^\s*($tag_all)(.*)$/)) 				
+			{
+				//println "balise autres : "+tag
+				tag = m[0][1];
+				line = m[0][2];
+				output.print("\n$tag\n");
+				if(!unknowned.contains(tag))
+				{
+					unknowned.add(tag)
+					//println "Warning: unknown tag $tag in line : "+line;
+				}
+				
+			}
+			else{			
+				println "Error in "+ line;
+				line = " ";
+			}
+			firstTest=false;
+		}
+		// End
+	}
+}
Index: tmp/org.txm.groovy.core/src/groovy/tests/advancedPartition.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/tests/advancedPartition.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/tests/advancedPartition.groovy	(revision 321)
@@ -0,0 +1,719 @@
+package tests
+
+import org.txm.rcpapplication.views.*
+import org.txm.searchengine.cqp.corpus.*
+
+
+/*
+import java.awt.Dimension
+import java.util.UUID;
+
+import javax.swing.JFrame;
+import javax.swing.JOptionPane;
+
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.MemCqiClient
+import org.txm.searchengine.cqp.MemCqiServer
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.functions.ReferencePattern
+import org.txm.rcpapplication.views.CorporaView
+import org.txm.searchengine.cqp.corpus.query.Query
+*/
+
+
+/**
+ * Create partition with advanced CQL queries and autoname the parts.
+ * Can create partitions by defining a multi-level structural units hierarchy or by defining several properties values for one structural unit.
+ * Can define some structural units to remove from the resulting parts subcorpus.
+ *
+ */
+
+// TODO : Log
+println "******************************** Starting ********************************************";
+
+
+// *************************** User parameters ************************************************************
+
+
+// Test 1 : partition en excluant des sections
+def CORPUS_NAME = "LIVRETOPERA2"								// The name of the corpus to partition
+
+def SUBCORPUS_NAME = null										// The name of the subcorpus to partition
+//def SUBCORPUS_NAME = "TEST"										// The name of the subcorpus to partition if needed
+// FIXME: subcorpus management is bugged, when trying to create a CA on a Partition created on a subcorpus, it doesn't work because of the getLexicon() method which throws a CqiCqpErrorErrorGeneral exception
+
+PartitionQueriesGenerator.STRUCTURAL_UNITS = ['sp']; 					// Applying the partition on these structural units
+PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['who'];		// Applying the partition on these properties of structural units defined above
+
+PartitionQueriesGenerator.STRUCTURAL_UNITS_TO_IGNORE = ['speaker'];	// These structural units will be removed from the partition. NOTE : If doing that you need to define an EXPAND_TARGET
+																		// if you want some subcorpus parts that will manage sequential positions queries
+
+PartitionQueriesGenerator.EXPAND_TARGET = 'lg'; 						// Expand the results to this structural unit parent target. If some structural units to ignore are defined
+																		// and the expand target is upper on the hierarchy than them, the structural units to ignore WON'T be ignored
+
+//PartitionQueriesGenerator.PARTITION_NAME = 'gugu';					// The partition name. If empty or not defined, the partition will be autonamed regarding of the structural units and properties
+
+
+
+
+// Test 2 : partition sur arborescence (multi-niveau sur 2 niveaux)
+//def CORPUS_NAME = "LIVRETOPERA10TEXTES"
+//PartitionQueriesGenerator.STRUCTURAL_UNITS = ['text', 'div1'];
+//PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['id', 'name'];
+
+
+// Test 3 : partition sur arborescence (multi-niveau sur 3 niveaux)
+//def CORPUS_NAME = "LIVRETOPERA10TEXTES"
+//PartitionQueriesGenerator.STRUCTURAL_UNITS = ['text', 'div1', 'div2'];
+//PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['id', 'name', 'name'];
+
+
+// Test 4 : partition sur arborescence (multi-niveau sur 4 niveaux)
+// Ajouter le 'n' permet par exemple ici de trier les parties par ordre de scène car sinon le tri est problématique, ex : "SCENE II" passe avant "SCENE PREMIERE"
+//def CORPUS_NAME = "LIVRETOPERA2"
+//PartitionQueriesGenerator.STRUCTURAL_UNITS = ['text', 'div1', 'div2', 'div2'];
+//PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['id', 'name', 'n', 'name'];
+
+
+// Test 5 : partitions croisées (sur plusieurs propriétés d'une même structure)
+//def CORPUS_NAME = "DISCOURS"
+//PartitionQueriesGenerator.STRUCTURAL_UNITS = ['text', 'text'];
+//PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['loc', 'type'];
+
+
+
+// Tests
+//def CORPUS_NAME = "LIVRETOPERA10TEXTES"
+//PartitionQueriesGenerator.STRUCTURAL_UNITS = ['div1', 'sp'];
+//PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['name', 'who'];
+
+// Tests
+//def CORPUS_NAME = "LIVRETOPERA10TEXTES"
+//PartitionQueriesGenerator.STRUCTURAL_UNITS = ['div1'];
+//PartitionQueriesGenerator.STRUCTURAL_UNITS_PROPERTIES = ['n'];
+//PartitionQueriesGenerator.PART_NAMES_PREFIX = 'act_';
+
+
+
+
+// *************************** Debug parameters ************************************************************
+
+
+PartitionQueriesGenerator.DEBUG = 0; 	// If DEBUG != 0 then partition is not created,
+										// script only outputs the created queries and part names strings in console
+
+
+
+// *************************** End of parameters ************************************************************
+
+
+
+
+
+
+
+// Running
+def partition = PartitionQueriesGenerator.createPartition(CORPUS_NAME, SUBCORPUS_NAME);
+
+
+// Refreshing the RCP component
+if(partition != null)	{
+	monitor.syncExec(new Runnable() {
+			public void run() {
+				CorporaView.refresh();
+				CorporaView.expand(partition.getParent());
+			}
+		});
+}
+
+
+
+
+
+/**
+ * Create a list of queries and part names regarding the structural units, structural units properties, structural units to ignore user defined lists and expand target value specified.
+ * @author s
+ *
+ */
+public class PartitionQueriesGenerator	{
+
+	public static int DEBUG = 0;								// si DEBUG != 0, alors les requêtes sont affichées mais la partition n'est pas créée
+
+	public static String PARTITION_NAME = '';
+	public static String[] STRUCTURAL_UNITS = [];				// Liste des unités structurelles sur lesquelles effectuer la partition, ex: ['text', 'div1']
+	public static String[] STRUCTURAL_UNITS_PROPERTIES = [];	// Propriétés des unités structurelles sur lesquelles effectuer la partition, ex : ['id', 'name']
+	public static String[] STRUCTURAL_UNITS_TO_IGNORE = [];		// Structure à ignorer, ex. CQL : !speaker
+	public static String PART_NAMES_PREFIX = '';
+	public static String EXPAND_TARGET = null;					// Expand to target, englobe les empans jusqu'à la balise parente spécifiée. NOTE : Le expand entre en conflit avec les sections à ignorer.
+																// Si la target est à un niveau supérieur aux balises à ignorer, il les remet dans liste de résultats CWB et elles ne sont donc pas ignorées
+
+
+
+	public static ArrayList<String> queries = new ArrayList<String>();
+	public static ArrayList<String> partNames = new ArrayList<String>();
+
+
+	/**
+	 * Init the generator and process.
+	 * @param corpusName
+	 */
+	public static Partition createPartition(String corpusName, String subcorpusName)	{
+
+		if(STRUCTURAL_UNITS.size() > 0 && STRUCTURAL_UNITS.size() == STRUCTURAL_UNITS_PROPERTIES.size())	{
+
+			// TODO : Log
+			println '**************************************************************************************************************'
+			println 'Creating the queries on corpus "' + corpusName + "'" ;
+
+			Corpus corpus = CorpusManager.getCorpusManager().getCorpus(corpusName);
+
+
+			// Subcorpora
+			if(subcorpusName != null)	{
+				corpus = corpus.getSubcorpusByName(subcorpusName);
+			}
+
+
+
+			// Recursing through the corpus and subcorpus
+			process(corpus, 0, '', '');
+
+			// Finalizing the queries
+			finalizeQueries();
+
+
+			// TODO : Debug
+			// Displaying the partition name
+			println '';
+			println 'PARTITION_NAME: ' + PartitionQueriesGenerator.PARTITION_NAME;
+
+			// Displaying the queries
+			println 'Queries (count = ' + PartitionQueriesGenerator.queries.size() + '):';
+			for(query in PartitionQueriesGenerator.queries)	{
+				println query;
+			}
+			// Displaying the part names
+			println 'Partnames (count = ' + PartitionQueriesGenerator.partNames.size() + '):';
+			for(partName in PartitionQueriesGenerator.partNames)	{
+				print partName + ' / ';
+			}
+
+
+			// TODO : Log
+			println 'Queries created.';
+
+			// Creating the partition
+			if(DEBUG == 0 && queries.size() == partNames.size()) {
+				return corpus.createPartition(PARTITION_NAME, queries, partNames);
+			}
+
+		}
+		else	{
+			// TODO : Log
+			println 'Structural units count or structural units properties count error.';
+			return null
+		}
+
+	}
+
+
+
+
+	/**
+	 * Recurse through structural units and structural units properties of corpus and create the queries and the part names.
+	 * @param corpus the corpus or subcorpus
+	 * @param index the index for recursion
+	 * @param tmpQuery the temporary query for creating subcorpus part
+	 * @param tmpPartName the temporary part name of the subcorpus part
+	 */
+	protected static void process(Corpus corpus, int index, String tmpQuery, String tmpPartName)	{
+
+
+		// End of array
+		if(index >= STRUCTURAL_UNITS.size())	{
+
+			queries.add(tmpQuery);
+			partNames.add(PART_NAMES_PREFIX + tmpPartName);
+
+			return;
+		}
+
+		StructuralUnit su = corpus.getStructuralUnit(STRUCTURAL_UNITS[index]);
+		StructuralUnitProperty sup = su.getProperty(STRUCTURAL_UNITS_PROPERTIES[index]);
+
+		// TODO : Log
+		println ''
+		if(index == 0)	{
+			println 'Pocessing Structural Unit Property "' + sup.getFullName() + '" on mother corpus "' + corpus.getName() + '"';
+		}
+		else	{
+			println 'Pocessing Structural Unit Property "' + sup.getFullName() + '" on subcorpus part "' + tmpPartName + '"';
+		}
+		println ''
+
+
+		// Creating the queries parts for each structural units properties values
+		//for (supValue in sup.getOrderedValues()) { // TODO : signaler bug Matthieu, on ne devrait pas être obligé de repasser le sous-corpus à la méthode car sup a déjà été créée depuis le sous-corpus ? getValues() bugge aussi
+		for (supValue in sup.getOrderedValues(corpus)) {
+
+			// TODO : Log
+			println ''
+			println 'Value "' + supValue + '"';
+			println ''
+
+
+			// Getting the subcorpus linked to the structural unit property value
+			Subcorpus tmpSubcorpus = corpus.createSubcorpusWithQueryString(su, sup, supValue, "tmp" + UUID.randomUUID());
+
+			// Partition conditions and part name separators
+			String and = '';
+			String underscore = '';
+			if(tmpQuery != '')	{
+				underscore = '_';
+				and = ' & ';
+			}
+
+
+			process(tmpSubcorpus, index + 1, (tmpQuery + and + '_.' + sup.getFullName() + '="' + supValue + '"'), tmpPartName + underscore + supValue);
+
+			// Deleting the temporary subcorpus
+			// TODO : bug : cette méthode ne supprime pas le corpus sans doute car il faut que le sous-corpus ne contienne pas d'autres sous-corpus ? le delete() en revanche fonctionne.
+//			corpus.dropSubcorpus(tmpSubcorpus);
+			tmpSubcorpus.delete();
+
+		}
+	}
+
+
+	/**
+	 * 	Autoname the partition.
+	 * @param partitionName
+	 */
+	protected static void autoNamePartition(String partitionName)	{
+
+		// Structural units names and properties
+		for(int i = 0; i < STRUCTURAL_UNITS.size(); i++)	{
+			partitionName +=  STRUCTURAL_UNITS[i] + '_' + STRUCTURAL_UNITS_PROPERTIES[i] + '.';
+		}
+
+		// Structural units to ignore
+		for(int i = 0; i < STRUCTURAL_UNITS_TO_IGNORE.size(); i++)	{
+			partitionName +=  'NOT_' + STRUCTURAL_UNITS_TO_IGNORE[i] + '.';
+		}
+
+
+		// Removing last point in name
+		PARTITION_NAME = partitionName.substring(0, partitionName.length() - 1);
+	}
+
+
+	/**
+	 * Finalize the queries.
+	 */
+	protected static void finalizeQueries()	{
+
+		String expandTo = '';
+		// Expanding to user defined target
+		if(EXPAND_TARGET != null && EXPAND_TARGET != '')	{
+			expandTo = ' expand to ' + EXPAND_TARGET;
+		}
+		// Expanding to last child structural unit in user defined hierarchy
+		else if(STRUCTURAL_UNITS_TO_IGNORE.size() == 0)	{
+			expandTo = ' expand to ' + STRUCTURAL_UNITS[STRUCTURAL_UNITS.size() - 1];
+		}
+
+		// Autonaming the partition
+		if(PARTITION_NAME == '')	{
+			autoNamePartition(PARTITION_NAME);
+			// Finalizing partition name
+			PARTITION_NAME += expandTo.replace(' expand to', '.EXPAND TO').replace(' ', '_');
+		}
+
+
+		// Finalizing queries
+		for(int j = 0; j < queries.size(); j++)	{
+
+			String queryEnd = '';
+
+			// Removing some sections
+			for(sectionToIgnore in STRUCTURAL_UNITS_TO_IGNORE)	{
+				queryEnd += ' & !' + sectionToIgnore;
+			}
+
+			queryEnd += ']' + expandTo;
+
+			queries.set(j, '[' +  queries.get(j) + queryEnd);
+		}
+	}
+
+}
+
+
+
+
+
+
+/*
+// Test dialogue de confirmation avant création de la partition
+JFrame frame = new JFrame('test');
+//frame.setMinimumSize(new Dimension(400, 400));
+//frame.setVisible(true);
+//frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+int n = JOptionPane.showConfirmDialog(
+	frame, "Would you like green eggs and ham?",
+	"An Inane Question",
+	JOptionPane.YES_NO_OPTION);
+if (n == JOptionPane.YES_OPTION) {
+frame.setTitle("Ewww!");
+} else if (n == JOptionPane.NO_OPTION) {
+frame.setTitle("Me neither!");
+} else {
+frame.setTitle("Come on -- tell me!");
+}*/
+
+
+
+
+
+//// Récupération des unités structurelles et création de la liste des requêtes CQL
+//int i = 0;
+//ArrayList<String> queries = new ArrayList<String>();
+//ArrayList<String> partNames = new ArrayList<String>();
+//
+//
+//int currentQueriesCount;
+//if(STRUCTURAL_UNITS.size() > 0 && STRUCTURAL_UNITS.size() == STRUCTURAL_UNITS_PROPERTIES.size())	{
+//
+//	// Parcours des unités strcuturelles à traiter
+//	for (suName in STRUCTURAL_UNITS) {
+//
+//		// Définition auto du nom de partition
+//		PARTITION_NAME += suName + '_';
+//
+//
+//		StructuralUnit su = corpus.getStructuralUnit(suName);
+//
+//
+//		// TODO : Debug
+//		println ''
+//		println 'Structural Unit Name: ' + su.getName();
+//
+//
+//		// Récupération des propriétés d'unités structurelles
+//		for (StructuralUnitProperty sup in su.getProperties())	{
+//
+//
+//			if(STRUCTURAL_UNITS_PROPERTIES[i] != '' && sup.getName() == STRUCTURAL_UNITS_PROPERTIES[i])	{
+//
+//				// Définition auto du nom de partition
+//				PARTITION_NAME += STRUCTURAL_UNITS_PROPERTIES[i] + ".";
+//
+//				// TODO : Debug
+//				println "Structural Unit Property: " + sup.getName() + " ";
+//
+//
+//				// Récupération des valeurs des propriétés d'unités structurelles
+//
+//				//for (supValue in sup.getOrderedValues()) {
+//				for (supValue in sup.getOrderedValues()) {
+//
+//					// Récupération du sous-corpus lié à la valeur de propriété de structure
+//					Subcorpus tmpSubcorpus = corpus.createSubcorpusWithQueryString(su, sup, supValue, "tmp");
+//
+//					String query = suName + '_' + STRUCTURAL_UNITS_PROPERTIES[i] + '="' + supValue + '"';
+//
+//
+//					queries.add(query);
+//					partNames.add(supValue);
+//
+//					// TODO : Debug
+//					print supValue + " ";
+//
+//					// Suppression du sous-corpus temporaire
+//					corpus.dropSubcorpus(tmpSubcorpus);
+//
+//				}
+//
+//			}
+//		}
+//
+//
+////		// Ajout des niveaux inférieurs
+////		for(int i = 1; i < STRUCTURAL_UNITS.size(); i++)	{
+////
+////			for(int j = 0; j < queries.size(); j++)	{
+////				queries.set(j, queries.get(j) + ' & _.' + STRUCTURAL_UNITS[i] + '_' + STRUCTURAL_UNITS_PROPERTIES[i] + '=".*"');
+////
+////				// Noms de partitions
+////				partNames.set(j, partNames.get(j) + '_' + STRUCTURAL_UNITS[i] + '_'  + STRUCTURAL_UNITS_PROPERTIES[i]);
+////			}
+////		}
+//
+//
+//
+//
+//
+//
+//		i++;
+//	//	}
+//	//	else	{
+//	//
+//	//	}
+//
+//	}
+//
+//
+//
+//
+//
+//	// Clôture des requêtes
+//	for(int j = 0; j < queries.size(); j++)	{
+//
+//		String queryEnd = '';
+//
+//		// Suppression de sections
+//		for(sectionToIgnore in STRUCTURAL_UNITS_TO_IGNORE)	{
+//			queryEnd += ' & !' + sectionToIgnore;
+//		}
+//
+//		queryEnd += ']';
+//
+//		// Expand to target
+//		if(EXPAND_TARGET != null && EXPAND_TARGET != '')	{
+//			queryEnd += ' expand to ' + EXPAND_TARGET;
+//		}
+//
+//		queries.set(j, '[_.' +  queries.get(j) + queryEnd);
+//	}
+//}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+////for (suName in STRUCTURAL_UNITS) {
+//if(STRUCTURAL_UNITS.size() > 0 && STRUCTURAL_UNITS.size() == STRUCTURAL_UNITS_PROPERTIES.size())	{
+//
+//	suName = STRUCTURAL_UNITS[0]
+//	//if(STRUCTURAL_UNITS.size() > 1)	{
+//
+//	// Définition auto du nom de partition
+//	PARTITION_NAME += suName + '_';
+//
+//
+//	// TODO : Debug
+//	println ''
+//	println "Structural Unit Name: $suName"
+//
+//	// Récupération des valeurs des propriétés d'unités structurelles
+//	for (sup in discours.getStructuralUnitProperties(suName.asType(String)))	{
+//
+//		supName = STRUCTURAL_UNITS_PROPERTIES[0];
+//
+//		if(supName != '' && sup.getName() == supName)	{
+//
+//			// Définition auto du nom de partition
+//			PARTITION_NAME += supName + ".";
+//
+//			// TODO : Debug
+//			println "Structural Unit Property: " + supName + " ";
+//
+//			for (supValue in sup.getOrderedValues()) {
+//
+//				String query = '[_.' + suName + '_' + supName + '="' + supValue + '"';
+//
+//				// Suppression de sections
+//				for(sectionToIgnore in STRUCTURAL_UNITS_TO_IGNORE)	{
+//					query += ' & !' + sectionToIgnore;
+//				}
+//
+//
+//
+//				queries.add(query);
+//				partNames.add(supValue);
+//
+//				// TODO : Debug
+//				print supValue + " ";
+//			}
+//		}
+//	}
+//
+//
+//	// Ajout des niveaux inférieurs
+//	for(int i = 1; i < STRUCTURAL_UNITS.size(); i++)	{
+//
+//		for(int j = 0; j < queries.size(); j++)	{
+//			queries.set(j, queries.get(j) + ' & _.' + STRUCTURAL_UNITS[i] + '_' + STRUCTURAL_UNITS_PROPERTIES[i] + '=".*"');
+//
+//			// Noms de partitions
+//			partNames.set(j, partNames.get(j) + '_' + STRUCTURAL_UNITS[i] + '_'  + STRUCTURAL_UNITS_PROPERTIES[i]);
+//		}
+//	}
+//
+//
+//	// Clôture des requêtes
+//	for(int j = 0; j < queries.size(); j++)	{
+//
+//		String queryEnd = ']';
+//
+//		// Expand to target
+//		if(EXPAND_TARGET != null && EXPAND_TARGET != '')	{
+//			queryEnd += ' expand to ' + EXPAND_TARGET;
+//		}
+//
+//		queries.set(j, queries.get(j) + queryEnd);
+//	}
+//
+//
+//
+//	//i++;
+////	}
+////	else	{
+////
+////	}
+//
+////}
+//}
+
+
+
+// Fonctionnelle pour 1 niveau de hiérarchie
+//// Récupération des unités structurelles et création de la liste des requêtes CQL
+//int i = 0;
+//String supName;
+//ArrayList queries = new ArrayList();
+//ArrayList partNames = new ArrayList();
+//
+//for (suName in STRUCTURAL_UNITS) {
+//
+//
+//	// Définition auto du nom de partition
+//	NAME += suName + '_';
+//
+//
+//
+//	println "Structural Unit Name: $suName"
+//
+//	// Récupération des valeurs des propriétés d'unités structurelles
+//	for (sup in discours.getStructuralUnitProperties(suName.asType(String)))	{
+//
+//		supName = STRUCTURAL_UNITS_PROPERTIES[i];
+//
+//		if(supName != '' && sup.getName() == supName)	{
+//
+//			// Définition auto du nom de partition
+//			NAME += supName + ".";
+//
+//			// TODO : Debug
+//			println "Structural Unit Property: " + supName + " ";
+//
+//			for (supValue in sup.getOrderedValues()) {
+//
+//				String query = '[_.' + suName + '_' + supName + '="' + supValue + '"';
+//
+//				// Suppression de sections
+//				for(sectionToIgnore in STRUCTURAL_UNITS_TO_IGNORE)	{
+//					query += ' & !' + sectionToIgnore;
+//				}
+//
+//				 query += ']';
+//
+//				// Expand to target
+//				if(EXPAND_TARGET != null && EXPAND_TARGET != '')	{
+//					query += ' expand to ' + EXPAND_TARGET;
+//				}
+//
+//
+//				queries.add(query);
+//				partNames.add(supValue);
+//
+//				// TODO : Debug
+//				print supValue + " ";
+//			}
+//		}
+//	}
+//	i++;
+//}
+
+
+
+
+//// start
+//if (queries.size() == partnames.size()) {
+////	def discours = CorpusManager.getCorpusManager().getCorpus(CORPUS)
+//	def partition;
+//	if (SUBCORPUS == null) {
+//		println "partition build with $discours"
+//		partition = discours.createPartition(NAME, queries.as, PARTNAMES)
+//	} else {
+//		def subcorpus
+//		if (SUBCORPUSQUERY == null) {
+//			println "partition build with subcorpus by name $SUBCORPUS"
+//			subcorpus = discours.getSubcorpusByName(SUBCORPUS)
+//			if (subcorpus == null) {
+//				println "Error: SUBCORPUS NOT FOUND with name : $SUBCORPUS"
+//			}
+//		} else {
+//			println "partition build with subcorpus with query $SUBCORPUSQUERY"
+//			subcorpus = discours.createSubcorpus(new Query(SUBCORPUSQUERY), SUBCORPUS)
+//		}
+//		partition = subcorpus.createPartition(NAME, QUERIES, PARTNAMES)
+//	}
+//
+//	def list = Arrays.asList(partition.getPartSizes())
+//	println "Partition created $NAME: "+list+" parts"
+//	// println "Total size: "+list.sum()+" - is equal to (sub)corpus size : "+(list.sum() == partition.getCorpus().getSize())
+//    println "Total size: "+list.sum()+" - (sub)corpus size : "+(partition.getCorpus().getSize())
+//
+//	monitor.syncExec(new Runnable() {
+//	public void run() {
+//		CorporaView.refresh();
+//		CorporaView.expand(partition.getParent());
+//	}
+//});
+//} else {
+//	println "QUERIES.size() != PARTNAMES.size() = "+QUERIES.size()+" != "+PARTNAMES.size()
+//}
+
+
+
+
+// parameters
+//def CORPUS_NAME = "LIVRETOPERA10TEXTES"
+////def SUBCORPUS = "NOMDUSOUSCORPUS"
+//def SUBCORPUS = null // si partition sur corpus
+////def SUBCORPUSQUERY = '"je" expand to s'
+//def SUBCORPUSQUERY = null // si sous-corpus par nom
+
+
+
+
+
+
+//def TEXT = "01_ACHILLE ET POLYXENE"
+// def TEXT = "02_ZEPHIRE ET FLORE"
+/*def QUERIES = [
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="1" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="2" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="3" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="4" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="5" & _.sp_who=".*"]+</lg>',
+]
+def PARTNAMES = [
+"acte_1",
+"acte_2",
+"acte_3",
+"acte_4",
+"acte_5",
+]*/
Index: tmp/org.txm.groovy.core/src/groovy/tests/tests_cql.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/tests/tests_cql.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/tests/tests_cql.groovy	(revision 321)
@@ -0,0 +1,318 @@
+package tests
+import java.util.UUID;
+
+import org.txm.Toolbox
+import org.txm.searchengine.cqp.MemCqiClient
+import org.txm.searchengine.cqp.MemCqiServer
+import org.txm.searchengine.cqp.corpus.*
+import org.txm.functions.ReferencePattern
+import org.txm.rcpapplication.views.*
+import org.txm.searchengine.cqp.corpus.query.Query
+
+// Debug
+println "******************************** Starting ********************************************";
+
+
+// Tests suite de requêtes directement sur le serveur et création manuelle du sous-corpus
+
+// parameters
+def CORPUS = "LIVRETOPERA10TEXTES"
+//def SUBCORPUS = "NOMDUSOUSCORPUS"
+// def SUBCORPUS = null // si partition sur corpus
+//def SUBCORPUSQUERY = '"je" expand to s'
+//def SUBCORPUSQUERY = null // si sous-corpus par nom
+
+
+def discours = CorpusManager.getCorpusManager().getCorpus(CORPUS)
+def subcorpusCqpId = 'S' + UUID.randomUUID().toString();
+
+try {
+	def server = (MemCqiServer)Toolbox.getCqiServer();
+	
+//	server.query('show corpora;');
+//	server.query("info " + CORPUS + ";");
+	
+	server.query(CORPUS + ";");
+	
+	//client.query(SUBCORPUSQUERY);
+
+	// Suppression de variable de résultats
+//	server.query("discard A;");
+//	server.query("discard B;");
+//	server.query("discard C;");
+	
+//	server.query('C = /region[sp,a]::a.sp_who="IPHIS";');
+
+	
+// Test mono ligne : ne fonctionne pas en mode server
+//	server.query('A = /region[sp]; B = /region[speaker]; C = difference B A;');
+		
+	
+		
+//	server.query('A = /region[sp];');
+//	server.query('B = /region[speaker];');
+//	server.query('C = difference A B;');
+
+	// Tests options d'affichage
+	server.query('show cd;');
+	server.query('show +div1 +div2 +sp +lg +l +speaker;'); // Affichage des balises XML dans les listes de résultats
+	//server.query('set ShowTagAttributes on;');
+	
+	// Tests
+//	server.query('A = /region[sp,a]::a.sp_who="THALIE";');
+//	server.query('B = /region[speaker];');
+//	server.query(subcorpusCqpId + ' = diff A B;');
+
+	
+	//server.query(subcorpusCqpId + ' = B;');
+	
+	
+	
+//	server.query(subcorpusCqpId + ' = <sp>[(_.) != "speaker"]+</sp>;');
+	//server.query(subcorpusCqpId + ' = <div1>[]*<sp>[]+</sp>[]*</div1>;');
+
+	//server.query(subcorpusCqpId + ' = <div1>[!speaker]*</div1>;');
+	
+// server.query(subcorpusCqpId + ' = <lg>[_.div1_n=".*" & _.sp_who="THALIE"]+</lg>;');	
+//	server.query(subcorpusCqpId + ' = /region[i];');
+	
+
+	// Tests de requête sur le sous-corpus créé
+	//server.query('set MatchingStrategy shortest;'); // standard, shortest, longest, traditional
+	//server.query(subcorpusCqpId + ' = [div1 & _.sp_who="THALIE" & !speaker];');
+	//server.query(subcorpusCqpId + ' = [_.div1_name=".*" & !speaker];');
+	//server.query(subcorpusCqpId + ' = [_.div1_name=".*" & !speaker] expand to lg;');
+	server.query(subcorpusCqpId + ' = [div1 & !speaker] expand to lg;');
+	
+	
+	//server.query('group ' + subcorpusCqpId + ' match word;');
+
+	server.query(subcorpusCqpId + ';');
+	
+		
+	//server.query(subcorpusCqpId + ' = [] !;');
+	//server.query(subcorpusCqpId + ';');
+	//server.query(subcorpusCqpId + ' = [] !;');
+//	server.query(subcorpusCqpId + ' = [] expand to div1 & !speaker;');
+	
+	
+	
+	//server.query(subcorpusCqpId + ' = [div1 & !speaker];');
+	//server.query(subcorpusCqpId + ' = [div1 & !speaker] expand to lg;');
+	//server.query(subcorpusCqpId + ' = [div1 & !speaker] expand to lg;');
+	
+	//server.query(subcorpusCqpId + ' = [div1 & !speaker];');
+	
+	//server.query(subcorpusCqpId + ' = [div1 &_.sp_who="THALIE" & !speaker];');
+	
+	//server.query(subcorpusCqpId + ' = [div1 &_.sp_who="THALIE" & !speaker];');
+	//server.query(subcorpusCqpId + ' = "fin" [];');
+	//server.query(subcorpusCqpId + ' = [div1 & _.sp_who="THALIE" & !speaker];');
+	
+	// Affichage du sous-corpus
+	//server.query("cat " + subcorpusCqpId + " 0 10;")
+	server.query("cat " + subcorpusCqpId + ";")
+	
+	//server.cqpQuery(discours.getCqpId(), 'subcorpusid', SUBCORPUSQUERY);
+
+		
+	
+	// Affichage des erreurs
+	try {
+		System.out.println("client.getLastCqiError() : "+server.getLastCqiError());
+	} catch(Exception e) { System.out.println("Exception : "+e);}
+	try {
+		System.out.println("client.getLastCQPError() : "+server.getLastCQPError());
+	} catch(Exception e) { System.out.println("Exception : "+e);}
+
+	
+	
+	// Création manuelle du sous-corpus
+//	Subcorpus subcorpus = new Subcorpus(subcorpusCqpId, "subcorpus" + subcorpusCqpId.toString().substring(0, 5), discours, new Query()); // FIXME : est-ce que cela pose un problème de passer une Query() vide ici ?
+//	discours.subcorpora.add(subcorpus);
+//	subcorpus.registerToParent();
+	
+	// Création manuelle d'une partition
+//	subcorpus.createPartition(NAME, QUERIES, PARTNAMES)
+	
+	
+}
+ catch(Exception e) {
+	  System.out.println("Exception : "+e);
+  }
+
+
+
+// Refresh de l'interface
+monitor.syncExec(new Runnable() {
+	public void run() {
+		CorporaView.refresh();
+		//CorporaView.expand(partition.getParent());
+	}
+});
+
+
+
+
+
+
+//// parameters
+//def CORPUS = "LIVRETOPERA10TEXTES"
+////def SUBCORPUS = "NOMDUSOUSCORPUS"
+//def SUBCORPUS = null // si partition sur corpus
+////def SUBCORPUSQUERY = '"je" expand to s'
+//def SUBCORPUSQUERY = null // si sous-corpus par nom
+//
+//
+//
+//// Liste des unités strcuturelles
+////def HIERARCHY = ['text_id', 'div1_n', 'sp_who', 'lg'];
+//def STRUCTURAL_UNITS = ['text', 'div1', 'sp'];
+//def STRUCTURAL_UNITS_PROPERTIES = ['id', 'n', 'who'];
+////def TARGET = 'lg';
+//
+//String NAME = '';
+
+
+
+
+//def TEXT = "01_ACHILLE ET POLYXENE"
+// def TEXT = "02_ZEPHIRE ET FLORE"
+/*def QUERIES = [
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="1" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="2" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="3" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="4" & _.sp_who=".*"]+</lg>',
+'<lg>[_.text_id="' + TEXT + '" & _.div1_n="5" & _.sp_who=".*"]+</lg>',
+]
+def PARTNAMES = [
+"acte_1",
+"acte_2",
+"acte_3",
+"acte_4",
+"acte_5",
+]*/
+
+
+//def discours = CorpusManager.getCorpusManager().getCorpus(CORPUS)
+//
+//
+//// Récupération des unités structurelles
+//def i = 0;
+//def supName;
+//def QUERIES;
+//def query;
+//
+//for (suName in STRUCTURAL_UNITS) {
+//
+//
+//	// Définition auto du nom de partition
+//	NAME += suName + '_';
+//
+//	//query = '';
+//	
+//	
+//	println "Structural Unit Name: $suName"
+//	
+//	// Récupération des valeurs des propriétés d'unités structurelles
+//	for (sup in discours.getStructuralUnitProperties(suName.asType(String)))	{
+//		
+//		supName = STRUCTURAL_UNITS_PROPERTIES[i];
+//		
+//		if(supName != '' && sup.getName() == supName)	{
+//			
+//			// Définition auto du nom de partition
+//			NAME += supName + ".";
+//			
+//			println "Structural Unit Property: $supName";
+//			
+//			for (supValue in sup.getOrderedValues()) {
+//			
+//				//query += '<' + TARGET + '>' + supName;
+//				
+//				
+//				print "$supValue "
+//			}
+//		}
+//	}
+//	i++;
+//}
+//
+//// Suppression du dernier underscore ou point
+//NAME = NAME.substring(0, NAME.length() - 1);
+//// Debug
+//println "NAME: " + NAME;
+
+
+
+
+
+
+
+
+// start
+/*if (QUERIES.size() == PARTNAMES.size()) {
+
+	def partition;
+
+	
+	// Test unités structurelles
+	for (t in discours.getStructuralUnits()) {
+		println "Structural Unit: $t"
+		
+		// Test propriétés d'unités structurelles
+		for (g in discours.getStructuralUnitProperties(t.asType(String))) {
+			
+			println "\tStructural Unit Property: $g"
+			
+			// Test valeurs des propriétés d'unités structurelles
+			for (h in g.getOrderedValues()) {
+				
+				print "$h, "
+				
+			}
+			
+		}
+		
+	}
+	
+		
+	// Test propriétés
+	for (t in discours.getProperties()) {
+		println "Property: $t"
+	}
+	
+	
+	if (SUBCORPUS == null) {
+		println "partition build with $discours"
+		partition = discours.createPartition(NAME, QUERIES, PARTNAMES)
+	} else {
+		def subcorpus
+		if (SUBCORPUSQUERY == null) {
+			println "partition build with subcorpus by name $SUBCORPUS"
+			subcorpus = discours.getSubcorpusByName(SUBCORPUS)
+			if (subcorpus == null) {
+				println "Error: SUBCORPUS NOT FOUND with name : $SUBCORPUS"
+			}
+		} else {
+			println "partition build with subcorpus with query $SUBCORPUSQUERY"
+			subcorpus = discours.createSubcorpus(new Query(SUBCORPUSQUERY), SUBCORPUS)
+		}
+		partition = subcorpus.createPartition(NAME, QUERIES, PARTNAMES)
+	}
+	
+	def list = Arrays.asList(partition.getPartSizes())
+	println "Partition created $NAME: "+list+" parts"
+	// println "Total size: "+list.sum()+" - is equal to (sub)corpus size : "+(list.sum() == partition.getCorpus().getSize())
+    println "Total size: "+list.sum()+" - (sub)corpus size : "+(partition.getCorpus().getSize())
+	
+	monitor.syncExec(new Runnable() {
+	public void run() {
+		CorporaView.refresh();
+		CorporaView.expand(partition.getParent());
+	}
+});
+} else {
+	println "QUERIES.size() != PARTNAMES.size() = "+QUERIES.size()+" != "+PARTNAMES.size()
+}*/
+
Index: tmp/org.txm.groovy.core/src/groovy/tests/tests_groovy.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/tests/tests_groovy.groovy	(revision 0)
+++ tmp/org.txm.groovy.core/src/groovy/tests/tests_groovy.groovy	(revision 321)
@@ -0,0 +1 @@
+print(Progression5)
\ No newline at end of file
Index: tmp/org.txm.groovy.core/build.properties
===================================================================
--- tmp/org.txm.groovy.core/build.properties	(revision 0)
+++ tmp/org.txm.groovy.core/build.properties	(revision 321)
@@ -0,0 +1,5 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .,\
+               bin/

Property changes on: tmp/org.txm.groovy.core/build.properties
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.groovy.core/.settings/org.eclipse.jdt.core.prefs
===================================================================
--- tmp/org.txm.groovy.core/.settings/org.eclipse.jdt.core.prefs	(revision 0)
+++ tmp/org.txm.groovy.core/.settings/org.eclipse.jdt.core.prefs	(revision 321)
@@ -0,0 +1,7 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.6

Property changes on: tmp/org.txm.groovy.core/.settings/org.eclipse.jdt.core.prefs
___________________________________________________________________
Added: svn:mime-type
   + text/plain

