Laboratoire ICAR » Plateforme TXM

   <feature url="features/org.txm.ahc.feature_1.0.0.201806251537.jar" id="org.txm.ahc.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.analec.feature_1.0.0.201806251537.jar" id="org.txm.analec.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.annotation.feature_1.0.0.201806251537.jar" id="org.txm.annotation.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.annotation.kr.feature_1.0.0.201806251537.jar" id="org.txm.annotation.kr.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.backtomedia.feature_1.0.0.201806251537.jar" id="org.txm.backtomedia.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.ca.feature_1.0.0.201806251537.jar" id="org.txm.ca.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.chartsengine.jfreechart.feature_1.0.0.201806251537.jar" id="org.txm.chartsengine.jfreechart.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.chartsengine.r.feature_1.0.0.201806251537.jar" id="org.txm.chartsengine.r.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.concordance.feature_1.0.0.201806251537.jar" id="org.txm.concordance.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.cooccurrence.feature_1.0.0.201806251537.jar" id="org.txm.cooccurrence.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.cql2lsa.feature_1.0.0.201806251537.jar" id="org.txm.cql2lsa.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.dictionary.feature_1.0.0.201806251537.jar" id="org.txm.dictionary.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.edition.feature_1.0.0.201806251537.jar" id="org.txm.edition.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.groovy.feature_1.0.0.201806251537.jar" id="org.txm.groovy.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.index.feature_1.0.0.201806251537.jar" id="org.txm.index.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.internalview.feature_1.0.0.201806251537.jar" id="org.txm.internalview.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.lexicaltable.feature_1.0.0.201806251537.jar" id="org.txm.lexicaltable.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.partition.feature_1.0.0.201806251537.jar" id="org.txm.partition.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.progression.feature_1.0.0.201806251537.jar" id="org.txm.progression.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.properties.feature_1.0.0.201806251537.jar" id="org.txm.properties.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.rcp.feature_1.0.0.201806251537.jar" id="org.txm.rcp.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.referencer.feature_1.0.0.201806251537.jar" id="org.txm.referencer.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.searchengine.cqp.feature_1.0.0.201806251537.jar" id="org.txm.searchengine.cqp.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.specificities.feature_1.0.0.201806251537.jar" id="org.txm.specificities.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.statsengine.r.feature_1.0.0.201806251537.jar" id="org.txm.statsengine.r.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.svn.feature_1.0.0.201806251537.jar" id="org.txm.svn.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.textbalance.feature_1.0.0.201806251537.jar" id="org.txm.textbalance.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.tigersearch.feature_1.0.0.201806251537.jar" id="org.txm.tigersearch.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.treetagger.feature_1.0.0.201806251537.jar" id="org.txm.treetagger.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.treetagger.files.feature_1.0.0.201806251537.jar" id="org.txm.treetagger.files.feature" version="1.0.0.201806251537"/>

   <feature url="features/org.txm.wordcloud.feature_1.0.0.201806251537.jar" id="org.txm.wordcloud.feature" version="1.0.0.201806251537"/>

   <extension

         point="org.txm.annotation.core.AnnotationEngine">

      <AnnotationEngine

            class="org.txm.treetagger.core.TreeTaggerEngine"

            description="Call TreeTagger treetagger train processus to annotate corpus">

      </AnnotationEngine>

   </extension>

org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate

org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve

org.eclipse.jdt.core.compiler.debug.lineNumber=generate

org.eclipse.jdt.core.compiler.debug.localVariable=generate

org.eclipse.jdt.core.compiler.debug.sourceFile=generate

eclipse.preferences.version=1

groovy.compiler.level=25

groovy.script.filters=**/*.dsld,y,**/*.gradle,n

	<classpathentry exported="true" kind="con" path="GROOVY_SUPPORT"/>

	<classpathentry exported="true" kind="con" path="GROOVY_DSL_SUPPORT"/>

Require-Bundle: org.txm.annotation.core;visibility:=reexport

Export-Package: org.txm.treetagger.core,

 org.txm.treetagger.core.preferences

		<nature>org.eclipse.jdt.groovy.core.groovyNature</nature>

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2013-12-06 16:57:35 +0100 (Fri, 06 Dec 2013) $

// $LastChangedRevision: 2583 $

// $LastChangedBy: mdecorde $

//

package org.txm.importer.xmltxm

import java.io.File

import java.text.DateFormat

import java.util.Date

import org.txm.Toolbox

import org.txm.importer.*

import org.txm.importer.cwb.*

import org.txm.objects.*

import org.txm.stat.utils.ConsoleProgressBar

import org.txm.utils.DeleteDir;

import org.txm.utils.LangDetector;

import org.txm.utils.treetagger.TreeTagger

// TODO: Auto-generated Javadoc

/**

 * Annotate and replace the TEI-TXM files of the folder $rootDirFile/txm with TreeTagger.

 * creates $rootDirFile/interp and $rootDirFile/treetagger

 *

 */

class AnnotateCQP {

	/** The debug. */

	boolean debug = true;

	/**

	 * Sets the debug.

	 */

	public void setDebug() { debug=true; }

	String id;

	/**

	 * Apply tt.

	 *

	 * @param ttsrcfile the ttsrcfile

	 * @param ttoutfile the ttoutfile

	 * @param modelfile the modelfile

	 * @return true, if successful

	 */

	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile)

	{

		try {

			File infile = ttsrcfile;

			File outfile = ttoutfile;

			// TODO: why org.txm.utils.treetagger.TreeTagger tt = ..., throw IllegalAccessError ???

			String opt = Toolbox.getPreference(Toolbox.TREETAGGER_OPTIONS);

			if (opt == null) opt = "";

			String[] options = opt.split("  ");

			def tt = new TreeTagger(Toolbox.getPreference(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/", options);

			tt.settoken();

			tt.setlemma();

			tt.setquiet();

			tt.setsgml();

			tt.setnounknown();

			tt.seteostag("<s>");

			tt.treetagger(modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())

			//infile.delete();

		}

		catch(Exception e)

		{

			e.printStackTrace();

			return false;

		}

		return true;

	}

	/**

	 * Run.

	 *

	 * @param rootDirFile the root dir file

	 * @param modelfilename the modelfilename

	 * @return true, if successful

	 */

	public boolean run(File binDir, File txmDir,  String modelfilename)

	{

		//test if modelfile exists

		if(debug) {

			println "rootDirFile "+binDir

			println "txmDir "+txmDir

			println "TREETAGGER INSTALL PATH : "+Toolbox.getPreference(Toolbox.TREETAGGER_INSTALL_PATH)

			println "TREETAGGER MODELS PATH : "+Toolbox.getPreference(Toolbox.TREETAGGER_MODELS_PATH)

		}

		//test if the Toolbox know TreeTagger

		if (!new File(Toolbox.getPreference(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/").exists())

		{

			println("Could not find TreeTagger binaries in "+Toolbox.getPreference(Toolbox.TREETAGGER_INSTALL_PATH)+"/bin/")

			return false;

		}

		//cleaning

		File annotDir = new File(binDir, "annotations")

		annotDir.mkdir();

		//BUILD TT FILE READY TO BE TAGGED

		List<File> files = txmDir.listFiles()

		// get model file and check it

		File modelfile = new File(Toolbox.getPreference(Toolbox.TREETAGGER_MODELS_PATH), modelfilename);

		if (debug)

			println "model file : "+modelfile;

		if (!modelfile.exists()) {

			println "Skipping ANNOTATE: Incorrect modelfile path: "+modelfile;

			if(System.getProperty("os.name").startsWith("Windows"))

				println "Windows users: Windows might be hiding files extension. To see them, in the explorer parameters."

			return false;

		}

		//APPLY TREETAGGER

		println("Applying $modelfilename TreeTagger model on dir: "+txmDir+ " ("+files.size()+" files)")

		if (files == null || files.size() == 0)

			return false;

		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size());

		for (File f : files) {

			cpb.tick();

			File infile = f;

			File outfile = new File(new File(binDir, "annotations"), f.getName());

			if (outfile.exists() && // outfile exists

				outfile.lastModified() >= infile.lastModified() && // outfile is more recent 

				outfile.length() > infile.length()) { // outfile is bigger

				// skip

			} else {

				if (!applyTT(infile, outfile, modelfile)) {

					System.out.println("Failed to apply treetagger on file "+f);

					return false;

				}

			}

		}

		println("")

//		if (DeleteDir.deleteDirectory(txmDir)) {

//			if (annotDir.renameTo(txmDir)) {

//

//			} else {

//				println "Could not rename  'annotations' directory to 'txm' directory"

//				return false

//			}

//		} else {

//			println "Could not delete txmDir: $txmDir (and could not copy 'annotations' directory"

//		}

		return true;

	}

}

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-10-03 15:30:36 +0200 (lun. 03 oct. 2016) $

// $LastChangedRevision: 3313 $

// $LastChangedBy: mdecorde $

//

package org.txm.importer.xmltxm

import java.io.File

import java.text.DateFormat

import java.util.Date

import java.util.concurrent.*

import org.txm.Toolbox

import org.txm.importer.*

import org.txm.importer.cwb.*

import org.txm.objects.*

import org.txm.stat.utils.ConsoleProgressBar;

import org.txm.treetagger.core.preferences.TreeTaggerPreferences

import org.txm.utils.LangDetector;

import org.txm.utils.logger.Log;

import org.txm.utils.treetagger.TreeTagger

/**

 * Annotate and replace the TEI-TXM files of the folder $rootDirFile/txm with TreeTagger.

 * creates $rootDirFile/interp and $rootDirFile/treetagger

 *

 */

class Annotate {

	boolean cancelNow = false;

	/** The report file. */

	File reportFile;//contains the txm:application tag content

	/** The resp person. */

	String respPerson;

	/** The resp id. */

	String respId;

	/** The resp desc. */

	String respDesc;

	/** The resp date. */

	String respDate;

	/** The resp when. */

	String respWhen;

	/** The app ident. */

	String appIdent;

	/** The app version. */

	String appVersion;

	/** The distributor. */

	String distributor;

	/** The publi stmt. */

	String publiStmt;

	/** The source stmt. */

	String sourceStmt;

	/** The types. */

	def types;

	/** The types title. */

	def typesTITLE;

	/** The types desc. */

	def typesDesc;

	/** The types tagset. */

	def typesTAGSET;

	/** The types web. */

	def typesWEB;

	/** The idform. */

	String idform;

	/** The debug. */

	boolean debug = false;

	File modelsDirectory;

	public Annotate() {

		modelsDirectory = new File(TreeTaggerPreferences.getString(TreeTaggerPreferences.MODELS_PATH, TreeTaggerPreferences.PREFERENCES_NODE)); // default models directory is set in the Toolbox

	}

	/**

	 * Sets the debug.

	 */

	public void setDebug() {

		debug = true;

	}

	String id;

	/**

	 * Inits the tt outfile infos.

	 *

	 * @param rootDirFile the root dir file

	 * @param modelfile the modelfile

	 */

	public void initTTOutfileInfos(File rootDirFile, File modelfile, String modelfilename)

	{

		initTTOutfileInfos(rootDirFile, modelfile, modelfilename, null);

	}

	/**

	 * Inits the tt outfile infos.

	 *

	 * @param rootDirFile the root dir file

	 * @param modelfile the modelfile

	 * @param properties : 2 element array that contains the word properties to create. It can be null (the modelfilename will be used)

	 */

	public void initTTOutfileInfos(File rootDirFile, File modelfile, String modelfilename, String[] properties)

	{

		id = modelfilename;

		String[] split = id.split("\\.");

		if (split.length > 0) id = split[0];

		if (id.equals("??")) id = "xx"

		reportFile = new File(rootDirFile,"NLPToolsParameters.xml");

		respPerson = System.getProperty("user.name");

		respId = "txm";

		respDesc = "NLP annotation tool";

		respDate = DateFormat.getDateInstance(DateFormat.SHORT, Locale.UK).format(new Date());

		respWhen = DateFormat.getDateInstance(DateFormat.FULL, Locale.UK).format(new Date());

		appIdent = "TreeTagger";

		appVersion = "3.2";

		distributor = "";

		publiStmt = """""";

		sourceStmt = """""";

		if (properties != null && properties.length == 2) {

			types = [properties[0],properties[1]];

			typesTITLE = [properties[0],properties[1]];

		} else {

			types = [id+"pos",id+"lemma"];

			typesTITLE = [id+"pos",id+"lemma"];

		}

		//TODO: the tagset, website and description should be referenced in the model catalog

		if(modelfile.getName() == "rgaqcj.par") {

			typesDesc = ["CATTEX pos tagset built with BFM texts","fr lemma of the model "+modelfile+" - "]

			typesTAGSET = ["http://bfm.ens-lyon.fr/IMG/pdf/Cattex2009_Manuel.pdf",""]

			typesWEB = ["http://bfm.ens-lyon.fr/",""]

		} else {

			typesDesc = ["pos tagset built from model "+modelfile,id+" lemma of the model "+modelfile+" - "]

			typesTAGSET = ["",""]

			typesWEB = ["",""]

		}

		idform ="w";

	}

	/**

	 * Apply tt.

	 *

	 * @param ttsrcfile the ttsrcfile

	 * @param ttoutfile the ttoutfile

	 * @param modelfile the modelfile

	 * @return true, if successful

	 */

	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile) {

		applyTT(ttsrcfile, ttoutfile, modelfile, null)

	}

	/**

	 * Apply tt.

	 *

	 * @param ttsrcfile the ttsrcfile

	 * @param ttoutfile the ttoutfile

	 * @param modelfile the modelfile

	 * @param options, if null use value set in Toolbox preferences

	 * @return true, if successful

	 */

	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile, String[] options)

	{

		try {

			File infile = ttsrcfile;

			File outfile = ttoutfile;

			// TODO: why org.txm.utils.treetagger.TreeTagger tt = ..., throw IllegalAccessError ???

			String opt = TreeTaggerPreferences.getString(TreeTaggerPreferences.OPTIONS, TreeTaggerPreferences.PREFERENCES_NODE);

			if (options == null) {

				if (opt == null) opt = "";

				options = opt.split("  ");

			}

			def tt = new TreeTagger(TreeTaggerPreferences.getString(TreeTaggerPreferences.INSTALL_PATH, TreeTaggerPreferences.PREFERENCES_NODE)+"/bin/", options);

			tt.settoken();

			tt.setlemma();

			tt.setquiet();

			tt.setsgml();

			tt.setnounknown();

			tt.seteostag("<s>");

			tt.debug(debug);

			tt.treetagger(modelfile.getAbsolutePath(), infile.getAbsolutePath(), outfile.getAbsolutePath())

			infile.delete();

		} catch(Exception e) {

			Log.printStackTrace(e);

			System.out.println("Failed to apply TreeTagger on $f input file with the $modelfile model file.");

			return false;

		}

		return true;

	}

	/**

	 * Write standoff file.

	 *

	 * @param ttoutfile the ttoutfile

	 * @param posfile the posfile

	 * @return true, if successful

	 */

	public boolean writeStandoffFile(File ttoutfile, File posfile)

	{

		def encoding ="UTF-8";

		def transfo = new CSV2W_ANA();

		//println("build w-interp "+ttfile.getName()+ ">>"+posfile.getName())

		transfo.setAnnotationTypes( types, typesDesc, typesTAGSET, typesWEB, idform);

		transfo.setResp(respId, respDesc, respDate, respPerson, respWhen);

		transfo.setApp(appIdent, appVersion);

		transfo.setTarget(ttoutfile.getAbsolutePath(), reportFile);

		transfo.setInfos(distributor,  publiStmt, sourceStmt);

		return transfo.process( ttoutfile, posfile, encoding );

	}

	/**

	 * Run step by step : build TT src files, run TT, build xml-standoff files, inject standoff annotations

	 *

	 * @param rootDirFile the root dir file

	 * @param modelfilename the modelfilename

	 * @return true, if successful

	 */

	public boolean run(File binDir, File txmDir,  String modelfilename)

	{

		//test if modelfile exists

		if (debug) {

			println "rootDirFile "+binDir

			println "txmDir "+txmDir

			println "TREETAGGER INSTALL PATH : "+TreeTaggerPreferences.getString(TreeTaggerPreferences.INSTALL_PATH, TreeTaggerPreferences.PREFERENCES_NODE);

			println "TREETAGGER MODELS PATH : "+TreeTaggerPreferences.getString(TreeTaggerPreferences.MODELS_PATH, TreeTaggerPreferences.PREFERENCES_NODE)

		}

		//test if the Toolbox know TreeTagger

		if (!new File(TreeTaggerPreferences.getString(TreeTaggerPreferences.INSTALL_PATH, TreeTaggerPreferences.PREFERENCES_NODE)+"/bin/").exists()) {

			println("Could not find TreeTagger binaries in "+TreeTaggerPreferences.getString(TreeTaggerPreferences.INSTALL_PATH, TreeTaggerPreferences.PREFERENCES_NODE)+"/bin/")

			return false;

		}

		String langAll = null

		String lang;

		if (modelfilename.startsWith("??")) {

			langAll = new LangDetector(binDir).getLang();

			println "General lang $langAll"

		}

		//cleaning

		new File(binDir, "annotations").deleteDir();

		new File(binDir, "annotations").mkdir();

		new File(binDir, "treetagger").deleteDir();

		new File(binDir, "treetagger").mkdir();

		ArrayList<String> milestones = [];

		//BUILD TT FILE READY TO BE TAGGED

		List<File> files = txmDir.listFiles()

		println("Building TT source files ("+files.size()+") from directory "+txmDir)

		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

		for (File f : files) {

			cpb.tick()

			File srcfile = f;

			File resultfile = new File(binDir, "treetagger/"+f.getName()+".tt");

			if(debug)

				println "build tt src : "+srcfile+" >> "+resultfile

			def ttsrcbuilder = new BuildTTSrc(srcfile.toURI().toURL())

			if (!ttsrcbuilder.process(resultfile, null))

				System.out.println("Failed to build tt src file of "+srcfile);

		}

		if (cancelNow) return;

		File modelDirectory = new File(TreeTaggerPreferences.getString(TreeTaggerPreferences.MODELS_PATH, TreeTaggerPreferences.PREFERENCES_NODE));

		if (!modelDirectory.exists()) {

			println "Skipping ANNOTATE: TreeTagger language model file directory not found: "+modelDirectory.getAbsolutePath();

			return false;

		} else 	if (!modelDirectory.canRead()) {

			println "Skipping ANNOTATE: impossible to access the TreeTagger language model file directory: "+modelDirectory.getAbsolutePath();

			return false;

		}

		println("")

		//Convert encoding if needed

		//APPLY TREETAGGER

		files = new File(binDir, "treetagger").listFiles()

		println("Applying $modelfilename TreeTagger model on dir: "+new File(binDir, "treetagger")+ " ("+files.size()+" files)")

		if (files == null || files.size() == 0)

			return false;

		File modelfile;

		cpb = new ConsoleProgressBar(files.size())

		for (File f : files) {

			String tmpModelFileName = modelfilename

			if (modelfilename.startsWith("??")) {

				lang = langAll;

				if (f.length() > LangDetector.MINIMALSIZE) {

					lang = new LangDetector(f).getLang();

					//println "guessing lang $f : $lang"

				}

				tmpModelFileName = lang+".par"

			}

			modelfile = new File(modelsDirectory, tmpModelFileName);

			if (debug)

				println "model file : "+modelfile;

			File

			if (!modelfile.exists()) {

				println "Skipping ANNOTATE: '$modelfile' TreeTagger language model file not found."

				if(System.getProperty("os.name").startsWith("Windows") || System.getProperty("os.name").startsWith("Mac"))

					println "Windows&Mac users: the operating system might be hiding file extensions. Use your file explorer to check the file name."

				return false;

			} else if (!modelfile.canRead()) {

				println "Skipping ANNOTATE: impossible to access the '$modelfile' TreeTagger language model file."

				return false;

			}

			//			if (modelfile.getName().equals("sp.par")) {//UTF >> Latin1

			//				if(debug)

			//					println "fix encoding for model "+modelfile

			//				new EncodingConverter(f, "UTF-8", "ISO-8859-1")

			//			}

			cpb.tick()

			File infile = f;

			File outfile = new File(f.getParent(),f.getName()+"-out.tt");

			if (!applyTT(infile, outfile, modelfile)) {

				return false;

			}

			//			//Reconvert encoding if needed

			//			if (modelfile.getName().equals("sp.par")) {

			//				if(debug)

			//					println "convert "+f+" latin1 >> UTF-8"

			//				new EncodingConverter(f, "ISO-8859-1", "UTF-8")

			//			}

		}

		println("")

		if (cancelNow) return;

		//BUILD STAND-OFF FILES

		if (modelfile == null) {

			println "no model applied"

			return false;

		}

		initTTOutfileInfos(binDir, modelfile, modelfilename);

		files = new File(binDir, "treetagger").listFiles()// now contains the result files of TT

		println "Building stdoff files ("+files.size()+") from dir:"+new File(binDir, "treetagger")+" to "+new File(binDir, "annotations");

		if (files == null || files.size() == 0)

			return false;

		cpb = new ConsoleProgressBar(files.size())

		for (File ttrezfile : files) {

			cpb.tick()

			File annotfile = new File(binDir, "annotations/"+ttrezfile.getName()+"-STOFF.xml");

			if (!writeStandoffFile(ttrezfile, annotfile))

				println("Failed to build standoff file of "+ttrezfile);

		}

		println("")

		if (cancelNow) return;

		//INJECT ANNOTATIONS

		List<File> interpfiles = new File(binDir, "annotations").listFiles();

		List<File> txmfiles = txmDir.listFiles();

		if (txmfiles == null) {

			println "No file to annotate in "+txmDir.getAbsolutePath()

			return false;

		}

		interpfiles.sort(); // same order

		txmfiles.sort(); //same order

		println "Injecting stdoff files ("+interpfiles.size()+") data from "+new File(binDir, "annotations")+ " to xml-txm files of "+txmDir;

		if (interpfiles == null || interpfiles.size() == 0)

			return false;

		cpb = new ConsoleProgressBar(interpfiles.size())

		for (int i = 0 ; i < interpfiles.size() ; i++) {

			cpb.tick()

			File srcfile = txmfiles.get(i);

			File pos1file = interpfiles.get(i);

			File temp = File.createTempFile("Annotate", "temp", srcfile.getParentFile());

			def builder = new AnnotationInjection(srcfile.toURI().toURL(), pos1file.toURI().toURL());

			if (!builder.process(temp)) {

				return false;

			}

			builder = null;

			//println "renaming files..."

			if (!(srcfile.delete() && temp.renameTo(srcfile)))

				println "Warning can't rename file "+temp+" to "+srcfile

		}

		println("")

		return true;

	}

	public void setModelsDirectory(File modelsDirectory) {

		this.modelsDirectory = modelsDirectory;

	}

	/**

	 * Run file by file. Allow to have one different lang per file. Default behavior add new word properties

	 *

	 * @param binDir

	 * @param txmDir

	 * @param lang associate a file name with a model filename

	 * @return true, if successful

	 */

	public boolean run(File binDir, File txmDir, HashMap<String, String> langs)

	{

		return run(binDir, txmDir, langs, false, new String[0], new String[0]);

	}

	/**

	 * Run file by file. Allow to have one different lang per file

	 *

	 * @param binDir 

	 * @param txmDir

	 * @param lang associate a file name with a model filename

	 * @param replace, replace or create a word property

	 * @return true, if successful

	 */

	public boolean run(File binDir, File txmDir, HashMap<String, String> langs, boolean replace, String[] properties, String[] options)

	{

		if (!new File(TreeTaggerPreferences.getString(TreeTaggerPreferences.INSTALL_PATH, TreeTaggerPreferences.PREFERENCES_NODE)+"/bin/").exists()) {

			println("Path to TreeTagger is wrong "+TreeTaggerPreferences.getString(TreeTaggerPreferences.INSTALL_PATH, TreeTaggerPreferences.PREFERENCES_NODE)+"/bin/")

			return true;

		}

		List<File> listfiles = txmDir.listFiles();

		//cleaning

		File annotDir = new File(binDir,"annotations");

		annotDir.deleteDir();

		annotDir.mkdir();

		File ptreetaggerDir = new File(binDir,"ptreetagger");

		ptreetaggerDir.deleteDir();

		ptreetaggerDir.mkdir();

		File treetaggerDir = new File(binDir,"treetagger");

		treetaggerDir.deleteDir();

		treetaggerDir.mkdir();

		int cores = Runtime.getRuntime().availableProcessors()

		int coresToUse = Math.max(1.0, cores * 0.7)

		ExecutorService pool = Executors.newFixedThreadPool(coresToUse)

		def files = txmDir.listFiles()

		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

		for (File teiFile : files) {

			int counter = 1;

			ThreadFile t = new ThreadFile("TT_"+counter++, teiFile) {

						public void run() {

							if (cancelNow) return;

							if (langs.get(f.getName()) == null) {

								println "Error: no lang defined for file $f"

								return;

							}

							String lang = langs.get(f.getName());

							run(f, lang, binDir, txmDir, replace, properties, options, annotDir, ptreetaggerDir, treetaggerDir)

							cpb.tick();

						}

					};

			pool.execute(t)

		}

		pool.shutdown()

		pool.awaitTermination(10, TimeUnit.HOURS)

		println ""

		return true;

	}

	public boolean run(File f, String lang, File binDir, File txmDir) {

		File annotDir = new File(binDir,"annotations");

		annotDir.mkdir();

		File ptreetaggerDir = new File(binDir,"ptreetagger");

		ptreetaggerDir.mkdir();

		File treetaggerDir = new File(binDir,"treetagger");

		treetaggerDir.mkdir();

		return run(f, lang, binDir, txmDir, false, new String[0], new String[0], annotDir, ptreetaggerDir, treetaggerDir)

	}

	public boolean run(File f, String lang, File binDir, File txmDir, boolean replace, String[] properties, String[] options, File annotDir, File ptreetaggerDir, File treetaggerDir) {

		File modelfile = new File(modelsDirectory, lang+".par");

		if (!modelfile.exists()) {

			println "Error: No Modelfile available for lang "+modelfile+". Continue import process ";

			return false;

		}

		File annotfile = new File(annotDir, f.getName()+"-STDOFF.xml");

		File ttsrcfile = new File(ptreetaggerDir, f.getName()+"-src.tt");

		File ttrezfile = new File(treetaggerDir, f.getName()+"-out.tt");

		//println ("TT with $model "+f+"+"+annotfile+" > "+ttsrcfile+" > "+ttrezfile);

		//BUILD TT FILE READY TO BE TAGGED

		def builder = new BuildTTSrc(f.toURL());

		builder.process(ttsrcfile, null);

		//Apply TT

		applyTT(ttsrcfile, ttrezfile, modelfile, options);

		//CREATE STANDOFF FILES

		initTTOutfileInfos(binDir, modelfile, lang, properties);

		writeStandoffFile(ttrezfile, annotfile)

		//INJECT ANNOTATIONS

		File tmpFile = new File(txmDir, "temp_"+f.getName())

		builder = new AnnotationInjection(f.toURL(), annotfile.toURL(), replace);

		builder.process(tmpFile);

		if (!(f.delete() && tmpFile.renameTo(f))) println "Warning can't rename file "+tmpFile+" to "+f

		return f.exists();

	}

	public void setCancelNow() {

		cancelNow = true;

	}

	public class ThreadFile extends Thread {

		File f;

		public ThreadFile(String name, File f) {

			this.setName(name)

			this.f = f;

		}

	}

}

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-03-02 14:17:06 +0100 (mer., 02 mars 2016) $

// $LastChangedRevision: 3134 $

// $LastChangedBy: mdecorde $

//

package org.txm.importer.xmltxm

import javax.xml.stream.*

import org.txm.importer.StaxIdentityParser

import org.txm.importer.filters.*

import org.txm.importer.PersonalNamespaceContext

/**

 * The Class AnnotationInjection.

 *

 * @author mdecorde

 * 

 * inject annotation from a stand-off file into a xml-tei-txm file

 */

public class AnnotationInjectionFromTTOutput extends StaxIdentityParser {

	public static String TXMNS = "http://textometrie.org/1.0"

	/** The xml reader factory. */

	private def factory;

	/** The links. */

	private List<String> links;

	/** The linkparsers. key=type*/

	private HashMap<String, XMLStreamReader> linkparsers;

	/** The anaurl. */

	private def anaurl;

	/** The anainput data. */

	private def anainputData;

	/** The anafactory. */

	private XMLInputFactory anafactory = XMLInputFactory.newInstance();

	/** The anaparser. */

	private XMLStreamReader anaparser;

	private XMLStreamReader headerparser;

	/** The resp stmt id. */

	String respStmtID = "";

	/** The present taxonomies. */

	ArrayList<String> presentTaxonomies = new ArrayList();

	/**

	 * Instantiates a new annotation injection.

	 *

	 * @param url the xml-tei-txm file

	 * @param anaurl the stand-off file

	 */

	public AnnotationInjectionFromTTOutput(URL url, URL anaurl) {

		super(url); // init reader and writer

		try {

			this.anaurl = anaurl;

			factory = XMLInputFactory.newInstance();

			this.buildLinkParsers();// build a parser per linkgroup

		} catch (XMLStreamException ex) {

			System.out.println(ex);

		} catch (IOException ex) {

			System.out.println("IOException while parsing ");

		}

	}

	private void getHeaderInfos(String containertag, boolean captureTheTag)

	{

		anainputData = new BufferedInputStream(anaurl.openStream());

		headerparser = anafactory.createXMLStreamReader(anainputData);

		boolean start = false;

		String localname;

		for (int event = headerparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = headerparser.next())

		{

			//String prefix = headerparser.getNamespaceURI();

			if (event == XMLStreamConstants.START_ELEMENT) {

				localname = headerparser.getLocalName();

				if (captureTheTag && localname == containertag) // start copy  after the tag

					start = true;

				if (start) { // copy header

					String prefix = headerparser.getPrefix();

					if (prefix.length() > 0)

						writer.writeStartElement(Nscontext.getNamespaceURI(prefix), localname)

					else

						writer.writeStartElement(localname);

					for (int i = 0 ; i < headerparser.getNamespaceCount(); i++)

						writer.writeNamespace(headerparser.getNamespacePrefix(i), headerparser.getNamespaceURI(i));

					for (int i = 0 ; i < headerparser.getAttributeCount(); i++)

						writer.writeAttribute(headerparser.getAttributeLocalName(i), headerparser.getAttributeValue(i));

				}

				if (!captureTheTag && localname == containertag) // start copy  after the tag

					start = true;

			} else if (event == XMLStreamConstants.END_ELEMENT) {

				localname = headerparser.getLocalName();

				if (!captureTheTag && localname == containertag)

					break;// stop looping

				if (start)

					writer.writeEndElement();

				if (captureTheTag && localname == containertag)

					break;// stop looping

			} else if (event == XMLStreamConstants.CHARACTERS) {

				if (start)

					writer.writeCharacters(headerparser.getText());

			} else if (event == XMLStreamConstants.COMMENT) {

				if (start)

					writer.writeComment(headerparser.getText());

			}

		}

		headerparser.close();

	}

	/**

	 * find all refs.

	 *

	 * @return the list of link parser

	 */

	private List<String> findGrpLink()

	{

		ArrayList<String> links = new ArrayList<String>();

		anainputData = anaurl.openStream();

		anaparser = anafactory.createXMLStreamReader(anainputData);

		for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {

			if (event == XMLStreamConstants.START_ELEMENT) {

				if (anaparser.getLocalName().matches("linkGrp")) {

					String targetsvalue = anaparser.getAttributeValue(0)

					if (links.contains(targetsvalue)) {

						System.err.println("Warning: Multiple group declaration : "+targetsvalue+" has already been added, the first one will be used")

					} else {

						links.add(targetsvalue); // add the taxonomy type

					}

				} else if (anaparser.getLocalName().matches("respStmt")) {

					respStmtID = anaparser.getAttributeValue(0); // one attribute (id) only

				}

			}

		}

		anaparser.close();

		return links;

	}

	/**

	 * Builds the link parsers.

	 * I need to know what groups exists to build a parser per taxonomy and go to the first link element

	 */

	private void buildLinkParsers()

	{

		// link group of the standoff file

		links = findGrpLink();

		linkparsers = new HashMap<String, XMLStreamReader>();

		// build one parser per link group

		for (String link : links) { // build a parser per group

			anainputData = new BufferedInputStream(anaurl.openStream());

			linkparsers.put(link, anafactory.createXMLStreamReader(anainputData));

		}

		//for each parser

		for (String link : links) {

			anaparser = linkparsers.get(link);

			for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {

				if (event == XMLStreamConstants.START_ELEMENT) {

					if (anaparser.getLocalName().equals("linkGrp")) { // position the parser to the right group

						String targetsvalue = anaparser.getAttributeValue(0)

						if (targetsvalue.equals(link))

							break; // next element is a link start tag

					}

				}

			}

		}

	}

	/**

	 * get the next tei:link value of a tei:LinkGrp.

	 *

	 * @param link the link

	 * @return the next ana

	 */

	private String getNextAnaValue(String link, String wordId)

	{

		anaparser = linkparsers.get(link);

		def m;

		for (int event = anaparser.next(); event != XMLStreamConstants.END_DOCUMENT; event = anaparser.next()) {

			if (event == XMLStreamConstants.START_ELEMENT) {

				if (anaparser.getLocalName().matches("link")) {

					String targetsvalue = anaparser.getAttributeValue(0)

					if ((m = targetsvalue =~ /#(.*) #(.*)/)) { // balise externe

						def g1 = m[0][1];

						def g2 = m[0][2];

						String anavalue = g2;

						anavalue = anavalue.replace("&lt;", "<")

						return anavalue;

					} else {

						System.err.println("Error: getNextAna(): link target is not well formed:  = "+anaparser.getAttributeValue(0));

					}

				}

			}

		}

		return "";

	}

	/**

	 * build the ana tags of a word.

	 *

	 * @param wordId the word id

	 * @return the ana tag

	 */

	private void writeAnaTag(String wordId)

	{

		String anabalises ="\n";

		for (String link : links) {

			writer.writeStartElement(TXMNS, "ana");

			writer.writeAttribute("resp", "#"+respStmtID);

			writer.writeAttribute("type", "#"+link);

			writer.writeCharacters(getNextAnaValue(link, wordId));

			writer.writeEndElement(); // txm:ana

		}

	}

	String wordId;

	boolean flagSourceDesc = false;

	protected void processStartElement()

	{

		if (parser.getLocalName().matches("taxonomy")) {

			String taxo = parser.getAttributeValue(0) // taxonomy type

			presentTaxonomies.add(taxo);

		}

		for (int i= 0 ; i < parser.getAttributeCount() ;i++ ) {

			if (parser.getAttributeLocalName(i) == "id") {

				wordId = parser.getAttributeValue(i);

				break

			}

		}

		super.processStartElement();

	}

	boolean applicationWritten = false;

	boolean taxonomiesWritten = false;

	protected void processEndElement() {

		switch (parser.getLocalName()) {

			case "w":

				writeAnaTag(wordId);

				break;

			case "appInfo":

				applicationWritten = true;

				getHeaderInfos("appInfo", false);

				break;

			case "classDecl":

				taxonomiesWritten = true;

				getHeaderInfos("classDecl", false);

				break;

			case "encodingDesc":

				if (!applicationWritten) {

					writer.writeStartElement("appInfo");

					getHeaderInfos("appInfo", false);

					writer.writeEndElement(); // appInfo

				}

				if (!taxonomiesWritten) {

					writer.writeStartElement("classDecl");

					getHeaderInfos("classDecl", false);

					writer.writeEndElement(); // classDecl

				}

				break;

			case "titleStmt":

				if (flagSourceDesc) {

					//output.write(this.respStmt+"\n")

					getHeaderInfos("respStmt", true);

					flagSourceDesc = false;

					break;

				}

				break;

		}

		super.processEndElement();

	}

	/** The declarenamespace. */

	boolean declarenamespace = false;

	/**

	 * Declare namespace.

	 *

	 * @return the java.lang. object

	 */

	private declareNamespace() {

		if (!declarenamespace) {

			writer.writeDefaultNamespace("http://www.tei-c.org/ns/1.0");

			writer.writeNamespace("txm", TXMNS);

			declarenamespace = true;

		}

	}

	/**

	 * The main method.

	 *

	 * @param args the arguments

	 */

	public static void main(String[] args) {

		String rootDir = "~/xml/rgaqcj/";

		new File(rootDir + "/injection/").mkdir();

		def milestones = [

			"tagUsage",

			"pb",

			"lb",

			"catRef"]// the tags who

		File srcfile = new File(rootDir, "/anainline/", "roland.xml");

		File pos1file = new File(rootDir, "/pos/", "rolandTT1-w-ana.xml");

		File src2file = new File(rootDir, "/injection/", "roland.xml");

		File pos2file = new File(rootDir, "/pos/", "rolandTT2-w-ana.xml");

		println("process file : " + srcfile + " with : " + pos1file);

		def builder = new AnnotationInjectionFromTTOutput(srcfile.toURI().toURL(),

				pos1file.toURI().toURL(), milestones);

		builder.transfomFile(new File(rootDir + "/injection/", "roland.xml"));

		println("process file : " + src2file + " with : " + pos1file);

		builder = new AnnotationInjectionFromTTOutput(src2file.toURI().toURL(), pos2file.toURI().toURL(),

				milestones);

		builder.transfomFile(rootDir + "/injection/", "roland-FINAL.xml");

		return;

	}

}

<html>

<body>

<p>Manages xml-tei-txm files</p>

</body>

</html>

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2017-01-24 18:11:42 +0100 (mar. 24 janv. 2017) $

// $LastChangedRevision: 3400 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.importer.xmltxm

import java.text.DateFormat;

import java.util.Date;

import java.util.ArrayList;

import javax.xml.stream.*;

import java.net.URL;

import org.txm.Toolbox;

import org.txm.importer.filters.*;

import org.txm.treetagger.core.preferences.TreeTaggerPreferences

// TODO: Auto-generated Javadoc

/**

 * The Class BuildTTSrc.

 *

 * @author mdecorde

 * build the TT source for tigerSearch

 */

public class BuildTTSrc {

	/** The url. */

	private def url;

	/** The input data. */

	private def inputData;

	/** The factory. */

	private def factory;

	/** The parser. */

	private XMLStreamReader parser;

	/** The output. */

	private BufferedWriter output;

	/**

	 * Instantiates a new builds the tt src.

	 * uses XML-TXM V2

	 *

	 * @param url the url of the file to process

	 */

	public BuildTTSrc(URL url) {

		try {

			this.url = url;

			inputData = url.openStream();

			factory = XMLInputFactory.newInstance();

			parser = factory.createXMLStreamReader(inputData);

		} catch (XMLStreamException ex) {

			System.out.println(ex);

		} catch (IOException ex) {

			System.out.println("IOException while parsing ");

		}

	}

	/**

	 * Creates the output.

	 *

	 * @param outfile the outfile

	 * @return true, if successful

	 */

	private boolean createOutput(File outfile) {

		try {

			File f = outfile;

			output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile),

					"UTF-8"));

			return true;

		} catch (Exception e) {

			System.out.println(e.getLocalizedMessage());

			return false;

		}

	}

	/**

	 * Process.

	 *

	 * @param outfile the outfile

     * @param formtype, if multiple form, use this param to choose the correct one, if null takes the first form found

	 * @return true, if successful

	 */

	public boolean process(File outfile, String formtype) {

		if (!createOutput(outfile))

			return false;

		boolean flagform = false; // to catch the content of the form tag

		boolean firstform = false; // to know if its the first form of the w element

		String form = ""; // the content of the form tag

		String lastopenlocalname = "";

		String localname = "";

		StringBuffer buffer = new StringBuffer();

		try {

			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

				switch (event) {

					case XMLStreamConstants.START_ELEMENT:

						localname = parser.getLocalName();

						switch (localname) {

							case "w":

								//firstform = true;

								break;

							case "form":

//								if (firstform) {

//									if (formtype != null) {

//										if(parser.getAttributeCount() > 0 

//											&& parser.getAttributeValue(0).equals(formtype)) // only one attribute in form, type

//											flagform = true;

//									}

//									else

									flagform = true;

									form = "";

									firstform = false;

								//}

								break;

							case "s": // TreeTagger can use s tags

								buffer.append("<s>\n");

								break;

						}

						break;

					case XMLStreamConstants.END_ELEMENT:

						localname = parser.getLocalName();

						switch (localname) {

							case "form":

								flagform = false;

								form = form.trim()

								if (form.length() == 0) buffer.append("__EMPTY__\n");

								else buffer.append(form.replace("\n", "").replace("<", "&lt;")+ "\n");

								//buffer.append(form+ "\n"); // its a txt file no need to use entities

								break;

							case "s":

								buffer.append("</s>\n");

								break;

						}

						break;

					case XMLStreamConstants.CHARACTERS:

						if (flagform) {

							if (parser.getText().length() > 0)

								form += parser.getText();

						}

						break;

				}

			}

			String str = buffer.toString()

			if ("false".equals(TreeTaggerPreferences.getString(TreeTaggerPreferences.FIX_APOSTROPHES, TreeTaggerPreferences.PREFERENCES_NODE))) {

				str = str.replace("’", "'").replace("‘", "'");

			}

			output.write(str)

			output.close();

			parser.close();

			inputData.close();

		} catch (Exception ex) {

			System.out.println(ex);

			return false;

		} 

		return true;

	}

	/**

	 * The main method.

	 *

	 * @param args the arguments

	 */

	public static void main(String[] args) {

		String rootDir = "~/xml/rgaqcj/";

		// new File(rootDir+"/identity/").mkdir();

		ArrayList<String> milestones = new ArrayList<String>();// the tags who

		// you want them

		// to stay

		// milestones