Laboratoire ICAR » Plateforme TXM

 org.txm.importer.cqp,

		publishedImports.put(IMPORTERPACKAGE+"cqp","cqpLoader.groovy");

scripts["CWB"] = "cqpLoader.groovy"

println("build cqp RGAQCJ");

			//We don't want to concatenate the cqp files

	 * Transfom file cqp.

		new File(binDir,"/cqp/").deleteDir();

		new File(binDir,"/cqp/").mkdir();

		String cqpName;

			cqpName = basename+"_"+corpusIDS.get(group);

			createOutput(rootDir+"/cqp", "${cqpName}.cqp");

			output.write("<txmcorpus id=\"${cqpName}\" lang=\"$lang\">\n")

			new File(rootDir, "txm/"+cqpName.toUpperCase()).mkdir();

			cqpName = basename+"_"+corpusIDS.get(group);

				String corpusname = (cqpName).toLowerCase();

				builder.transfomFileWtc(rootDir+"/cqp",cqpName+".cqp");

				anaTypesPerWtc[cqpName] = builder.getAnaTypes();

				File txmCorpusDir = new File(rootDir, "txm/"+cqpName.toUpperCase())

			createOutput(rootDir+"/cqp", basename+"_"+corpusIDS.get(group)+".cqp");

		def cqpFiles = [:]

		for (File cqpfile : new File(rootDir, "cqp").listFiles()) {

			String corpusname = cqpfile.getName();

				cqpFiles[corpusname.toLowerCase()] = outDir + "/cqp/"+cqpfile.getName();

				cwbEn.run(outDirTxm + "/data/"+corpusname.toUpperCase()+"/", outDir + "/cqp/"+cqpfile.getName(), regPath, pAttributes, sAttributes);

							File cqpFile1 = new File(cqpFiles[corpusName.getName()]);

							File cqpFile2 = new File(cqpFiles[targetName.getName()]);

							BuildAlignOut bao = new BuildAlignOut(cqpFile1, cqpFile2);

	 * Transfom file cqp.

		new File(rootDir+"/cqp/","bvh.cqp").delete();//cleaning&preparing

		new File(rootDir,"/cqp/").deleteDir();

		new File(rootDir,"/cqp/").mkdir();

				builder.transfomFileWtc(rootDir+"/cqp","bvh.cqp");

			cwbEn.run(outDirTxm + "/data", outDir + "/cqp/"+"bvh.cqp", regPath, pAttributes, sAttributes);

		File cqpFile = new File(binDir,"cqp/${corpusname}.cqp");

		new File(binDir, "cqp.").deleteDir();

		new File(binDir, "cqp").mkdirs();

		if (createOutput(cqpFile)) {

		XMLTXM2WTC cqpbuilder = null;

			cqpbuilder = new XMLTXM2WTC(f.toURI().toURL());

			cqpbuilder.setTextInfo(txtname, corpusname, "project");

			cqpbuilder.setBalisesToKeep(["text","p","s"]);

			cqpbuilder.setSendToPAttributes(["s":["id"], "p":["id"]]);

			cqpbuilder.setLang(lang);

			if (!cqpbuilder.transformFile(cqpFile)) {

		if (createOutput(cqpFile)) {

		if (cqpbuilder == null) {

		List<String> pAttributesList = cqpbuilder.getpAttributs();

		List<String> sAttributesList = cqpbuilder.getsAttributs();

					outDir + "/cqp/"+corpusname+".cqp",

	 * Transfom file cqp.

		new File(rootDir+"/cqp/","bfm3tt.cqp").delete();//cleaning&preparing

		new File(rootDir,"/cqp/").deleteDir();

		new File(rootDir,"/cqp/").mkdir();

				builder.transfomFileWtc(rootDir+"/cqp","bfm3tt.cqp");

		 cwbEn.run(outDirTxm + "/data", outDir + "/cqp/"+"bfm3tt.cqp", outDirTxm + "/registry/"+"bfm3tt",pAttributes, sAttributes);

	 * Transfom file cqp.

		new File(rootDir+"cqp/").mkdir();

		File f = new File(rootDir+"cqp/","RGAQCJ.cqp");

			builder.transfomFileWtc(rootDir+"cqp","RGAQCJ.cqp");

				cwbEn.run(outDirTxm + "data/"+"RGAQCJ", outDir +"cqp/"+"RGAQCJ"+".cqp", outDirTxm + "registry/"+"rgaqcj",pAttributes, sAttributes);

				cwbEn.run(outDirTxm + "data/"+"RGAQCJ", outDir + "/cqp/"+"RGAQCJ.cqp", outDirTxm + "registry/"+"rgaqcj",pAttributes, sAttributes);

		new File(rootDir+"/cqp/").mkdir();

		File resultfile = new File(rootDir+"/cqp/","roland-xmltag.cqp");

		def outDir =rootDir+"cqp/";

			cwbEn.run(outDirTxm + "data/"+"ROLANDXML", outDir +"roland-xmltag.cqp", outDirTxm + "registry/"+"rolandxml",pAttributes, sAttributes);

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2012-01-05 14:27:34 +0100 (jeu., 05 janv. 2012) $

// $LastChangedRevision: 2096 $

// $LastChangedBy: mdecorde $

//

package org.txm.importer.cqp

import org.txm.Toolbox;

import org.txm.importer.cwb.*

import org.txm.importer.*;

import org.txm.scripts.*;

import org.txm.scripts.teitxm.*;

import org.txm.utils.io.FileCopy;

import org.txm.utils.treetagger.TreeTagger;

import java.util.ArrayList;

import javax.xml.stream.*;

import java.net.URL;

import java.io.File;

import java.util.HashMap;

import java.util.List;

// TODO: Auto-generated Javadoc

/**

 * The Class compiler.

 */

class compiler

{

	/** The debug. */

	boolean debug = false;

	/** The dir. */

	private def dir;

	File srcWTCFile, srcRegistryFile;

	public def pAttributesList = [];

	public def sAttributesList = [];

	public compiler(File cqpFile, File registryFile) {

		this.srcWTCFile = cqpFile;

		this.srcRegistryFile = registryFile

	}

	/**

	 * Sets the debug.

	 *

	 * @return the java.lang. object

	 */

	public setDebug()

	{

		debug =true;

	}

	/**

	 * Run.

	 *

	 * @param rootDirFile the root dir file

	 * @param basename the basename

	 * @return true, if successful

	 */

	public boolean run(File binDir, String corpusname)

	{

		if (!(CwbEncode.isExecutableAvailable() && CwbMakeAll.isExecutableAvailable())) {

			println ("Error: CWB executables not well set.")

			return false;

		}

		if (!binDir.exists()) {

			println ("binary directory does not exists: "+binDir)

			return false;

		}

		if (!srcWTCFile.exists()) {

			println "Error: cannot find the WTC file $srcWTCFile"

			return false;

		}

		File cqpFile = new File(binDir,"cqp/"+corpusname+".cqp");

		new File(binDir, "cqp").deleteDir();

		new File(binDir, "cqp").mkdir();

		new File(binDir, "data").deleteDir();

		new File(binDir, "data").mkdir();

		new File(binDir, "registry").mkdir();

		FileCopy.copy(srcWTCFile, cqpFile);

		//2- Import into CWB

		def outDir = binDir.getAbsolutePath()+"/";

		CwbEncode cwbEn = new CwbEncode();

		CwbMakeAll cwbMa = new CwbMakeAll();

		cwbEn.setDebug(debug);

		cwbMa.setDebug(debug);

		String[] pAttributes;

		String[] sAttributes;

		if (srcRegistryFile == null) {

			println "WARNING: No registry file found in source directory"

			println "We'll search for positional attributes and structural attributes in the WTC file"

			// s attributes

			BuildCwbEncodeArgsFromWTC argsgetter = new BuildCwbEncodeArgsFromWTC(); // XML stream

			argsgetter.process(cqpFile); //$NON-NLS-1$

			sAttributesList = argsgetter.getSAttributes();

			// p attributes

			int nbAttr = -1;

			File tmp = File.createTempFile("txm", ".cqp", cqpFile.getParentFile());

			int wcounter = 1;

			println "Adding the 'id' property to the WTC file and getting word properties number."

			tmp.withWriter("UTF-8") { writer ->

				cqpFile.eachLine("UTF-8") { line ->

					if (!line.startsWith("<")) {

						if (nbAttr == -1) nbAttr = line.split("\t").size();

						writer.println(line+"\tw_"+(wcounter++))

					} else {

						writer.println(line)

					}

					writer.flush();

				}

			}

			cqpFile.delete()

			tmp.renameTo(cqpFile)

			System.out.println("Found "+(nbAttr-1)+" word properties, $nbAttr with the 'id'");

			for (int i = 1; i < nbAttr ; i++) {

				pAttributesList << "p$i";

			}

			pAttributesList << "id"

		} else {

			ReadRegistryFile reader = new ReadRegistryFile(srcRegistryFile);

			pAttributesList = reader.getPAttributes();

			sAttributesList = reader.getSAttributes();

			pAttributesList.remove(0) // remove word

			if (!pAttributesList.contains("id")) {

				System.out.println("Error: The registry file does not declare the 'id' word property");

				return false;

			}

		}

		pAttributes = pAttributesList; // cast to array

		sAttributes = sAttributesList; // cast to array

		println "pAttrs : "+Arrays.toString(pAttributes)

		println "sAttrs : "+Arrays.toString(sAttributes)

		try {

			cwbEn.setDebug(debug);

			cwbMa.setDebug(debug);

			String regPath =outDir + "/registry/"+corpusname.toLowerCase()

			cwbEn.run(outDir + "/data/$corpusname", 

					cqpFile.getAbsolutePath(),

					regPath, pAttributes, sAttributes);

			if (!new File(regPath).exists()) {

				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

				return false;

			}

			cwbMa.run(corpusname, outDir + "/registry");

			return true;

		} catch (Exception ex) {System.out.println(ex); return false;}

		return true;

	}

	/**

	 * The main method.

	 *

	 * @param args the arguments

	 */

	public static void main(String[] args)

	{

		File dir = new File("~/xml/perrault/txm/");

		List<File> files = dir.listFiles();

		new compiler().run(files);

	}

}

package org.txm.importer.cqp

import org.txm.importer.cwb.BuildCwbEncodeArgs;

import java.io.File;

import java.util.ArrayList;

import org.txm.objects.*;

import org.txm.scripts.teitxm.*;

import org.txm.*;

import org.txm.utils.i18n.*;

import org.txm.metadatas.*;

import org.txm.utils.io.FileCopy;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

String userDir = System.getProperty("user.home");

def MONITOR;

boolean debug = org.txm.utils.logger.Log.isPrintingErrors();

BaseParameters params;

try {params = paramsBinding;MONITOR=monitor} catch (Exception)

{	println "DEV MODE";//exception means we debug

	debug = true

	params = new BaseParameters(new File(userDir, "xml/cqp/import.xml"))

	params.load()

	if (!org.txm.Toolbox.isInitialized()) {

		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows

		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));

		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows

		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));

		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");

		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");

		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows

		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));

	}

}

if (params == null) { println "no parameters. Aborting"; return; }

String corpusname = params.getCorpusName();

Element corpusElem = params.corpora.get(corpusname);

String basename = params.name;

String rootDir = params.rootDir;

String lang = corpusElem.getAttribute("lang");

String model = lang

String encoding = corpusElem.getAttribute("encoding");

boolean annotate = "true" == corpusElem.getAttribute("annotate");

String xsl = params.getXsltElement(corpusElem).getAttribute("xsl")

def xslParams = params.getXsltParams(corpusElem);

int wordsPerPage = params.getWordsPerPage("default")

boolean build_edition = params.getDoEdition("default")

File srcDir = new File(rootDir);

File binDir = new File(Toolbox.getTXMHOMEPATH(),"corpora/"+basename);

binDir.deleteDir();

binDir.mkdirs();

if (!binDir.exists()) {

	println "Could not create binDir "+binDir

	return;

}

// FAKE: will contains no XML-TXM files

File txmDir = new File(binDir, "txm/$corpusname");

txmDir.deleteDir();

txmDir.mkdirs();

// Will contains the edition files

File htmlDir = new File(binDir, "HTML/$corpusname");

htmlDir.deleteDir();

htmlDir.mkdirs();

// get the cqp file and the registry file

File cqpFile;

File registryFile;

println "Looking for files to process..."

for(File f : srcDir.listFiles()) {

	if (f.isHidden()) continue;

	if (f.isDirectory()) continue;

	if (f.getName().endsWith(".cqp")) {

		if (cqpFile == null) {

			cqpFile = f;

			println "Using WTC file : $cqpFile"

		}

	} else if (f.getName().endsWith(".xml")) {

	} else if (!f.getName().contains(".")){ // its the registry file

		if (registryFile == null) {

			registryFile = f;

			println "Using registry file : $f"

		}

	}

}

if (cqpFile == null) {

	println "No WTC file in source directory: "+srcDir;

	return false;

}

// get pAttributes an run cwb-encode + cwb-makeall

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(25, "COMPILING")

println "-- COMPILING - Building Search Engine indexes"

def c = new compiler(cqpFile, registryFile)

c.setDebug(debug)

if (!c.run(binDir, corpusname)) {

	println "Import stopped"

	return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

new File(binDir,"HTML/$corpusname").deleteDir();

new File(binDir,"HTML/$corpusname").mkdirs();

if (build_edition) {

	println "-- EDITION - Building edition"

	if (MONITOR != null) MONITOR.worked(20, "EDITION - Building edition")

	File outdir = new File(binDir,"/HTML/$corpusname/default/");

	outdir.mkdirs();

	print "."

	List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

	List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

	File binWtcFile = new File(binDir,"cqp/"+corpusname+".cqp");

	def ed = new pager(binWtcFile, outdir, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, registryFile !=null, c.pAttributesList);

	def allIndexes = ed.getIdx();

	def allPages = ed.getPageFiles();

	for (String txtname : ed.getTextNames()) {

		def txtIndexes = allIndexes[txtname];

		def txtPages = allPages[txtname];

		Element text = params.addText(params.corpora.get(corpusname), txtname, cqpFile);

		Element edition = params.addEdition(text, "default", outdir.getAbsolutePath(), "html");

		for (i = 0 ; i < txtPages.size();) {

			File f = txtPages.get(i);

			String wordid = txtIndexes.get(i);

			params.addPage(edition, ""+(++i), wordid);

		}

	}

}

println ""

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

File paramFile = new File(binDir, "import.xml");

DomUtils.save(params.root.getOwnerDocument(), paramFile);readyToLoad = true;

<html>

<body>

<p>CWB import module.</p>

</body>

</html>

package org.txm.importer.cqp;

//Copyright © - ANR Textométrie - http://textometrie.ens-lyon.fr

//

//This file is part of the TXM platform.

//

//The TXM platform is free software: you can redistribute it and/or modif y

//it under the terms of the GNU General Public License as published by

//the Free Software Foundation, either version 3 of the License, or

//(at your option) any later version.

//

//The TXM platform is distributed in the hope that it will be useful,

//but WITHOUT ANY WARRANTY; without even the implied warranty of

//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

//GNU General Public License for more details.

//

//You should have received a copy of the GNU General Public License

//along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

//

//

//

//$LastChangedDate: 2012-06-01 17:47:31 +0200 (ven., 01 juin 2012) $

//$LastChangedRevision: 2185 $

//$LastChangedBy: mdecorde $

//

import org.txm.importer.*;

import org.xml.sax.Attributes;

import org.txm.importer.filters.*;

import java.io.File;

import java.io.IOException;

import java.util.ArrayList;

import javax.xml.parsers.SAXParserFactory;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.parsers.SAXParser;

import javax.xml.stream.*;

import java.net.URL;

import org.xml.sax.InputSource;

import org.xml.sax.helpers.DefaultHandler;

/** Build WTC corpus simple edition from a WTC file. 

 * 

 * @author mdecorde

 */

class pager {

	List<String> NoSpaceBefore;

	/** The No space after. */

	List<String> NoSpaceAfter;

	/** The wordcount. */

	int wordcount = 0;

	/** The pagecount. */

	int pagecount = 0;

	/** The wordmax. */

	int wordmax = 0;

	/** The basename. */

	String basename = "";

	String txtname = "";

	File outdir;

	/** The wordid. */

	String wordid;

	/** The first word. */

	boolean firstWord = true;

	/** The wordvalue. */

	String wordvalue;

	/** The interpvalue. */

	String interpvalue;

	/** The lastword. */

	String lastword = " ";

	/** The wordtype. */

	String wordtype;

	/** The flagform. */

	boolean flagform = false;

	/** The flaginterp. */

	boolean flaginterp = false;

	/** The url. */

	private def url;

	/** The input data. */

	private def inputData;

	/** The factory. */

	private def factory;

	/** The parser. */

	private XMLStreamReader parser;

	/** The writer. */

	OutputStreamWriter writer;

	/** The pagedWriter. */

	OutputStreamWriter pagedWriter = null;

	/** The cqpFile. */

	File cqpFile;

	/** The outfile. */

	File outfile;

	/** The pages. */

	def pages = [:];

	/** The idxstart. */

	def idxstart = [:]

	String editionPage;

	ArrayList<Integer> splitTUs; // contains the tu ids used to split pages

	boolean shouldSplit = false;

	boolean useSplitTUs = false;

	boolean hasWordId = false;

	int noWordIdProperty = -1;

	/**

	 * Instantiates a new pager.

	 *

	 * @param cqpFile the cqpFile

	 * @param outfile the outfile

	 * @param NoSpaceBefore the no space before

	 * @param NoSpaceAfter the no space after

	 * @param max the max

	 * @param basename the basename

	 */

	pager(File cqpFile, File outdir, List<String> NoSpaceBefore,

	List<String> NoSpaceAfter, int max, String basename, boolean hasWordId, List pAttrs) {

		this.editionPage = editionPage;

		this.basename = basename;

		this.txtname = "text1";

		this.outdir = outdir;

		this.wordmax = max;

		this.NoSpaceBefore = NoSpaceBefore;

		this.NoSpaceAfter = NoSpaceAfter;

		this.cqpFile = cqpFile;

		this.hasWordId = hasWordId;

		if (hasWordId) {

			noWordIdProperty = pAttrs.indexOf("id") + 1;

			if (noWordIdProperty == 0)

			hasWordId = false;

		}

		process();

	}

	private void closeMultiWriter()

	{

		if (pagedWriter != null) {

			if (firstWord) { // there was no words

				this.idxstart[txtname] = ["w_0"]

				pagedWriter.write("<span id=\"w_0\"/>");

			}

			pagedWriter.write("</p>\n")

			pagedWriter.write("</body>");

			pagedWriter.write("</html>");

			pagedWriter.close();

		}

	}

	/**

	 * Creates the next output.

	 *

	 * @return true, if successful

	 */

	private boolean createNextOutput()

	{

		wordcount = 0;

		shouldSplit = false;

		try {

			closeMultiWriter();

			File outfile = new File(outdir, txtname+"_"+(++pagecount)+".html")

			//println "outfile: "+outfile

			if (pages[txtname] == null) pages[txtname] = []

			pages[txtname] << outfile;

			firstWord = true; // waiting for next word

			pagedWriter = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(outfile)) , "UTF-8");

			pagedWriter.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");

			pagedWriter.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"   \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n")

			pagedWriter.write("<html>");

			pagedWriter.write("<head>");

			pagedWriter.write("<title>"+basename.toUpperCase()+" $txtname Edition - Page "+pagecount+"</title>");

			pagedWriter.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"tmx.css\"/>");

			pagedWriter.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\"/>");

			pagedWriter.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>");

			pagedWriter.write("</head>");

			pagedWriter.write("<body>");

			return true;

		} catch (Exception e) {

			System.out.println(e.getLocalizedMessage());

			return false;

		}

	}

	/**

	 * Creates the output.

	 *

	 * @param outfile the outfile

	 * @return true, if successful

	 */

	private boolean createOutput() {

		try {

			return createNextOutput();

		} catch (Exception e) {

			System.out.println(e.getLocalizedMessage());

			return false;

		}

	}

	/**

	 * Gets the page files.

	 *

	 * @return the page files

	 */

	public def getPageFiles() {

		return pages;

	}

	/**

	 * Gets the idx.

	 *

	 * @return the idx

	 */

	public def getIdx() {

		return idxstart;

	}

	def texts = []

	/**

	 * Process.

	 */

	void process() {

		String localname = "";

		String lastword= "";

		createNextOutput();

		int wcounter = 1;

		int txtwcounter = 1;

		BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(cqpFile) , "UTF-8"));

		String line = reader.readLine();

		int cline = 0;

		while (line != null) {

			pagedWriter.flush();

			cline++;

			//println "line: "+line

			if (line.startsWith("<")) {

				if (line.startsWith("<lb/") || line.startsWith("<br/") || line.startsWith("</p>") || 

					line.startsWith("<lb>") || line.startsWith("<br>")) {

					pagedWriter.write("<br/>");

				} 

				if (line.startsWith("<text>") || line.startsWith("<text ")) {

					String tmp = line;

					int idx = tmp.indexOf("id=\"");

					if (idx > 0) {

						tmp = tmp.substring(idx+4);

						//println "tmp1: "+tmp

						int idx2 = tmp.indexOf("\"");

						//println "tmp2: "+tmp

						if (idx2 > 0) {

							tmp = tmp.substring(0, idx2);

							txtname = tmp

							texts << txtname;

							pagecount = 0;

							createNextOutput();

						}

					}

				} else if (line.startsWith("<s>") || line.startsWith("<s ")) {

					pagedWriter.write(line.replaceAll("<s", "<sent "));

				} else if (line.startsWith("</s>")) {

					pagedWriter.write("</sent>");

				}  else if (line.startsWith("<pb/>") || line.startsWith("<pb>")) {

					pagedWriter.write(line);

					txtwcounter = 1;

					createNextOutput();

				} else {

					pagedWriter.write(line);

				}

			} else { // word

				if (txtwcounter > wordmax) {

					txtwcounter = 1;

					createNextOutput();

				}

				def split = line.split("\t");

				if (split == null || split.size() == 0) {

					// empty line

				} else {

					String wordid;

					if (hasWordId) {

						if (split.size() <= noWordIdProperty) { // +1 since line contains word

							println "Error: line $cline"

						} else {

							wordid = split[noWordIdProperty];

						}

					} else {

						wordid = "w_"+(wcounter++);

					}

					txtwcounter++;

					String wordvalue = split[0];

					if (firstWord) {

						firstWord = false;

						if (this.idxstart[txtname] == null) {

							this.idxstart[txtname] = []

						}

						this.idxstart[txtname] << [wordid];

					}

					int l = lastword.length();

					String endOfLastWord = "";

					if (l > 0) {

						endOfLastWord = lastword.subSequence(l-1, l);

					}

					String interpvalue = "";

					if (split.size() > 1) {

						interpvalue = split[1..split.size()-1].join(" ")

						interpvalue = interpvalue.replace("&", "&amp;").replace("<", "&lt;").replace("\"","&quot;");

					}

					if (NoSpaceBefore.contains(wordvalue) ||

					NoSpaceAfter.contains(lastword) ||

					wordvalue.startsWith("-") ||

					NoSpaceAfter.contains(endOfLastWord)) {

						pagedWriter.write("<span title=\""+interpvalue+"\" id=\""+wordid+"\">");

					} else {

						pagedWriter.write(" <span title=\""+interpvalue+"\" id=\""+wordid+"\">");

					}

					pagedWriter.write(wordvalue.replace("&", "&amp;").replace("\"","&quot;").replace("<", "&lt;")+"</span>\n");

					pagedWriter.flush()

					lastword = wordvalue;

				}

			}

			line = reader.readLine();

		}

		closeMultiWriter();

	}

	def getTextNames() {

		return texts;

	}

}

	 * Transfom file cqp.

		new File(rootDir+"/cqp/","corptef.cqp").delete();//cleaning&preparing

		new File(rootDir,"/cqp/").deleteDir();

		new File(rootDir,"/cqp/").mkdir();

				builder.transfomFileWtc(rootDir+"/cqp","corptef.cqp");

					new File(outDir, "cqp/"+"corptef.cqp").getAbsolutePath(),

		// "C:\Documents and Settings\alavrent\TXM\cwb\bin\cwb-encode" -d "C:\Documents and Settings\alavrent\TXM\corpora\corptef\data" -f "C:\Documents and Settings\alavrent\TXM\corpora\corptef\cqp\corpte.cqp" -R "C:\Documents and Settings\alavrent\TXM\corpora\corptef\registry\corptef" -c utf8 -xsB -xsB -P ttpos -P ttlemme -P id -P pb -P lb -P orig -P sic -P abbr -S text:0+id+titre+auteur+datecompo+ssiecle+domaine+genre+forme+dialecte+base+project -S front:0+n -S body:0+n -S ab:0+n+rend -S div:0+id+type -S p:0+n -S back:0+n

		File cqpDir = new File(binDir, "cqp")

		new File(binDir, "cqp").delete();

		cqpDir.mkdir();

		if (!new File(binDir, "cqp").exists()) {

			println "Can't create cqp directory"

		File cqpFile = new File(cqpDir, corpusname.toLowerCase()+".cqp")

		if (!createOutput(cqpFile)) return false;

		if (!createOutput(cqpFile)) return false;

					cqpFile.getAbsolutePath(),

		createOutput(new File(outdir, "cqp/"+corpusname.toLowerCase()+".cqp"));

 * build cqp

		File cqpFile = new File(binDir,"cqp/"+corpusname+".cqp");

		new File(binDir, "cqp").deleteDir();

		new File(binDir, "cqp").mkdir();

		if (createOutput(cqpFile)) {

		XMLTXM2WTC cqpbuilder = null;

				cqpbuilder = new XMLTXM2WTC(f.toURI().toURL());

				cqpbuilder.setTextInfo(txtname, corpusname, "default");

				cqpbuilder.setBalisesToKeep(["text", "s", "p", "lb"]);

				cqpbuilder.setSendToPAttributes(["lb":["n"], "p":["n"], "s":["n"]])

				cqpbuilder.setLang(lang);

				if (!cqpbuilder.transformFile(cqpFile)) {

		if (createOutput(cqpFile)) {

		if (cqpbuilder == null) {

		List<String> pAttributesList = cqpbuilder.getpAttributs();

		List<String> sAttributesList = cqpbuilder.getsAttributs();

			cwbEn.run(rootDir + "/data/${corpusname}", rootDir + "/cqp/"+corpusname+".cqp", regPath,pAttributes, sAttributes);

	 * Transfom file cqp.

	public boolean transfomFileWtc(File cqpFile)

		createOutput(cqpFile);

		File cqpFile = new File(binDir,"cqp/"+corpusname.toLowerCase()+".cqp");

		new File(rootDir,"/cqp/").deleteDir();

		new File(rootDir,"/cqp/").mkdir();

		if (createOutput(cqpFile)) {

				if(!builder.transfomFileWtc(cqpFile))

		if (createOutput(cqpFile)) {

				cqpFile.getAbsolutePath(), 

	 * Transfom file cqp.