Laboratoire ICAR » Plateforme TXM

	boolean debug = false

				id = 1 // first position in sentence

				for (i = match.start ; i <= match.end ; i++) { // loop n times, n = mathc length

					writer.write(""+id+"\t"+words.get(iposition)+"\t"+lemmestr+"\t_\t"+posstr+"\t_\t_\t_\t_\tXmlID="+idsList.get(iposition)+"\n");

import org.txm.core.preferences.TBXPreferences

								String interpvalue = null;

								def tooltipProperties = pager.project.getEditionDefinition("default").get(TBXPreferences.EDITION_DEFINITION_TOOLTIP_PROPERTIES, "*");

								println tooltipProperties

								if (tooltipProperties.equals("*")) {

									interpvalue = "- "+anaValues.entrySet().join("\n- ")+"\n- "+wordid

								} else {

									for (String p : tooltipProperties.split(",")) {

										interpvalue += "- "+p+"="+anaValues.get(p)+"\n"

									}

									interpvalue += "- "+wordid

								}

import java.text.NumberFormat;

			return this.getName() + " T=" + NumberFormat.getInstance().format(this.getSize());

import java.text.NumberFormat;

				return "CQP ID "+this.pID+" T " + NumberFormat.getInstance().format(this.getSize());

import org.txm.core.preferences.TBXPreferences;

import org.txm.core.preferences.TXMPreferences;

	private Text defaultEditionTooltipPropertiesText;

			tmpLabel = toolkit.createLabel(sectionClient, "Tooltip properties");

			tmpLabel.setToolTipText("comma separated list (comma*=all)");

			gdata = getLabelGridData();

			gdata.indent = 20;

			tmpLabel.setLayoutData(gdata);

			defaultEditionTooltipPropertiesText = toolkit.createText(sectionClient, "", SWT.BORDER); //$NON-NLS-1$

			gdata = getTextGridData();

			gdata.colspan = 2;

			defaultEditionTooltipPropertiesText.setLayoutData(gdata);

		if (defaultEditionTooltipPropertiesText != null) {

			defaultEditionTooltipPropertiesText.setText(project.getEditionDefinition("default").get(TBXPreferences.EDITION_DEFINITION_TOOLTIP_PROPERTIES, "*").replace(" ", ""));

		}

			if (defaultEditionTooltipPropertiesText != null) {

				project.getEditionDefinition("default").set(TBXPreferences.EDITION_DEFINITION_TOOLTIP_PROPERTIES, defaultEditionTooltipPropertiesText.getText().replace(" ", ""));

			}

	@Option(name="outputDirectory", usage="an example file", widget="Folder", required=true, def="outputDirectory")

	File outputDirectory;

	@Option(name="encoding", usage="sentenceProperty", widget="String", required=true, def="UTF-8")

	String encoding = "UTF-8";

	 @Option(name="sentenceStructure", usage="sentenceProperty", widget="String", required=true, def="s")

	String sentenceStructure;

	@Option(name="posProperty", usage="if set posProperty used to fill the UPOS ud property", widget="String", required=true, def="frpos")

	String posProperty;

	@Option(name="lemmaProperty", usage="if set lemmaProperty used to fill the LEMMA ud property", widget="String", required=true, def="frlemma")

	String lemmaProperty;

		outputDirectory.mkdirs();

		if (outputDirectory == null || !outputDirectory.exists() || !outputDirectory.isDirectory()) {

			Log.warning("Error: conllu result directory does not exists: " + outputDirectory);

			new ToCoNLL2009().process(outputDirectory, mainCorpus, corpus.getStructuralUnit(sentenceStructure), corpus.getProperty("word"), corpus.getProperty(lemmaProperty), corpus.getProperty(posProperty), encoding);

		} catch (Exception e) {

// Copyright © 2010-2020 ENS de Lyon., University of Franche-Comté

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$ 

//

package org.txm.conllu.rcp.commands;

import java.io.File;

import java.io.IOException;

import java.io.PrintWriter;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.HashSet;

import org.apache.commons.lang.StringUtils;

import org.eclipse.core.commands.AbstractHandler;

import org.eclipse.core.commands.ExecutionEvent;

import org.eclipse.core.commands.ExecutionException;

import org.eclipse.jface.viewers.IStructuredSelection;

import org.eclipse.osgi.util.NLS;

import org.eclipse.ui.handlers.HandlerUtil;

import org.kohsuke.args4j.Option;

import org.txm.rcp.swt.widget.parameters.ParametersDialog;

import org.txm.searchengine.cqp.CQPSearchEngine;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.clientExceptions.InvalidCqpIdException;

import org.txm.searchengine.cqp.clientExceptions.UnexpectedAnswerException;

import org.txm.searchengine.cqp.corpus.CQPCorpus;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.corpus.StructuralUnit;

import org.txm.searchengine.cqp.corpus.WordProperty;

import org.txm.searchengine.cqp.corpus.query.CQLQuery;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.utils.i18n.LangFormater;

import org.txm.utils.io.IOUtils;

import org.txm.utils.logger.Log;

/**

 * Export the conllu properties and CQP words into a conllu corpus of several files (one per text)

 * 

 * @author mdecorde.

 */

public class ExportCorpusAsFullCoNLLU extends AbstractHandler {

	public static final String ID = ExportCorpusAsFullCoNLLU.class.getName();

	@Option(name = "conlluResultDirectory", usage = "conlluResultDirectory", widget = "Folder", required = true, def = "conllu-result-directory")

	File conlluResultDirectory;

	@Option(name = "propertiesPrefix", usage = "optional prefix for the properties to create", widget = "String", required = true, def = "ud-")

	String propertiesPrefix;

	@Option(name = "separator", usage = "Options", widget = "Separator", required = true, def = "comment properties")

	Boolean separator = false;

	@Option(name = "insertParagraphs", usage = "Insert paragraph marks in the CoNLLU corpus", widget = "Boolean", required = true, def = "true")

	Boolean insertParagraphs = false;

	@Option(name = "detectGap", usage = "Insert gap comment using the CQP 'gap' property", widget = "Boolean", required = true, def = "true")

	Boolean detectGap = false;

	@Option(name = "separator3", usage = "Options", widget = "Separator", required = true, def = "tokens options")

	Boolean separator3 = false;

	@Option(name = "insertNoSpaceAfter", usage = "Insert the NoSpaceAfter misc property if not initially in the CoNLLU corpus", widget = "Boolean", required = true, def = "true")

	Boolean insertNoSpaceAfter = true;

	@Option(name = "insertTokenWithoutUdAnnotations", usage = "if checked words without ud annotations are exported as well", widget = "Boolean", required = false, def = "false")

	Boolean insertTokenWithoutUdAnnotations;

	// "form", "lemma", "upos", "xpos", "feats", "head", "deprel", "deps", "misc" };

	@Option(name = "separator_properties", usage = "Options", widget = "Separator", required = true, def = "fixing UD properties")

	Boolean separator_properties = false;

	@Option(name = "defaultFormPropertyName", usage = "optional CQP property to fix the missing 'form' ud property", widget = "String", required = false, def = "")

	String defaultFormPropertyName;

	@Option(name = "defaultLemmaPropertyName", usage = "optional CQP property to fix the missing 'lemma' ud property", widget = "String", required = false, def = "")

	String defaultLemmaPropertyName;

	@Option(name = "defaultUposPropertyName", usage = "optional CQP property to fix the missing 'upos' ud property", widget = "String", required = false, def = "")

	String defaultUposPropertyName;

	@Option(name = "defaultXposPropertyName", usage = "optional CQP property to fix the missing 'xpos' ud property", widget = "String", required = false, def = "")

	String defaultXposPropertyName;

	@Option(name = "defaultFeatsPropertyName", usage = "optional CQP property to fix the missing 'feats' ud property", widget = "String", required = false, def = "")

	String defaultFeatsPropertyName;

	@Option(name = "defaultHeadPropertyName", usage = "optional CQP property to fix the missing 'head' ud property", widget = "String", required = false, def = "")

	String defaultHeadPropertyName;

	@Option(name = "defaultDeprelPropertyName", usage = "optional CQP property to fix the missing 'deprel' ud property", widget = "String", required = false, def = "")

	String defaultDeprelPropertyName;

	@Option(name = "defaultDepsPropertyName", usage = "optional CQP property to fix the missing 'deps' ud property", widget = "String", required = false, def = "")

	String defaultDepsPropertyName;

	@Option(name = "defaultMiscPropertyName", usage = "optional CQP property to fix the missing 'misc' ud property", widget = "String", required = false, def = "")

	String defaultMiscPropertyName;

	@Option(name = "separator2", usage = "Options", widget = "Separator", required = true, def = "sentence fix options")

	Boolean separator2 = false;

	@Option(name = "openingPunct", usage = "optional prefix for the properties to create", widget = "String", required = true, def = "[\\-–«‘“\\(]")

	String openingPunct;

	/**

	 * the UD property suffixes, will be used to create the CQP properties like propertiesPrefix + suffix

	 */

	public static String[] propNames = { "id", "form", "lemma", "upos", "xpos", "feats", "head", "deprel", "deps", "misc" };

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.core.commands.AbstractHandler#execute(org.eclipse.core.commands.ExecutionEvent)

	 */

	@Override

	public Object execute(final ExecutionEvent event) throws ExecutionException {

		IStructuredSelection selection = (IStructuredSelection) HandlerUtil.getCurrentSelection(event);

		Object s = selection.getFirstElement();

		if (!(s instanceof MainCorpus)) {

			Log.warning("Selection is not a corpus. Aborting.");

			return null;

		}

		if (!ParametersDialog.open(this)) {

			return null;

		}

		conlluResultDirectory.mkdirs();

		if (conlluResultDirectory == null || !conlluResultDirectory.exists() || !conlluResultDirectory.isDirectory()) {

			Log.warning("Error: conllu result directory does not exists: " + conlluResultDirectory);

			return null;

		}

		CQPCorpus corpus = (CQPCorpus) s;

		MainCorpus mainCorpus = corpus.getMainCorpus();

		try {

			return exportAnnotationsAsCorpus(mainCorpus, conlluResultDirectory, propertiesPrefix, openingPunct, insertTokenWithoutUdAnnotations,

					defaultFormPropertyName, defaultLemmaPropertyName, defaultUposPropertyName, defaultXposPropertyName,

					defaultFeatsPropertyName, defaultHeadPropertyName, defaultDeprelPropertyName, defaultDepsPropertyName,

					defaultMiscPropertyName,

					detectGap, insertParagraphs, insertNoSpaceAfter);

		}

		catch (Exception e) {

			Log.warning(e);

			Log.printStackTrace(e);

		}

		return null;

	}

	/**

	 * export the corpus in a directory of conllu files (one per text)

	 * 

	 * @param mainCorpus

	 * @param conlluResultDirectory

	 * @param prefix

	 * @param openingPunct

	 * @param insertTokenWithoutUdAnnotations

	 * @param defaultFormPropertyName

	 * @param defaultLemmaPropertyName

	 * @param defaultUposPropertyName

	 * @param defaultXposPropertyName

	 * @param detectGap

	 * @param insertParagraphs

	 * @param insertNoSpaceAfter

	 * @return the number of annotation exported

	 * @throws UnexpectedAnswerException

	 * @throws IOException

	 * @throws CqiServerError

	 * @throws CqiClientException

	 * @throws InvalidCqpIdException

	 */

	public static int exportAnnotationsAsCorpus(MainCorpus mainCorpus, File conlluResultDirectory, String prefix, String openingPunct, boolean insertTokenWithoutUdAnnotations,

			String defaultFormPropertyName, String defaultLemmaPropertyName, String defaultUposPropertyName, String defaultXposPropertyName,

			String defaultFeatsPropertyName, String defaultHeadPropertyName, String defaultDeprelPropertyName, String defaultDepsPropertyName,

			String defaultMiscPropertyName,

			boolean detectGap, boolean insertParagraphs, boolean insertNoSpaceAfter)

			throws UnexpectedAnswerException,

			IOException,

			CqiServerError,

			CqiClientException, InvalidCqpIdException {

		if (!conlluResultDirectory.exists()) {

			conlluResultDirectory.mkdirs();

		}

		int numberOfWordsWritten = 0;

		int numberOfSentencesWritten = 0;

		int numberOfTextsWritten = 0;

		String[] textIds = mainCorpus.getCorpusTextIdsList();

		int[] start_limits = mainCorpus.getTextStartLimits();

		int[] end_limits = mainCorpus.getTextEndLimits();

		String lang = mainCorpus.getLang();

		// HashSet<String> beforeSpacesRules = new HashSet<>(LangFormater.getNoSpaceBefore(mainCorpus.getLang()));

		// HashSet<String> afterSpacesRules = new HashSet<>(LangFormater.getNoSpaceAfter(mainCorpus.getLang()));

		for (String p : propNames) {

			WordProperty wp = mainCorpus.getProperty(prefix + p);

			if (wp == null) {

				Log.warning("Error: cannot find the Conllu property: " + prefix + p);

				return 0;

			}

		}

		if (insertTokenWithoutUdAnnotations && (defaultFormPropertyName == null || mainCorpus.getProperty(defaultFormPropertyName) == null)) {

			Log.warning("Error: the defaultFormPropertyName parameter needs to be set if insertTokenWithoutUdAnnotations is set to true");

			return 0;

		}

		for (int iText = 0; iText < start_limits.length; iText++) {

			// Build corpus positions

			int[] positions = new int[end_limits[iText] - start_limits[iText] + 1];

			int tmp = 0;

			for (int n = start_limits[iText]; n <= end_limits[iText]; n++) {

				positions[tmp++] = n;

			}

			numberOfWordsWritten += positions.length;

			// Get UD properties

			WordProperty wp;

			wp = mainCorpus.getProperty(prefix + "id");

			String[] tmpValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			int[] idValues = new int[tmpValues.length];

			for (int iId = 0; iId < tmpValues.length; iId++) {

				if (tmpValues[iId].length() > 0 && !tmpValues[iId].equals("_") && !tmpValues[iId].equals("__UNDEF__")) {

					idValues[iId] = Integer.parseInt(tmpValues[iId]);

				}

				else {

					idValues[iId] = 0;

				}

			}

			tmpValues = null;

			WordProperty formWordProperty = mainCorpus.getProperty(prefix + "form");

			String[] formValues = CQPSearchEngine.getCqiClient().cpos2Str(formWordProperty.getQualifiedName(), positions);

			fixUNDEFValues(formValues);

			wp = mainCorpus.getProperty(prefix + "lemma");

			String[] lemmaValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(lemmaValues);

			wp = mainCorpus.getProperty(prefix + "upos");

			String[] uposValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(uposValues);

			wp = mainCorpus.getProperty(prefix + "xpos");

			String[] xposValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(xposValues);

			wp = mainCorpus.getProperty(prefix + "feats");

			String[] featsValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(featsValues);

			wp = mainCorpus.getProperty(prefix + "head");

			// String[] headValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			tmpValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			int[] headValues = new int[tmpValues.length];

			for (int iId = 0; iId < tmpValues.length; iId++) {

				if (tmpValues[iId].length() > 0 && !tmpValues[iId].equals("_") && !tmpValues[iId].equals("__UNDEF__")) {

					headValues[iId] = Integer.parseInt(tmpValues[iId]);

				}

				else {

					headValues[iId] = -1;

				}

			}

			tmpValues = null;

			wp = mainCorpus.getProperty(prefix + "deprel");

			String[] deprelValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(deprelValues);

			wp = mainCorpus.getProperty(prefix + "deps");

			String[] depsValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(depsValues);

			wp = mainCorpus.getProperty(prefix + "misc");

			String[] miscValues = CQPSearchEngine.getCqiClient().cpos2Str(wp.getQualifiedName(), positions);

			fixUNDEFValues(miscValues);

			HashSet<Integer> paragraphsStartPositions = new HashSet<>();

			if (insertParagraphs) {

				StructuralUnit p_struct = mainCorpus.getStructuralUnit("p");

				if (p_struct == null) {

					Log.warning(NLS.bind("** insertParagraph parameter is set, but there are no *p* structure (no paragraph) in the {0} CQP corpus. The insertParagraph parameter will be ignored.",

							mainCorpus));

				}

				else {

					for (int position : mainCorpus.query(new CQLQuery("<p> [_.text_id=\"" + textIds[iText] + "\"]"), "textParagraphPositions", false).getStarts()) {

						paragraphsStartPositions.add(position);

					}

				}

			}

			HashMap<Integer, String> sentidStartPositions = getNonUNDEFPositionsAndValues(mainCorpus, prefix+"sentid");

			HashMap<Integer, String> newdocidStartPositions = getNonUNDEFPositionsAndValues(mainCorpus, prefix+"newdocid");

			// build sentence, first pass using UD word sentence positions

			ArrayList<ArrayList<Integer>> sentences = new ArrayList<>();

			ArrayList<Integer> tmpSentence = new ArrayList<>();

			for (int p = 0; p < positions.length; p++) {

				// System.out.println("p=" + p + " id=" + idValues[p] + " form=" + formValues[p] + " lemma=" + lemmaValues[p] + " upos=" + uposValues[p] + " xpos=" + xposValues[p] + " feats="

				// + featsValues[p] + " head="

				// + headValues[p] + " deprel=" + deprelValues[p] + " deps=" + depsValues[p] + " misc=" + miscValues[p]);

				if (sentidStartPositions.containsKey(p)) { // new ud sentence

					if (tmpSentence.size() > 0) {

						sentences.add(new ArrayList<>(tmpSentence));

					}

					// System.out.println("new sentence: " + " id=" + idValues[p] + " form=" + formValues[p] + " lemma=" + lemmaValues[p] + " upos=" + uposValues[p] + " xpos=" + xposValues[p] + "

					// feats="

					// + featsValues[p] + " head="

					// + headValues[p] + " deprel=" + deprelValues[p] + " deps=" + depsValues[p] + " misc=" + miscValues[p]);

					tmpSentence.clear();

				}

				if (insertTokenWithoutUdAnnotations) {

					tmpSentence.add(p); // insert all tokens

				}

				else if (idValues[p] != 0) {

					tmpSentence.add(p); // insert all tokens

				}

			}

			positions = null; // free memory

			// fixing sentences

			for (int s = 0; s < sentences.size(); s++) {

				// fix only ud sentences limits

				ArrayList<Integer> sentence = sentences.get(s);

				if (sentidStartPositions.get(sentence.get(0)) == null) {

					continue; // this is not a UD sentence

				}

				int max = -1;

				int imax = 0;

				for (int ip = 0; ip < sentence.size(); ip++) {

					int p = sentence.get(ip);

					if (max < idValues[p]) {

						max = idValues[p];

						imax = ip;

					}

				}

				ArrayList<Integer> newSentence = new ArrayList<>();

				for (int ip = imax + 1; ip < sentence.size(); ip++) {

					newSentence.add(sentence.get(ip));

					sentence.remove(ip);

					ip--;

				}

				if (newSentence.size() == 1) { // the new sentence size is 1, resinsert it

					sentence.addAll(newSentence);

					newSentence.clear();

				}

				if (newSentence.size() > 0) {

					//System.out.println("INSERT " + newSentence);

					sentences.add(s + 1, newSentence);

				}

			}

			if (tmpSentence.size() > 0) { // add last sentence

				sentences.add(new ArrayList<>(tmpSentence));

			}

			// fixing sentence __NULL__ ud properties

			for (int iSentence = 0; iSentence < sentences.size(); iSentence++) {

				ArrayList<Integer> sentence = sentences.get(iSentence);

				int[] sentencePositions = new int[sentence.size()];

				for (int p = 0; p < sentence.size(); p++) {

					sentencePositions[p] = sentence.get(p);

				}

				// get CQP values fixing "form", "lemma", "upos", "xpos", "feats", "head", "deprel", "deps"

				String[] ids = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty("id").getQualifiedName(), sentencePositions);

				String[] words = null;

				if (defaultFormPropertyName != null && defaultFormPropertyName.length() > 0) {

					words = getDefaultValues(mainCorpus, defaultFormPropertyName, sentencePositions);

				}

				String[] lemmas = null;

				if (defaultLemmaPropertyName != null && defaultLemmaPropertyName.length() > 0) {

					lemmas = getDefaultValues(mainCorpus, defaultLemmaPropertyName, sentencePositions);

				}

				String[] uposs = null;

				if (defaultUposPropertyName != null && defaultUposPropertyName.length() > 0) {

					uposs = getDefaultValues(mainCorpus, defaultUposPropertyName, sentencePositions);

				}

				String[] xposs = null;

				if (defaultXposPropertyName != null && defaultXposPropertyName.length() > 0) {

					xposs = getDefaultValues(mainCorpus, defaultXposPropertyName, sentencePositions);

				}

				String[] feats = null;

				if (defaultFeatsPropertyName != null && defaultFeatsPropertyName.length() > 0) {

					feats = getDefaultValues(mainCorpus, defaultFeatsPropertyName, sentencePositions);

				}

				String[] heads = null;

				if (defaultHeadPropertyName != null && defaultHeadPropertyName.length() > 0) {

					heads = getDefaultValues(mainCorpus, defaultHeadPropertyName, sentencePositions);

				}

				String[] deprels = null;

				if (defaultDeprelPropertyName != null && defaultDeprelPropertyName.length() > 0) {

					deprels = getDefaultValues(mainCorpus, defaultDeprelPropertyName, sentencePositions);

				}

				String[] depss = null;

				if (defaultDepsPropertyName != null && defaultDepsPropertyName.length() > 0) {

					depss = getDefaultValues(mainCorpus, defaultDepsPropertyName, sentencePositions);

				}

				String[] miscs = null;

				if (defaultMiscPropertyName != null && defaultMiscPropertyName.length() > 0) {

					miscs = getDefaultValues(mainCorpus, defaultMiscPropertyName, sentencePositions);

				}

				// String[] feats = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty(featsCorrPropertyName).getQualifiedName(), sentencePositions);

				// String[] head = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty(headCorrPropertyName).getQualifiedName(), sentencePositions);

				// String[] deprel = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty(deprelCorrPropertyName).getQualifiedName(), sentencePositions);

				// String[] deps = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty(deprelCorrPropertyName).getQualifiedName(), sentencePositions);

				// fix ud properties using CQP values

				for (int ip = 0; ip < sentence.size(); ip++) {

					int p = sentence.get(ip);

					// new word

					if (miscValues[p].equals("_")) {

						miscValues[p] = "XmlId=" + ids[ip];

					}

					// { "id", "form", "lemma", "upos", "xpos", "feats", "head", "deprel", "deps", "misc" };

					if (words != null && formValues[p].equals("_")) {

						formValues[p] = words[ip];

					}

					if (lemmas != null && lemmaValues[p].equals("_")) {

						lemmaValues[p] = lemmas[ip];

					}

					if (uposs != null && uposValues[p].equals("_")) {

						uposValues[p] = uposs[ip];

					}

					if (xposs != null && xposValues[p].equals("_")) {

						xposValues[p] = xposs[ip];

					}

					if (feats != null && featsValues[p].equals("_")) {

						featsValues[p] = feats[ip];

					}

					if (heads != null && headValues[p] < 0) {

						headValues[p] = Integer.parseInt(heads[ip]);

					}

					if (deprels != null && deprelValues[p].equals("_")) {

						deprelValues[p] = deprels[ip];

					}

					if (depss != null && depsValues[p].equals("_")) {

						depsValues[p] = depss[ip];

					}

					if (miscs != null && miscValues[p].equals("_")) {

						miscValues[p] = miscs[ip];

					}

				}

				if (insertNoSpaceAfter) {

					for (int ip = 0; ip < sentence.size(); ip++) { // fix SpaceAfter. !!! this needs to be done after ud properties are fixed

						int p = sentence.get(ip);

						// insertion activated

						if (!miscValues[p].contains("SpaceAfter=")) { // only update if not present

							if (LangFormater.isSpaceAfterNotNeeded(formValues[p], lang)) {

								miscValues[p] += "|SpaceAfter=No";

							}

							else if (formValues.length > (p + 1) && LangFormater.isSpaceBeforeNotNeeded(formValues[p + 1], lang)) {

								// if next token needs a space before, set SpaceAfter=Yes to the previous token

								miscValues[p] += "|SpaceAfter=No";

							}

						}

					}

				}

				// fixing sentence punct limits

				while (sentence.size() > 0 && iSentence > 0 && formValues[sentence.get(0)].matches("\\p{P}") && !formValues[sentence.get(0)].matches(openingPunct)) {

					// System.out.println("FIXING: first non-openingPunct position " + formValues[sentence.get(0)] + " in " + iSentence);

					int p2 = sentence.remove(0);

					sentences.get(iSentence - 1).add(p2);

				}

				//

				while (sentence.size() > 0 && iSentence + 1 < sentences.size() && formValues[sentence.get(sentence.size() - 1)].matches(openingPunct)) {

					// System.out.println("FIXING: last openingPunct position " + formValues[sentence.get(sentence.size() - 1)] + " in " + iSentence);

					int p2 = sentence.remove(sentence.size() - 1);

					sentences.get(iSentence + 1).add(0, p2);

				}

				if (sentence.size() == 0) { // sentence was depleted after fixing it

					sentences.remove(iSentence);

					iSentence--;

					continue;

				}

			}

			for (int s = 0; s < sentences.size(); s++) {

				// fix only ud sentences limits

				ArrayList<Integer> sentence = sentences.get(s);

				HashMap<Integer, Integer> oldToNewIds = new HashMap<>();

				for (int ip = 0; ip < sentence.size(); ip++) { // computing old to new ids

					int p = sentence.get(ip);

					if (idValues[p] != 0) { // store "old id -> new id"

						oldToNewIds.put(idValues[p], (ip + 1)); // from 1 to N

					}

				}

				// fixing head and set missing head to 0 and root

				for (int ip = 0; ip < sentence.size(); ip++) {

					int p = sentence.get(ip);

					// fixing id value

					idValues[p] = (ip + 1);  // from 1 to N

					// fixing head values

					if (oldToNewIds.containsKey(headValues[p])) {

						headValues[p] = oldToNewIds.get(headValues[p]);

					}

					else if (headValues[p] != 0) { // new word, set to default values

						headValues[p] = 0;

						deprelValues[p] = "_";

						depsValues[p] = "_";

					}

				}

			}

			// writing sentences

			File resultConlluFile = new File(conlluResultDirectory, textIds[iText] + ".conllu");

			PrintWriter writer = IOUtils.getWriter(resultConlluFile);

			int iParagraph = 1;

			for (int iSentence = 0; iSentence < sentences.size(); iSentence++) {

				ArrayList<Integer> sentence = sentences.get(iSentence);

				int[] sentencePositions = new int[sentence.size()];

				for (int p = 0; p < sentence.size(); p++) {

					sentencePositions[p] = sentence.get(p);

				}

				String[] gap = null;

				if (detectGap && mainCorpus.getProperty("gap") != null) {

					gap = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty("gap").getQualifiedName(), sentencePositions);

				}

				String[] tokens = new String[sentence.size()];

				for (int ip = 0; ip < sentence.size(); ip++) {

					tokens[ip] = formValues[sentence.get(ip)];

				}

				if (insertNoSpaceAfter) {

					writer.println("# text = " + LangFormater.format(StringUtils.join(tokens, " "), mainCorpus.getLang()));

				}

				else {

					writer.println("# text = " + StringUtils.join(tokens, " "));

				}

				if (newdocidStartPositions.containsKey(sentence.get(0))) {

					writer.println("# newdoc id = " + newdocidStartPositions.get(sentence.get(0)));

				}

				else {

					writer.println("# newdoc id = " + textIds[iText]);

				}

				boolean foundSentId = false;

				for (int ip : sentence) {

					if (!foundSentId && sentidStartPositions.containsKey(ip)) {

						writer.println("# sent_id = " + sentidStartPositions.get(ip));

						foundSentId = true;

					}

				}

				if (!foundSentId) { // no sent_id found

					writer.println("# sent_id = " + textIds[iText] + "-" + (iSentence + 1) + ".new");

				}

				if (paragraphsStartPositions.contains(sentence.get(0))) { // paragraphsStartPositions is empty if the injectParagraph option is not set

					writer.println("# newpar id = " + iParagraph);

					iParagraph++;

				}

				for (int ip = 0; ip < sentence.size(); ip++) {

					int p = sentence.get(ip);

					// { "id", "form", "lemma", "upos", "xpos", "feats", "head", "deprel", "deps", "misc" };

					writer.println(idValues[p] + "\t" + formValues[p] + "\t" + lemmaValues[p] + "\t" + uposValues[p]

							+ "\t" + xposValues[p] + "\t" + featsValues[p] + "\t" + headValues[p] + "\t" + deprelValues[p]

							+ "\t" + depsValues[p] + "\t" + miscValues[p]);

					if (gap != null && gap[ip].equals("next")) {

						writer.println("# gap");

					}

				}

				writer.println("");

				numberOfSentencesWritten++;

			}

			writer.close();

			System.out.println(" Text done: " + resultConlluFile);

			numberOfTextsWritten++;

		}

		System.out.println("# words written: " + numberOfWordsWritten);

		System.out.println("# sentences written: " + numberOfSentencesWritten);

		System.out.println("# texts written: " + numberOfTextsWritten);

		return numberOfWordsWritten;

	}

	private static String[] getDefaultValues(MainCorpus mainCorpus, String property, int[] positions) throws UnexpectedAnswerException, IOException, CqiServerError, CqiClientException {

		String[] values = CQPSearchEngine.getCqiClient().cpos2Str(mainCorpus.getProperty(property).getQualifiedName(), positions);

		for (int iupos = 0; iupos < values.length; iupos++) { // recode the || CQP multiple values to ud multiple values

			if (values[iupos].length() > 2 && values[iupos].startsWith("|") && values[iupos].endsWith("|")) {

				values[iupos] = values[iupos].substring(1, values[iupos].length() - 1);

			}

		}

		return values;

	}

	private static HashMap<Integer, String> getNonUNDEFPositionsAndValues(MainCorpus mainCorpus, String property) throws UnexpectedAnswerException, IOException, CqiServerError, CqiClientException {

		HashMap<Integer, String> sentidStartPositions = new HashMap<>();

		int[] ids = CQPSearchEngine.getCqiClient().regex2Id(mainCorpus.getProperty(property).getQualifiedName(), "(?!__UNDEF__).+");

		String[] strs = CQPSearchEngine.getCqiClient().id2Str(mainCorpus.getProperty(property).getQualifiedName(), ids);

		for (int iId = 0; iId < ids.length; iId++) {

			int id = ids[iId];

			int[] pp = CQPSearchEngine.getCqiClient().id2Cpos(mainCorpus.getProperty(property).getQualifiedName(), id);

			for (int p : pp) {

				sentidStartPositions.put(p, strs[iId]);

			}

		}

		return sentidStartPositions;

	}

	private static void fixUNDEFValues(String[] values) {

		for (int i = 0; i < values.length; i++) {

			if (values[i].equals("__UNDEF__") || values[i].equals("") || values[i].equals("|_|")) {

				values[i] = "_";

			}

		}

	}

}

package org.txm.conllu.rcp.commands;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.io.Writer;

import java.util.List;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.CQPCorpus;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.StructuralUnit;

import org.txm.searchengine.cqp.corpus.StructuralUnitProperty;

import org.txm.searchengine.cqp.corpus.WordProperty;

import org.txm.searchengine.cqp.corpus.query.CQLQuery;

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.utils.ConsoleProgressBar;

public class ToCoNLL2009 {

	boolean debug = false;

	/**

	 * To CoNLL2009.

	 *

	 * @param outfile the outfile, where is result is saved

	 * @param corpus the corpus or sub-corpus to export

	 * @param sentenceUnit the StructuralUnitProperty that identify a sentence

	 * @param word the word property

	 * @param lemme the lemme property

	 * @param pos the pos property

	 * @param encoding the encoding of the outfile

	 * @return true, if successful

	 * @throws CqiServerError 

	 * @throws IOException 

	 * @throws CqiClientException 

	 */

	public boolean process(File outdir, CQPCorpus corpus, StructuralUnit sentenceUnit, Property word, Property lemme, Property pos, String encoding) throws CqiClientException, IOException, CqiServerError {

		String sstruct = "";

		if (sentenceUnit != null) {

			sstruct = "expand to "+sentenceUnit.getName();

		}

		String[] texts = corpus.getCorpusTextIdsList();

		WordProperty idProperty = corpus.getProperty("id");

		ConsoleProgressBar cpb = new ConsoleProgressBar(texts.length);

		for (String text : texts) {

			cpb.tick();

			List<Match> matches = corpus.query(new CQLQuery("[_.text_id=\""+text+"\"]"+sstruct), "TMPEXPORTCONLL", false).getMatches();

			if (debug) System.out.println(matches.size());

			int npositions = 0;

			for (Match match : matches) npositions += match.size() + 1;

			if (debug) System.out.println("npositions="+npositions);

			int[] positions = new int[npositions+1];

			int i = 0;

			for (Match match : matches) {

				for (int p : match.getRange())

					positions[i++] = p;

			}

			String[] words = CorpusManager.getCorpusManager().getCqiClient().cpos2Str(word.getQualifiedName(), positions);

			String[] idsList = CorpusManager.getCorpusManager().getCqiClient().cpos2Str(idProperty.getQualifiedName(), positions);

			String[] lemmes = null;

			if (lemme != null) {

				lemmes = CorpusManager.getCorpusManager().getCqiClient().cpos2Str(lemme.getQualifiedName(), positions);

			}

			String[] poss = null;

			if (pos != null) {

				poss =  CorpusManager.getCorpusManager().getCqiClient().cpos2Str(pos.getQualifiedName(), positions);

			}

			File outfile = new File(outdir, text+".conllu");

			Writer writer = new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8"); //$NON-NLS-1$;

			String lemmestr;

			String posstr;

			int id = 0;

			int iposition = 0;

			for (Match match : matches) {

				writer.write("\n");

				writer.write("# newdoc id="+text+"\n");

				writer.write("# sent_id="+match+"\n");

				id = 1; // first position in sentence

				for (i = match.getStart() ; i <= match.getEnd() ; i++) { // loop n times, n = mathc length

					if (lemme != null) {

						lemmestr = lemmes[iposition];

					} else {

						lemmestr = "_";

					}

					if (pos != null) {

						posstr = poss[iposition];

					} else {

						posstr = "_";

					}

					//print(""+id+"\t"+words.get(iposition)+"\t"+lemmestr+"\t_\t"+posstr+"\t_\t_\t_\t_\t_\t_\t_\t_\t_\t_\n");

					writer.write(""+id+"\t"+words[iposition]+"\t"+lemmestr+"\t_\t"+posstr+"\t_\t_\t_\t_\tXmlID="+idsList[iposition]+"\n");

					id++; // next word

					iposition++; // nextposition

				}

			}

			writer.flush();

			writer.close();

		}

		cpb.done();

		return true;

	}

	public static void main(String[] args) throws CqiClientException, IOException, CqiServerError {

		CQPCorpus corpus = null;

		File outfile = new File("/home/mdecorde/Bureau/conn_export.tsv");

		String encoding = "UTF-8";

		String sentenceProperty = "s_id";

		String posProperty = "frpos";

		String lemmaProperty = "frlemma";

		String[] split = sentenceProperty.split("_", 2);

		StructuralUnit s = null;//corpus.getStructuralUnit(split[0]).getProperty(split[1])

		Property word = null;//corpus.getProperty("word")

		Property lemma = null;//corpus.getProperty(lemmaProperty)

		Property pos = null;//corpus.getProperty(posProperty)

		//		if (s == null) { println "Error sentence property: $sentenceProperty"; return}

		//		if (word == null) { println "Error no word property"; return}

		//		if (pos == null) { println "Error pos property: $posProperty"; return}

		//		if (lemma == null) { println "Error lemma property: $lemmaProperty"; return}

		ToCoNLL2009 processor = new ToCoNLL2009();

		processor.process(outfile, corpus, s, word, lemma, pos, encoding);

	}

}

      <command

            categoryId="TreeSearch4TXM.commands.category"

            defaultHandler="org.txm.conllu.rcp.commands.ExportCorpusAsFullCoNLLU"

            id="org.txm.conllu.rcp.commands.ExportCorpusAsFullCoNLLU"

            name="Export as a full CoNLL-U corpus...">

      </command>

         <command

               commandId="org.txm.conllu.rcp.commands.ExportCorpusAsFullCoNLLU"

               icon="icons/functions/UD.png"

               style="push">

            <visibleWhen

                  checkEnabled="false">

               <reference

                     definitionId="OneMainCorpusSelected">

               </reference>

            </visibleWhen>

         </command>

	public static final String EDITION_DEFINITION_TOOLTIP_PROPERTIES = "edition_definition_tooltip_properties"; //$NON-NLS-1$

	public String get(String key, String defaultValue) {

		return node.get(key, defaultValue);

	public final String get(String key) {

		return get(key, "");

	}