Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2020 ENS de Lyon., University of Franche-Comté

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$ 

//

package org.txm.tigersearch.commands;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileFilter;

import java.io.IOException;

import java.util.HashMap;

import javax.xml.stream.XMLStreamException;

import org.eclipse.core.commands.AbstractHandler;

import org.eclipse.core.commands.ExecutionEvent;

import org.eclipse.core.commands.ExecutionException;

import org.eclipse.jface.viewers.IStructuredSelection;

import org.eclipse.ui.handlers.HandlerUtil;

import org.kohsuke.args4j.Option;

import org.txm.core.messages.TXMCoreMessages;

import org.txm.rcp.commands.workspace.UpdateCorpus;

import org.txm.rcp.swt.widget.parameters.ParametersDialog;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.CQPCorpus;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.utils.io.FileCopy;

import org.txm.utils.io.IOUtils;

import org.txm.utils.logger.Log;

import org.txm.xml.xmltxm.XMLTXMWordPropertiesInjection;

import cern.colt.Arrays;

/**

 * Import CONNLU annotations into a TXM corpus

 * 

 * IF the corpus already contains CONNLU annotations, they are replaced

 * 

 * @author mdecorde.

 */

public class ImportCONNLUAnnotations extends AbstractHandler {

	public static final String ID = ImportCONNLUAnnotations.class.getName();

	@Option(name = "connluDirectory", usage = "connluDirectory", widget = "Folder", required = true, def = "connlu-directory")

	File connluDirectory;

	@Option(name = "propertiesPrefix", usage = "optional prefix for the properties to create", widget = "String", required = true, def = "ud")

	String propertiesPrefix;

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.core.commands.AbstractHandler#execute(org.eclipse.core.commands.ExecutionEvent)

	 */

	@Override

	public Object execute(final ExecutionEvent event) throws ExecutionException {

		IStructuredSelection selection = (IStructuredSelection) HandlerUtil.getCurrentSelection(event);

		Object s = selection.getFirstElement();

		if (!(s instanceof MainCorpus)) {

			Log.warning("Selection is not a corpus. Aborting.");

			return null;

		}

		ParametersDialog.open(this);

		if (connluDirectory == null || !connluDirectory.exists() || !connluDirectory.isDirectory() || connluDirectory.listFiles().length == 0) {

			Log.warning("Error: connlu directory is empty: " + connluDirectory);

			return null;

		}

		CQPCorpus corpus = (CQPCorpus) s;

		MainCorpus mainCorpus = corpus.getMainCorpus();

		try {

			return importAnnotations(mainCorpus, connluDirectory, propertiesPrefix);

		}

		catch (Exception e) {

			Log.warning(e);

			e.printStackTrace();

		}

		return null;

	}

	/**

	 * 

	 * if import CONNLU annotations in the corpus with the same name already exists, it is replaced

	 * 

	 * @param corpus

	 * @param connluDirectory

	 * @param propertiesPrefix

	 * @return the number of imported annotations

	 * @throws CqiClientException

	 * @throws CqiServerError

	 * @throws IOException

	 * @throws XMLStreamException

	 */

	public static int importAnnotations(MainCorpus mainCorpus, File connluDirectory, String propertiesPrefix) throws IOException,

			CqiServerError, CqiClientException, XMLStreamException {

		Log.info(TXMCoreMessages.bind("Importing CONNL-u annotations of {0} in {1} using the ''{2}'' prefix...", connluDirectory, mainCorpus, propertiesPrefix));

		File[] files = connluDirectory.listFiles(new FileFilter() {

			@Override

			public boolean accept(File file) {

				return file.isFile() && file.getName().endsWith(".conllu");

			}

		});

		int nTextProcessed = 0;

		int nWords = 0;

		int nWordsInserted = 0;

		for (File coonluFile : files) {

			String textid = coonluFile.getName().substring(0, coonluFile.getName().length() - 7);

			Log.info("** processing text: " + textid);

			File xmltxmFile = mainCorpus.getProject().getText(textid).getXMLTXMFile();

			File xmltxmUpdatedFile = new File(System.getProperty("java.io.tmpdir"), xmltxmFile.getName());

			XMLTXMWordPropertiesInjection processor = new XMLTXMWordPropertiesInjection(xmltxmFile);

			HashMap<String, HashMap<String, String>> rules = new HashMap<>();

			processor.setProperties(rules);

			BufferedReader reader = IOUtils.getReader(coonluFile);

			String line = reader.readLine();

			String[] propNames = { "id", "form", "lemma", "upos", "xpos", "feats", "head", "deprel", "deps", "misc" };

			int nWords2 = 0;

			int nLine = 0;

			while (line != null) {

				nLine++;

				if (line.startsWith("#") || line.length() == 0) {

					line = reader.readLine();

					continue; // comment

				}

				String[] split = line.split("\t", 10);

				if (split.length < 10) {

					Log.warning("Error: line " + nLine + " : " + line + " -> " + Arrays.toString(split) + " len=" + split.length);

					line = reader.readLine();

					continue; // comment

				}

				String id = split[9];

				int from = id.indexOf("XmlId=") + 6;

				if (from < 6) {

					Log.warning("Error: line " + nLine + " with no 'XmlId=': " + line);

					line = reader.readLine();

					continue;

				}

				id = id.substring(from);

				// System.out.println("ID=" + id);

				if (id.contains("-")) continue; // TODO to manage later

				HashMap<String, String> properties = new HashMap<>();

				for (int i = 0; i < split.length; i++) {

					properties.put(propertiesPrefix + propNames[i], split[i]); // add the property name using the prefix

				}

				processor.addProperty(id, properties);

				nWords2++;

				line = reader.readLine();

			}

			reader.close();

			if (nWords2 == 0) {

				Log.warning("** No annotation found in " + coonluFile);

			}

			nWords += nWords2;

			Log.info("** loading annotations from : " + coonluFile);

			if (processor.process(xmltxmUpdatedFile)) {

				if (xmltxmFile.delete() && FileCopy.copy(xmltxmUpdatedFile, xmltxmFile)) {

				}

				else {

					Log.warning("** Warning: annotation import failed for replace the corpus XML-TXM file: " + xmltxmFile + ". TEMP file: " + xmltxmUpdatedFile);

					return 0;

				}

			}

			else {

				Log.warning("** Warning: annotation import failed for text: " + textid);

				return 0;

			}

			if (processor.getNInsertions() == 0) {

				Log.warning("** No annotation imported in " + textid);

			}

			nWordsInserted += processor.getNInsertions();

			nTextProcessed++;

		}

		if (nTextProcessed == 0) {

			Log.warning("** No text to process. Aborting.");

			return 0;

		}

		if (nWords == 0) {

			Log.warning("** No annotation to import in corpus. Aborting.");

			return 0;

		}

		if (nWordsInserted == 0) {

			Log.warning("** No annotation imported. Aborting.");

			return 0;

		}

		Log.info("XML-TXM source files updated. Updating indexes...");

		UpdateCorpus.update(mainCorpus);

		Log.info("Done.");

		return 0;

	}

}

// Copyright © 2010-2020 ENS de Lyon., University of Franche-Comté

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$ 

//

package org.txm.tigersearch.commands;

import java.io.File;

import java.io.IOException;

import java.io.RandomAccessFile;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

import org.apache.log4j.BasicConfigurator;

import org.eclipse.core.commands.AbstractHandler;

import org.eclipse.core.commands.ExecutionEvent;

import org.eclipse.core.commands.ExecutionException;

import org.eclipse.jface.dialogs.MessageDialog;

import org.eclipse.jface.viewers.IStructuredSelection;

import org.eclipse.swt.SWT;

import org.eclipse.swt.widgets.DirectoryDialog;

import org.eclipse.ui.handlers.HandlerUtil;

import org.txm.searchengine.cqp.AbstractCqiClient;

import org.txm.searchengine.cqp.CQPSearchEngine;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.clientExceptions.UnexpectedAnswerException;

import org.txm.searchengine.cqp.corpus.CQPCorpus;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.searchengine.ts.InternalCorpusQueryManagerLocal2;

import org.txm.searchengine.ts.TSCorpus;

import org.txm.searchengine.ts.TSCorpusManager;

import org.txm.utils.ConsoleProgressBar;

import org.txm.utils.DeleteDir;

import org.txm.utils.io.FileCopy;

import org.txm.utils.io.IOUtils;

import org.txm.utils.logger.Log;

import ims.tiger.corpus.Sentence;

import ims.tiger.corpus.T_Node;

import ims.tiger.index.reader.Index;

import ims.tiger.index.reader.IndexException;

import ims.tiger.index.writer.IndexBuilderErrorHandler;

import ims.tiger.index.writer.SimpleErrorHandler;

import ims.tiger.index.writer.XMLIndexing;

import ims.tiger.query.api.QueryIndexException;

import ims.tiger.query.processor.CorpusQueryProcessor;

/**

 * Import TIGERSearch annotations into a TXM corpus

 * 

 * IF the corpus alreasy wontains TIGER annotations, they are replaced

 * 

 * The annotations are given using a TIGERSEarch binary corpus OR a TIGER source directory using a "main.xml" file

 * 

 * @author mdecorde.

 */

public class ExportCorpusAsCONNLU extends AbstractHandler {

	public static final String ID = "org.txm.rcp.commands.function.ComputeTSIndex"; //$NON-NLS-1$

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.core.commands.AbstractHandler#execute(org.eclipse.core.commands.ExecutionEvent)

	 */

	@Override

	public Object execute(final ExecutionEvent event) throws ExecutionException {

		IStructuredSelection selection = (IStructuredSelection) HandlerUtil.getCurrentSelection(event);

		Object s = selection.getFirstElement();

		if (s instanceof MainCorpus) {

			CQPCorpus corpus = (CQPCorpus) s;

			MainCorpus mainCorpus = corpus.getMainCorpus();

			File tigerCorpusDirectory = null;

			DirectoryDialog dialog = new DirectoryDialog(HandlerUtil.getActiveShell(event), SWT.OPEN);

			String path = dialog.open();

			if (path == null) {

				Log.warning("Aborting annotation importation.");

				return null;

			}

			else {

				tigerCorpusDirectory = new File(path);

			}

			File tigerDirectory = new File(mainCorpus.getProjectDirectory(), "tiger");

			File tigerCorpusExistingDirectory = new File(tigerDirectory, tigerCorpusDirectory.getName());

			if (tigerCorpusExistingDirectory.exists()) {

				boolean doIt = MessageDialog.openConfirm(HandlerUtil.getActiveShell(event), "Replace existing annotations", "TIGERSearch annotations already exists, replace them ?");

				if (!doIt) {

					Log.warning("Aborting annotation importation.");

					return null;

				}

			}

			if (new File(tigerCorpusDirectory, "word.lexicon").exists() && new File(tigerCorpusDirectory, "corpus_config.xml").exists()) {

				// ok this is a TIGERSearch binary corpus

			}

			else {

				// need to build a TIGERSearch binary corpus

				File tigerBinaryCorpusDirectory = new File(tigerCorpusDirectory, "tiger");

				if (!buildTIGERCorpus(mainCorpus, tigerCorpusDirectory, tigerBinaryCorpusDirectory)) {

					Log.warning("Aborting annotation importation.");

					return null;

				}

				tigerCorpusDirectory = new File(tigerBinaryCorpusDirectory, corpus.getName());

			}

			try {

				return importAnnotations(mainCorpus, tigerCorpusDirectory, "editionId");

			}

			catch (Exception e) {

				e.printStackTrace();

				return null;

			}

		}

		else {

			Log.warning("Selection is not a corpus. Aborting.");

			return null;

		}

	}

	private boolean buildTIGERCorpus(MainCorpus corpus, File sourceDirectory, File tigerDir) {

		tigerDir.mkdirs();

		File configfile = new File(tigerDir, "tigersearch.logprop");

		if (!configfile.exists()) {

			TSCorpus.createLogPropFile(tigerDir);

		}

		BasicConfigurator.configure();

		File master = new File(sourceDirectory, "main.xml");

		if (!master.exists()) master = new File(sourceDirectory, "master.xml");

		if (!master.exists()) {

			Log.warning("Error: Can't create TIGERSearch corpus: no main or master file found in " + sourceDirectory);

			return false;

		}

		String uri = master.getAbsolutePath(); // TIGER corpus source root file

		File tigerBinDir = new File(tigerDir, corpus.getName());

		tigerBinDir.mkdirs();

		try {

			IndexBuilderErrorHandler handler = new SimpleErrorHandler(tigerBinDir.getAbsolutePath()) {

				@Override

				public void setMessage(String message) {}

				@Override

				public void setNumberOfSentences(int number) {}

				@Override

				public void setProgressBar(int value) {}

			};

			XMLIndexing indexing = new XMLIndexing(corpus.getName(), uri, tigerBinDir.getAbsolutePath(), handler, false);

			indexing.startIndexing();

			File logs = new File(tigerBinDir, "indexing.log");

			String txt = IOUtils.getText(logs);

			if (txt.contains("Error in corpus graph ")) {

				Log.warning("Error while importing TIGER corpus: " + txt);

				return false;

			}

		}

		catch (Exception e) {

			System.out.println(e.getMessage());

			return false;

		}

		return true;

	}

	/**

	 * 

	 * if aTIGER corpus with the same name already exists, it is replaced

	 * 

	 * @param corpus

	 * @param tigerCorpusDirectory

	 * @return the number of imported annotations

	 * @throws IndexException

	 * @throws QueryIndexException

	 * @throws CqiClientException

	 * @throws CqiServerError

	 * @throws IOException

	 * @throws UnexpectedAnswerException

	 */

	public static int importAnnotations(MainCorpus corpus, File tigerCorpusDirectory, String wordIdAttribute) throws IndexException, QueryIndexException, UnexpectedAnswerException, IOException,

			CqiServerError,

			CqiClientException {

		// TXM corpus files

		File tigerDirectory = new File(corpus.getProjectDirectory(), "tiger");

		File tigerCorpusExistingDirectory = new File(tigerDirectory, corpus.getName());

		DeleteDir.deleteDirectory(tigerCorpusExistingDirectory);

		tigerCorpusExistingDirectory.mkdirs();

		File configfile = new File(tigerDirectory, "tigersearch.logprop");

		if (!configfile.exists()) {

			TSCorpus.createLogPropFile(tigerDirectory);

		}

		AbstractCqiClient CQI = CQPSearchEngine.getCqiClient();

		TSCorpusManager manager = new TSCorpusManager(tigerCorpusDirectory.getParentFile(), configfile);

		TSCorpus tcorpus = manager.getCorpus(tigerCorpusDirectory.getName());

		InternalCorpusQueryManagerLocal2 tigermanager = tcorpus.manager;

		CorpusQueryProcessor processor = tigermanager.getQueryProcessor();

		Index index = processor.getIndex();

		int size = 0;

		for (int nr = 0; nr < index.getNumberOfGraphs(); nr++) {

			size += index.getNumberOfTNodes(nr);

		}

		if (size == 0) {

			Log.warning("No word found in the TIGERSearch corpus: " + tigerCorpusDirectory + ". Aborting.");

			return 0;

		}

		Log.info("Importing " + size + " word annotations...");

		// compute start position of sentences

		int[] starts = new int[index.getNumberOfGraphs()];

		for (int i = 0; i < index.getNumberOfGraphs(); i++) {

			starts[i] = 0;

			if (i > 0) {

				starts[i] += index.getNumberOfTNodes(i - 1) + starts[i - 1];

			}

		}

		File offsetsFile = new File(tigerCorpusExistingDirectory, "offsets.data");

		RandomAccessFile offsetsRAFile = new RandomAccessFile(offsetsFile, "rw");

		FileChannel offsetsFileChannel = offsetsRAFile.getChannel();

		MappedByteBuffer offsetsMapped = offsetsFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size * Integer.BYTES);

		// out.putInt(positions[i])

		File presencesFile = new File(tigerCorpusExistingDirectory, "presences.data");

		RandomAccessFile presencesRAFile = new RandomAccessFile(presencesFile, "rw");

		FileChannel presencesFileChannel = presencesRAFile.getChannel();

		MappedByteBuffer presencesMapped = presencesFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size);

		int numberOfWordsAnnotated = 0;

		// for each sentence

		ConsoleProgressBar cpb = new ConsoleProgressBar(index.getNumberOfGraphs());

		for (int nr = 0; nr < index.getNumberOfGraphs(); nr++) {

			cpb.tick();

			int sent_size = index.getNumberOfTNodes(nr);

			Sentence sent = tcorpus.manager.getSentence(nr);

			String[] ids = new String[sent_size];

			int[] tigerPositions = new int[sent_size];

			for (int t = 0; t < sent_size; t++) {

				T_Node terminal = (T_Node) sent.getTerminalAt(t);

				ids[t] = terminal.getFeature(wordIdAttribute);

				// try fixing ID

				if (ids[t].startsWith("w")) {

					if (!ids[t].startsWith("w_")) {

						ids[t] = "w_" + ids[t].substring(1);

					}

				}

				else {

					ids[t] = "w_" + ids[t];

				}

				tigerPositions[t] = starts[nr] + t;

				// System.out.println("T id="+terminal.getID());

			}

			int[] ids_idx = CQI.str2Id(corpus.getProperty("id").getQualifiedName(), ids);

			Integer[] cqpPositions = new Integer[sent_size];

			Integer[] offsets = new Integer[sent_size];

			for (int t = 0; t < sent_size; t++) {

				if (ids_idx[t] >= 0) {

					int[] positions = CQI.id2Cpos(corpus.getProperty("id").getQualifiedName(), ids_idx[t]);

					if (positions.length > 1) {

						Log.warning("Warning: multiple CQP positions for word_id=" + ids[t]);

					}

					cqpPositions[t] = positions[0]; // take the first position

				}

				else { // word not in the CQP corpus

					Log.warning("Could not find word for id=" + ids[t]);

					cqpPositions[t] = null;

				}

				if (cqpPositions[t] != null) {

					offsets[t] = cqpPositions[t] - tigerPositions[t];

				}

				else {

					offsets[t] = null;

				}

			}

			// System.out.println("ids="+Arrays.toString(ids));

			// System.out.println("cqp indexes="+Arrays.toString(ids_idx));

			// System.out.println("tiger positions="+Arrays.toString(tigerPositions));

			// System.out.println("cqp positions="+Arrays.toString(cqpPositions));

			// System.out.println("offsets="+Arrays.toString(offsets));

			// writing data to offset and presences files

			for (int t = 0; t < sent_size; t++) {

				if (offsets[t] != null) {

					numberOfWordsAnnotated++;

					presencesMapped.put((byte) 1);

					offsetsMapped.putInt(offsets[t]);

				}

				else {

					presencesMapped.put((byte) 0);

					offsetsMapped.putInt(0);

				}

			}

		}

		cpb.done();

		offsetsFileChannel.close();

		offsetsRAFile.close();

		presencesFileChannel.close();

		presencesRAFile.close();

		Log.info("Finalizing TIGERSearch corpus");

		if (numberOfWordsAnnotated > 0) {

			FileCopy.copyFiles(tigerCorpusDirectory, tigerCorpusExistingDirectory);

			Log.info("Done. " + numberOfWordsAnnotated + " words annotated.");

		}

		else {

			Log.warning("Warning: no words could be aligned with the CQP corpus. Aborting");

		}

		return numberOfWordsAnnotated;

	}

}

      <command

            categoryId="TIGERSearch4TXM.commands.category"

            defaultHandler="org.txm.tigersearch.commands.ImportCONNLUAnnotations"

            id="org.txm.tigersearch.commands.ImportCONNLUAnnotations"

            name="Import CONNL-u Annotations...">

      </command>

      <command

            categoryId="TIGERSearch4TXM.commands.category"

            defaultHandler="org.txm.tigersearch.commands.ExportCorpusAsCONNLU"

            id="org.txm.tigersearch.commands.ExportCorpusAsCONNLU"

            name="Export CONNL-u Annotations...">

      </command>

            <command

                  commandId="org.txm.tigersearch.commands.ImportCONNLUAnnotations"

                  icon="icons/functions/UDplus.png"

                  style="push">

               <visibleWhen

                     checkEnabled="false">

                  <or>

                     <test

                           forcePluginActivation="true"

                           property="org.txm.rcp.testers.TIGERSearchReady"

                           value="TIGERSearchReady">

                     </test>

                     <reference

                           definitionId="OneMainCorpusSelected">

                     </reference>

                  </or>

               </visibleWhen>

            </command>

      <menuContribution

            locationURI="menu:menu.file.export">

         <command

               commandId="org.txm.tigersearch.commands.ExportCorpusAsCONNLU"

               icon="icons/functions/UD.png"

               label="Export corpus as CONNL-u..."

               style="push">

            <visibleWhen

                  checkEnabled="false">

               <reference

                     definitionId="OneMainCorpusSelected">

               </reference>

            </visibleWhen>

         </command>

      </menuContribution>

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2016-03-02 14:17:06 +0100 (mer., 02 mars 2016) $

// $LastChangedRevision: 3134 $

// $LastChangedBy: mdecorde $

//

package org.txm.xml.xmltxm;

import java.io.File;

import java.io.IOException;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.HashMap;

import java.util.List;

import java.util.Map.Entry;

import javax.xml.stream.XMLStreamException;

import org.txm.xml.IdentityHook;

import org.txm.xml.XMLProcessor;

import org.txm.xml.XPathHookActivator;

import org.txm.xml.XPathsHookActivator;

/**

 * The Class AnnotationInjection.

 *

 * @author mdecorde

 * 

 *         inject annotation from a stand-off file into a xml-tei-txm file

 */

public class XMLTXMWordPropertiesInjection extends XMLProcessor {

	HashMap<String, HashMap<String, String>> rules;

	XPathHookActivator activator;

	IdentityHook hook;

	int nInsertions = 0;

	public XMLTXMWordPropertiesInjection(File infile) throws IOException, XMLStreamException {

		super(infile);

	}

	public void addProperty(String id, HashMap<String, String> properties) throws IOException, XMLStreamException {

		if (rules != null) {

			rules.put(id, properties);

		}

	}

	public void setProperties(HashMap<String, HashMap<String, String>> rules) throws IOException, XMLStreamException {

		this.rules = rules;

		activator = new XPathHookActivator<>(hook, "//w");

		hook = new IdentityHook("word_hook", activator, this) {

			String id;

			boolean inAna = false;

			boolean inForm = false;

			boolean inW = false;

			ArrayList<String[]> anaValues = new ArrayList<>();

			ArrayList<String[]> formValues = new ArrayList<>();

			StringBuilder resp = new StringBuilder(), type = new StringBuilder(), value = new StringBuilder();

			@Override

			public boolean deactivate() {

				return true;

			}

			@Override

			public boolean _activate() {

				id = parser.getAttributeValue(null, "id");

				if (id != null && rules.containsKey(id)) {

					anaValues.clear(); // empty ana values

					formValues.clear(); // empty form values

					nInsertions++;

					return true;

				}

				else {

					return false;

				}

			}

			@Override

			protected void processStartElement() throws XMLStreamException, IOException {

				if (localname.equals("ana")) {

					// store values

					inAna = true;

					resp.setLength(0);

					resp.append(parser.getAttributeValue(null, "resp"));

					type.setLength(0);

					type.append(parser.getAttributeValue(null, "type"));

					value.setLength(0);

					return; // write ana later

				}

				else if (localname.equals("form")) {

					// store values

					inForm = true;

					type.setLength(0);

					if (parser.getAttributeValue(null, "type") != null) {

						type.append(parser.getAttributeValue(null, "type"));

					}

					value.setLength(0);

					return; // write form later

				}

				else {

					if (localname.equals("w")) inW = true;

					super.processStartElement();

				}

			}

			@Override

			protected void processCharacters() throws XMLStreamException {

				if (inAna || inForm) {

					value.append(parser.getText());

				}

				else if (inW) {

				}

				else {

					super.processCharacters();

				}

			}

			@Override

			protected void processEndElement() throws XMLStreamException {

				if (localname.equals("w")) {

					// update ana values

					HashMap<String, String> values = rules.get(id);

					for (String[] l : anaValues) { // update existing values

						if (values.containsKey(l[0])) {

							l[2] = values.get(l[0]);

							values.remove(l[0]);

						}

					}

					for (Entry<String, String> e : values.entrySet()) { // create new values

						anaValues.add(new String[] { "#" + e.getKey(), "#txm", e.getValue() });

					}

					// write forms

					for (String[] l : formValues) {

						writer.writeStartElement("txm:form");

						if (l[0].length() > 0) {

							writer.writeAttribute("type", l[0]);

						}

						writer.writeCharacters(l[1]);

						writer.writeEndElement();

					}

					writer.writeCharacters("\n");

					// write anas

					for (String[] l : anaValues) {

						writer.writeStartElement("txm:ana");

						writer.writeAttribute("resp", l[1]);

						writer.writeAttribute("type", l[0]);

						writer.writeCharacters(l[2]);

						writer.writeEndElement();

					}

					writer.writeCharacters("\n");

					// let parent writer the end of the w element

					inW = false;

					super.processEndElement();

				}

				else if (localname.equals("ana")) {

					anaValues.add(new String[] { type.toString(), resp.toString(), value.toString() });

					inAna = false;

					return; // write ana later

				}

				else if (localname.equals("form")) {

					formValues.add(new String[] { type.toString(), value.toString() });

					inForm = false;

					return; // write form later

				}

				else {

					super.processEndElement();

				}

			}

		};

	}

	public int getNInsertions() {

		return nInsertions;

	}

	public static void main(String[] args) {

		File infile = new File(System.getProperty("user.home"), "xml/srcmfmadridxmltxm/alexis-txm.xml");

		File outfile = new File(System.getProperty("user.home"), "xml/srcmfmadridxmltxm/alexis-txm-out.xml");

		try {

			XMLTXMWordPropertiesInjection processor = new XMLTXMWordPropertiesInjection(infile);

			HashMap<String, HashMap<String, String>> rules = new HashMap<>();

			HashMap<String, String> props = new HashMap<>();

			props.put("lol", "lolvalue");

			rules.put("w19_1303", props);

			processor.setProperties(rules);

			processor.process(outfile);

		}

		catch (Exception e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

	}

}

Export-Package: .,

 EDU.oswego.cs.dl.util.concurrent,

 EDU.oswego.cs.dl.util.concurrent.misc,

 antlr,

 antlr.ASdebug,

 antlr.actions.cpp,

 antlr.actions.csharp,

 antlr.actions.java,

 antlr.actions.python,

 antlr.build,

 antlr.collections,

 antlr.collections.impl,

 antlr.debug,

 antlr.debug.misc,

 antlr.preprocessor,

 ch.randelshofer.quaqua,

 ch.randelshofer.quaqua.border,

 ch.randelshofer.quaqua.color,

 ch.randelshofer.quaqua.colorchooser,

 ch.randelshofer.quaqua.datatransfer,

 ch.randelshofer.quaqua.ext.base64,

 ch.randelshofer.quaqua.ext.batik.ext.awt,

 ch.randelshofer.quaqua.ext.batik.ext.awt.image,

 ch.randelshofer.quaqua.ext.batik.ext.awt.image.codec.tiff,

 ch.randelshofer.quaqua.ext.batik.ext.awt.image.codec.util,

 ch.randelshofer.quaqua.ext.batik.ext.awt.image.renderable,

 ch.randelshofer.quaqua.ext.batik.ext.awt.image.rendered,

 ch.randelshofer.quaqua.ext.batik.i18n,

 ch.randelshofer.quaqua.ext.batik.util,

 ch.randelshofer.quaqua.ext.nanoxml,

 ch.randelshofer.quaqua.filechooser,

 ch.randelshofer.quaqua.icon,

 ch.randelshofer.quaqua.images,

 ch.randelshofer.quaqua.jaguar,

 ch.randelshofer.quaqua.jaguar.filechooser,

 ch.randelshofer.quaqua.jaguar.images,

 ch.randelshofer.quaqua.leopard,

 ch.randelshofer.quaqua.leopard.filechooser,

 ch.randelshofer.quaqua.leopard.images,

 ch.randelshofer.quaqua.osx,

 ch.randelshofer.quaqua.panther,

 ch.randelshofer.quaqua.panther.filechooser,

 ch.randelshofer.quaqua.panther.images,

 ch.randelshofer.quaqua.subset,

 ch.randelshofer.quaqua.tiger,

 ch.randelshofer.quaqua.tiger.filechooser,

 ch.randelshofer.quaqua.tiger.images,

 ch.randelshofer.quaqua.util,

 com.birosoft.liquid,

 com.birosoft.liquid.borders,

 com.birosoft.liquid.icons,

 com.birosoft.liquid.skin,

 com.jgoodies.forms.builder,

 com.jgoodies.forms.debug,

 com.jgoodies.forms.factories,

 com.jgoodies.forms.layout,

 com.jgoodies.forms.util,

 com.jgoodies.looks,

 com.jgoodies.looks.common,

 com.jgoodies.looks.plastic,

 com.jgoodies.looks.plastic.icons,

 com.jgoodies.looks.plastic.theme,

 com.jgoodies.looks.windows,

 com.jgoodies.looks.windows.icons,

 com.jgoodies.looks.windows.icons.xp,

 com.sun.star.accessibility,

 com.sun.star.animations,

 com.sun.star.auth,

 com.sun.star.awt,

 com.sun.star.awt.grid,

 com.sun.star.awt.tree,

 com.sun.star.beans,

 com.sun.star.bridge,

 com.sun.star.bridge.oleautomation,

 com.sun.star.chart,

 com.sun.star.chart2,

 com.sun.star.chart2.data,

 com.sun.star.configuration,

 com.sun.star.configuration.backend,

 com.sun.star.connection,

 com.sun.star.container,

 com.sun.star.corba,

 com.sun.star.corba.giop,

 com.sun.star.corba.iiop,

 com.sun.star.corba.iop,

 com.sun.star.datatransfer,

 com.sun.star.datatransfer.clipboard,

 com.sun.star.datatransfer.dnd,

 com.sun.star.deployment,

 com.sun.star.deployment.test,

 com.sun.star.deployment.ui,

 com.sun.star.document,

 com.sun.star.drawing,

 com.sun.star.drawing.framework,

 com.sun.star.embed,

 com.sun.star.form,

 com.sun.star.form.binding,

 com.sun.star.form.inspection,

 com.sun.star.form.runtime,

 com.sun.star.form.submission,

 com.sun.star.form.validation,

 com.sun.star.formula,

 com.sun.star.frame,

 com.sun.star.frame.status,

 com.sun.star.gallery,

 com.sun.star.geometry,

 com.sun.star.graphic,

 com.sun.star.i18n,

 com.sun.star.inspection,

 com.sun.star.installation,

 com.sun.star.io,

 com.sun.star.java,

 com.sun.star.lang,

 com.sun.star.ldap,

 com.sun.star.lib.uno.typedesc,

 com.sun.star.lib.uno.typeinfo,

 com.sun.star.lib.util,

 com.sun.star.linguistic2,

 com.sun.star.loader,

 com.sun.star.logging,

 com.sun.star.mail,

 com.sun.star.media,

 com.sun.star.mozilla,

 com.sun.star.office,

 com.sun.star.oooimprovement,

 com.sun.star.packages,

 com.sun.star.packages.manifest,

 com.sun.star.packages.zip,

 com.sun.star.plugin,

 com.sun.star.presentation,

 com.sun.star.rdf,

 com.sun.star.reflection,

 com.sun.star.registry,

 com.sun.star.rendering,

 com.sun.star.report,

 com.sun.star.report.inspection,

 com.sun.star.report.meta,

 com.sun.star.resource,

 com.sun.star.scanner,

 com.sun.star.script,

 com.sun.star.script.browse,

 com.sun.star.script.provider,

 com.sun.star.sdb,

 com.sun.star.sdb.application,

 com.sun.star.sdb.tools,

 com.sun.star.sdbc,

 com.sun.star.sdbcx,

 com.sun.star.security,

 com.sun.star.setup,

 com.sun.star.sheet,

 com.sun.star.smarttags,

 com.sun.star.style,

 com.sun.star.svg,

 com.sun.star.sync,

 com.sun.star.sync2,

 com.sun.star.system,

 com.sun.star.table,

 com.sun.star.task,

 com.sun.star.test,

 com.sun.star.test.bridge,

 com.sun.star.test.performance,

 com.sun.star.text,

 com.sun.star.ucb,

 com.sun.star.ui,

 com.sun.star.ui.dialogs,

 com.sun.star.uno,

 com.sun.star.uri,

 com.sun.star.util,

 com.sun.star.util.logging,

 com.sun.star.view,

 com.sun.star.xforms,

 com.sun.star.xml,

 com.sun.star.xml.crypto,

 com.sun.star.xml.crypto.sax,

 com.sun.star.xml.csax,

 com.sun.star.xml.dom,

 com.sun.star.xml.dom.events,

 com.sun.star.xml.dom.views,

 com.sun.star.xml.input,

 com.sun.star.xml.sax,

 com.sun.star.xml.wrapper,

 com.sun.star.xml.xpath,

 com.sun.star.xsd,

 drafts.com.sun.star.form,

 images,

 jline,

 jsesh.macSpecific,

 jsesh.softwares.rtfCleaner,

 junit.extensions,

 junit.framework,

 junit.runner,

 junit.textui,

 org.apache.log4j,

 org.apache.log4j.chainsaw,

 org.apache.log4j.config,

 org.apache.log4j.helpers,

 org.apache.log4j.jdbc,

 org.apache.log4j.jmx,

 org.apache.log4j.lf5,

 org.apache.log4j.lf5.config,

 org.apache.log4j.lf5.util,

 org.apache.log4j.lf5.viewer,

 org.apache.log4j.lf5.viewer.categoryexplorer,

 org.apache.log4j.lf5.viewer.configure,

 org.apache.log4j.lf5.viewer.images,

 org.apache.log4j.net,

 org.apache.log4j.nt,

 org.apache.log4j.or,

 org.apache.log4j.or.jms,

 org.apache.log4j.or.sax,

 org.apache.log4j.spi,

 org.apache.log4j.varia,

 org.apache.log4j.xml,

 org.apache.tools.ant,

 org.apache.tools.ant.dispatch,

 org.apache.tools.ant.filters,

 org.apache.tools.ant.filters.util,

 org.apache.tools.ant.helper,

 org.apache.tools.ant.input,

 org.apache.tools.ant.launch,

 org.apache.tools.ant.listener,

 org.apache.tools.ant.loader,

 org.apache.tools.ant.property,

 org.apache.tools.ant.taskdefs,

 org.apache.tools.ant.taskdefs.compilers,

 org.apache.tools.ant.taskdefs.condition,

 org.apache.tools.ant.taskdefs.cvslib,

 org.apache.tools.ant.taskdefs.email,

 org.apache.tools.ant.taskdefs.rmic,

 org.apache.tools.ant.types,

 org.apache.tools.ant.types.conditions,

 org.apache.tools.ant.types.mappers,

 org.apache.tools.ant.types.resources,

 org.apache.tools.ant.types.resources.comparators,

 org.apache.tools.ant.types.resources.selectors,

 org.apache.tools.ant.types.selectors,

 org.apache.tools.ant.types.selectors.modifiedselector,

 org.apache.tools.ant.types.spi,

 org.apache.tools.ant.util,

 org.apache.tools.ant.util.facade,

 org.apache.tools.ant.util.regexp,

 org.apache.tools.bzip2,

 org.apache.tools.mail,

 org.apache.tools.tar,

 org.apache.tools.zip,

 org.artofsolving.jodconverter,

 org.artofsolving.jodconverter.cli,

 org.artofsolving.jodconverter.document,

 org.artofsolving.jodconverter.office,

 org.artofsolving.jodconverter.process,

 org.artofsolving.jodconverter.util,

 org.dom4j,

 org.dom4j.bean,

 org.dom4j.datatype,

 org.dom4j.dom,

 org.dom4j.dtd,

 org.dom4j.io,

 org.dom4j.jaxb,

 org.dom4j.rule,

 org.dom4j.rule.pattern,

 org.dom4j.swing,

 org.dom4j.tree,

 org.dom4j.util,

 org.dom4j.xpath,

 org.dom4j.xpp,

 org.hamcrest,

 org.hamcrest.core,

 org.hamcrest.internal,

 org.jdesktop.layout,

 org.json,

 org.jsoup,

 org.jsoup.helper,

 org.jsoup.internal,

 org.jsoup.nodes,

 org.jsoup.parser,

 org.jsoup.safety,

 org.jsoup.select,

 org.junit,

 org.junit.experimental.results,

 org.junit.experimental.runners,

 org.junit.experimental.theories,

 org.junit.experimental.theories.internal,

 org.junit.experimental.theories.suppliers,

 org.junit.internal,

 org.junit.internal.builders,

 org.junit.internal.matchers,

 org.junit.internal.requests,

 org.junit.internal.runners,

 org.junit.internal.runners.model,

 org.junit.internal.runners.statements,

 org.junit.matchers,

 org.junit.runner,

 org.junit.runner.manipulation,

 org.junit.runner.notification,

 org.junit.runners,

 org.junit.runners.model,

 org.knallgrau.utils.textcat,

 org.knallgrau.utils.textcat.language_fp,

 org.mozilla.universalchardet,

 org.mozilla.universalchardet.prober,

 org.mozilla.universalchardet.prober.contextanalysis,

 org.mozilla.universalchardet.prober.distributionanalysis,

 org.mozilla.universalchardet.prober.sequence,

 org.mozilla.universalchardet.prober.statemachine,

 org.objectweb.asm.tree,

 org.objectweb.asm.tree.analysis,

 org.postgresql,

 org.postgresql.copy,

 org.postgresql.core,

 org.postgresql.core.v2,

 org.postgresql.core.v3,

 org.postgresql.ds,

 org.postgresql.ds.common,

 org.postgresql.fastpath,

 org.postgresql.geometric,

 org.postgresql.gss,

 org.postgresql.hostchooser,

 org.postgresql.jdbc,

 org.postgresql.jdbc2,

 org.postgresql.jdbc2.optional,

 org.postgresql.jdbc3,

 org.postgresql.largeobject,

 org.postgresql.osgi,

 org.postgresql.ssl,

 org.postgresql.ssl.jdbc4,

 org.postgresql.sspi,

 org.postgresql.translation,

 org.postgresql.util,

 org.postgresql.xa,

 org.qenherkhopeshef.graphics.bitmaps,

 org.qenherkhopeshef.graphics.emf,

 org.qenherkhopeshef.graphics.eps,

 org.qenherkhopeshef.graphics.generic,

 org.qenherkhopeshef.graphics.pict,

 org.qenherkhopeshef.graphics.rtfBasicWriter,

 org.qenherkhopeshef.graphics.svg,

 org.qenherkhopeshef.graphics.utils,

 org.qenherkhopeshef.graphics.vectorClipboard,

 org.qenherkhopeshef.graphics.wmf,

 org.sqlite,

 org.sqlite.core,

 org.sqlite.date,

 org.sqlite.javax,

 org.sqlite.jdbc3,

 org.sqlite.jdbc4,

 org.sqlite.util,

 org.txm,

 org.txm.core,

 org.txm.core.engines,

 org.txm.core.messages,

 org.txm.core.preferences,

 org.txm.core.results,