Laboratoire ICAR » Plateforme TXM

import java.io.File;

import java.io.FileNotFoundException;

import java.io.UnsupportedEncodingException;

import org.txm.utils.io.IOUtils;

	public InternalCorpusQueryManagerLocal2 manager = null;

	public static boolean createLogPropFile(File directory) {

		directory.mkdirs();

		File logprop = new File(directory, "tigersearch.logprop");

		try {

			IOUtils.write(logprop, "# Default log configuration of the TIGERSearch suite\n" + 

					"log4j.rootLogger=SEVERE,Logfile\n" + 

					"log4j.logger.ims.tiger.gui.tigersearch.TIGERSearch=SEVERE\n" + 

					"log4j.appender.Logfile=org.apache.log4j.RollingFileAppender\n" + 

					"log4j.appender.Logfile.File=\\${user.home}/tigersearch/tigersearch.log\n" + 

					"log4j.appender.Logfile.MaxFileSize=500KB\n" + 

					"log4j.appender.Logfile.MaxBackupIndex=1\n" + 

					"log4j.appender.Logfile.layout=org.apache.log4j.PatternLayout\n" + 

					"log4j.appender.Logfile.layout.ConversionPattern=%5r %-5p [%t] %c{2} - %m%n");

		} catch (Exception e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

			return false;

		}

		return true;

	}

import java.io.RandomAccessFile;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

		File offsetsFile = new File(tigerDirectory, corpus.getRootCorpusBuild().getID()+"/offsets.data");

		RandomAccessFile offsetsRAFile = null;

		FileChannel offsetsFileChannel = null;

		MappedByteBuffer offsetsMapped = null;

		if (offsetsFile.exists()) {

			offsetsRAFile = new RandomAccessFile(offsetsFile, "rw");

			offsetsFileChannel = offsetsRAFile.getChannel();

			offsetsMapped = offsetsFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, offsetsFileChannel.size());

		}

		//out.putInt(positions[i])

		File presencesFile = new File(tigerDirectory, corpus.getRootCorpusBuild().getID()+"/presences.data");

		RandomAccessFile presencesRAFile = null;

		FileChannel presencesFileChannel = null;

		MappedByteBuffer presencesMapped = null;

		if (presencesFile.exists()) {

			presencesRAFile = new RandomAccessFile(presencesFile, "rw");

			presencesFileChannel = presencesRAFile.getChannel();

			presencesMapped = presencesFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, presencesFileChannel.size());

		}

					int left = sent_start+index.getLeftCorner(sent, match[i]);

					if (offsetsMapped != null) { // the TIGER token is not in the CQP corpus

						left += offsetsMapped.getInt(left*Integer.BYTES);

//						System.out.println("left="+left+" offset="+offsetsMapped.getInt(left*Integer.BYTES));

					}

					int right = sent_start+index.getRightCorner(sent, match[i]);

					if (offsetsMapped != null) { // the TIGER token is not in the CQP corpus

						right += offsetsMapped.getInt(right*Integer.BYTES);

					}

					TIGERMatch tigerMatch = new TIGERMatch(left, right);

		if (presencesRAFile != null) presencesRAFile.close();

		if (presencesFileChannel != null) presencesFileChannel.close();

		if (offsetsRAFile != null) offsetsRAFile.close();

		if (offsetsFileChannel != null) offsetsFileChannel.close();

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$ 

//

package org.txm.tigersearch.commands;

import java.io.File;

import java.io.IOException;

import java.io.RandomAccessFile;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

import java.util.Arrays;

import org.eclipse.core.commands.AbstractHandler;

import org.eclipse.core.commands.ExecutionEvent;

import org.eclipse.core.commands.ExecutionException;

import org.eclipse.jface.dialogs.MessageDialog;

import org.eclipse.jface.viewers.IStructuredSelection;

import org.eclipse.swt.SWT;

import org.eclipse.swt.widgets.DirectoryDialog;

import org.eclipse.ui.handlers.HandlerUtil;

import org.txm.searchengine.cqp.AbstractCqiClient;

import org.txm.searchengine.cqp.CQPSearchEngine;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.clientExceptions.UnexpectedAnswerException;

import org.txm.searchengine.cqp.corpus.CQPCorpus;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.searchengine.ts.InternalCorpusQueryManagerLocal2;

import org.txm.searchengine.ts.TSCorpus;

import org.txm.searchengine.ts.TSCorpusManager;

import org.txm.utils.DeleteDir;

import org.txm.utils.io.FileCopy;

import org.txm.utils.logger.Log;

import ims.tiger.corpus.Sentence;

import ims.tiger.corpus.T_Node;

import ims.tiger.index.reader.Index;

import ims.tiger.index.reader.IndexException;

import ims.tiger.query.api.QueryIndexException;

import ims.tiger.query.processor.CorpusQueryProcessor;

// TODO: Auto-generated Javadoc

/**

 * open the TIGERSearch Editor 

 * @author mdecorde.

 */

public class ImportTIGERAnnotations extends AbstractHandler {

	public static final String ID = "org.txm.rcp.commands.function.ComputeTSIndex"; //$NON-NLS-1$

	/* (non-Javadoc)

	 * @see org.eclipse.core.commands.AbstractHandler#execute(org.eclipse.core.commands.ExecutionEvent)

	 */

	@Override

	public Object execute(final ExecutionEvent event) throws ExecutionException {

		IStructuredSelection selection = (IStructuredSelection) HandlerUtil.getCurrentSelection(event);

		Object s = selection.getFirstElement();

		if (s instanceof CQPCorpus) {

			CQPCorpus corpus = (CQPCorpus)s;

			MainCorpus mainCorpus = corpus.getMainCorpus();

			File tigerCorpusDirectory = null;

			DirectoryDialog dialog = new DirectoryDialog(HandlerUtil.getActiveShell(event), SWT.OPEN);

			String path = dialog.open();

			if (path == null) {

				return null;

			} else {

				tigerCorpusDirectory = new File(path);

			}

			File tigerDirectory = new File(corpus.getProjectDirectory(), "tiger");

			File tigerCorpusExistingDirectory = new File(tigerDirectory, tigerCorpusDirectory.getName());

			if (tigerCorpusExistingDirectory.exists()) {

				boolean doIt = MessageDialog.openConfirm(HandlerUtil.getActiveShell(event), "Replace existing annotations", "TIGERSearch annotations already exists, replace them ?");

				if (!doIt) {

					Log.warning("Aborting annotations import.");

					return null;

				}

			}

			try {

				return importAnnotations(mainCorpus, tigerCorpusDirectory);

			} catch (Exception e) {

				// TODO Auto-generated catch block

				e.printStackTrace();

				return null;

			}

		} else {

			Log.warning("Selection is not a corpus. Aborting.");

			return null;

		}

	}

	/**

	 * 

	 * if aTIGER corpus with the same name already exists, it is replaced

	 * 

	 * @param corpus

	 * @param tigerCorpusDirectory

	 * @return the number of imported annotations

	 * @throws IndexException 

	 * @throws QueryIndexException 

	 * @throws CqiClientException 

	 * @throws CqiServerError 

	 * @throws IOException 

	 * @throws UnexpectedAnswerException 

	 */

	public static int importAnnotations(MainCorpus corpus, File tigerCorpusDirectory) throws IndexException, QueryIndexException, UnexpectedAnswerException, IOException, CqiServerError, CqiClientException {

		// TXM corpus files

		File tigerDirectory = new File(corpus.getProjectDirectory(), "tiger");

		File tigerCorpusExistingDirectory = new File(tigerDirectory, tigerCorpusDirectory.getName());

		DeleteDir.deleteDirectory(tigerCorpusExistingDirectory);

		tigerCorpusExistingDirectory.mkdirs();

		File configfile = new File(tigerDirectory, "tigersearch.logprop");

		if (!configfile.exists()) {

			TSCorpus.createLogPropFile(tigerDirectory);

		}

		AbstractCqiClient CQI = CQPSearchEngine.getCqiClient();

		TSCorpusManager manager = new TSCorpusManager(tigerCorpusDirectory.getParentFile(), configfile);

		TSCorpus tcorpus = manager.getCorpus(tigerCorpusDirectory.getName());

		InternalCorpusQueryManagerLocal2 tigermanager = tcorpus.manager;

		CorpusQueryProcessor processor = tigermanager.getQueryProcessor();

		Index index = processor.getIndex();

		int size = 0;

		for (int nr = 0 ; nr < index.getNumberOfGraphs() ; nr++) {

			size += index.getNumberOfTNodes(nr);

		}

		if (size == 0) {

			Log.warning("No word found in the TIGERSearch corpus: "+tigerCorpusDirectory+". Aborting.");

			return 0;

		}

		Log.info("Importing "+size+" word annotations...");

		// compute start position of sentences

		int[] starts = new int[index.getNumberOfGraphs()];

		for (int i = 0 ; i < index.getNumberOfGraphs() ; i++) {

			starts[i] = 0;

			if (i > 0) {

				starts[i] += index.getNumberOfTNodes(i-1) + starts[i-1];

			}

		}

		File offsetsFile = new File(tigerCorpusExistingDirectory, "offsets.data");

		RandomAccessFile offsetsRAFile = new RandomAccessFile(offsetsFile, "rw");

		FileChannel offsetsFileChannel = offsetsRAFile.getChannel();

		MappedByteBuffer offsetsMapped = offsetsFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size*Integer.BYTES);

		//out.putInt(positions[i])

		File presencesFile = new File(tigerCorpusExistingDirectory, "presences.data");

		RandomAccessFile presencesRAFile = new RandomAccessFile(presencesFile, "rw");

		FileChannel presencesFileChannel = presencesRAFile.getChannel();

		MappedByteBuffer presencesMapped = presencesFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size);

		int numberOfWordsAnntoated = 0;

		// for each sentence

		for (int nr = 0 ; nr < index.getNumberOfGraphs() ; nr++) {

			int sent_size = index.getNumberOfTNodes(nr);

			Sentence sent = tcorpus.manager.getSentence(nr);

			String[] ids = new String[sent_size];

			int[] tigerPositions = new int[sent_size];

			for (int t = 0 ; t < sent_size ; t++) {

				T_Node terminal = (T_Node)sent.getTerminalAt(t);

				ids[t] = terminal.getFeature("editionId");

				// try fixing ID

				if (ids[t].startsWith("w")) {

					if (!ids[t].startsWith("w_")) {

						ids[t] = "w_"+ids[t].substring(1);

					}

				} else {

					ids[t] = "w_"+ids[t];

				}

				tigerPositions[t] = starts[nr]+t;

				//System.out.println("T id="+terminal.getID());

			}

			int[] ids_idx = CQI.str2Id(corpus.getProperty("id").getQualifiedName(), ids);

			Integer[] cqpPositions = new Integer[sent_size];

			Integer[] offsets = new Integer[sent_size];

			for (int t = 0 ; t < sent_size ; t++) {

				if (ids_idx[t] >= 0) {

					int[] positions = CQI.id2Cpos(corpus.getProperty("id").getQualifiedName(), ids_idx[t]);

					if (positions.length > 1) {

						Log.warning("Warning: multiple CQP positions for word_id="+ids[t]);

					}

					cqpPositions[t] = positions[0]; // take the first position

				} else { // word not in the CQP corpus

					cqpPositions[t] = null;

				}

				if (cqpPositions[t] != null) {

					offsets[t] = cqpPositions[t] - tigerPositions[t];

				} else {

					offsets[t] = 0;

				}

			}

//			System.out.println("ids="+Arrays.toString(ids));

//			System.out.println("cqp indexes="+Arrays.toString(ids_idx));

//			System.out.println("tiger positions="+Arrays.toString(tigerPositions));

//			System.out.println("cqp positions="+Arrays.toString(cqpPositions));

//			System.out.println("offsets="+Arrays.toString(offsets));

			// writing data to offset and presences files

			for (int t = 0 ; t < sent_size ; t++) {

				offsetsMapped.putInt(offsets[t]);

				if (offsets[t] != null) {

					numberOfWordsAnntoated++;

					presencesMapped.put((byte)1);

				} else {

					presencesMapped.put((byte)0);

				}

			}

		}

		offsetsFileChannel.close();

		offsetsRAFile.close();

		presencesFileChannel.close();

		presencesRAFile.close();

		if (numberOfWordsAnntoated > 0) {

			FileCopy.copyFiles(tigerCorpusDirectory, tigerCorpusExistingDirectory);

			Log.warning("Done. "+numberOfWordsAnntoated+" words annotated.");

		} else {

			Log.warning("Warning: no words could be aligned with the CQP corpus. Aborting");

		}

		return numberOfWordsAnntoated;

	}

}

      <command

            categoryId="TIGERSearch4TXM.commands.category"

            defaultHandler="org.txm.tigersearch.commands.ImportTIGERAnnotations"

            id="org.txm.tigersearch.commands.ImportTIGERAnnotations"

            name="Import TIGERSearch Annotations...">

      </command>

      <menuContribution

            locationURI="menu:menu.corpus?after=menu.corpus.build">

         <command

               commandId="org.txm.tigersearch.commands.ImportTIGERAnnotations"

               icon="icons/functions/TSplus.png"

               style="push">

            <visibleWhen

                  checkEnabled="false">

               <or>

                  <test

                        forcePluginActivation="true"

                        property="org.txm.rcp.testers.TIGERSearchReady"

                        value="TIGERSearchReady">

                  </test>

                  <reference

                        definitionId="OneMainCorpusSelected">

                  </reference>

               </or>

            </visibleWhen>

         </command>

      </menuContribution>