Laboratoire ICAR » Plateforme TXM

import org.txm.searchengine.cqp.corpus.MainCorpus

				// re-align TIGER word indexes with the CQP word indexes using the TS@editionId and CQP@id properties

				File tigerDirectory = new File(this.binaryDirectory, "tiger");

				File tigerCorpusExistingDirectory = new File(this.binaryDirectory, "tiger/"+corpusName);

				MainCorpus corpus = this.project.getFirstChild(MainCorpus.class);

				corpus.compute(false); // load  the corpus in CQP

				TIGERSearchEngine.writeOffsetDataFiles(corpus, "editionId", tigerCorpusExistingDirectory, tigerDirectory, tigerCorpusExistingDirectory)

import java.io.RandomAccessFile;

import java.nio.channels.FileChannel;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.utils.ConsoleProgressBar;

import ims.tiger.corpus.Sentence;

import ims.tiger.corpus.T_Node;

import ims.tiger.query.api.QueryIndexException;

import ims.tiger.query.processor.CorpusQueryProcessor;

	public static int writeOffsetDataFiles(MainCorpus corpus, String wordIdAttribute, File tigerCorpusDirectory, File tigerDirectory, File tigerCorpusExistingDirectory) throws IndexException, QueryIndexException, UnexpectedAnswerException, IOException, CqiServerError, CqiClientException {

		// TXM corpus files

		File configfile = new File(tigerDirectory, "tigersearch.logprop");

		TSCorpusManager manager = new TSCorpusManager(tigerCorpusDirectory.getParentFile(), configfile);

		TSCorpus tcorpus = manager.getCorpus(tigerCorpusDirectory.getName());

		InternalCorpusQueryManagerLocal2 tigermanager = tcorpus.manager;

		CorpusQueryProcessor processor = tigermanager.getQueryProcessor();

		AbstractCqiClient CQI = CQPSearchEngine.getCqiClient();

		Index index = processor.getIndex();

		int size = 0;

		for (int nr = 0; nr < index.getNumberOfGraphs(); nr++) {

			size += index.getNumberOfTNodes(nr);

		}

		if (size == 0) {

			Log.warning("No word found in the TIGERSearch corpus: " + tigerCorpusDirectory + ". Aborting.");

			return 0;

		}

		Log.info("Importing " + size + " word annotations...");

		// compute start position of sentences

		int[] starts = new int[index.getNumberOfGraphs()];

		for (int i = 0; i < index.getNumberOfGraphs(); i++) {

			starts[i] = 0;

			if (i > 0) {

				starts[i] += index.getNumberOfTNodes(i - 1) + starts[i - 1];

			}

		}

		File offsetsFile = new File(tigerCorpusExistingDirectory, "offsets.data");

		RandomAccessFile offsetsRAFile = new RandomAccessFile(offsetsFile, "rw");

		FileChannel offsetsFileChannel = offsetsRAFile.getChannel();

		MappedByteBuffer offsetsMapped = offsetsFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size * Integer.BYTES);

		// out.putInt(positions[i])

		File presencesFile = new File(tigerCorpusExistingDirectory, "presences.data");

		RandomAccessFile presencesRAFile = new RandomAccessFile(presencesFile, "rw");

		FileChannel presencesFileChannel = presencesRAFile.getChannel();

		MappedByteBuffer presencesMapped = presencesFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size);

		int numberOfWordsAnnotated = 0;

		// for each sentence

		ConsoleProgressBar cpb = new ConsoleProgressBar(index.getNumberOfGraphs());

		for (int nr = 0; nr < index.getNumberOfGraphs(); nr++) {

			cpb.tick();

			int sent_size = index.getNumberOfTNodes(nr);

			Sentence sent = tcorpus.manager.getSentence(nr);

			String[] ids = new String[sent_size];

			int[] tigerPositions = new int[sent_size];

			for (int t = 0; t < sent_size; t++) {

				T_Node terminal = (T_Node) sent.getTerminalAt(t);

				ids[t] = terminal.getFeature(wordIdAttribute);

				// try fixing ID

				if (ids[t].startsWith("w")) {

					if (!ids[t].startsWith("w_")) {

						ids[t] = "w_" + ids[t].substring(1);

					}

				}

				else {

					ids[t] = "w_" + ids[t];

				}

				tigerPositions[t] = starts[nr] + t;

				// System.out.println("T id="+terminal.getID());

			}

			int[] ids_idx = CQI.str2Id(corpus.getProperty("id").getQualifiedName(), ids);

			Integer[] cqpPositions = new Integer[sent_size];

			Integer[] offsets = new Integer[sent_size];

			for (int t = 0; t < sent_size; t++) {

				if (ids_idx[t] >= 0) {

					int[] positions = CQI.id2Cpos(corpus.getProperty("id").getQualifiedName(), ids_idx[t]);

					if (positions.length > 1) {

						Log.warning("Warning: multiple CQP positions for word_id=" + ids[t]);

					}

					cqpPositions[t] = positions[0]; // take the first position

				}

				else { // word not in the CQP corpus

					Log.warning("Could not find word for id=" + ids[t]);

					cqpPositions[t] = null;

				}

				if (cqpPositions[t] != null) {

					offsets[t] = cqpPositions[t] - tigerPositions[t];

				}

				else {

					offsets[t] = null;

				}

			}

			// System.out.println("ids="+Arrays.toString(ids));

			// System.out.println("cqp indexes="+Arrays.toString(ids_idx));

			// System.out.println("tiger positions="+Arrays.toString(tigerPositions));

			// System.out.println("cqp positions="+Arrays.toString(cqpPositions));

			// System.out.println("offsets="+Arrays.toString(offsets));

			// writing data to offset and presences files

			for (int t = 0; t < sent_size; t++) {

				if (offsets[t] != null) {

					numberOfWordsAnnotated++;

					presencesMapped.put((byte) 1);

					offsetsMapped.putInt(offsets[t]);

				}

				else {

					presencesMapped.put((byte) 0);

					offsetsMapped.putInt(0);

				}

			}

		}

		cpb.done();

		offsetsFileChannel.close();

		offsetsRAFile.close();

		presencesFileChannel.close();

		presencesRAFile.close();

		return numberOfWordsAnnotated;

	}

import java.io.FileNotFoundException;

import org.txm.searchengine.ts.TIGERSearchEngine;

		int numberOfWordsAnnotated = TIGERSearchEngine.writeOffsetDataFiles(corpus, wordIdAttribute, tigerCorpusDirectory, tigerDirectory, tigerCorpusExistingDirectory);

		if (numberOfWordsAnnotated > 0) { // copy the TIGERcorpus to import