Laboratoire ICAR » Plateforme TXM

package org.txm.index.core;

import org.eclipse.ui.plugin.AbstractUIPlugin;

import org.osgi.framework.BundleContext;

/**

 * The activator class controls the plug-in life cycle

 */

public class Activator extends AbstractUIPlugin {

	// The plug-in ID

	public static final String PLUGIN_ID = "org.txm.index.core"; //$NON-NLS-1$

	// The shared instance

	private static Activator plugin;

	/**

	 * The constructor

	 */

	public Activator() {

	}

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.ui.plugin.AbstractUIPlugin#start(org.osgi.framework.BundleContext)

	 */

	public void start(BundleContext context) throws Exception {

		super.start(context);

		plugin = this;

	}

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.ui.plugin.AbstractUIPlugin#stop(org.osgi.framework.BundleContext)

	 */

	public void stop(BundleContext context) throws Exception {

		plugin = null;

		super.stop(context);

	}

	/**

	 * Returns the shared instance

	 *

	 * @return the shared instance

	 */

	public static Activator getDefault() {

		return plugin;

	}

}

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2015-11-19 14:29:08 +0100 (Thu, 19 Nov 2015) $

// $LastChangedRevision: 3056 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.functions.index;

import java.text.Collator;

import java.util.Comparator;

import org.txm.searchengine.cqp.corpus.Corpus;

// TODO: Auto-generated Javadoc

/**

 * The Class LineComparator.

 */

public class LineComparator implements Comparator<Line> {

	/** The collator. */

	protected Collator collator;

	/**

	 * The Enum SortMode.

	 */

	public enum SortMode { // the sort methods

		/** The FREQ. */

 FREQ, // compare only frequency

		/** The UNIT. */

 UNIT, // compare only units

		/** The FREQUNIT. */

 FREQUNIT, // compare frequency then units

		/** The UNITFREQ. */

 UNITFREQ

		// compare units then frequency

	}

	/** The currentsortmode. */

	SortMode currentsortmode = SortMode.FREQUNIT;

	/** The reverse. */

	boolean reverse;

	/**

	 * Instantiates a new line comparator.

	 *

	 * @param mode the mode

	 * @param reverse the reverse

	 */

	public LineComparator(SortMode mode, boolean reverse) {

		setSortByFrequence(mode);

		setReverse(reverse);

	}

	/**

	 * set if the sort must be reverted.

	 *

	 * @param reverse the new reverse

	 */

	private void setReverse(boolean reverse) {

		this.reverse = reverse;

	}

	/* (non-Javadoc)

	 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)

	 */

	@Override

	public int compare(Line arg0, Line arg1) {

		int sort = 0;

		switch (currentsortmode) {

		case FREQUNIT:

			sort = arg0.getFrequency() - arg1.getFrequency();

			if (sort == 0)

				sort = -collator.compare(arg0.toString(), arg1.toString());

			break;

		case UNITFREQ:

			sort = collator.compare(arg0.toString(), arg1.toString());

			if (sort == 0)

				sort = arg0.getFrequency() - arg1.getFrequency();

			break;

		case FREQ:

			sort = arg0.getFrequency() - arg1.getFrequency();

			break;

		case UNIT:

			sort = collator.compare(arg0.toString(), arg1.toString());

			break;

		}

		if (reverse)

			sort = -sort;

		return sort;

	}

	/**

	 * set the sort method.

	 *

	 * @param mode the new sort by frequence

	 */

	public void setSortByFrequence(SortMode mode) {

		currentsortmode = mode;

	}

	/**

	 * Initialize.

	 *

	 * @param corpus the corpus

	 */

	public void initialize(Corpus corpus) {

		collator = Collator.getInstance(corpus.getLocale());

		collator.setStrength(Collator.TERTIARY);

	}

}

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2016-10-21 11:09:43 +0200 (Fri, 21 Oct 2016) $

// $LastChangedRevision: 3323 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.functions.index;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.LinkedHashMap;

import java.util.List;

import org.apache.commons.lang.StringUtils;

import org.txm.HasResults;

import org.txm.Messages;

import org.txm.Toolbox;

import org.txm.functions.Function;

import org.txm.functions.TXMResult;

import org.txm.searchengine.cqp.ICqiClient;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.Lexicon;

import org.txm.searchengine.cqp.corpus.Part;

import org.txm.searchengine.cqp.corpus.Partition;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.QueryResult;

import org.txm.searchengine.cqp.corpus.StructuralUnitProperty;

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.searchengine.cqp.corpus.query.Query;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.stat.engine.r.RWorkspace;

import org.txm.stat.engine.r.RWorkspaceException;

import org.txm.utils.logger.Log;

// TODO: Auto-generated Javadoc

/**

 * compute an index of a corpus or a partition 

 * 

 * may contains LexicalTable built from its values

 * @author mdecorde.

 */

public class Index extends Function implements TXMResult, HasResults  {

	/** The partition. */

	protected Partition partition;

	/** The corpus. */

	protected Corpus corpus;

	/** The query. */

	protected Query query;

	/** The props. */

	protected List<Property> props;

	/** The partnames. */

	protected List<String> partnames = new ArrayList<String>();

	/** The currentpartid. */

	protected int currentpartid = 0;

	/** The altered. */

	boolean altered = false;

	/** The n lines. */

	protected int nLines = 0;

	/** The Fmax. */

	protected int Fmax = 0;

	/** The Fmin. */

	protected int Fmin = 9999999;

	/** The Filter vmax. */

	protected int FilterVmax = 999999999;

	/** The Filter fmin. */

	protected int FilterFmin = 1;

	/** The Filter fmax. */

	protected int FilterFmax = 999999999;

	/** The prop separator. */

	protected String propSeparator = "_"; // the separator of properties values //$NON-NLS-1$

	/** The counts. */

	protected LinkedHashMap<String, ArrayList<Integer>> counts = new LinkedHashMap<String, ArrayList<Integer>>();

	/** The lines. */

	protected List<Line> lines = new ArrayList<Line>();

	/** proxy to get the string values of cqp tokens. */

	//private Map<Property, CqpDataProxy> cache;

	/** The symbol. */

	private String symbol;

	/**

	 * compute the index with a lexicon.

	 *

	 * @param corpus the corpus

	 * @param property the property

	 */

	public Index(Corpus corpus, Property property) {

		init(corpus, property);

		altered = false;

	}

	/** The lexicon. */

	protected Lexicon lexicon;

	/** The writer. */

	private BufferedWriter writer;

	/**

	 * Inits the Index using a lexicon.

	 *

	 * @param corpus the corpus

	 * @param property the property

	 */

	private void init(Corpus corpus, Property property) {

		try {

			partnames.add(corpus.getName());

			this.query = new Query("[]"); //$NON-NLS-1$

			this.corpus = corpus;

			this.props = new ArrayList<Property>();

			this.props.add(property);

			lexicon = corpus.getLexicon(property);

			this.nLines = lexicon.nbrOfToken();

			String[] forms = lexicon.getForms();

			int[] freqs = lexicon.getFreq();

			int[] ids = lexicon.getIds();

			this.lines = new ArrayList<Line>();

			int nbOfToken = lexicon.nbrOfToken();

			if (nbOfToken > 0) // no lines

				for (int i = 0; i < forms.length && i < freqs.length; i++) {

					Line l = new Line();

					List<String> values = new ArrayList<String>(1);

					values.add(forms[i]);

					l.put(property, values);

					int[] c = new int[1];

					c[0] = freqs[i];

					l.setCounts(c, freqs[i] / lexicon.nbrOfToken());

					l.putIds(property, new int[] { ids[i] });

					lines.add(l);

					if (Fmin > freqs[i])

						Fmin = freqs[i];

					if (Fmax < freqs[i])

						Fmax = freqs[i];

					// getAllLines();

				}

		} catch (Exception e) {

			// TODO Auto-generated catch block

			org.txm.utils.logger.Log.printStackTrace(e);

		}

	}

	/**

	 * compute a index, given a corpus, a query and analysis properties.

	 *

	 * @param corpus the corpus

	 * @param query the query

	 * @param props the props

	 * @throws CqiClientException 

	 * @throws CqiServerError 

	 * @throws IOException 

	 */

	public Index(Corpus corpus, Query query, List<Property> props) throws CqiClientException, IOException, CqiServerError {

		//		if (props.size() == 1 && query.getQueryString().equals("[]")) //$NON-NLS-1$

		//		{

		//			init(corpus, props.get(0));

		//			return;

		//		}

		partnames.add(corpus.getName());

		this.corpus = corpus; // the corpus which we compute on

		this.query = query; // the query of the selection

		if (query.getQueryString() == "") //$NON-NLS-1$

			query = new Query("[]"); //$NON-NLS-1$

		this.props = new ArrayList<Property>(props);

		if (!scanCorpus(corpus)) {

			lines = new ArrayList<Line>();

			return;

		}

		for (Line line : lines)// for each Line set its count

		{

			int[] c = new int[partnames.size()];

			for (int i = 0; i < partnames.size(); i++)

				c[i] = counts.get(line.getSignature()).get(i);

			line.setCounts(c, -1);

		}

		getAllLines();

		setLineCounts();

		altered = false;

	}

	/**

	 * compute a index per Part.

	 *

	 * @param partition the partition

	 * @param query the query

	 * @param props the props

	 * @throws CqiClientException 

	 * @throws CqiServerError 

	 * @throws IOException 

	 */

	public Index(Partition partition, Query query, List<Property> props) throws CqiClientException, IOException, CqiServerError {

		this.partition = partition;

		this.corpus = partition.getCorpus(); // the corpus which we compute on

		this.query = query; // the query of the selection

		if (query.getQueryString() == "") //$NON-NLS-1$

			query = new Query("[]"); //$NON-NLS-1$

		this.props = new ArrayList<Property>(props);

		for (Part part : partition.getParts()) {

			scanCorpus(part);

			currentpartid++;

			partnames.add(part.getName());

		}

		setLineCounts();

		getAllLines();

		// this.sortLines(LineComparator.SortMode.FREQUNIT);

		altered = false;

	}

	/**

	 * tell if the index has been computed with a partition or not.

	 *

	 * @return true, if is computed with partition

	 */

	public boolean isComputedWithPartition() {

		return partition != null;

	}

	/**

	 * update the lines counts.

	 */

	protected void setLineCounts() {

		for (Line line : lines)// for each Line set its count

		{

			int[] c = new int[partnames.size()];

			for (int i = 0; i < partnames.size(); i++)

				if (counts.get(line.getSignature()).size() <= i)

					c[i] = 0;

				else

					c[i] = counts.get(line.getSignature()).get(i);

			line.setCounts(c, -1);

		}

	}

	/**

	 * count tokens.

	 *

	 * @param corp the corp

	 * @return true, if successful

	 * @throws CqiClientException 

	 * @throws CqiServerError 

	 * @throws IOException 

	 */

	protected boolean scanCorpus(Corpus corp) throws CqiClientException, IOException, CqiServerError {

		// get the cqp result of the query

		long time = System.currentTimeMillis();

		QueryResult result = corp.query(query, "index", true); //$NON-NLS-1$

		boolean isTargetUsed = Toolbox.getCqiClient().subCorpusHasField(result.getQualifiedCqpId(), ICqiClient.CQI_CONST_FIELD_TARGET);

		int nbresults = result.getNMatch();

		this.nLines += nbresults; // get number of tokens

		// System.out.println("nLines : "+nLines);

		List<Match> matches = null;

		if (nbresults > 0)

			matches = result.getMatches(0, nbresults - 1); // get the

		// indexes

		// sequences of

		// result's

		// tokens

		else

			matches = new ArrayList<Match>();

		// List<Integer> beginingOfKeywordsPositions = new

		// ArrayList<Integer>();

		// List<Integer> lengthOfKeywords = new ArrayList<Integer>();

		// Map<Property, List<List<String>>> keywordsViewPropValues = new

		// HashMap<Property, List<List<String>>>();

		// count matches

		// time = System.currentTimeMillis();

		List<Integer> allpositions = new ArrayList<Integer>();

		for (int j = 0; j < nbresults; j++) {

			Match match = matches.get(j);

			// beginingOfKeywordsPositions.add(match.getStart()); // get the

			// first index

			// lengthOfKeywords.add(match.size());// get the last index

			if (isTargetUsed) {

				allpositions.add(match.getTarget());

			} else {

				for (int i = match.getStart(); i <= match.getEnd(); i++)

					allpositions.add(i);

			}

		}

		// System.out.println("get string data");

		// time = System.currentTimeMillis();

		// for (Property property : props) {// for each property get the

		// string values of the tokens

		// keywordsViewPropValues.put(property,

		// cache.get(property).getData(beginingOfKeywordsPositions,

		// lengthOfKeywords));

		// }

		// System.out.println("took "+(System.currentTimeMillis()-time));

		// System.out.println("get count data");

		int[] allpositionsarray = new int[allpositions.size()];

		int pcount = 0;

		for (int p : allpositions)

			allpositionsarray[pcount++] = p;

		// time = System.currentTimeMillis();

		HashMap<Property, int[]> propsId = new HashMap<Property, int[]>();

		for (Property property : props) {

			try {

				if (property instanceof StructuralUnitProperty) {

					int[] structs = CorpusManager.getCorpusManager()

							.getCqiClient().cpos2Struc(

									property.getQualifiedName(),

									allpositionsarray);

					propsId.put(property, structs);

				} else {

					int[] indices = CorpusManager.getCorpusManager()

							.getCqiClient().cpos2Id(

									property.getQualifiedName(),

									allpositionsarray);

					propsId.put(property, indices);

				}

			} catch (Exception e) {

				org.txm.utils.logger.Log.printStackTrace(e);

				result.drop();

				return false;

			}

		}

		// System.out.println("Time recup indices "+(System.currentTimeMillis()-time));

		int currentIndex = 0;

		// time = System.currentTimeMillis();

		for (int i = 0; i < nbresults; i++) {

			Line line = new Line();

			Match match = matches.get(i);

			int size = match.size();

			if (isTargetUsed) size = 1;

			for (int p = 0; p < props.size(); p++) {

				Property property = props.get(p);

				int[] allprosids = propsId.get(property);

				int[] ids = new int[size];

				System.arraycopy(allprosids, currentIndex, ids, 0, size);

				line.putIds(property, ids);

			}

			currentIndex += size;

			String signature = line.getSignature();

			if (counts.containsKey(signature)) // if the counts contains the

				// signature, increment its

				// corresponding value

			{

				while (counts.get(signature).size() <= currentpartid)

					counts.get(signature).add(0);

				int c = counts.get(signature).get(currentpartid) + 1;

				counts.get(signature).set(currentpartid, c);

			} else // else initialize count of the signature to 1

			{

				// System.out.println("add new sign "+signature+" of line "+line.toString());

				ArrayList<Integer> tmp = new ArrayList<Integer>();

				for (int j = 0; j < currentpartid + 1; j++)

					tmp.add(0);

				counts.put(signature, tmp);

				counts.get(signature).set(currentpartid, 1);

				lines.add(line);

			}

		}

		result.drop();

		// System.out.println("Time count lines "+(System.currentTimeMillis()-time));

		// System.out.println("took "+(System.currentTimeMillis()-time));

		return true;

	}

	/**

	 * return the lines from le "start"th one to the "end"th one.

	 *

	 * @param start the start

	 * @param end the end

	 * @return the lines

	 */

	public List<Line> getLines(int start, int end) {

		long time = System.currentTimeMillis();

		List<Line> selectedLines = new ArrayList<Line>();

		if (lines.size() > 0) {

			start = Math.max(0, start);

			end = Math.min(end, lines.size());

			selectedLines = lines.subList(start, end);

			int p = 0;

			for (Property property : props) {// for each property get the string values

				// values of the tokens

				int len = 0;

				for (Line l : selectedLines) {

					len += l.UnitsIds.get(p).length;

				}

				int[] indices = new int[len]; // build the array of indices

				len = 0;

				for (Line l : selectedLines) {

					int[] ids = l.UnitsIds.get(p);

					System.arraycopy(ids, 0, indices, len, ids.length);

					len += ids.length;

				}

				String[] strs = null;

				try {

					if (property instanceof StructuralUnitProperty) {

						strs = CorpusManager.getCorpusManager().getCqiClient()

								.struc2Str(property.getQualifiedName(), indices);

					} else {

						strs = CorpusManager.getCorpusManager().getCqiClient()

								.id2Str(property.getQualifiedName(), indices);

					}

				} catch (Exception e) {

					org.txm.utils.logger.Log.printStackTrace(e);

					return null;

				}

				len = 0;

				for (Line l : selectedLines) {

					int[] ids = l.UnitsIds.get(p);

					String[] lstr = new String[ids.length];

					System.arraycopy(strs, len, lstr, 0, ids.length);

					if (l.UnitsProperty.size() == props.size()) continue; // the line is already initialized

						l.put(property, Arrays.asList(lstr));

					len += ids.length;

				}

				p++;

			}

		}

		// System.out.println("Time get lines "+(System.currentTimeMillis()-time));

		return selectedLines;

	}

	/**

	 * return all the lines of the index.

	 *

	 * @return the all lines

	 */

	public List<Line> getAllLines() {

		return getLines(0, lines.size());

	}

	/**

	 * return the cqp expression build with the lines between "from" and "to"

	 * TODO: finish implementation.

	 *

	 * @param from the from

	 * @param to the to

	 * @return the query

	 */

	public Query buildQuery(int from, int to) {

		return new Query(""); //$NON-NLS-1$

	}

	/**

	 * Gets the query.

	 *

	 * @return the query used

	 */

	public Query getQuery() {

		return query;

	}

	/**

	 * Gets the v.

	 *

	 * @return the number of entrie in the index

	 */

	public int getV() {

		return lines.size();

	}

	/**

	 * Gets the t.

	 *

	 * @return the number of tokens returned by the selection

	 */

	public int getT() {

		return nLines;

	}

	/**

	 * Sort lines.

	 *

	 * @param mode the mode

	 * @param reverse the reverse

	 */

	public void sortLines(LineComparator.SortMode mode, boolean reverse) {

		this.altered = true;

		LineComparator lc = new LineComparator(mode, reverse);

		lc.initialize(corpus);

		Collections.sort(lines, lc);

	}

	/**

	 * remove line which frenquency is not in the inverval [Fmin,Fmax].

	 *

	 * @param Fmin the fmin

	 * @param Fmax the fmax

	 */

	public void filterLines(int Fmin, int Fmax) {

		int before = lines.size();

		for (int i = 0; i < lines.size(); i++)// for each line

		{

			Line line = lines.get(i);

			int f = line.getFrequency();

			if (f < Fmin || f > Fmax) // if its frequency is not in the

				// interval, remove it

			{

				nLines -= line.getFrequency();

				lines.remove(i);

				i--;

			}

		}

		this.FilterFmin = Fmin;

		this.FilterFmax = Fmax;

		this.updateFminFmax();

		altered = true;

	}

	/**

	 * keep the vmax lines more frequents.

	 *

	 * @param vmax the vmax

	 */

	public void cut(int vmax) {

		this.acquireSemaphore();

		// assume the lines are sorted

		int before = lines.size();

		this.lines = this.lines.subList(0, Math.min(lines.size(), vmax));

		this.FilterVmax = vmax;

		updateFminFmax();

		altered = true;

		this.releaseSemaphore();

	}

	/**

	 * checks all lines to update Fmin and Fmax.

	 */

	private void updateFminFmax() {

		Fmin = 9999;

		Fmax = 1;

		for (int i = 0; i < lines.size(); i++) {

			Line line = lines.get(i);

			int f = line.getFrequency();

			if (f < Fmin)

				Fmin = f;

			if (f > Fmax)

				Fmax = f;

		}

	}

	/**

	 * Write all the lines on a writer.

	 *

	 * @param outfile the outfile

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @return true, if successful

	 */

	public boolean toTxt(File outfile, String encoding, String colseparator, String txtseparator) {

		try {

			toTxt(outfile, 0, lines.size(), encoding, colseparator, txtseparator);

		} catch (Exception e) {

			System.err.println(Messages.Index_7 + Log.toString(e));

			return false;

		}

		return true;

	}

	/**

	 * Write the lines between from and to on a writer.

	 *

	 * @param outfile the outfile

	 * @param from The first line to be written

	 * @param to The last line to be writen

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @throws CqiClientException the cqi client exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 */

	public void toTxt(File outfile, int from, int to, String encoding, String colseparator, String txtseparator)

			throws CqiClientException, IOException {

		// NK: writer declared as class attribute to perform a clean if the operation is interrupted

		this.writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile),

				encoding)); 

		// if ("UTF-8".equals(encoding)) writer.write('\ufeff'); // UTF-8 BOM

		String header = ""; //$NON-NLS-1$

		for (Property p : props)

			header += (p + propSeparator);

		header = txtseparator+ header.substring(0, header.length() - 1) +txtseparator;

		header += colseparator+ txtseparator+ "F" + txtseparator; //$NON-NLS-1$

		if (partnames.size() > 1)

			for (int j = 0; j < partnames.size(); j++)

				header += colseparator + txtseparator+ partnames.get(j).replace(txtseparator, txtseparator+txtseparator)+txtseparator; 

		header += "\n"; //$NON-NLS-1$

		writer.write(header);

		// for(Line ligne: lines)

		for (int i = from; i < to; i++) {

			Line ligne = lines.get(i);

			writer.write(txtseparator+ ligne.toString().replace(txtseparator, txtseparator+txtseparator)+ txtseparator + colseparator + ligne.getFrequency()); 

			if (partnames.size() > 1)

				for (int j = 0; j < partnames.size(); j++)

					writer.write(colseparator + ligne.getFrequency(j)); 

			writer.write("\n"); //$NON-NLS-1$

		}

		writer.flush();

		writer.close();

	}

	/**

	 * Write the lines between from and to on a writer.

	 *

	 * @param outfile the outfile

	 * @param from The first line to be written

	 * @param to The last line to be writen

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @throws CqiClientException the cqi client exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 */

	public void toTSVDictionnary(File outfile, String colseparator, String encoding) throws CqiClientException, IOException {

		//String colseparator = "\t";

		this.writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), encoding)); 

		String header = ""; //$NON-NLS-1$

		for (Property p : props) {

			header += (p + colseparator);

		}

		//header = header.substring(0, header.length() - 1);

		header += "F"; //$NON-NLS-1$

		if (partnames.size() > 1) {

			for (int j = 0; j < partnames.size(); j++) {

				header += colseparator + partnames.get(j);

			}

		}

		header += "\n"; //$NON-NLS-1$

		writer.write(header);

		for (Line ligne : lines) {

			for (int i = 0 ; i < props.size() ; i++) {

				writer.write(StringUtils.join(ligne.getUnitsProperties().get(i), " ") + colseparator);

			}

			writer.write(Integer.toString(ligne.getFrequency()));

			if (partnames.size() > 1) {

				for (int j = 0; j < partnames.size(); j++) {

					writer.write(colseparator + ligne.getFrequency(j));

				}

			}

			writer.write("\n"); //$NON-NLS-1$

		}

		writer.flush();

		writer.close();

	}

	/**

	 * dump the index in the console.

	 *

	 * @throws CqiClientException the cqi client exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 */

	public void toConsole() throws CqiClientException, IOException {

		System.out.println(Messages.Index_1 + (lines.size()));

		toConsole(0, lines.size());

	}

	/**

	 * dump a part of the index.

	 *

	 * @param from the from

	 * @param to the to

	 * @throws CqiClientException the cqi client exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 */

	public void toConsole(int from, int to) throws CqiClientException,

	IOException {

		String header = ""; //$NON-NLS-1$

		for (Property p : props)

			header += (p + propSeparator);

		header = header.substring(0, header.length() - 1);

		header += "\tF"; //$NON-NLS-1$

		if (partnames.size() > 1)

			for (int j = 0; j < partnames.size(); j++)

				header += "\t" + partnames.get(j); //$NON-NLS-1$

		System.out.println(header);

		for (int i = from; i < to; i++) {

			Line ligne = lines.get(i);

			System.out.print(ligne + "\t" + ligne.getFrequency()); //$NON-NLS-1$ 

			if (partnames.size() > 1)

				for (int j = 0; j < partnames.size(); j++)

					System.out.print("\t" + ligne.getFrequency(j)); //$NON-NLS-1$

			System.out.print("\n"); //$NON-NLS-1$

		}

		System.out.flush();

	}

	/**

	 * Gets the name.

	 *

	 * @return the name of this index

	 */

	public String getName() {

		String name = query.getQueryString() + ":"; //$NON-NLS-1$

		for (Property s : props)

			name += s + propSeparator;

		if (props.size() > 0)

			name = name.substring(0, name.length() - 1);

		return name;

	}

	/**

	 * Gets the corpus.

	 *

	 * @return the corpus

	 */

	public Corpus getCorpus() {

		return this.corpus;

	}

	/**

	 * Gets the properties.

	 *

	 * @return the properties

	 */

	public List<Property> getProperties() {

		return this.props;

	}

	/**

	 * Gets the fmax.

	 *

	 * @return the fmax

	 */

	public int getFmax() {

		return Fmax;

	}

	/**

	 * Gets the fmin.

	 *

	 * @return the fmin

	 */

	public int getFmin() {

		return Fmin;

	}

	/**

	 * Gets the filter fmin.

	 *

	 * @return the filter fmin

	 */

	public int getFilterFmin() {

		return FilterFmin;

	}

	/**

	 * Gets the filter fmax.

	 *

	 * @return the filter fmax

	 */

	public int getFilterFmax() {

		return FilterFmax;

	}

	/**

	 * Gets the filter vmax.

	 *

	 * @return the filter vmax

	 */

	public int getFilterVmax() {

		return FilterVmax;

	}

	/**

	 * Equals.

	 *

	 * @param voc the voc

	 * @return true, if successful

	 */

	public boolean equals(Index voc) {

		return this.query.equals(voc.getQuery())

				&& this.props.equals(voc.getProperties())

				&& this.Fmin == voc.getFmin() && this.Fmax == voc.getFmax();

	}

	/**

	 * Gets the partnames.

	 *

	 * @return the partnames

	 */

	public List<String> getPartnames() {

		return partnames;

	}

	/**

	 * Gets the partition.

	 *

	 * @return the partition

	 */

	public Partition getPartition() {

		return partition;

	}

	/**

	 * Checks if is altered.

	 *

	 * @return true, if is altered

	 */

	public boolean isAltered() {

		// TODO Auto-generated method stub

		return altered;

	}

	/** The novoc. */

	protected static int novoc = 1;

	/** The prefix r. */

	protected static String prefixR = "Index"; //$NON-NLS-1$

	/**

	 * As r matrix.

	 *

	 * @return the string

	 * @throws RWorkspaceException the r workspace exception

	 */

	public String asRMatrix() throws RWorkspaceException {

		symbol = prefixR+novoc;

		ArrayList<String> colnames = new ArrayList<String>();

		colnames.add("F"); //$NON-NLS-1$

		if (partnames.size() > 1)

			for (int j = 0; j < partnames.size(); j++)

				colnames.add(partnames.get(j)); 

		//System.out.println("cols: "+colnames);

		String[] keywords = new String[this.lines.size()];

		int[] freq = new int[this.lines.size()];

		ArrayList<int[]> partfreqs = new ArrayList<int[]>(partnames.size());

		if (partnames.size() > 1)

			for (int j = 0; j < partnames.size(); j++)

				partfreqs.add(new int[this.lines.size()]);

		for (int i = 0 ; i < lines.size() ; i++) 

		{

			Line ligne = lines.get(i);

			freq[i] = ligne.getFrequency(); 

			keywords[i] = ligne.toString();

			if (partnames.size() > 1)

				for (int j = 0; j < partnames.size(); j++)

				{

					partfreqs.get(j)[i] = ligne.getFrequency(j);

				}

		}

		RWorkspace rw = RWorkspace.getRWorkspaceInstance();

		if (partnames.size() > 1)

			for (int j = 0; j < partnames.size(); j++)

				rw.addVectorToWorkspace("vocpartfreqs"+j, partfreqs.get(j)); //$NON-NLS-1$

		rw.addVectorToWorkspace("vocfreq", freq); //$NON-NLS-1$

		rw.addVectorToWorkspace("vockeywords", keywords); //$NON-NLS-1$

		rw.addVectorToWorkspace("voccolnames", colnames.toArray(new String[colnames.size()])); //$NON-NLS-1$

		int ncol = 1;

		if (partnames.size() > 1)

			ncol += partnames.size();

		int nrow = lines.size();

		String partscmd = ""; //$NON-NLS-1$

		if (partnames.size() > 1)

			for (int j = 0; j < partnames.size(); j++)

				partscmd +=", vocpartfreqs"+j; //$NON-NLS-1$

		rw.eval(symbol+ "<- matrix(data = c(vocfreq"+partscmd+"), nrow = "+nrow+", ncol = "+ncol+")"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$

		rw.eval("colnames("+symbol+" ) <- voccolnames"); //$NON-NLS-1$ //$NON-NLS-2$

		rw.eval("rownames("+symbol+" ) <- vockeywords"); //$NON-NLS-1$ //$NON-NLS-2$

		rw.eval(symbol+ "<- list(data="+symbol+")"); //$NON-NLS-1$ //$NON-NLS-2$

		novoc++;

		return symbol;

	}

	/**

	 * Gets the symbol.

	 *

	 * @return the symbol

	 */

	public String getSymbol()

	{

		return symbol;

	}

	/**

	 * If the index has been build with corpus + property, the method returns the lexicon used.

	 *

	 * @return the lexicon

	 */

	public Object getLexicon() {

		return lexicon;

	}

	/** The properties to process. */

	HashSet<String> propertiesToProcess;

//	/**

//	 * Render.

//	 *

//	 * @param line the line

//	 * @return the string

//	 */

//	public String render(Line line)

//	{

//		if(propertiesToProcess == null)

//			propertiesToProcess = TxmRenderer.getPropertyToRender(this, "jsesh"); //$NON-NLS-1$

//		//System.out.println("props to process: "+propertiesToProcess);

//

//		ArrayList<Boolean> torender = new ArrayList<Boolean>();

//		for(Property prop : line.properties)

//			if(propertiesToProcess.contains(prop.getName()))

//				torender.add(true);

//			else

//				torender.add(false);

//		//System.out.println("to render: "+torender);

//

//		String rez = ""; //$NON-NLS-1$

//		int len = 0; // get number of token per line

//		for (int i = 0; i < line.properties.size(); ) {

//			len = line.UnitsProperty.get(i).size();

//			break;

//		}

//		// TODO: bug concat props

//		for (int i = 0; i < len; i++)// for each token

//		{

//			for (int j = 0; j < line.properties.size(); j++) {

//				if(torender.get(j))

//				{

//					//System.out.println("render mdc : "+line.UnitsProperty.get(j).get(i));

//					rez += TxmRenderer.jseshrenderer.render(line.UnitsProperty.get(j).get(i));

//				}

//				else

//				{

//					//System.out.println("tostring : "+line.UnitsProperty.get(j).get(i));

//					rez += line.UnitsProperty.get(j).get(i);

//				}

//

//				if (j < line.properties.size() - 1) {

//					rez += rez+"_"; //$NON-NLS-1$

//				}

//			}

//			if (i < len - 1)

//				rez += " "; //$NON-NLS-1$

//		}

//		return rez;	

//

//	}

	public void setIsAltered(boolean b) {

		this.altered = b;

	}

	public HasResults getParent() {

		if (partition != null) return partition;

		return corpus;

	}

	@Override

	public void clean() {

		try {

			this.writer.flush();

			this.writer.close();	

		} catch (IOException e) {

			// TODO Auto-generated catch block

			org.txm.utils.logger.Log.printStackTrace(e);

		}

	}

	@Override

	public boolean delete() {

		return getParent().removeResult(this);

	}

	ArrayList<Object> results = new ArrayList<Object>();

	@Override

	public List<Object> getResults() {

		return results;

	}

	@Override

	public boolean removeResult(Object result) {

		return results.remove(result);

	}

	@Override

	public List<HasResults> getSubHasResults() {

		return new ArrayList<HasResults>();

	}

	@Override

	public void storeResult(Object result) {

		results.add(result);

	}

}

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2015-11-19 14:29:08 +0100 (Thu, 19 Nov 2015) $

// $LastChangedRevision: 3056 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.functions.index;

import java.util.ArrayList;

import java.util.List;

import org.txm.functions.index.Line;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.query.Occurrence;

// TODO: Auto-generated Javadoc

/**

 * contains the signature of the line all properties value and counts @ author

 * mdecorde.

 */

public class Line {

	/** The Units property. */

	public List<List<String>> UnitsProperty; // the values, for each property

												// there is a list of property

												// values

	/** The Units ids. */

												public List<int[]> UnitsIds; // the values, for each property there is a

									// list of property values

	/** The properties. */

									public List<Property> properties = new ArrayList<Property>(); // the

																	// properties

																	// name

	// UnitsProperty and properties have the same order

	/** The signature. */

																	private String signature;

	/** The counts. */

	private int[] counts;

	/** The total. */

	private int total;

	/** The frelatif. */

	private float frelatif = 0;

	/** The str. */

	private String str;

	/**

	 * Instantiates a new line.

	 */

	public Line() {

		this.UnitsProperty = new ArrayList<List<String>>();

		UnitsIds = new ArrayList<int[]>();

	}

	/**

	 * add line's values for the given property.

	 *

	 * @param p the p

	 * @param values the values

	 */

	public void put(Property p, List<String> values) {

		UnitsProperty.add(values);

		// properties.add(p);

	}

	/**

	 * add line's values for the given property.

	 *

	 * @param p the p

	 * @param ids the ids

	 */

	public void putIds(Property p, int[] ids) {

		UnitsIds.add(ids);

		properties.add(p);

	}

	/**

	 * return the signature of a Line, an id to compare them quickly if L1 ==

	 * L2, L1.signature = L2.signature else L1.signature != L2.signature

	 *

	 * @return the signature

	 */

	public String getSignature() {

		if (signature == null) {

			//System.out.println("getSignature() props size "+properties.size());

			signature = ""; //$NON-NLS-1$

			for (int i = 0; i < properties.size(); i++)

				for (int s : UnitsIds.get(i))

					signature += "[" + s + "]"; //$NON-NLS-1$ //$NON-NLS-2$

		}

		return signature;

	}

	/**

	 * Sets the counts.

	 *

	 * @param c the c

	 * @param f the f

	 */

	public void setCounts(int[] c, float f) {

		counts = c;

		total = 0;

		for (int i = 0; i < c.length; i++)

			total += c[i];

		frelatif = f;

	}

	/**

	 * Gets the frequency.

	 *

	 * @return the frequency

	 */

	public int getFrequency() {

		return total;

	}

	/**

	 * Gets the frequency.

	 *

	 * @param i the i

	 * @return the frequency

	 */

	public int getFrequency(int i) {

		if (i >= counts.length || i < 0)

			return -1;

		return counts[i];

	}

	/**

	 * Gets the properties.

	 *

	 * @return the properties

	 */

	public List<Property> getProperties() {

		return this.properties;

	}

	/**

	 * Gets the units properties.

	 *

	 * @return the units properties

	 */

	public List<List<String>> getUnitsProperties() {

		return this.UnitsProperty;

	}

	/**

	 * Gets the relative frenquency.

	 *

	 * @return the relative frenquency

	 */

	public float getRelativeFrenquency() {

		return frelatif ;

	}

	/* (non-Javadoc)

	 * @see java.lang.Object#toString()

	 */

	@Override

	public String toString() {

		if (str == null)

			str = this.toString("_"); //$NON-NLS-1$

		return str;

	}

	/**

	 * To string.

	 *

	 * @param separator the separator

	 * @return the string

	 */

	public String toString(String separator) {

		String rez = ""; //$NON-NLS-1$

		int len = 0;

		for (int i = 0; i < properties.size(); ) {

			len = UnitsProperty.get(i).size();

			break;

		}

		for (int i = 0; i < len; i++)// for each token

		{