Laboratoire ICAR » Plateforme TXM

Export-Package: org.txm.cooccurrence.core.functions,

 org.txm.cooccurrence.core.functions.comparators,

 org.txm.cooccurrence.core.preferences

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2017-01-24 18:11:42 +0100 (Tue, 24 Jan 2017) $

// $LastChangedRevision: 3400 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.cooccurrence.core.functions;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.io.Writer;

import java.util.ArrayList;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.Set;

import org.rosuda.REngine.REXPMismatchException;

import org.txm.HasResults;

import org.txm.Messages;

import org.txm.cooccurrence.core.functions.comparators.CLineComparator;

import org.txm.functions.Function;

import org.txm.functions.TXMResult;

import org.txm.functions.concordances.Concordance;

import org.txm.functions.concordances.Line;

import org.txm.index.core.functions.Index;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.clientExceptions.UnexpectedAnswerException;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.QueryResult;

import org.txm.searchengine.cqp.corpus.StructuralUnit;

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.searchengine.cqp.corpus.query.Query;

import org.txm.searchengine.cqp.serverException.CqiServerError;

import org.txm.specificities.core.functions.Specificites;

import org.txm.specificities.core.functions.SpecificitesResult;

import org.txm.stat.StatException;

import org.txm.stat.data.LexicalTable;

import org.txm.stat.engine.r.RWorkspace;

import org.txm.stat.engine.r.RWorkspaceException;

import org.txm.stat.engine.r.data.LexicalTableImpl;

import org.txm.utils.logger.Log;

// TODO: Auto-generated Javadoc

/**

 * Compute a coocurrence from a concordance.

 *

 * @author mdecorde

 * 

 */

public class Cooccurrence extends Function implements TXMResult  {

	/** The voc. */

	Index voc;

	/** The conc. */

	Concordance conc;

	/**

	 * The reference corpus to use = the R symbol that point to a matrix WordxFreqs

	 */

	String referenceCorpus;

	/** The conclines. */

	List<org.txm.functions.concordances.Line> conclines;

	/** The corpus. */

	Corpus corpus;

	/** The query. */

	Query query;

	String query_occ = "[]"; //$NON-NLS-1$

	/** The properties. */

	List<Property> properties;

	/** The limit. */

	StructuralUnit limit;

	/** The maxleft. */

	int maxleft;

	/** The maxright. */

	int maxright;

	/** The minleft. */

	int minleft = 1;

	/** The minright. */

	int minright = 1;

	/** The lines. */

	List<CLine> lines = new ArrayList<CLine>();

	/** The occproperties. */

	HashMap<String, List<String>> occproperties;

	/** The count. */

	HashMap<String, Integer> count;

	/** The dist. */

	HashMap<String, Float> dist;

	/** The freq. */

	HashMap<String, Integer> freq;

	/** The scores. */

	HashMap<String, Double> scores;

	/** The FA. */

	int FA = -1;

	/** The P. */

	int P = -1;

	/** The number of keyword. */

	int numberOfKeyword = 0;

	/** The seuil_freq. */

	int seuil_freq = -1;

	/** The seuil_count. */

	int seuil_count = -1;

	/** The seuil_score. */

	double seuil_score = -1;

	/** The contextquery. */

	private Query contextquery;

	/** The minf. */

	private int minf;

	/** The minscore. */

	private double minscore;

	/** The mincof. */

	private int mincof;

	/** The anticontextquery. */

	private Query anticontextquery;

	/** The include xpivot. */

	private boolean includeXpivot;

	//System.out.println("Matches: focus: "+m1.size()+" full: "+m2.size()+" anti: "+m3.size());

	//System.out.println("T matches : "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

	/** The distances. */

	HashMap<String, Double> distances = new HashMap<String, Double>();

	// contains the sum of distances

	/** The distancescounts. */

	HashMap<String, Integer> distancescounts = new HashMap<String, Integer>();

	// contains the sum of distances

	/** The counts. */

	HashMap<String, Integer> counts = new HashMap<String, Integer>();

	/** The indexfreqs. */

	HashMap<String, Integer> indexfreqs = new HashMap<String, Integer>();

	// contains the number of encounter

	/** The counted. */

	HashMap<Integer, Integer> counted = new HashMap<Integer, Integer>();

	/** The m1. */

	private List<Match> m1;

	//contains the list of positions already counteds

	/** The m2. */

	private List<Match> m2;

	/** The m3. */

	private List<Match> m3;

	/** The allsignaturesstr. */

	private HashMap<Integer, String> allsignaturesstr;

	/** The lt. */

	private LexicalTable lt;

	/** The keys to string. */

	private HashMap<String, String> keysToString;

	/** The symbol. */

	private String symbol;

	/** The writer. */

	private BufferedWriter writer;

	private boolean buildLexicalTableWithCooccurrents;

	/**

	 * specif y exactly on which concordance lines compute the cooc.

	 *

	 * @param conclines the conclines

	 */

	public Cooccurrence(List<Line> conclines) {

		if (conclines.size() > 0) {

			initConcInfos(conclines.get(0).getConcordance(), conclines);

		}

	}

	/**

	 * the cooc will be computed on all lines.

	 *

	 * @param conc the conc

	 */

	public Cooccurrence(Concordance conc) {

		initConcInfos(conc);

	}

	//

	//	/**

	//	 * build the conc and compute on all lines.

	//	 *

	//	 * @param corpus the corpus

	//	 * @param query the query

	//	 * @param properties the properties

	//	 * @param limit the limit

	//	 * @param leftlimit the leftlimit

	//	 * @param rightlimit the rightlimit

	//	 */

	//	public Cooccurrence(Corpus corpus, Query query, List<Property> properties,

	//			StructuralUnit limit, int leftlimit, int rightlimit) {

	//		try {

	//			Concordance conc = new Concordance(corpus, query,

	//					properties.get(0), properties,

	//					new ReferencePattern(), new ReferencePattern(),

	//					leftlimit, rightlimit);

	//			// corpus.storeResult(conc);

	//			initConcInfos(conc);

	//		} catch (CqiClientException e) {

	//			// TODO Auto-generated catch block

	//			org.txm.utils.logger.Log.printStackTrace(e);

	//		}

	//

	//	}

	/**

	 * Instantiates a new cooccurrence.

	 *

	 * @param corpus the corpus

	 * @param query the query

	 * @param properties the properties

	 * @param limit the limit

	 * @param maxleft the maxleft

	 * @param minleft the minleft

	 * @param minright the minright

	 * @param maxright the maxright

	 * @param minf the minf

	 * @param mincof the mincof

	 * @param minscore the minscore

	 * @param includeXpivot the include xpivot

	 * @throws Exception the exception

	 */

	public Cooccurrence(Corpus corpus, Query query, List<Property> properties,

			StructuralUnit limit, int maxleft, int minleft, int minright,

			int maxright, int minf, int mincof, double minscore, boolean includeXpivot, boolean buildLexicalTableWithCooccurrents)

					throws Exception {

		long time = System.currentTimeMillis();

		// System.out.println(Messages.Cooccurrence_0);

		this.corpus = corpus;

		this.query = query;

		//System.out.println("cooc params= corpus: "+corpus+" query: "+query+" props: "+properties+" limit: "+limit+" maxleft: "+maxleft+" minleft: "+minleft+" minright: "+minright+" maxright: "+maxright);

		this.properties = properties;

		this.limit = limit;

		this.minleft = minleft;

		this.maxleft = maxleft;

		this.minright = minright;

		this.maxright = maxright;

		this.minf = minf;

		this.minscore = minscore;

		this.mincof = mincof;

		this.includeXpivot = includeXpivot;

		this.buildLexicalTableWithCooccurrents = buildLexicalTableWithCooccurrents;

		//System.out.println("-- Done"); //$NON-NLS-1$

	}

	public boolean getIncludeXPivot() {

		return includeXpivot;

	}

	/**

	 * Gets the number of keyword.

	 *

	 * @return the number of keyword

	 */

	public int getNumberOfKeyword()

	{

		return numberOfKeyword;

	}

	public void setReferenceCorpus(String symbol) {

		referenceCorpus = symbol;

	}

	public void setCoocQuery(String q) {

		query_occ = q;

	}

	/**

	 * Process.

	 *

	 * @throws CqiClientException the cqi client exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 * @throws CqiServerError the cqi server error

	 * @throws StatException the stat exception

	 */

	public void process() throws CqiClientException, IOException, CqiServerError, StatException

	{

		stepQueryLimits();

		stepGetMatches();

		stepBuildSignatures();

		stepCount();

		stepBuildLexicalTable();

		stepGetScores();

		clearMemory();

		getLines();

	}

	public void clearMemory() {

		distances = null;

		distancescounts = null;

		counts = null;

		indexfreqs = null;

		counted = null;

		m1 = null;

		m2 = null;

		m3 = null;

		allsignaturesstr = null;

		lt = null;

		keysToString = null;

		occproperties = null;

		count = null;

		dist = null;

		freq = null;

		scores = null;

	}

	/**

	 * Step query limits.

	 *

	 * @return true, if successful

	 */

	public boolean stepQueryLimits()

	{

		if (limit != null) // structural context

		{

			String tempquery =""; //$NON-NLS-1$

			String lname = limit.getName();

			if (minleft > 0 ) { // test if there is a left context

				tempquery += "(<"	+ lname + ">[]* </" + lname + ">){" + (maxleft) + "," + (maxleft) + "}"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$

			}

			// (<p>[]*</p>){0, 50} "je" (<p>[]*</p>){0, 50}

			tempquery +=" <" + lname + ">[]* " + query.getQueryString() + " []* </" + lname + "> "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$

			if (minright > 0) { // test if there is a right context

				tempquery+="(<" + lname + ">[]* </" + lname + ">){" + (maxright) + "," + (maxright) + "}";  //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$

			}

			this.contextquery = new Query(tempquery);

			if (includeXpivot) {

				String anticontextquerystring = ""; //$NON-NLS-1$

				if (minleft > 1)// minleft = 2..N

					anticontextquerystring += "(<" + lname + ">[]* </" + lname + ">){" + (minleft - 1) + "," + (minleft - 1) + "} <" + lname + ">[]* "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$

				anticontextquerystring += query.getQueryString();

				if (minright > 1) // minright = 2..N

					anticontextquerystring += " []* </" + lname + "> (<" + lname + ">[]* </" + lname + ">){" + (minright - 1) + "," + (minright - 1) + "}"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$

				this.anticontextquery = new Query(anticontextquerystring);

			} else {

				String anticontextquerystring = ""; //$NON-NLS-1$

				if (minleft > 0)// minleft = 2..N

					anticontextquerystring += "(<" + lname + ">[]* </" + lname + ">){" + (minleft - 1) + "," + (minleft - 1) + "} <" + lname + ">[]* "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$

				anticontextquerystring += query.getQueryString();

				if (minright > 0) // minright = 2..N

					anticontextquerystring += " []* </" + lname + "> (<" + lname + ">[]* </" + lname + ">){" + (minright - 1) + "," + (minright - 1) + "}"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$

				this.anticontextquery = new Query(anticontextquerystring);

			}

		} else // word context

		{

			String tempquery =""; //$NON-NLS-1$

			if (minleft > 0 ) { // test if there is a left context

				tempquery += "[]{"	+ maxleft + "," + maxleft + "} "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			}

			tempquery+= query.getQueryString();

			if (minright > 0) { // test if there is a right context

				tempquery+=" []{" + maxright + "," + maxright + "} "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			}

			this.contextquery = new Query(tempquery); 

			String anticontextquerystring = ""; //$NON-NLS-1$

			if (minleft > 1)

				anticontextquerystring += "[]{" + (minleft - 1) + ", " + (minleft - 1) + "}"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			anticontextquerystring += query.getQueryString();

			if (minright > 1)

				anticontextquerystring += "[]{" + (minright -1) + "," + (minright -1)+ "} "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			this.anticontextquery = new Query(anticontextquerystring);

		}

		return true;

	}

	/**

	 * Step get matches.

	 *

	 * @return true, if successful

	 * @throws CqiClientException 

	 */

	public boolean stepGetMatches() throws CqiClientException 

	{

		QueryResult r1 = corpus.query(query, "CoocFocusQuery", false); // keywords positions //$NON-NLS-1$

		QueryResult r2 = corpus.query(contextquery, "CoocContextFocusQuery", false); // max context //$NON-NLS-1$

		QueryResult r3 = corpus.query(anticontextquery, "CoocAntiContextFocusQuery", false); // no context //$NON-NLS-1$

		m1 = r1.getMatches();

		if (Thread.interrupted()) return false; // stop if interrupted by user

		numberOfKeyword = m1.size();

		m2 = r2.getMatches();

		if (Thread.interrupted()) return false; // stop if interrupted by user

		m3 = r3.getMatches();

		if (Thread.interrupted()) return false; // stop if interrupted by user

		//		System.out.println(query+" M1 size: "+m1.size());

		//		System.out.println(contextquery+" M2 size: "+m2.size());

		//		System.out.println(anticontextquery+" M3 size: "+m3.size());

		r1.drop();

		r2.drop();

		r3.drop();

		return true;

	}

	/**

	 * Step build signatures.

	 *

	 * @return true, if successful

	 * @throws UnexpectedAnswerException the unexpected answer exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 * @throws CqiServerError the cqi server error

	 */

	public boolean stepBuildSignatures() throws UnexpectedAnswerException, IOException, CqiServerError

	{

		allsignaturesstr = new HashMap<Integer, String>();

		Set<Integer> allpositions = new HashSet<Integer>(); // no duplicates

		for (Match n : m2) {

			for (int i = n.getStart(); i <= n.getEnd(); i++)

				allpositions.add(i);

		}

		//System.out.println("Position set: "+allpositions.size());

		int[] allpositionsarray = new int[allpositions.size()];

		int pcount = 0;

		for (int p : allpositions) {

			allpositionsarray[pcount++] = p;

		}

		HashMap<Property, int[]> propsId = new HashMap<Property, int[]>();

		// HashMap<Property, String[]> propsValues = new HashMap<Property, String[]>();

		for (Property property : properties) {

			int[] indices = CorpusManager.getCorpusManager().getCqiClient()

					.cpos2Id(property.getQualifiedName(), allpositionsarray);

			// String[] values =

			// CorpusManager.getCorpusManager().getCqiClient().cpos2Str(property.getQualifiedName(),allpositionsarray);

			propsId.put(property, indices);

			// propsValues.put(property, values);

			//System.out.println("all "+property+" indices: "+propsId.get(property).length);

		}

		if (Thread.interrupted()) return false; // stop if interrupted by user

		//System.out.println("T values + ids: "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		pcount = 0;

		for (int position : allpositionsarray) {

			//String sign = ""; //$NON-NLS-1$

			String signstr = ""; //$NON-NLS-1$

			for (Property property : properties) {

				signstr += "[" + propsId.get(property)[pcount] + "]"; //$NON-NLS-1$ //$NON-NLS-2$

				//signstr+="["+propsValues.get(property)[pcount]+"]"; //$NON-NLS-1$ //$NON-NLS-2$

			}

			// allsignatures.put(position, sign);

			allsignaturesstr.put(position, signstr);

			pcount++;

		}

		return true;

	}

	/**

	 * Step count.

	 *

	 * @return true, if successful

	 */

	public boolean stepCount()

	{

		ArrayList<Integer> keepedPosition = new ArrayList<Integer>();

		int startsearchM2 = 0; // optimisation: m2 is ordered

		int startsearchM3 = 0; // optimisation: m3 is ordered

		//time = System.currentTimeMillis();

		for (Match m : m1) { // for each match = for each focus

			//System.out.println("for match m: "+m);

			Match n = null; // the match which contains the context

			Match o = null; // the match which does not contain the context

			boolean matchFound = false;

			for (int i = startsearchM2; i < m2.size(); i++) { // find n

				n = m2.get(i);

				if (n.getStart() <= m.getStart() && m.getEnd() <= n.getEnd()) {

					startsearchM2 = i;	

					matchFound = true;

					break;

				}

			}

			if (Thread.interrupted()) return false; // stop if interrupted by user

			//System.out.println("found n: "+n);

			for (int i = startsearchM3; i < m3.size(); i++)// find next match m3 contained by m2

			{

				o = m3.get(i);

				if (o.getStart() <= m.getStart() && m.getEnd() <= o.getEnd()) {

					startsearchM3 = i;

					matchFound = matchFound && true;

					break;

				}

			}

			if (Thread.interrupted()) return false; // stop if interrupted by user

			//System.out.println("found o: "+o);

			if (!matchFound) {

				//System.out.println(Messages.Cooccurrence_1 + n + " ; " + o); //$NON-NLS-1$ 

				continue;

			}

			int start = n.getStart();

			int size = n.getEnd() - start +1;

			//if (size > 0)

			//	size++;

			int ignore = 0;

			int chevauche = 0;

			// System.out.println("Process focus:"+m+" with maxcontext:"+n+" and anticontext:"+o);

			// System.out.println("NbOccs "+(size));

			int[] positions = new int[size];

			int noOcc = 0;

			//System.out.println("positions");

			// System.out.println("start: "+(start)+" end:"+n.getEnd());

			for (int position = start; position <= n.getEnd(); position++) 

				// creates the list of positions, anticontext not yet removed

			{

				positions[noOcc++] = position;

				//System.out.print(" "+position);

			}

			if (Thread.interrupted()) return false; // stop if interrupted by user

			//System.out.println();

			noOcc = 0;

			for (int position : positions) {

				// String signature = allsignatures.get(position);

				String signaturestr = allsignaturesstr.get(position);

				if (o.getStart() <= position && position <= o.getEnd()) { 

					// ignore positions in the anticontext positions

					ignore++;

					continue;

				}

				if (!distances.containsKey(signaturestr)) {

					distances.put(signaturestr, 0.0);

					distancescounts.put(signaturestr, 0);

				}

				if (counted.containsKey(position)) {

					// ignore positions already counted

					chevauche++;

					counted.put(position, counted.get(position) + 1);

					// continue;

				} else {

					counted.put(position, 1);

					if (!counts.containsKey(signaturestr))

						counts.put(signaturestr, 0);

					counts.put(signaturestr, (counts.get(signaturestr)) + 1);

					keepedPosition.add(position);

					double dist;

					if (position < m.getStart())

						dist = m.getStart() - position;

					else

						dist = position - m.getStart();

					distances.put(signaturestr, (distances.get(signaturestr))

							+ dist);

					distancescounts.put(signaturestr, (distancescounts

							.get(signaturestr)) + 1);

				}

				noOcc++;

				if (Thread.interrupted()) return false; // stop if interrupted by user

			}

			// System.out.println("nb Occ ignored: "+ignore);

			// System.out.println("nb Occ chevauche: "+chevauche);

		}

		//System.out.println("T counts : "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		allsignaturesstr = null; // no more need

		counted = null;

		return true;

	}

	/**

	 * Step build lexical table.

	 *

	 * @return true, if successful

	 * @throws RWorkspaceException the r workspace exception

	 */

	public boolean stepBuildLexicalTable() throws RWorkspaceException

	{

		String[] colnames = {

				corpus.getName() + "-" + query.getQueryString(), query.getQueryString() }; //$NON-NLS-1$

		keysToString = new HashMap<String, String>();

		//time = System.currentTimeMillis();

		Index voc = null;

		for (Object rez : corpus.getResults()) {//TODO: fix usages of index for cooc

			if (rez instanceof Index) {

				Index rezvoc = (Index) rez;

				if (rezvoc.getProperties().equals(properties)) {

					if (rezvoc.getQuery().equals(new Query(""+query_occ+""))) { //$NON-NLS-1$ //$NON-NLS-2$

						if (!rezvoc.isAltered()) {

							voc = rezvoc;

							break;

						}

					}

				}

			}

		}

		if (Thread.interrupted()) return false; // stop if interrupted by user

		if (voc == null) {

			if (properties.size() == 1 && "[]".equals(query_occ)) { //$NON-NLS-1$

				//System.out.println("build lexicon of props: "+properties);

				voc = new Index(corpus, properties.get(0));

			} else {

				//System.out.println("build index of props: "+properties);

				try {

					voc = new Index(corpus, new Query(query_occ), properties);

				} catch (Exception e){

					Log.severe("Error while computing Index for the cooccurrence: "+e.getLocalizedMessage());

					return false;

				}

			}

			corpus.storeResult(voc);

		}

		if (Thread.interrupted()) return false; // stop if interrupted by user

		// ALTER THE INDEX IF A REFERENCE CORPUS IS SET

		if(referenceCorpus != null && referenceCorpus.length() > 0) {

			//voc.toTxt(new File("/home/mdecorde/TEMP/before.tsv"), "UTF-8", "\t", "");

			try {

				voc.setIsAltered(true);

				String[] ref_forms = RWorkspace.getRWorkspaceInstance().eval("rownames("+referenceCorpus+")").asStrings(); //$NON-NLS-1$ //$NON-NLS-2$

				int[] ref_freqs = RWorkspace.getRWorkspaceInstance().eval(referenceCorpus+"[,1]").asIntegers(); //$NON-NLS-1$

				if (ref_forms.length != ref_freqs.length) {

					System.out.println(Messages.Cooccurrence_22);

					return false;

				}

				HashMap<String, Integer> ref_counts = new HashMap<String, Integer>();

				for(int i = 0 ; i < ref_forms.length ; i++) ref_counts.put(ref_forms[i], ref_freqs[i]);

				for (org.txm.index.core.functions.Line l : voc.getAllLines()) {

					String key = l.toString();

					if (ref_counts.containsKey(key)) {

						int[] f = {ref_counts.get(key)};

						l.setCounts(f, 0);

					}

				}

				//voc.toTxt(new File("/home/mdecorde/TEMP/after.tsv"), "UTF-8", "\t", "");

			} catch (REXPMismatchException e) {

				// TODO Auto-generated catch block

				org.txm.utils.logger.Log.printStackTrace(e);

				return false;

			}

		}

		if (Thread.interrupted()) return false; // stop if interrupted by user

		List<org.txm.index.core.functions.Line> vocLines = voc.getAllLines();

		int[][] freqs;

		String[] rownames;

		if (buildLexicalTableWithCooccurrents) {

			freqs = new int[counts.keySet().size()][2];

			rownames = new String[counts.keySet().size()];

		} else { // all words

			freqs = new int[vocLines.size()][2];

			rownames = new String[vocLines.size()];

		}

		int i = 0;

		//System.out.println("T voc : "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		// System.out.println("nb lines voc "+voclines.size());

		//System.out.println("counts keys: "+counts.keySet());

		for (org.txm.index.core.functions.Line l : vocLines) {

			//System.out.println("L sign '"+l.getSignature()+"'");

			if (counts.keySet().contains(l.getSignature())) {

				keysToString.put(l.toString(), l.getSignature());

				rownames[i] = l.toString();

				//System.out.println("set rowname: "+l.toString());

				//System.out.println("("+l.getSignature()+", "+l.toString()+") : "+l.getFrequency()+" - "+counts.get(l.getSignature()));

				int count = counts.get(l.getSignature());

				int tot = l.getFrequency();

				indexfreqs.put(l.toString(), tot);

				freqs[i][0] = tot - count;

				freqs[i][1] = count;

				i++;

			} else if (!buildLexicalTableWithCooccurrents) {

				keysToString.put(l.toString(), l.getSignature());

				rownames[i] = l.toString();

				//System.out.println("set rowname: "+l.toString());

				//System.out.println("("+l.getSignature()+", "+l.toString()+") : "+l.getFrequency()+" - "+counts.get(l.getSignature()));

				int tot = l.getFrequency();

				indexfreqs.put(l.toString(), tot);

				freqs[i][0] = tot;

				freqs[i][1] = 0;

				i++;

			}

		}

		if (Thread.interrupted()) return false; // stop if interrupted by user

		//time = System.currentTimeMillis();

		if (freqs.length == 0) {

			System.out.println(Messages.Cooccurrence_23);

		}

		//		try {

		//			PrintWriter writer = IOUtils.getWriter("/home/mdecorde/test_cooc.txt");

		//			//writer.println("Build LT: ");

		//			//writer.println("freqs: "+Arrays.toString(freqs));

		//			//writer.println("Rows: "+);

		//			//String rows = Arrays.toString();

		//			int nrow = rownames.length;

		//			int ncol = colnames.length;

		//			for (int ii = 0 ; ii < nrow ; ii++) {

		//				writer.write(rownames[ii]);

		//				for (int j = 0 ; j < ncol ; j++) {

		//					writer.write("\t"+freqs[ii][j]);

		//				}

		//				writer.write("\n");

		//			}

		//			writer.close();

		//			//writer.println("Cols: "+Arrays.toString(colnames));

		//		} catch(Exception e) {e.printStackTrace();}

		lt = LexicalTableImpl.createLexicalTable(freqs, properties.get(0), rownames, colnames, 1);

		lt.setCorpus(corpus);

		//		if(referenceCorpus != null && referenceCorpus.length() > 0) {

		//			//lt.removeCol(0, false);

		//			lt.setReference(referenceCorpus);

		//			lt.exchangeColumns(1,2);

		//		}

		return true;

	}

	/**

	 * Step get scores.

	 *

	 * @return true, if successful

	 * @throws CqiClientException the cqi client exception

	 * @throws StatException the stat exception

	 */

	public boolean stepGetScores() throws CqiClientException, StatException

	{

		SpecificitesResult specif = Specificites.specificites(lt, 1000);

		//System.out.println("Specif N part: "+specif.getNbrPart()); //$NON-NLS-1$

		//System.out.println("Specif N lines number: "+specif.getSpecificitesIndex().length); //$NON-NLS-1$

		//System.out.println("T specif e: "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		//specif.toTxt(new File("~/Bureau/coocresults/specif Cooc")); //$NON-NLS-1$

		String[] specifrownames = specif.getTypeNames();

		double[][] scores = specif.getSpecificitesIndex();

		//System.out.println("Nb specif result: "+specif.getSpecificitesIndex().length);

		int iimax = Math.min(specifrownames.length, scores.length);

		for (int ii = 0; ii < iimax; ii++) { // counts.keySet())

			String signaturestr = keysToString.get(specifrownames[ii]);

			ArrayList<String> props = new ArrayList<String>();

			if (properties.size() > 1) {

				String[] splited = specifrownames[ii].split("_", properties.size()); //$NON-NLS-1$

				for (int p = 0; p < properties.size(); p++) {

					props.add(splited[p]);

				}

			} else {

				props.add(specifrownames[ii]);

			}

			//			if(specifrownames[ii].equals("(") || specifrownames[ii].equals(")"))

			//			{

			//				System.out.println("rowname: "+specifrownames[ii]);

			//				System.out.println("props: "+props);

			//				System.out.println("counts: "+counts.get(signaturestr));

			//				System.out.println("speciffreq: "+indexfreqs.get(specifrownames[ii]));

			//				System.out.println("specif score: "+scores[ii][1]);

			//				System.out.println("distance: "+distances.get(signaturestr));

			//				System.out.println("distance count: "+distancescounts.get(signaturestr));

			//			}

			if (counts.containsKey(signaturestr)) {

				CLine cline = new CLine(this, specifrownames[ii], props, counts

						.get(signaturestr), indexfreqs.get(specifrownames[ii]), scores[ii][1],

						((float) (distances.get(signaturestr) / distancescounts

								.get(signaturestr))) - 1.0f, -1);

				//System.out.println(cline);

				if (cline.freq >= this.minf	&& cline.nbocc >= this.mincof && Math.abs(cline.score) >= this.minscore)

				{

					if (cline.score >= Integer.MAX_VALUE - 5)

						cline.score = Float.MAX_EXPONENT;

					lines.add(cline);

				}

			}

			//System.out.println(signaturestr+"\t"+voclines.get(ii).getFrequency()+"\t"+counts.get(signaturestr)+"\t"+(distances.get(signaturestr)/distancescounts.get(signaturestr))+"\t"+scores[ii][1]); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$

		}

		return true;

	}

	/**

	 * Sort.

	 *

	 * @param comparator the comparator

	 * @throws CqiClientException the cqi client exception

	 */

	public void sort(CLineComparator comparator) throws CqiClientException {

		comparator.initialize(corpus);

		if (lines.size() > 0) {

			Collections.sort(lines, comparator);

		}

	}

	/**

	 * Inits the conc infos.

	 *

	 * @param conc the conc

	 * @return true, if successful

	 */

	private boolean initConcInfos(Concordance conc) {

		int from = 0;

		int to = conc.getNLines() - 1;

		return initConcInfos(conc, from, to);

	}

	/**

	 * Inits the conc infos.

	 *

	 * @param conc the conc

	 * @param from the from

	 * @param to the to

	 * @return true, if successful

	 */

	private boolean initConcInfos(Concordance conc, int from, int to) {

		try {

			this.conc = conc;

			conclines = conc.getLines(from, to);

			return initConcInfos(conc, conclines);

		} catch (Exception e) {

			org.txm.utils.logger.Log.printStackTrace(e);

		}

		return false;

	}

	/**

	 * Inits the conc infos.

	 *

	 * @param conc the conc

	 * @param conclines the conclines

	 * @return true, if successful

	 */

	private boolean initConcInfos(Concordance conc, List<Line> conclines) {

		try {

			this.conc = conc;

			corpus = conc.getCorpus();

			this.P = corpus.getSize()

					/ (conc.getLeftContextSize() + conc.getRightContextSize());

			this.FA = conc.getNLines();

			corpus = conc.getCorpus();

			query = conc.getQuery();

			// System.out.println("CONC PROPS :"+conc.getViewProperties());

			properties = conc.getViewProperties();

			limit = null;

			maxleft = conc.getLeftContextSize();

			maxright = conc.getRightContextSize();

			this.conclines = conclines;

			return true;

		} catch (CqiClientException e) {

			org.txm.utils.logger.Log.printStackTrace(e);

		}

		return false;

	}

	/**

	 * Sets the thresfold.

	 *

	 * @param freq the freq

	 * @param count the count

	 * @param score the score

	 */

	public void setThresfold(int freq, int count, double score) {

		seuil_freq = freq;

		seuil_count = count;

		seuil_score = score;

	}

	/**

	 * Compute.

	 *

	 * @return true, if successful

	 */

	public boolean compute() {

		if (conclines == null || conc == null || P == -1 || FA == -1

				|| properties == null || corpus == null) {

			System.err.println(Messages.Cooccurrence_5 + conc);

			return false;

		}

		// System.out.println("seuils: freq="+seuil_freq+", count="+seuil_count+", score="+seuil_score);

		String queryfreq = "["; //$NON-NLS-1$

		for (Property p : properties) {

			queryfreq += " " + p.getName() + "=\".*\" |"; //$NON-NLS-1$ //$NON-NLS-2$

		}

		queryfreq = queryfreq.substring(0, queryfreq.length() - 1) + "]"; //$NON-NLS-1$

		try {

			voc = new Index(corpus, new Query(queryfreq), properties);

		} catch (Exception e){

			Log.severe("Error while computing Index for the cooccurrence: "+e.getLocalizedMessage());

			return false;

		}

		// voc.getCorpus().storeResult(voc);

		count = new HashMap<String, Integer>(voc.getV());

		dist = new HashMap<String, Float>(voc.getV());

		freq = new HashMap<String, Integer>(voc.getV());

		scores = new HashMap<String, Double>(voc.getV());

		occproperties = new HashMap<String, List<String>>(voc.getV());

		// System.out.println("Lignes");

		// System.out.println("leftC\tKeyword\trightC");

		for (Line concline : conclines) {

			// System.out.println(concline.leftContextToString()+"\t"+concline.keywordToString()+"\t"+concline.rightContextToString());

			Map<Property, List<String>> propsvalue = concline

					.getLeftCtxViewProperties();

			countOcc(propsvalue, false);

			propsvalue = concline.getRightCtxViewProperties();

			countOcc(propsvalue, true);

		}

		for (String cooc : count.keySet()) {

			if (count.get(cooc) >= seuil_count && freq.get(cooc) >= seuil_freq) {

				// System.out.println("compute dist et score de : "+cooc);

				// calcul dist moyenne avant de rectif ier count

				dist.put(cooc, dist.get(cooc) / count.get(cooc));

				if (count.get(cooc) > freq.get(cooc))

					count.put(cooc, freq.get(cooc));

				// calcul score du cooc

				double score = ProbaBinom(FA, freq.get(cooc), P,

						count.get(cooc));

				if (score >= seuil_score) {

					long mode = calcmode(FA, freq.get(cooc), P, count.get(cooc));

					CLine line = new CLine(this, cooc, occproperties.get(cooc),

							count.get(cooc), freq.get(cooc), score, dist

							.get(cooc), mode);

					lines.add(line);

				}

			}

		}

		// clear memory

		count = null;

		dist = null;

		freq = null;

		scores = null;

		occproperties = null;

		return true;

	}

	/**

	 * Prints the.

	 */

	public void print() {

		// System.out.println("Lines "+lines.size());

		System.out.println("FA : " + FA); //$NON-NLS-1$

		System.out.println("P : " + P); //$NON-NLS-1$

		System.out.println(Messages.Cooccurrence_6);

		for (CLine line : lines)

			System.out.println(line.resume("\t", "")); //$NON-NLS-1$ //$NON-NLS-2$

	}

	/**

	 * Count occ.

	 *

	 * @param propsvalue the propsvalue

	 * @param rightcontext the rightcontext

	 */

	public void countOcc(Map<Property, List<String>> propsvalue,

			boolean rightcontext) {

		// System.out.println("countOcc (R="+rightcontext+") "+propsvalue);

		Property key = properties.get(0);

		List<String> iterationlist = propsvalue.get(key);

		for (int i = 0; i < iterationlist.size(); i++)// pr chq mot

		{

			// build occ

			String occ = ""; //$NON-NLS-1$

			for (Property p : properties)

				occ += propsvalue.get(p).get(i) + "_"; //$NON-NLS-1$

			occ = occ.substring(0, occ.length() - 1);

			if (occproperties.get(occ) == null) {

				ArrayList<String> values = new ArrayList<String>();

				for (Property p : properties)

					values.add(propsvalue.get(p).get(i));

				occproperties.put(occ, values);

			}

			// System.out.println("occ '"+occ+"'");

			// update nbocc

			if (count.get(occ) == null)

				count.put(occ, 0);

			count.put(occ, count.get(occ) + 1);

			// update dist

			if (dist.get(occ) == null)

				dist.put(occ, 0.0f);

			if (rightcontext) {

				dist.put(occ, dist.get(occ) + i + 1);

			} else {

				dist.put(occ, dist.get(occ) + iterationlist.size() - i);

			}

			// update freq

			if (freq.get(occ) == null) {

				// System.out.println("compute freq of "+occ);

				int occfreq = -1; // calcul avec l'index

				for (org.txm.index.core.functions.Line l : voc

						.getLines(0, voc.getV())) {

					if (l.toString().equals(occ)) {

						// System.out.println("FOUND "+occ);

						occfreq = l.getFrequency();

						break;

					}

				}

				freq.put(occ, occfreq);

			}

		}

	}

	/**

	 * The Class CLine.

	 */

	public class CLine {

		/** The cooc. */

		Cooccurrence cooc;

		/** The occ. */

		public String occ;

		/** The freq. */

		public int freq;

		/** The nbocc. */

		public int nbocc;

		/** The score. */

		public double score;

		/** The distmoyenne. */

		public float distmoyenne;

		/** The mode. */

		public long mode;

		/** The props. */

		public List<String> props;

		/** The id. */

		public int id;

		/* (non-Javadoc)

		 * @see java.lang.Object#toString()

		 */

		@Override

		public String toString()

		{

			return occ+Messages.Cooccurrence_4+freq+Messages.Cooccurrence_7+nbocc+Messages.Cooccurrence_8+score+Messages.Cooccurrence_9+distmoyenne+Messages.Cooccurrence_10+props;

		}

		/**

		 * Instantiates a new c line.

		 *

		 * @param cooc the cooc

		 * @param occ the occ

		 * @param props the props

		 * @param nbocc the nbocc

		 * @param freq the freq

		 * @param score the score

		 * @param distmoyenne the distmoyenne

		 * @param mode the mode

		 */

		public CLine(Cooccurrence cooc, String occ, List<String> props,

				int nbocc, int freq, double score, Float distmoyenne, long mode) {

			this.occ = occ;

			this.props = props;

			this.freq = freq;

			this.nbocc = nbocc;

			this.score = score;

			this.distmoyenne = distmoyenne;

			this.mode = mode;

			this.cooc = cooc;

		}

		/**

		 * Sets the count and dist.

		 *

		 * @param count the count

		 * @param dist the dist

		 */

		public void setCountAndDist(int count, int dist) {

			this.nbocc = count;

			this.distmoyenne = dist;

		}

		/**

		 * Adds the txt sep.

		 *

		 * @param str the str

		 * @param sep the sep

		 * @return the string

		 */

		private String addTxtSep(String str, String sep)

		{

			return sep+str.replace(sep, sep+sep)+sep;

		}

		/**

		 * Resume.

		 *

		 * @param colseparator the colseparator

		 * @param txtseparator the txtseparator

		 * @return the string

		 */

		public String resume(String colseparator, String txtseparator) {

			return addTxtSep(""+occ, txtseparator) //$NON-NLS-1$

					+ colseparator + freq + colseparator + nbocc + colseparator + score + colseparator + distmoyenne; 

		}

		/**

		 * Sets the freq.

		 *

		 * @param freq the new freq

		 */

		public void setFreq(int freq) {

			this.freq = freq;

		}

		/**

		 * Sets the score.

		 */

		public void setScore()// FB == freq, R = nbocc

		{

			this.score = (this.nbocc + this.freq + this.distmoyenne);

		}

		/**

		 * Gets the cooc.

		 *

		 * @return the cooc

		 */

		public Cooccurrence getCooc() {

			return cooc;

		}

	}

	/**

	 * Calcmode.

	 *

	 * @param inf the inf

	 * @param ing the ing

	 * @param s the s

	 * @param cf the cf

	 * @return the long

	 */

	long calcmode(long inf, long ing, long s, long cf) {

		return (long) Math.floor((double) ((inf + 1) * (ing + 1))

				/ (inf + ing + s + 2));

	}

	// Maths

	/**

	 * Proba binom.

	 *

	 * @param inf the inf

	 * @param ing the ing

	 * @param s the s

	 * @param cf the cf

	 * @return the double

	 */

	public static double ProbaBinom(long inf, long ing, long s, long cf) {

		long mode = (long) Math.floor((double) ((inf + 1) * (ing + 1))

				/ (inf + ing + s + 2));

		long f, g, k;

		double P = 0.0, p, lp, dnm;

		int lpflag = 1;

		if (cf <= mode)

			return (1.0);

		if (inf < ing) {

			f = inf;

			g = ing;

		} else {

			f = ing;

			g = inf;

		}

		;

		dnm = rbicoln((int) (f + g + s), (int) g);

		for (k = cf; k <= f; k++) {

			p = Math.exp(rbicoln((int) f, (int) k)

					+ rbicoln((int) (s + g), (int) (g - k)) - dnm);

			// if (lpflag > 0) { // rentre dedans de toute facon

			lp = p;

			lpflag = 0;

			// };

			if ((lp / p) < maxprec)

				break;

			lp = p;

			P += p;

		}

		;

		return (P);

		/*

		 * for (k = 0; k < cf; k++) { p = exp(rbicoln(f,

		 * k)+rbicoln(s+g,g-k)-rbicoln(f+g+s, g)); P += p; }; return(fabs(1-P));

		 */

	}

	/** The arbicoln. */

	static double[][] arbicoln = new double[101][101];

	/** The cof. */

	static double[] cof = { 76.18009172947146, -86.50532032941677,

		24.01409824083091, -1.231739572450155, 0.1208650973866179e-2,

		-0.5395239384953e-5 };

	/** The a. */

	static double[] a = new double[101];

	/** The maxprec. */

	static double maxprec = 0.0;

	static {// void init_rbicoln()

		int i, j;

		for (i = 0; i < 101; i++)

			for (j = 0; j < 101; j++)

				arbicoln[i][j] = -1;

	}

	/**

	 * Gammln.

	 *

	 * @param xx the xx

	 * @return the double

	 */

	static double gammln(double xx) {

		double x, y, tmp, ser;

		int j;

		y = x = xx;

		tmp = x + 5.5;

		tmp -= (x + 0.5) * Math.log(tmp);

		ser = 1.000000000190015;

		for (j = 0; j <= 5; j++)

			ser += cof[j] / ++y;

		return -tmp + Math.log(2.5066282746310005 * ser / x);

	}

	/**

	 * Factln.

	 *

	 * @param n the n

	 * @return the double

	 */

	static double factln(int n) {

		if (n < 0)

			System.err.println(Messages.Cooccurrence_102);

		if (n <= 1)

			return 0.0;

		if (n <= 100)

			return (a[n] > 0) ? a[n] : (a[n] = gammln(n + 1.0));

			else

				return gammln(n + 1.0);

	}

	/**

	 * Rbicoln.

	 *

	 * @param n the n

	 * @param k the k

	 * @return the double

	 */

	static double rbicoln(int n, int k) {

		if (n < 0)

			System.err.println(Messages.Cooccurrence_103);

		if (k < 0)

			System.err.println(Messages.Cooccurrence_103);

		if (n <= 100 && k <= 100)

			return (arbicoln[n][k] >= 0) ? arbicoln[n][k]

					: (arbicoln[n][k] = factln(n) - factln(k) - factln(n - k));

			else

				return factln(n) - factln(k) - factln(n - k);

	}

	/**

	 * Gets the lines.

	 *

	 * @return the lines

	 */

	public List<CLine> getLines() {

		return lines;

	}

	int numberOfCooccurrents = -1;

	/**

	 * Gets the lines.

	 *

	 * @return the lines

	 */

	public int getNumberOfCooccurrents() {

		if (numberOfCooccurrents == -1) {

			numberOfCooccurrents = 0;

			for (CLine line : lines)

				numberOfCooccurrents += line.nbocc;

		}

		return numberOfCooccurrents;

	}

	/**

	 * Gets the lines.

	 *

	 * @return the lines

	 */

	public int getNumberOfDifferentCooccurrents() {

		if (lines != null) return lines.size();

		return 0;

	}

	/**

	 * Gets the corpus.

	 *

	 * @return the corpus

	 */

	public Corpus getCorpus() {

		return corpus;

	}

	/**

	 * Gets the query.

	 *

	 * @return the query

	 */

	public Query getQuery() {

		return query;

	}

	/**

	 * To txt.

	 *

	 * @param outfile the outfile

	 * @param encoding the encoding

	 * @return true, if successful

	 */

	public boolean toTxt(File outfile, String encoding) {

		return toTxt(outfile, encoding, "\t", ""); //$NON-NLS-1$ //$NON-NLS-2$

	}

	/**

	 * To txt.

	 *

	 * @param outfile the outfile

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @return true, if successful

	 */

	public boolean toTxt(File outfile, String encoding, String colseparator, String txtseparator) {

		try {

			// NK: writer declared as class attribute to perform a clean if the operation is interrupted

			this.writer = new BufferedWriter(new OutputStreamWriter(

					new FileOutputStream(outfile), encoding)); 

			// if ("UTF-8".equals(encoding)) writer.write('\ufeff'); // UTF-8 BOM

			toTxt(writer, colseparator, txtseparator);

		} catch (Exception e) {

			Log.severe(Messages.Cooccurrence_107 + e);

			return false;

		}

		return true;

	}

	/**

	 * To txt.

	 *

	 * @param writer the writer

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @return true, if successful

	 */

	public boolean toTxt(Writer writer, String colseparator, String txtseparator) {

		try {

			//Occ	Freq	CoFreq	Score	MeanDist

			writer.write( txtseparator+Messages.Cooccurrence_11+txtseparator+colseparator+

					txtseparator+Messages.Cooccurrence_12+txtseparator+colseparator+

					txtseparator+Messages.Cooccurrence_13+txtseparator+colseparator+

					txtseparator+Messages.Cooccurrence_14+txtseparator+colseparator+

					txtseparator+Messages.Cooccurrence_15+txtseparator+"\n");//colseparator+ //$NON-NLS-1$

			//txtseparator+Messages.Cooccurrence_16+txtseparator+"\n"); //$NON-NLS-1$

			for (CLine line : lines) 

				writer.write(line.resume(colseparator, txtseparator) + "\n"); //$NON-NLS-1$

			writer.close();

		} catch (IOException e) {

			Log.severe(Messages.Cooccurrence_107 + e);

			return false;

		}

		return true;

	}

	/**

	 * Gets the name.

	 *

	 * @return the name

	 */

	public String getName() {

		return query + Messages.Cooccurrence_3

				+ (maxleft-1) + ", " + (maxright-1) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ 

	}

	/**

	 * Gets the fA.

	 *

	 * @return the fA

	 */

	public int getFA() {

		return this.FA;

	}