Laboratoire ICAR » Plateforme TXM

import org.txm.searchengine.cqp.corpus.QueryResult;

import org.txm.searchengine.cqp.corpus.StructuralUnit;

import org.txm.searchengine.cqp.corpus.StructuralUnitProperty;

import org.txm.searchengine.cqp.corpus.WordProperty;

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.searchengine.cqp.corpus.query.CQLQuery;

 * 

 */

public class Cooccurrence extends TXMResult {

	/** The nocooc. */

	protected static int nocooc = 1;

	/** The prefix r. */

	protected static String prefixR = "Cooccurrences"; //$NON-NLS-1$

	/** The allsignaturesstr. */

	private HashMap<Integer, String> allsignaturesstr;

	/** The anticontextquery. */

	private CQLQuery anticontextquery;

	private boolean buildLexicalTableWithCooccurrents;

	/** The conclines. */

	List<org.txm.concordance.core.functions.Line> conclines;

	/** The conc. */

	Concordance concordance;

	/** The contextquery. */

	private CQLQuery contextQuery;

	/** The count. */

	HashMap<String, Integer> count;

	// contains the sum of distances

	/** The counts. */

	HashMap<String, Integer> counts = new HashMap<>();

	/** The dist. */

	HashMap<String, Float> dist;

	// System.out.println("Matches: focus: "+m1.size()+" full: "+m2.size()+" anti: "+m3.size());

	// System.out.println("T matches : "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

	/** The distances. */

	HashMap<String, Double> distances = new HashMap<>();

	/** The FA. */

	int FA = -1;

	/** The freq. */

	HashMap<String, Integer> freq;

	/** The index. */

	Index index;

	/** The indexfreqs. */

	HashMap<String, Integer> indexfreqs = new HashMap<>();

	/** The keys to string. */

	private HashMap<String, String> keysToString;

	/** The lines. */

	List<CLine> lines = new ArrayList<>();

	/** The lt. */

	private LexicalTableImpl lt;

	/** The m1. */

	private List<Match> m1;

	/** The m2. */

	private List<Match> m2;

	/** The m3. */

	private List<Match> m3;

	int numberOfCooccurrents = -1;

	/** The number of keyword. */

	int numberOfKeyword = 0;

	/** The occproperties. */

	HashMap<String, List<String>> occproperties;

	/** The P. */

	int P = -1;

	/** The reference corpus to use = the R symbol that point to a matrix WordxFreqs. */

	String referenceCorpus;

	/** The scores. */

	HashMap<String, Double> scores;

	/** The symbol. */

	private String symbol;

	/** The writer. */

	private BufferedWriter writer;

	@Parameter(key = CooccurrencePreferences.QUERY_FILTER)

	protected String pCooccurentQueryFilter = "[]"; //$NON-NLS-1$

	/** The mincof. */

	@Parameter(key = CooccurrencePreferences.MIN_COUNT)

	protected Integer pFCoocFilter;

	/** The minf. */

	@Parameter(key = TXMPreferences.F_MIN)

	protected Integer pFminFilter;

	/** The include xpivot. */

	@Parameter(key = CooccurrencePreferences.INCLUDE_X_PIVOT)

	protected Boolean pIncludeXpivot;

	/** The maxleft. */

	@Parameter(key = CooccurrencePreferences.MAX_LEFT)

	protected Integer pMaxLeftContextSize;

	/** The maxright. */

	@Parameter(key = CooccurrencePreferences.MAX_RIGHT)

	protected Integer pMaxRightContextSize;

	/** The minleft. */

	@Parameter(key = CooccurrencePreferences.MIN_LEFT)

	protected Integer pMinLeftContextSize;

	/** The minright. */

	@Parameter(key = CooccurrencePreferences.MIN_RIGHT)

	protected Integer pMinRightContextSize;

	/** The cooccurrents properties to display. */

	@Parameter(key = CooccurrencePreferences.UNIT_PROPERTIES)

	protected List<WordProperty> pProperties;

	/** The keyword query. */

	@Parameter(key = CooccurrencePreferences.QUERY)

	protected CQLQuery pQuery;

	/** The minscore. */

	@Parameter(key = CooccurrencePreferences.MIN_SCORE)

	protected Float pScoreMinFilter;

	/**

	 * The structural unit context limit.

	 * In null then the unit property is used.

	 */

	@Parameter(key = CooccurrencePreferences.STRUCTURAL_UNIT_LIMIT, electric = false)

	protected StructuralUnit pStructuralUnitLimit;

	/**

	 * Creates a not computed cooccurrence from the specified corpus.

	 * 

	public Cooccurrence(CQPCorpus parent) {

		super(parent);

	}

	/**

	 * Creates a not computed cooccurrence from a parameters node.

	 * 

	public Cooccurrence(String parametersNodePath) {

		super(parametersNodePath);

	}

	@Override

	protected boolean _compute(TXMProgressMonitor monitor) throws CqiClientException, IOException, CqiServerError, StatException {

		// System.out.println("cooc: "+corpus+" "+query+" "+properties+" "+limit+" "+maxLeft+" "+minLeft+" "+minRight+" "+maxRight+" "+minFreq+" "+minCof+" "+minScore+" "+includeXpivot);

		monitor.setTask(CooccurrenceCoreMessages.info_buildingQueries);

		// clear data

		try {

			this.numberOfCooccurrents = -1;

		}

		catch (Exception e) {

		}

		if (!this.stepQueryLimits()) {

			return false;

		}

		monitor.setTask(CooccurrenceCoreMessages.info_retreivingMatches);

		if (!this.stepGetMatches()) {

			return false;

		}

		monitor.worked(20);

		monitor.setTask(CooccurrenceCoreMessages.info_buildingLineSignatures);

		if (!this.stepBuildSignatures()) {

			return false;

		}

		monitor.worked(20);

		monitor.setTask(CooccurrenceCoreMessages.info_counting);

		if (!this.stepCount()) {

			return false;

		}

		monitor.worked(20);

		monitor.setTask(CooccurrenceCoreMessages.info_buildingLexicalTable);

		if (!this.stepBuildLexicalTable(monitor)) {

			return false;

		}

		monitor.worked(10);

		monitor.setTask(CooccurrenceCoreMessages.info_computingSpecificitiesScores);

		if (!this.stepGetScores()) {

			return false;

		}

		this.clearMemory();

		monitor.done();

		return true;

	}

	@Override

	public boolean loadParameters() throws CqiClientException {

		pProperties = (List<WordProperty>) Property.stringToProperties(getCorpus(), this.getStringParameterValue(TXMPreferences.UNIT_PROPERTIES));

		pStructuralUnitLimit = this.getCorpus().getStructuralUnit(this.getStringParameterValue(CooccurrencePreferences.STRUCTURAL_UNIT_LIMIT));

		return true;

	}

	@Override

	public boolean saveParameters() {

		this.saveParameter(TXMPreferences.UNIT_PROPERTIES, Property.propertiesToString(this.pProperties));

		if (pQuery != null) {

			this.saveParameter(TXMPreferences.QUERY, pQuery.getQueryString());

		}

		if (pStructuralUnitLimit != null) {

			this.saveParameter(CooccurrencePreferences.STRUCTURAL_UNIT_LIMIT, this.pStructuralUnitLimit.getName());

		}

		return true;

	}

	/**

	 * As r matrix.

	 *

	 */

	public String asRMatrix() throws RWorkspaceException {

		symbol = prefixR + nocooc;

		String[] occ = new String[this.lines.size()];

		int[] freq = new int[this.lines.size()];

		int[] cofreq = new int[this.lines.size()];

		double[] score = new double[this.lines.size()];

		double[] dist = new double[this.lines.size()];

		int i = 0;

		for (CLine line : this.lines) {

			occ[i] = line.occ;

			dist[i] = line.distmoyenne;

			i++;

		}

		RWorkspace rw = RWorkspace.getRWorkspaceInstance();

		rw.addVectorToWorkspace("coococc", occ); //$NON-NLS-1$

		rw.addVectorToWorkspace("coocfreq", freq); //$NON-NLS-1$

		rw.addVectorToWorkspace("cooccofreq", cofreq); //$NON-NLS-1$

		rw.addVectorToWorkspace("coocscore", score); //$NON-NLS-1$

		rw.addVectorToWorkspace("coocmeandist", dist); //$NON-NLS-1$

		rw.eval(symbol + "<- matrix(data = c(coocfreq, cooccofreq, coocscore, coocmeandist), nrow = " + this.lines.size() + ", ncol = 4)"); //$NON-NLS-1$ //$NON-NLS-2$

		rw.eval("rownames(" + symbol + " ) <- coococc"); //$NON-NLS-1$ //$NON-NLS-2$

		rw.eval("colnames(" + symbol + " ) <- c('freq', 'cofreq', 'score', 'dist')"); //$NON-NLS-1$ //$NON-NLS-2$

		rw.eval(symbol + "<- list(data=" + symbol //$NON-NLS-1$

				+ ")"); //$NON-NLS-1$

		nocooc++;

		return symbol;

	}

	@Override

	public boolean canCompute() {

		if (pQuery == null || pQuery.isEmpty()) {

			Log.fine("No query set.");

			return false;

		}

		if (pProperties == null) {

			Log.fine("No properties set.");

			return false;

		}

		if (getCorpus() == null) {

			Log.fine("No corpus set.");

			return false;

		}

		if (pProperties.size() == 0) {

			Log.fine("No properties filled.");

			return false;

		}

		return true;

	}

	@Override

	public void clean() {

		try {

			org.txm.utils.logger.Log.printStackTrace(e);

		}

	}

	// FIXME: useless?

	public void clearMemory() {

		if (distances != null) distances.clear();

		if (dist != null) dist.clear();

		if (freq != null) freq.clear();

		if (scores != null) scores.clear();

		lt = null;

	}

	// /**

	// * Count occ.

	// *

	// }

	// }

	// }

	/**

	 * Gets the corpus.

	 *

	public CQPCorpus getCorpus() {

		return (CQPCorpus) this.getParent();

	}

	@Override

	public String getDetails() {

		Object[] params = new Object[] { this.getParent(), this.pQuery, this.pProperties, this.pStructuralUnitLimit, (this.pMinLeftContextSize - 1), (this.pMaxLeftContextSize - 1),

				(this.pMaxRightContextSize - 1), this.pFminFilter, this.pFCoocFilter, this.pScoreMinFilter };

		return NLS.bind(CooccurrenceCoreMessages.info_details, params);

	}

	@Override

	public String getName() {

		try {

			return this.getSimpleName();

		}

	}

	@Override

	public String getSimpleName() {

		if (this.pQuery != null && !this.pQuery.isEmpty()) {

			StringBuffer output = new StringBuffer();

			output.append(this.pQuery.asString());

			output.append(WordProperty.asString(this.pProperties));

			if (this.pMaxLeftContextSize > 0 && this.pMaxRightContextSize > 0) {

				output.append(" " + (this.pMaxLeftContextSize - 1) + " " + (this.pMaxRightContextSize - 1)); //$NON-NLS-1$ //$NON-NLS-2$

			}

			output.append(TXMCoreMessages.formatMinFilter(this.pFminFilter));

			output.append(TXMCoreMessages.formatMinFilter(this.pFCoocFilter));

			output.append(TXMCoreMessages.formatMinFilter(this.pScoreMinFilter));

			// TODO: SJ: improve the hiding or display of value according to the default preferences values

			// output.append(TXMCoreMessages.formatMinFilter(this.pFminFilter, CooccurrencePreferences.getInstance().getInt(CooccurrencePreferences.F_MIN) + 1));

			// output.append(TXMCoreMessages.formatMinFilter(this.pFCoocFilter, CooccurrencePreferences.getInstance().getInt(CooccurrencePreferences.MIN_COUNT) + 1));

			// output.append(TXMCoreMessages.formatMinFilter(this.pScoreMinFilter, CooccurrencePreferences.getInstance().getDouble(CooccurrencePreferences.MIN_SCORE) + 1));

			return output.toString();

		}

		else {

			return this.getEmptyName();

		}

	}

	@Override

	public String getComputingStartMessage() {

				(this.pMaxLeftContextSize - 1), (this.pMaxRightContextSize - 1), this.pFminFilter, this.pFCoocFilter, this.pScoreMinFilter, this.getCorpus().getName());

	}

	@Override

	public String getComputingDoneMessage() {

		if (this.lines.isEmpty()) {

			return TXMCoreMessages.bind(CooccurrenceCoreMessages.P0CooccurentsForP1Occurrences, TXMCoreMessages.formatNumber(this.lines.size()), TXMCoreMessages.formatNumber(this.numberOfKeyword));

		}

	}

	/**

	 * Gets the fA.

	 *

	public int getFA() {

		return this.FA;

	}

	public boolean getIncludeXPivot() {

		return pIncludeXpivot;

	}

	/**

	 * Gets the lines.

	 *

	public List<CLine> getLines() {

		return lines;

	}

	/**

	 * Gets the max left.

	 *

	public int getMaxLeft() {

		return pMaxLeftContextSize;

	}

	/**

	 * Gets the max right.

	 *

	public int getMaxRight() {

		return pMaxRightContextSize;

	}

	/**

	 * Gets the min left.

	 *

	public int getMinLeft() {

		return pMinLeftContextSize;

	}

	/**

	 * Gets the min right.

	 *

	public int getMinRight() {

		return pMinRightContextSize;

	}

	/**

	 * Gets the lines.

	 *

		}

		return numberOfCooccurrents;

	}

	/**

	 * Gets the lines.

	 *

		}

		return 0;

	}

	/**

	 * Gets the number of keyword.

	 *

	public int getNumberOfKeyword() {

		return numberOfKeyword;

	}

	/**

	 * Gets the p.

	 *

	public int getP() {

		return P;

	}

	/**

	 * Gets the properties.

	 *

	public List<WordProperty> getProperties() {

		return pProperties;

	}

	/**

	 * Gets the query.

	 *

	public CQLQuery getQuery() {

		return pQuery;

	}

	/**

	 * Gets the structural unit limit.

	 *

	public StructuralUnit getStructuralUnitLimit() {

		return this.pStructuralUnitLimit;

	}

	/**

	 * Gets the symbol.

	 *

	public String getSymbol() {

		return this.symbol;

	}

	/**

	 * Inits the conc infos.

	 *

	 * @return true, if successful

	 */

	private boolean initConcInfos(Concordance conc) {

		int from = 0;

		int to = conc.getNLines() - 1;

		return initConcInfos(conc, from, to);

	}

	/**

	 * Inits the conc infos.

	 *

	 * @return true, if successful

	 */

	private boolean initConcInfos(Concordance conc, int from, int to) {

		try {

			this.concordance = conc;

			conclines = conc.getLines(from, to);

		}

		return false;

	}

	/**

	 * Inits the conc infos.

	 *

	 * @return true, if successful

	 */

	private boolean initConcInfos(Concordance conc, List<Line> conclines) {

		try {

			CQPCorpus corpus = this.getCorpus();

			this.concordance = conc;

		}

		return false;

	}

	/**

	 * Prints the.

	 */

		for (CLine line : lines)

			System.out.println(line.resume("\t", "")); //$NON-NLS-1$ //$NON-NLS-2$

	}

	public void setCoocQuery(String q) {

		pCooccurentQueryFilter = q;

	}

	/**

	 * Sets the max left.

	 *

	public void setMaxLeft(int maxleft) {

		this.pMaxLeftContextSize = maxleft;

	}

	/**

	 * Sets the max right.

	 *

	public void setMaxRight(int maxright) {

		this.pMaxRightContextSize = maxright;

	}

	/**

	 * Sets the min left.

	 *

	public void setMinLeft(int minleft) {

		this.pMinLeftContextSize = minleft;

	}

	/**

	 * Sets the min right.

	 *

	public void setMinRight(int minright) {

		this.pMinRightContextSize = minright;

	}

	public void setParameters(CQLQuery query, List<WordProperty> properties, StructuralUnit limit, int maxLeft, int minLeft, int minRight,

			int maxRight, int minFreq, float minScore, int minCof, boolean includeXpivot, boolean buildLexicalTableWithCooccurrents) {

		this.pQuery = query;

		this.pProperties = properties;

		this.pStructuralUnitLimit = limit;

		this.pIncludeXpivot = includeXpivot;

		this.buildLexicalTableWithCooccurrents = buildLexicalTableWithCooccurrents;

	}

	public void setIncludeXpivot(boolean b) {

		pIncludeXpivot = b;

	}

	@Override

	public boolean setParameters(TXMParameters parameters) {

		try {

			CQPCorpus corpus = this.getCorpus();

			boolean includeXpivot = parameters.getBoolean(CooccurrencePreferences.INCLUDE_X_PIVOT);

			String queryString = ""; //$NON-NLS-1$

			if (parameters.get(TXMPreferences.QUERY) != null) {

				queryString = parameters.get(TXMPreferences.QUERY).toString();

			}

			CQLQuery query = new CQLQuery(queryString);

			StructuralUnit limit = (StructuralUnit) parameters.get(CooccurrencePreferences.STRUCTURAL_UNIT_LIMIT);

			Object propsParam = parameters.get(CooccurrencePreferences.UNIT_PROPERTIES);

			List<WordProperty> properties = null;

			if (propsParam instanceof List<?>) {

			else if (propsParam instanceof String) {

				properties = (List<WordProperty>) Property.stringToProperties(corpus, propsParam.toString());

			}

			int maxLeft = parameters.getInteger(CooccurrencePreferences.MAX_LEFT);

			int minLeft = parameters.getInteger(CooccurrencePreferences.MIN_LEFT);

			int maxRight = parameters.getInteger(CooccurrencePreferences.MAX_RIGHT);

			int minCof = parameters.getInteger(CooccurrencePreferences.MIN_COUNT);

			int minFreq = parameters.getInteger(TXMPreferences.F_MIN);

			boolean buildLexicalTableWithCooccurrents = parameters.getBoolean(CooccurrencePreferences.PARTIAL_LEXICAL_TABLE);

			this.setParameters(query, properties, limit, maxLeft, minLeft, minRight, maxRight, minFreq, minScore, minCof, includeXpivot, buildLexicalTableWithCooccurrents);

		}

		catch (Exception e) {

			System.out.println("Error while setting cooccurrence parameters: " + e.getLocalizedMessage());

		}

		return true;

	}

	public void setReferenceCorpus(String symbol) {

		referenceCorpus = symbol;

	}

	/**

	 * Sets the structural unit limit.

	 *

	public void setStructuralUnitLimt(StructuralUnit su) {

		pStructuralUnitLimit = su;

	}

	/**

	 * Sets the thresfold.

	 *

	 * @param score the score

	 */

	public void setThresfold(int freq, int count, float score) {

		pFminFilter = freq;

		pFCoocFilter = count;

		pScoreMinFilter = score;

	}

	/**

	 * Sort.

	 *

			Collections.sort(lines, comparator);

		}

	}

	/**

	 * Step build lexical table.

	 *

	 * @throws RWorkspaceException the r workspace exception

	 */

	public boolean stepBuildLexicalTable(TXMProgressMonitor monitor) throws RWorkspaceException {

		CQPCorpus corpus = this.getCorpus();

		String[] colnames = { corpus.getName() + "-" + pQuery.getQueryString(), pQuery.getQueryString() }; //$NON-NLS-1$

		keysToString = new HashMap<>();

		// time = System.currentTimeMillis();

		for (TXMResult rez : corpus.getChildren(Index.class)) { // TODO: fix usages of index for cooc

			Index rezvoc = (Index) rez;

			if (rezvoc.getProperties().equals(pProperties)) {

				if (rezvoc.getQuery().equals(new CQLQuery(pCooccurentQueryFilter))) {

					if (rezvoc.getFilterFmax() == null) {

				}

			}

		}

		// TODO: MD: implement a sibling dependency system

		if (index == null) {

			try {

				return false;

			}

		}

		// ALTER THE INDEX IF A REFERENCE CORPUS IS SET -> this change the base frequencies

		if (referenceCorpus != null && referenceCorpus.length() > 0) {

			// voc.toTxt(new File("/home/mdecorde/TEMP/before.tsv"), "UTF-8", "\t", "");

				return false;

			}

		}

		List<org.txm.index.core.functions.Line> vocLines = index.getAllLines();

		int[][] freqs;

		String[] rownames;

			freqs = new int[vocLines.size()][2];

			rownames = new String[vocLines.size()];

		}

		int i = 0;

		// System.out.println("T voc : "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		// System.out.println("nb lines voc "+voclines.size());

				int count = counts.get(l.getSignature());

				int tot = l.getFrequency();

				indexfreqs.put(l.toString(), tot);

				freqs[i][0] = tot - count;

				freqs[i][1] = count;

				i++;

				rownames[i] = l.toString();

				// System.out.println("set rowname: "+l.toString());

				// System.out.println("("+l.getSignature()+", "+l.toString()+") : "+l.getFrequency()+" - "+counts.get(l.getSignature()));

				int tot = l.getFrequency();

				indexfreqs.put(l.toString(), tot);

				freqs[i][0] = tot;

				freqs[i][1] = 0;

				i++;

		}

		index.delete(); // no more needed

		index = null;

		// time = System.currentTimeMillis();

		if (freqs.length == 0) {

			System.out.println(CooccurrenceCoreMessages.errorColonNoCooccurrents);

		// writer.close();

		// //writer.println("Cols: "+Arrays.toString(colnames));

		// } catch(Exception e) {e.printStackTrace();}

		lt = new LexicalTableImpl(freqs, rownames, colnames);

		// if(referenceCorpus != null && referenceCorpus.length() > 0) {

		// //lt.removeCol(0, false);

		// lt.setReference(referenceCorpus);

		// }

		return true;

	}

	/**

	 * Step build signatures.

	 *

			}

		}

		// System.out.println("Position set: "+allpositions.size());

		int[] allpositionsarray = new int[allpositions.size()];

		int pcount = 0;

		for (int p : allpositions) {

			allpositionsarray[pcount++] = p;

		}

		HashMap<Property, int[]> propsId = new HashMap<>();

		// HashMap<Property, String[]> propsValues = new HashMap<Property, String[]>();

			// String[] values =

			// CorpusManager.getCorpusManager().getCqiClient().cpos2Str(property.getQualifiedName(),allpositionsarray);

			propsId.put(property, indices);

			// propsValues.put(property, values);

			// System.out.println("all "+property+" indices: "+propsId.get(property).length);

		}

		// System.out.println("T values + ids: "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		pcount = 0;

		for (int position : allpositionsarray) {

			// String sign = ""; //$NON-NLS-1$

			// allsignatures.put(position, sign);

			allsignaturesstr.put(position, signstr);

			pcount++;

		}

		return true;

	}

	/**

	 * Step count.

	 *

	 */

	public boolean stepCount() {

		// ArrayList<Integer> keepedPosition = new ArrayList<>();

		int startsearchM2 = 0; // optimisation: m2 is ordered

		int startsearchM3 = 0; // optimisation: m3 is ordered

		// time = System.currentTimeMillis();

		HashMap<Integer, Integer> positionsDistances = new HashMap<>();

		for (Match m : m1) { // for each match = for each focus

			if (m.getTarget() >= 0) { // if target is set focus on target position

				m.setStart(m.getTarget());

				m.setEnd(m.getTarget());

			}

			if (Thread.interrupted()) return false; // stop if interrupted by user

			// System.out.println("found n: "+n);

			for (int i = startsearchM3; i < m3.size(); i++) { // find next match m3 contained by m2

				o = m3.get(i);

				if (o.getStart() <= m.getStart() && m.getEnd() <= o.getEnd()) {

					startsearchM3 = i;

				}

			}

			// System.out.println("found o: "+o);

			if (!matchFound) {

				continue;

			}

			int start = n.getStart();

			int size = n.getEnd() - start + 1;

			// if (size > 0)

			// System.out.println("NbOccs "+(size));

			int[] positions = new int[size];

			int noOcc = 0;

			// System.out.println("positions");

			// System.out.println("start: "+(start)+" end:"+n.getEnd());

			for (int position = start; position <= n.getEnd(); position++) {

				if (o.getStart() <= position && position <= o.getEnd()) {

					// ignore positions in the anticontext positions

					continue;

				}

				int dist;

				if (position < m.getStart()) {

					dist = m.getStart() - position - 1;

					positionsDistances.put(position, dist);

				}

			}

			// System.out.println("nb Occ ignored: "+ignore);

			// System.out.println("nb Occ chevauche: "+chevauche);

		}

		// store and count distances for each position signature

		int noOcc = 0;

		for (int position : positionsDistances.keySet()) { // cooccurrent words positions

			// String signature = allsignatures.get(position);

			String signaturestr = allsignaturesstr.get(position);

			int dist = positionsDistances.get(position);

			if (distances.containsKey(signaturestr)) {

				distances.put(signaturestr, (distances.get(signaturestr)) + dist);

			else {

				distances.put(signaturestr, 0.0);

			}

			if (counts.containsKey(signaturestr)) {

				counts.put(signaturestr, (counts.get(signaturestr)) + 1);

			}

			else {

				counts.put(signaturestr, 1);

			}

			// if ("[1599]".equals(signaturestr)) {

			// System.out.println("p=" + position + " d=" + dist + " total(d)=" + distances.get(signaturestr) + " c=" + counts.get(signaturestr));

			// }

			// }

			noOcc++;

		}

		// System.out.println("T counts : "+(System.currentTimeMillis()- time)); //$NON-NLS-1$

		allsignaturesstr = null; // no more need

		// counted = null;

		return true;

	}

	/**

	 * Step get matches.

	 *

		else {

			r3 = corpus.query(anticontextquery, "CoocAntiContextFocusQuery", false); // no context //$NON-NLS-1$

		}

		m1 = r1.getMatches();

		numberOfKeyword = m1.size();

		m2 = r2.getMatches();

		m3 = r3.getMatches();

		Log.finest("R1 size=" + m1.size());

		Log.finest("R2 size=" + m2.size());

		Log.finest("R3 size=" + m3.size());

		// System.out.println(query+" M1 size: "+m1.size());

		// System.out.println(contextquery+" M2 size: "+m2.size());

		// System.out.println(anticontextquery+" M3 size: "+m3.size());

		r1.drop();

		if (r2 != r1) {

			r2.drop();

		}

		}

		return true;

	}

	/**

	 * Step get scores.

	 *

	 * @throws StatException the stat exception

	 */

	public boolean stepGetScores() throws CqiClientException, StatException {

		SpecificitiesR specif = new SpecificitiesR(lt);

		// System.out.println("Specif N part: "+specif.getNbrPart()); //$NON-NLS-1$

		// System.out.println("Specif N lines number: "+specif.getSpecificitesIndex().length); //$NON-NLS-1$

		String[] specifrownames = specif.getRowNames().asStringsArray();

		double[][] scores = specif.getScores();

		// System.out.println("Nb specif result: "+specif.getSpecificitesIndex().length);

		int iimax = Math.min(specifrownames.length, scores.length);

		for (int ii = 0; ii < iimax; ii++) { // counts.keySet())

			String signaturestr = keysToString.get(specifrownames[ii]);

			ArrayList<String> props = new ArrayList<>();

			if (pProperties.size() > 1) {

				String[] splited = specifrownames[ii].split("_", pProperties.size()); //$NON-NLS-1$

				for (int p = 0; p < pProperties.size(); p++) {

					props.add(splited[p]);

				}

						indexfreqs.get(specifrownames[ii]), scores[ii][1], // freq

						((float) (distances.get(signaturestr) / counts.get(signaturestr))), // mean distance

						-1);

				// select the line

				if (cline.freq >= this.pFminFilter && cline.nbocc >= this.pFCoocFilter && cline.score >= 0 && cline.score >= this.pScoreMinFilter) {

					if (cline.score >= Integer.MAX_VALUE - 5) {

		}

		return true;

	}

	/**

	 * Step query limits build the context, anticontext and keyword queries using the context size and xincludepivot and structuralLimit parameters

	 *

	 */

	public boolean stepQueryLimits() {

		// structural context

		String fixedQuery = CQLQuery.fixQuery(pQuery.getQueryString());

		if (pStructuralUnitLimit != null) {

			String tempquery = ""; //$NON-NLS-1$

			String lname = pStructuralUnitLimit.getName();

			// test if there is a left context

			if (pMinLeftContextSize > 0) {

				tempquery += "(<" + lname + ">[]* </" + lname + ">){" + (pMaxLeftContextSize) + "," + (pMaxLeftContextSize) + "}"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$

			}

			// (<p>[]*</p>){0, 50} "je" (<p>[]*</p>){0, 50}

			tempquery += " <" + lname + ">[]* " + fixedQuery + " []* </" + lname + "> "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$

			// test if there is a right context

			if (pMinRightContextSize > 0) {

				tempquery += "(<" + lname + ">[]* </" + lname + ">){" + (pMaxRightContextSize) + "," + (pMaxRightContextSize) + "}";  //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$

			}

			this.contextQuery = new CQLQuery(tempquery);

			if (pIncludeXpivot) {

				String anticontextquerystring = ""; //$NON-NLS-1$

				// minleft = 2..N

			}

		}

		else { // word context

			String tempquery = ""; //$NON-NLS-1$

			if (pMinLeftContextSize > 0) { // test if there is a left context

				tempquery += "[]{" + pMaxLeftContextSize + "," + pMaxLeftContextSize + "} "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			}

			tempquery += fixedQuery;

			if (pMinRightContextSize > 0) { // test if there is a right context

				tempquery += " []{" + pMaxRightContextSize + "," + pMaxRightContextSize + "} "; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			}

		}

		return true;

	}

	/**

	 * To txt.

	 *

	public boolean toTxt(File outfile, String encoding) throws Exception {

		return toTxt(outfile, encoding, "\t", ""); //$NON-NLS-1$ //$NON-NLS-2$

	}

	/**

	 * To txt.

	 *

		}

		return true;

	}

	/**

	 * To txt.

	 *

			Log.severe(TXMCoreMessages.bind(TXMCoreMessages.error_error, e));

			return false;

		}

		return true;

	}

	/**

	 * Sets the query.

	 * 

	public void setQuery(CQLQuery query) {

		this.pQuery = query;

	}

	/**

	 * Creates a CQL query string from the specified lines.

	 *

	 * @return the query

	 */

	public String createQuery(List<CLine> lines) {

		if (this.getQuery().isEmpty()) {

			return ""; //$NON-NLS-1$

		}

		int nbProps = this.getProperties().size();

		List<WordProperty> props = this.getProperties();

		for (int p = 0; p < nbProps; p++) {

			}

		}

		query += "] "; //$NON-NLS-1$

		int maxempan = Math.max(this.getMaxLeft(), this.getMaxRight());

		if (this.getIncludeXPivot() && maxempan == 0) maxempan = 1;

		String maxempanstr = "within " + maxempan + " "; //$NON-NLS-1$ //$NON-NLS-2$

		if (this.getStructuralUnitLimit() != null) maxempanstr += this.getStructuralUnitLimit().getName();

		String pquery = CQLQuery.fixQuery(this.getQuery().getQueryString());

		if (this.getMaxLeft() == 0) {

			query = "" + pquery + " []* " + query + " " + maxempanstr; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

		else {

			query = "(" + pquery + " []* " + query + ") | (" + query + " []* " + pquery + ") " + maxempanstr; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$

		}

		return query;

	}

	/**

	 * The Class CLine.

	 */

	public class CLine {

		/** The cooc. */

		Cooccurrence cooc;

		/** The distmoyenne. */

		public float distmoyenne;

		/** The freq. */

		public int freq;

		/** The id. */

		public int id;

		/** The mode. */

		public long mode;

		/** The nbocc. */

		public int nbocc;

		/** The occ. */

		public String occ;

		/** The props. */

		public List<String> props;

		/** The score. */

		public double score;

		/**

		 * Instantiates a new c line.

		 *

			this.mode = mode;

			this.cooc = cooc;

		}

		/**

		 * Adds the txt sep.

		 *

		private String addTxtSep(String str, String sep) {

			return sep + str.replace(sep, sep + sep) + sep;

		}

		/**

		 * Gets the cooc.

		 *

		public Cooccurrence getCooc() {

			return cooc;

		}

		/**

		 * Resume.

		 *

			return addTxtSep("" + occ, txtseparator) //$NON-NLS-1$

					+ colseparator + freq + colseparator + nbocc + colseparator + score + colseparator + distmoyenne;

		}

		/**

		 * Sets the count and dist.

		 *

			this.nbocc = count;

			this.distmoyenne = dist;

		}

		/**

		 * Sets the freq.

		 *

		public void setFreq(int freq) {

			this.freq = freq;

		}

		/**

		 * Sets the score.

		 */

		public void setScore() {// FB == freq, R = nbocc

			this.score = (this.nbocc + this.freq + this.distmoyenne);

		}

		@Override

		public String toString() {

			return occ + CooccurrenceCoreMessages.fColon + freq + CooccurrenceCoreMessages.occColon + nbocc + CooccurrenceCoreMessages.scoreColon + score + CooccurrenceCoreMessages.meanDistColon

					+ distmoyenne + CooccurrenceCoreMessages.propertiesColon + props;