Laboratoire ICAR » Plateforme TXM

import org.txm.lexicon.core.corpusengine.cqp.Lexicon

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

	public static final String COOCQUERYFILTER = PREFERENCES_PREFIX + "cooc_query_filter"; //$NON-NLS-1$

	@Parameter(key=CooccurrencePreferences.COOCQUERYFILTER)

	/** The reference corpus to use = the R symbol that point to a matrix WordxFreqs. */

	String referenceCorpus;

	/** The scores. */

	HashMap<String, Double> scores;

	/** The symbol. */

	private String symbol;

	/** The writer. */

	private BufferedWriter writer;

		if (this.getParent() != null)

		else return this.getSimpleName();

//		pCooccurentQueryFilter = this.getStringParameterValue(CooccurrencePreferences.COOCQUERYFILTER);

//		pFCoocFilter = this.getIntParameterValue(CooccurrencePreferences.MIN_COUNT);

//		pFminFilter = this.getIntParameterValue(CooccurrencePreferences.MIN_FREQ);

//		pIncludeXpivot  = this.getBooleanParameterValue(CooccurrencePreferences.INCLUDE_X_PIVOT);

//		pMaxLeftContextSize = this.getIntParameterValue(CooccurrencePreferences.MAX_LEFT);

//		pMaxRightContextSize = this.getIntParameterValue(CooccurrencePreferences.MAX_RIGHT);

//		pMinLeftContextSize = this.getIntParameterValue(CooccurrencePreferences.MIN_LEFT);

//		pMinRightContextSize = this.getIntParameterValue(CooccurrencePreferences.MIN_RIGHT);

//		pScoreMinFilter = this.getIntParameterValue(CooccurrencePreferences.MIN_SCORE);

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

import java.io.IOException;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.serverException.CqiServerError;

	public TreeNode getRoot()

	{

		return treenodes;

	public boolean compute() throws CqiClientException, IOException, CqiServerError

	{

		if (treenodes.check())

		else

	@Override

	public boolean canCompute() {

		return corpus != null && properties != null && properties.size() > 0;

	}

	public boolean setParameters(TXMParameters parameters) {

		// TODO Auto-generated method stub

		System.err.println("Summary.setParameters(): not yet implemented.");

		return true;

	}

	@Override

	public boolean loadParameters() {

		// TODO Auto-generated method stub

		System.err.println("Summary.loadParameters(): not yet implemented.");

		return true;

	}

	@Override

	public boolean saveParameters() {

		// TODO Auto-generated method stub

		System.err.println("Summary.saveParameters(): not yet implemented.");

		return true;

	}

	@Override

	protected boolean _compute() throws Exception {

		System.err.println("Summary.compute() not implemented");

		return false;

	}

	@Override

	public void setProperties(ArrayList<StructuralUnitProperty> properties2) {

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

			buffer.append("Result: " + result.toString() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$

			buffer.append("Direct children count = " + result.getResults().size() + ", direct visible children count = " + result.getChildren(true).size() + ", children count = " + result.getDeepChildren().size());

			for (Property p : selectedProps)

				if (p instanceof StructuralUnitProperty)

import org.txm.lexicon.core.corpusengine.cqp.Lexicon;

		str.append("Path = " +  preferences .absolutePath() + ")\n");

	public static final DateFormat ID_TIME_FORMAT = new SimpleDateFormat("YYMMDD");

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2016-09-19 10:31:00 +0200 (Mon, 19 Sep 2016) $

// $LastChangedRevision: 3298 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.lexicon.core.corpusengine.cqp;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.io.UnsupportedEncodingException;

import java.util.Arrays;

import java.util.Map;

import org.eclipse.core.runtime.IProgressMonitor;

import org.txm.core.messages.TXMCoreMessages;

import org.txm.core.results.TXMParameters;

import org.txm.core.results.TXMResult;

import org.txm.lexicon.core.messages.LexiconCoreMessages;

import org.txm.searchengine.cqp.ICqiClient;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.Subcorpus;

import org.txm.searchengine.cqp.corpus.query.Query;

import org.txm.statsengine.core.StatException;

import org.txm.statsengine.core.data.Vector;

import org.txm.statsengine.r.core.data.VectorImpl;

import org.txm.utils.logger.Log;

// TODO: Auto-generated Javadoc

//  TODO should be put into stat.data package ?

/**

 * Represent a frequency list according to a {@link Corpus} (or a.

 *

 * {@link Subcorpus}) and a {@link Property}.

 * @author sloiseau

 */

public class Lexicon extends TXMResult {

	/** The nolex. */

	protected static int nolex = 1;

	/** The prefix r. */

	protected static String prefixR = "Lexicon_"; //$NON-NLS-1$

	/** The forms. */

	private String[] forms;

	/** The freqs. */

	private int[] freqs;

	/** The ids. */

	private int[] ids;

	/** The number of tokens. */

	int numberOfTokens = -1;

	/** The property. */

	private Property pProperty;

	/** The symbol. */

	private String symbol;

	/** The writer. */

	private OutputStreamWriter writer;

	private Corpus corpus;

//	/**

//	 * Find or build a lexicon given a Corpus (MainCorpus or SubCorpus).

//	 * 

//	 * @param corpus

//	 * @param property

//	 * @return a Lexicon. May return null if the lexicon forms or freqs are null.

//	 * @throws Exception 

//	 */

//	public static Lexicon getLexicon(Corpus corpus, Property property) throws Exception {

//		HashSet<Object> results = corpus.getStoredData(Lexicon.class);

//		for (Object result : results) {

//			Lexicon lex = (Lexicon)result;

//			if (lex.getProperty().equals(property)) {

//				return lex;

//			}

//		}

//

//		Lexicon lex = new Lexicon(corpus);

//		lex.setParameters(property);

//		if (lex.compute(null) && lex.getForms() != null && lex.getFreq() != null) {

//			corpus.storeData(lex);

//			return lex;

//		} else {

//			return null;

//		}

//	}

	public Lexicon(Corpus corpus) {

		super(corpus);

		this.corpus = corpus;

	}

	@Override

	public boolean saveParameters() throws Exception {

		// TODO Auto-generated method stub

		return true;

	}

	@Override

	public boolean loadParameters() throws Exception {

		// TODO Auto-generated method stub

		return true;

	}

	@Override

	public void clean() {

		// TODO Auto-generated method stub

	}

	@Override

	public boolean canCompute() throws Exception {

		return corpus != null && pProperty != null;

	}

	@Override

	protected boolean _compute() throws Exception {

		if (corpus instanceof MainCorpus) {

			return computeWithMainCorpus((MainCorpus)corpus, pProperty, monitor);

		}

		else if (corpus instanceof Subcorpus) {

			return computewithSubCorpus((Subcorpus)corpus, pProperty, monitor);

		}

		else {

			System.out.println("Error: Lexicon parent is neither a Maincorpus nor a Subcorpus.");

			return false;

		}

	}

	/**

	 * Gets the lexicon relative to a given property.

	 * 

	 * @param property

	 *            the property

	 * 

	 * @return the lexicon

	 * 

	 * @throws CqiClientException

	 *             the cqi client exception

	 */

	protected boolean computeWithMainCorpus(MainCorpus corpus, Property property, IProgressMonitor monitor) throws CqiClientException {

		// System.out.println("in "+this.getCqpId()+" look for cached lexicon "+property);

		// System.out.println("not found");

		this.subTask("Computing lexicon size...");

		Log.finest(TXMCoreMessages.LEXICON + corpus.getName());

		int lexiconSize;

		try {

			lexiconSize = CorpusManager.getCorpusManager().getCqiClient().lexiconSize(property.getQualifiedName());

		} catch (Exception e) {

			throw new CqiClientException(e);

		}

		int[] ids = new int[lexiconSize];

		for (int i = 0; i < ids.length; i++) {

			ids[i] = i;

		}

		int[] freqs;

		try {

			this.subTask("Computing lexicon frequencies...");

			freqs = CorpusManager.getCorpusManager().getCqiClient().id2Freq(property.getQualifiedName(), ids);

		} catch (Exception e) {

			throw new CqiClientException(e);

		}

		init(corpus, property, freqs, ids);

		return true;

	}

	/**

	 * 

	 * @param corpus

	 * @param property

	 * @param monitor

	 * @return

	 * @throws CqiClientException

	 */

	protected boolean computewithSubCorpus(Subcorpus corpus, Property property, IProgressMonitor monitor) throws CqiClientException {

		//System.out.println("not found");

		Log.finest(TXMCoreMessages.SUBCORPUS_LEXICON + corpus.getName());

		long start = System.currentTimeMillis();

		int[][] fdist = null;

		Subcorpus tmp = null;

		try {

			this.subTask("Computing lexicon frequencies...");

			tmp = corpus.createSubcorpus(new Query("[]"), "S"+corpus.getNextSubcorpusCounter(), true); //$NON-NLS-1$

			if (tmp != null) {

				fdist = CorpusManager.getCorpusManager().getCqiClient().fdist1(

						tmp.getQualifiedCqpId(), 0,

						ICqiClient.CQI_CONST_FIELD_MATCH, property.getName());

				corpus.dropSubcorpus(tmp); // drop the subcorpus only if correctly created

			}

			//System.out.println("nb lines: "+fdist.length);

		} catch (Exception e) {

			throw new CqiClientException(e);

		} finally {

			if (tmp != null) {

				try {corpus.dropSubcorpus(tmp);}

				catch (Exception e2) {}

			}

		}

		int lexiconSize = fdist.length;

		int[] freqs = new int[lexiconSize];

		int[] ids = new int[lexiconSize];

		for (int i = 0; i < fdist.length; i++) {

			ids[i] = fdist[i][0];

			freqs[i] = fdist[i][1];

		}

		init(corpus, property, freqs, ids);

		return true;

	}

	/**

	 * Convert the Lexicon into a Vector object.

	 *

	 * @return the vector

	 * @throws StatException the stat exception

	 */

	public Vector asVector() throws StatException {

		String symbol = prefixR + (nolex++);

		VectorImpl v = new VectorImpl(freqs, symbol);

		v.setRNames(getForms());

		this.symbol = v.getSymbol();

		return v;

	}

	/**

	 * Compute number of tokens. / this.nbr

	 */

	private void computeNumberOfTokens() {

		numberOfTokens = 0;

		for (int i = 0; i < freqs.length; i++) {

			numberOfTokens += freqs[i];

			// System.out.println(numberOfTokens);

			// if (freqs[i] != 1) System.out.println(freqs[i]);

		}

	}

	@Override

	public boolean delete() {

		if (corpus != null) {

			corpus.removeData(this);

		}

		return true;

	}

	/**

	 * Dump lexicon forms and frequencies in a String.

	 *

	 * @param col the col

	 * @param txt the txt

	 * @return the string

	 */

	public String dump(String col, String txt) {

		StringBuffer buffer = new StringBuffer();

		getForms();

		for (int i = 0; i < forms.length; i++) {

			buffer.append(txt+ forms[i].replace(txt, txt+txt) + txt + col + freqs[i] + "\n"); //$NON-NLS-1$ 

		}

		return buffer.toString();

	}

	/* (non-Javadoc)

	 * @see java.lang.Object#equals(java.lang.Object)

	 */

	@Override

	public boolean equals(Object obj) {

		if (!(obj instanceof Lexicon)) {

			return false;

		}

		Lexicon other = (Lexicon) obj;

		if (other.nbrOfType() != this.nbrOfType()) {

			return false;

		}

		return (Arrays.equals(freqs, other.getFreq()) && Arrays.equals(getForms(), other.getForms()));

	}

	/**

	 * The corpus or subcorpus this lexicon is build on.

	 * 

	 * @return the corpus

	 */

	public Corpus getCorpus() {

		return corpus;

	}

	public String getDetails() {

		return  this.corpus.getName() + " " + this.pProperty.getName(); //$NON-NLS-1$

	}

	//TODO: move this into a Lexicon chart renderer

	//	/**

	//	 * Draw a pareto graphic with this frequency list and record it into the

	//	 * provided filename into svg format.

	//	 *

	//	 * @param file where to save the pareto graphic.

	//	 * @return the pareto graphic

	//	 * @throws StatException if anything goes wrong.

	//	 */

	//	public void getParetoGraphic(File file) throws StatException {

	//		String rName = asVector().getSymbol();

	//		String expr = "pareto(" + rName + ")"; //$NON-NLS-1$ //$NON-NLS-2$

	//		try {

	//			RWorkspace.getRWorkspaceInstance().plot(file, expr, RDevice.SVG);

	//		} catch (Exception e) {

	//			throw new StatException(e);

	//		}

	//	}

	/**

	 * The dif ferent types in the lexicon, the type at the index <code>j</code>

	 * of this array have the frequency at index <code>j</code> in the array

	 * returned by {@link #getFreq()}.

	 * 

	 * @return types as an array of <code>String</code>

	 */

	public String[] getForms() {

		if (forms == null) {

			if(ids == null) {

				return new String[0];

			}

			try {

				forms = CorpusManager.getCorpusManager().getCqiClient().id2Str(pProperty.getQualifiedName(), ids);

			} catch (Exception e) {

				// TODO Auto-generated catch block

				org.txm.utils.logger.Log.printStackTrace(e);

			}

		}

		return forms;

	}

	/**

	 * The dif ferent types in the lexicon, the type at the index <code>j</code>

	 * of this array have the frequency at index <code>j</code> in the array

	 * returned by {@link #getFreq()}.

	 *

	 * @param number the number

	 * @return types as an array of <code>String</code>

	 */

	public String[] getForms(int number) {

		//System.out.println("Lexicon("+this.property+" get forms. number="+number+", ids len="+ids.length);

		if (forms == null) {

			try {

				number = Math.min(number, ids.length);

				if (number <= 0) {

					return new String[0];

				}

				int[] subpositions = new int[number];

				System.arraycopy(ids, 0, subpositions, 0, number);

				return CorpusManager.getCorpusManager().getCqiClient().id2Str(pProperty.getQualifiedName(), subpositions);

			} catch (Exception e) {

				// TODO Auto-generated catch block

				org.txm.utils.logger.Log.printStackTrace(e);

				return null;

			}

		} else {

			number = Math.min(number, ids.length);

			if (number <= 0) {

				return new String[0];

			}

			String[] subforms = new String[number];

			System.arraycopy(ids, 0, subforms, 0, number);

			return subforms;

		}

	}

	/**

	 * The dif ferent frequencies in the lexicon. See {@link #getForms()}.

	 * 

	 * @return frequencies as an array of <code>int</code>

	 */

	public int[] getFreq() {

		return freqs;

	}

	/**

	 * return the ids of the entries.

	 *

	 * @return types as an array of <code>String</code>

	 */

	public int[] getIds() {

		return ids;

	}

	public String getName() {

		try {

			return LexiconCoreMessages.RESULT_TYPE + ": " + this.corpus.getSimpleName() + ": " + this.getSimpleName();	

		}

		catch(Exception e) {

		}

		return ""; //$NON-NLS-1$

	}

	/**

	 * The property this lexicon is build on.

	 * 

	 * @return the property

	 */

	public Property getProperty() {

		return pProperty;

	}

	public String getSimpleName() {

		try {

			return this.getProperty().getName();

		}

		catch(Exception e) {

		}

		return "";

	}

	/**

	 * Gets the symbol.

	 *

	 * @return the symbol

	 */

	public String getSymbol() {

		return this.symbol;

	}

	/**

	 * Hack frequencies using a map to set forms and frequencies

	 *

	 * @param corpus the corpus

	 * @param pProperty the property

	 * @param map the map

	 * {@link Corpus#getLexicon(Property)} or

	 * {@link Subcorpus#getLexicon(Property)}.

	 */

	public boolean hack(Map<String, Integer> map) {

		if (map.size() != forms.length) return false;

		//super(corpus);

		int size = map.size();

		int[] freqs = new int[size];

		String[] forms = map.keySet().toArray(new String[] {});

		for (int i = 0; i < forms.length; i++) {

			freqs[i] = map.get(forms[i]);

		}

		this.freqs = freqs;

		return true;

	}

	/**

	 * Protected on purpose: should be accessed through others initializer.

	 *

	 * @param corpus the corpus

	 * @param property the property

	 * @param freq the freq

	 * @param ids the ids

	 * {@link Corpus#getLexicon(Property)} or

	 * {@link Subcorpus#getLexicon(Property)}.

	 */

	protected void init(TXMResult corpus, Property property, int[] freq, int[] ids) {

		if (freq.length != ids.length) {

			throw new IllegalArgumentException(LexiconCoreMessages.Lexicon_0);

		}

		this.freqs = freq;

		this.ids = ids;

		this.forms = null;

		this.pProperty = property;

		this.corpus = (Corpus) corpus;

	}

	/**

	 * Number of tokens (sum of all the frequencies) in the corpus.

	 * 

	 * @return the size of the corpus or subcorpus.

	 */

	public int nbrOfToken() {

		if (numberOfTokens <= 0) {

			computeNumberOfTokens();

		}

		return numberOfTokens;

	}

	/**

	 * Number of dif ferent types in the frequency list.

	 * 

	 * @return number of types in the corpus or subcorpus.

	 */

	public int nbrOfType() {

		return freqs.length;

	}

	public void setParameters(Property property) {

		this.pProperty = property;

	}

	@Override

	public boolean setParameters(TXMParameters parameters) {

		try {

			Property p = (Property) parameters.get("properties");

			this.setParameters(p);

		} catch (Exception e) {

			Log.printStackTrace(e);

			return false;

		}

		return true;

	}

	/**

	 * Sets the symbol.

	 *

	 * @param symbol the new symbol

	 */

	public void setSymbol(String symbol) {

		this.symbol = symbol;

	}

	@Override

	public String toString() {

		return LexiconCoreMessages.Lexicon_3 + getName();

	}

	/**

	 * To txt.

	 *

	 * @param outfile the outfile

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @return true, if successful

	 */

	@Deprecated

	public boolean toTxt(File outfile, String encoding, String colseparator, String txtseparator) {

		// NK: writer declared as class attribute to perform a clean if the operation is interrupted

		// OutputStreamWriter writer;

		try {

			this.writer = new OutputStreamWriter(new FileOutputStream(outfile),

					encoding); 

		} catch (UnsupportedEncodingException e1) {

			org.txm.utils.logger.Log.printStackTrace(e1);

			return false;

		} catch (FileNotFoundException e1) {

			org.txm.utils.logger.Log.printStackTrace(e1);

			return false;

		}

		try {

			writer.write(this.dump(colseparator, txtseparator));

			writer.close();

		} catch (IOException e) {

			org.txm.utils.logger.Log.printStackTrace(e);

			return false;

		}

		return true;

	}

	public void setProperty(Property property) {

		this.pProperty = property;

	}

}