Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2016-09-19 10:31:00 +0200 (Mon, 19 Sep 2016) $

// $LastChangedRevision: 3298 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.index.core.functions;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.io.UnsupportedEncodingException;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Map;

import org.eclipse.core.runtime.IProgressMonitor;

import org.txm.core.messages.TXMCoreMessages;

import org.txm.core.preferences.TXMPreferences;

import org.txm.core.results.Parameter;

import org.txm.core.results.TXMParameters;

import org.txm.core.results.TXMResult;

import org.txm.index.core.messages.IndexCoreMessages;

import org.txm.searchengine.cqp.ICqiClient;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.corpus.Part;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.Subcorpus;

import org.txm.searchengine.cqp.corpus.query.CQLQuery;

import org.txm.statsengine.core.StatException;

import org.txm.statsengine.core.data.Vector;

import org.txm.statsengine.r.core.data.VectorImpl;

import org.txm.utils.logger.Log;

//  TODO should be put into stat.data package ?

/**

 * Represent a frequency list according to a {@link Corpus} (or a.

 *

 * {@link Subcorpus}) and a {@link Property}.

 * 

 * @author sjacquot

 * @author sloiseau

 */

public class ___Lexicon2 extends Index {

	/** The nolex. */

	protected static int nolex = 1;

	/** The prefix r. */

	protected static String prefixR = "Lexicon_"; //$NON-NLS-1$

	/** The forms. */

	private String[] forms;

	/** The freqs. */

	private int[] freqs;

	/** The ids. */

	private int[] ids;

	/** The number of tokens. */

	int numberOfTokens = -1;

	/** The symbol. */

	private String symbol;

	/** The writer. */

	private OutputStreamWriter writer;

	/**

	 * The property.

	 */

	@Parameter(key=TXMPreferences.UNIT_PROPERTY)

	protected Property pProperty;

	/**

	 * Creates a not computed lexicon from the specified corpus.

	 * @param parent

	 */

	public ___Lexicon2(TXMResult parent)	{

		super(parent);

	}

	/**

	 * Creates a not computed lexicon from the specified uuid node name.

	 * @param uuid

	 */

	public ___Lexicon2(String uuid)	{

		super(uuid);

	}

	/**

	 * 

	 * @param uuid

	 * @param parent

	 */

	public ___Lexicon2(String uuid, TXMResult parent)	{

		super(uuid, parent);

	}

	@Override

	public Corpus getParent() {

		return (Corpus)super.getParent();

	}

	@Override

	public boolean loadParameters() {

		super.loadParameters();

		try {

			//this.persistable = false; // FIXME: remove that later

			String p = this.getStringParameterValue(TXMPreferences.UNIT_PROPERTY);

			if (p != null && p.length() > 0) {

				Corpus parent = getParent();

				if (parent != null) {

					this.pProperty = parent.getProperty(p);

				}

				else {

					return false;

				}

			}

		}

		catch (CqiClientException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

			return false;

		}

		return true;

	}

	@Override

	public boolean saveParameters() {

		super.saveParameters();

		// FIXME: became useless? check that it's well automated by autoSaveParametersFromAnnotations()?

		//this.saveParameter(TXMPreferences.UNIT_PROPERTY, this.pProperty);

		return true;

	}

	@Override

	public void clean() {

		super.clean();

		// TODO Auto-generated method stub

	}

	@Override

	public boolean canCompute() {

		return super.canCompute() && (pProperty != null);

	}

	@Override

	protected boolean _compute() throws Exception {

		if (this.getParent() instanceof MainCorpus) {

			return computeWithMainCorpus((MainCorpus)this.getParent(), pProperty, monitor);

		}

		else if (this.getParent() instanceof Subcorpus) {

			return computewithSubCorpus((Subcorpus)this.getParent(), pProperty, monitor);

		}

		else {

			Log.severe("Error: Lexicon parent is neither a Maincorpus nor a Subcorpus."); //$NON-NLS-1$

			return false;

		}

	}

	/**

	 * Gets the lexicon relative to a given property.

	 * 

	 * @param property

	 *            the property

	 * 

	 * @return the lexicon

	 * 

	 * @throws CqiClientException

	 *             the cqi client exception

	 */

	protected boolean computeWithMainCorpus(MainCorpus corpus, Property property, IProgressMonitor monitor) throws CqiClientException {

		// System.out.println("in "+this.getCqpId()+" look for cached lexicon "+property);

		// System.out.println("not found");

		this.subTask("Computing lexicon size...");

		Log.finest(IndexCoreMessages.LEXICON_RESULT_TYPE + corpus.getName());

		int lexiconSize;

		try {

			lexiconSize = CorpusManager.getCorpusManager().getCqiClient().lexiconSize(property.getQualifiedName());

		} catch (Exception e) {

			throw new CqiClientException(e);

		}

		int[] ids = new int[lexiconSize];

		for (int i = 0; i < ids.length; i++) {

			ids[i] = i;

		}

		int[] freqs;

		try {

			this.subTask("Computing lexicon frequencies...");

			freqs = CorpusManager.getCorpusManager().getCqiClient().id2Freq(property.getQualifiedName(), ids);

		} catch (Exception e) {

			throw new CqiClientException(e);

		}

		this.init(corpus, property, freqs, ids);

		return true;

	}

	/**

	 * 

	 * @param corpus

	 * @param property

	 * @param monitor

	 * @return

	 * @throws CqiClientException

	 */

	// FIXME: why this method needs to create and delete some new subcorpus???? the computing can't be done directly on the corpus argument???

	// eg. dist = CorpusManager.getCorpusManager().getCqiClient().fdist1(corpus.getQualifiedCqpId(), 0,	ICqiClient.CQI_CONST_FIELD_MATCH, property.getName());

	protected boolean computewithSubCorpus(Subcorpus corpus, Property property, IProgressMonitor monitor) throws CqiClientException {

		if (corpus instanceof Part) {

			this.persistable = false;

		}

		//System.out.println("not found");

		Log.finest(TXMCoreMessages.SUBCORPUS_LEXICON + corpus.getName());

		//long start = System.currentTimeMillis();

		int[][] fdist = null;

		Subcorpus tmp = null;

		try {

			this.subTask("Computing lexicon frequencies...");

			tmp = corpus.createSubcorpus(new CQLQuery("[]"), "S" + corpus.getNextSubcorpusCounter(), true); //$NON-NLS-1$

			if (tmp != null) {

				fdist = CorpusManager.getCorpusManager().getCqiClient().fdist1(tmp.getQualifiedCqpId(), 0, ICqiClient.CQI_CONST_FIELD_MATCH, property.getName());

				corpus.dropSubcorpus(tmp);

				tmp.delete();

			}

			//System.out.println("nb lines: "+fdist.length);

		} catch (Exception e) {

			throw new CqiClientException(e);

		} finally {

			if (tmp != null) {

				try {

					corpus.dropSubcorpus(tmp);

					tmp.delete();

				}

				catch (Exception e2) {

				}

			}

		}

		int lexiconSize = fdist.length;

		int[] freqs = new int[lexiconSize];

		int[] ids = new int[lexiconSize];

		for (int i = 0; i < fdist.length; i++) {

			ids[i] = fdist[i][0];

			freqs[i] = fdist[i][1];

		}

		init(corpus, property, freqs, ids);

		return true;

	}

	/**

	 * Convert the Lexicon into a Vector object.

	 *

	 * @return the vector

	 * @throws StatException the stat exception

	 */

	public Vector asVector() throws StatException {

		String symbol = prefixR + (nolex++);

		VectorImpl v = new VectorImpl(freqs, symbol);

		v.setRNames(getForms());

		this.symbol = v.getSymbol();

		return v;

	}

	/**

	 * Compute number of tokens. / this.nbr

	 */

	private void computeNumberOfTokens() {

		numberOfTokens = 0;

		for (int i = 0; i < freqs.length; i++) {

			numberOfTokens += freqs[i];

		}

	}

	@Override

	public boolean delete() {

		super.delete();

		// FIXME: deprecated + anyway if it's still useful after the TXMObject nerf it should be in the clean() method

		if (this.parent != null) {

			this.getParent().removeData(this);

		}

		// end of fixme 

		return true;

	}

	/**

	 * Dump lexicon forms and frequencies in a String.

	 *

	 * @param col the col

	 * @param txt the txt

	 * @return the string

	 */

	public String dump(String col, String txt) {

		StringBuffer buffer = new StringBuffer();

		getForms();

		for (int i = 0; i < forms.length; i++) {

			buffer.append(txt + forms[i].replace(txt, txt + txt) + txt + col + freqs[i] + "\n"); //$NON-NLS-1$ 

		}

		return buffer.toString();

	}

	/* (non-Javadoc)

	 * @see java.lang.Object#equals(java.lang.Object)

	 */

	@Override

	public boolean equals(Object obj) {

		if (!(obj instanceof ___Lexicon2)) {

			return false;

		}

		___Lexicon2 other = (___Lexicon2) obj;

		if (other.nbrOfType() != this.nbrOfType()) {

			return false;

		}

		return (Arrays.equals(freqs, other.getFreq()) && Arrays.equals(getForms(), other.getForms()));

	}

	public String getDetails() {

		try {

			return this.getParent().getName() + " " + this.pProperty.getName(); //$NON-NLS-1$

		}

		catch (Exception e) {

			return this.getCurrentName();

		}

	}

	//TODO: move this into a Lexicon chart renderer

	//	/**

	//	 * Draw a pareto graphic with this frequency list and record it into the

	//	 * provided filename into svg format.

	//	 *

	//	 * @param file where to save the pareto graphic.

	//	 * @return the pareto graphic

	//	 * @throws StatException if anything goes wrong.

	//	 */

	//	public void getParetoGraphic(File file) throws StatException {

	//		String rName = asVector().getSymbol();

	//		String expr = "pareto(" + rName + ")"; //$NON-NLS-1$ //$NON-NLS-2$

	//		try {

	//			RWorkspace.getRWorkspaceInstance().plot(file, expr, RDevice.SVG);

	//		} catch (Exception e) {

	//			throw new StatException(e);

	//		}

	//	}

	/**

	 * The dif ferent types in the lexicon, the type at the index <code>j</code>

	 * of this array have the frequency at index <code>j</code> in the array

	 * returned by {@link #getFreq()}.

	 * 

	 * @return types as an array of <code>String</code>

	 */

	public String[] getForms() {

		if (forms == null) {

			if(ids == null) {

				return new String[0];

			}

			try {

				forms = CorpusManager.getCorpusManager().getCqiClient().id2Str(pProperty.getQualifiedName(), ids);

			} catch (Exception e) {

				// TODO Auto-generated catch block

				org.txm.utils.logger.Log.printStackTrace(e);

			}

		}

		return forms;

	}

	/**

	 * The different types in the lexicon, the type at the index <code>j</code>

	 * of this array have the frequency at index <code>j</code> in the array

	 * returned by {@link #getFreq()}.

	 *

	 * @param number the number

	 * @return types as an array of <code>String</code>

	 */

	public String[] getForms(int number) {

		//System.out.println("Lexicon("+this.property+" get forms. number="+number+", ids len="+ids.length);

		if (forms == null) {

			try {

				number = Math.min(number, ids.length);

				if (number <= 0) {

					return new String[0];

				}

				int[] subpositions = new int[number];

				System.arraycopy(ids, 0, subpositions, 0, number);

				return CorpusManager.getCorpusManager().getCqiClient().id2Str(pProperty.getQualifiedName(), subpositions);

			} catch (Exception e) {

				// TODO Auto-generated catch block

				org.txm.utils.logger.Log.printStackTrace(e);

				return null;

			}

		} else {

			number = Math.min(number, ids.length);

			if (number <= 0) {

				return new String[0];

			}

			String[] subforms = new String[number];

			System.arraycopy(ids, 0, subforms, 0, number);

			return subforms;

		}

	}

	/**

	 * The dif ferent frequencies in the lexicon. See {@link #getForms()}.

	 * 

	 * @return frequencies as an array of <code>int</code>

	 */

	public int[] getFreq() {

		return freqs;

	}

	/**

	 * return the ids of the entries.

	 *

	 * @return types as an array of <code>String</code>

	 */

	public int[] getIds() {

		return ids;

	}

	@Override

	public String getName() {

		try {

			return IndexCoreMessages.LEXICON_RESULT_TYPE + ": " + this.parent.getSimpleName() + ": " + this.getSimpleName();	

		}

		catch(Exception e) {

		}

		return ""; //$NON-NLS-1$

	}

	/**

	 * The property this lexicon is build on.

	 * 

	 * @return the property

	 */

	public Property getProperty() {

		return pProperty;

	}

	@Override

	public String getSimpleName() {

		try {

			return this.getProperty().getName();

		}

		catch(Exception e) {

			return this.getEmptyName();

		}

	}

	/**

	 * Gets the symbol.

	 *

	 * @return the symbol

	 */

	public String getSymbol() {

		return this.symbol;

	}

	/**

	 * Hack frequencies using a map to set forms and frequencies

	 *

	 * @param corpus the corpus

	 * @param pProperty the property

	 * @param map the map

	 * {@link Corpus#getLexicon(Property)} or

	 * {@link Subcorpus#getLexicon(Property)}.

	 */

	public boolean hack(Map<String, Integer> map) {

		if (map.size() != forms.length) {

			return false;

		}

		//super(corpus);

		int size = map.size();

		int[] freqs = new int[size];

		String[] forms = map.keySet().toArray(new String[] {});

		for (int i = 0; i < forms.length; i++) {

			freqs[i] = map.get(forms[i]);

		}

		this.freqs = freqs;

		return true;

	}

	/**

	 * Protected on purpose: should be accessed through others initializer.

	 *

	 * @param corpus the corpus

	 * @param property the property

	 * @param freq the freq

	 * @param ids the ids

	 * {@link Corpus#getLexicon(Property)} or

	 * {@link Subcorpus#getLexicon(Property)}.

	 */

	protected void init(TXMResult corpus, Property property, int[] freq, int[] ids) {

		if (freq.length != ids.length) {

			throw new IllegalArgumentException(IndexCoreMessages.Lexicon_0);

		}

		this.freqs = freq;

		this.ids = ids;

		this.forms = null;

		this.pProperty = property;

	}

	/**

	 * Number of tokens (sum of all the frequencies) in the corpus.

	 * 

	 * @return the size of the corpus or subcorpus.

	 */

	public int nbrOfToken() {

		if (numberOfTokens <= 0) {

			computeNumberOfTokens();

		}

		return numberOfTokens;

	}

	/**

	 * Number of different types in the frequency list.

	 * 

	 * @return number of types in the corpus or subcorpus.

	 */

	public int nbrOfType() {

		try {

			return freqs.length;

		}

		catch (Exception e) {

			return 0;

		}

	}

	public void setParameters(Property property) {

		this.pProperty = property;

	}

	@Override

	public boolean setParameters(TXMParameters parameters) {

		try {

			Property p = (Property) parameters.get("properties");

			this.setParameters(p);

		} catch (Exception e) {

			Log.printStackTrace(e);

			return false;

		}

		return true;

	}

	/**

	 * Sets the symbol.

	 *

	 * @param symbol the new symbol

	 */

	public void setSymbol(String symbol) {

		this.symbol = symbol;

	}

	@Override

	public String toString() {

		return IndexCoreMessages.Lexicon_3 + this.getCurrentName();

	}

	/**

	 * To txt.

	 *

	 * @param outfile the outfile

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @return true, if successful

	 */

	@Deprecated

	public boolean toTxt(File outfile, String encoding, String colseparator, String txtseparator) {

		// NK: writer declared as class attribute to perform a clean if the operation is interrupted

		// OutputStreamWriter writer;

		try {

			this.writer = new OutputStreamWriter(new FileOutputStream(outfile),

					encoding); 

		} catch (UnsupportedEncodingException e1) {

			org.txm.utils.logger.Log.printStackTrace(e1);

			return false;

		} catch (FileNotFoundException e1) {

			org.txm.utils.logger.Log.printStackTrace(e1);

			return false;

		}

		try {

			writer.write(this.dump(colseparator, txtseparator));

			writer.close();

		} catch (IOException e) {

			org.txm.utils.logger.Log.printStackTrace(e);

			return false;

		}

		return true;

	}

	/**

	 * Sets the unit property.

	 * @param property the unit property

	 */

	public void setProperty(Property property) {

		this.pProperty = property;

	}

	/**

	 * Gets a Lexicon from the specified corpus.

	 * If a Lexicon child exists in the Corpus fro the specified property, returns it otherwise creates and computes a new Lexicon.

	 * @param corpus

	 * @param property

	 * @return

	 * @throws Exception

	 */

	public static ___Lexicon2 getLexicon(Corpus corpus, Property property, IProgressMonitor monitor) throws Exception {

		___Lexicon2 lexicon = null;

		// recycling parent Lexicon if exists 

		ArrayList<___Lexicon2> partLexicons = (ArrayList<___Lexicon2>)corpus.getChildren(___Lexicon2.class);

		for (int i = 0; i < partLexicons.size(); i++) {

			if(partLexicons.get(i).getProperty() == property)	{

				lexicon = partLexicons.get(i);

				break;

			}

		}

		// creating new Lexicon

		if(lexicon == null || lexicon.getProperty() != property)	{

			lexicon = new ___Lexicon2(corpus);

			lexicon.setProperty(property);

		}

		if(!lexicon.hasBeenComputedOnce())	{

			lexicon.compute(monitor);

		}

		return lexicon;

	}

//	/**

//	 * Find or build a lexicon given a Corpus (MainCorpus or SubCorpus).

//	 * 

//	 * @param corpus

//	 * @param property

//	 * @return a Lexicon. May return null if the lexicon forms or freqs are null.

//	 * @throws Exception 

//	 */

//	public static Lexicon getLexicon(Corpus corpus, Property property) throws Exception {

//		HashSet<Object> results = corpus.getStoredData(Lexicon.class);

//		for (Object result : results) {

//			Lexicon lex = (Lexicon)result;

//			if (lex.getProperty().equals(property)) {

//				return lex;

//			}

//		}

//

//		Lexicon lex = new Lexicon(corpus);

//		lex.setParameters(property);

//		if (lex.compute(null) && lex.getForms() != null && lex.getFreq() != null) {

//			corpus.storeData(lex);

//			return lex;

//		} else {

//			return null;

//		}

//	}

	@Override

	public String getResultype() {

		return IndexCoreMessages.LEXICON_RESULT_TYPE;

	}

}

 * Computes an index of a corpus or a partition. 

 * @author mdecorde

	// FIXME: should be in an exporter extension

	@Deprecated

		if (parent instanceof Corpus && "[]".equals(pQuery.getQueryString()) && pProperties.size() == 1) {

			if (this.writer != null) {

	@Deprecated

	// FIXME: should be in an exporter extension

	@Deprecated

	// FIXME: should be in an exporter extension

	@Deprecated

	// FIXME: should be in an exporter extension

	@Deprecated

				return '[' + matcher.group(1) + '=' + '"' + matcher.group(2) + '"' + ']';