Laboratoire ICAR » Plateforme TXM

#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/)

#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/)

 * Computes an index of a corpus or a partition.

public class PartitionIndex extends TXMResult {

	/** The counts. */

	protected LinkedHashMap<String, ArrayList<Integer>> counts = new LinkedHashMap<String, ArrayList<Integer>>();

	/** The currentpartid. */

	protected int currentpartid = 0;

	/** The current Fmax value. */

	protected int Fmax = 0;

	/** The current Fmin value. */

	protected int Fmin = 9999999;

	/** The current lines. */

	protected List<Line> lines = new ArrayList<Line>();

	/** The current number of lines. */

	protected int nTotalTokens = 0;

	/** The partnames. The corpus name if built with a Corpus or the parts names if built with a Partition */

	protected List<String> partnames = new ArrayList<String>();

	/** The writer. */

	// FIXME: should be in an exporter extension

	@Deprecated

	private BufferedWriter writer;

	 * Maximum frequency filter value.

	@Parameter(key = TXMPreferences.F_MAX)

	protected Integer pFmaxFilter;

	/**

	 * Minimum frequency filter value.

	 */

	@Parameter(key = TXMPreferences.F_MIN)

	protected Integer pFminFilter;

	 * Number of lines to display per page.

	 */

	@Parameter(key = TXMPreferences.N_LINES_PER_PAGE)

	protected Integer pNLinesPerPage;

	/**

	 * The word properties to display.

	 */

	@Parameter(key = TXMPreferences.UNIT_PROPERTIES)

	protected List<WordProperty> pProperties;

	/**

	 * The string used to separated property values.

	 */

	@Parameter(key = IndexPreferences.PROPERTIES_SEPARATOR)

	protected String pPropertiesSeparator;

	/**

	 * The CQP query.

	 */

	@Parameter(key = TXMPreferences.QUERY)

	protected IQuery pQuery;

	/**

	 * The line index of the current index page.

	 */

	@Parameter(key = IndexPreferences.N_TOP_INDEX)

	private Integer pTopIndex;

	/**

	 * The vmax filter value parameter.

	 */

	@Parameter(key = TXMPreferences.V_MAX)

	protected Integer pVmaxFilter;

	/**

		partnames.clear();

			partnames.add(part.getName());

		partnames.clear();

		partnames.clear();

			partnames.add(part.getName());

	 * Creates a CQL query string from the specified Index lines.

	 *

	 * @param lines

	 * @return the query

	 */

	public static String createQuery(List<Line> lines) {

		String query = ""; //$NON-NLS-1$

		if (lines.size() == 0) {

			return query;

		}

		Line line = lines.get(0);

		int nbToken = line.getUnitsProperties().get(0).size();

		int nbProps = line.getProperties().size();

		int nbLines = lines.size();

		List<Property> props = line.getProperties();

		for (int t = 0; t < nbToken; t++) {

			query += "["; //$NON-NLS-1$

			for (int p = 0; p < nbProps; p++) {

				if (props.get(p) instanceof StructuralUnitProperty) {

					query += "_." + ((StructuralUnitProperty) props.get(p)).getFullName() + "=\""; //$NON-NLS-1$ //$NON-NLS-2$

				}

				else {

					query += props.get(p) + "=\""; //$NON-NLS-1$

				}

				for (int l = 0; l < nbLines; l++) {

					line = lines.get(l);

					List<List<String>> values = line.getUnitsProperties();

					String s = values.get(p).get(t);

					s = CQLQuery.addBackSlash(s);

					query += s + "|"; //$NON-NLS-1$

				}

				query = query.substring(0, query.length() - 1);

				query += "\" & "; //$NON-NLS-1$

			}

			query = query.substring(0, query.length() - 3);

			query += "] "; //$NON-NLS-1$

		}

		query = query.substring(0, query.length() - 1);

		return query;

	}

	/**

	 * Creates a Query list from the specified Index lines.

	 * 

	 * @param lines

	 * @return

	 */

	public static List<CQLQuery> createQueries(List<Line> lines) {

		List<CQLQuery> queries = new ArrayList<CQLQuery>();

		for (Line line : lines) {

			String query = ""; //$NON-NLS-1$

			int nbToken = line.getUnitsProperties().get(0).size();

			int nbProps = line.getProperties().size();

			List<List<String>> values = line.getUnitsProperties();

			List<Property> props = line.getProperties();

			for (int t = 0; t < nbToken; t++) {

				query += "["; //$NON-NLS-1$

				for (int p = 0; p < nbProps; p++) {

					query += props.get(p).getName() + "=\""; //$NON-NLS-1$

					String s = values.get(p).get(t);

					s = CQLQuery.addBackSlash(s);

					query += s;

					query += "\" & "; //$NON-NLS-1$

				}

				query = query.substring(0, query.length() - 3);

				query += "] "; //$NON-NLS-1$

			}

			queries.add(new CQLQuery(query));

		}

		return queries;

	}

	/**

		HashMap<String, Integer> ref_counts = new HashMap<String, Integer>();

	@Override

	public boolean saveParameters() {

		this.saveParameter(TXMPreferences.UNIT_PROPERTIES, WordProperty.propertiesToString(pProperties));

		if (pQuery != null) {

			this.saveParameter(TXMPreferences.QUERY, pQuery.getQueryString());

		}

		return true;

	}

	@Override

	public boolean loadParameters() {

		this.pProperties = (List<WordProperty>) Property.stringToProperties(getCorpus(), this.getStringParameterValue(TXMPreferences.UNIT_PROPERTIES));

		this.pQuery = new CQLQuery(this.getStringParameterValue(TXMPreferences.QUERY));

		return true;

	}

	@Override

	public void clean() {

		try {

			if (this.writer != null) {

				this.writer.flush();

				this.writer.close();

			}

		}

		catch (IOException e) {

			org.txm.utils.logger.Log.printStackTrace(e);

		}

	}

	 * keep the vmax lines more frequents.

	 *

	 */

	public void cut() {

		if (pVmaxFilter == null) {

			return;

		}

		this.acquireSemaphore();

		Log.fine("Cutting Tmax=" + pVmaxFilter);

		// assume the lines are sorted

		// int before = lines.size();

		this.lines = this.lines.subList(0, Math.min(lines.size(), pVmaxFilter));

		this.updateFminFmax();

		this.releaseSemaphore();

	}

	/**

	/**

	 * Removes lines with frequency not in [Fmin,Fmax] range.

	 *

	 **/

	public void filterLines() {

		if (!(pFminFilter > 0 && pFmaxFilter > 0 && pFminFilter <= pFmaxFilter)) {

			return;

		}

		Log.fine("Filtering Fmin = " + pFminFilter + " and Fmax = " + pFmaxFilter); //$NON-NLS-1$ //$NON-NLS-2$

		for (int i = 0; i < lines.size(); i++) { // for each line

			Line line = lines.get(i);

			int f = line.getFrequency();

			if (f < pFminFilter) { // if its frequency is not in the interval, remove it

				nTotalTokens -= line.getFrequency();

				lines.remove(i);

				i--;

				continue; // no need to go further the line is removed

			}

			if (f > pFmaxFilter) { // if its frequency is not in the interval, remove it

				nTotalTokens -= line.getFrequency();

				lines.remove(i);

				i--;

			}

		}

		this.updateFminFmax();

	}

	/**

	 * Gets the filter fmax.

	 *

	 * @return the filter fmax

	 */

	public Integer getFilterFmax() {

		return pFmaxFilter;

	}

	 * Gets the filter fmin.

	 *

	 * @return the filter fmin

	 */

	public Integer getFilterFmin() {

		return pFminFilter;

	}

	/**

	 * Gets the filter vmax.

	 *

	 * @return the filter vmax

	 */

	public Integer getFilterVmax() {

		return pVmaxFilter;

	}

	/**

	 * Gets the fmax.

	 *

	 * @return the fmax

	 */

	public int getFmax() {

		return Fmax;

	}

	/**

	 * Gets the fmin.

	 *

	 * @return the fmin

	 */

	public int getFmin() {

		return Fmin;

	}

	/**

	/**

	 * return the lines from le "start"th one to the "end"th one.

	 *

	 * @param start the start

	 * @param end the end

	 * @return the lines

	 */

	public List<Line> getLines(int start, int end) {

		// long time = System.currentTimeMillis();

		List<Line> selectedLines = new ArrayList<Line>();

		if (lines.size() > 0) {

			start = Math.max(0, start);

			end = Math.min(end, lines.size());

			selectedLines = lines.subList(start, end);

			int p = 0;

			// for each property get the string values of the tokens

			for (Property property : pProperties) {

				int len = 0;

				for (Line l : selectedLines) {

					len += l.UnitsIds.get(p).length;

				}

				int[] indices = new int[len]; // build the array of indices

				len = 0;

				for (Line l : selectedLines) {

					int[] ids = l.UnitsIds.get(p);

					System.arraycopy(ids, 0, indices, len, ids.length);

					len += ids.length;

				}

				String[] strs = null;

				try {

					if (property instanceof StructuralUnitProperty) {

						strs = CorpusManager.getCorpusManager().getCqiClient().struc2Str(property.getQualifiedName(), indices);

					}

					else {

						strs = CorpusManager.getCorpusManager().getCqiClient().id2Str(property.getQualifiedName(), indices);

					}

				}

				catch (Exception e) {

					org.txm.utils.logger.Log.printStackTrace(e);

					return null;

				}

				len = 0;

				for (Line l : selectedLines) {

					int[] ids = l.UnitsIds.get(p);

					String[] lstr = new String[ids.length];

					System.arraycopy(strs, len, lstr, 0, ids.length);

					if (l.UnitsProperty.size() == pProperties.size()) continue; // the line is already initialized

					l.put(property, Arrays.asList(lstr));

					len += ids.length;

				}

				p++;

			}

		}

		// System.out.println("Time get lines "+(System.currentTimeMillis()-time));

		return selectedLines;

	}

	 * return all the lines of the index.

	 *

	 * @return the all lines

	 */

	public List<Line> getAllLines() {

		return getLines(0, lines.size());

	}

	/**

			int[] c = new int[partnames.size()];

			for (int i = 0; i < partnames.size(); i++) {

				return IndexCoreMessages.RESULT_TYPE + ": " + this.parent.getSimpleName() + ": " + this.getSimpleName();

		return partnames;

	/**

	 * Gets the properties.

	 *

	 * @return the properties

	 */

	public List<WordProperty> getProperties() {

		return this.pProperties;

	}

	/**

	 * 

	 * @return

	 */

	public String getPropertySeparator() {

		return pPropertiesSeparator;

	}

	/**

	 * Gets the query.

	 *

	 * @return the query used

	 */

	public IQuery getQuery() {

		return pQuery;

	}

	/**

	 * Gets the number of tokens found.

	 *

	 * @return the number of tokens returned by the selection

	 */

	public int getT() {

		return nTotalTokens;

	}

	 * 

	 * @return

	 */

	public int getTopIndex() {

		return pTopIndex;

	}

	/**

	 * Gets the v.

	 *

	 * @return the number of entries in the index

	 */

	public int getV() {

		return lines.size();

	}

	/**

	 * count tokens.

	 *

	 * @param corpus the corpus to scan

	 * @return true, if successful

	 * @throws CqiClientException

	 * @throws CqiServerError

	 * @throws IOException

	 */

	protected boolean scanCorpus(CQPCorpus corpus) throws Exception {

		// get the cqp result of the query

		// long time = System.currentTimeMillis();

		Selection result = null;

		if (pQuery instanceof CQLQuery) {

			result = corpus.query((CQLQuery) pQuery, "index", true); //$NON-NLS-1$

		}

		else {

			result = pQuery.getSearchEngine().query(corpus, pQuery, "index", true); //$NON-NLS-1$

		}

		boolean isTargetUsed = result.isTargetUsed();

		int nbresults = result.getNMatch();

		this.nTotalTokens += nbresults; // get number of tokens

		// System.out.println("nLines : "+nLines);

		List<? extends Match> matches = null;

		if (nbresults > 0) {

			matches = result.getMatches(0, nbresults - 1); // get the indexes sequences of result's tokens

		}

		else {

			matches = new ArrayList<Match>();

		}

		// count matches

		// time = System.currentTimeMillis();

		List<Integer> allpositions = new ArrayList<Integer>();

		for (int j = 0; j < nbresults; j++) {

			Match match = matches.get(j);

			// beginingOfKeywordsPositions.add(match.getStart()); // get the

			// first index

			// lengthOfKeywords.add(match.size());// get the last index

			if (isTargetUsed) {

				allpositions.add(match.getTarget());

			}

			else {

				for (int i = match.getStart(); i <= match.getEnd(); i++) {

					allpositions.add(i);

				}

			}

		}

		// System.out.println("get string data");

		// time = System.currentTimeMillis();

		// for (Property property : props) {// for each property get the

		// string values of the tokens

		// keywordsViewPropValues.put(property,

		// cache.get(property).getData(beginingOfKeywordsPositions,

		// lengthOfKeywords));

		// }

		// System.out.println("took "+(System.currentTimeMillis()-time));

		// System.out.println("get count data");

		int[] allpositionsarray = new int[allpositions.size()];

		int pcount = 0;

		for (int p : allpositions) {

			allpositionsarray[pcount++] = p;

		}

		// time = System.currentTimeMillis();

		HashMap<Property, int[]> propsId = new HashMap<Property, int[]>();

		for (Property property : pProperties) {

			try {

				if (property instanceof StructuralUnitProperty) {

					int[] structs = CorpusManager.getCorpusManager().getCqiClient().cpos2Struc(property.getQualifiedName(), allpositionsarray);

					propsId.put(property, structs);

				}

				else {

					int[] indices = CorpusManager.getCorpusManager().getCqiClient().cpos2Id(property.getQualifiedName(), allpositionsarray);

					propsId.put(property, indices);

				}

			}

			catch (Exception e) {

				org.txm.utils.logger.Log.printStackTrace(e);

				result.drop();

				return false;

			}

		}

		// System.out.println("Time recup indices "+(System.currentTimeMillis()-time));

		int currentIndex = 0;

		// time = System.currentTimeMillis();

		for (int i = 0; i < nbresults; i++) {

			Line line = new Line();

			Match match = matches.get(i);

			int size = match.size();

			if (isTargetUsed) {

				size = 1;

			}

			for (int p = 0; p < pProperties.size(); p++) {

				Property property = pProperties.get(p);

				int[] allprosids = propsId.get(property);

				int[] ids = new int[size];

				System.arraycopy(allprosids, currentIndex, ids, 0, size);

				line.putIds(property, ids);

			}

			currentIndex += size;

			String signature = line.getSignature();

			// if the counts contains the signature, increment its corresponding value

			if (counts.containsKey(signature)) {

				while (counts.get(signature).size() <= currentpartid) {

					counts.get(signature).add(0);

				}

				int c = counts.get(signature).get(currentpartid) + 1;

				counts.get(signature).set(currentpartid, c);

			}

			// else initialize count of the signature to 1

			else {

				// System.out.println("add new sign "+signature+" of line "+line.toString());

				ArrayList<Integer> tmp = new ArrayList<Integer>();

				for (int j = 0; j < currentpartid + 1; j++) {

					tmp.add(0);

				}

				counts.put(signature, tmp);

				counts.get(signature).set(currentpartid, 1);

				lines.add(line);

			}

		}

		result.drop();

		// System.out.println("Time count lines "+(System.currentTimeMillis()-time));

		// System.out.println("took "+(System.currentTimeMillis()-time));

		return true;

	}

	 * @param nLinesPerPage

	 */

	public void setNLinesPerPage(int nLinesPerPage) {

		this.pNLinesPerPage = Math.max(nLinesPerPage, 1);

	}

	/**

	 * 

	/**

	 * Sets the query.

	 * 

	 * @param query

	 */

	public void setQuery(CQLQuery query) {

		this.pQuery = query;

	}

	/**

	 * Sets the properties.

	 * 

	 * @param properties

	 */

	public void setProperties(List<WordProperty> properties) {

		this.pProperties = properties;

	}

	/**

	 * Sets the property.

	 * Clears all existing properties.

	 * 

	 * @param property

	 */

	public void setProperty(WordProperty property) {

		List<WordProperty> properties = new ArrayList<WordProperty>();

		properties.add(property);

		this.setProperties(properties);

	}

	public void setParameters(CQLQuery query, List<WordProperty> props, Integer filterFmin, Integer filterFmax, Integer filterVmax, Integer nLinesPerPage) {

		this.pQuery = query;

		this.pProperties = props;

		if (filterFmax != null) this.pFmaxFilter = filterFmax;

		if (filterFmin != null) this.pFminFilter = filterFmin;

		if (filterVmax != null) this.pVmaxFilter = filterVmax;

		if (nLinesPerPage != null) this.pNLinesPerPage = nLinesPerPage;

	}

	@Override

	public boolean setParameters(TXMParameters parameters) {

		try {

			List<WordProperty> props = (List<WordProperty>) parameters.get("props"); //$NON-NLS-1$

			CQLQuery query = (CQLQuery) parameters.get("query"); //$NON-NLS-1$

			Integer filterFmin = (Integer) parameters.get("filterFmin"); //$NON-NLS-1$

			Integer filterFmax = (Integer) parameters.get("filterFmax"); //$NON-NLS-1$

			Integer filterVmax = (Integer) parameters.get("filterVmax"); //$NON-NLS-1$

			Integer nLinesPerPage = (Integer) parameters.get("nLinesPerPage"); //$NON-NLS-1$

			this.setParameters(query, props, filterFmin, filterFmax, filterVmax, nLinesPerPage);

		}

		catch (Exception e) {

			Log.severe("Error while setting parameters of Index: " + e.getLocalizedMessage()); //$NON-NLS-1$

			Log.printStackTrace(e);

			return false;

		}

		return true;

	}

	public void setTopLine(int i) {

		pTopIndex = Math.max(i, 0);

	}

	public void setVMax(int maxFilter) {

		this.pVmaxFilter = maxFilter;

	}

	 * Sort lines.

	 *

	 * @param mode the mode

	 * @param reverse the reverse

	 */

	public void sortLines(LineComparator.SortMode mode, boolean reverse) {

		LineComparator lc = new LineComparator(mode, reverse);

		lc.initialize(this.getCorpus());

		Collections.sort(lines, lc);

		this.pTopIndex = 0; // return to the first page

	}

	/**

	 * dump the index in the console.

	 *

	 * @throws CqiClientException the cqi client exception

	 * @throws IOException Signals that an I/O exception has occurred.

	 */

	public void toConsole() throws CqiClientException, IOException {

		System.out.println(NLS.bind(IndexCoreMessages.consoleColonP0, (lines.size())));

		toConsole(0, lines.size());

	}

	/**

		if (partnames.size() > 1) {

			for (int j = 0; j < partnames.size(); j++) {

				header += "\t" + partnames.get(j); //$NON-NLS-1$

			if (partnames.size() > 1) {

				for (int j = 0; j < partnames.size(); j++) {

		if (partnames.size() > 1) {

			for (int j = 0; j < partnames.size(); j++) {

				header += colseparator + txtseparator + partnames.get(j).replace(txtseparator, txtseparator + txtseparator) + txtseparator;

			if (partnames.size() > 1) {

				for (int j = 0; j < partnames.size(); j++) {

	/**

	 * Write all the lines on a writer.

	 *

	 * @param outfile the outfile

	 * @param encoding the encoding

	 * @param colseparator the colseparator

	 * @param txtseparator the txtseparator

	 * @return true, if successful

	 */

	// FIXME: should be in an exporter extension

	@Deprecated

	public boolean toTxt(File outfile, String encoding, String colseparator, String txtseparator) {

		try {

			toTxt(outfile, 0, lines.size(), encoding, colseparator, txtseparator);

		}

		catch (Exception e) {

			Log.severe(TXMCoreMessages.bind(IndexCoreMessages.error_failedToExportLexiconColonP0, Log.toString(e)));

			return false;

		}

		return true;

	}

	/**

	 * checks all lines to update Fmin and Fmax.

	 */

	private void updateFminFmax() {

		Fmin = Integer.MAX_VALUE;

		Fmax = Integer.MIN_VALUE;

		for (int i = 0; i < lines.size(); i++) {

			Line line = lines.get(i);

			int f = line.getFrequency();

			if (f < Fmin) {

				Fmin = f;

			}

			if (f > Fmax) {

				Fmax = f;

			}

		}

	}

	protected LinkedHashMap<String, ArrayList<Integer>> counts = new LinkedHashMap<String, ArrayList<Integer>>();

	protected List<Line> lines = new ArrayList<Line>();

	protected final boolean _compute() throws Exception {

				lines = new ArrayList<Line>();

		List<CQLQuery> queries = new ArrayList<CQLQuery>();

		HashMap<String, Integer> ref_counts = new HashMap<String, Integer>();

		if (getCorpus() == null) {

		List<Line> selectedLines = new ArrayList<Line>();

			matches = new ArrayList<Match>();

		List<Integer> allpositions = new ArrayList<Integer>();

		HashMap<Property, int[]> propsId = new HashMap<Property, int[]>();

				ArrayList<Integer> tmp = new ArrayList<Integer>();

		List<WordProperty> properties = new ArrayList<WordProperty>();