Laboratoire ICAR » Plateforme TXM

<?xml version="1.0" encoding="UTF-8"?>

<?eclipse version="3.4"?>

<plugin>

   <extension

         point="org.txm.searchengine.core.SearchEngine">

      <SearchEngine

            class="org.txm.searchengine.cqp.CQPEngine">

      </SearchEngine>

   </extension>

</plugin>

eclipse.preferences.version=1

org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled

org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7

org.eclipse.jdt.core.compiler.compliance=1.7

org.eclipse.jdt.core.compiler.problem.assertIdentifier=error

org.eclipse.jdt.core.compiler.problem.enumIdentifier=error

org.eclipse.jdt.core.compiler.source=1.7

<?xml version="1.0" encoding="UTF-8"?>

<classpath>

	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>

	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins">

		<accessrules>

			<accessrule kind="accessible" pattern="**"/>

		</accessrules>

	</classpathentry>

	<classpathentry kind="src" path="src"/>

	<classpathentry kind="output" path="bin"/>

</classpath>

Manifest-Version: 1.0

Bundle-ManifestVersion: 2

Bundle-Name: CQP

Bundle-SymbolicName: org.txm.searchengine.cqp.core;singleton:=true

Bundle-Version: 1.1.0.qualifier

Require-Bundle: org.eclipse.ui,

 org.eclipse.core.runtime,

 org.txm.core;bundle-version="0.7.0",

 org.txm.searchengine.core;bundle-version="1.0.0",

 org.txm.libs.cqp;bundle-version="1.0.0"

Bundle-RequiredExecutionEnvironment: JavaSE-1.6

Bundle-ClassPath: res/,

 .

Bundle-Vendor: Textometrie.org

Export-Package: org.txm.functions.diagnostic,

 org.txm.functions.internal,

 org.txm.functions.preview,

 org.txm.functions.selection,

 org.txm.functions.summary,

 org.txm.importer.cwb,

 org.txm.searchengine.cqp,

 org.txm.searchengine.cqp.clientExceptions,

 org.txm.searchengine.cqp.corpus,

 org.txm.searchengine.cqp.corpus.query,

 org.txm.searchengine.cqp.serverException,

 org.txm.test

<?xml version="1.0" encoding="UTF-8"?>

<projectDescription>

	<name>org.txm.searchengine.cqp.core</name>

	<comment></comment>

	<projects>

	</projects>

	<buildSpec>

		<buildCommand>

			<name>org.eclipse.jdt.core.javabuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

		<buildCommand>

			<name>org.eclipse.pde.ManifestBuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

		<buildCommand>

			<name>org.eclipse.pde.SchemaBuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

	</buildSpec>

	<natures>

		<nature>org.eclipse.pde.PluginNature</nature>

		<nature>org.eclipse.jdt.core.javanature</nature>

	</natures>

</projectDescription>

package cwbinstaller;

import org.eclipse.jface.resource.ImageDescriptor;

import org.eclipse.ui.plugin.AbstractUIPlugin;

import org.osgi.framework.BundleContext;

/**

 * The activator class controls the plug-in life cycle

 */

public class Activator extends AbstractUIPlugin {

	// The plug-in ID

	public static final String PLUGIN_ID = "CQP"; //$NON-NLS-1$

	// The shared instance

	private static Activator plugin;

	/**

	 * The constructor

	 */

	public Activator() {

	}

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.ui.plugin.AbstractUIPlugin#start(org.osgi.framework.BundleContext)

	 */

	public void start(BundleContext context) throws Exception {

		super.start(context);

		plugin = this;

	}

	/*

	 * (non-Javadoc)

	 * @see org.eclipse.ui.plugin.AbstractUIPlugin#stop(org.osgi.framework.BundleContext)

	 */

	public void stop(BundleContext context) throws Exception {

		plugin = null;

		super.stop(context);

	}

	/**

	 * Returns the shared instance

	 *

	 * @return the shared instance

	 */

	public static Activator getDefault() {

		return plugin;

	}

	/**

	 * Returns an image descriptor for the image file at the given

	 * plug-in relative path

	 *

	 * @param path the path

	 * @return the image descriptor

	 */

	public static ImageDescriptor getImageDescriptor(String path) {

		return imageDescriptorFromPlugin(PLUGIN_ID, path);

	}

}

package cwbinstaller.commands;

import java.io.File;

import org.eclipse.core.commands.ExecutionEvent;

import org.eclipse.core.commands.ExecutionException;

import org.eclipse.jface.util.Util;

import org.osgi.framework.Version;

import org.txm.core.preferences.TBXPreferences;

import org.txm.core.preferences.TXMPreferences;

import org.txm.utils.BundleUtils;

import org.txm.utils.logger.Log;

import cwbinstaller.CQPPreferences;

public class DoInstallStep extends org.txm.rcp.commands.TxmCommand {

	protected String name = "cwbbinaries.commands.DoInstallStep"; //$NON-NLS-1$

	@Override

	public Object execute(ExecutionEvent event) throws ExecutionException {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public void install() {	

	}

	@Override

	public void preInstall() {	

		String CQPFRAGMENT = "CQP."+System.getProperty("osgi.os");

		Log.info("CQP.DoInstallStep.install()");

		String saved = TXMPreferences.getString(CQPPreferences.VERSION, CQPPreferences.PREFERENCES_NODE);

		Version currentVersion = BundleUtils.getBundleVersion(CQPFRAGMENT); // the CQPFRAGMENT plugin contains the right version

		if (saved != null && saved.length() > 0) {

			Version savedVersion = new Version(saved);

			if (currentVersion.compareTo(savedVersion) <= 0) {

				Log.info("No post-installation of CQP to do");

				return; // nothing to do

			}

		}

		System.out.println("Updating CWB preferences for CQP version="+currentVersion);

		String os = "win";

		String ext = "";

		if (Util.isWindows()) { //$NON-NLS-1$

			if (System.getProperty("os.arch").contains("64")) { os = "win64"; } 

			else { os = "win32"; }

			ext = ".exe";

		} else if (Util.isMac()) { //$NON-NLS-1$

			os = "macosx";

		} else {

			os = "linux32";

			if (System.getProperty("os.arch").contains("64")) 

				os = "linux64";

		}

		File bundleDir = BundleUtils.getBundleFile(CQPFRAGMENT);

		if (bundleDir == null) {

			System.out.println("Error while retrieving TreeTaggerInstaller bundle directory.");

			return;

		}

		File cwbDir = new File(bundleDir, "res");

		File initFile = new File(cwbDir, "cqpserver.init");

		File OSDir = new File(cwbDir, os);

		File execFile = new File(OSDir, "cqpserver"+ext);

		TXMPreferences.putString(TBXPreferences.PREFERENCES_NODE, TBXPreferences.CQI_SERVER_PATH_TO_CQPLIB, OSDir.getAbsolutePath());

		TXMPreferences.putString(TBXPreferences.PREFERENCES_NODE, TBXPreferences.CQI_SERVER_PATH_TO_INIT_FILE, initFile.getAbsolutePath());

		TXMPreferences.putString(TBXPreferences.PREFERENCES_NODE, TBXPreferences.CQI_SERVER_PATH_TO_EXECUTABLE, execFile.getAbsolutePath());

		if (!Util.isWindows()) { 

			try {

				Log.info("Setting execution file rights to: "+OSDir.getAbsolutePath());

				Runtime.getRuntime().exec("chmod -R +x "+OSDir.getAbsolutePath()).waitFor();

			} catch (Exception e) {

				System.out.println("Error while setting execution file rights to: "+OSDir.getAbsolutePath());

				e.printStackTrace();

			}

		}

		if (!execFile.canExecute()) {

			System.out.println("File right setting error, cannot execute: "+execFile);

		}

		Log.warning("SearchEngine preferences set with: "+OSDir.getAbsolutePath()+ ", "+initFile.getAbsolutePath()+ " and "+execFile.getAbsolutePath());

		TXMPreferences.putString(CQPPreferences.PREFERENCES_NODE, CQPPreferences.VERSION, currentVersion.toString());

	}

}

package cwbinstaller;

import org.eclipse.core.runtime.preferences.DefaultScope;

import org.osgi.service.prefs.Preferences;

import org.txm.core.preferences.TXMPreferences;

public class CQPPreferences extends TXMPreferences {

	public static final String PREFERENCES_NODE = "cwbinstaller";

	public static final String VERSION = "cwbinstaller.commands.version";

	@Override

	public void initializeDefaultPreferences() {

		Preferences preferences = DefaultScope.INSTANCE.getNode(PREFERENCES_NODE);

		preferences.put(VERSION, "0.0");

	}

}

package org.txm.functions.internal;

import java.util.ArrayList;

import java.util.Collection;

import java.util.HashMap;

import java.util.List;

import org.txm.Toolbox;

import org.txm.core.messages.TXMCoreMessages;

import org.txm.core.results.TXMResult;

import org.txm.searchengine.cqp.CQPEngine;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.QueryResult;

import org.txm.searchengine.cqp.corpus.StructuralUnit;

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.searchengine.cqp.corpus.query.Query;

public class InternalView extends TXMResult  {

	Corpus corpus;

	List<Property> properties;

	StructuralUnit struct;

	List<Property> structProperties;

	int currentStruct = -1;

	int nmatches = 0;

	List<Match> matches = new ArrayList<Match>();

	public InternalView(Corpus corpus, List<Property> properties, StructuralUnit struct, List<Property> structProperties) throws CqiClientException {

		this.corpus = corpus;

		this.properties = properties;

		this.struct = struct;

		this.structProperties = structProperties;

		//find struct start-end

		Query query = new Query("<"+struct.getName()+"> [] expand to "+struct); //$NON-NLS-1$ //$NON-NLS-2$

		//System.out.println(query);

		QueryResult result = corpus.query(query, "test", false); //$NON-NLS-1$

		nmatches = result.getNMatch();

		if (nmatches > 0) {

			matches = result.getMatches();

		}

		result.drop();

		//System.out.println(matches);

	}

	public String getStructInfos() {

		try {

			if (structProperties.size() > 0) {

				String str = TXMCoreMessages.InternalView_3;

				for (Property sup : structProperties) {

					int[] array = {getCurrentSegmentNo()};

					str += " "+sup.getName()+"="+CQPEngine.getCqiClient().struc2Str(sup.getQualifiedName(), array)[0]; //$NON-NLS-1$ //$NON-NLS-2$

				}

				return str;

			}

		} catch (Exception e) {

			System.out.println(TXMCoreMessages.InternalView_6+e.getMessage());

			org.txm.utils.logger.Log.printStackTrace(e);

		}

		return TXMCoreMessages.InternalView_7;

	}

	public HashMap<Property, List<String>> getNext() throws CqiClientException {

		int next = currentStruct + 1;

		if (next < nmatches) {

			return getSegment(next);

		} 

		return new HashMap<Property, List<String>>();

	}

	public HashMap<Property, List<String>> getPrevious() throws CqiClientException {

		int previous = currentStruct - 1;

		if (previous >= 0 && previous < nmatches) {

			return getSegment(previous);

		}

		return new HashMap<Property, List<String>>();

	}

	public HashMap<Property, List<String>> getFirst() throws CqiClientException {

		if (0 < nmatches) {

			return getSegment(0);

		}

		return new HashMap<Property, List<String>>();

	}

	public HashMap<Property, List<String>> getLast() throws CqiClientException {

		if (nmatches - 1 >= 0) {

			return getSegment(nmatches - 1);

		}

		return new HashMap<Property, List<String>>();

	}

	public HashMap<Property, List<String>> getSegment(int i) throws CqiClientException {

		HashMap<Property, List<String>> rez = new HashMap<Property, List<String>>();

		if (i >= 0 && i < nmatches) {

			Match m = matches.get(i);

			Collection<? extends Integer> positions = m.getRange();

			int[] p = new int[positions.size()];

			int c = 0;

			for(int v : positions) p[c++] = v;

			for (Property prop : properties) {

				rez.put(prop, Match.getValuesForProperty(prop, p));

			}

			currentStruct = i;

		}

		return rez;

	}

	public void printNext() throws CqiClientException {

		HashMap<Property, List<String>> seg = getNext();

		if (seg.size() > 0)

			System.out.println(seg);

	}

	public void printPrevious() throws CqiClientException {

		HashMap<Property, List<String>> seg = getPrevious();

		if (seg.size() > 0)

			System.out.println(seg);

	}

	public void printSeg(HashMap<Property, List<String>> seg) {

		Collection<List<String>> values = seg.values();

		if(values.size() == 0) return;

		for(List<String> list : values) {

			for(String value : list)

				System.out.print("\t"+value); //$NON-NLS-1$

			System.out.println();

		}

		System.out.println();

	}

	public Corpus getCorpus() {

		return corpus;

	}

	public StructuralUnit getStructuralUnit() {

		return struct;

	}

	public List<Property> getProperties() {

		return properties;

	}

	public int getCurrentSegmentNo() {

		return currentStruct;

	}

	public int getNSegment() {

		return nmatches;

	}

	public int getSegmentLength() {

		if (matches.size() == 0) return 0;

		Match m = matches.get(currentStruct);

		return m.size();

	}

	public boolean isFirst() {

		return currentStruct == 0;

	}

	public boolean isLast() {

		return currentStruct == nmatches - 1;

	}

	public String getLocationTail() {

		return " / "+nmatches; //$NON-NLS-1$

	}

	public String getCurrentLocation() {

		return ""+(currentStruct+1); //$NON-NLS-1$

	}

	public String getDetails() {

		// TODO Auto-generated method stub

		return corpus.getName()+" "+struct.getName()+" "+(currentStruct+1); //$NON-NLS-1$ //$NON-NLS-2$

	}

	public void setProperties(List<Property> properties2) {

		this.properties = properties2;

	}

	public void setStructProperties(List<Property> properties2) {

		this.structProperties = properties2;

	}

	@Override

	public String getName() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public String getSimpleName() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public void clean() {

		// TODO Auto-generated method stub

	}

}

<html>

<body>

<p>Internal view</p>

</body>

</html>

package org.txm.functions.summary;

import java.io.IOException;

import java.util.ArrayList;

import java.util.Arrays;

import org.txm.Toolbox;

import org.txm.core.messages.TXMCoreMessages;

import org.txm.core.results.TXMResult;

import org.txm.objects.Edition;

import org.txm.objects.Page;

import org.txm.objects.Text;

import org.txm.searchengine.cqp.CQPEngine;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.QueryResult;

import org.txm.searchengine.cqp.corpus.StructuralUnitProperty;

import org.txm.searchengine.cqp.corpus.query.Query;

import org.txm.searchengine.cqp.serverException.CqiServerError;

public class Summary extends TXMResult  {

	MainCorpus corpus;

	TreeNode treenodes = new TreeNode("root"); //$NON-NLS-1$

	private ArrayList<StructuralUnitProperty> properties;

	public Summary(MainCorpus corpus)

	{

		this.corpus = corpus;

	}

	public TreeNode getRoot()

	{

		return treenodes;

	}

	public boolean init(ArrayList<StructuralUnitProperty> properties) throws CqiClientException, IOException, CqiServerError

	{

		this.properties = properties;

		treenodes.end = corpus.getSize();

		treenodes.id = corpus.getName();

		treenodes.start = 0;

		treenodes.parent = null;

		// get all type of structures

		for (int j = 0 ; j < properties.size() ; j++) {

			String suname = properties.get(j).getStructuralUnit().getName();

			//System.out.println("getting : "+properties.get(j).getFullName());

			QueryResult result = corpus.query(new Query("<"+properties.get(j).getFullName()+">[]"), "summary", false); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			int[] starts = result.getStarts();

			result.drop();

			result = corpus.query(new Query("[]</"+properties.get(j).getFullName()+">"), "summary", false); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			int[] ends = result.getStarts();

			result.drop();

			//System.out.println("starts: "+Arrays.toString(starts));

			//System.out.println("ends: "+Arrays.toString(ends));

			int[] strcutpos = CQPEngine.getCqiClient().cpos2Struc(properties.get(j).getQualifiedName(), starts);

			String[] infos = CQPEngine.getCqiClient().struc2Str(properties.get(j).getQualifiedName(), strcutpos);

			//allMatches.put(suname, list);

			if (starts.length != ends.length) {

				System.out.println(suname+": starts.length "+starts.length+" != ends.length "+ends.length); //$NON-NLS-1$ //$NON-NLS-2$

				return false;

			}

			for (int i = 0 ; i < starts.length ; i++) {

				//System.out.println("suname: "+suname+" start: "+m.getStart());

				TreeNode node = new TreeNode();

				node.start = starts[i];

				node.end = ends[i];

				node.struct = suname;

				node.id = infos[i];

				treenodes.add(node);

			}

		}

		//this.print();

		//System.out.println("sorting... "+treenodes.size()+" nodes");

		processNodeList(treenodes);

		//System.out.println("checking result..."+treenodes.size()+" nodes");

		if (treenodes.check())

			return true;

		else

			return false;

	}

	private void processNodeList(TreeNode nodes)

	{

		//System.out.println("+processing node list: "+nodes.size());

		if (nodes.size() == 0)

			return;

		// 1st loop

		//System.out.println("1st loop: "+nodes.size());

		for (int i = 0 ; i < nodes.size() ; i++) {

			TreeNode node = nodes.get(i);

			if (node.end == -1)

				System.out.println("ERROR: node.end = -1"); //$NON-NLS-1$

			for (int j = 0 ; j < nodes.size() ; j++) {

				if (j != i) {

					TreeNode n2 = nodes.get(j);

					if (n2.start >= node.start && n2.end <= node.end) {

						n2.parent = node;

						node.add(n2);

						nodes.remove(j);

						j--;

						if (j < i)

							i--;

					}

				}

			}

		}

		//process sublists

		//System.out.println("processing sublists: "+nodes.size());

		for (int i = 0 ; i < nodes.size() ; i++) {

			TreeNode node = nodes.get(i);

			processNodeList(node);

		}

	}

	public ArrayList<StructuralUnitProperty> getProperties()

	{

		return properties;

	}

	public void print()

	{

		for (TreeNode node : treenodes)

			System.out.println("\n"+node); //$NON-NLS-1$

	}

	public Page getDefaultEditionPage(TreeNode node) throws Exception

	{

		StructuralUnitProperty text_id = corpus.getStructuralUnit("text").getProperty("id"); //$NON-NLS-1$ //$NON-NLS-2$

		Property w_id =  corpus.getProperty("id"); //$NON-NLS-1$

		int[] positions = {node.getStart()};

		final String[] wordids = CQPEngine.getCqiClient().cpos2Str(w_id.getQualifiedName(), positions);

		int[] textidstruct = CQPEngine.getCqiClient().cpos2Struc(text_id.getQualifiedName(), positions);

		String[] textids = CQPEngine.getCqiClient().struc2Str(text_id.getQualifiedName(), textidstruct);

		if (wordids.length == 1 && textids.length == 1) {

			Text text = corpus.getText(textids[0]);

			if (text == null) {

				System.out.println(TXMCoreMessages.Summary_13+textids[0]+TXMCoreMessages.Summary_14);

				return null;

			}

			Edition edition = text.getEdition("default"); //$NON-NLS-1$

			if (edition == null) {

				if (text.getEditions().size() == 0) {

					System.out.println("No edition for text: "+text);

				} else {

					for (Edition ed : text.getEditions()) {

						edition = ed;

						break;

					}

				}

			}

			if (edition == null) {

				System.out.println(TXMCoreMessages.Summary_15);

				return null;

			}

			node.wordid = wordids[0];

			return edition.getPageForWordId(wordids[0]);

		} else {

			System.out.println(TXMCoreMessages.Summary_16+Arrays.toString(textids)+TXMCoreMessages.Summary_17+Arrays.toString(wordids));

			return null;

		}

	}

	public class TreeNode extends ArrayList<TreeNode>

	{

		/**

		 * 

		 */

		private static final long serialVersionUID = 1L;

		public TreeNode parent;

		boolean leaf = false;

		String struct;

		String id;

		int start = -1;

		int end = -1;

		public String wordid;

		public TreeNode()

		{

		}

		public boolean check() {

			for (TreeNode node : this) {

				if(node.start >= this.start && node.end <= this.end && (node.end-node.start) < (this.end-this.start))

					return node.check();

				else

					return false;

			}

			return true;

		}

		public TreeNode(String struct) {

			this();

			this.struct = struct;

		}

		@Override

		public String toString()

		{

			String str = struct+"("+start+", "+end+")"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

			if(size() > 0)

				str +="\n\t"+super.toString(); //$NON-NLS-1$

			return str;

		}

		public TreeNode getParent()

		{

			return parent;

		}

		public String getInfo() {

			return id;

		}

		public int getStart() {

			return start;

		}

		public String getWordId() {

			return wordid;

		}

	}

	@Override

	public String getName() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public String getSimpleName() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public String getDetails() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public void clean() {

		// TODO Auto-generated method stub

	}

}

<html>

<body>

<p>Prototype of Preview functionnaly: shows the begining of a corpus or a partition.</p>

</body>

</html>

package org.txm.functions.preview;

import java.util.Collection;

import java.util.List;

import org.txm.core.results.TXMResult;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.Partition;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.query.Match;

public class Preview extends TXMResult  {

	private Corpus corpus;

	private Partition partition;

	private Property prop;

	private int n;

	public Preview(Corpus corpus, Property prop, int n) {

		this.corpus = corpus;

		this.prop = prop;

		this.n = n;

	}

	public Preview(Partition partition, Property prop, int n) {

		this.partition = partition;

		this.prop = prop;

		this.n = n;

	}

	public boolean compute() {

		if (partition != null) {

			return computePartition();

		} else {

			return computeCorpus();

		}

	}

	protected boolean computeCorpus() {

		boolean ok = false;

		return ok;

	}

	protected boolean computePartition() {

		boolean ok = false;

		List<Match> matches = corpus.getMatches();

		for (Match m : matches) {

			if (m.getLength() >= n) {

				ok = true;

			}

		}

		return ok;

	}

	private String getLine(Corpus corpus) {

		List<Match> matches = corpus.getMatches();

		if (matches.size() == 0) return ""; //$NON-NLS-1$

		Match firstOkMatch = matches.get(0);

		for (Match m : matches) {

			if (m.getLength() >= n) {

				Collection<? extends Integer> positions = m.getRange();

			}

		}

		return "";//getString(firstOkMatch); //$NON-NLS-1$

	}

	@Override

	public String getName() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public String getSimpleName() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public String getDetails() {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public void clean() {

		// TODO Auto-generated method stub

	}

}

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate: 2016-09-19 10:31:00 +0200 (Mon, 19 Sep 2016) $

// $LastChangedRevision: 3298 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.functions.diagnostic;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.io.UnsupportedEncodingException;

import java.util.ArrayList;

import java.util.Collections;

import java.util.HashMap;

import java.util.LinkedHashSet;

import java.util.List;

import java.util.Locale;

import org.eclipse.osgi.util.NLS;

import org.txm.core.messages.TXMCoreMessages;

import org.txm.functions.Function;

import org.txm.objects.Base;

import org.txm.searchengine.cqp.AbstractCqiClient;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.corpus.Corpus;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.Part;

import org.txm.searchengine.cqp.corpus.Partition;

import org.txm.searchengine.cqp.corpus.Property;

import org.txm.searchengine.cqp.corpus.StructuralUnit;

import org.txm.searchengine.cqp.corpus.StructuralUnitProperty;

import org.txm.utils.logger.Log;

// TODO: Auto-generated Javadoc

/**

 * compute a diagnostic of Corpus, sub corpus and partition @ author mdecorde.

 */

public class Diagnostic extends Function {

	// Base base;

	/** The corpus. */

	Corpus corpus;

	/** The partition. */

	Partition partition;

	/** The maxvalue. */

	int maxvalue = 20;

	// Numbers

	/** The T. */

	int T = 0; // number of word

	/** The N properties. */

	int NProperties = 0; // number of word properties

	//HashMap<String, Integer> propertiesCounts = new HashMap<String, Integer>(); // properties counts

	/** The properties values. */

	HashMap<String, List<String>> propertiesValues = new HashMap<String, List<String>>(); // properties values

	/** The N structures. */

	int NStructures = 0; // number of structures

	/** The structures counts. */

	HashMap<String, Integer> structuresCounts = new HashMap<String, Integer>(); // structures counts

	/** The internal architecture. */

	List<String> internalArchitecture = new ArrayList<String>(); // wtc structures = propertiesCounts keys but ordered

	/** The hierarchie counts. */

	HashMap<String, Integer> hierarchieCounts = new HashMap<String, Integer>(); // hierarchic structures counts

	/** The internal architecture properties. */

	HashMap<String, HashMap<String, List<String>>> internalArchitectureProperties = new HashMap<String, HashMap<String, List<String>>>(); // hierarchic structures counts

	/** The internal architecture properties counts. */

	HashMap<String, HashMap<String, Integer>> internalArchitecturePropertiesCounts = new HashMap<String, HashMap<String, Integer>>(); // hierarchic structures counts

	// Annotations description

	/** The annotations types. */

	HashMap<String, Integer> annotationsTypes = new HashMap<String, Integer>(); // for each annotation its description

	/** The annotations origins. */

	HashMap<String, Integer> annotationsOrigins = new HashMap<String, Integer>(); // for each annoation its origin description

	// Import description

	/** The import description. */

	String importDescription = ""; //$NON-NLS-1$

	/** The properties. */

	List<Property> properties;

	/** The structures. */

	List<StructuralUnit> structures;

	/**

	 * Instantiates a new diagnostic.

	 *

	 * @param corpus a MainCorpus or a SubCorpus

	 * @param maxvalue the number of values of properties shown

	 */

	public Diagnostic(Corpus corpus, int maxvalue) {

		// long start = System.currentTimeMillis();

		// base = b;

		super(corpus);

		this.corpus = corpus;

		if (maxvalue <= 0)

			maxvalue = 10;

		this.maxvalue = maxvalue;

	}

	/**

	 * Process.

	 */

	public void process() {

		stepGeneralInfos();

		stepLexicalProperties();

		stepStructuralUnits();

	}

	/**

	 * Step general infos.

	 */

	public void stepGeneralInfos()

	{

		try {

			properties = corpus.getProperties();

			NProperties = properties.size();

		} catch (CqiClientException e1) {

			Log.severe(TXMCoreMessages.Diagnostic_1 + corpus.getName()

					+ TXMCoreMessages.Diagnostic_4);

			return;

		}

		try {

			T = corpus.getSize();// corpus.getLexicon(corpus.getProperty("id")).nbrOfToken();

		} catch (CqiClientException e1) {

			Log.severe(TXMCoreMessages.Diagnostic_6 + corpus.getName()

					+ TXMCoreMessages.Diagnostic_8);

			return;

		} 

	}

	/**

	 * step2.

	 * randomly position are choose

	 */

	public void stepLexicalProperties()

	{

		AbstractCqiClient cqiClient = CorpusManager.getCorpusManager().getCqiClient();

		for (Property p : properties) {

			//int size;

			try {

				//size = CorpusManager.getCorpusManager().getCqiClient().lexiconSize(p.getQualifiedName());

				//propertiesCounts.put(p.getName(), size);

				//List<String> list = Arrays.asList(corpus.getLexicon(p).getForms(this.maxvalue));

				int[] positions = new int[Math.min(maxvalue, T)];

				for(int i = 0 ; i < Math.min(maxvalue, T) ; i++)

					positions[i] = i;

				//ArrayList<String> values = new ArrayList<String>();

				LinkedHashSet<String> values = new LinkedHashSet<String>(cqiClient.getSingleData(p, positions));

				propertiesValues.put(p.getName(), new ArrayList<String>(values));

			} catch (Exception e) {

				org.txm.utils.logger.Log.printStackTrace(e);

			}

			// System.out.println("Lexi size "+p+" : "+(System.currentTimeMillis()-

			// start));

			// start = System.currentTimeMillis();

		}

	}

	/**

	 * Step structural units.

	 */

	public void stepStructuralUnits()

	{

		try {

			structures = corpus.getOrderedStructuralUnits();

		} catch (CqiClientException e1) {

			Log.severe(TXMCoreMessages.Diagnostic_1 + corpus.getName()

					+ TXMCoreMessages.Diagnostic_11);

			return;

		}

		NStructures = structures.size();

		for (StructuralUnit su : structures) { // for each structural unit

			internalArchitecture.add(su.getName());

			hierarchieCounts.put(su.getName(), -1);

			internalArchitectureProperties.put(su.getName(),

					new HashMap<String, List<String>>());

			internalArchitecturePropertiesCounts.put(su.getName(),

					new HashMap<String, Integer>());

			if (su.getProperties().size() > 0) {

				for (StructuralUnitProperty sup : su.getProperties())  {// for each of its property

					try {

						List<String> allvalues = sup.getValues(corpus);

						internalArchitectureProperties.get(su.getName()).put(

								sup.getName(),

								allvalues.subList(0, Math.min(allvalues.size(),

										maxvalue)));

						internalArchitecturePropertiesCounts.get(su.getName())

						.put(sup.getName(), allvalues.size());

					} catch (Exception e) {

						ArrayList<String> list = new ArrayList<String>(1);

						list.add(TXMCoreMessages.Diagnostic_2);

						internalArchitectureProperties.get(su.getName()).put(

								sup.getName(), list);

						internalArchitecturePropertiesCounts.get(su.getName())

						.put(sup.getName(), -1);

					}

				}

			}

		}

	}

	/**

	 * Instantiates a new diagnostic.

	 *

	 * @param partition the partition

	 * @throws CqiClientException the cqi client exception

	 */

	public Diagnostic(Partition partition) throws CqiClientException {

		corpus = partition.getCorpus();

		this.partition = partition;

		int T = corpus.getSize();

		System.out.println(TXMCoreMessages.Diagnostic_12);

		int nbParts = partition.getNPart();

		List<String> partNames = partition.getPartNames();

		List<Integer> partSizes = new ArrayList<Integer>();

		List<List<Property>> partProperties = new ArrayList<List<Property>>();

		List<String> partQueries = new ArrayList<String>();

		for (Part part : partition.getParts()) {

			partSizes.add(part.getSize());

			partProperties.add(part.getProperties());

			partQueries.add(part.getQuery().getQueryString());

		}

	}

	/**

	 * dump the results of the diagnostic in a String containing HTML code.

	 *

	 * @return the string

	 */

	public String htmldump() {

		StringBuffer buff = new StringBuffer();

		buff.append("<? version=\"1.0\" encoding=\"UTF-8\"?>\n" + //$NON-NLS-1$

				"<html>" + //$NON-NLS-1$

				"<head>" + //$NON-NLS-1$

				"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>" + //$NON-NLS-1$

				"</head>" + //$NON-NLS-1$

				"<body>\n"); //$NON-NLS-1$

		buff.append("<h2 style'font-family:\"Arial\";'>"+TXMCoreMessages.Diagnostic_3 + this.corpus.getName() + "</h2>\n"); //$NON-NLS-2$ //$NON-NLS-1$

		if (corpus.getDescription() != null && corpus.getDescription().length() > 0) {

			buff.append("<p>"+corpus.getDescription()+"</p>"); //$NON-NLS-1$ //$NON-NLS-2$

		}

		buff.append("<h3 style'font-family:\"Arial\";'>"+TXMCoreMessages.Diagnostic_5+ "</h3>\n"); //$NON-NLS-1$ //$NON-NLS-2$ 

		// counts

		buff.append("<ul>\n"); //$NON-NLS-1$

		buff.append("<li>" + TXMCoreMessages.Diagnostic_7 + T + "</li>\n"); //$NON-NLS-1$ //$NON-NLS-2$ 

		buff

		.append("<li>" + TXMCoreMessages.Diagnostic_9 + (NProperties - 1) + "</li>\n"); //$NON-NLS-2$ //$NON-NLS-1$

		buff.append("<li>" + TXMCoreMessages.Diagnostic_16 + NStructures + "</li>\n"); //$NON-NLS-2$ //$NON-NLS-1$

		buff.append("</ul>\n"); //$NON-NLS-1$

		// Propriétés d'occurrences

		buff.append("<h3 style'font-family:\"Arial\";'>"+NLS.bind(TXMCoreMessages.Diagnostic_14, maxvalue) + "</h3>\n"); //$NON-NLS-1$ //$NON-NLS-2$ 

		ArrayList<String> properties = new ArrayList<String>(propertiesValues.keySet());

		Collections.sort(properties);

		buff.append("<ul>\n"); //$NON-NLS-1$

		for (String s : properties) {

			if (!s.equals("id")) //$NON-NLS-1$

			{

				//buff.append("<li> " + s + " (" + propertiesCounts.get(s) + ") : "); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

				buff.append("<li> " + s + " : "); //$NON-NLS-1$ //$NON-NLS-2$ 

				for (String v : propertiesValues.get(s))

					buff.append(v + ", "); //$NON-NLS-1$

				buff.append("..."); //$NON-NLS-1$

				buff.append("</li>\n"); //$NON-NLS-1$ 

			}

		}

		buff.append("</ul>\n"); //$NON-NLS-1$

		// Propriété de structures

		buff.append("<h3 style'font-family:\"Arial\";'>"+NLS.bind(TXMCoreMessages.Diagnostic_22, maxvalue)+"</h3>\n"); //$NON-NLS-1$ //$NON-NLS-2$

		buff.append("<ul>\n"); //$NON-NLS-1$

		for (String s : internalArchitecture) {

			StringBuffer subbuffer = new StringBuffer();

			if (s.equals("txmcorpus")) //$NON-NLS-1$

				continue;//ignore the txmcorpus structure

			properties = new ArrayList<String>(internalArchitectureProperties.get(s).keySet());

			subbuffer.append("<li> " + s + "\n<ul>\n"); //$NON-NLS-1$ //$NON-NLS-2$

			if (internalArchitectureProperties.get(s).keySet().size() == 0)

				subbuffer.append("<li>"+TXMCoreMessages.Diagnostic_26+"</li>\n"); //$NON-NLS-1$ //$NON-NLS-2$ 

			//System.out.println("struct: "+s+" >> "+properties);

			Collections.sort(properties);

			//System.out.println("struct: "+s+" >sort> "+properties);

			int nbOfValues = 0;

			for (String ps : properties) {

				if (!ps.equals("project") && !ps.equals("base") ) //$NON-NLS-1$ //$NON-NLS-2$ 

				{

					//System.out.println("struct: "+s+" >> "+ps);

					int valuecount = internalArchitecturePropertiesCounts.get(s).get(ps);

					ArrayList<String> values = new ArrayList<String>(

							internalArchitectureProperties.get(s).get(ps));

					subbuffer.append("<li> " + ps + " (" + valuecount + ") = "); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

					//System.out.println("VALUES: "+values);

					Collections.sort(values);

					for (int i = 0 ; i < values.size() ; i++) {

						nbOfValues++;

						String psv = values.get(i);

						if (psv != null && psv.length() > 0)

							subbuffer.append(psv.replace("<", "&lt;")); //$NON-NLS-1$ //$NON-NLS-2$

						else

							subbuffer.append("\"\""); //$NON-NLS-1$

						if (i == values.size() -1)

							subbuffer.append("."); //$NON-NLS-1$

						else

							subbuffer.append(", "); //$NON-NLS-1$

						if (i >= maxvalue) {

							subbuffer.append("..."); //$NON-NLS-1$

							break;

						}

					}

					subbuffer.append("</li>\n"); //$NON-NLS-1$

				}

			}

			subbuffer.append("</ul>\n</li>\n"); //$NON-NLS-1$

			if (nbOfValues > 0) {

				buff.append(subbuffer);

			}

		}

		buff.append("</ul>\n"); //$NON-NLS-1$

		buff.append("</body>\n"); //$NON-NLS-1$

		buff.append("</html>\n"); //$NON-NLS-1$

		return buff.toString();

	}

	/**

	 * dump the result in the console.

	 *

	 * @param maxvalue the maxvalue

	 * @return the string

	 */

	public String dump(int maxvalue) {

		StringBuffer buff = new StringBuffer();

		buff.append(TXMCoreMessages.Diagnostic_40);

		buff.append(TXMCoreMessages.Diagnostic_41 + String.format(Locale.FRANCE,"%,d", T) + "\n"); //$NON-NLS-1$ 

		buff.append(TXMCoreMessages.Diagnostic_43 + String.format("%,d", NProperties) + "\n"); //$NON-NLS-1$ 

		/*for (String s : propertiesCounts.keySet())

			if (!s.equals("id")) //$NON-NLS-1$

				buff.append(" - " + s + " " + propertiesCounts.get(s) + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

				*/

		buff.append(TXMCoreMessages.Diagnostic_49 + String.format("%,d", NStructures) + "\n"); //$NON-NLS-1$ 

		for (String s : structuresCounts.keySet())

			buff.append(" - " + s + "\n");//+structuresCounts.get(s)+"\n"); //$NON-NLS-1$ //$NON-NLS-2$

		/*

		 * buff.append("* Structural Units counts\n"); for(String s :

		 * internalArchitecture)

		 * buff.append(" - "+s+" "+hierarchieCounts.get(s)+"\n");

		 */

		buff.append(TXMCoreMessages.Diagnostic_0);

		for (String s : internalArchitecture) {

			buff.append(" - " + s + "\n"); //$NON-NLS-1$ //$NON-NLS-2$

			for (String ps : internalArchitectureProperties.get(s).keySet()) {

				if (!(s.equals("text") && (ps.equals("project") || ps.equals("base") || ps.equals("id")))) //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$

				{

					buff

					.append("   - " + ps + "(" + internalArchitectureProperties.get(s).get(ps).size() + ") = "); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

					int valuecount = 0;

					for (String psv : internalArchitectureProperties.get(s)

							.get(ps)) {

						buff.append(psv);

						if (valuecount++ >= maxvalue) {

							buff.append("..."); //$NON-NLS-1$

							break;

						}

						if (valuecount < internalArchitectureProperties.get(s)

								.get(ps).size())

							buff.append(", "); //$NON-NLS-1$

					}

					buff.append("\n"); //$NON-NLS-1$

				}

			}

		}

		return buff.toString();

	}

	/**

	 * dump the result in the console with maxvalue=30.

	 *

	 * @return the string

	 */

	public String dump() {

		return dump(30);

	}

	/**

	 * Gets the name.

	 *

	 * @return the name

	 */

	public String getName() {

		return corpus.getName();

	}

	/**

	 * Gets the base.

	 *

	 * @return the base

	 */

	public Base getBase() {

		return corpus.getBase();

	}

	/**

	 * Gets the t.

	 *

	 * @return the t

	 */

	public int getT() {

		return T;

	}

	/**

	 * Gets the n properties.

	 *

	 * @return the n properties

	 */

	public int getNProperties() {

		return NProperties;

	}

	/**

	 * Gets the internal architecture.

	 *

	 * @return the internal architecture

	 */

	public List<String> getInternalArchitecture() {

		return internalArchitecture;

	}

	/*public HashMap<String, Integer> getPropertiesCounts() {

		return propertiesCounts;

	}*/

	/**

	 * Gets the n structures.

	 *

	 * @return the n structures

	 */

	public int getNStructures() {

		return NStructures;

	}

	/**

	 * Gets the structures counts.

	 *

	 * @return the structures counts

	 */

	public HashMap<String, Integer> getStructuresCounts() {

		return structuresCounts;

	}

	/**

	 * Gets the annotations types.

	 *

	 * @return the annotations types

	 */

	public HashMap<String, Integer> getAnnotationsTypes() {

		return annotationsTypes;

	}

	/**

	 * Gets the annotations origins.

	 *

	 * @return the annotations origins

	 */

	public HashMap<String, Integer> getAnnotationsOrigins() {

		return annotationsOrigins;

	}

	/**