Laboratoire ICAR » Plateforme TXM

               OSGI-INF/,\

               groovy/

 org.txm.groovy.core;bundle-version="1.0.0";visibility:=reexport,

 org.txm.searchengine.core

// Copyright © 2019 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// STANDARD DECLARATIONS

package org.txm.macro.tiger.exploit

import groovy.transform.Field

import org.txm.searchengine.core.SearchEnginesManager

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.ts.TIGERSearchEngine

def scriptName = this.class.getSimpleName()

def selection = []

for (def s : corpusViewSelections) {

	if (s instanceof CQPCorpus) selection << s

	else if (s instanceof Partition) selection.addAll(s.getParts())

}

if (selection.size() == 0) {

	println "** $scriptName: please select a Corpus or a Partition in the Corpus view: "+corpusViewSelections

	return false

} else {

	for (def c : selection) c.compute(false)

}

@Field @Option(name="tiger_query_A", usage="A Full TIGERSearch query", widget="Text", required=true, def="[]")

		String tiger_query_A

		@Field @Option(name="tiger_query_B", usage="A Full TIGERSearch query", widget="Text", required=true, def="[]")

		String tiger_query_B

@Field @Option(name="count_subgraph", usage="A Full TIGERSearch query", widget="Boolean", required=true, def="true")

		def count_subgraph

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

		debug

if (!ParametersDialog.open(this)) return	

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

def results_A = gse.run(TIGERMatchesMacro, ["args":["tiger_query":tiger_query_A, "count_subgraph":count_subgraph, "debug":debug],

	"selection":selection,

	"selections":selections,

	"corpusViewSelection":corpusViewSelection,

	"corpusViewSelections":corpusViewSelections,

	"monitor":monitor])

def results_B = gse.run(TIGERMatchesMacro,	["args":["tiger_query":tiger_query_B, "count_subgraph":count_subgraph, "debug":debug],

	"selection":selection,

	"selections":selections,

	"corpusViewSelection":corpusViewSelection,

	"corpusViewSelections":corpusViewSelections,

	"monitor":monitor])

println results_A

println results_B

for (def corpus : selection) {

	String.format( "%.2f", coef * 100.0d )

	println "R = ${results_A[corpus]} / ${results_B[corpus]} = "+String.format( "%.2f", results_A[corpus] / results_B[corpus])

}

println "Done."

// Copyright © 2019 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// STANDARD DECLARATIONS

package org.txm.macro.tiger.exploit

import groovy.transform.Field

import org.txm.searchengine.core.SearchEnginesManager

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.ts.TIGERSearchEngine

def scriptName = this.class.getSimpleName()

def selection = []

for (def s : corpusViewSelections) {

	if (s instanceof CQPCorpus) selection << s

	else if (s instanceof Partition) selection.addAll(s.getParts())

}

if (selection.size() == 0) {

	println "** $scriptName: please select a Corpus or a Partition in the Corpus view: "+corpusViewSelections

	return false

} else {

	for (def c : selection) c.compute(false)

}

@Field @Option(name="tiger_query", usage="A Full TIGERSearch query", widget="Text", required=true, def="[]")

		String tiger_query

@Field @Option(name="count_subgraph", usage="A Full TIGERSearch query", widget="Boolean", required=true, def="true")

		def count_subgraph

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

		debug

if (!ParametersDialog.open(this)) return	

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

TIGERSearchEngine tse = SearchEnginesManager.getTIGERSearchEngine()

println tse

println "\t"+selection.join("\t")

print "F"

def results = new LinkedHashMap()

for (def corpus : selection) {

def root = corpus.getRootCorpusBuild();

File buildDirectory = new File(root.getProjectDirectory(), "tiger");

	if (!tse.hasIndexes(corpus)) {

		println "Warning: skipping $corpus: no TIGERSearch indexes found."

		continue;

	}

	def tcorpus = tse.getTSCorpus(corpus);

	def sentences_min_max = tse.getSentMinMax(corpus);

	def mresult = tcorpus.manager.processQuery(tiger_query, sentences_min_max[0], sentences_min_max[1], 9999999);

	int size = 0;

	if (count_subgraph) {

		size = mresult.submatchSize();

	} else {

		size = mresult.size();

	}

	results[corpus] = size

	print "\t"+size

}

println ""

println "Done."

return results

// Copyright © 2019 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author mdecorde

// STANDARD DECLARATIONS

package org.txm.macro.tiger.exploit

import groovy.transform.Field

import java.util.List

import org.txm.searchengine.core.EmptySelection

import org.txm.searchengine.core.SearchEnginesManager

import org.txm.searchengine.cqp.AbstractCqiClient

import org.txm.searchengine.cqp.CQPSearchEngine

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.ts.TIGERSearchEngine

import org.txm.searchengine.ts.TSCorpus

import org.txm.searchengine.ts.TSResult

import org.txm.utils.ConsoleProgressBar

import org.txm.utils.logger.Log

import ims.tiger.corpus.Sentence

import ims.tiger.query.api.*;

def scriptName = this.class.getSimpleName()

def selection = []

for (def s : corpusViewSelections) {

	if (s instanceof CQPCorpus) selection << s

	else if (s instanceof Partition) selection.addAll(s.getParts())

}

if (selection.size() == 0) {

	println "** $scriptName: please select a Corpus or a Partition in the Corpus view: "+corpusViewSelections

	return false

} else {

	for (def c : selection) c.compute(false)

}

@Field @Option(name="tiger_query", usage="A Full TIGERSearch query", widget="Text", required=true, def="[]")

		String tiger_query

@Field @Option(name="labels", usage="List of TIGER labels separated with ','", widget="String", required=true, def="A,B,C")

		def labels

@Field @Option(name="properties", usage="List of properties separated with ','", widget="String", required=true, def="word, word, word")

		def properties

@Field @Option(name="count_subgraph", usage="A Full TIGERSearch query", widget="Boolean", required=true, def="true")

		def count_subgraph

@Field @Option(name="sort_column", usage="choose between the len or freq columns", widget="StringArray", metaVar="freq	labels", required=true, def="freq")

		String sort_column

		@Field @Option(name="max_lines", usage="choose between the len or freq columns", widget="Integer", metaVar="freq	labels", required=true, def="-1")

		int max_lines

@Field @Option(name="debug", usage="Show internal variable content", widget="StringArray", metaVar="OFF	ON	ALL	REALLY ALL", required=true, def="OFF")

		debug

if (!ParametersDialog.open(this)) return

if (debug == "OFF") debug = 0; else if (debug == "ON") debug = 1; else if (debug == "ALL") debug = 2 else if (debug == "REALLY ALL") debug = 3

TIGERSearchEngine tse = SearchEnginesManager.getTIGERSearchEngine()

if (labels.length() == 0) {

	println "Error: no label given. Aborting"

	return false;

} else {

	labels = labels.split(",")

}

if (properties.length() == 0) {

	println "Error: no property given. Aborting"

	return false;

} else {

	properties = properties.split(",")

}

if (properties.size() != labels.size()) {

	println "Error: the number of labels and properties does not matches. labels=$labels properties=$properties (${labels.size()} != {properties.size()})"

	return false;

}

def missing = []

for (def l : labels) {

	if (!tiger_query.contains("#${l}:")) {

		missing << l

	}

}

if (missing.size() > 0) {

	println "Error: $missing label(s) not found in query: $tiger_query"

	return false

}

def results = new LinkedHashMap()

def total_counts = [:] // List<String> -> Integer

ConsoleProgressBar cpb = new ConsoleProgressBar(selection.size())

for (def corpus : selection) {

	def counts = [:] // List<String> -> Integer

	def root = corpus.getRootCorpusBuild();

	File buildDirectory = new File(root.getProjectDirectory(), "tiger");

	if (!tse.hasIndexes(corpus)) {

		println "Warning: skipping $corpus: no TIGERSearch indexes found."

		continue;

	}

	def tcorpus = tse.getTSCorpus(corpus);

	def index = tcorpus.getIndex()

	def sentences_min_max = tse.getSentMinMax(corpus);

	MatchResult mresult = tcorpus.manager.processQuery(tiger_query, sentences_min_max[0], sentences_min_max[1], 9999999);

	List<String> variables = java.util.Arrays.asList(mresult.getVariableNames());

	def iVariables = []

	for (def l : labels) {

		iVariables << variables.indexOf(l)

	}

	def matches = mresult.matches

	int size = matches.keySet().size();

	if (debug > 0) println "$size sentences matched."

	for (int sent : matches.keySet()) { // the matching sentences

		if (!mresult.isMatchingSentence(sent)) continue;

		int sent_submatch_size = mresult.getSentenceSubmatchSize(sent);

		//println "sent no $sent with $sent_submatch_size submatches : "

		def sentence = index.getSentence(sent)

		for (int j = 0 ; j < sent_submatch_size ; j++) {

			//println "submatch no $j"

			def positions = mresult.getSentenceSubmatchAt(sent, j)

			def strings = []

//			println "	positions=$positions"

//			println "	properties=$properties"

//			println "	iVariables=$iVariables"

			int t = 0;

			for (int iV : iVariables) {

				def node = sentence.getNode(positions[iV])

				String value = node.getFeature(properties[t])

				t++

				if (value == null) {

					//println "Warning: no '${properties[iV]}' feature found in node="+node.getFeatures()

					value = "NA"

				}

				strings << value

			}

			if (!counts.containsKey(strings)) {

				counts[strings] = 0;

			}

			if (!total_counts.containsKey(strings)) {

				total_counts[strings] = 0;

			}

			counts[strings] = counts[strings] + 1

			total_counts[strings] = total_counts[strings] + 1

			if (!count_subgraph) break;

		}

	}

	results[corpus] = counts

	cpb.tick()

	// DEBUG

	//	def keys = []

	//	keys.addAll(counts.keySet())

	//	if ("freq".equals(sort_column)) {

	//		keys = keys.sort() {-counts[it]}

	//	} else {

	//		keys = keys.sort()

	//	}

	//

	//	for (def k : keys) {

	//		println "${k.join('_')}	${counts[k]}"

	//	}

}

cpb.done()

def keys = []

keys.addAll(total_counts.keySet())

if ("freq".equals(sort_column)) {

	keys = keys.sort() {-total_counts[it]}

} else {

	keys = keys.sort()

}

println properties.join(", ")+"\t"+"F\t"+selection.join("\t")

int nline = 0;

for (def k : keys) {

	print "${k.join('_')}	${total_counts[k]}"

	if (selection.size() > 1) {

		for (def corpus : selection) {

			def v = results[corpus][k];

			if (v == null) v = 0;

			print "\t"+v

		}

	}

	println ""

	nline++

	if (max_lines > 0 && nline >= max_lines) {

		println "... (${keys.size() - max_lines})"

		break;

	}

}

		scriptsPackageDirectory = new File(userDirectory, "org/txm/macro/tiger");

		scriptsPackageDirectory2 = new File(systemDirectory, "org/txm/macro/tiger");

		scriptsPackageDirectory.mkdirs();

		scriptsPackageDirectory2.mkdirs();

		BundleUtils.copyFiles(bundle_id, "groovy", "org/txm/macro/tiger", "", scriptsPackageDirectory);

		BundleUtils.copyFiles(bundle_id, "groovy", "org/txm/macro/tiger", "", scriptsPackageDirectory2);

import ims.tiger.index.reader.Index;

import ims.tiger.index.reader.IndexException;

import java.io.IOException;

import java.io.RandomAccessFile;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

	// Additional data for corpus alignment with TXM base corpus (CQP corpus)

	RandomAccessFile offsetsRAFile = null;

	FileChannel offsetsFileChannel = null;

	MappedByteBuffer offsetsMapped = null; // one offset per tiger position

	RandomAccessFile presencesRAFile = null;

	FileChannel presencesFileChannel = null;

	MappedByteBuffer presencesMapped = null; // one 0/1 boolean per tiger position

	private int[] sentence_starts;

	@Override

	public void finalize() {

		try {

			close();

		} catch(Exception e) {

			e.printStackTrace();

		}

	}

			File offsetsFile = new File(regpath, corpusId+"/offsets.data");

			if (offsetsFile.exists()) {

				offsetsRAFile = new RandomAccessFile(offsetsFile, "rw");

				offsetsFileChannel = offsetsRAFile.getChannel();

				offsetsMapped = offsetsFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, offsetsFileChannel.size());

			}

			//out.putInt(positions[i])

			File presencesFile = new File(regpath, corpusId+"/presences.data");

			if (presencesFile.exists()) {

				presencesRAFile = new RandomAccessFile(presencesFile, "rw");

				presencesFileChannel = presencesRAFile.getChannel();

				presencesMapped = presencesFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, presencesFileChannel.size());

			}

	public String getID() {

		return this.id;

	}

	public void close() {

		try {

			if (presencesRAFile != null) presencesRAFile.close();

			if (presencesFileChannel != null) presencesFileChannel.close();

			if (offsetsRAFile != null) offsetsRAFile.close();

			if (offsetsFileChannel != null) offsetsFileChannel.close();

			if (sentence_starts != null) sentence_starts = null;

			if (results != null) results.clear();

		} catch (IOException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

	}

	public int getOffset(int tigerPosition) {

		if (offsetsMapped != null) {

			return offsetsMapped.getInt(tigerPosition*Integer.BYTES);

		} else {

			return 0;

		}

	}

	public int[] getOffsets(int tigerPositions[]) {

		int[] ret = new int[tigerPositions.length];

		if (offsetsMapped != null) {

			for (int i = 0 ; i < tigerPositions.length ; i++) {

				ret[i] =  offsetsMapped.getInt(tigerPositions[i]*Integer.BYTES);

			}

		}

		return ret;

	}

	public MappedByteBuffer getOffsetsMapped() {

		return offsetsMapped;

	}

	public MappedByteBuffer getPresencesMapped() {

		return presencesMapped;

	}

	public int getPresence(int tigerPosition) {

		if (presencesMapped != null) {

			return presencesMapped.getInt(tigerPosition*Integer.BYTES);

		} else {

			return 0;

		}

	}

	public int[] getPresences(int tigerPositions[]) {

		int[] ret = new int[tigerPositions.length];

		if (presencesMapped != null) {

			for (int i = 0 ; i < tigerPositions.length ; i++) {

				ret[i] =  presencesMapped.getInt(tigerPositions[i]*Integer.BYTES);

			}

		}

		return ret;

	}

	public Index getIndex() {

		InternalCorpusQueryManagerLocal2 tigermanager = this.manager;

		CorpusQueryProcessor processor = tigermanager.getQueryProcessor();

		return processor.getIndex();

	}

	public int[] getSentenceStartPositions() throws IndexException {

		if (sentence_starts != null) {

			return sentence_starts;

		}

		Index index = getIndex();

		sentence_starts = new int[index.getNumberOfGraphs()];

		for (int i = 0 ; i < index.getNumberOfGraphs() ; i++) {

			sentence_starts[i] = 0;

			if (i > 0) {

				sentence_starts[i] += index.getNumberOfTNodes(i-1) + sentence_starts[i-1];

			}

		}

		return sentence_starts;

	}

	public TSProperty getTProperty(String name) {

		return new TSProperty(this, name, true);

	}

	public TSProperty getNTProperty(String name) {

		return new TSProperty(this, name, false);

	}

import java.io.IOException;

import java.util.HashMap;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.clientExceptions.UnexpectedAnswerException;

import org.txm.searchengine.cqp.serverException.CqiServerError;

	HashMap<CorpusBuild, TSCorpus> corpora = null;

	public TSCorpus getTSCorpus(CorpusBuild corpus) {

		CorpusBuild root = corpus.getRootCorpusBuild();

		TSCorpus tscorpus = corpora.get(root);

		if (tscorpus != null) {

			return tscorpus;

		}

		File tigerDirectory = new File(root.getProjectDirectory(), "tiger");

		File configfile = new File(tigerDirectory, "tigersearch.logprop");

		TSCorpusManager manager = new TSCorpusManager(tigerDirectory, configfile);

		tscorpus = manager.getCorpus(root.getID());

		if (tscorpus != null) {

			corpora.put(root, tscorpus);

			return tscorpus;

		} else {

			return null;

		}

	}

	public TSCorpus removeTSCorpus(CorpusBuild corpus) {

		CorpusBuild root = corpus.getRootCorpusBuild();

		return corpora.remove(root);

	}

	 * @param cqpCorpus the targeted CQPCorpus 

	 * @return the first sentence and last sentence id (from 0 to N, N the number of sentences). WARNING: this is not the list of sentences in the targeted CQPCorpus. unless the CQPcorpus is contigues

	 * 

	 * @throws UnexpectedAnswerException

	 * @throws IOException

	 * @throws CqiServerError

	 * @throws CqiClientException

	 */

	public static int[] getSentMinMax(CQPCorpus cqpCorpus) throws UnexpectedAnswerException, IOException, CqiServerError, CqiClientException {

		AbstractCqiClient CQI = CQPSearchEngine.getCqiClient();

		List<org.txm.searchengine.cqp.corpus.query.Match> matches = cqpCorpus.getMatches();

		if (matches.size() == 0) {

			return new int[]{0,0};

		}

		int[] cpos = new int[] {matches.get(0).getStart(), matches.get(matches.size()-1).getEnd()};

		int[] structs = CQI.cpos2Struc(cqpCorpus.getStructuralUnit("s").getProperty("n").getQualifiedName(), cpos);

		if (structs.length == 0) {

			return new int[]{0,0};

		}

		int sent_min = structs[0];

		int sent_max = structs[structs.length-1];

		return new int[]{sent_min, sent_max};

	}

	/**

	 * 

		corpora = new HashMap<CorpusBuild, TSCorpus>();

		if (corpora != null) {

			for (TSCorpus corpus : corpora.values()) {

				corpus.close(); // free memory (mmap, etc.)

			}

			corpora.clear();

		}

		TSCorpus tcorpus = this.getTSCorpus(corpus);

		Index index = tcorpus.getIndex();

		int[] starts = tcorpus.getSentenceStartPositions();

		MappedByteBuffer offsetsMapped = tcorpus.getOffsetsMapped();

		//MappedByteBuffer offsetsMapped = tcorpus.getOffsetsMapped();

				//				System.out.println("  sent="+sent_start+ " matches="+Arrays.toString(match)+" ipivot="+iPivot);

						//						System.out.println("left="+left+" offset="+offsetsMapped.getInt(left*Integer.BYTES));

		if (corpus == null) return false;

		CorpusBuild root = corpus.getRootCorpusBuild();

		File buildDirectory = new File(root.getProjectDirectory(), "tiger");

				new File(buildDirectory, root.getID()).exists();

		if (r instanceof MainCorpus && "clean".equals(state)) { // the CQP corpus has been deleted by the user

		} else if (r instanceof Project && "clean".equals(state)) {  // the Project has been deleted by the user

package org.txm.searchengine.ts;

import org.txm.searchengine.core.SearchEngineProperty;

import org.txm.searchengine.cqp.corpus.query.Match;

import ims.tiger.index.reader.Index;

import ims.tiger.query.processor.CorpusQueryProcessor;

public class TSProperty implements SearchEngineProperty {

	TSCorpus tcorpus;

	String name;

	boolean T;

	public TSProperty(TSCorpus corpus, String name, boolean T) {

		this.tcorpus = corpus;

		this.name = name;

		this.T = T;

	}

	@Override

	public String getName() {

		return name;

	}

	@Override

	public String getFullName() {

		return tcorpus.getHeader().getCorpus_ID()+"_"+name;

	}

	public String getValue(Match m) {

		if (m == null) return null;

		InternalCorpusQueryManagerLocal2 tigermanager = tcorpus.manager;

		CorpusQueryProcessor processor = tigermanager.getQueryProcessor();

		Index index = processor.getIndex();

		//TODO not finished

		return null;

	}

}