Laboratoire ICAR » Plateforme TXM

<?xml version="1.0" encoding="UTF-8"?>

<classpath>

	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>

	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins">

		<accessrules>

			<accessrule kind="accessible" pattern="**"/>

		</accessrules>

	</classpathentry>

	<classpathentry kind="src" path="src"/>

	<classpathentry kind="lib" path="libs/hsqldb.jar"/>

	<classpathentry kind="lib" path="libs/postgresql-9.4.1207.jre6.jar"/>

	<classpathentry kind="lib" path="libs/sqlite-jdbc-3.8.11.2.jar"/>

	<classpathentry kind="output" path="bin"/>

</classpath>

<?xml version="1.0" encoding="UTF-8"?>

<persistence version="2.1" xmlns="http://xmlns.jcp.org/xml/ns/persistence" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/persistence http://xmlns.jcp.org/xml/ns/persistence/persistence_2_1.xsd">	

		<persistence-unit name="HSQLKRPERSISTENCE" transaction-type="RESOURCE_LOCAL">

		<provider>org.eclipse.persistence.jpa.PersistenceProvider</provider>

		<class>org.txm.annotation.kr.core.Annotation</class>

		<class>org.txm.annotation.kr.core.repository.AnnotationType</class>

		<class>org.txm.annotation.kr.core.repository.TypedValue</class>

<!--	

		<class>org.txm.functions.dictionary_jpa.EntryId</class>

		<class>org.txm.functions.dictionary_jpa.EntryProperty</class>

		<class>org.txm.functions.dictionary_jpa.EntryPropertyType</class>

-->

		<properties>

			<property name="javax.persistence.jdbc.driver" value="org.hsqldb.jdbcDriver"/>

			<property name="javax.persistence.jdbc.url" value="jdbc:hsqldb:file:txmdb/db;shutdown=true;hsqldb.write_delay=false;hsqldb.lock_file=false;hsqldb.applog=0;hsqldb.sqllog=0"/>

			<property name="javax.persistence.jdbc.user" value="SA"/>

<!-- 			<property name="eclipselink.logging.parameters" value="false"/>

 			<property name="eclipselink.logging.level" value="ON" />

			<property name="eclipselink.jdbc.read-connections.min" value="1" />

			<property name="eclipselink.jdbc.write-connections.min" value="1" />

			<property name="eclipselink.jdbc.batch-writing" value="JDBC" />

-->

<!-- Logging 

			<property name="eclipselink.logging.file" value="output.log"/>

			<property name="eclipselink.logging.logger" value="JavaLogger"/>

-->

<!-- 		

			<property name="eclipselink.logging.level" value="FINE" />

			<property name="eclipselink.logging.timestamp" value="false" />

			<property name="eclipselink.logging.session" value="false" />

			<property name="eclipselink.logging.thread" value="false" /> 

-->

			<!-- <property name="eclipselink.ddl-generation" value="drop-and-create-tables"/> -->

			<!-- <property name="eclipselink.ddl-generation.output-mode" value="database"/> -->

		</properties>

	</persistence-unit>

</persistence>

Manifest-Version: 1.0

Require-Bundle: org.txm.core;bundle-version="0.7.0";visibility:=reexport,

 org.txm.searchengine.cqp.core;visibility:=reexport,

 org.txm.utils;bundle-version="1.0.0";visibility:=reexport,

 javax.persistence;bundle-version="2.1.0";visibility:=reexport,

 org.eclipse.persistence.asm;bundle-version="3.3.1";visibility:=reexport,

 org.eclipse.persistence.jpa;bundle-version="2.6.0";visibility:=reexport,

 org.eclipse.persistence.jpa.jpql;bundle-version="2.6.0";visibility:=reexport,

 org.eclipse.persistence.antlr;bundle-version="3.2.0";visibility:=reexport,

 org.eclipse.osgi;bundle-version="3.10.2";visibility:=reexport,

 org.txm.searchengine.core;bundle-version="1.0.0";visibility:=reexport,

 org.eclipse.ui.workbench;visibility:=reexport,

 org.txm.annotation.core;visibility:=reexport

Export-Package: org.hsqldb,

 org.hsqldb.auth,

 org.hsqldb.dbinfo,

 org.hsqldb.error,

 org.hsqldb.index,

 org.hsqldb.jdbc,

 org.hsqldb.jdbc.pool,

 org.hsqldb.lib,

 org.hsqldb.lib.java,

 org.hsqldb.lib.tar,

 org.hsqldb.map,

 org.hsqldb.navigator,

 org.hsqldb.persist,

 org.hsqldb.resources,

 org.hsqldb.result,

 org.hsqldb.rights,

 org.hsqldb.rowio,

 org.hsqldb.scriptio,

 org.hsqldb.server,

 org.hsqldb.types,

 org.hsqldb.util,

 org.postgresql,

 org.postgresql.copy,

 org.postgresql.core,

 org.postgresql.core.v2,

 org.postgresql.core.v3,

 org.postgresql.ds,

 org.postgresql.ds.common,

 org.postgresql.fastpath,

 org.postgresql.geometric,

 org.postgresql.gss,

 org.postgresql.hostchooser,

 org.postgresql.jdbc,

 org.postgresql.jdbc2,

 org.postgresql.jdbc2.optional,

 org.postgresql.jdbc3,

 org.postgresql.largeobject,

 org.postgresql.osgi,

 org.postgresql.ssl,

 org.postgresql.ssl.jdbc4,

 org.postgresql.sspi,

 org.postgresql.translation,

 org.postgresql.util,

 org.postgresql.xa,

 org.sqlite,

 org.sqlite.core,

 org.sqlite.date,

 org.sqlite.javax,

 org.sqlite.jdbc3,

 org.sqlite.jdbc4,

 org.sqlite.util,

 org.txm.annotation.kr.core,

 org.txm.annotation.kr.core.conversion,

 org.txm.annotation.kr.core.preferences,

 org.txm.annotation.kr.core.repository,

 org.txm.annotation.kr.core.storage.temporary,

 org.txm.annotation.kr.core.temporary

Bundle-ActivationPolicy: lazy

Bundle-ClassPath: .,libs/hsqldb.jar,libs/postgresql-9.4.1207.jre6.jar,

 libs/sqlite-jdbc-3.8.11.2.jar

Bundle-Version: 1.0.0.qualifier

Bundle-Name: org.txm.annotation.kr.core

Bundle-ManifestVersion: 2

Bundle-Activator: org.txm.annotation.kr.core.Activator

Bundle-SymbolicName: org.txm.annotation.kr.core;singleton:=true

Meta-Persistence: META-INF/persistence.xml

Bundle-RequiredExecutionEnvironment: JavaSE-1.7

<?xml version="1.0" encoding="UTF-8"?>

<projectDescription>

	<name>org.txm.annotation.kr.core</name>

	<comment></comment>

	<projects>

	</projects>

	<buildSpec>

		<buildCommand>

			<name>org.eclipse.jdt.core.javabuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

		<buildCommand>

			<name>org.eclipse.pde.ManifestBuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

		<buildCommand>

			<name>org.eclipse.pde.SchemaBuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

	</buildSpec>

	<natures>

		<nature>org.eclipse.pde.PluginNature</nature>

		<nature>org.eclipse.jdt.core.javanature</nature>

	</natures>

</projectDescription>

package org.txm.annotation.kr.core.temporary;

import org.junit.runner.RunWith;

import org.junit.runners.Suite;

import org.junit.runners.Suite.SuiteClasses;

import org.txm.StartToolbox;

@RunWith(Suite.class)

@SuiteClasses({ StartToolbox.class, CreateAnnotation.class, DeleteAnnotation.class,

		UpdateAnnotation.class })

public class AllTests {

}

package org.txm.annotation.kr.core.temporary;

import static org.junit.Assert.*;

import org.junit.Test;

public class DeleteAnnotation {

	@Test

	public void test() {

		fail("Not yet implemented");

	}

}

package org.txm.annotation.kr.core.temporary;

import static org.junit.Assert.*;

import org.junit.Test;

public class UpdateAnnotation {

	@Test

	public void test() {

		fail("Not yet implemented");

	}

}

package org.txm.annotation.kr.core.temporary;

import static org.junit.Assert.*;

import java.util.HashMap;

import org.junit.Test;

import org.txm.Toolbox;

import org.txm.annotation.kr.core.storage.temporary.TemporaryAnnotationManager;

import org.txm.searchengine.cqp.clientExceptions.CqiClientException;

import org.txm.searchengine.cqp.clientExceptions.InvalidCqpIdException;

import org.txm.searchengine.cqp.corpus.CorpusManager;

import org.txm.searchengine.cqp.corpus.MainCorpus;

public class CreateAnnotation {

	@Test

	public void test() throws CqiClientException, InvalidCqpIdException {

		if (!Toolbox.isInitialized()) fail("Toolbox not initialized.");

		MainCorpus corpus = CorpusManager.getCorpusManager().getCorpus("VOEUX");

		if (corpus == null) fail("Corpus Voeux not loaded.");

		HashMap<String, Object> properties = TemporaryAnnotationManager.getInitialisationProperties(this.getClass(), corpus);

		properties.put("eclipselink.persistencexml", System.getProperty("user.home")+"/workspace442/org.txm.core/META-INF/persistence.xml");

		TemporaryAnnotationManager tam = new TemporaryAnnotationManager(corpus, properties);

		System.out.println(tam);

	}

}

package org.txm.annotation.kr.core;

import java.text.DateFormat;

import java.text.SimpleDateFormat;

import java.util.Comparator;

import java.util.StringTokenizer;

public class AnnotationComparator implements Comparator<Annotation> {

	public int compare(Annotation a1, Annotation a2) {

	    // comparer e1 et e2

		DateFormat dateformat =  new SimpleDateFormat("yyyy-MM-dd");

		//System.out.println("Date 1 : "+a1.getDate() + " => "+dateformat.format(a1.getDate()));

		//System.out.println("Date 2 : "+a2.getDate() + " => "+dateformat.format(a2.getDate()));

		StringTokenizer tokenizer1 = new StringTokenizer(dateformat.format(a1.getDate()), "-");

		StringTokenizer tokenizer2 = new StringTokenizer(dateformat.format(a2.getDate()), "-");

		for(int i = 0 ; i < 2 && tokenizer1.hasMoreTokens() && tokenizer2.hasMoreTokens() ; ++i) {

			String token1 = tokenizer1.nextToken();

			String token2 = tokenizer2.nextToken();

			int valint1 = new Integer(token1).intValue();

			int valint2 = new Integer(token2).intValue();

			//System.out.println(valint1+" | "+valint2);

			if (valint1<valint2){

				return -1;

			}else {

				if(valint1>valint2){

					return 1;

				}else {

					//System.out.println("idem ["+i+"]");

				}

			}

		}

		return 0;

	  }

	public static void main(String[] args) {

		AnnotationComparator comp = new AnnotationComparator();

		Annotation a1 = new Annotation("truc", "bidule", 3, 10);

		Annotation a2 = new Annotation("truc", "bidule", 6, 8);

		int ret = comp.compare(a1, a2);

		System.out.println("Le retour est : "+ret);

	}

}

package org.txm.annotation.kr.core;

import java.util.HashMap;

import javax.persistence.EntityManager;

public class DatabasePersistenceManager {

	//Object can be Corpus or KnowledgeRepository

	protected HashMap<Object, EntityManager> managers;

	public static final String PERSISTENCE_UNIT_NAME = "HSQLKRPERSISTENCE";

	public static String ACCESS_SQL = "sql";

	public static String ACCESS_FILE = "file";

	public static String ACCESS_SPARQL = "sparql";

	/**

	 * Instantiates a new database manager.

	 */

	public DatabasePersistenceManager() {

		managers = new HashMap<Object, EntityManager>() ;

	}

	/**

	 * The Object can be a Corpus or a KnowledgeRepository

	 * @param obj

	 * @return

	 */

	public EntityManager getJPAEntityManager(Object obj){

		if (managers.containsKey(obj)) {

			return managers.get(obj);

		}

		return null;

	}

	public void closeManager(Object key) {

		if (!managers.keySet().contains(key)) return;

		managers.get(key).close();

		managers.remove(key);

	}

	public void closeAllManagers() {

		for (Object key : managers.keySet()) {

			EntityManager m = managers.get(key);

			m.flush();

			m.createNativeQuery("SHUTDOWN;");

			m.close();

		}

		managers.clear();

	}

}

package org.txm.annotation.kr.core;

import java.io.File;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.HashMap;

import java.util.List;

import org.eclipse.core.runtime.IProgressMonitor;

import org.txm.annotation.kr.core.repository.AnnotationEffect;

import org.txm.annotation.kr.core.repository.AnnotationType;

import org.txm.annotation.kr.core.repository.TypedValue;

import org.txm.annotation.kr.core.storage.temporary.TemporaryAnnotationManager;

import org.txm.core.engines.Engine;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.searchengine.cqp.corpus.query.Match;

import org.txm.utils.logger.Log;

/**

 * Manage annotations and is able to return annotation saved in JPA

 * 

 * @author mdecorde

 *

 */

public class AnnotationManager implements Engine {

	MainCorpus corpus;

	TemporaryAnnotationManager tempManager;

	CQPAnnotationManager cqpManager;

	boolean dirty = false;

	public AnnotationManager(MainCorpus mainCorpus){

		this.corpus = mainCorpus;

	}

	public TemporaryAnnotationManager getTemporaryManager(){

		return tempManager;

	}

	public CQPAnnotationManager getCQPManager(){

		return cqpManager;

	}

	public boolean saveAnnotations() throws Exception{

		List<Annotation> annots = tempManager.getAnnotations();

		if (annots.isEmpty()) {

			System.out.println("No annotation to save. Aborting.");

			dirty = false;

			return true;

		}

		AnnotationWriter writer = new AnnotationWriter(corpus);

		if (writer.writeAnnotations(annots)) {

			Log.info("Annotations succesfully written. Deleting temporary annotations...");

			tempManager.deleteAnnotations();

			dirty = false;

			return true;

		}

		return false;

	}

	public boolean exportAnnotationsToSyMoGIH(File resultZipFile) throws Exception{

		AnnotationWriter writer = new AnnotationWriter(corpus);

		if (writer.writeAnnotationsInStandoff(resultZipFile)) {

			Log.info("Annotations succesfully written in "+resultZipFile);

			return true;

		}

		return false;

	}

	//TODO: not ended?

	/**

	 * Deletes the annotations stored in the temporary annotation manager

	 * @param type

	 * @param job

	 * @return

	 * @throws Exception

	 */

	public boolean deleteAnnotations(AnnotationType type, IProgressMonitor job) throws Exception {

		List<Annotation> temporaryAnnotations = null;

		List<Annotation> cqpAnnotations = null;

		try {

			temporaryAnnotations = tempManager.getAnnotations(type);

			tempManager.getEntityManager().getTransaction().begin();

			for (Annotation a : temporaryAnnotations){

				if (job != null && job.isCanceled()) {

					System.out.println("Delete annotation canceled.");

					return false;

				}

				tempManager.deleteAnnotation(type, a.getStart(), a.getEnd());

			}

			tempManager.getEntityManager().getTransaction().commit();

			cqpAnnotations = cqpManager.getAnnotations(type);

			tempManager.getEntityManager().getTransaction().begin();

			for (Annotation a : cqpAnnotations) {

				if (job != null && job.isCanceled()) {

					System.out.println("Delete annotation canceled.");

					return false;

				}

				String value = cqpManager.getCQPAnnotationValue(a.getStart(), a.getEnd(), type);

				if (value != null) {

					tempManager.createAnnotationNoCommit(type, new TypedValue("#del", "#del", type.getId()),  a.getStart(), a.getEnd());

				} else {

					tempManager.deleteAnnotationNoCommit(type, a.getStart(), a.getEnd());

				}

			}

			dirty = true;

			tempManager.getEntityManager().getTransaction().commit();

		} catch(Exception e) {

			e.printStackTrace();

			return false;

		}

		return true;

	}

	public boolean deleteAnnotations(AnnotationType type, List<Match> matches, IProgressMonitor job) throws Exception {

		try {

			tempManager.getEntityManager().getTransaction().begin();

			for (Match m : matches) {

				if (job != null && job.isCanceled()) {

					System.out.println("Delete annotation canceled.");

					return false;

				}

				String value = cqpManager.getCQPAnnotationValue(m.getStart(), m.getEnd(), type);

				if (value != null) {

					tempManager.createAnnotationNoCommit(type, new TypedValue("#del", "#del", type.getId()),  m.getStart(), m.getEnd());

				} else {

					tempManager.deleteAnnotationNoCommit(type, m.getStart(), m.getEnd());

				}

			}

			dirty = true;

			tempManager.getEntityManager().getTransaction().commit();

		} catch (Exception e) {

			e.printStackTrace();

			return false;

		}

		return true;

	}

	/**

	 * Returns the annotation saved in the temporary database and in the CQP corpus indexes

	 * 

	 * CQP Annotations must be shadowed by temporary annotations of the same type and positions

	 */

	public List<Annotation> getAnnotationsForMatches(AnnotationType type, List<Match> matches, boolean overlap) {

		List<Annotation> temporaryAnnotations = null;

		List<Annotation> resultAnnotations = new ArrayList<Annotation>();

		try {

			temporaryAnnotations = tempManager.getAnnotations(type, matches, null, false, overlap);

			temporaryAnnotations = tempManager.getAnnotationsForMatches(matches, temporaryAnnotations, overlap);

			List<Annotation> cqpAnnotations = cqpManager.getAnnotationsForMatches(matches, type, overlap);

		//	System.out.println("Temporary annotations: "+temporaryAnnotations);

		//	System.out.println("CQP annotations: "+cqpAnnotations);

			if (cqpAnnotations.size() != matches.size() || temporaryAnnotations.size() != matches.size()) {

				System.out.println("ERROR in getAnnotationsForMatches methods! ");

				return new ArrayList<Annotation>(matches.size());

			}

			// merge the 2 results

			for (int i = 0 ; i < matches.size() ; i++) {

				if (cqpAnnotations.get(i) == null && temporaryAnnotations.get(i) == null) {

					resultAnnotations.add(null);

				} else if (temporaryAnnotations.get(i) != null) {

					resultAnnotations.add(temporaryAnnotations.get(i));

				} else if (cqpAnnotations.get(i) != null) {

					resultAnnotations.add(cqpAnnotations.get(i));

				} else {

					resultAnnotations.add(null);

				}

			}

		} catch (Exception e) {

			e.printStackTrace();

			return new ArrayList<Annotation>(matches.size());

		}

		return resultAnnotations;

	}

	public void clearInstance() {

		try {

			tempManager.close();

		} catch(Exception e) {

			System.out.println("Fail to clear AnnotationManager instance: "+e);

			org.txm.utils.logger.Log.printStackTrace(e);

		}

	}

	public void checkData() {

		try {

			tempManager.checkData();

		} catch (Exception e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

	}

	public boolean hasChanges() {

		return tempManager.hasChanges();

	}

	/**

	 * 

	 * @param annotSelectedType not null

	 * @param annotSelectedTypedValue not null

	 * @param matches not null

	 * @param job may be null

	 * @return

	 */

	public HashMap<Match,List<Annotation>> createAnnotations(AnnotationType annotSelectedType,

			TypedValue annotSelectedTypedValue, List<Match> matches, IProgressMonitor job) {

		HashMap<Match,List<Annotation>> allAnnotationsThatCollides = new HashMap<Match, List<Annotation>>();

		tempManager.getEntityManager().getTransaction().begin(); // warning

		for (Match match : matches) {

			allAnnotationsThatCollides.put(match, new ArrayList<Annotation>());

			if (job != null && job.isCanceled()) { // check if user canceled the job

				System.out.println("Affect annotation canceled.");

				return null;

			}

			try {

				List<Annotation> cqpAnnotations = null;

				if (annotSelectedType.getEffect().equals(AnnotationEffect.SEGMENT)) {

					cqpAnnotations = cqpManager.getAnnotations(null, match, null, true); // get all annotations

					// remove A)the wrapping annotations and B) the annotation with same type and same positions

					for (int i = 0 ; i < cqpAnnotations.size() ; i++) {

						Annotation a = cqpAnnotations.get(i);

						// exact match + exact type

						if (a.getType().equals(annotSelectedType.getId()) && a.getStart() == match.getStart() && a.getEnd() == match.getEnd()) {

							cqpAnnotations.remove(i);

							i--;

						} else if (!a.getType().equals(annotSelectedType.getId()) && (// different type and inner or outer wrap

								(a.getStart() <= match.getStart() && match.getEnd() <= a.getEnd()) ||

								(match.getStart() <= a.getStart() && a.getEnd() <= match.getEnd())

								)) {

							cqpAnnotations.remove(i);

							i--;

						}

					}

				} else {

					// no need to test collision (AnnotationType=TOKEN)

					cqpAnnotations = new ArrayList<Annotation>();

				}

				if (cqpAnnotations.size() > 0) {

					allAnnotationsThatCollides.get(match).addAll(cqpAnnotations);

					continue; // don't create annotation, process next match

				} else { // test with temporary annotation manager 

					List<Annotation> tempAnnotations = null;

					if (match.getTarget() >= 0) { // the @ operator has been used, annotate only the @position

						tempAnnotations = tempManager.createAnnotationNoCommit(annotSelectedType, annotSelectedTypedValue, match.getTarget(), match.getTarget());

					} else {

						tempAnnotations = tempManager.createAnnotationNoCommit(annotSelectedType, annotSelectedTypedValue, match.getStart(), match.getEnd());

					}

					if (tempAnnotations.size() > 0)

						allAnnotationsThatCollides.get(match).addAll(tempAnnotations);

				}

			} catch (Exception e) {

				Log.printStackTrace(e);

				System.out.println("Error during annotation creation: "+e);

			}

			if (allAnnotationsThatCollides.get(match).size() == 0) allAnnotationsThatCollides.remove(match); // keep only colision lists

		}

		dirty = true;

		tempManager.getEntityManager().getTransaction().commit(); // warning

		// test if there are CQP annotations for the 

		return allAnnotationsThatCollides;

	}

	public List<Annotation> getAnnotations(AnnotationType type, int start, int end, boolean overlap) {

		List<Annotation> temporaryAnnotations = null;

		List<Annotation> cqpAnnotations = null;

		try {

			temporaryAnnotations = tempManager.getAnnotations(type, Arrays.asList(new Match(start, end)), null, false, overlap);

			cqpAnnotations = cqpManager.getAnnotations(type, start, end, overlap);

			int i = 0;

			for (Annotation a : cqpAnnotations) {

				while (temporaryAnnotations.get(i).getStart() < a.getStart()) {

					i++;

				}

				temporaryAnnotations.add(i, a);

			}

		} catch(Exception e) {

		}

		return temporaryAnnotations;

	}

	public List<Annotation> getAnnotations(AnnotationType type, int i, int j) {

		return getAnnotations(type, i, j, false);

	}

	public void closeAll() {

		Log.info("Closing annotation manager of "+corpus);

		tempManager.close();

		cqpManager.close();

	}

	public boolean isOpen() {

		return tempManager.getEntityManager() != null && tempManager.getEntityManager().isOpen();

	}

	@Override

	public String getName() {

		return "Annotation";

	}

	@Override

	public boolean getState() {

		return isOpen();

	}

	@Override

	public boolean initialize() throws Exception {

		tempManager = new TemporaryAnnotationManager(corpus);

		dirty = tempManager.getAnnotations().size() > 0;

		cqpManager = new CQPAnnotationManager(corpus);

		return false;

	}

	@Override

	public boolean start(IProgressMonitor arg0) throws Exception {

		return true;

	}

	@Override

	public boolean stop() throws Exception {

		return true;

	}

	/**

	 * 

	 * @return

	 */

	public boolean isDirty() {

		return dirty;

	}

}

package org.txm.annotation.kr.core;

import java.io.Serializable;

import javax.persistence.*;

@Embeddable

public class AnnotationPK implements Serializable {

	private static final long serialVersionUID = -2360693333015275209L;

	//corresponding to the start and end positions (in the corpus)

	private int startpos;

	private int endpos;

	private String refType;

	public AnnotationPK() {

	}

	public AnnotationPK(int start, int end, String refType) {

		this.startpos = start;

		this.endpos = end;

		this.refType = refType;

	}

	public void setRefType(String refType) {

		this.refType = refType;

	}

	public void setStartPosition(int start) {

		this.startpos = start;

	}

	public void setEndPosition(int end) {

		this.endpos = end;

	}

	public String getRefType() {

		return refType;

	}

	public int getStartPosition() {

		return startpos;

	}

	public int getEndPosition() {

		return endpos;

	}

	public boolean equals(Object obj) {

		if (obj == null) return false;

		if (obj == this) return true;

		if (!(obj instanceof AnnotationPK)) return false;

		AnnotationPK other = (AnnotationPK) obj;

		return startpos == other.startpos && endpos == other.endpos&& refType.equals(other.refType);

	}

	public int hashCode() {

		return refType.hashCode()+startpos+endpos;

	}

	public String toString() {

		return getRefType() + "["+getStartPosition()+"-"+getEndPosition()+"]" ;

	}

}

package org.txm.annotation.kr.core.conversion;

import java.io.BufferedReader;

import java.io.File;

import java.io.IOException;

import java.util.HashSet;

import java.util.LinkedHashMap;

import java.util.regex.Pattern;

import javax.xml.stream.XMLStreamException;

import org.apache.commons.lang.StringUtils;

import org.txm.Toolbox;

import org.txm.core.preferences.TBXPreferences;

import org.txm.searchengine.cqp.corpus.MainCorpus;

import org.txm.stat.utils.ConsoleProgressBar;

import org.txm.utils.io.IOUtils;

public class CorpusRuledConvertion {

	LinkedHashMap<Pattern, String> rules = new LinkedHashMap<Pattern, String>();

	private String oldType;

	private String newType;

	public CorpusRuledConvertion(File conversionFile, String oldType, String newType) throws IOException {

		this.oldType = oldType;

		this.newType = newType;

		BufferedReader reader = IOUtils.getReader(conversionFile);

		String line = reader.readLine();

		while (line != null) {

			int idx = line.indexOf("\t");

			if (idx > 0) {

				String k = line.substring(0, idx);

				String v = line.substring(idx +1);

				rules.put(Pattern.compile(k), v);

			}

			line = reader.readLine();

		}

		System.out.println("Conversion rules: "+rules);

	}

	public CorpusRuledConvertion(LinkedHashMap<Pattern, String> rules,

			String oldType, String newType) {

		this.oldType = oldType;

		this.newType = newType;

		this.rules = rules;

	}

	public boolean process(MainCorpus corpus) throws XMLStreamException, IOException {

		File binaryCorpusDirectory = corpus.getBaseDirectory();

		File txmDirectory = new File(binaryCorpusDirectory, "txm");

		if (!txmDirectory.exists()) {

			System.out.println("'txm' directory not found in "+binaryCorpusDirectory.getAbsolutePath());

			return false;

		}

		File txmCorpusDirectory = new File(txmDirectory, corpus.getName());

		if (!txmCorpusDirectory.exists()) {

			System.out.println("'"+corpus.getName()+"' corpus directory not found in "+txmDirectory.getAbsolutePath());

			return false;

		}

		File[] files = txmCorpusDirectory.listFiles();

		if (files == null || files.length == 0) {

			System.out.println("No file in "+txmCorpusDirectory);

			return false;

		}

		ConsoleProgressBar cpb = new ConsoleProgressBar(files.length);

		for (File xmlFile : files) {

			cpb.tick();

			if (xmlFile.isDirectory()) continue;

			if (xmlFile.isHidden()) continue;

			if (!xmlFile.getName().endsWith(".xml")) continue;

			File tmpFile = new File(xmlFile.getParentFile(), "tmp_"+xmlFile.getName());

			XMLTXMFileRuledConversion converter = new XMLTXMFileRuledConversion(xmlFile, rules, oldType, newType, XMLTXMFileRuledConversion.ABANDON);

			if (converter.process(tmpFile)) {

				xmlFile.delete();

				//				try {

				//					FileCopy.copy(tmpFile, new File("/tmp/"+tmpFile.getName()));

				//				} catch (IOException e) {

				//					// TODO Auto-generated catch block

				//					e.printStackTrace();

				//				}

				HashSet<String> errors = converter.getNoMatchValues();

				if (errors.size() > 0) {

					System.out.println("Some values did not match rule:");

					int i = 0;

					for (String error : errors) {

						System.out.println("\t"+error);

						if (i >= 10) break;

					}

					if (errors.size() > 10) {

						try {

							File errorFile = new File(Toolbox.getTXMHOMEPATH(), "errors.txt");

							IOUtils.write(errorFile, StringUtils.join(errors, "\t"));

							System.out.println("More errors, see "+errorFile.getAbsolutePath());

						} catch (Exception e) {

							e.printStackTrace();

						}

					}

					return false;

				}

				tmpFile.renameTo(xmlFile);

				if (tmpFile.exists()) {

					System.out.println("Could not replace original file with the result file. "+xmlFile+ " with "+tmpFile);

					return false;

				}

			} else {

				System.out.println("Fail to process "+xmlFile);

				return false;

			}

		}

		System.out.println(); // end of ConsoleProgressBar

		return true;

	}

}

package org.txm.annotation.kr.core.conversion;

import java.io.File;

import java.io.IOException;

import java.util.HashSet;

import java.util.LinkedHashMap;

import java.util.regex.Pattern;

import javax.xml.stream.XMLStreamException;

import org.txm.importer.StaxIdentityParser;

public class XMLTXMFileRuledConversion extends StaxIdentityParser {

	protected LinkedHashMap<Pattern, String> rules = new LinkedHashMap<Pattern, String>();

	protected String oldType;

	protected String newType;

	public static final String DELETE = "supprimer";

	public static final String COPY = "copier";

	public static final String ABANDON = "abandon";

	HashSet<String> noMatchValues = new HashSet<String>();

	public XMLTXMFileRuledConversion(File infile, LinkedHashMap<Pattern, String> rules, String oldType, String newType, String mode) throws IOException, XMLStreamException {

		super(infile);

		this.rules = rules;

		this.oldType = oldType;

		this.newType = newType;

		this.mode = mode;

		if (!this.newType.startsWith("#")) this.newType = "#"+this.newType;

		if (!this.oldType.startsWith("#")) this.oldType = "#"+this.oldType;

	}

	boolean inW = false, inAna = false, inForm;

	LinkedHashMap<String, String> anaValues = new LinkedHashMap<String, String>();

	LinkedHashMap<String, String> anaResps = new LinkedHashMap<String, String>();

	String typeName = null;

	String respName = null;

	String formValue, typeValue = null;

	private Object mode;

	@Override

	public void processStartElement() throws XMLStreamException, IOException {

		if (!inW) super.processStartElement(); // don't write W content

		if (localname.equals("w")) {

			inW = true;

			anaValues.clear();

			anaResps.clear();

			//initialize the new type to a empty value in case there is transformation rule

			anaValues.put(newType, "");

			anaResps.put(newType, "#txm_recode");

		} else if (localname.equals("ana")) {

			inAna = true;

			typeName = parser.getAttributeValue(null, "type");

			respName = parser.getAttributeValue(null, "resp");

			anaResps.put(typeName, respName);

			//if (typeName != null) typeName = typeName.substring(1); // remove #

			typeValue = "";

		} else if (localname.equals("form")) {

			inForm = true;

			formValue = "";

		} 

	}

	@Override

	public void processCharacters() throws XMLStreamException {

		if (inW && inAna) typeValue+=parser.getText();

		else if (inW && inForm) formValue+=parser.getText();

		else super.processCharacters();

	}

	@Override

	public void processEndElement() throws XMLStreamException {

		if (localname.equals("w")) {

			inW = false;

			// write W content

			try {

				// get the value to test

				String value = null;

				if (oldType.equals("word")) {

					value = formValue;

				} else {

					value = anaValues.get(oldType);

				}

				if (newType.equals("word")) { // update form property

					updateFormValueIfMatch(value);

				} else { // update another word property

					if (value != null) {

						updateAnaValuesIfMatch(value);

					}

				}

				// write the word element

				writer.writeStartElement("txm:form");

				writer.writeCharacters(formValue);

				writer.writeEndElement();

				for (String k : anaValues.keySet()) {

					String resp = anaResps.get(k);

					if (resp == null) resp = "#txm_recode";

					writer.writeStartElement("txm:ana");

					writer.writeAttribute("resp", resp);

					writer.writeAttribute("type", k);

					writer.writeCharacters(anaValues.get(k));

					writer.writeEndElement();

				}

			} catch (XMLStreamException e) {

				e.printStackTrace();

			}

		} else if (localname.equals("ana")) {

			anaValues.put(typeName, typeValue);

			inAna = false;

		} else if (localname.equals("form")) {

			inForm = false;

		} 

		if (!inW) super.processEndElement(); // don't write W content

	}

	protected void updateFormValueIfMatch(String value) {

		for (Pattern rule : rules.keySet()) {

			if (rule.matcher(value).matches()) {

				formValue = rules.get(rule);

				return; // ok stop

			}

		}

		noMatchValues.add(value);

	}

	protected void updateAnaValuesIfMatch(String value) {

		for (Pattern rule : rules.keySet()) {

			if (rule.matcher(value).matches()) {

				value = rules.get(rule);

				anaValues.put(newType, value); // do a replace if newType == oldType :-)

				anaResps.put(newType, "#txm_recode");

				return;

			}

		}

		if (DELETE.equals(mode)) anaValues.put(newType, ""); // do a replace if newType == oldType :-)

		else if (ABANDON.equals(mode)) anaValues.put(newType, "ERROR("+value+")"); // do a replace if newType == oldType :-)

		noMatchValues.add(value);

	}

	public HashSet<String> getNoMatchValues() {

		return noMatchValues;

	}

	public static void main(String args[]) {

		try {

		File xmlFile = new File("/home/mdecorde/TXM/corpora/XTZTEXTUALPLANS/txm/XTZTEXTUALPLANS/test.xml");

		File tmpFile = new File("/home/mdecorde/TXM/corpora/XTZTEXTUALPLANS/txm/XTZTEXTUALPLANS/test-o.xml");

		String oldType = "type";

		String newType = "type";

		LinkedHashMap<Pattern, String> rules = new LinkedHashMap<Pattern, String>();

		rules.put(Pattern.compile("w"), "WORD");

		rules.put(Pattern.compile("x.+"), "XWORD");

		rules.put(Pattern.compile("y"), "YWORD");

		rules.put(Pattern.compile("y.*"), "YMULTIWORD");

		XMLTXMFileRuledConversion converter = new XMLTXMFileRuledConversion(xmlFile, rules, oldType, newType, ABANDON);

		System.out.println(converter.process(tmpFile));

		} catch (Exception e) {

			e.printStackTrace();

		}

	}

}

package org.txm.annotation.kr.core;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.net.MalformedURLException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLOutputFactory;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.XMLStreamWriter;

import org.txm.annotation.kr.core.repository.AnnotationType;

import org.txm.importer.StaxIdentityParser;

/**

 * The Class AnnotationStandoffInjector.

 *

 * @author sgedzelman, mdecorde

 *

 * copy a XML-TXM file without annotations elements

 * and creates annotations XML-TEI-SymoGIH annotation files for each annotation author

 * 

 */

public class AnnotationSyMoGIHWriter extends StaxIdentityParser {

	File xmlStandOffDirectory;

	boolean debug = false;

	String currentRef ;

	String currentAuthor ;

	String currentDate;

	String currentStartPos ;

	String currentEndPos ;

	//read xmlFile, to find annotations and update/write to xmlstandofffile

	//order annotations by annotator

	////// order annotations by date

	HashSet<String> types = new HashSet<String>();

	ArrayList<String> positions;

	HashMap<String, ArrayList<String>> annotationsPositions;

	XMLStreamWriter currentWriter;

	XMLStreamWriter standoffWriter;

	String currentType;

	boolean startAnnotation = false;

	private String filename;

	// author -> date -> annotation_values

	HashMap<String, HashMap<String, ArrayList<String[]>>> allannotations = new HashMap<String, HashMap<String, ArrayList<String[]>>>();

	HashMap<String, HashMap<String, ArrayList<ArrayList<String>>>> allannotationspositions = new HashMap<String, HashMap<String, ArrayList<ArrayList<String>>>>();

	private String textid;

	/**

	 * 

	 * @param xmlStandOffFile

	 * @param tokenAnnotations 

	 * @param annotationsToAddByStartPos

	 * @param annotationsToAddByEndPos

	 * @param corpus_start_position

	 * @param debug

	 * @throws XMLStreamException 

	 * @throws IOException 

	 */

	public AnnotationSyMoGIHWriter(String textid, File xmlFile, File xmlStandOffDirectory, List<AnnotationType> types, boolean debug) throws IOException, XMLStreamException {

		super(xmlFile.toURI().toURL()); // init reader and writer

		this.textid = textid;

		this.filename = xmlFile.getName();

		this.debug = debug;

		this.xmlStandOffDirectory = xmlStandOffDirectory;

		factory = XMLInputFactory.newInstance();

		annotationsPositions = new HashMap<String, ArrayList<String>>();

		//System.out.println("AnnotationStandoff -  ");

		for (AnnotationType type : types){

			this.types.add(type.getId()); //.getName().toLowerCase());

			//System.out.println("Annotation Types in TXM : "+type.getName().toLowerCase() +" vs "+ type.getId());

		}

	}

	/*<TEI xmlns="http://www.tei-c.org/ns/1.0">

    <teiHeader>

        <fileDesc>

            <titleStmt>

                <title>Title</title>

            </titleStmt>

            <publicationStmt>

                <p>Publication Information</p>

            </publicationStmt>

            <sourceDesc>

                <p>Ce document permet l'annotation sémantique de tous les textes concernant l'association avec des unités de connaissance</p>

            </sourceDesc>

        </fileDesc>

    </teiHeader>

    <text>

        <body>

            <div>

                <div>

                    <!-- La date dans le header indique la date d'annotation -->

                    <head>

                        <date type="annotation_date" when="2016-06-16"/>

                    </head>

                    <span type="identification d'entités nommées" ana="CoAc56389" 

                          target="#w_article_baip_1254-0714_1850_num_01_005_974_tei_2152 

                                  #w_article_baip_1254-0714_1850_num_01_005_974_tei_2153 

                                  #w_article_baip_1254-0714_1850_num_01_005_974_tei_2154 

                                  #w_article_baip_1254-0714_1850_num_01_005_974_tei_2155 

                                  #w_article_baip_1254-0714_1850_num_01_005_974_tei_2156" />

                </div>

            </div>

         </body>

      </text>

 </TEI>*/

	protected void processStartElement() throws XMLStreamException, IOException {

		//<coac author="gazelledess" ref="CoAc397" start="5" end="5">

		/*if(localname.startsWith("actr")){

			System.out.println("Check existence of actr in Corpus !!!! "+localname);

		}*/

		boolean foundAnnot = false;

		if (types.contains(localname) && parser.getPrefix().equals("txm")) { // don't write txm annotation elements

			//System.out.println(" START "+ localname);

			foundAnnot = true;

			currentType = localname;

			//<txm:actr author="gazelledess" ref="PhileasFogg" date="2016-09-05" start="56" end="57">

			currentAuthor = parser.getAttributeValue(null, "author");

			currentRef = parser.getAttributeValue(null, "ref");

			currentStartPos = parser.getAttributeValue(null, "start");

			currentEndPos = parser.getAttributeValue(null, "end");

			currentDate = parser.getAttributeValue(null, "date");

			//annotation is here

			startAnnotation = true;

			positions = new ArrayList<String>();

			annotationsPositions.put(currentType, positions);

			// initialize allannotations

			if (!allannotations.containsKey(currentAuthor)) {

				allannotations.put(currentAuthor, new HashMap<String, ArrayList<String[]>>());

				allannotationspositions.put(currentAuthor, new HashMap<String, ArrayList<ArrayList<String>>>());

			}

			HashMap<String, ArrayList<String[]>> authorsAnnotation = allannotations.get(currentAuthor);

			HashMap<String, ArrayList<ArrayList<String>>> authorsAnnotationPositions = allannotationspositions.get(currentAuthor);

			if (!authorsAnnotation.containsKey(currentDate)) {

				authorsAnnotation.put(currentDate, new ArrayList<String[]>());

				authorsAnnotationPositions.put(currentDate, new ArrayList<ArrayList<String>>());

			}

			storeAnnotation();

		}

		if (!foundAnnot) {

			super.processStartElement(); /// continue writing in file all elements, except the tags that are now in stand-off files

			// get words ids of the current annotations

			if (localname.equals("w") && startAnnotation) {

				String id = parser.getAttributeValue(null, "id");

				for (String typeIn : annotationsPositions.keySet()) {

					positions = annotationsPositions.get(typeIn);

					positions.add(id);

					//System.out.println("Positions of w id="+posW+" for ["+typeIn+"] ");

				}

			}

		}

	}

	/**

	 * ends the current author stand-off file

	 * @param currentWriter

	 */

	private void writeEndStandOffFile(XMLStreamWriter currentWriter){

		//System.out.println("writeEndStandOffFile ...");

		try {

			currentWriter.writeEndElement(); 

			currentWriter.writeEndDocument();

			currentWriter.flush();

			currentWriter.close();

		} catch (XMLStreamException e) {

			e.printStackTrace();

		}

	}

	/**

	 * Create the stand-off file for one author

	 * @param file

	 * @return

	 */

	private XMLStreamWriter writeStartStandOffFile(File file){

		//System.out.println("writeStartStandOffFile ...");

		String ns = "http://www.tei-c.org/ns/1.0";    

		XMLOutputFactory output = XMLOutputFactory.newInstance();

		XMLStreamWriter writer = null ;

		try {

			writer = output.createXMLStreamWriter(new FileWriter(file));

			writer.writeStartDocument();

			writer.setPrefix("tei", ns);

			writer.setDefaultNamespace(ns);

			writer.writeStartElement("TEI");

			writer.writeStartElement("teiHeader");

			writer.writeStartElement("fileDesc");

			writer.writeStartElement("titleStmt");

			writer.writeStartElement("title");

			writer.writeCharacters(textid); 

			writer.writeEndElement(); // title

			writer.writeEndElement(); // titleStmt

			writer.writeStartElement("publicationStmt");

			writer.writeStartElement("p");

			writer.writeCharacters("PUBLICATION INFO à renseigner");

			writer.writeEndElement(); // p

			writer.writeEndElement(); // publicationStmt

			writer.writeStartElement("sourceDesc");

			writer.writeStartElement("p");

			writer.writeCharacters("Ce document permet l'annotation sémantique de tous les textes, par auteur");

			writer.writeEndElement(); // p

			writer.writeEndElement(); // sourceDesc

			writer.writeEndElement(); // </fileDesc>

			writer.writeStartElement("encodingDesc");

			writer.writeStartElement("projectDesc");

			writer.writeStartElement("p");

			writer.writeCharacters("Annotations created by "+currentAuthor+", for the use in Symogih XML platform");

			writer.writeEndElement(); // p

			writer.writeEndElement(); // </projectDesc>

			writer.writeEndElement(); // </encodingDesc>

			writer.writeEndElement(); // </teiHeader>

			writer.writeStartElement("text");

			writer.writeStartElement("body");

			writer.writeCharacters("\n");

			writer.writeStartElement("div");

			writer.writeCharacters("\n");

		} catch (XMLStreamException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		} catch (IOException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

		return writer;

	}

	private void storeAnnotation() {

		allannotations.get(currentAuthor).get(currentDate).add(new String[]{currentDate, currentRef, currentType});

	}

	private void storeAnnotationPositions() {

		allannotationspositions.get(currentAuthor).get(currentDate).add(positions);

	}

	/**

	 * write stand-off annotation

	 */

	private void writeStartAnnotationToStandoffFile(String[] data, ArrayList<String> positions) {

		//System.out.println("writeStartAnnotationToStandoffFile ...");

		try {

			currentWriter.writeStartElement("span");

			currentWriter.writeAttribute("type","named_entities_identifications");

			currentWriter.writeAttribute("ana", data[1]);

			StringBuffer listWids = new StringBuffer();

			for (String posW : positions) {

				listWids.append("#"+posW+" ");

			}

			currentWriter.writeAttribute("target", listWids.toString().trim());

			currentWriter.writeComment("type="+data[2]);

			currentWriter.writeEndElement(); // span

			currentWriter.writeCharacters("\n");

		} catch (XMLStreamException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

	}

	protected void processEndElement() throws XMLStreamException {

		boolean foundAnnot = false;

		if (types.contains(localname) && parser.getPrefix().equals("txm")) { // skip annotation end element

			//System.out.println(" END "+ localname);

			foundAnnot = true;

			//annotation ends here

			storeAnnotationPositions();

			if (annotationsPositions.containsKey(localname)) {

				annotationsPositions.remove(localname);