Laboratoire ICAR » Plateforme TXM

			println "Failed to prepare files - Aborting";

			return;

		}

		println "-- Xml Validation"

		if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

		cpb.done()

		if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

		boolean annotationSuccess = false;

		if (annotate) {

 * The Class compiler.

 */

class compiler {

	boolean ADD_TEXTID_TO_REF = true

	/** The input data. */

	private def inputData;

	/** The factory. */

	private def factory;

	/** The parser. */

	private XMLStreamReader parser;

	/** The output. */

	OutputStreamWriter output;

	/** The basename. */

	String corpusname;

	/** The projectname. */

	String projectname

	/** The outdir. */

	String outdir;

	/** The debug. */

	boolean debug = false;

	/** The indexInterviewer: index interviewer speech if true. */

	boolean indexInterviewer = true;

	/** The trans. */

	HashMap<String, ArrayList<Pair<String, String>>> trans;

	/** The speakers. */

	HashMap<String, ArrayList<Pair<String, String>>> speakers;

	/** The speakersname. */

	HashMap<String, String> speakersname = new HashMap<String, String>();

	/** The topics. */

	HashMap<String, ArrayList<Pair<String, String>>> topics;

	/** The interviewers regex */

	def interviewers = null

	static LinkedHashSet<String> sectionAttrs;

	static LinkedHashSet<String> spAttrs;

	static LinkedHashSet<String> uAttrs;

	/** The anatypes. */

	private static anatypes = []

	private static anavalues = [:]

	/**

	 * Removes the interviewers.

	 *

	public setIndexInterviewer(boolean value) {

		this.indexInterviewer = value;

	}

	File cqpFile

	/**

	 * Run.

	 *

		this.outdir = binDir;

		this.corpusname = corpusname;

		this.projectname = projectname;

		anatypes = ["event"] // reset

		anavalues = [:] // reset

		sectionAttrs = new LinkedHashSet<String>() // reset section attributs set

		spAttrs = new LinkedHashSet<String>() // reset section attributs set

		uAttrs = new LinkedHashSet<String>() // reset section attributs set

		CorpusBuild corpus = project.getCorpusBuild(project.getName(), MainCorpus.class);

		if (corpus != null) {

			if (project.getDoUpdate()) {

		new File(binDir,"cqp").mkdirs()

		new File(binDir,"data").mkdirs()

		new File(binDir,"registry").mkdirs()

		// get all anatypes

		for (File f : xmlfiles) {

			getAnaTypes(f)

		}

		//println "ANATYPES: "+anatypes

		if (!createOutput(cqpFile)) return false;

		output.write("<txmcorpus lang=\"fr\">\n")

		output.close();

		println("Compiling "+xmlfiles.size()+" files")

		ConsoleProgressBar cpb = new ConsoleProgressBar(xmlfiles.size())

		for (File txmFile :xmlfiles) {

		if (!createOutput(cqpFile)) return false;

		output.write("</txmcorpus>\n")

		output.close();

		//2- Import into CWB

		File registryFile = new File(binDir, "registry/"+corpusname.toLowerCase())

		File dataDir = new File(binDir, "data/$corpusname")

		new File(binDir, "registry").mkdir();

		if (!new File(binDir, "registry").exists()) {

			println "Can't create registry directory"

			return false;

		}

		if (!(CwbEncode.isExecutableAvailable() && CwbMakeAll.isExecutableAvailable())) {

			println ("Error: CWB executables not well set.")

			return false;

		CwbMakeAll cwbMa = new CwbMakeAll();

		cwbEn.setDebug(debug);

		cwbMa.setDebug(debug);

		String uAttr = "u:0";

		for (String attr : uAttrs) {

			uAttr += "+"+attr

					if (ignoreTranscriberMetadata) {

						String meta =p.getFirst();

						if (meta != "scribe" && meta != "audio_filename" &&

							textAttr+="+"+meta

					} else {

						textAttr+="+"+p.getFirst()

				break;

			}

		}

		List<String> pargs = ["spk", "ref", "id", "entitytype", "entityid"]

		for (String ana : anatypes) if (!pargs.contains(ana)) pargs.add(ana)

		String[] pAttributes = pargs

		String[] sAttributes = ["txmcorpus:0+lang", uAttr , textAttr, "event:0+id+desc+type+extent", sectionAttr, spAttr];

		println "pAttributes: $pAttributes"

		println "sAttributes: $sAttributes"

		//return;

				return false;

			}

			cwbMa.run(corpusname, registryFile.getParent());

		} catch (Exception ex) {System.out.println(ex); return false;}

		if (project.getCleanAfterBuild()) {

			new File(project.getProjectDirectory(), "cqp").deleteDir()

		}

		return true;

	}

	/**

	 * Creates the output.

	 *

			return false;

		}

	}

	/** The text_id. */

	String text_id

	/** The u opened. */

	boolean uOpened = false;

	/** The idturn. */

	int idturn = 1;

	/** The idsection. */

	int idsection = 1;

	/** The idu. */

	int idu = 1;

	/** The idevent. */

	int idevent = 1;

	/** The events. */

	List<String> events = [];

	static int vEntityId = 0;

	static int vEntityIdCount = 1;

	/**

	 * Process.

	 *

	private boolean process(File xmlfile) {

		text_id = xmlfile.getName();

		text_id = text_id.substring(0, text_id.length() -4);

		idturn = 1;

		idsection = 1;

		idu = 1;

		boolean flagAna;

		boolean flagForm;

		boolean flagWord;

		String anatype = "";

		String anaresp = "";

		String anavalue = "";

		String formatedTime;

		LinkedHashMap<String, String> anahash = new LinkedHashMap<String, String>();

		String currentType;

		URL url = xmlfile.toURI().toURL();

		inputData = url.openStream();

		factory = XMLInputFactory.newInstance();

		parser = factory.createXMLStreamReader(inputData);

		String filename = xmlfile.getName()

		String textid = filename.substring(0, filename.length() - 4);

		createOutput(cqpFile);

		String localname;

		//get all metadatas declared before Episode tag

		speakers = new HashMap<String, ArrayList<Pair<String, String>>>();

		trans = new HashMap<String, ArrayList<Pair<String, String>>>();

		topics = new HashMap<String, ArrayList<Pair<String, String>>>();

		//println "parse infos"

		parseInfos();

		//		println "Trans: $trans"

		//		println "Topics: $topics"

		//		println "Speakers: $speakers"

		//		}

		//		println "Trans properties: "+transproperties

		List<String> localspeakers;

		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

			//print "event: "+event +" "

			switch (event) {

				case XMLStreamConstants.START_ELEMENT:

					localname = parser.getLocalName();

				//println localname

					switch(localname) {

						case "div":

							output.write("<div");

							for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {

								output.write(" "+name+"=\""+parser.getAttributeValue(i).replace("\"", """)+"\"");

								sectionAttrs << name

							}

							output.write ">\n"

							break;

						case "sp":

								output.write(" "+name+"=\""+parser.getAttributeValue(i).replace("\"", """)+"\"");

								spAttrs << name

							}

							output.write ">\n"

							break;

						case "u":

							for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {

								String name = parser.getAttributeLocalName(i).replace("_","").toLowerCase()

								output.write(" "+name+"=\""+parser.getAttributeValue(i).replace("\"", """)+"\"");

								}

								uAttrs << name

							}

							output.write ">\n"

							break;

						case "event":

							output.write("<event");

							writeAttributes();

							output.write ">\n"

							if (parser.getAttributeValue(null, "type") == "entities") {

								if (parser.getAttributeValue(null, "extent") == "begin") {

									vEntityType = parser.getAttributeValue(null, "desc");

					break;

				case XMLStreamConstants.END_ELEMENT:

					localname = parser.getLocalName();

					switch (localname) {

						case "text":

							output.write("</text>\n")

							String ref = (u_name+", "+formatedTime+""+isEnq)

							if (ADD_TEXTID_TO_REF) ref = textid+", "+ref

							vForm +="\t"+u_name+"\t"+ref

						// concat entity and entity ID

							vAna+= "\t"+vEntityType+"\t"+vEntityId;

						//concat ana values

							for (String type : anatypes) {

								def v = anavalues.get(type);

								if (v == null) v = "";

								if ("event" == type) {

									if (v.length() > 0)

										vAna+="\t#"+v;

									else

										vAna+="\t";

									//concat <Event> values

									if (vEvents != null && vEvents.length() > 0 && vEvents != "N/A")

										vAna += "#"+vEvents;

									vAna+="\t"+v;

								}

							}

							vForm = vForm.replaceAll("\n", "").replaceAll("&", "&amp;").replaceAll("<", "&lt;");

							if (interviewers != null && !indexInterviewer) { // we must remove some words

								if (!interviewers.matches(u_name)) { // keep what is now an interviewer

									output.write(vForm+"\t"+wordid+vAna+"\n");

							} else {

								output.write(vForm+"\t"+wordid+vAna+"\n");

							}

							vAna = "";

							vForm = "";

							break;

					break;

			}

		}

		parser.close();

		inputData.close();

		output.close();

		return true;

	}

	/** The u_name. */

	String u_name;

	/**

	 * Write start tag.

	 */

		writeAttributes();

		output.write ">\n"

	}

	/**

	 * Write attributes.

	 */

			output.write(" "+parser.getAttributeLocalName(i).replace("_","").toLowerCase()+"=\""+parser.getAttributeValue(i).replace("\"", """)+"\"");

		}

	}

	private void getAnaTypes(File xmlFile) {

		inputData = xmlFile.toURI().toURL().openStream();

		factory = XMLInputFactory.newInstance();

			}

		}

	}

	/**

	 * Write start tag.

	 *

		writeAttributes();

		output.write ">\n"

	}

	/**

	 * Write end tag.

	 */

	private void writeEndTag() {

		output.write("</"+parser.getLocalName().toLowerCase()+">\n");

	}

	/** The ignore transcriber metadata. */

	boolean ignoreTranscriberMetadata = false;

	/**

	 * Sets the ignore transcriber metadata.

	 *

	public void setIgnoreTranscriberMetadata(boolean state) {

		this.ignoreTranscriberMetadata = state;

	}

	public static int MAXATTRIBUTEVALUELENGTH = 8191;

	/**

		for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

			if (event == XMLStreamConstants.START_ELEMENT) {

				localname = parser.getLocalName();

				switch (localname) {

					case "text":

						output.write("<text project=\""+projectname+"\" base=\""+corpusname+"\"")

							String value = parser.getAttributeValue(i).replace("\"", """);

							if (value.length() > MAXATTRIBUTEVALUELENGTH) {

								println "WARNING: attribute value is too long ( > $MAXATTRIBUTEVALUELENGTH). The value will be truncated to: $value"

							}

							if (ignoreTranscriberMetadata) {

								if (parser.getAttributeLocalName(i) != "scribe" &&

									output.write(" "+parser.getAttributeLocalName(i).replace("_","").toLowerCase()+"=\""+value+"\"");

								}

							} else {

								output.write(" "+parser.getAttributeLocalName(i).replace("_","").toLowerCase()+"=\""+value+"\"");

							}

						}

						output.write ">\n"

						ArrayList list = new ArrayList<Pair<String, String>>()

						trans.put("trans", list);

						for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {

							list.add(new Pair(parser.getAttributeLocalName(i).replace("_","").toLowerCase(), parser.getAttributeValue(i)));

							if (parser.getAttributeLocalName(i).equals("interviewer-id-regex"))

						if (id != null) {

							ArrayList list = new ArrayList<Pair<String, String>>()

							speakers.put(id, list);

							for (int i = 0 ; i < parser.getAttributeCount() ; i ++) {

								list.add(new Pair(parser.getAttributeLocalName(i), parser.getAttributeValue(i)));

							}

			}

		}

	}

	/**

	 * Sets the debug.

	 */

		this.normalizeFormValues = n

	}

		this.xmlFile = xmlFile

		this.cqpFile = cqpFile

		this.textname = textname

	 * @return true, if successful

	 */

	public boolean process() {

		if (!createOutput(cqpFile)) {

			return false

		}

		boolean foundtei = false;

		boolean foundtext = false;

		int nWords = 0;

		try {

			String localname;

			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

				switch (event) {

					case XMLStreamConstants.START_ELEMENT:

						localname = parser.getLocalName().toLowerCase();

						if ("tei".equals(localname)) foundtei = true;

						switch (localname) {

							case "text":

								if (!foundtei || !foundtext) break;

								output.write("<"+localname)

								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

									String attrname = parser.getAttributeLocalName(i)

									if (normalizeAttributeValues) {

										attrvalue = attrvalue.trim()

									}

								}

								if (parser.getAttributeCount() == 0) { // add the n attribute

									if (!ncounts.containsKey(localname)) ncounts.put(localname, 0)

									int ncount = ncounts.get(localname)

									ncounts.put(localname, ncount+1)

									output.write(" n=\""+ncount+QUOTE)

								}

								output.write(">\n")

						}

						break;

					case XMLStreamConstants.END_ELEMENT:

						localname = parser.getLocalName().toLowerCase();

						switch (localname) {

							case WTAG:

								for (String type : anatypes) {

	String wtag;

	boolean debug = false;

		inputData = xmlFile.toURI().toURL().openStream()

		factory = XMLInputFactory.newInstance()

		parser = factory.createXMLStreamReader(inputData)

import org.txm.searchengine.cqp.corpus.*

class XTZCompiler extends Compiler {

	SAttributesListener sattrsListener; // store scanned structures

	String regPath;

	String corpusname;

	String wtag;

	boolean doNormalizeAttributeValues = false;

	boolean doNormalizeAnaValues = true;

	public XTZCompiler(ImportModule module) {

		super(module);

		corpusname = module.getProject().getName();

		regPath = module.getBinaryDirectory().getAbsolutePath() + "/registry/"+corpusname.toLowerCase()

		wtag = module.getProject().getTokenizerWordElement();

		doNormalizeAttributeValues = "true".equals(module.getProject().getPreferencesScope().getNode("import").get(TBXPreferences.NORMALISEATTRIBUTEVALUES, "false"))

		doNormalizeAnaValues = "true".equals(module.getProject().getPreferencesScope().getNode("import").get(TBXPreferences.NORMALISEANAVALUES, "false"))

	}

	/**

	 * the Text list

	 */

	def initialTypesValues;

	@Override

	public void _process() {

		if (orderedTextIDs == null) orderedTextIDs = module.getProject().getTextsID() ;

		Project project = module.getProject();

		texts = orderedTextIDs.collect() { id -> module.getProject().getText(id) }

		textsToProcess = getTextsToProcess(texts)

		// get all structures

		sattrsListener = new SAttributesListener() // will store the structure and properties declaration

		sattrsListener.W = wtag

		File registryFile = new File(regPath)

		initialTypesValues = new HashSet<String>()

			corpus.setName(project.getName());

		}

		corpus.setDescription("Built with the XTZ import module");

		if (!doScanStep()) return;

		if (!doCQPStep()) return;

		if (!doCWBEncodeStep()) return;

		if (!doCWBMakeAllStep()) return;

			new File(module.getBinaryDirectory(), "cqp").deleteDir()

		}

		isSuccessFul = true;

	}

	/**

	 * Scan all XML-TXM files to find out structures and word properties

	 */

	public boolean doScanStep() {

		println "-- Scanning structures&properties to create for "+texts.size()+" texts..."

		//		def initialTypesValues = new HashSet<String>()

		//		initialTypesValues.addAll(sattrsListener.getAnatypes())

		// get all word properties

		ConsoleProgressBar cpb = new ConsoleProgressBar(texts.size())

		for (Text t : texts) {

				return false;

			}

		}

		if (initialTypesValues.size() == sattrsListener.getAnatypes().size()

			// no new property

		} else {

			if (module.isUpdatingCorpus()) {

			textsToProcess.clear()

			textsToProcess.addAll(texts)

		}

		println ""

		return true;

	}

	def getTextsToProcess(def texts) {

		def textsToProcess = texts.findAll() { text ->

			File xmlFile = text.getXMLTXMFile()

			String textname = text.getName()

			File cqpFile = new File(cqpDirectory, textname + ".cqp")

			cqpFiles << cqpFile // insert cqp files to concat later

			// skip step if cqpFile exists AND is more recent than the XML-TXM File

			if (!cqpFile.exists() || xmlFile.lastModified() >= cqpFile.lastModified()) {

				return true

			}

			if (!text.isDirty() && !mustBuild) {

				Log.finer("skipping .cqp step of $text");

				return false

			}

			return true

		}

		return textsToProcess

	}

	def cqpFiles = [] // ordered cqp files to concat before calling cwb-encode

	int cqpFilesUpdated = 0;

	public boolean doCQPStep() {

		cqpDirectory.mkdir(); // if not created

		println "-- Building CQP files ${textsToProcess.size()}/${texts.size()}..."

		ConsoleProgressBar cpb = new ConsoleProgressBar(textsToProcess.size())

		cqpFilesUpdated = 0;

		for (Text text : textsToProcess) {

			cpb.tick();

			File xmlFile = text.getXMLTXMFile()

			String textname = text.getName()

			File cqpFile = new File(cqpDirectory, textname + ".cqp")

			cqpFilesUpdated++

			step.setNormalizeAnaValues(doNormalizeAnaValues)

			step.setNormalizeAttributeValues(doNormalizeAttributeValues)

			if (!step.process()) {

		println ""

		return true;

	}

	public boolean doCWBEncodeStep() {

		println "-- Running cwb-encode..."

		DeleteDir.deleteDirectory(outputDirectory);

		outputDirectory.mkdirs();

		dataDirectory.mkdirs();

		DeleteDir.deleteDirectory(registryDirectory);

		registryDirectory.mkdirs();

		CwbEncode cwbEn = new CwbEncode()

		cwbEn.setDebug(debug)

		List<String> pargs = ["id"]

		for (String ana : sattrsListener.getAnatypes()) {

			if (ana == "word") continue; // no need to be added, cwb will declared it automatically

			if (ana == "id") continue; // no need to be added, we did it already

			pargs.add(ana)

		}

		String[] pAttrs = pargs

		def structs = sattrsListener.getStructs()

		def structsProf = sattrsListener.getProfs()

		if (debug) {

			println structs

			println structsProf

		}

		List<String> sargs = new ArrayList<String>()

		def tmpTextAttrs = []

		for (String name : structs.keySet()) {

			if (name == "txmcorpus") continue;

			if (name == "text") {

				for (String value : structs.get(name)) // append the attributes

					tmpTextAttrs << value // added after

				continue;

			}

			String concat = name+":"+structsProf.get(name); // append the depth

			for (String attributeName : structs.get(name)) { // append the attributes

				concat += "+"+attributeName.toLowerCase();

			}

			if (structs.get(name).size() == 0) {

				concat += "+n";

			} else {

					concat += "+n"

				}

			}

			if ((name == "p" || name == "body" || name == "back" || name == "front")

				concat += "+n"

			}

			sargs.add(concat)

		}

		String textSAttributes = "text:0+id+base+project";

		for (String name : tmpTextAttrs) {

			if (!("id".equals(name) || "base".equals(name) || "project".equals(name))) {

				textSAttributes += "+"+name.toLowerCase()

			}

		}

		sargs.add(textSAttributes)

		sargs.add("txmcorpus:0+lang")

		sargs.sort()

		String[] sAttributes = sargs

		String[] pAttributes = pAttrs

		println " Word properties: "+pAttributes.join(', ')

				println "Fail to write the master cqp file: "+allcqpFile

				return false;

			}

			new File(regPath).delete()// ensure the registry file is deleted

			if (!cwbEn.run(outputDirectory.getAbsolutePath() + "/$corpusname",

				println "** cwb-encode did not ends well. Please activate a finer log level to see more details."

				return false;

			}

			allcqpFile.delete(); // clean

		} catch (Exception e) {

			println "Error while running cwb-encode: "+e

		println ""

		return true;

	}

	public boolean doCWBMakeAllStep() {

		println "-- Running cwb-makeall..."

		try {

			CwbMakeAll cwbMa = new CwbMakeAll();

			cwbMa.setDebug(debug);

			if (!new File(regPath).exists()) {

				println "Error: The registry file was not created: $regPath. See https://groupes.renater.fr/wiki/txm-users/public/faq"

				return false;

				println "** cwb-makeall did not ends well. Activate finer logs to see details."

				return false;

			}

			// remove milestones from CWB registry and data files

			FixMilestoneDeclarations fm = new FixMilestoneDeclarations(

					new File(regPath), new File(outputDirectory.getAbsolutePath(), corpusname));

			edition.setIndex(outputDirectory.getAbsolutePath());

			try {

				if (!ed.process()) {

					println "Fail to build 'facs' edition for text: $txmFile"

					continue;

package org.txm.importer.xtz;

public class Step {

	}

	public boolean process() {

	private Text noteElementsText;

	private Text milestoneElementsText;

	public TextualPlansSection(ImportFormEditor editor, FormToolkit toolkit2, ScrolledForm form2, Composite parent, int style) {

		super(editor, toolkit2, form2, parent, style, "textualplans");

		milestoneElementsText = toolkit.createText(sectionClient, "", SWT.BORDER); //$NON-NLS-1$

		gdata = getTextGridData();

		milestoneElementsText.setLayoutData(gdata);

	}

	@Override

			outsideTextElementsToEditText.setText(project.getTextualPlan("OutSideTextTagsAndKeepContent"));

			noteElementsText.setText(project.getTextualPlan("Note"));

			milestoneElementsText.setText(project.getTextualPlan("MileStones"));

		}

	}

			project.setTextualPlan("MileStones", milestoneElementsText.getText().trim());

			return true;

		}

		params.put(PREBUILD, true);

		params.put(QUERIES, true);

		params.put(UI, true);

		params.put(OPTIONS, true);

		sectionsPerImportModule.put("transcriberLoader.groovy", params); //$NON-NLS-1$

		names.put("transcriberLoader.groovy", "XML-TRS + CSV"); //$NON-NLS-1$ //$NON-NLS-2$