Laboratoire ICAR » Plateforme TXM

				tokenizer.setRetokenize("true" == project.getTokenizerParameter("doRetokenizeStep", "false"))

			if (s.startsWith("^^") && s.length() > 2) {

			if (s.startsWith("*") && s.length() > 1) {

			if ((s.contains("(") || s.contains(")")) && && s.length() > 2) // contains ( or )

		boolean retokenize = "true" == module.getProject().getTokenizerParameter("doRetokenizeStep", "false")

				tokenizer.setRetokenize(retokenize)

import org.txm.importer.PersonalNamespaceContext

	boolean retokenize = false

	LinkedHashMap<String, String>retokenizedWordProperties = new LinkedHashMap()

	Pattern reg_outside_text_tags_keep_content;

			if (!reg_word_tags.matcher(localname).matches()) {

				writer.writeCharacters("\n");

			}

		XMLInputFactory.newInstance();

		parser = inputfactory.createXMLStreamReader(inputData);

		//		println "reg_outside_text_tags_keep_content=$reg_outside_text_tags_keep_content"

		//		println "reg_outside_text_tags=$reg_outside_text_tags"

		//		println "reg_note_content=$reg_note_content"

						if (retokenize) {

							retokenizedWordProperties.clear()

							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

								retokenizedWordProperties[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i)

							}

						} else {

							wordcount++;

							donothing(event, wordcount);

							//ignorecontent = true;

							insideword = true;

						}

						if (retokenize) {

							retokenizedWordProperties.clear()

						} else {

							//ignorecontent = false;

							insideword = false;

							writer.writeEndElement();

							writer.writeCharacters("\n");

						}

	public void setRetokenize(boolean retokenize) {

		this.retokenize = retokenize

	}

				iterate(s);

			}

			//	if (DEBUG){println "  > $s"}

			//	if (DEBUG) println "CLITIC found: $s ->"+ m

			//	if (DEBUG) println "Elision found: $s ->"+ m.group(1)+" + "+m.group(2)+" + "+m.group(3)

			//			int sep = s.indexOf("'");

			//			if (sep < 0)

			//				sep = s.indexOf("’");

			//			if (sep < 0)

			//				sep = s.indexOf("‘");

			//	if(DEBUG){println "REG '...' found: $s -> "+m.group(1)+" + "+m.group(2)+" + "+m.group(3)}

			//		if(DEBUG){println "Other found: "+s}

			//			if (reg_punct_other.matcher(s).matches())

			//				writer.writeAttribute("type","pon");

			//			else

			//				writer.writeAttribute("type","w");

		for (String attr : retokenizedWordProperties.keySet()) {

			if ("id" == attr) {

				writer.writeAttribute("previous-id", retokenizedWordProperties[attr]);

			} else if ("n" == attr) {

				writer.writeAttribute("previous-n", retokenizedWordProperties[attr]);

			} else {

				writer.writeAttribute(attr, retokenizedWordProperties[attr]);

			}

		}

		//		File SVNDIR = new File(System.getProperty("user.home"), "xml/tokenizernum");

		//		Runtime.getRuntime().exec("svn", "update", SVNDIR.getAbsolutePath());

		//		File inputFile = new File(SVNDIR, "baye1.xml")

		//		File outputFile = new File(SVNDIR, "baye1-t.xml")

		//		SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inputFile, outputFile, "fr")

		//		tokenizer.DEBUG = true

		//		tokenizer.setNote("note")

		//		//tokenizer.setOutSideTextTags("teiHeader")

		//		tokenizer.setOutSideTextTagsAndKeepContent("teiHeader")

		//		println outputFile.toString() + " : "+tokenizer.process();

		 File inputDir = new File("/home/mdecorde/xml/bvhepistemon2016")

		 File outputDir = new File("/home/mdecorde/xml/bvhepistemon2016/out-fast")

		 File outputDir2 = new File("/home/mdecorde/xml/bvhepistemon2016/out-slow")

		 File outputDir3 = new File("/home/mdecorde/xml/bvhepistemon2016/out-fast2")

		 File inputFile = new File(inputDir, "1542_RabelaisGrgt.xml");

		 outputDir.deleteDir()

		 outputDir2.deleteDir()

		 outputDir3.deleteDir()

		 outputDir.mkdir()

		 outputDir2.mkdir()

		 outputDir3.mkdir()

		 long time;

		 for (String lang : ["none"]) {

		 println "LANG: $lang"

		 time = System.currentTimeMillis();

		 File outputFile = new File(outputDir, "${lang}.xml")

		 SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inputFile, outputFile, lang)

		 tokenizer.process();

		 println "FAST $lang: "+(System.currentTimeMillis()-time)

		 //			time = System.currentTimeMillis();

		 //			File outputFile3 = new File(outputDir3, "${lang}.xml")

		 //			FastSimpleTokenizerXml tokenizer3 = new FastSimpleTokenizerXml(inputFile, outputFile3, lang)

		 //			tokenizer3.process();

		 //			println "FAST2 $lang: "+(System.currentTimeMillis()-time)

		 //			

		 //			time = System.currentTimeMillis();

		 //			File outputFile2 = new File(outputDir2, "${lang}.xml")

		 //			OldSimpleTokenizerXml tokenizer2 = new OldSimpleTokenizerXml(inputFile, outputFile2, lang)

		 //			tokenizer2.process();

		 //			println "SLOW $lang: "+(System.currentTimeMillis()-time)

		 }

		 */

		String lang = "fr"

		File inFile = new File(System.getProperty("user.home"), "xml/vocapia/test.trs")

		File outFile = new File(System.getProperty("user.home"), "xml/vocapia/test-retokenized.trs")

		println "processing "+inFile

		SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(inFile, outFile, lang)

		tokenizer.setRetokenize(true)

		//tokenizer.setDEBUG false

		tokenizer.process();

		println "Done"

package org.txm.macro.transcription

import javax.xml.stream.*

import org.txm.importer.PersonalNamespaceContext

import org.txm.xml.IdentityHook

import org.txm.xml.*

import java.io.BufferedOutputStream

import java.io.FileOutputStream

import java.io.IOException

import java.net.URL

import java.util.*

import java.util.Map.Entry

import java.util.regex.Pattern

class FixTranscription extends XMLProcessor {

	LocalNamesHookActivator activator;

	IdentityHook hook;

	boolean retokenizeWords

	public FixTranscription(File xmlfile, boolean retokenizeWords) {

		super(xmlfile)

		this.retokenizeWords = retokenizeWords

		activator = new LocalNamesHookActivator<>(hook, ["w", "Turn", "Sync"]);

		hook = new IdentityHook("word_hook", activator, this) {

					boolean inTurn = false;

					boolean inW = false;

					StringBuilder wordBuffer = new StringBuilder();

					String currentTime;

					LinkedHashMap turnInfos = new LinkedHashMap()

					LinkedHashMap wInfos = new LinkedHashMap()

					boolean other

					@Override

					public boolean deactivate() {

						return true;

					}

					@Override

					public boolean _activate() {

						return true;

					}

					@Override

					protected void processStartElement() throws XMLStreamException, IOException {

						if (localname.equals("Turn")) {

							// store values

							inTurn = true;

							turnInfos.clear()

							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

								turnInfos[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i)

							}

							currentTime = turnInfos["startTime"]

							super.processStartElement();

						} else if (localname.equals("Sync")) {

							currentTime = parser.getAttributeValue(null, "time")

							super.processStartElement();

						} else if (localname.equals("w")) {

							// store values

							inW = true;

							wInfos.clear()

							for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

								wInfos[parser.getAttributeLocalName(i)] = parser.getAttributeValue(i)

							}

							String time = parser.getAttributeValue(null, "time")

							if (time != null && time.length() > 0) {

								currentTime = time

							}

							wordBuffer.setLength(0);

							return; // write w later

						}

						else {

							super.processStartElement();

						}

					}

					@Override

					protected void processCharacters() throws XMLStreamException {

						if (inW) {

							wordBuffer.append(parser.getText())

						}

						else {

							super.processCharacters();

						}

					}

					@Override

					protected void processEndElement() throws XMLStreamException {

						if (localname.equals("w")) {

							inW = false

							String word = wordBuffer.toString().trim()

							if (!other && word.startsWith("*")) {

								//close current Turn and start a 'other' Turn

								writer.writeEndElement() // current Turn

								writer.writeCharacters("\n")

								def tmpInfos = new LinkedHashMap()

								for (String attr : turnInfos.keySet()) tmpInfos[attr] = turnInfos[attr]

								tmpInfos["orig-speaker"] = turnInfos["speaker"]

								tmpInfos["speaker"] = "other"

								tmpInfos["startTime"] = currentTime

								writer.writeStartElement("Turn")

								for (String attr : tmpInfos.keySet()) {

									writer.writeAttribute(attr, tmpInfos[attr])

								}

								other = true

								word = word.substring(1)

							}

							boolean shouldCloseOtherTurn = false;

							if (other && word.endsWith("*")) {

								shouldCloseOtherTurn = true;

								word = word.substring(0, word.length()-1)

								other = false

							}

							// split before re-writing the word

							def puncts = []

							if (retokenizeWords) {

//								while (word.length() > 0 && word.matches("\\p{Punct}.+")) {

//									puncts << word.substring(0, 1)

//									word = word.substring(1, word.length())

//								}

								//						//vocapia fix

								//						Pattern reg = Pattern.compile("([^']+')(.+)")

								//						def m = reg.matcher(word)

								//						while (word.length() > 0 && m.matches()) {

								//							puncts << m.group(1)

								//							word = m.group(2)

								//							m = reg.matcher(word)

								//						}

							}

							for (def punct : puncts) { // pre-retokenize if any

								writer.writeStartElement("w")

								for (String attr : wInfos.keySet()) {

									writer.writeAttribute(attr, wInfos[attr])

								}

								writer.writeCharacters(punct)

								writer.writeEndElement() // w

								writer.writeCharacters("\n")

							}

							puncts = []

							if (retokenizeWords) {

//								while (word.length() > 0 && word.matches(".+\\p{Punct}")) {

//									puncts << word.substring(word.length()-1, word.length())

//									word = word.substring(0, word.length()-1)

//								}

							}

							if ("XXX".equals(word)) { // <Event desc="XXX" type="unknown" extent="next"/>

								writer.writeStartElement("event") // start the initial word

								writer.writeAttribute("desc", "XXX from "+wInfos["start"] + " to "+wInfos["end"])

								writer.writeAttribute("type", "unknown")

								writer.writeAttribute("extent", "instantaneous")

								writer.writeEndElement() // event

								word = "" // don't write the word

							}

							if (word.length() > 0) {

								writer.writeStartElement("w") // start the initial word

								for (String attr : wInfos.keySet() ) {

									writer.writeAttribute(attr, wInfos[attr])

								}

								writer.writeCharacters(word)

								writer.writeEndElement() // w

							}

							for (String punct : puncts) {  // post-retokenize if any

								writer.writeStartElement("w")

								for (String attr : wInfos.keySet() ) {

									writer.writeAttribute(attr, wInfos[attr])

								}

								writer.writeCharacters(punct)

								writer.writeEndElement() // w

							}

							if (shouldCloseOtherTurn) {

								shouldCloseOtherTurn = false;

								//close the current 'other' Turn and restart the actual Turn

								writer.writeEndElement() // current 'other' Turn

								writer.writeStartElement("Turn") // rebuild the orig Turn and fix its start-end infos

								turnInfos["startTime"] = wInfos["end"] // fix the startTime using the current word end time

								for (String attr : turnInfos.keySet()) {

									writer.writeAttribute(attr, turnInfos[attr])

								}

								other = false

							}

						} else {

							super.processEndElement();

						}

					}

				}

	}

	public static void main(String[] args) {

		File infile = new File("/home/mdecorde/xml/vocapia","test.trs")

		File outfile = new File("/home/mdecorde/xml/vocapia","test-fixed.trs")

		def processor = new FixTranscription(infile, true)

		println processor.process(outfile)

	}

}

package org.txm.macro.transcription

import java.time.LocalTime

import java.time.format.DateTimeFormatter

import org.txm.utils.*

import org.txm.utils.logger.*

@Field @Option(name="trsFile", usage="A single vocapia XML file", widget="FileOpen", required=false, def="")

		File trsFile;

@Field @Option(name="trsDirectory", usage="A Vocapia XML files directory to process", widget="Folder", required=false, def="")

		File trsDirectory;

@Field @Option(name="resultDirectory", usage="The result directory", widget="Folder", required=false, def="")

		File resultDirectory;

if (!ParametersDialog.open(this)) return;

if (trsDirectory.equals(resultDirectory) || trsFile.getParentFile().equals(resultDirectory)) {

	return false;

}

resultDirectory.mkdirs();

def trsFiles = []

if (trsDirectory != null && trsDirectory.exists()) {

	println "Processing TRS directory: $trsDirectory"

	for (File file : trsDirectory.listFiles()) {

		if (file.getName().toLowerCase().endsWith(".trs")) {

			trsFiles << file

		}

	}

} else if (trsFile != null && trsFile.exists()) {

	println "Processing TRS file: $trsFile"

	trsFiles << trsFile

}

if (trsFiles.size() == 0) {

	println "No XML file found for parameters trsFile=$trsFile and trsDirectory=$trsDirectory"

	return false

}

ConsoleProgressBar cpb = new ConsoleProgressBar(trsFiles.size())

for (File trsFile : trsFiles) {

	cpb.tick()

	FixTranscription fixer = new FixTranscription(trsFile)

	String name = FileUtils.stripExtension(trsFile)

	File outFile = new File(resultDirectory, name+".trs")

	if (!fixer.process(outFile)) {

		println "WARNING: ERROR WHILE PROCESSING: "+trsFile

		return false

	}

}

cpb.done()

println "Done: "+trsFiles.size()+" files processed. Result files in $resultDirectory"

		def winfos = new LinkedHashMap()

		def turninfos = new LinkedHashMap()

								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

									writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i))

								}

								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

									String name = parser.getAttributeLocalName(i)

									if (name == "stime") name = "startTime"

									else if (name == "etime") name = "endTime"

									else if (name == "spkid") name = "speaker"

									writer.writeAttribute(name, parser.getAttributeValue(i))

								def startValue = parser.getAttributeValue(null, "stime");

								winfos.clear()

								for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

									String name = parser.getAttributeLocalName(i)

									if (name == "stime") continue

										winfos[name] = parser.getAttributeValue(i)

								}

								winfos["time"] = startValue

								winfos["start"] = startValue

								winfos["end"] = endValue

							// write <w> at the end of <Word> tag

										puncts << word.substring(0 ,1)

										word = word.substring(1)

									// fix "d'abord" like words

										puncts.add(0, word.substring(word.length()-1, word.length()))

								if (word.length() > 0) {

									writer.writeStartElement("w") // start the initial word

									for (String attr : winfos.keySet()) {

										writer.writeAttribute(attr, winfos[attr])

									}

									writer.writeCharacters(word)

									writer.writeEndElement() // w

									writer.writeCharacters("\n")

								}

								for (String punct : puncts) {  // write post-retokenize if any

									for (String attr : winfos.keySet()) {

										writer.writeAttribute(attr, winfos[attr])

									}

		println processor.process(outfile, true)