Laboratoire ICAR » Plateforme TXM

import org.apache.commons.lang.StringUtils;

import org.txm.importer.cwb.ReadRegistryFile;

	 * @throws InvalidCqpIdException the invalid CQP id exception

				if (!compute(false)) {

					Log.warning(NLS.bind("Warning: the {0} corpus won't work correctly", this.getID()));

					throw new IllegalStateException(NLS.bind("{0} CQP MainCorpus not instanciate correctly.", this.getID()));

				}

				throw new IllegalStateException(e);

			try { // fix the absolute paths in the registry file

			// check if all corpus index files are present

			ReadRegistryFile rrf = new ReadRegistryFile(this.registryFile);

			ArrayList<String> errors = rrf.isCorpusBuildValid(this.dataDirectory);

			if (errors.size() > 0) {

				Log.warning(TXMCoreMessages.bind("Error: some {0} index files are missing : {1}.", this.getID(), StringUtils.join(errors, ", ")));

				return false;

			}

			List<String> tmp = Arrays.asList(CQPSearchEngine.getCqiClient().listCorpora());

import org.txm.utils.logger.Log;

 * 

		sattrs = new HashMap<>();

		sattrsProfs = new HashMap<>();

			}

			else if (line.startsWith("STRUCTURE ")) {

				// println split

					if (sname.matches(".+[1-9]") && sattrs.containsKey(sname.substring(0, sname.length() - 1))) { // recursive structure

						sname = sname.substring(0, sname.length() - 1);

						sattrsProfs.put(sname, sattrsProfs.get(sname) + 1);

					}

					else {

				}

				else {

					if (satt.matches(".+[1-9]") && sattrs.get(sname).contains(satt.substring(0, satt.length() - 1))) {

					}

					else {

			String tmp = "" + sattr + ":" + sattrsProfs.get(sattr);

				tmp += "+" + attr;

	 * Test the CQP index files of each p-attribute and s-attribute properties

	 * @param dataDirectory the directory where the binary files should be found

	 * @return true if all CQP files are present

	 */

	public ArrayList<String> isCorpusBuildValid(File dataDirectory) {

		if (pAttributes == null) {

			read();

		}

		ArrayList<String> errors = new ArrayList<>();

		// test p-attributes

		String[] exts = { ".corpus", ".lexicon", ".corpus.cnt", ".corpus.rdx", ".corpus.rev", ".lexicon.idx", ".lexicon.srt" };

		for (String p : pAttributes) {

			for (String ext : exts) {

				File f = new File(dataDirectory, p + ext);

				if (!f.exists()) {

					// System.out.println("MISSING: " + f.exists() + " " + f.getAbsolutePath());

					errors.add(f.getName());

				}

			}

		}

		String[] sexts = { ".rng" };

		String[] spexts = { ".avs", ".avx", ".rng" };

		for (String s : sattrs.keySet()) {

			for (String ext : sexts) {

				File f = new File(dataDirectory, s + ext);

				if (!f.exists()) {

					// System.out.println("MISSING: " + f.exists() + " " + f.getAbsolutePath());

					errors.add(f.getName());

				}

			}

			for (String sp : sattrs.get(s)) {

				for (String ext : spexts) {

					File f = new File(dataDirectory, s + "_" + sp + ext);

					if (!f.exists()) {

						// System.out.println("MISSING: " + f.exists() + " " + f.getAbsolutePath());

						errors.add(f.getName());

					}

				}

			}

		}

		return errors;

	}

	/**

	 * 

		File registry = new File(System.getProperty("user.home"), "runtime-rcpapplication.product/corpora/VOEUX/registry/voeux");

		File data = new File(System.getProperty("user.home"), "runtime-rcpapplication.product/corpora/VOEUX/data/VOEUX");

		System.out.println("pAttributes: " + reader.getPAttributes());

		System.out.println("sAttributes Map: " + reader.getSAttributesMap());

		System.out.println("sAttributes: " + reader.getSAttributes());

		System.out.println("Validation: " + reader.isCorpusBuildValid(data));

			catch (Throwable e) {

				Log.warning("Internal error: could not initialize result: " + resultNodePath + ": " + e);

				Log.printStackTrace(e);

			if (!getCorpus().hasBeenComputedOnce()) {

				Log.warning("Error: CQP corpus is not available: " + getCorpus());

				return false;

			}

package org.txm.macro.transcription

import javax.xml.stream.*

import org.txm.importer.PersonalNamespaceContext

import java.io.BufferedOutputStream

import java.io.FileOutputStream

import java.net.URL

import java.util.regex.Pattern

class Vocapia2Transcriber {

	File xmlfile;

	protected BufferedOutputStream output;

	protected XMLStreamWriter writer;

	public Vocapia2Transcriber(File xmlfile) {

		this.xmlfile = xmlfile;

	}

	public boolean process(File outfile, boolean retokenizeWords) {

		if (!xmlfile.exists()) return false;

		output = new BufferedOutputStream(new FileOutputStream(outfile), 16 * 1024);

		writer = XMLOutputFactory.newInstance().createXMLStreamWriter(output, "ISO-8859-1");// create a new file

		writer.setNamespaceContext(new PersonalNamespaceContext());

		URL url = xmlfile.toURI().toURL();

		String filename = outfile.getName()

		filename = filename.substring(0, filename.length()-4); // remove ".cqp"

		def inputData = url.openStream();

		XMLInputFactory factory = XMLInputFactory.newInstance();

		XMLStreamReader parser = factory.createXMLStreamReader(inputData);

		boolean flagWord = false

		def winfos = [:]

		boolean other = false;

		String word = ""

		try {

			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

				switch (event) {

					case XMLStreamConstants.START_ELEMENT:

						String localname = parser.getLocalName()

						switch(localname) {

							case "AudioDoc": // <AudioDoc name="xyz" path="xyz.flac"> -> <Trans scribe="see Proc elements" audio_filename="xyz.flac" version="see Proc elements" version_date="see Proc elements">

								writer.writeStartDocument("ISO-8859-1", "1.0")

								writer.writeStartElement("Trans")

								writer.writeAttribute("audio_filename", parser.getAttributeValue(null, "path"))

								break

							case "Proc": // <Proc name="scribe" version="date" editor="AAA"/>

							//continue writing the "Trans" element

								if ("scribe" == parser.getAttributeValue(null, "name")) {

									writer.writeAttribute("scribe", parser.getAttributeValue(null, "editor"))

									writer.writeAttribute("version", parser.getAttributeValue(null, "version"))

									writer.writeAttribute("version_date", parser.getAttributeValue(null, "version"))

								}

								break;

							case "SpeakerList": // <SpeakerList> -> <Speakers>

								writer.writeCharacters("\n") // after <Trans>

								writer.writeStartElement("Speakers")

								writer.writeCharacters("\n")

								break;

							case "Speaker": // <Speaker ch="1" dur="531.38" gender="X" spkid="Enquêtrice" lang="fre" lconf="1.00" nw="1586" tconf="0.95"/> -> <Speaker id="spk1" name="enq4" check="no" dialect="native" accent="" scope="local"/>

								writer.writeStartElement("Speaker")

								writer.writeAttribute("id", parser.getAttributeValue(null, "spkid"))

								writer.writeAttribute("name", parser.getAttributeValue(null, "spkid"))

								writer.writeAttribute("check", "")

								writer.writeAttribute("dialect", parser.getAttributeValue(null, "lang"))

								writer.writeAttribute("accent", parser.getAttributeValue(null, "gender"))

								writer.writeAttribute("scope", "local")

								writer.writeEndElement()

								writer.writeCharacters("\n")

								break;

							case "SegmentList":

								writer.writeStartElement("Episode")

							//<Section type="report" startTime="0" endTime="3617.593">

								writer.writeStartElement("Section")

								break;

							case "SpeechSegment": // <SpeechSegment ch="1" sconf="1.00" stime="9.94" etime="43.81" spkid="Enquêtrice" lang="fre" lconf="1.00" trs="1">

								writer.writeStartElement("Turn")

								writer.writeAttribute("speaker", parser.getAttributeValue(null, "spkid"))

								writer.writeAttribute("startTime", parser.getAttributeValue(null, "stime"))

								writer.writeAttribute("endTime", parser.getAttributeValue(null, "etime"))

								writer.writeCharacters("\n")

								writer.writeStartElement("Sync")

								writer.writeAttribute("time", parser.getAttributeValue(null, "stime"))

								writer.writeEndElement() // Sync

								writer.writeCharacters("\n")

								break;

							case "Word":

								flagWord = true

								word = ""

							// store w infos in case the word must be splited

								def endValue = String.format(Locale.US, "%.2f", (Double.parseDouble(parser.getAttributeValue(null, "stime")) + Double.parseDouble(parser.getAttributeValue(null, "dur"))))

								winfos = ["time":parser.getAttributeValue(null, "stime"), "start": parser.getAttributeValue(null, "stime"), "end":endValue]

								break

						}

						break;

					case XMLStreamConstants.END_ELEMENT:

						String localname = parser.getLocalName()

						switch(localname) {

							case "AudioDoc": // <AudioDoc name="xyz" path="xyz.flac"> -> <Trans scribe="see Proc elements" audio_filename="xyz.flac" version="see Proc elements" version_date="see Proc elements">

								writer.writeEndElement() // Trans

								break

							case "Proc": // <Proc name="scribe" version="date" editor="AAA"/>

								break

							case "SpeakerList": // <SpeakerList> -> <Speakers>

								writer.writeEndElement()

								writer.writeCharacters("\n")

								break

							case "Speaker": // <Speaker ch="1" dur="531.38" gender="X" spkid="Enquêtrice" lang="fre" lconf="1.00" nw="1586" tconf="0.95"/> -> <Speaker id="spk1" name="enq4" check="no" dialect="native" accent="" scope="local"/>

							//already closed

								break

							case "SegmentList":

								writer.writeEndElement() // Section

								writer.writeEndElement() // Episode

								writer.writeCharacters("\n")

								break

							case "SpeechSegment": // <SpeechSegment ch="1" sconf="1.00" stime="9.94" etime="43.81" spkid="Enquêtrice" lang="fre" lconf="1.00" trs="1">

								writer.writeEndElement() // Turn

								writer.writeCharacters("\n")

								break

							case "Word":

								flagWord = false

								word = word.trim()

								if (word.startsWith("*")) {

									other = true

									word = word.substring(1)

								}

								String otherAttributeValue = Boolean.toString(other) // set now

								if (other && word.endsWith("*")) {

									word = word.substring(0, word.length()-1)

									other = false

								}

							// split before the word

								def puncts = []

								if (retokenizeWords) {

									while (word.length() > 0 && word.matches("\\p{Punct}.+")) {

										puncts << word.substring(0, 1)

										word = word.substring(1, word.length())

									}

									Pattern reg = Pattern.compile("([^']+')(.+)")

									def m = reg.matcher(word)

									while (word.length() > 0 && m.matches()) {

										puncts << m.group(1)

										word = m.group(2)

										m = reg.matcher(word)

									}

								}

								for (def punct : puncts) { // pre-retokenize if any

									writer.writeStartElement("w")

									writer.writeAttribute("time", winfos["time"])

									writer.writeAttribute("start", winfos["start"])

									writer.writeAttribute("end", winfos["start"])

									writer.writeCharacters(punct)

									writer.writeEndElement() // w

									writer.writeCharacters("\n")

								}

								puncts = []

								if (retokenizeWords) {

									while (word.length() > 0 && word.matches(".+\\p{Punct}")) {

										puncts << word.substring(word.length()-1, word.length())

										word = word.substring(0, word.length()-1)

									}

								}

								writer.writeStartElement("w") // start the initial word

								writer.writeAttribute("time", winfos["time"])

								writer.writeAttribute("start", winfos["start"])

								writer.writeAttribute("end", winfos["end"])

								writer.writeAttribute("other", otherAttributeValue)

								writer.writeCharacters(word)

								writer.writeEndElement() // w

								writer.writeCharacters("\n")

								for (String punct : puncts) {  // post-retokenize if any

									writer.writeStartElement("w")

									writer.writeAttribute("time", winfos["time"])

									writer.writeAttribute("start", winfos["end"])

									writer.writeAttribute("end", winfos["end"])

									writer.writeCharacters(punct)

									writer.writeEndElement() // w

									writer.writeCharacters("\n")

								}

								break

						}

						break

					case XMLStreamConstants.CHARACTERS:

						if (flagWord) {

							word += parser.getText()

						}

						break

				}

			}

		} catch (Exception e) {

			println "Error while processing XML File "+xmlfile+": "

			e.printStackTrace();

			println "At: "+parser.getLocation();

			println "See: "+outfile.getAbsolutePath()

		}

		output.flush()

		writer.close()

		output.close()

		//writer.close()

		parser.close()

		//println "$xmlfile -> $outfile"

		return true;

	}

	public static void main(String[] args) {

		File infile = new File("/home/mdecorde/xml/vocapia","test.xml")

		File outfile = new File("/home/mdecorde/xml/vocapia","test.trs")

		def processor = new Vocapia2Transcriber(infile)

		println processor.process(outfile)

	}

}

package org.txm.macro.transcription

import java.time.LocalTime

import java.time.format.DateTimeFormatter

import org.txm.utils.*

import org.txm.utils.logger.*

@Field @Option(name="vocapiaFile", usage="A single vocapia XML file", widget="FileOpen", required=false, def="")

		File vocapiaFile;

@Field @Option(name="vocapiaDirectory", usage="A Vocapia XML files directory to process", widget="Folder", required=false, def="")

		File vocapiaDirectory;

@Field @Option(name="resultDirectory", usage="The result directory", widget="Folder", required=false, def="")

		File resultDirectory;

@Field @Option(name="retokenize_words", usage="retokenize words prefixed or postfixed with puunctuations", widget="Boolean", required=true, def="true")

		Boolean retokenize_words;

if (!ParametersDialog.open(this)) return;

resultDirectory.mkdirs();

def xmlFiles = []

if (vocapiaDirectory != null && vocapiaDirectory.exists()) {

	println "Processing directory: $vocapiaDirectory"

	for (File file : vocapiaDirectory.listFiles()) {

		if (file.getName().toLowerCase().endsWith(".xml")) {

			xmlFiles << file

		}

	}

} else if (vocapiaFile != null && vocapiaFile.exists()) {

	println "Processing file: $vocapiaFile"

	xmlFiles << vocapiaFile

}

if (xmlFiles.size() == 0) {

	println "No XML file found for parameters vocapiaFile=$vocapiaFile and vocapiaDirectory=$vocapiaDirectory"

	return false

}

ConsoleProgressBar cpb = new ConsoleProgressBar(xmlFiles.size())

for (File xmlFile : xmlFiles) {

	cpb.tick()

	Vocapia2Transcriber v2t = new Vocapia2Transcriber(xmlFile)

	String name = FileUtils.stripExtension(xmlFile)

	File outFile = new File(resultDirectory, name+".trs")

	if (!v2t.process(outFile, retokenize_words)) {

		println "WARNING: ERROR WHILE PROCESSING: "+xmlFile

		return false

	}

}

cpb.done()

println "Done: "+xmlFiles.size()+" files processed. Result files in $resultDirectory"

// @author Sebastien Jacquot

// STANDARD DECLARATIONS

package org.txm.macro

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcpapplication.swt.widget.parameters.*

import org.txm.searchengine.cqp.clientExceptions.*

import org.txm.searchengine.cqp.corpus.*

import org.txm.searchengine.cqp.corpus.query.*

import org.apache.commons.lang.time.StopWatch

import java.util.Arrays

import org.jfree.chart.renderer.xy.*

import org.jfree.chart.renderer.*

import org.jfree.chart.plot.*

import org.jfree.data.xy.*

import org.jfree.chart.axis.*

import java.awt.*;

import java.awt.geom.*;

import org.jfree.chart.labels.*

import org.txm.ca.core.chartsengine.jfreechart.themes.highcharts.renderers.*

import org.txm.ca.rcp.editors.*

import org.txm.libs.office.ReadODS

import org.txm.ca.core.chartsengine.jfreechart.datasets.*

import org.jfree.chart.renderer.AbstractRenderer

println "editor: "+editor

if (!(editor instanceof CAEditor)) {

	println "editor is not a CA editor: $editor, Run the macro with F12 when the editor is selected :-)"

	return

}

ica = editor.getCA();

chart = ica.getChart();

plot = chart.getXYPlot();

dataset = plot.getDataset(); 

// overrides some dataset methods to return inverted X coordinates for columns and rows

plot.setDataset(new CAXYDataset(ica) {

        public Number getX(int series, int item) {

                if(item == -1)        {

                        System.out.println("CAXYDataset.getX()");

                }

                // Rows

                if(series == 0) {

                        return -this.rowCoordinates[item][this.axis1];

                }

                // Cols

                else {

                        return -this.columnCoordinates[item][this.axis1];

                }

        }

        /**

         * Gets the minimum value in the specified series according to the specified axis.

         * @param series

         * @param axis

         * @return

         */

        public double getMinValue(int series, int axis)        {

                double minValue = 0;

                double tmpMinValue;

                double[][] coordinates = this.rowCoordinates;

                if(series != 0)        {

                        coordinates = this.columnCoordinates;

                }

                for(int i = 0; i < coordinates.length; i++) {

                        tmpMinValue = coordinates[i][axis];

						// invert X coordinate

                		if(axis == 0)        {

                			tmpMinValue = -tmpMinValue;

                		}

                        if(tmpMinValue < minValue)        {

                                minValue = tmpMinValue;

                        }

                }

                return minValue;

        }

        /**

         * Gets the maximum value in the specified series according to the specified axis.

         * @param series

         * @param axis

         * @return

         */

        public double getMaxValue(int series, int axis)        {

                double maxValue = 0;

                double tmpMaxValue;

                double[][] coordinates = this.rowCoordinates;

                if(series != 0)        {

                        coordinates = this.columnCoordinates;

                }

                for(int i = 0; i < coordinates.length; i++) {

                        tmpMaxValue = coordinates[i][axis];

						// invert X coordinate

                		if(axis == 0)        {

                			tmpMaxValue = -tmpMaxValue;

                		}

                        if(tmpMaxValue > maxValue)        {

                                maxValue = tmpMaxValue;

                        }

                }

                return maxValue;

        }

}

);

// update the limits dotted borders

ica.getChartCreator().createCAFactorialMapChartLimitsBorder(chart);

	boolean overlapingTurn = false // need to skip fist Sync when multiple locutors in Turn

				overlapingTurn = false

					if (speakersname.containsKey(vSpeaker)) {

						//vSpeaker = speakersname.get(vSpeaker);

					} else {

						localspeakers = vSpeaker.split(" ")

						if (localspeakers.size() > 1) { // only one speaker

							overlapingTurn = true

				overlapingTurn = vSpeaker.contains(" ") // need to skip fist Sync when multiple locutors in Turn

				writer.writeAttribute("overlap", ""+overlapingTurn)

//				if (overlapingTurn) { // need to skip fist Sync when multiple locutors in Turn

//					overlapingTurn = false;

//				} else {

//					

//				}

				if (!overlapingTurn) {

			writer.writeCharacters("\n")

		//println "ATTRIBUTES="+attributes+" vSpeaker='$vSpeaker'"

//			if (vSpeaker.startsWith("#") && vSpeaker.endsWith("?")) { // don't show "N/A" vSpeaker

				//writeAttributes();

//			} else {

//				

//			}

									if (!metadataGroups.containsKey(g)) {

										metadataGroups[g] = []

									}

						//store attributes values in HashMap

						// write metadata HTML

							String spid = parser.getAttributeValue(null, "who");

								//writer.writeEmptyElement("br"); // write all overlaping speakers

								//writeSpeaker(""+parser.getAttributeValue(null, "who"), false)

							String spk = parser.getAttributeValue(null, "who")

								writeSpeaker(parser.getAttributeValue(null, "who"), overlapping)

							if (overlapping) previousSPK = null

								if (parser.getAttributeValue(null, "extent") == "end") {

								}

								else if (parser.getAttributeValue(null, "extent") == "begin") {

								}

									if (parser.getAttributeValue(null, "type") == "pronounce")

								else {

								}

							if (interpvalue != null) {

							}

							if (events.size() > 0) {

							}

							if (nextEvent.length() > 0) {

							if (NoSpaceBefore.contains(wordvalue) ||

					if (flagform) {

						if (parser.getText().length() > 0) {

						}

					}

					if (flaginterp) {

						if (parser.getText().length() > 0) {

						}

					}

			def anaValues = [:]

			def anaType = ""

			def anaResp = ""

			def anaValue = new StringBuilder()

								anaValues.clear()

								anaType = getAttributeValue(parser, null, "type").substring(1)

								anaResp = getAttributeValue(parser, null, "resp").substring(1)

								anaValue.setLength(0)

								if (anaValues[anaType] == null || "src".equals(anaResp)) {

									anaValues[anaType] = anaValue.toString().trim()

								}

								String interpvalue = anaValues.entrySet().join(", ")

							anaValue.append(parser.getText())

		/** The lastword. */

		def anaValues = [:]

		def anaType = ""

		def anaResp = ""

		def anaValue =  new StringBuilder()

							anaValues.clear()

							anaType = parser.getAttributeValue(null, "type").substring(1)

							anaResp = parser.getAttributeValue(null, "resp").substring(1)

							anaValue.setLength(0)

							if (anaValues[anaType] == null || "src".equals(anaResp)) {

								anaValues[anaType] = anaValue.toString().trim()

							}

							String interpvalue = anaValues.entrySet().join(", ")

						anaValue.append(parser.getText());

import org.txm.utils.io.FileCopy

//			println "TEMP REMOVED SURROGATE FIX"

//			FileCopy.copy(f, outputFile)