Laboratoire ICAR » Plateforme TXM

package org.txm.macro.transcription

import java.nio.charset.Charset

import java.time.LocalTime

import java.time.format.DateTimeFormatter

import org.eclipse.core.internal.localstore.IsSynchronizedVisitor

import org.txm.utils.*

import org.txm.utils.logger.*

@Field @Option(name="metadataFile", usage="Tableau des metadonnées de sections", widget="FileOpen", required=true, def="")

		File metadataFile;

@Field @Option(name="trsDirectory", usage="Dossier qui contient les fichiers TRS", widget="Folder", required=true, def="")

		File trsDirectory;

@Field @Option(name="joinTRSColumn", usage="Colonne de jointure de transcription", widget="String", required=true, def="Lien notice principale")

		def joinTRSColumn

@Field @Option(name="startTimeColumn", usage="Colonne de timing de début de section", widget="String", required=true, def="antract_debut")

		def startTimeColumn = "antract_debut"

@Field @Option(name="endTimeColumn", usage="Colonne de timing de fin de section", widget="String", required=true, def="antract_fin")

		def endTimeColumn = "antract_fin"

@Field @Option(name="typeColumns", usage="Colonnes des métadonnées de type de section", widget="String", required=true, def="Titre propre")

		def typeColumns

@Field @Option(name="topicColumns", usage="Colonnes des métadonnées de topic de section", widget="String", required=true, def="Date de diffusion")

		def topicColumns

@Field @Option(name="metadataColumns", usage="Colonnes de metadonnées de section", widget="String", required=true, def="Titre propre;Date de diffusion;Identifiant de la notice;Notes du titre;Type de date;Durée;Genre;Langue VO / VE;Nature de production;Producteurs (Aff.);Thématique;Nom fichier segmenté (info);antract_video;antract_debut;antract_fin;antract_duree;antract_tc_type;antract_tc_date;Résumé;Séquences;Descripteurs (Aff. Lig.);Générique (Aff. Lig.)")

		def metadataColumns

@Field @Option(name="metadataColumnsGroups", usage="Colonnes des gruopes de metadonnées de section", widget="String", required=true, def="metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;secondary;secondary;secondary;secondary;secondary;secondary;secondary;text;text;text;text")

		def metadataColumnsGroups

@Field @Option(name="fixSectionsLimits", usage="Correction des limites de sections du tableau de metadonnees", widget="Boolean", required=true, def="true")

		def fixSectionsLimits

@Field @Option(name="sectionsMergeActivationThreashold", usage="marge d'erreur de corrections des limites de sections", widget="Float", required=true, def="1.0")

		def sectionsMergeActivationThreashold

@Field @Option(name="fixTurnsLimits", usage="Correction des limites de sections du tableau de metadonnees", widget="Boolean", required=true, def="true")

		def fixTurnsLimits

@Field @Option(name="turnsCutActivationThreashold", usage="marge d'erreur de corrections des limites de tours", widget="Float", required=true, def="0.1")

		def turnsCutActivationThreashold

@Field @Option(name="debug", usage="show debug messages", widget="String", required=true, def="false")

		def debug

if (!ParametersDialog.open(this)) return;

debug = true

typeColumns = typeColumns.split(";")

topicColumns = topicColumns.split(";")

metadataColumns = metadataColumns.split(";")

metadataColumnsGroups = metadataColumnsGroups.split(";")

if (metadataColumns.size() != metadataColumnsGroups.size()) {

	println "ERROR in metadata declarations&groups:"

	println "COLUMNS: "+metadataColumns

	println "GROUPS : "+metadataColumnsGroups

	return

}

if (!trsDirectory.exists()) {

	println "$trsDirectory not found"

	return

}

println "Loading data from $metadataFile..."

TableReader reader = new TableReader(metadataFile)//, "\t".charAt(0), Charset.forName("UTF-8")

reader.readHeaders()

def header = reader.getHeaders()

if (!header.contains(joinTRSColumn)) {

	println "No TRS ID $joinTRSColumn column found"

	return

}

if (!header.contains(startTimeColumn)) {

	println "No start time $startTimeColumn column found"

	return

}

if (!header.contains(endTimeColumn)) {

	println "No end time $endTimeColumn column found"

	return

}

for (def col : metadataColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No $col column found"

		return

	}

}

for (def col : typeColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No type $col column found"

		return

	}

}

for (def col : topicColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No topic $col column found"

		return

	}

}

File outputDirectory = new File(trsDirectory, "out")

println "Writing result to $outputDirectory..."

dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_TIME

def strTotime(def str) {

	if (str.lastIndexOf(":") == -1) {

		return null

	}

	bonusFrame = Integer.parseInt(str.substring(str.lastIndexOf(":")+1))

	//if (str.contains("135475")) println "ERROR $str in $infos -> $bonusFrame"

	if (bonusFrame > 25) {

		bonusFrame=0;

	}

	totalFrame = str.substring(0, str.lastIndexOf(":"))

	LocalTime time1 = LocalTime.parse(totalFrame, dateTimeFormatter)

	totalFrame = (time1.getHour()*60*60) + (time1.getMinute()*60) + time1.getSecond()

	def ret = totalFrame + (bonusFrame/25)

	return ret

}

try {

	def sectionGroupsToInsert = [:]

	println "Reading data..."

	while (reader.readRecord()) { // loading & sorting sections

		String id = reader.get(joinTRSColumn).trim()

		if (id.endsWith(".mp4")) id = id.substring(0, id.length()-4)

		if (id.length() == 0) continue;

		if (!sectionGroupsToInsert.containsKey(id)) {

			sectionGroupsToInsert[id] = []

		}

		def section = sectionGroupsToInsert[id]

		if (reader.get(startTimeColumn) != null && reader.get(startTimeColumn).length() > 0) { // ignore non timed sections

			def m = [:]

			for (def todo : ["topic":topicColumns, "type":typeColumns]) {

				def data = []

				for (def col : todo.value) {

					if (reader.get(col).trim().length() > 0) {

						data << reader.get(col).trim().replace("\n", "")

					}

				}

				m[todo.key] = data.join("\t")

			}

			def metadataList = []

			def metadataGroupList = []

			for (int i = 0 ;  i < metadataColumns.size() ; i++) {

				def col = metadataColumns[i]

				String c = AsciiUtils.buildAttributeId(col)

				m[c] = reader.get(col)

				metadataList << c

				metadataGroupList << metadataColumnsGroups[i]

			}

			m["metadata"] = metadataList.join("|")

			m["metadata_groups"] = metadataGroupList.join("|")

			m["startTime"] = strTotime(reader.get(startTimeColumn))

			m["endTime"] = strTotime(reader.get(endTimeColumn))

			m["synchronized"] = "true"

			section << [m["startTime"], m["endTime"], m]

		}

	}

	println "Inserting sections... "+sectionGroupsToInsert.size()

	ConsoleProgressBar cpb = new ConsoleProgressBar(sectionGroupsToInsert.keySet().size())

	for (String id : sectionGroupsToInsert.keySet()) {

		File trsFile = new File(trsDirectory, id+".trs")

		if (!trsFile.exists()) {

			cpb.tick()

			continue

		}

		if (debug) println "== $id =="

		else cpb.tick()

		//println "Processing $id..."

		sections = sectionGroupsToInsert[id]

		sections = sections.sort() { a, b -> a[0] <=> b[0] ?: a[1] <=> -b[1] } // negative second test for sections inclusion

		if (fixSectionsLimits) {

			if (debug) println "Fixing sections of $id"

			for (int iSection = 1 ; iSection < sections.size() ; iSection++) {

				//println sections[iSection]

				if (Math.abs(sections[iSection][0] - sections[iSection - 1][1]) < sectionsMergeActivationThreashold) {

					if (debug) println "s=$iSection start <- end : "+sections[iSection][0]+ " <- "+sections[iSection - 1][1]

					sections[iSection][0] = sections[iSection - 1][1] // fix the start time with the previous section end time

				}

			}

		}

		// Open input file

		slurper = new groovy.util.XmlParser(false, true, true);

		slurper.setFeature("http://apache.org/xml/features/disallow-doctype-decl", false) // allow DTD declaration

		slurper.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "all"); // allow to read DTD from local file

		trs = slurper.parse(trsFile.toURI().toString())

		trsEpisodes = trs.Episode // 1

		if (trsEpisodes.size() > 1) {

			println "Error: multiple Episode node in $trsFile"

			continue

		}

		def trsEpisode = trsEpisodes[0]

		def trsSections =  trs.Episode.Section // 1

		if (trsSections.size() > 1) {

			println "Error: multiple Section node in $trsFile"

			continue

		}

		trsSection = trsSections[0]

		turns = trsSection.Turn

		newSections = []

		iSection = 0;

		currentSection = null

		currentNode = null

		foundSection=null

		isTurnSynchronized=false

		cutCheck=false

		// boucle sur les tours dans l'ordre

		for (iTurn = 0 ; iTurn < turns.size() ; iTurn++) {

			turn = turns[iTurn]

			start = Float.parseFloat(turn.@startTime)

			end = Float.parseFloat(turn.@endTime)

			//println "Turn: $iTurn ($start, $end)"

			// Etape 1 : y aura-t-il besoin de couper le tour, et dans quelle section est le tour (ou sa première partie)

			foundSection = null;

			for (int i = iSection ; i < sections.size() ; i++) {

				// if section_end < turn_start OU |turn_start - section_end| < turn_threshold

				if (sections[i][1] < start || Math.abs(start - sections[i][1]) < turnsCutActivationThreashold) { // Turn is before section

					// Cas 1 : la section est complètement avant (modulo la marge)

				} else {

					// Cas 2 : on est arrivés à la section à considérer

					iSection = i

					// if section_start > turn_end OU |section_start - turn_end| < turn_threshold

					if (sections[i][0] > end || Math.abs(sections[i][0] - end) < turnsCutActivationThreashold) { // Section is before section

						// Cas 2.1 : la section est complètement après (modulo la marge) (et les suivantes le seront aussi)

						foundSection=null

						isTurnSynchronized=false

						cutCheck=false

					} else {

						// if |section_start - turn_start| > turn_threshold

						if (Math.abs(start - sections[i][0]) > turnsCutActivationThreashold) { // Section is before section

							// Cas 2.2 : la section commence significativement après le début du tour (le début est non synchronisé)

							foundSection=sections[i] // (c'est la première section rencontrée, mais elle sera pour le tour suivant)

							isTurnSynchronized=false

							cutCheck=true

						} else {

							// Cas 2.3 : le début du tour est dans la section (on n'a pas besoin de chercher d'autres sections car si ce n'est pas la seule on coupera le tour et ce sera un autre tour).

							foundSection=sections[i]  // (c'est la section qui commence le tour, au moins)

							isTurnSynchronized=true

							cutCheck=true

						}

					}

					break; // stop searching and set iSection to accelerate next search

				}

			}

			// Etape 3 : positionne *le* tour dans *le* noeud (cf. ~ l.247-264 ?)

			// (on ne gère qu'un seul tour et un seul noeud à chaque itération de la boucle tour,

			// puisqu'on a retaillé le tour pour qu'il ne concerne pas plusieurs noeuds)

			if (foundSection != null) {  // on complète ou on ajoute une div.

				if (foundSection != currentSection || currentSection == null) {

					if (currentNode != null && currentNode.@synchronized == "false") {

						def tmp = currentNode.Turn

						currentNode.@endTime = tmp[-1].@endTime

					}

					currentSection = foundSection

					currentNode = new Node(trsEpisode, "Section", currentSection[2])

				}

			} else {  // on complète ou on ajoute un noeud (div) non synchronisé.

				currentNode = new Node(trsEpisode, "Section", ["type":"Sujet non synchronisé", "startTime":sections[iSection][0], "endTime":sections[iSection][1], "synchronized":"false"] )

				currentSection = null;

			}

			// Etape 2 : on coupe le tour s'il y a besoin (cf. ~l.267-315 ?)

			if (cutCheck) {

				if (isTurnSynchronized) {

					cutTurn(true) // iSection++ quand on coupe

				} else {

					if (foundSection != null) {

						cutTurn(false)

					}

				}

			}

			trsSection.remove(turn)

			currentNode.append(turn)

		}

		outputDirectory.mkdir()

		File outfile = new File(outputDirectory, trsFile.getName())

		outfile.withWriter("UTF-8") { writer ->

			writer.write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE Trans SYSTEM "trans-14.dtd">\n')

			def printer = new groovy.util.XmlNodePrinter(new PrintWriter(writer))

			printer.setPreserveWhitespace(true)

			printer.print(trs)

		}

	}

	cpb.done()

	reader.close()

	println "Done."

} catch(Exception e) {

	println "Error: "+e

	Log.printStackTrace(e)

}

def cutTurn(def incrementISection) {

	def children = turn.children()

	Node newTurnKaNode = null;//new Node(trsEpisode, "Turn", currentSection[2])

	def startSection = currentNode.@startTime

	if (startSection instanceof String) startSection = Float.parseFloat(currentNode.@startTime)

	def endSection = currentNode.@endTime

	if (endSection instanceof String) endSection = Float.parseFloat(currentNode.@endTime)

	//println "Cut the last turn if necessary"

	for (int iChildren = 0 ; iChildren < children.size() ; iChildren++) {

		def c = children[iChildren]

		if (c instanceof String) continue;

		if (newTurnKaNode != null) { // append the remaining children to the new turn

			turn.remove(c)

			newTurnKaNode.append(c)

			if (debug) c.@moved="yes"

			iChildren--

		} else {

			if ("w".equals(c.name())) {

				def start2 = Float.parseFloat(c.@startTime)

				def end2 = Float.parseFloat(c.@endTime)

				if (start2 > endSection && Math.abs(start2 - endSection) > turnsCutActivationThreashold) {

					if (debug) println "cut with a w at ($start2, $end2) for section ("+startSection+", "+endSection+")"

					newTurnKaNode = new Node(trsSection, "Turn", ["startTime":""+start2, "endTime":""+turn.@endTime, "speaker":turn.@speaker])

					turns.add(iTurn, newTurnKaNode)

					iTurn--

					if (debug) newTurnKaNode.@created = "yes"

					turn.@endTime = ""+start2;

					turn.remove(c)

					newTurnKaNode.append(c)

					if (debug) c.@moved="yes"

					if (incrementISection) iSection++

					iChildren--

				}

			} else if ("Sync".equals(c.name())) {

				def start2 = c.@time

				if (start2 instanceof String) start2 = Float.parseFloat(c.@time)

				def end2 = start2

				if (start2 > endSection && Math.abs(start2 - endSection) > turnsCutActivationThreashold) {

					if (debug) println "cut with a Sync at ($start2, $end2) for section ("+startSection+", "+endSection+")"

					newTurnKaNode = new Node(trsSection, "Turn", ["startTime":""+start2, "endTime":""+turn.@endTime, "speaker":turn.@speaker])

					turns.add(iTurn, newTurnKaNode)

					iTurn--

					if (debug) newTurnKaNode.@created = "yes"

					turn.@endTime = ""+start2;

					turn.remove(c)

					if (debug) c.@moved="yes"

					if (incrementISection) iSection++

					newTurnKaNode.append(c)

					iChildren--

				}

			} else {

				// no time to check

			}

		}

	}

}

@Field @Option(name="fixTurnsLimits", usage="Correction des limites de sections du tableau de metadonnees", widget="Boolean", required=true, def="true")

		def fixTurnsLimits

@Field @Option(name="turnsCutActivationThreashold", usage="marge d'erreur de corrections des limites de tours", widget="Float", required=true, def="0.1")

		def turnsCutActivationThreashold

debug = true

	while (reader.readRecord()) { // loading & sorting sections

				if (Math.abs(sections[iSection][0] - sections[iSection - 1][1]) < sectionsMergeActivationThreashold) {

			println "Error: multiple Episode node in $trsFile"

			println "Error: multiple Section node in $trsFile"

			def foundSection = null;

				} else if (sections[i][1] < start) { // Section is before section

					foundSection = sections[i]

			if (foundSection == null) {

				if (foundSection != currentSection) {

					currentSection = foundSection

		}

		//remove the initial section which is empty now or not

		trsEpisode.remove(trsSection)

		if (fixTurnsLimits) {

			if (debug) println "Fixing Turn limits..."

			def turnToInsert = null;

			sections = trs.Episode.Section

			for (int i = 0 ; i < sections.size() -1 ; i++) { // browse created sections but stop before the last one (whichc can not be fixed)

				def section = sections[i]

				if (turnToInsert != null) {

					if (debug) println "Moving part-of turn: "+turnToInsert+" in section ("+section.@startTime+", "+section.@endTime+")"

					section.children().add(0, turnToInsert) // insert the slited part of the turn in the section

				}

				def startSection = section.@startTime // Float.parseFloat(section.@startTime)

				def endSection = section.@endTime //  Float.parseFloat(section.@endTime)

				if (startSection instanceof String) startSection = Float.parseFloat(section.@startTime)

				if (endSection instanceof String) endSection = Float.parseFloat(section.@endTime)

				turnToInsert = null

				turns = section.Turn

				if (turns.size() == 0) continue;

				def turn = turns[-1]

					if (c instanceof String) continue;

							if (start2 > endSection && Math.abs(start2 - endSection) > turnsCutActivationThreashold) {

								if (debug) println "cut with a w at [$start2, $end2] for section ("+startSection+", "+endSection+")"

							if (start2 > endSection && Math.abs(start2 - endSection) > turnsCutActivationThreashold) {

								if (debug) println "cut with a Sync at [$start2, $end2] for section "+endSection

				turnToInsert = newTurnKaNode

package org.txm.macro.nlp;

	import org.kohsuke.args4j.*

	import groovy.transform.Field

	import java.nio.charset.Charset

	import org.txm.rcpapplication.swt.widget.parameters.*

	import org.txm.utils.*

	import javax.xml.stream.*

	@Field @Option(name="inputDirectory", usage="TXT directory", widget="Folder", required=true, def="dir")

	File inputDirectory

	@Field @Option(name="encoding", usage="File encoding", widget="String", required=false, def="UTF-8")

	String encoding

	@Field @Option(name="debug", usage="Debug mode", widget="Boolean", required=false, def="false")

	Boolean debug

	if (!ParametersDialog.open(this)) return

	encoding = encoding.trim()

	outputDirectory = new File(inputDirectory, "xml")

	outputDirectory.mkdir()

	println "Processing: "+inputDirectory

	XMLOutputFactory factory = XMLOutputFactory.newInstance()

	def files = inputDirectory.listFiles().sort{ it.name }

	if (files == null || files.length == 0) {

		println "Error: no file to process in $inputDirectory"

		return false;

	}

	for (File inputfile : files.sort()) {

		if (inputfile.isDirectory() || inputfile.isHidden() || !inputfile.getName().endsWith(".tt")) continue // ignore

		println " file: "+inputfile

		name = inputfile.getName()

		idx = name.lastIndexOf(".")

		if (idx > 0) name = name.substring(0, idx)

		outputfile = new File(outputDirectory, name+".xml")

		Writer output = new OutputStreamWriter(new FileOutputStream(outputfile) , "UTF-8")

		XMLStreamWriter writer = factory.createXMLStreamWriter(output)

		writer.writeStartDocument("UTF-8","1.0")

		writer.writeCharacters("\n")

		writer.writeStartElement("text")

		writer.writeCharacters("\n")

		inSent = 0

		nLine = 0

		inputfile.eachLine(encoding) { line ->

			nLine++

			if (debug) {

				println nLine+": "+line

			}

			def sent = false

	        def sent_pattern = ~/^([^\t]+)\t(SENT)\t([^\t]+)$/

	        def sent_m = (line =~ sent_pattern)

	        if (sent_m) sent = true

			if (sent && inSent) {

				writer.writeStartElement("w")

				writer.writeAttribute("frpos", sent_m[0][2])

				writer.writeAttribute("frlemma", sent_m[0][3])

				writer.writeCharacters(sent_m[0][3])

				writer.writeEndElement() // close w		

				writer.writeComment("\n")

				writer.writeEndElement() // close s

				writer.writeEndElement() // close p (sent)

				writer.writeCharacters("\n")

				inSent=0

			} else {

				if (!inSent) {

					writer.writeStartElement("p")

					writer.writeStartElement("s")

					inSent=1

				}

				def w_pattern = ~/^([^\t]+)\t([^\t]+)\t([^\t]+)$/

	        	def w_m = (line =~ w_pattern)

	        	if (w_m.size() > 0) {

	        		writer.writeStartElement("w")

					writer.writeAttribute("frpos", w_m[0][2])

					writer.writeAttribute("frlemma", w_m[0][3])

					writer.writeCharacters(w_m[0][1])

					writer.writeEndElement() // close w

					writer.writeCharacters("\n")

				} else {

				    writer.writeStartElement("w")

					writer.writeAttribute("frpos", "NAM")

					writer.writeAttribute("frlemma", "<unknown>")

					writer.writeCharacters(line)

					writer.writeEndElement() // close w

					writer.writeCharacters("\n")

					println "** TT2XMLInDirectory: unknown [word] line pattern, "+nLine+": <"+line+">."

				}

			}

		}

		if (inSent) {

			writer.writeEndElement() // s

			writer.writeEndElement() // p

			writer.writeCharacters("\n")

		}

		writer.writeEndElement()

		writer.writeCharacters("\n")

		writer.close()

		output.close()

	}

	, "fixSectionsLimits":true

	, "sectionsMergeActivationThreashold":4.0

	, "fixTurnsLimits":true

	, "turnsCutActivationThreashold":1.0

							if (enableCollapsibles && parser.getAttributeCount() > 2) {

								pagedWriter.writeAttribute("class", "transcription-table collapsiblecontent")

								pagedWriter.writeAttribute("style", "display:none;")

							} else {

								pagedWriter.writeAttribute("class", "transcription-table");

							}

						System.out.println(NLS.bind(TXMCoreMessages.corpusUpdateDoneInP0, ExecTimer.stop()));

						System.out.println(NLS.bind(TXMCoreMessages.corpusImportDoneInP0, ExecTimer.stop()));

			if (ams.get(c) != null && ams.get(c).tempManager != null && ams.get(c).tempManager.getEntityManager() != null ) {

				ams.get(c).tempManager.close(); // free files

			}

				Log.warning("Warning: error while deleting the corpus: "+e);

				Log.printStackTrace(e);

		Label label = toolkit.createLabel(sectionClient, "CQP Structure properties projections", SWT.WRAP);