Laboratoire ICAR » Plateforme TXM

package org.txm.macro.csv

import org.kohsuke.args4j.*

import groovy.transform.Field

import java.nio.charset.Charset

import org.txm.rcp.swt.widget.parameters.*

import org.txm.utils.*

import javax.xml.stream.*

import java.net.URL

@Field @Option(name="inputFile", usage="CSV File", widget="File", required=false, def="file.xlsx")

		File inputFile;

@Field @Option(name="outputDirectory", usage="output directory", widget="File", required=false, def="directory")

		File outputDirectory;

@Field @Option(name="textSelector", usage="column_to_test=regexp", widget="String", required=false, def="Type de notice=Notice sommaire")

		def textSelector;

@Field @Option(name="structureSelector", usage="column_to_test=regexp", widget="String", required=false, def="Type de notice=Notice sujet")

		def structureSelector;

@Field @Option(name="textIDColumn", usage="text id column", widget="String", required=false, def="Identifiant de la notice")

		def textIDColumn;

@Field @Option(name="joinColumn", usage="jointure column, values should point to the textIDColumn values", widget="String", required=false, def="Lien notice principale")

		def joinColumn;

@Field @Option(name="textMetadataColumnList", usage="text metadata columns", widget="String", required=false, def="Identifiant de la notice,Date de diffusion,Durée,Genre,Identifiant Matériels (info.)")

		def textMetadataColumnList;

@Field @Option(name="textContentColumnList", usage="text content columns", widget="String", required=false, def="Titre propre,Résumé,Descripteurs (Aff. Lig.),Descripteurs (Aff. Col.),Séquences")

		def textContentColumnList;

@Field @Option(name="structureTag", usage="structure metadata columns", widget="String", required=false, def="div")

		def structureTag;

@Field @Option(name="structureMetadataColumnList", usage="structure metadata columns", widget="String", required=false, def="Identifiant de la notice,Date de diffusion,Durée,Genre,Identifiant Matériels (info.)")

		def structureMetadataColumnList;

@Field @Option(name="structureContentColumnList", usage="structure content columns", widget="String", required=false, def="Titre propre,Résumé,Descripteurs (Aff. Lig.),Descripteurs (Aff. Col.),Séquences")

		def structureContentColumnList;

@Field @Option(name="metadataDateColumnList", usage="metadata columns of type=Date", widget="String", required=false, def="")

		def metadataDateColumnList;

@Field @Option(name="prefixContentColumnList", usage="metadata columns of type=Date", widget="String", required=false, def="")

		def prefixContentColumnList;

@Field @Option(name="listContentColumnList", usage="metadata columns of type=Date", widget="String", required=false, def="")

		def listContentColumnList;

//@Field @Option(name="structureOrderColumn", usage="structure column coding structure order", widget="String", required=false, def="Titre propre,Résumé,Descripteurs (Aff. Lig.),Descripteurs (Aff. Col.),Séquences")

//		def structureOrderColumn;

if (!ParametersDialog.open(this)) return;

textMetadataColumnList = textMetadataColumnList.split(",")

textContentColumnList = textContentColumnList.split(",")

structureMetadataColumnList = structureMetadataColumnList.split(",")

structureContentColumnList = structureContentColumnList.split(",")

metadataDateColumnList = metadataDateColumnList.split(",")

prefixContentColumnList = prefixContentColumnList.split(",")

listContentColumnList = listContentColumnList.split(",")

int ti = textSelector.indexOf("=")

String p1 = textSelector.substring(0, ti)

String p2 = textSelector.substring(ti+1)

textSelector = [p1, p2]

structureSelector = [structureSelector.substring(0, structureSelector.indexOf("=")), structureSelector.substring(structureSelector.indexOf("=")+1)]

println "textIDColumn, joinColumn=$textIDColumn, $joinColumn"

println "textMetadataColumnList columns: $textMetadataColumnList"

println "textContentColumnList columns: $textContentColumnList"

println "structureMetadataColumnList columns: $structureMetadataColumnList"

println "structureContentColumnList columns: $structureContentColumnList"

println "text selector="+textSelector

println "structure selector="+structureSelector

println "structureTag="+structureTag

TableReader reader = new TableReader(inputFile);

if (!reader.readHeaders()) {

	println "Error: no header"

	return

}

def headers = Arrays.asList(reader.getHeaders())

println "table columns: $headers"

def ok = true

for (def list : ["selection":[textIDColumn, joinColumn], "textMetadataColumnList":textMetadataColumnList,

	"textContentColumnList":textContentColumnList, "structureMetadataColumnList":structureMetadataColumnList,

	"structureContentColumnList":structureContentColumnList]) {

	for (String m : list.value) {

		m = m.trim()

		if (!headers.contains(m)) {

			println "Error: missing ${list.key} column: $m"

			ok = false

		}

	}

}

if (!ok) { return; }

// group by text

def texts = new LinkedHashMap()

def nRecord = 0

while (reader.readRecord()) {

	nRecord++

	//println "record="+reader.getRecord().get(textSelector[0])+" "+reader.getRecord().get(structureSelector[0])

	String id = reader.get(textIDColumn).trim()

	String join = reader.get(joinColumn).trim()

	String textSelectorValue = reader.get(textSelector[0]).trim()

	String structureSelectorValue = reader.get(structureSelector[0]).trim()

	if (textSelectorValue != null && textSelectorValue.matches(textSelector[1])) {

		if (!texts.containsKey(id)) texts[id] = []

		texts[id].add(0, reader.getRecord())

	} else if (structureSelectorValue != null && structureSelectorValue.matches(structureSelector[1])) {

		if (!texts.containsKey(join)) texts[join] = []

		texts[join].add(reader.getRecord())

	} else {

		// ignore

	}

}

println "N lines: "+nRecord

println "N groups: "+texts.size()

if (texts.size()  == 0) {

	println "No text found. Aborting."

	return

}

outputDirectory.mkdir()

for (def id : texts.keySet()) {

	def toWrite = texts[id]

	def text = toWrite[0]

	String textSelectorValue = text.get(textSelector[0]).trim()

	if (textSelectorValue != null && textSelectorValue.matches(textSelector[1])) {

		println "Processing text: $id"

		File outputfile = new File(outputDirectory, id+".xml")

		XMLOutputFactory factory = XMLOutputFactory.newInstance()

		FileOutputStream output = new FileOutputStream(outputfile)

		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8")

		writer.writeStartDocument("UTF-8","1.0")

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("TEI")

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("teiHeader")

		writer.writeEndElement() // teiHeader

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("text")

		writer.writeAttribute("id", id)

		for (String att : textMetadataColumnList) {

			if (att in metadataDateColumnList) {

				writeMetadataDate(text, att, writer)

			} else {

				writer.writeAttribute(AsciiUtils.buildAttributeId(att), text.get(att)) // struct

			}

		}

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("metadata")

		writer.writeStartElement("list")

		writer.writeAttribute("type", "unordered")

		writer.writeCharacters("\n")

		textMetadataColumnList.each { att ->

			writer.writeStartElement("item")

			writer.writeCharacters(att+" : "+text.get(att).replaceAll("\n", ";"))

			writer.writeEndElement() // item

			writer.writeCharacters("\n")

		}

		writer.writeEndElement() // list

		writer.writeEndElement() // metadata

		writer.writeCharacters("\n")

		for (String att : textContentColumnList) {

			if (att in prefixContentColumnList) {

				writeContentPrefix(text, att, writer)

			} else if (att in listContentColumnList) {

				writeContentList(text, att, writer)

			} else {

				writer.writeStartElement("p");

				writer.writeAttribute("type", att.trim())

				writer.writeCharacters("\n") // simple XML formating

				writer.writeStartElement("head")

				writer.writeStartElement("hi")

				writer.writeCharacters(att+" : ")

				writer.writeEndElement() // hi

				writer.writeEndElement() // head

				writer.writeCharacters(text.get(att)) // get textColumnList content

				writer.writeEndElement() // t

				writer.writeCharacters("\n") // simple XML formating)

			}

		}

		int pb_n = 1;

		for (int i = 1 ; i < toWrite.size() ; i++) {

			def record = toWrite[i]

			writer.writeEmptyElement("pb") // <pb/>

			writer.writeAttribute("n", ""+pb_n++)

			writer.writeStartElement(structureTag)

			for (String att : structureMetadataColumnList) {

				if (att in metadataDateColumnList) {

					writeMetadataDate(record, att, writer)

				} else {

					writer.writeAttribute(AsciiUtils.buildAttributeId(att), record.get(att)) // struct

				}

			}

			writer.writeCharacters("\n")

			writer.writeStartElement("metadata")

			writer.writeStartElement("list")

			writer.writeAttribute("type", "unordered")

			writer.writeCharacters("\n")

			structureMetadataColumnList.each { att ->

				writer.writeStartElement("item")

				writer.writeCharacters(att+" : "+record.get(att).replaceAll("\n", ";"))

				writer.writeEndElement() // item

				writer.writeCharacters("\n")

			}

			writer.writeEndElement() // list

			writer.writeEndElement() // metadata

			writer.writeCharacters("\n")

			for (String att : structureContentColumnList) {

				if (att in prefixContentColumnList) {

					writeContentPrefix(text, att, writer)

				} else if (att in listContentColumnList) {

					writeContentList(text, att, writer)

				} else {

					writer.writeStartElement("p");

					writer.writeAttribute("type", att.trim())

					writer.writeAttribute("id", record.get(textIDColumn))

					writer.writeCharacters("\n") // simple XML formating

					writer.writeStartElement("head")

					writer.writeStartElement("hi")

					writer.writeCharacters(att+" : ")

					writer.writeEndElement() // hi

					writer.writeEndElement() // head

					writer.writeCharacters(record.get(att)) // get textColumnList content

					writer.writeEndElement() // t

					writer.writeCharacters("\n") // simple XML formating

				}

			}

			writer.writeEndElement() // struct

			writer.writeCharacters("\n") // simple XML formating

		}

		writer.writeEndElement() // text

		writer.writeCharacters("\n") // simple XML formating

		writer.writeEndElement() // TEI

		writer.close()

		output.close()

		reader.close()

	} else {

		// error

		println "ERROR: '$id' text group with  no text line"

	}

}

def writeContentList(def record, def att, def writer) {

	writer.writeCharacters("\n")

	found = false

	def value = record.get(att)

	value.findAll( /(?s)([^\n]+?)[\n]/ ).each { desc ->

		found = true

		writer.writeCharacters("\t")

		writer.writeStartElement("p")

		writer.writeAttribute("rend", "list")

		matches = (desc =~ /(?s)([^\n]+?)[\n]/)

		writer.writeCharacters(matches[0][1])

		writer.writeEndElement() // p

		writer.writeCharacters("\n")

	}

	if (!found) {

		writer.writeCharacters("\t")

		writer.writeStartElement("p")

		writer.writeAttribute("rend", "no-list")

		writer.writeCharacters(value)

		writer.writeEndElement() // p

		writer.writeCharacters("\n")

	}

}

def writeContentPrefix(def record, def att, def writer) {

	writer.writeCharacters("\n")

	writer.writeStartElement("list")

	writer.writeAttribute("rend", "prefixes")

	writer.writeAttribute("type", "unordered")

	writer.writeCharacters("\n")

	found = false

	def value = record.get(att)

	value.findAll( /(?s)[A-Z]{3}:? *([^;\n]+?) +[;\n]/ ).each { desc ->

		found = true

		writer.writeCharacters("\t")

		writer.writeStartElement("item")

		matches = (desc =~ /(?s)([A-Z]{3}):? *([^;\n]+?) +[;\n]/)

		writer.writeAttribute("type", matches[0][1])

		writer.writeStartElement("span")

		writer.writeCharacters(matches[0][1]+" ")

		writer.writeEndElement() // span

		writer.writeCharacters(matches[0][2])

		writer.writeEndElement() // item

		writer.writeCharacters("\n")

	}

	if (!found) {

		writer.writeCharacters(value)

	}

	writer.writeEndElement() // list

	writer.writeCharacters("\n")

}

def writeMetadataDate(def record, def att, def writer) {

	String value = record.get(att)

	String att_normalized = AsciiUtils.buildAttributeId(att)

	matches = (value =~ /([0-9]{2})\/([0-9]{2})\/([0-9]{4})/)

	writer.writeAttribute(att_normalized+"jour", matches[0][1])

	writer.writeAttribute(att_normalized+"joursemaine", new java.text.SimpleDateFormat('EEEE').format(Date.parse("dd/MM/yyyy", value)))

	writer.writeAttribute(att_normalized+"mois", matches[0][2])

	writer.writeAttribute(att_normalized+"annee", matches[0][3])

	writer.writeAttribute(att_normalized+"tri", matches[0][3]+"-"+matches[0][2]+"-"+matches[0][1])

}

/*

 String name = inputFile.getName()

 int idx = name.lastIndexOf(".")

 if (idx > 0) name = name.substring(0, idx)

 */

							String metadata = parser.getAttributeValue(null,"metadata");

							if (type != null || desc != null) {

								if (type != null || type.length() ==0) {

									writer.writeCharacters(type+": "+desc);

								} else {

									writer.writeCharacters(desc);

								}

								writer.writeEndElement(); // h3

								if (metadata != null && metadata.length() > 0) { // the metadata to show

									writer.writeStartElement("ul");

									for (def m : metadata.split("\t")) {

										writer.writeStartElement("li");

										writer.writeCharacters(m);

										writer.writeEndElement(); // li

									writer.writeEndElement(); // ul

								} else if (parser.getAttributeCount() > 1) { // process all attributes

									writer.writeStartElement("ul");

									for (int i = 0 ; i < parser.getAttributeCount() ; i++) {

										String name = parser.getAttributeLocalName(i);

										if (!"type".equals(name)

										&& !"topic".equals(name)

										&& !"startTime".equals(name)

										&& !"endTime".equals(name)) {

											writer.writeStartElement("li");

											writer.writeCharacters(""+name+": "+parser.getAttributeValue(i));

											writer.writeEndElement(); // li

										}

									}

									writer.writeEndElement(); // ul

import org.txm.utils.ConsoleProgressBar

def directory = new File("/home/mdecorde/TEMP/ANTRACT/AF/trs")

println "Fixing $directory"

def files = directory.listFiles()

ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

for (File trsFile : files) {

//File trsFile = new File(directory, "AFE86004868.trs")

	cpb.tick()

	if (!trsFile.getName().endsWith(".trs")) {

		continue;

	}

	String content = trsFile.getText("UTF-8")

	content = content.replaceAll("punct=\"([^\"]+)\"\">", "punct=\"\$1\">")

	content = content.replaceAll("<unk>", "???")

	content = content.replaceAll(" Time=\"", " time=\"")

	trsFile.setText(content, "UTF-8")

}

cpb.done()

println "Done"

import java.nio.charset.Charset

import org.txm.utils.AsciiUtils

import org.txm.utils.ConsoleProgressBar

import org.txm.utils.CsvReader

import java.time.LocalTime

import java.time.format.DateTimeFormatter

File metadataFile = new File("/home/mdecorde/TEMP/ANTRACT/AF/metadata.tsv")

File trsDirectory = new File("/home/mdecorde/TEMP/ANTRACT/AF/trs")

def idTRSColumn = "Lien notice principale"

def typeColumns = ["Identifiant de la notice"]

def topicColumns = ["Titre propre", "Genre", "antract_debut", "antract_fin"]

def startTimeColumn = "antract_debut"

def endTimeColumn = "antract_fin"

//def metadataColumns = ["Identifiant de la notice", "Titre propre", "antract_debut", "antract_fin"]

def metadataColumns = ["Date de diffusion", "Descripteurs (Aff. Col.)", "Durée", "Générique (Aff. Col.)", "Genre", "Identifiant de la notice", "Langue VO / VE ", "Lien notice principale", "Nature de production ", "Nom fichier segmenté (info)", "Notes du titre ", "Producteurs (Aff.)", "Résumé", "Séquences", "Thématique", "Titre propre", "Type de date", "Type de notice"]

if (!trsDirectory.exists()) {

	println "$trsDirectory not found"

	return

}

CsvReader reader = new CsvReader(metadataFile.getAbsolutePath(), "\t".charAt(0), Charset.forName("UTF-8"))

reader.readHeaders()

def header = reader.getHeaders()

if (!header.contains(idTRSColumn)) {

	println "No TRS ID $idTRSColumn column found"

	return

}

if (!header.contains(startTimeColumn)) {

	println "No start time $startTimeColumn column found"

	return

}

if (!header.contains(endTimeColumn)) {

	println "No end time $endTimeColumn column found"

	return

}

for (def col : metadataColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No $col column found"

		return

	}

}

for (def col : typeColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No type $col column found"

		return

	}

}

for (def col : topicColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No topic $col column found"

		return

	}

}

File outputDirectory = new File(trsDirectory, "out")

println "Writing result to $outputDirectory..."

dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_TIME

def strTotime(def str) {

	if (str.lastIndexOf(":") == -1) {

		return null

	}

	bonusFrame = Integer.parseInt(str.substring(str.lastIndexOf(":")+1))

	//if (str.contains("135475")) println "ERROR $str in $infos -> $bonusFrame"

	if (bonusFrame > 25) {

		bonusFrame=0;

	}

	totalFrame = str.substring(0, str.lastIndexOf(":"))

	LocalTime time1 = LocalTime.parse(totalFrame, dateTimeFormatter)

	totalFrame = (time1.getHour()*60*60) + (time1.getMinute()*60) + time1.getSecond()

		def ret = totalFrame + (bonusFrame/25)

		return ret

	}

def sectionGroupsToInsert = [:]

while (reader.readRecord()) {

	String id = reader.get(idTRSColumn).trim()

	if (id.endsWith(".mp4")) id = id.substring(0, id.length()-4)

	if (id.length() == 0) continue;

	if (!sectionGroupsToInsert.containsKey(id)) {

		sectionGroupsToInsert[id] = []

	}

	def section = sectionGroupsToInsert[id]

	if (reader.get(startTimeColumn) != null && reader.get(startTimeColumn).length() > 0) { // ignore non timed sections

		def m = [:]

		for (def todo : ["metadata":metadataColumns, "topic":topicColumns, "type":typeColumns]) {

			def data = []

			for (def col : todo.value) {

				if (reader.get(col).trim().length() > 0) {

					data << col+": "+reader.get(col).trim().replace("\n", "")

				}

			}

			m[todo.key] = data.join("\t")

		}

		m["startTime"] = strTotime(reader.get(startTimeColumn))

		m["endTime"] = strTotime(reader.get(endTimeColumn))

		m["synchronized"] = "true"

		section << [m["startTime"], m["endTime"], m]

	}

}

println "N sections: "+sectionGroupsToInsert.size()

ConsoleProgressBar cpb = new ConsoleProgressBar(sectionGroupsToInsert.keySet().size())

for (String id : sectionGroupsToInsert.keySet()) {

	cpb.tick()

	File trsFile = new File(trsDirectory, id+".trs")

	if (!trsFile.exists()) {

		continue

	}

	//println "Processing $id..."

	def sections = sectionGroupsToInsert[id]

	sections = sections.sort() { a, b -> a[0] <=> b[0] ?: a[1] <=> b[1] }

	// Open input file

	def slurper = new XmlParser();

	slurper.setFeature("http://apache.org/xml/features/disallow-doctype-decl", false)

	def trs = slurper.parse(trsFile.toURI().toString())

	def trsEpisodes = trs.Episode // 1

	if (trsEpisodes.size() > 1) {

		println "multiple Episode node in $trsFile"

		continue

	}

	def trsEpisode = trsEpisodes[0]

	def trsSections =  trs.Episode.Section // 1

	if (trsSections.size() > 1) {

		println "multiple Section node in $trsFile"

		continue

	}

	def trsSection = trsSections[0]

	def turns = trsSection.Turn

	def newSections = []

	def iSection = 0;

	def currentSection = null

	def currentNode = null

	for (def turn : turns) {

		def start = Float.parseFloat(turn.@startTime)

		def end = Float.parseFloat(turn.@endTime)

		def found = null;

		for (int i = iSection ; i < sections.size() ; i++) {

			if (end < sections[i][0]) { // Turn is before section

			} else if (sections[i][1] < start) { // Turn is before section

			} else {

				found = sections[i]

				iSection = i

				break; // stop searching and set iSection to accelerate next search

			}

		}

		if (found == null) {

			if (currentSection != null || currentNode == null) {

				currentNode = new Node(trsEpisode, "Section", ["type":"non synchronisée", "topic":"none", "startTime":turn.@startTime, "endTime":"", "synchronized":"false"] )

				currentSection = null;

			}

		} else {

			if (found != currentSection) {

				if (currentNode != null && currentNode.@synchronized == "false") {

					def tmp = currentNode.Turn

					currentNode.@endTime = tmp[-1].@endTime

				}

				currentSection = found

				currentNode = new Node(trsEpisode, "Section", currentSection[2])

				//trsEpisode.appendNode(currentNode)

			}

		}

		trsSection.remove(turn)

		currentNode.append(turn)

	}

	//remove the initial section

	trsEpisode.remove(trsSection)

	outputDirectory.mkdir()

	File outfile = new File(outputDirectory, trsFile.getName())

	outfile.withWriter("UTF-8") { writer ->

		writer.write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE Trans SYSTEM "trans-14.dtd">\n')

		def printer = new groovy.util.XmlNodePrinter(new PrintWriter(writer))

		printer.setPreserveWhitespace(true)

		printer.print(trs)

	}

}

cpb.done()

		ConsoleProgressBar cpb = new ConsoleProgressBar(trsfiles.size())

			cpb.tick()

		cpb.done()

				cpb = new ConsoleProgressBar(trsfiles.size())

						cpb.tick()

				cpb.done()

		def files = txmDir.listFiles()

		println "Tokenizing entities "+files.length+" files"

		cpb = new ConsoleProgressBar(files.length)

		for (File pfile : files) {

			cpb.tick()

		cpb.done()

		println "Tokenizing "+files.length+" files from $txmDir"

		cpb = new ConsoleProgressBar(files.length)

		for (File pfile : files) {

			cpb.tick()

		cpb.done()

		files = tokenizedDir.listFiles()

		println("Building XML-TXM ("+files.length+" files)")

		cpb = new ConsoleProgressBar(files.length)

		for (File tfile : files) {

			cpb.tick()

		cpb.done()

import org.txm.utils.ConsoleProgressBar

		ConsoleProgressBar cpb = new ConsoleProgressBar(xmlfiles.size())

			if (f.exists()) {

				cpb.tick()

			}

		cpb.done()

		new File(module.getBinaryDirectory(),"tokenized").deleteDir()