Laboratoire ICAR » Plateforme TXM

package org.txm.macro.transcription

import java.nio.charset.Charset

import java.time.LocalTime

import java.time.format.DateTimeFormatter

import org.txm.utils.*

@Field @Option(name="metadataFile", usage="Tableau des metadonnées de sections", widget="FileOpen", required=true, def="")

File metadataFile;

@Field @Option(name="trsDirectory", usage="Dossier qui contient les fichiers TRS", widget="Folder", required=true, def="")

File trsDirectory;

@Field @Option(name="joinTRSColumn", usage="Colonne de jointure de transcription", widget="String", required=true, def="Lien notice principale")

def joinTRSColumn

@Field @Option(name="startTimeColumn", usage="Colonne de timing de début de section", widget="String", required=true, def="antract_debut")

def startTimeColumn = "antract_debut"

@Field @Option(name="endTimeColumn", usage="Colonne de timing de fin de section", widget="String", required=true, def="antract_fin")

def endTimeColumn = "antract_fin"

@Field @Option(name="typeColumns", usage="Colonnes des métadonnées de type de section", widget="String", required=true, def="Titre propre")

def typeColumns

@Field @Option(name="topicColumns", usage="Colonnes des métadonnées de topic de section", widget="String", required=true, def="Date de diffusion")

def topicColumns

@Field @Option(name="metadataColumns", usage="Colonnes de metadonnées de section", widget="String", required=true, def="Titre propre;Date de diffusion;Identifiant de la notice;Notes du titre;Type de date;Durée;Genre;Langue VO / VE;Nature de production;Producteurs (Aff.);Thématique;Nom fichier segmenté (info);antract_video;antract_debut;antract_fin;antract_duree;antract_tc_type;antract_tc_date;Résumé;Séquences;Descripteurs (Aff. Lig.);Générique (Aff. Lig.)")

def metadataColumns

@Field @Option(name="metadataColumnsGroups", usage="Colonnes des gruopes de metadonnées de section", widget="String", required=true, def="metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;secondary;secondary;secondary;secondary;secondary;secondary;secondary;text;text;text;text")

def metadataColumnsGroups

if (!ParametersDialog.open(this)) return;

typeColumns = typeColumns.split(";")

topicColumns = topicColumns.split(";")

metadataColumns = metadataColumns.split(";")

metadataColumnsGroups = metadataColumnsGroups.split(";")

if (metadataColumns.size() != metadataColumnsGroups.size()) {

	println "ERROR in metadata declarations&groups:"

	println "COLUMNS: "+metadataColumns

	println "GROUPS : "+metadataColumnsGroups

	return

}

if (!trsDirectory.exists()) {

	println "$trsDirectory not found"

	return

}

println "Loading data from $metadataFile..."

TableReader reader = new TableReader(metadataFile)//, "\t".charAt(0), Charset.forName("UTF-8")

reader.readHeaders()

def header = reader.getHeaders()

if (!header.contains(joinTRSColumn)) {

	println "No TRS ID $joinTRSColumn column found"

	return

}

if (!header.contains(startTimeColumn)) {

	println "No start time $startTimeColumn column found"

	return

}

if (!header.contains(endTimeColumn)) {

	println "No end time $endTimeColumn column found"

	return

}

for (def col : metadataColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No $col column found"

		return

	}

}

for (def col : typeColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No type $col column found"

		return

	}

}

for (def col : topicColumns) {

	if (!header.contains(endTimeColumn)) {

		println "No topic $col column found"

		return

	}

}

File outputDirectory = new File(trsDirectory, "out")

println "Writing result to $outputDirectory..."

dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_TIME

def strTotime(def str) {

	if (str.lastIndexOf(":") == -1) {

		return null

	}

	bonusFrame = Integer.parseInt(str.substring(str.lastIndexOf(":")+1))

	//if (str.contains("135475")) println "ERROR $str in $infos -> $bonusFrame"

	if (bonusFrame > 25) {

		bonusFrame=0;

	}

	totalFrame = str.substring(0, str.lastIndexOf(":"))

	LocalTime time1 = LocalTime.parse(totalFrame, dateTimeFormatter)

	totalFrame = (time1.getHour()*60*60) + (time1.getMinute()*60) + time1.getSecond()

		def ret = totalFrame + (bonusFrame/25)

		return ret

	}

def sectionGroupsToInsert = [:]

println "Reading data..."

while (reader.readRecord()) {

	String id = reader.get(joinTRSColumn).trim()

	if (id.endsWith(".mp4")) id = id.substring(0, id.length()-4)

	if (id.length() == 0) continue;

	if (!sectionGroupsToInsert.containsKey(id)) {

		sectionGroupsToInsert[id] = []

	}

	def section = sectionGroupsToInsert[id]

	if (reader.get(startTimeColumn) != null && reader.get(startTimeColumn).length() > 0) { // ignore non timed sections

		def m = [:]

		for (def todo : ["topic":topicColumns, "type":typeColumns]) {

			def data = []

			for (def col : todo.value) {

				if (reader.get(col).trim().length() > 0) {

					data << reader.get(col).trim().replace("\n", "")

				}

			}

			m[todo.key] = data.join("\t")

		}

		def metadataList = []

		def metadataGroupList = []

		for (int i = 0 ;  i < metadataColumns.size() ; i++) {

			def col = metadataColumns[i]

			String c = AsciiUtils.buildAttributeId(col)

			m[c] = reader.get(col)

			metadataList << c

			metadataGroupList << metadataColumnsGroups[i]

		}

		m["metadata"] = metadataList.join("|") 

		m["metadata_groups"] = metadataGroupList.join("|")

		m["startTime"] = strTotime(reader.get(startTimeColumn))

		m["endTime"] = strTotime(reader.get(endTimeColumn))

		m["synchronized"] = "true"

		section << [m["startTime"], m["endTime"], m]

	}

}

println "Inserting sections... "+sectionGroupsToInsert.size()

ConsoleProgressBar cpb = new ConsoleProgressBar(sectionGroupsToInsert.keySet().size())

for (String id : sectionGroupsToInsert.keySet()) {

	cpb.tick()

	File trsFile = new File(trsDirectory, id+".trs")

	if (!trsFile.exists()) {

		continue

	}

	//println "Processing $id..."

	def sections = sectionGroupsToInsert[id]

	sections = sections.sort() { a, b -> a[0] <=> b[0] ?: a[1] <=> b[1] }

	// Open input file

	def slurper = new groovy.util.XmlParser(false, true, true);

	slurper.setFeature("http://apache.org/xml/features/disallow-doctype-decl", false) // allow DTD declaration

	slurper.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "all"); // allow to read DTD from local file

	def trs = slurper.parse(trsFile.toURI().toString())

	def trsEpisodes = trs.Episode // 1

	if (trsEpisodes.size() > 1) {

		println "multiple Episode node in $trsFile"

		continue

	}

	def trsEpisode = trsEpisodes[0]

	def trsSections =  trs.Episode.Section // 1

	if (trsSections.size() > 1) {

		println "multiple Section node in $trsFile"

		continue

	}

	def trsSection = trsSections[0]

	def turns = trsSection.Turn

	def newSections = []

	def iSection = 0;

	def currentSection = null

	def currentNode = null

	for (def turn : turns) {

		def start = Float.parseFloat(turn.@startTime)

		def end = Float.parseFloat(turn.@endTime)

		def found = null;

		for (int i = iSection ; i < sections.size() ; i++) {

			if (end < sections[i][0]) { // Turn is before section

			} else if (sections[i][1] < start) { // Turn is before section

			} else {

				found = sections[i]

				iSection = i

				break; // stop searching and set iSection to accelerate next search

			}

		}

		if (found == null) {

			if (currentSection != null || currentNode == null) {

				currentNode = new Node(trsEpisode, "Section", ["type":"Sujet non synchronisé", "startTime":turn.@startTime, "endTime":"", "synchronized":"false"] )

				currentSection = null;

			}

		} else {

			if (found != currentSection) {

				if (currentNode != null && currentNode.@synchronized == "false") {

					def tmp = currentNode.Turn

					currentNode.@endTime = tmp[-1].@endTime

				}

				currentSection = found

				currentNode = new Node(trsEpisode, "Section", currentSection[2])

			}

		}

		trsSection.remove(turn)

		currentNode.append(turn)

	}

	//remove the initial section

	trsEpisode.remove(trsSection)

	outputDirectory.mkdir()

	File outfile = new File(outputDirectory, trsFile.getName())

	outfile.withWriter("UTF-8") { writer ->

		writer.write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE Trans SYSTEM "trans-14.dtd">\n')

		def printer = new groovy.util.XmlNodePrinter(new PrintWriter(writer))

		printer.setPreserveWhitespace(true)

		printer.print(trs)

	}

}

cpb.done()

reader.close()

println "Done."

package org.txm.macro.projects.antract

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.macro.transcription.*

import org.txm.utils.io.FileCopy

import org.txm.macro.projects.antract.BuildAFMetadataMacro

import org.txm.macro.projects.antract.BuildAFMetadataMacro

@Field @Option(name="tableFile", usage="Tableau des metadonnées de sections", widget="FileOpen", required=true, def="all.xlsx") // /home/mdecorde/TEMP/ANTRACT/AF/all.xlsx

File tableFile;

@Field @Option(name="trsDirectory", usage="Dossier qui contient les fichiers TRS à corriger", widget="Folder", required=true, def="AF")

def trsDirectory

if (!ParametersDialog.open(this)) return;

File workingDirectory = tableFile.getParentFile()

// extract infos for sujets and emissions from the main table file

gse.runMacro(BuildAFMetadataMacro, ["tableFile":tableFile, 

				"buildSujetsMetadata": true,

				"buildEmissionsMetadata": true])

File emissionsFile = new File(workingDirectory, "emissions.xlsx")

File sujetsFile = new File(workingDirectory, "sujets.xlsx")

// fix TRS files in the trsDirectory directory

gse.runMacro(FixINATRSMacro, ["trsDirectory":trsDirectory])

// insert the section in the TRS files

gse.runMacro(AddSectionsFromTableMacro, ["metadataFile": sujetsFile

	, "trsDirectory": trsDirectory

	, "joinTRSColumn": "Lien notice principale"

	, "startTimeColumn": "antract_debut"

	, "endTimeColumn": "antract_fin"

	, "typeColumns": "Titre propre"

	, "topicColumns": "Date de diffusion"

	, "metadataColumns": "Titre propre;Date de diffusion;Identifiant de la notice;Notes du titre;Type de date;Durée;Genre;Langue VO / VE;Nature de production;Producteurs (Aff.);Thématique;Nom fichier segmenté (info);antract_video;antract_debut;antract_fin;antract_duree;antract_tc_type;antract_tc_date;Résumé;Séquences;Descripteurs (Aff. Lig.);Générique (Aff. Lig.)"

	, "metadataColumnsGroups": "metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;metadata;secondary;secondary;secondary;secondary;secondary;secondary;secondary;text;text;text;text"])

//copy the emissions file in the source directory

FileCopy.copy(emissionsFile, new File(new File(trsDirectory, "out"), "metadata.xlsx"))

//done \o/

println "Done: import to finalize using the "+new File(trsDirectory, "out")+" directory."

package org.txm.macro.projects.antract

import java.io.File

import java.nio.charset.Charset

import java.time.LocalTime

import java.time.format.DateTimeFormatter

import java.util.Arrays

import java.util.HashMap

import java.util.List

import org.txm.libs.msoffice.ReadExcel

import org.txm.utils.*

@Field @Option(name="tableFile", usage="Tableau des metadonnées de sections", widget="FileOpen", required=true, def="all.xlsx") // /home/mdecorde/TEMP/ANTRACT/AF/all.xlsx

File tableFile;

@Field @Option(name="buildSujetsMetadata", usage="Build the sujets metadata", widget="Boolean", required=true, def="true")

boolean buildSujetsMetadata;

@Field @Option(name="buildEmissionsMetadata", usage="Build the emissions metadata", widget="Boolean", required=true, def="true")

boolean buildEmissionsMetadata;

if (!ParametersDialog.open(this)) return;

File table2File = null;

System.out.println("opening $tableFile...");

ReadExcel excel = new ReadExcel(tableFile, null);

if (buildEmissionsMetadata) {

	//emissions

	table2File = new File(tableFile.getParentFile(), "emissions.xlsx");

	HashMap<String, String> lineRules = new HashMap<>(); // line tests to select line to keep

	List<String> columnsSelection; // list of columns to keep

	HashMap<String, String> columnsToCopy = new HashMap<>();

	HashMap<String, String> columnsToRenameRules = new HashMap<>();

	HashMap<String, String[]> searchAndReplaceRules = new HashMap<>();

	columnsSelection = Arrays.asList(

	"Identifiant de la notice", "Titre propre", "Notes du titre", "Date de diffusion", "Durée", "Nom fichier segmenté (info)", "antract_video",

	"antract_debut","antract_fin","antract_duree","antract_tc_type","antract_tc_date");

	lineRules.put("Type de notice", "Notice sommaire");

	columnsToRenameRules.put("Identifiant de la notice", "id");

	columnsToCopy.put("Notes du titre", "subtitle");

	columnsToCopy.put("Titre propre", "title"); 

	columnsToCopy.put("Date de diffusion", "textorder");

	columnsToCopy.put("Date de diffusion", "annee");

	searchAndReplaceRules.put("textorder", ["([0-9][0-9])/([0-9][0-9])/([0-9][0-9][0-9][0-9])", '$3$2$1'] as String[]); // not working yet

	searchAndReplaceRules.put("annee", ["([0-9][0-9])/([0-9][0-9])/([0-9][0-9][0-9][0-9])", '$3'] as String[]); // not working yet

	process(excel, table2File, lineRules, columnsSelection, columnsToCopy, searchAndReplaceRules, columnsToRenameRules)

}

if (buildSujetsMetadata) {

	// sujets

	table2File = new File(tableFile.getParentFile(), "sujets.xlsx");

	HashMap<String, String> lineRules = new HashMap<>(); // line tests to select line to keep

	List<String> columnsSelection; // list of columns to keep

	HashMap<String, String> columnsToCopy = new HashMap<>();

	HashMap<String, String> columnsToRenameRules = new HashMap<>();

	HashMap<String, String[]> searchAndReplaceRules = new HashMap<>();

	columnsSelection = Arrays.asList(

		"Identifiant de la notice", "Titre propre", "Notes du titre", "Lien notice principale",

		"Date de diffusion", "Type de date", "Durée", "Genre", "Langue VO / VE", "Nature de production", "Producteurs (Aff.)", "Thématique",

		"Nom fichier segmenté (info)", "antract_video", "antract_debut", "antract_fin", "antract_duree", "antract_tc_type", "antract_tc_date",

		"Résumé", "Séquences", "Descripteurs (Aff. Lig.)", "Générique (Aff. Lig.)");

	lineRules.put("Type de notice", "Notice sujet");

	process(excel, table2File, lineRules, columnsSelection, columnsToCopy, searchAndReplaceRules, columnsToRenameRules)

}

def process(ReadExcel excel, File table2File, def lineRules, def columnsSelection, def columnsToCopy, def searchAndReplaceRules, def columnsToRenameRules) {

	System.out.println("Writing: $table2File");

	table2File.delete();

	ReadExcel excel2 = new ReadExcel(table2File, null);

	println " Selecting $columnsSelection with lines matching $lineRules"

	if (!excel.extractTo(excel2, lineRules, columnsSelection)) {

		System.out.println("FAIL");

		return;

	}

	if (columnsToCopy.size() > 0) {

		System.out.println(" Copying column: " + columnsToCopy);

		excel2.copyColumns(columnsToCopy);

		println " WARNING: ReadExcel.copyColumns() not implemented"

	}

	if (searchAndReplaceRules.size() > 0) {

		System.out.println(" Search&replace column: " + searchAndReplaceRules);

		excel2.searchAndReplaceInLines(searchAndReplaceRules);

	}

	if (columnsToRenameRules.size() > 0) {

		System.out.println(" Renaming column: " + columnsToRenameRules);

		excel2.renameColumns(columnsToRenameRules);

	}

	System.out.println(" Saving&Closing...");

	excel2.save();

	excel2.close();

	excel.close();

	System.out.println("Done: $table2File");

	return table2File.exists()

}

package org.txm.macro.projects.antract

import org.txm.utils.ConsoleProgressBar

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

@Field @Option(name="trsDirectory", usage="Dossier qui contient les fichiers TRS à corriger", widget="Folder", required=true, def="AF")

def trsDirectory

if (!ParametersDialog.open(this)) return;

println "Fixing $trsDirectory"

def files = trsDirectory.listFiles()

ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

for (File trsFile : files) {

//File trsFile = new File(directory, "AFE86004868.trs")

	cpb.tick()

	if (!trsFile.getName().endsWith(".trs")) {

		continue;

	}

	String content = trsFile.getText("UTF-8")

	content = content.replaceAll("punct=\"([^\"]+)\"\">", "punct=\"\$1\">")

	content = content.replaceAll("<unk>", "???")

	content = content.replaceAll(" Time=\"", " time=\"")

	trsFile.setText(content, "UTF-8")

}

cpb.done()

println "Done"