Laboratoire ICAR » Plateforme TXM

	boolean debug

					if (debug) println "Recoding id $id -> $newId and name $name -> newName"

			int nReplace = 0

					nReplace++

			if (debug) println "$nReplace replacements"

		File infile = new File("/home/mdecorde/xml/vocapia/testé input","test.xml")

		File outfile = new File("/home/mdecorde/xml/vocapia/testé oh","testé.trs")

		outfile.getParentFile().mkdir()

		println processor.process(outfile)

package org.txm.macro.transcription

import java.nio.charset.Charset

import java.time.LocalTime

import java.time.format.DateTimeFormatter

import org.txm.utils.*

import org.txm.utils.logger.*

@Field @Option(name="trsDirectory", usage="Dossier qui contient les fichiers TRS", widget="Folder", required=true, def="")

		File trsDirectory;

@Field @Option(name="resultDirectory", usage="Dossier résultat TRS", widget="Folder", required=true, def="")

		File resultDirectory;

@Field @Option(name="wordElement", usage="The marker, spaces included", widget="String", required=true, def="w")

		def wordElement

@Field @Option(name="newEventMarker", usage="The marker, spaces included", widget="String", required=true, def="")

		def newEventMarker

@Field @Option(name="eventType", usage="The marker, spaces included", widget="String", required=true, def="")

		def eventType

@Field @Option(name="eventDescription", usage="The marker, spaces included", widget="String", required=false, def="")

		def eventDescription

@Field @Option(name="eventExtent", usage="The marker, spaces included", widget="String", required=true, def="instantaneous")

		def eventExtent

@Field @Option(name="debug", usage="activate debug messages", widget="Boolean", required=true, def="false")

		boolean debug

if (!ParametersDialog.open(this)) return;

if (!trsDirectory.exists()) {

	println "$trsDirectory not found"

	return

}

println "Writing result to $resultDirectory..."

def newEventMarkerRegex = /$newEventMarker/

try {

	def trsFiles = trsDirectory.listFiles().findAll() { it.getName().toLowerCase().endsWith(".trs") }

	if (trsFiles.size() == 0) {

		println "No TRS file to process in $trsDirectory"

		return;

	}

	ConsoleProgressBar cpb = new ConsoleProgressBar(trsFiles.size())

	for (File trsFile : trsFiles) {

		if (debug) println "== $trsFile =="

		else cpb.tick()

		// Open input file

		def slurper = new groovy.util.XmlParser(false, true, true);

		//slurper.setFeature("http://apache.org/xml/features/disallow-doctype-decl", false) // allow DTD declaration

		//slurper.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "all"); // allow to read DTD from local file

		def trs = slurper.parse(trsFile.toURI().toString())

		def trsEpisodes = trs.Episode // 1

		if (trsEpisodes.size() > 1) {

			println "Error: multiple Episode node in $trsFile"

			continue

		}

		def turns = trs.Episode.Section.Turn

		for (int iTurn = 0 ; iTurn < turns.size() ; iTurn++) {

			def turn = turns[iTurn]

			def children = turn.children()

			for (int i = 0 ; i < children.size() ; i++) {

				def w = children[i]

				if (wordElement.equals(w.name())) {

					String wContent = w.text()

					if (wContent ==~ newEventMarkerRegex) {

						if (debug) println "Create event $w with $wContent"

						Node replace = new Node(null, "event");

						replace.@type = eventType

						replace.@desc = eventDescription

						replace.@extent = eventExtent

						w.replaceNode(replace)

					}

				}

			}

		}

		resultDirectory.mkdir()

		File outfile = new File(resultDirectory, trsFile.getName())

		outfile.withWriter("UTF-8") { writer ->

			writer.write('<?xml version="1.0" encoding="UTF-8"?>\n')

			def printer = new groovy.util.XmlNodePrinter(new PrintWriter(writer))

			printer.setPreserveWhitespace(true)

			printer.print(trs)

		}

	}

	cpb.done()

	println "Done."

} catch(Exception e) {

	println "Error: "+e

	Log.printStackTrace(e)

}

@Field @Option(name="debug", usage="show debug messages", widget="String", required=true, def="false")

def debug

		if (debug) println "== $id =="

		else cpb.tick()

		if (debug) println "== $trsFile =="

		else cpb.tick()

			//if (debug) println "TURN: "

					if (debug) println "New section at $turn with $node child node"

@Field @Option(name="debug", usage="The result directory", widget="Boolean", required=false, def="false")

		boolean debug;

	if (debug) println "== $xmlFile =="

	else cpb.tick()

package org.txm.macro.table

import org.kohsuke.args4j.*

import groovy.transform.Field

import java.nio.charset.Charset

import org.txm.rcpapplication.swt.widget.parameters.*

import org.txm.utils.*

import javax.xml.stream.*

import java.net.URL

import org.apache.poi.ss.usermodel.*

import org.apache.poi.hssf.usermodel.*

import org.apache.poi.xssf.usermodel.*

import org.apache.poi.ss.usermodel.WorkbookFactory

import org.apache.poi.ss.util.*

@Field @Option(name="inputDirectory", usage="répertoire des fichiers Excel à traiter", widget="Folder", required=true, def="")

def inputDirectory

@Field @Option(name="sheetName", usage="sheet name (if no name is given the first sheet will be used)", widget="String", required=false, def="")

def sheetName

@Field @Option(name="rootTag", usage="Root tag name", widget="String", required=false, def="root")

def rootTag

@Field @Option(name="textTag", usage="Text tag name", widget="String", required=false, def="unit")

def textTag

@Field @Option(name="metadataColumnList", usage="metadataColumnList column list separated with comma", widget="String", required=false, def="meta1,meta2")

def metadataColumnList

@Field @Option(name="dateColumnList", usage="date columns list separated by comma", widget="String", required=false, def="meta1,meta2")

def dateColumnList

@Field @Option(name="textColumnList", usage="textColumnList column list separated with comma", widget="String", required=false, def="textColumnList1,textColumnList2")

def textColumnList

if (!ParametersDialog.open(this)) return

if (!inputDirectory.exists()) {

	println "** ExcelDir2XML: no '"+inputDirectory.name+"' directory found. Aborting."

	return false

}

if (!inputDirectory.canRead()) {

	println "** ExcelDir2XML: '"+inputDirectory.name+"' directory not readable. Aborting."

	return false

}

def f = []

inputDirectory.eachFileMatch(~/.*xlsx/) { f << it }

if (f.size() == 0) {

	println "** ExcelDir2XML: no .xlsx file found. Aborting."

	return false

}

try {

f.sort { it.name }.each { inputFile ->

	res = gse.run(Excel2XMLMacro, ["args":[

"inputFile":inputFile,

"sheetName":sheetName,

"metadataColumnList":metadataColumnList,

"dateColumnList":dateColumnList,

"textColumnList":textColumnList,

"rootTag":rootTag,

"textTag":textTag,

				"selection":selection,

				"selections":selections,

				"corpusViewSelection":corpusViewSelection,

				"corpusViewSelections":corpusViewSelections,

				"monitor":monitor]])

			if (!res) println "** problem calling Excel2XMLMacro."

}

} catch (Exception e) {

	println "** ExcelDir2XML: unable to read input files. Aborting."

	println e.getLocalizedMessage()

	println e.printStackTrace()

	return false

}

return true

package org.txm.macro.table

import org.kohsuke.args4j.*

import groovy.transform.Field

import java.nio.charset.Charset

import org.txm.rcpapplication.swt.widget.parameters.*

import org.txm.utils.*

import javax.xml.stream.*

import java.net.URL

import java.text.SimpleDateFormat

import org.apache.poi.ss.usermodel.*

import org.apache.poi.hssf.usermodel.*

import org.apache.poi.xssf.usermodel.*

import org.apache.poi.ss.usermodel.WorkbookFactory

import org.apache.poi.ss.util.*

def stringToIndent = { str -> org.txm.utils.AsciiUtils.buildAttributeId(org.txm.utils.AsciiUtils.convertNonAscii(str)).toLowerCase() }

// from http://www.java-connect.com/apache-poi-tutorials/read-all-type-of-excel-cell-value-as-string-using-poi

def getCellValueAsString = { cell ->

        strCellValue = null

        if (cell != null) {

            switch (cell.getCellType()) {

            case CellType.STRING:

                strCellValue = cell.toString()

                break

            case CellType.NUMERIC:

                if (DateUtil.isCellDateFormatted(cell)) {

                    SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy")

                    strCellValue = dateFormat.format(cell.getDateCellValue())

                } else {

                    value = cell.getNumericCellValue()

                    longValue = value.longValue()

                    strCellValue = new String(longValue.toString())

                }

                break

            case CellType.BOOLEAN:

                strCellValue = new String(new Boolean(cell.getBooleanCellValue()).toString())

                break

            case CellType.BLANK:

                strCellValue = ""

                break

            }

        }

        if (strCellValue == null) strCellValue = ""

        return strCellValue

}

@Field @Option(name="inputFile", usage="fichier Excel à traiter", widget="File", required=true, def="")

File inputFile

@Field @Option(name="sheetName", usage="sheet name (if no name is given the first sheet will be used)", widget="String", required=false, def="")

def sheetName

@Field @Option(name="rootTag", usage="root tag name", widget="String", required=false, def="root")

def rootTag

@Field @Option(name="textTag", usage="line unit tag name", widget="String", required=false, def="unit")

def textTag

@Field @Option(name="metadataColumnList", usage="metadata columns list separated by comma", widget="String", required=false, def="meta1,meta2")

def metadataColumnList

@Field @Option(name="dateColumnList", usage="date columns list separated by comma", widget="String", required=false, def="meta1,meta2")

def dateColumnList

@Field @Option(name="textColumnList", usage="text columns list separated by comma", widget="String", required=false, def="textColumnList1,textColumnList2")

def textColumnList

if (!ParametersDialog.open(this)) return

if (!inputFile.exists()) {

	println "** Excel2XML: no '"+inputFile.name+"' file found. Aborting."

	return false

}

if (!inputFile.canRead()) {

	println "** Excel2XML: '"+inputFile.name+"' file not readable. Aborting."

	return false

}

try {

metadataColumnList = metadataColumnList.split(",").collect { it.trim() }

dateColumnList = dateColumnList.split(",").collect { it.trim() }

textColumnList = textColumnList.split(",").collect { it.trim() }

textTag = textTag.trim()

rootTag = rootTag.trim()

wb = WorkbookFactory.create(inputFile)

if (sheetName.length() == 0) {

	ws = wb.getSheetAt(0)

} else {

	ws = wb.getSheet(sheetName)

	if (ws == null) {

		println "** Excel2XML: no '"+sheetName+" found. Aborting."

		return false

	}

}

if (ws == null) {

	println "** Excel2XML: no sheet found. Aborting."

	return false

}

nRows = ws.getPhysicalNumberOfRows()

println nRows+" rows."

firstRow = ws.getRow(0)

colMax = firstRow.getLastCellNum()

headers = (0..colMax-1).collect { getCellValueAsString(firstRow.getCell(it)) }

println "Headers: $headers"

normalizedHeaders = headers.collect { stringToIndent(it) }

ok = true

metadataColumnList.each { m ->

	if (!headers.contains(m)) {

		println "** Excel2XML: missing metadataColumnList column: $m"

		ok = false

	}

}

textColumnList.each { t ->

	if (!headers.contains(t)) {

		println "** Excel2XML: missing textColumnList column: $t"

		ok = false

	}

}

if (!ok) { return false }

metadataColumnIndex = metadataColumnList.collect { headers.indexOf(it) }

dateColumnsIndex = dateColumnList.collect { headers.indexOf(it) }

textColumnIndex = textColumnList.collect { headers.indexOf(it) }

println "metadataColumnList = "+metadataColumnList

println "metadataColumnIndex = "+metadataColumnIndex

println "dateColumnList = "+dateColumnList

println "dateColumnsIndex = "+dateColumnsIndex

println "textColumnList = "+textColumnList

println "textColumnIndex = "+textColumnIndex

name = inputFile.getName()

idx = name.lastIndexOf(".")

if (idx > 0) name = name.substring(0, idx)

outputFile = new File(inputFile.getParentFile(), name+".xml")

factory = XMLOutputFactory.newInstance()

output = new FileOutputStream(outputFile)

writer = factory.createXMLStreamWriter(output, "UTF-8")

writer.writeStartDocument("UTF-8","1.0")

writer.writeCharacters("\n") // simple XML formating

writer.writeStartElement(rootTag)

writer.writeCharacters("\n")

pb_n = 1

(1..nRows-1).each { rowIndex ->

	writer.writeCharacters("  ")

	writer.writeEmptyElement("pb") // <pb/> to get one page per input line (don't forget high number of words per page in import module)

	writer.writeAttribute("n", ""+pb_n++)

	writer.writeCharacters("\n") // simple XML formating

	writer.writeCharacters("  ")

	writer.writeStartElement(textTag)

	metadataColumnIndex.each { colIndex -> // build an attribute for each metadata

		String s = getCellValueAsString(ws.getRow(rowIndex).getCell(colIndex));

		if (s == null) s ="";

		value = s.replaceAll("\n", ";").trim()

		writer.writeAttribute(normalizedHeaders[colIndex], value)

		if (colIndex in dateColumnsIndex) { // also split date attributes in day+month+year attributes

			matches = (value =~ /([0-9]{2})\/([0-9]{2})\/([0-9]{4})/)

			writer.writeAttribute(normalizedHeaders[colIndex]+"jour", matches[0][1])

			writer.writeAttribute(normalizedHeaders[colIndex]+"joursemaine", new java.text.SimpleDateFormat('EEEE').format(Date.parse("dd/MM/yyyy", value)))

			writer.writeAttribute(normalizedHeaders[colIndex]+"mois", matches[0][2])

			writer.writeAttribute(normalizedHeaders[colIndex]+"annee", matches[0][3])

		}

	}

	writer.writeCharacters("\n")

	writer.writeCharacters("    ")

	writer.writeStartElement("metadata")

	writer.writeStartElement("list")

	writer.writeAttribute("type", "unordered")

	writer.writeCharacters("\n")

 	metadataColumnIndex.each { colIndex ->

 		writer.writeStartElement("item")

 		writer.writeCharacters(headers[colIndex]+" : "+getCellValueAsString(ws.getRow(rowIndex).getCell(colIndex)).replaceAll("\n", ";"))

		writer.writeEndElement() // item

		writer.writeCharacters("\n")

	}

	writer.writeCharacters("    ")

	writer.writeEndElement() // list

	writer.writeEndElement() // head

	writer.writeCharacters("\n")

   	textColumnIndex.each { colIndex ->

    	writer.writeCharacters("    ")

    	writer.writeStartElement(normalizedHeaders[colIndex])

		writer.writeStartElement("p")

		writer.writeStartElement("head")

		writer.writeStartElement("hi")

		writer.writeCharacters(headers[colIndex]+" : ")

		writer.writeEndElement() // hi

		writer.writeEndElement() // head

		value = getCellValueAsString(ws.getRow(rowIndex).getCell(colIndex))

		if (value ==~ /(?s)^[A-Z]{3}:  [^;\n]+? +[;\n].*/) {

			value.findAll( /(?s)[A-Z]{3}:  ([^;\n]+?) +[;\n]/ ).each { desc ->

				writer.writeStartElement("descripteur")

				matches = (desc =~ /(?s)([A-Z]{3}):  ([^;\n]+?) +[;\n]/)

				writer.writeAttribute("type", matches[0][1])

				writer.writeCharacters(matches[0][2])

				writer.writeEndElement() // descripteur

			}

		} else {

			writer.writeCharacters(value)

		}

		writer.writeEndElement() // p

		writer.writeEndElement() // textColumn

		writer.writeCharacters("\n")

	}

	writer.writeCharacters("  ")

	writer.writeEndElement() // textTag

	writer.writeCharacters("\n")

}

writer.writeEndElement() // rootTag

writer.writeCharacters("\n")

writer.close()

output.close()

println "Result file: $outputFile"

} catch (Exception e) {

	println "** Excel2XML: unable to read input file. Aborting."

	println e.getLocalizedMessage()

	println e.printStackTrace()

	return false

}

return true

					def startOtherReg = /^(.*)\*([^\p{Zs}]+.*)$/

					def endOtherReg = /^(.*[^\p{Zs}]+)\*(.*)$/

									println "Warning at "+getLocation()+" with $word: Found a starting * when one 'other' have been started at "+previousOtherStarting

									if (debug) println "- ligne "+parser.getLocation().getLine()+" : ouverture de other avec '$word' -> tours '$turnInfos'"

								if (debug) println "- ligne "+parser.getLocation().getLine()+" : fermeture de other avec '$word' -> tours '$turnInfos'"

									println "Warning at "+getLocation()+" with $word: Found a closing * when one 'other' have been closed at "+previousOtherStarting

println "CONVERTING VOCAPIA FILES TO TRS FILES..."

gse.runMacro(org.txm.macro.transcription.Vocapia2TranscriberMacro, ["vocapiaDirectory":vocapiaDirectory, "resultDirectory":trsDirectory, "debug":debug])

println "SPOTTING SECTION MARKS..."

println "SPOTTING 'OTHER' TURNS..."

	if (debug) println "== $file =="

	else cpb.tick()

println "CONVERTIGN WORD MARKERS TO EVENTS..."

	println "No XML file found in $trsDirectory"

	return false

}

File eventsDirectory = new File(resultDirectory, "events")

eventsDirectory.mkdir()

gse.runMacro(org.txm.macro.transcription.AddEventsFromWordMarkersMacro, ["trsDirectory":otherDirectory, "resultDirectory":eventsDirectory, "wordElement": "w", "newEventMarker":"XXX", "eventDescription":"termes incompréhensibles ou inaudibles", "eventType":"pi", "eventExtent": "instantaneous", "debug":debug])

println "NORMALIZING SPEAKER IDs..."

trsFiles = eventsDirectory.listFiles().findAll(){it.getName().toLowerCase().endsWith(".trs")}

if (trsFiles.size() == 0) {

	if (debug) println "== $file =="

	else cpb.tick()

	fixer.debug = debug

	eventsDirectory.deleteDir()