Laboratoire ICAR » Plateforme TXM

		for (def match : matches) npositions += match.length +1

				corpus.getLanguage());

import org.w3c.tidy.Tidy

// This macros needs the following libraries: jsoup-1.11.3.jar and jtidy

//org.txm.core.preferences.TXMPreferences.dump();

println TXMPreferences.getString(CQPPreferences.CQI_SERVER_PATH_TO_CQPLIB, CQPPreferences.PREFERENCES_NODE);

	for (File inputfile : inputDirectory.listFiles().sort()) {

package org.txm.macro.csv

import org.kohsuke.args4j.*

import groovy.transform.Field

import java.nio.charset.Charset

import org.txm.rcp.swt.widget.parameters.*

import org.txm.utils.*

import javax.xml.stream.*

import java.net.URL

@Field @Option(name="inputFile", usage="CSV File", widget="File", required=false, def="file.xlsx")

		File inputFile;

@Field @Option(name="outputDirectory", usage="output directory", widget="File", required=false, def="directory")

		File outputDirectory;

@Field @Option(name="textSelector", usage="column_to_test=regexp", widget="String", required=false, def="Type de notice=Notice sommaire")

		def textSelector;

@Field @Option(name="structureSelector", usage="column_to_test=regexp", widget="String", required=false, def="Type de notice=Notice sujet")

		def structureSelector;

@Field @Option(name="textIDColumn", usage="text id column", widget="String", required=false, def="Identifiant de la notice")

		def textIDColumn;

@Field @Option(name="joinColumn", usage="jointure column, values should point to the textIDColumn values", widget="String", required=false, def="Lien notice principale")

		def joinColumn;

@Field @Option(name="textMetadataColumnList", usage="text metadata columns", widget="String", required=false, def="Identifiant de la notice,Date de diffusion,Durée,Genre,Identifiant Matériels (info.)")

		def textMetadataColumnList;

@Field @Option(name="textContentColumnList", usage="text content columns", widget="String", required=false, def="Titre propre,Résumé,Descripteurs (Aff. Lig.),Descripteurs (Aff. Col.),Séquences")

		def textContentColumnList;

@Field @Option(name="structureTag", usage="structure metadata columns", widget="String", required=false, def="div")

		def structureTag;

@Field @Option(name="structureMetadataColumnList", usage="structure metadata columns", widget="String", required=false, def="Identifiant de la notice,Date de diffusion,Durée,Genre,Identifiant Matériels (info.)")

		def structureMetadataColumnList;

@Field @Option(name="structureContentColumnList", usage="structure content columns", widget="String", required=false, def="Titre propre,Résumé,Descripteurs (Aff. Lig.),Descripteurs (Aff. Col.),Séquences")

		def structureContentColumnList;

@Field @Option(name="metadataDateColumnList", usage="metadata columns of type=Date", widget="String", required=false, def="")

		def metadataDateColumnList;

@Field @Option(name="prefixContentColumnList", usage="metadata columns of type=Date", widget="String", required=false, def="")

		def prefixContentColumnList;

@Field @Option(name="listContentColumnList", usage="metadata columns of type=Date", widget="String", required=false, def="")

		def listContentColumnList;

//@Field @Option(name="structureOrderColumn", usage="structure column coding structure order", widget="String", required=false, def="Titre propre,Résumé,Descripteurs (Aff. Lig.),Descripteurs (Aff. Col.),Séquences")

//		def structureOrderColumn;

if (!ParametersDialog.open(this)) return;

textMetadataColumnList = textMetadataColumnList.split(",")

textContentColumnList = textContentColumnList.split(",")

structureMetadataColumnList = structureMetadataColumnList.split(",")

structureContentColumnList = structureContentColumnList.split(",")

metadataDateColumnList = metadataDateColumnList.split(",")

prefixContentColumnList = prefixContentColumnList.split(",")

listContentColumnList = listContentColumnList.split(",")

int ti = textSelector.indexOf("=")

String p1 = textSelector.substring(0, ti)

String p2 = textSelector.substring(ti+1)

textSelector = [p1, p2]

structureSelector = [structureSelector.substring(0, structureSelector.indexOf("=")), structureSelector.substring(structureSelector.indexOf("=")+1)]

println "textIDColumn, joinColumn=$textIDColumn, $joinColumn"

println "textMetadataColumnList columns: $textMetadataColumnList"

println "textContentColumnList columns: $textContentColumnList"

println "structureMetadataColumnList columns: $structureMetadataColumnList"

println "structureContentColumnList columns: $structureContentColumnList"

println "text selector="+textSelector

println "structure selector="+structureSelector

println "structureTag="+structureTag

TableReader reader = new TableReader(inputFile);

if (!reader.readHeaders()) {

	println "Error: no header"

	return

}

def headers = Arrays.asList(reader.getHeaders())

println "table columns: $headers"

def ok = true

for (def list : ["selection":[textIDColumn, joinColumn], "textMetadataColumnList":textMetadataColumnList,

	"textContentColumnList":textContentColumnList, "structureMetadataColumnList":structureMetadataColumnList,

	"structureContentColumnList":structureContentColumnList]) {

	for (String m : list.value) {

		m = m.trim()

		if (!headers.contains(m)) {

			println "Error: missing ${list.key} column: $m"

			ok = false

		}

	}

}

if (!ok) { return; }

// group by text

def texts = new LinkedHashMap()

def nRecord = 0

while (reader.readRecord()) {

	nRecord++

	//println "record="+reader.getRecord().get(textSelector[0])+" "+reader.getRecord().get(structureSelector[0])

	String id = reader.get(textIDColumn).trim()

	String join = reader.get(joinColumn).trim()

	String textSelectorValue = reader.get(textSelector[0]).trim()

	String structureSelectorValue = reader.get(structureSelector[0]).trim()

	if (textSelectorValue != null && textSelectorValue.matches(textSelector[1])) {

		if (!texts.containsKey(id)) texts[id] = []

		texts[id].add(0, reader.getRecord())

	} else if (structureSelectorValue != null && structureSelectorValue.matches(structureSelector[1])) {

		if (!texts.containsKey(join)) texts[join] = []

		texts[join].add(reader.getRecord())

	} else {

		// ignore

	}

}

println "N lines: "+nRecord

println "N groups: "+texts.size()

if (texts.size()  == 0) {

	println "No text found. Aborting."

	return

}

outputDirectory.mkdir()

for (def id : texts.keySet()) {

	def toWrite = texts[id]

	def text = toWrite[0]

	String textSelectorValue = text.get(textSelector[0]).trim()

	if (textSelectorValue != null && textSelectorValue.matches(textSelector[1])) {

		println "Processing text: $id"

		File outputfile = new File(outputDirectory, id+".xml")

		XMLOutputFactory factory = XMLOutputFactory.newInstance()

		FileOutputStream output = new FileOutputStream(outputfile)

		XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8")

		writer.writeStartDocument("UTF-8","1.0")

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("TEI")

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("teiHeader")

		writer.writeEndElement() // teiHeader

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("text")

		writer.writeAttribute("id", id)

		for (String att : textMetadataColumnList) {

			if (att in metadataDateColumnList) {

				writeMetadataDate(text, att, writer)

			} else {

				writer.writeAttribute(AsciiUtils.buildAttributeId(att), text.get(att)) // struct

			}

		}

		writer.writeCharacters("\n") // simple XML formating

		writer.writeStartElement("metadata")

		writer.writeStartElement("list")

		writer.writeAttribute("type", "unordered")

		writer.writeCharacters("\n")

		textMetadataColumnList.each { att ->

			writer.writeStartElement("item")

			writer.writeCharacters(att+" : "+text.get(att).replaceAll("\n", ";"))

			writer.writeEndElement() // item

			writer.writeCharacters("\n")

		}

		writer.writeEndElement() // list

		writer.writeEndElement() // metadata

		writer.writeCharacters("\n")

		for (String att : textContentColumnList) {

			if (att in prefixContentColumnList) {

				writeContentPrefix(text, att, writer)

			} else if (att in listContentColumnList) {

				writeContentList(text, att, writer)

			} else {

				writer.writeStartElement("p");

				writer.writeAttribute("type", att.trim())

				writer.writeCharacters("\n") // simple XML formating

				writer.writeStartElement("head")

				writer.writeStartElement("hi")

				writer.writeCharacters(att+" : ")

				writer.writeEndElement() // hi

				writer.writeEndElement() // head

				writer.writeCharacters(text.get(att)) // get textColumnList content

				writer.writeEndElement() // t

				writer.writeCharacters("\n") // simple XML formating)

			}

		}

		int pb_n = 1;

		for (int i = 1 ; i < toWrite.size() ; i++) {

			def record = toWrite[i]

			writer.writeEmptyElement("pb") // <pb/>

			writer.writeAttribute("n", ""+pb_n++)

			writer.writeStartElement(structureTag)

			for (String att : structureMetadataColumnList) {

				if (att in metadataDateColumnList) {

					writeMetadataDate(record, att, writer)

				} else {

					writer.writeAttribute(AsciiUtils.buildAttributeId(att), record.get(att)) // struct

				}

			}

			writer.writeCharacters("\n")

			writer.writeStartElement("metadata")

			writer.writeStartElement("list")

			writer.writeAttribute("type", "unordered")

			writer.writeCharacters("\n")

			structureMetadataColumnList.each { att ->

				writer.writeStartElement("item")

				writer.writeCharacters(att+" : "+record.get(att).replaceAll("\n", ";"))

				writer.writeEndElement() // item

				writer.writeCharacters("\n")

			}

			writer.writeEndElement() // list

			writer.writeEndElement() // metadata

			writer.writeCharacters("\n")

			for (String att : structureContentColumnList) {

				if (att in prefixContentColumnList) {

					writeContentPrefix(text, att, writer)

				} else if (att in listContentColumnList) {

					writeContentList(text, att, writer)

				} else {

					writer.writeStartElement("p");

					writer.writeAttribute("type", att.trim())

					writer.writeAttribute("id", record.get(textIDColumn))

					writer.writeCharacters("\n") // simple XML formating

					writer.writeStartElement("head")

					writer.writeStartElement("hi")

					writer.writeCharacters(att+" : ")

					writer.writeEndElement() // hi

					writer.writeEndElement() // head

					writer.writeCharacters(record.get(att)) // get textColumnList content

					writer.writeEndElement() // t

					writer.writeCharacters("\n") // simple XML formating

				}

			}

			writer.writeEndElement() // struct

			writer.writeCharacters("\n") // simple XML formating

		}

		writer.writeEndElement() // text

		writer.writeCharacters("\n") // simple XML formating

		writer.writeEndElement() // TEI

		writer.close()

		output.close()

		reader.close()

	} else {

		// error

		println "ERROR: '$id' text group with  no text line"

	}

}

def writeContentList(def record, def att, def writer) {

	writer.writeCharacters("\n")

	found = false

	def value = record.get(att)

	value.findAll( /(?s)([^\n]+?)[\n]/ ).each { desc ->

		found = true

		writer.writeCharacters("\t")

		writer.writeStartElement("p")

		writer.writeAttribute("rend", "list")

		matches = (desc =~ /(?s)([^\n]+?)[\n]/)

		writer.writeCharacters(matches[0][1])

		writer.writeEndElement() // p

		writer.writeCharacters("\n")

	}

	if (!found) {

		writer.writeCharacters("\t")

		writer.writeStartElement("p")

		writer.writeAttribute("rend", "no-list")

		writer.writeCharacters(value)

		writer.writeEndElement() // p

		writer.writeCharacters("\n")

	}

}

def writeContentPrefix(def record, def att, def writer) {

	writer.writeCharacters("\n")

	writer.writeStartElement("list")

	writer.writeAttribute("rend", "prefixes")

	writer.writeAttribute("type", "unordered")

	writer.writeCharacters("\n")

	found = false

	def value = record.get(att)

	value.findAll( /(?s)[A-Z]{3}:? *([^;\n]+?) +[;\n]/ ).each { desc ->

		found = true

		writer.writeCharacters("\t")

		writer.writeStartElement("item")

		matches = (desc =~ /(?s)([A-Z]{3}):? *([^;\n]+?) +[;\n]/)

		writer.writeAttribute("type", matches[0][1])

		writer.writeStartElement("span")

		writer.writeCharacters(matches[0][1]+" ")

		writer.writeEndElement() // span

		writer.writeCharacters(matches[0][2])

		writer.writeEndElement() // item

		writer.writeCharacters("\n")

	}

	if (!found) {

		writer.writeCharacters(value)

	}

	writer.writeEndElement() // list

	writer.writeCharacters("\n")

}

def writeMetadataDate(def record, def att, def writer) {

	String value = record.get(att)

	String att_normalized = AsciiUtils.buildAttributeId(att)

	matches = (value =~ /([0-9]{2})\/([0-9]{2})\/([0-9]{4})/)

	writer.writeAttribute(att_normalized+"jour", matches[0][1])

	writer.writeAttribute(att_normalized+"joursemaine", new java.text.SimpleDateFormat('EEEE').format(Date.parse("dd/MM/yyyy", value)))

	writer.writeAttribute(att_normalized+"mois", matches[0][2])

	writer.writeAttribute(att_normalized+"annee", matches[0][3])

	writer.writeAttribute(att_normalized+"tri", matches[0][3]+"-"+matches[0][2]+"-"+matches[0][1])

}

/*

 String name = inputFile.getName()

 int idx = name.lastIndexOf(".")

 if (idx > 0) name = name.substring(0, idx)

 */

			OpenImportForm.openfile(path);

import org.txm.rcp.views.*