Laboratoire ICAR » Plateforme TXM

 *

 */

class CoNLLUImporter extends XTZImporter {

	public CoNLLUImporter(ImportModule module) {

		super(module)

	}

	public final String merge(String orig, def sss) {

		int n = 0;

		for (String s : sss) {

			if (s != "" && s != "_") {

				def ssset = new HashSet(sss);

				if (ssset.size() == 1) return ssset.join(".")

				return sss.join(".")

			}

		}

		return orig;

	}

	@Override

	public void process() {

		File conlluSrcDirectory = inputDirectory

		boolean usenewdocid =  "true".equals(UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.IMPORT_USE_NEW_DOC_ID)); // THE conllu -> Tiger XSL MUST HAVE THE SAME BEHAVIOR BEFORE //

		if (usenewdocid) {

			conlluSrcDirectory = new File(outputDirectory.getParentFile().getParentFile(), "conllu")

			conlluSrcDirectory.deleteDir()

			conlluSrcDirectory.mkdirs()

			if (!splitCoNLLUFiles(inputDirectory, conlluSrcDirectory, project)) {

				return

			}

		}

		def files = conlluSrcDirectory.listFiles()

		files.sort()

		// Keep or not contractions

		String contractionsManagement =  UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.CONTRACTIONS_MANAGEMENT, UDPreferences.getInstance().getString(UDPreferences.CONTRACTIONS_MANAGEMENT));

							}

							lines.remove(i)

							i--

						}

								def splits = []

								for (int j = 0 ; j <= n ;j++) {

									def tmp = lines[i+j+1].split("\t", ImportCoNLLUAnnotations.UD_PROPERTY_NAMES.length);

									splits << tmp

								}

								for (int j = 2 ; j < 8 ; j++) {

									split[j] = merge(split[j], splits.collect(){it[j]})

								}

								//println "REMOVE non- $split"

								for (int j = 0 ; j <= n ;j++) {

									lines.remove(i+1)

								}

							}

						}

					}

				}

			}

		File metadataFile = Metadatas.findMetadataFile(module.sourceDirectory)

		File srcDirectory = new File(outputDirectory.getParentFile().getParentFile(), "conllu2tei")

		srcDirectory.deleteDir()

		srcDirectory.mkdirs()

		if (metadataFile != null && metadataFile.exists()) {

			File metadataFile2 = new File(srcDirectory, metadataFile.getName())

			FileCopy.copy(metadataFile, metadataFile2)

		}

		println "Convert CoNLL-U to XML-TEI..."

		convertCoNLLU2TEI(conlluSrcDirectory, srcDirectory, project)

		inputDirectory = srcDirectory // switch files source directory

		super.process()

	}

	public static def splitCoNLLUFiles(File inputDirectory, File srcDirectory, def project) {

		def files = inputDirectory.listFiles(new FilenameFilter() {

		if (files == null) {

			println "Aborting. No CONLL file found in $inputDirectory."

			return false

		}

		files.sort()

		ConsoleProgressBar cpb_texts = new ConsoleProgressBar(files.size())

		println "Splitting CoNLL-U files..."

		for (File master : files) {

			cpb_texts.tick()

			if (!master.getName().endsWith(".conllu")) {

				continue

			}

			String orig_text_id = FileUtils.stripExtension(master)

			String current_text_id = FileUtils.stripExtension(master)

			File conlluFile = new File(srcDirectory, current_text_id+".conllu")

			def writer = conlluFile.newWriter("UTF-8", true)

			master.eachLine("UTF-8") { line ->

				if (line.startsWith("# newdoc id = ")) {

					String text_id = line.substring("# newdoc id = ".length())

					if (!text_id.equals(current_text_id)) {

						writer.close()

						writer = conlluFile.newWriter("UTF-8", true)

					}

				}

				writer.println(line)

			}

			writer.close()

		cpb_texts.done()

		return true

	}

	public static def convertCoNLLU2TEI(File inputDirectory, File srcDirectory, def project) {

		def files = inputDirectory.listFiles()

		if (files == null) {

			println "Aborting. No CONLL file found in $inputDirectory."

			return false

		}

		files.sort()

		def properties = Arrays.asList(ImportCoNLLUAnnotations.UD_PROPERTY_NAMES)

		String prefix = UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.UDPREFIX, UDPreferences.getInstance().getString(UDPreferences.UDPREFIX));

		UDPreferences.getInstance().setProjectPreferenceValue(project, UDPreferences.UDPREFIX, prefix); // copy the current preference into the corpus preference

		def headPropertiesToProject = UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.IMPORT_HEAD_TO_PROJECT, UDPreferences.getInstance().getString(UDPreferences.IMPORT_HEAD_TO_PROJECT)).split(",") as Set

		def depsPropertiesToProject = UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.IMPORT_DEPS_TO_PROJECT, UDPreferences.getInstance().getString(UDPreferences.IMPORT_DEPS_TO_PROJECT)).split(",") as Set

		def formatSentences = "true" == UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.IMPORT_PRINT_NEWLINES_AFTER_SENTENCES, ""+UDPreferences.getInstance().getString(UDPreferences.IMPORT_PRINT_NEWLINES_AFTER_SENTENCES))

		String contractionsManagement =  UDPreferences.getInstance().getProjectPreferenceValue(project, UDPreferences.CONTRACTIONS_MANAGEMENT, UDPreferences.getInstance().getString(UDPreferences.CONTRACTIONS_MANAGEMENT));

		ConsoleProgressBar cpb_texts = new ConsoleProgressBar(files.size())

		println "Parsing CoNLL-U files..."

		for (File master : files) {

			cpb_texts.tick()

			if (!master.getName().endsWith(".conllu")) {

				continue

			}

			def content = [] // list of sentence

			String text_id = FileUtils.stripExtension(master)

			String sent_id = ""

			String par_id = "1"

			def comments = [] // /text/par/sent

			def words = []

			master.eachLine("UTF-8") { line ->

				if (line.startsWith("# newdoc id = ")) {

					// already set or ignored

				} else if (line.startsWith("# sent_id = ")) {

					if (words.size() > 0) {

						def sentence = [par_id, sent_id, words, comments]

						content.add(sentence)

						sent_id = ""

						par_id = "1"

						comments = []

						words = []

					}

				} else {

					LinkedHashMap<String, String> wProperties = new LinkedHashMap<String, String>()

					def split = line.split("\t", ImportCoNLLUAnnotations.UD_PROPERTY_NAMES.length)

					if (split.size() == properties.size()) {

						String id = split[0]

						for (int i = 0 ; i < split.size() ; i++) {

							wProperties[properties[i]] = split[i]

						}

						if (wProperties[properties[0]].contains(".")) { // id

							// empty node

						} else {

					}

				}

			}

			if (words.size() > 0) { // last sentence ?

				def sentence = [par_id, sent_id, words, comments]

				content.add(sentence)

			}

			if (content.size() == 0) {

				continue;

			}

			//println "${content.size()} sentences found."

			File xmlFile = new File(srcDirectory, text_id+".xml")

			// println "xmlFile=$xmlFile"

			BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(xmlFile))

			XMLOutputFactory factory = XMLOutputFactory.newInstance()

			XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8")

			writer.writeStartDocument("UTF-8","1.0")

			writer.writeStartElement ("TEI")

			writer.writeDefaultNamespace("http://www.tei-c.org/ns/1.0")

			writer.writeEndElement()

			writer.writeCharacters("\n")

			writer.writeStartElement ("text")

			writer.writeCharacters("\n")

			String current_par_id = null

			int wordCounter = 0

			for (def sentence : content) { // for all paragraph of the current text

				par_id = sentence[0]

				sent_id = sentence[1]

				words = sentence[2]

				comments = sentence[3]

				if (current_par_id == null || par_id != current_par_id) {

					if (current_par_id != null) {

						writer.writeEndElement() // p

					writer.writeStartElement ("p")

					writer.writeAttribute("id", par_id)

					writer.writeCharacters("\n")

					current_par_id = par_id

				}

				writer.writeStartElement ("s")

				writer.writeAttribute("id", sent_id)

				writer.writeCharacters("\n")

				for (def comment : comments) {

					writer.writeComment(comment.replace("--", "−−"))

					writer.writeCharacters("\n")

				}

				if (headPropertiesToProject.size() > 0 || depsPropertiesToProject.size() > 0) {

					LinkedHashMap sentencehash = new LinkedHashMap()

					//println "WORDS="+words

					//println "SENTENCE="+sentencehash

					ImportCoNLLUAnnotations.buildPropertiesProjections(sentencehash, headPropertiesToProject, depsPropertiesToProject)

				}

				if (formatSentences) {

				}

				for (def word : words) {

					if (word["id"].contains("-")) {

						writer.writeStartElement("seg")

						writer.writeCharacters("******")

						writer.writeEndElement() // span

					}

					String id = null

					wordCounter++

					writer.writeStartElement ("w")

						//println "WORD="+word

						writer.writeAttribute(prefix+p, word[p])

					}

					if (id != null) {

					} else {

						writer.writeAttribute("id", "w_"+text_id+"_"+wordCounter)

					}

					writer.writeCharacters(word["form"])

					writer.writeEndElement() // w

					writer.writeCharacters(" ")

				}

				if (formatSentences) writer.writeEndElement()

				writer.writeCharacters("\n")

				writer.writeEndElement() // s

			}

			if (current_par_id != null) {

				writer.writeEndElement() // p

				writer.writeCharacters("\n")

			}

			writer.writeEndElement() // text

			writer.writeCharacters("\n")

			writer.writeEndElement() // TEI

			writer.close()

		}

		cpb_texts.done()

		return true

	}

}

import java.io.File;

import java.io.IOException;

import java.util.List;

import org.apache.commons.lang.StringUtils;

	public static File print(File file, List<String> conll, String[] Tvalues, String[] NTvalues) {

		try {

			String bundle_id = "org.txm.conllu.core";

			File HTMLTEMPATE = BundleUtils.getFile(bundle_id, "template", "/", "index.html");

			File root = HTMLTEMPATE.getParentFile();

			String content = IOUtils.getText(HTMLTEMPATE);

			content = content.replace("HTMLROOTDIRECTORY", root.getAbsolutePath());

			IOUtils.write(file, content);

			//BundleUtils.copyFiles(bundle_id, "groovy", "org/txm/scripts/importer", "", scriptsPackageDirectory, true);