Laboratoire ICAR » Plateforme TXM

package org.txm.macro.frolex

import java.io.BufferedOutputStream

import java.io.File

import java.io.IOException

import java.io.InputStream

import java.io.PrintWriter

import java.net.URL

import java.util.ArrayList

import java.util.Arrays

import java.util.List

import javax.xml.stream.XMLInputFactory

import javax.xml.stream.XMLStreamConstants

import javax.xml.stream.XMLStreamException

import javax.xml.stream.XMLStreamReader

import org.txm.importer.ApplyXsl2

import org.txm.utils.io.IOUtils

//PARAMETERS

File xmlDirectory = new File(System.getProperty("user.home"), "TXM/results/lexique BFM/lgerm-processed")

File tsvFile = new File(System.getProperty("user.home"), "TXM/results/lexique BFM/lgerm-processed/dmf.tsv")

// START

factory = XMLInputFactory.newInstance()

writer = IOUtils.getWriter(tsvFile, "UTF-8")

writer.println("form\tcategory\tlemma\tlemma_src\tF_dmf")

File[] xmlFiles = ApplyXsl2.listFiles(xmlDirectory)

if (xmlFiles == null) return false

Arrays.sort(xmlFiles)

for (File xmlFile : xmlFiles) {

        if (!xmlFile.getName().endsWith(".xml")) continue

        if (!processFile(xmlFile)) {

                println("Error while parsing "+xmlFile)

                return false

        }

}

if (writer != null) writer.close()

return true

//END

class Entry {

        String lemma

        String lemma_source

        String category

        List<String> forms = new ArrayList<String>()

        List<String> freqs = new ArrayList<String>()

        public Entry() {}

        public Entry(String lemma2, String category2) {

                // TODO Auto-generated constructor stub

        }

        String toString() {

                StringBuilder str = new StringBuilder()

                if (forms.size() != freqs.size()) println("ERROR with entry="+lemma+" "+category+" "+forms+ " "+freqs)

                else if (lemma == null) println("ERROR with entry="+lemma+" "+category+" "+forms+ " "+freqs)

                else if (lemma_source == null) println("ERROR with entry="+lemma+" "+category+" "+forms+ " "+freqs)

                else if (category == null) println("ERROR with entry="+lemma+" "+category+" "+forms+ " "+freqs)

                else

                        for (int i = 0 ;  i < forms.size() ; i++) {

                                str.append(forms.get(i)+"\t"+category+"\t"+lemma+"\t"+lemma_source+"\t"+freqs.get(i)+"\n")

                        }

                return str.toString()

        }

}

boolean processFile(File xmlFile) throws IOException, XMLStreamException {

        URL inputurl = xmlFile.toURI().toURL()

        InputStream inputData = inputurl.openStream()

        XMLStreamReader parser = factory.createXMLStreamReader(inputData)

        try {

                Entry current = null

                String target = null

                boolean inLemmatizedForm = false

                String orthography = null

                boolean inOrthography = false

                String grammaticalCategory = null

                boolean inGrammaticalCategory = false

                String frequency = null

                boolean inFrequency = false

                String localname

                for (int event = parser.next() ; event != XMLStreamConstants.END_DOCUMENT ; event = parser.next()) {

                        switch (event) {

                                case XMLStreamConstants.START_ELEMENT:

                                        localname = parser.getLocalName()

                                        if ("lemmatizedForm".equals(localname)) {

                                                if (current != null) {

                                                        writer.print(current)

                                                }

                                                current = new Entry()

                                                inLemmatizedForm = true

                                        } else if ("orthography".equals(localname)) {

                                                inOrthography = true

                                                orthography = ""

                                                target = parser.getAttributeValue(null, "target")

                                        } else if ("grammaticalCategory".equals(localname)) {

                                                inGrammaticalCategory = true

                                                grammaticalCategory = ""

                                        } else if ("frequency".equals(localname)) {

                                                inFrequency = true

                                                frequency = ""

                                        }

                                        break

                                case XMLStreamConstants.CHARACTERS:

                                        if (inFrequency) frequency += parser.getText()

                                        else if (inGrammaticalCategory) grammaticalCategory += parser.getText()

                                        else if (inOrthography) orthography += parser.getText()

                                        break

                                case XMLStreamConstants.END_ELEMENT:

                                        localname = parser.getLocalName()

                                        if ("lemmatizedForm".equals(localname)) {

                                                inLemmatizedForm = false

                                        } else if ("orthography".equals(localname)) {

                                                inOrthography = false

                                                orthography = orthography.trim()

                                                if (inLemmatizedForm) { // lemma orthography

                                                        if (target.equals("DMF")) {

                                                                current.lemma = orthography

                                                                current.lemma_source = target

                                                        } else if (target.equals("LGeRM") && !"DMF".equals(current.lemma_source)) {

                                                                current.lemma = orthography

                                                                current.lemma_source = target

                                                        }

                                                } else { // form orthography

                                                        current.forms.add(orthography)

                                                }

                                        } else if ("grammaticalCategory".equals(localname)) {

                                                inGrammaticalCategory = false

                                                grammaticalCategory = grammaticalCategory.trim()

                                                current.category = grammaticalCategory

                                        } else if ("frequency".equals(localname)) {

                                                inFrequency = false

                                                frequency = frequency.trim()

                                                current.freqs.add(frequency)

                                        }

                                        break

                        }

                }

                if (current != null) { // last entry

                        writer.print(current)

                }

                parser.close()

                inputData.close()

        } catch(Exception e) {

                println("Unexpected error while parsing file "+xmlFile+" : "+e)

                println("Location line: "+parser.getLocation().getLineNumber()+" character: "+parser.getLocation().getColumnNumber())

                org.txm.utils.logger.Log.printStackTrace(e)

                //e.printStackTrace()

                parser.close()

                inputData.close()

                return false

        }

        return true

}

println "Done. DMF TSV file: $tsvFile"