Laboratoire ICAR » Plateforme TXM

// Script to restore punctuation etc. to results of TS query.

// Inputs:

// --- Tiger-XML Document node

// --- Java array:

// --- --- String [index][type] where:

// --- --- --- type == 0 gives the xml:id

// --- --- --- type == 1 gives the word form

// Process:

// --- Injects punctuation.

// Returns:

// --- Tiger-XML Document node.

// TXM package statement

package org.txm.scripts.importer.srcmf

import javax.xml.parsers.DocumentBuilderFactory

/**

 *

 * @author tmr

 *

 */

class PunctInjectImport

{

        static nsXml = new groovy.xml.Namespace('http://www.w3.org/XML/1998/namespace', 'xml')

        static nsTei = new groovy.xml.Namespace('http://www.tei-c.org/ns/1.0', 'tei')

        def tigerXml = null

        def txmIdWordTableFixed = []

        def outputFile = null

        /*

         * Call this constructor when no need of XML-TXM file for word ID table

         */

        PunctInjectImport(File tigerXmlFile, ArrayList txmIdWordTable) {

                def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()

                tigerXml = builder.parse(tigerXmlFile).documentElement

                txmIdWordTableFixed = txmIdWordTable

                outputFile = new File("output.xml")

        }

        /*

         * Call this constructor when the XML-TXM file is needed for the word list.

         */

        PunctInjectImport(File tigerXmlFile, File xmlTxmFile) {

                //

                def builder = DocumentBuilderFactory.newInstance().newDocumentBuilder()

                tigerXml = builder.parse(tigerXmlFile).documentElement

                txmIdWordTableFixed = []

                def xmlTxmGroovyDOM = new XmlParser().parse(xmlTxmFile)

                for (def w : xmlTxmGroovyDOM.text.'**'.findAll { it.name() == nsTei.'w' } ) {

                        def anId = null

                        anId = w.'@id'

                        if (! anId) {

                                anId = w.attribute(nsXml.id)

                        }

                        if (! anId) {

                                println "Error: ID attribute not located in XML TXM file"

                        }

                        txmIdWordTableFixed << [anId, getDeepText(w."txm:form"[0])]

                }

                // assert txmIdWordTableFixed.size() > 0

                outputFile = new File("output.xml")

        }

        def process () {

                def txmIdWordTable = new ArrayList(this.txmIdWordTableFixed)

                def allTs = toList(this.tigerXml.getElementsByTagName('t'))

                def tAttrs = getTAttrNames(allTs[0])

                def tNodeIdPrefix = allTs[0].getAttribute('id').tokenize('#').first()

                def addedWordIds = []

                int tIx = 0

                while (txmIdWordTable) {

                        def word = txmIdWordTable.remove(0)

                        if (tIx == allTs.size()) {

                                // End of TS file, but still words left in the BFM file.

                                addTNode(word, allTs.last(), tAttrs, tNodeIdPrefix, 'append')

                        }

                        else {

                                def tNode = allTs[tIx]

                                def tId = getTNodeId(tNode)

                                if (tId == word[0] && tNode.getAttribute('word') == word[1]) {

                                        // alles gut

                                        tIx += 1

                                }

                                else if (tId == word[0]) {

                                        println("Mismatched Ids! ($tId)")

                                        tIx += 1

                                }

                                else if (['#', '*'].contains(tNode.getAttribute('word')) ) {

                                        // SRCMF duplicata; try comparing word against the next tNode next time

                                        // around.

                                        txmIdWordTable.add(0, word)

                                        tIx += 1

                                }

                                // Check that the SRCMF corpus doesn't have a bug in it...

                                else if ( !(word[1] =~ /[\,\.\?\!\:\;\(\)\[\]\{\}]/)

                                && (allTs[0..tIx - 1].find{

                                        it.getAttribute('id') == "$tNodeIdPrefix#$tId"

                                })) {

                                        println "Warning: word ${tNode.getAttribute('word')}, id $tId appears twice in corpus!"

                                        txmIdWordTable.add(0, word)

                                        tIx += 1

                                }

                                // Check that there's not an extra word in the SRCMF corpus (rare, usu. a tokenisation change)

                                else if ( !(word[1] =~ /[\,\.\?\!\:\;\(\)\[\]\{\}]/)

                                && (allTs[tIx..-1].find{

                                        it.getAttribute('id') == "$tNodeIdPrefix#${word[0]}"

                                })) {

                                        println "Warning: word ${tNode.getAttribute('word')}, id $tId does not appear in BFM!"

                                        txmIdWordTable.add(0, word)

                                        tIx += 1

                                }

                                else if (addedWordIds.contains(tId)) {

                                        println "Warning: word ${tNode.getAttribute('word')}, id ${tId} out of sequence in SRCMF corpus!"

                                        txmIdWordTable.add(0, word)

                                        tIx += 1

                                }

                                else {

                                        // Insert word.  In the first instance, it will have the same parent as

                                        // the tNode before which it's being inserted.

                                        addTNode(word, allTs[tIx], tAttrs, tNodeIdPrefix, 'before')

                                        addedWordIds.add(word[0])

                                }

                        }

                }

                // Second phase: move punctuation into previous sentence,

                // dependent on sequence.

                def allTerminalses = toList(this.tigerXml.getElementsByTagName('terminals'))

                for (def i = 1 ; i < allTerminalses.size() ; i++) {

                        def ts = toList(allTerminalses[i].getElementsByTagName('t'))

                        def startPunc = true

                        def puncStack = []

                        while (ts && startPunc) {

                                if ((ts[0].getAttribute('word') =~ /[A-zÀ-ÿ0-9]/).size() == 0) {

                                        puncStack.add(ts.remove(0))

                                }

                                else {

                                        startPunc = false

                                }

                        }

                        // Now, treat the punctuation stack at the beginning of the sentence

                        if ( puncStack ) {

                                int moveLeft = 0

                                // First, identify LAST instance of sentence-final punctuation.

                                def puncString = puncStack.collect{ it.getAttribute('word')[0] }.join('')

                                def matches = puncString =~ /[\.\,\;\:\!\?\)\]\}»”’]/

                                if (matches.size() > 0) {

                                        moveLeft = puncString.lastIndexOf(matches[-1]) + 1

                                }

                                // Second, split pairs of straight quotes

                                matches = puncString =~ /(""|'')/ //"

                                if (matches.size() > 0) {

                                        moveLeft = [moveLeft, puncString.lastIndexOf(matches[-1][0]) + 1].max()

                                }

                                // Now, move moveLeft punctuation nodes to the end of the prev. sentence

                                ts = toList(allTerminalses[i].getElementsByTagName('t'))

                                for (def j = 0 ; j < moveLeft ; j++ ) {

                                        allTerminalses[i - 1].appendChild(ts[j])

                                }

                        }

                }

                outputFile.withWriter("UTF-8") { writer ->

                        writer.println(this.tigerXml)

                }

        }

        private addTNode(word, tNode, tAttrs, tNodeIdPrefix, where) {

                def newTNode = tNode.getOwnerDocument().createElement('t')

                for (def anAttr : tAttrs) {

                        if (anAttr == 'id') {

                                newTNode.setAttribute('id', "${tNodeIdPrefix}#${word[0]}")

                        }

                        else if (anAttr == 'word') {

                                newTNode.setAttribute('word', word[1])

                        }

                        else {

                                newTNode.setAttribute(anAttr, '--')

                        }

                }

                if (where == 'before') {

                        tNode.getParentNode().insertBefore(newTNode, tNode)

                }

                else if (where == 'append') {

                        tNode.getParentNode().appendChild(newTNode)

                }

                else {

                        throw new IllegalArgumentException('Bad before value')

                }

        }

        def getTAttrNames(tNode) {

                def nodeMap = tNode.attributes

                def nameList = []

                for ( def i = 0 ; i < nodeMap.getLength() ; i++ ) {

                        nameList.add( nodeMap.item(i).nodeName )

                }

                return nameList

        }

        def getTNodeId(tNode) {

                return tNode.getAttribute('id').tokenize('#').last()

        }

        def toList(def iterable) {

                return iterable.findAll {true};

        }

        def getDeepText(def node) {

                def str = ""

                node.children().each { child ->

                        if (child instanceof Node) {

                                str += getDeepText(child)

                        } else if (child instanceof String) {

                                str += child

                        }

                }

                return str

        }

        public static void main(String[] args)

        {

                def cli = new CliBuilder(

                                usage:'PunctInjectImport.groovy TigerXml.xml XmlTxm.xml'

                                )

                cli.h(longOpt:'help', 'Prints this message.')

                cli.o(longOpt:'output', args:1, argName:'outputfile.xml', 'Output to given file.')

                def options = cli.parse(args)

                def tigerXmlFile = null

                def xmlTxmFile = null

                if (options.arguments().size() == 2) {

                        tigerXmlFile = new File(options.arguments()[0])

                        xmlTxmFile = new File(options.arguments()[1])

                } else {

                        println 'Incorrect number of command line arguments... exiting'

                        println cli.usage()

                        System.exit(2)

                }

                def pii = new PunctInjectImport(tigerXmlFile, xmlTxmFile)

                if (options.o) {

                        pii.outputFile = new File(options.o)

                }

                pii.process()

        }

}