Laboratoire ICAR » Plateforme TXM

package org.txm.dictionary.functions.tsv

File workingDirectory = new File("/home/mdecorde/TXM/results/lexique BFM")

File frolextsv = new File(workingDirectory, "frolex.tsv")

/**

form        msd_afrlex        lemma        lemma_src

!        PON        !        

"        PON:cit        QUOTED        

 */

println "** BUILD BFM DICT"

bfmlex = new BuildBFMLex().run()

println "** BUILD AFR DICT"

afrlex = new BuildAFRLex().run()

def DEBUG = false

if (DEBUG) println "BFMLEX: "+bfmlex.size()

if (DEBUG) for (def d : bfmlex) println d

if (DEBUG) println "AFRLEX: "+afrlex.size()

if (DEBUG) for (def d : afrlex) println d

bfmcols = ["form","msd_bfm","F_bfm", "msd_cattex_conv"]

afrcols = ["form","msd_afrlex","lemma","lemma-source", "msd_cattex_conv"]

frocols = ["form", "F_bfm", "msd_afrlex", "msd_bfm", "msd_cattex_conv", "lemma", "lemma_src", "comment"]

int iBFM = 0;

int iAFR = 0;

int mergeCounter = 0;

all = new HashSet()

println "** MERGING BFM DICT WITH AFR DICT"

while (bfmlex.size() > 0 && afrlex.size() > 0) {

        def dafr = afrlex[0]

        def dbfm = bfmlex[0]

        if (DEBUG) println "data="+dafr+","+dbfm

        if (dafr[0] == dbfm[0]) { // same form

        //        println "FORM="+dafr[0]","+dbfm[0]

                if (dafr[4] == dbfm[3]) { // same msd ctx9-nca --> MERGE !!!

                        def current = ["","","","","","","",""]

                        current[0] = dafr[0] // same form

                        current[1] = dbfm[2] // F BFM 

                        current[2] = dafr[1] // same msd MSD AFRLEX (nca)

                        current[3] = dbfm[1] // MSD BFM CATTEX

                        current[4] = dafr[4] // MSD AFRLEX CATTEX CONV

                        current[5] = dafr[2] // AFR LEMMA

                        current[6] = dafr[3] // AFR LEMMA SRC

                        afrlex.remove(0)

                        bfmlex.remove(0)

                        mergeCounter++

                        if (DEBUG) println "MERGE: "+current

                        all.add(current)

                } else {

                        if (dafr[4].compareTo(dbfm[3]) < 0) { // insert afr data

                                insertAFRData();

                        } else { // insert bfm data

                                insertBFMData();

                        }

                }

        } else { // different form

                if (dafr[0].compareTo(dbfm[0]) < 0) { // insert afr data

                        insertAFRData();

                } else { // insert bfm data

                        insertBFMData();

                }

        }

}

if (afrlex.size() > 0) println "add remaining "+afrlex.size()+" AFRlex entries..."

while (afrlex.size() > 0) {

        insertAFRData();

}

if (bfmlex.size() > 0) println "add remaining "+bfmlex.size()+" BFM lex entries..."

while (bfmlex.size() > 0) {

        insertBFMData();

}

if (DEBUG) println frocols

if (DEBUG) for (def d : all) println d

println "Done, "+mergeCounter+" entries merged"

println "sorting..."

def data = []

data.addAll(all)

data.sort() {d1, d2 ->

        return d1[0].compareTo(d2[0])

}

println "Remove _nopos_ not alone..."

def NOPOS = "_nopos_"

String currentForm = null

def tmp = []

def tmpi = [];

def toRemove = []

for (int i = 0 ; i < data.size() ; i++) {

        def d = data[i]

        if (d[0] != currentForm) {

                if (currentForm != null && tmp.size() > 1) {

                        for (int j = 0 ; j < tmp.size() ; j++) {

                                def dtmp = tmp[j]

                                if (dtmp[3] == NOPOS) {

                                        toRemove << tmpi[j]

                                }

                        }

                }

                currentForm = d[0]

                tmp.clear()

                tmpi.clear()

        }

        tmp << d

        tmpi << i

}

if (currentForm != null && tmp.size() > 1) {

                        for (int j = 0 ; j < tmp.size() ; j++) {

                                def dtmp = tmp[j]

                                if (dtmp[3] == NOPOS) {

                                        toRemove << tmpi[j]

                                }

                        }

                }

if (DEBUG) for (def i : toRemove) println data[i]

for (int i = toRemove.size() -1 ; i >= 0 ; i--) {// revert loop !

        data.remove(toRemove[i])

}

println ""+toRemove.size()+" _nopos_ removed."

println "Done: "+data.size()+" entries"

if (DEBUG) for (def d : data) println d

println "Writing result in $frolextsv"

frolextsv.withWriter("UTF-8") { writer ->

        writer.println frocols.join("\t")

        for (def d : data) writer.println d.join("\t")

}

println "Done: "+data.size()+ "entries."

def insertAFRData() {

        def current = ["","","","","","","",""]

        def dafr = afrlex[0]

        afrlex.remove(0)

        current[0] = dafr[0] // FORM

        current[2] = dafr[1] // MSD NCA

        current[4] = dafr[4] // MSD CTX

        current[5] = dafr[2] // LEMMA

        current[6] = dafr[3] // LEMMA SRC

        //println "AFR INSERT: "+current+" with "+dafr

        all.add(current)

}

def insertBFMData() {

        def current = ["","","","","","","",""]

        def dbfm = bfmlex[0]

        bfmlex.remove(0)

        current[0] = dbfm[0] // FORM

        current[1] = dbfm[2] // FREQ

        current[3] = dbfm[1] // CTX

        current[4] = dbfm[3] // NCA CONV

        all.add(current)

}