Laboratoire ICAR » Plateforme TXM

package org.txm.dictionary.functions.tsv

import java.util.regex.Pattern;

DEBUG = false

File workingDirectory = new File("/home/mdecorde/TXM/results/lexique BFM")

File dmflextsv = new File(workingDirectory, "dmflex.tsv")

if (!dmflextsv.exists()) {

    println "Can't find $dmflextsv"

    return

}

File convCTX9DMF = new File(workingDirectory, "conv dmf ctx9-nca.tsv")

if (!convCTX9DMF.exists()) {

    println "Can't find $convCTX9DMF"

    return

}

Pattern pattern = Pattern.compile("\t");

String[] noposarray = ["_nopos_"]

def cols = ["form","category","dmf_lemma","dmf_lemma_old","source","source_lemma"]

def lines = dmflextsv.readLines("UTF-8");

println "Read file. "+lines.size()+" lines..."

def data = []

for (int i = 1 ; i < lines.size() ; i++) {

        String line = lines[i]

        List split = pattern.split(line, cols.size())

        if (split.size() >= cols.size()) {

                def d = [split[0], split[1], split[2], split[3], split[4], split[5]]

                data << d

                if (DEBUG) println d

        } else {

                println "Error with line '$line' gets '$split'"

        }

}

println "Done: "+data.size()+" entries"

println "Add new msd column..."

def conversion_rules2 = ConvRules.getConversionRulesFromFile(convCTX9DMF)

cols << "msd_cattex_conv"

def missing_conv = new HashSet()

n = data.size()

for (int i = 0 ; i < n ; i++) {

        def d = data[i]

        String msd_value = d[1];

        if (conversion_rules2.containsKey(msd_value)) {

                data.remove(i)

                i--;

                n--;

                for (def v : conversion_rules2.get(msd_value)) {

                        def dtmp = d.clone();

                        dtmp << v

                        data.add(dtmp)

                }

        } else {

                missing_conv << d[1]

                d << ""

        }

}

println "Done: "+data.size()+" entries"

if (missing_conv.size() > 0) {

        println "Missing conversion rules: "+missing_conv

}

if (DEBUG) for (def d : data) println d

println "Sort by form..."

data.sort() {d1, d2 ->

        return d1[0].compareTo(d2[0])

}

println "Done: "+data.size()+" entries"

if (DEBUG) for (def d : data) println d

//for (int i = 0 ; i < 200 ; i++) println data[(int)(Math.random()*data.size())]

n = data.size()

def hset = new HashSet()

def hsetform = new HashSet()

def hmap = [:]

for (int i = 0 ; i < n ; i++) {

        //if (data[i][0].length() == 0) continue;

        def cat = data[i][1]

        def form = data[i][0]

        hset << cat

        hsetform << form

        if (!hmap.containsKey(cat)) hmap[cat] = new HashSet()

        if (hmap[cat].size() < 10)        hmap[cat] << form

}

def resultFile = new File("formdmf.txt")

resultFile.delete()

for (String form : hsetform) {

        resultFile << form+"\n"

}

println "N form : "+hsetform.size()+" in "+resultFile.getAbsolutePath()

resultFile = new File("catdmf.txt")

resultFile.delete()

for (String cat : hset) {

        resultFile << cat+"\t"+hmap[cat]+"\n"

}

println "N cat : "+hset.size()+" in "+resultFile.getAbsolutePath()

return data