Laboratoire ICAR » Plateforme TXM

package org.txm.macro.urs

import org.txm.searchengine.cqp.corpus.Property

import org.txm.searchengine.cqp.corpus.Subcorpus

import org.txm.searchengine.cqp.corpus.query.Match

import org.txm.searchengine.cqp.corpus.query.CQLQuery

import visuAnalec.donnees.*

import visuAnalec.elements.*

import org.apache.commons.lang.StringUtils

static def isPropertyDefined(Class clazz, Corpus analecCorpus, String ursql) {

        if (ursql == null || ursql.length() == 0) return new HashSet()

        def params = getFilterParameters(ursql)

        def typeRegexp = params[0]

        def propRegexp = params[1]

        return isPropertyDefined(clazz, analecCorpus, typeRegexp, propRegexp)

}

static def isPropertyDefined(Class clazz, Corpus analecCorpus, String typeRegexp, String propRegexp) {

        def errors = new HashSet()

        if (propRegexp == null || propRegexp.length() == 0) return errors;

        Structure structure = analecCorpus.getStructure();

        for (def type : structure.getTypes(clazz)) {

                if (!type.matches(typeRegexp)) continue; // test only types matching with typeRegexp

                def props = structure.getNomsProps(clazz, type);

                boolean contains = false;

                for (def p : props) {

                        if (p.matches(propRegexp)) {

                                contains = true

                        }

                }

                if (!contains) errors << type

        }

        return errors

}

static def defineProperty(Class clazz, Corpus analecCorpus, String ursql, String newProperty) {

        def params = getFilterParameters(ursql)

        def typeRegexp = params[0]

        Structure structure = analecCorpus.getStructure();

        for (def type : structure.getTypes(clazz)) {

                if (!type.matches(typeRegexp)) continue; // test only types matching with typeRegexp

                def props = structure.getNomsProps(clazz, type)

                if (!props.contains(newProperty)) {

                        structure.ajouterProp(clazz, type, newProperty)

                }

        }

}

static def selectSchemas(def debug, Corpus analecCorpus, String schema_ursql, Integer minimum_schema_size, Integer maximum_schema_size) {

        if (maximum_schema_size <= 0) maximum_schema_size = Integer.MAX_VALUE;

        if (minimum_schema_size < 0) minimum_schema_size = 0;

        def allSchemas = []

        if (schema_ursql != null && schema_ursql.length() > 0) allSchemas = AnalecUtils.findAllInCorpus(debug, analecCorpus, Schema.class, schema_ursql)

        else allSchemas = analecCorpus.getTousSchemas()

        if (debug >= 2) println "allSchemas=${allSchemas.size()}"

        allSchemas = AnalecUtils.filterBySize(allSchemas, minimum_schema_size, maximum_schema_size);

        return allSchemas

}

static def selectSchemasInCorpus(def debug, Corpus analecCorpus, org.txm.searchengine.cqp.corpus.CQPCorpus corpus,

                String schema_ursql, Integer minimum_schema_size, Integer maximum_schema_size, boolean strictInclusion) {

        if (maximum_schema_size <= 0) maximum_schema_size = Integer.MAX_VALUE;

        if (minimum_schema_size < 0) minimum_schema_size = 0;

        def allSchemas = []

        if (schema_ursql != null && schema_ursql.length() > 0) allSchemas = AnalecUtils.findAllInCorpus(debug, analecCorpus, Schema.class, schema_ursql)

        else allSchemas = analecCorpus.getTousSchemas()

        def selectedSchemas = []

        for (Schema schema : allSchemas) {

                def selectedUnits = AnalecUtils.filterUniteByInclusion(debug, schema.getUnitesSousjacentes(), corpus.getMatches(), strictInclusion, 0)

                if (minimum_schema_size <= selectedUnits.size() && selectedUnits.size() <= maximum_schema_size ) {

                        selectedSchemas << schema

                }

        }

        return selectedSchemas

}

/**

 * select units from a selection of schema. If no schema critera are given, select all units then apply units critera

 *

 * @param debug

 * @param analecCorpus

 * @param corpus

 * @param schema_ursql

 * @param minimum_schema_size

 * @param maximum_schema_size

 * @param unit_ursql

 * @param cql_limit

 * @param strict_inclusion

 * @param position

 * @return

 */

static def selectUnitsInSchema(def debug, Corpus analecCorpus, org.txm.searchengine.cqp.corpus.CQPCorpus corpus,

                String schema_ursql, Integer minimum_schema_size, Integer maximum_schema_size,

                String unit_ursql, Integer position_in_schema, CQLQuery cql_limit, Boolean strict_inclusion, int position_in_matches) {

        def groupedUnits = []

        if (schema_ursql != null && schema_ursql.length() > 0) {

                def allSchema = null;

                if (schema_ursql != null && schema_ursql.length() > 0) allSchema = AnalecUtils.findAllInCorpus(debug, analecCorpus, Schema.class, schema_ursql)

                else allSchema = analecCorpus.getTousSchemas()

                if (debug >= 2) println "allSchema=${allSchema.size()}"

                groupedUnits = AnalecUtils.groupAllUnitesInElements(debug, allSchema, unit_ursql)

                if (debug >= 2) println "groupedUnits=${groupedUnits.size()}"

                groupedUnits = AnalecUtils.filterUniteByInclusionInSchema(debug, groupedUnits, position_in_schema)

                if (debug >= 2) println "groupedUnits=${groupedUnits.size()}"

        } else {

                groupedUnits = ["all":AnalecUtils.findAllInCorpus(debug, analecCorpus, Unite.class, unit_ursql)]

        }

        if (debug >= 2) println "groupedUnits=${groupedUnits.size()}"

        // limit units to corpus or cql_limit matches

        def matches = null

        if (cql_limit != null && !cql_limit.getQueryString().equals("\"\"")) {

                Subcorpus limitssubcorpus = corpus.createSubcorpus(cql_limit, corpus.getID().toUpperCase())

                matches = limitssubcorpus.getMatches();

                limitssubcorpus.delete();

        } else {

                matches = corpus.getMatches()

        }

        if (debug >= 2) println "matches=${matches}"

        def allUnits = []

        for (def k : groupedUnits.keySet()) {

                def selectedUnits = AnalecUtils.filterUniteByInclusion(debug, groupedUnits[k], matches, strict_inclusion, position_in_matches)

                if (minimum_schema_size <= selectedUnits.size() && selectedUnits.size() <= maximum_schema_size ) {

                        allUnits.addAll(selectedUnits)

                } else {

                }

        }

        if (debug >= 2) println "selectedUnits=${allUnits.size()}"

        Collections.sort(allUnits)

        return allUnits

}

/**

 * filter groups elements with the elements positions

 *

 *

 * @param groups [schema:units list]

 * @param distance 0=no selection, 1=first, 2=second, -1 last, -2 last-last

 * @return

 */

static def filterUniteByInclusionInSchema(def debug, def groups, Integer distance) {

        println "dist=$distance"

        if (distance == 0) return groups;

        if (distance > 0) distance = distance-1;

        def newGroups = [:]

        for (def k : groups.keySet()) {

                def group = groups[k]

                if (group.size() == 0) {

                        newGroups[k] = group;

                        continue;

                }

                def indexes = null

                if (distance >= 0) {

                        indexes = 0..Math.min(distance, group.size())

                } else {

                        indexes = Math.max(distance, -group.size())..-1

                }

                newGroups[k] = group[indexes];

        }

        return newGroups

}

static def getStartsEndsTargetsArrays(def selectedUnits) {

        int[] starts = new int[selectedUnits.size()]

        int[] ends = new int[selectedUnits.size()]

        int n = 0;

        for (def unite : selectedUnits) {

                starts[n] = unite.getDeb();

                ends[n] = unite.getFin();

                n++

        }

        return [starts, ends, null]

}

static int[] toIntArray(Unite u) {

        if (u.getDeb() > u.getFin()) // error

                return (u.getFin()..u.getDeb()).toArray(new int[u.getDeb()-u.getFin()])

        else

                return (u.getDeb()..u.getFin()).toArray(new int[u.getFin()-u.getDeb()])

}

static String toString(Element e) {

        Schema r = null;

        if (e.getClass() == Unite.class)

                return sprintf("%d-%d, %s", e.getDeb(), e.getFin(), e.getProps().sort())

        else if (e.getClass() == Relation.class)

                return sprintf("%s=%s -> %s", toString(e.getElt1()), toString(e.getElt2()), e.getProps().sort())

        else if (e.getClass() == Schema.class)

                return sprintf("%s=%d", e.getContenu().size(), e.getProps().sort())

}

static String toString(def CQI, def wordProperty, Element e) {

        Schema r = null;

        if (e.getClass() == Unite.class) {

                def form = StringUtils.join(CQI.cpos2Str(wordProperty.getQualifiedName(), toIntArray(e)), " ")

                return sprintf("%s %d-%d, %s", form, e.getDeb(), e.getFin(), e.getProps().sort())

        } else if (e.getClass() == Relation.class) {

                def form1 = StringUtils.join(CQI.cpos2Str(wordProperty.getQualifiedName(), toIntArray(e.getElt1())), " ")

                def form2 = StringUtils.join(CQI.cpos2Str(wordProperty.getQualifiedName(), toIntArray(e.getElt2())), " ")

                return sprintf("%s=%s -> %s", form1+" "+toString(e.getElt1()), form2+" "+toString(e.getElt2()), e.getProps().sort())

        } else if (e.getClass() == Schema.class) {

                return sprintf("%s=%d", e.getContenu().size(), e.getProps().sort())

        }

}

static def findAllInCorpus(def debug, def analecCorpus, Class elemClazz, String URSQL) {

        def params = getFilterParameters(URSQL)

        if (debug >= 2) println "PARAMS=$params"

        return findAllInCorpus(debug, analecCorpus, elemClazz, params[0], params[1], params[2], params[3])

}

static def findAllInCorpus(def debug, Corpus analecCorpus, Class elemClazz, String typeRegex, String propName, boolean eq, String valueRegex) {

        def allElements = null;

        if (elemClazz != null) {

                if (elemClazz == Unite.class)

                        allElements = analecCorpus.getToutesUnites()

                else if (elemClazz == Relation.class)

                        allElements = analecCorpus.getToutesRelations()

                else if (elemClazz == Schema.class)

                        allElements = analecCorpus.getTousSchemas()

        } else {

                allElements = [];

                allElements.addAll(analecCorpus.getToutesUnites())

                allElements.addAll(analecCorpus.getToutesRelations())

                allElements.addAll(analecCorpus.getTousSchemas())

        }

        return filterElements(debug, allElements, typeRegex, propName, eq, valueRegex);

}

static def filterBySize(def elements, Integer minimum_schema_size, Integer maximum_schema_size) {

        if (maximum_schema_size == null || maximum_schema_size <= 0) maximum_schema_size = Integer.MAX_VALUE;

        if (minimum_schema_size == null || minimum_schema_size < 0) minimum_schema_size = 0;

        def filteredElements = []

        for (Element e : elements) {

                Unite[] selectedUnits = e.getUnitesSousjacentes();

                int size = selectedUnits.length;

                if (minimum_schema_size <= selectedUnits.size() && selectedUnits.size() <= maximum_schema_size ) {

                        filteredElements << e

                }

        }

        return filteredElements

}

/**

 * group units by CQP match

 *

 * units are sorted for faster processing

 *

 * @param allUnites

 * @param matches

 * @param strict_inclusion

 * @return

 */

static def groupByMatch(def debug, def allUnites, def matches, boolean strict_inclusion) {

        if (debug >= 2) println "group "+allUnites.size()+" units with "+matches.size()+" strict=$strict_inclusion"

        //println allUnites.collect() {it -> it.getDeb()}

        allUnites = allUnites.sort() { a, b -> a.getDeb() <=> b.getDeb() ?: a.getFin() <=> b.getFin() }

        //println allUnites.collect() {it -> it.getDeb()}

        def unitsSize = allUnites.size()

        def iCurrentUnit = 0

        def selectedUnits = []

        def matchesSize = matches.size()

        def iCurrentMatch = 0

        def selectedUnitsPerMatch = new LinkedHashMap()

        selectedUnitsPerMatch[iCurrentMatch] = selectedUnits

        while (iCurrentMatch < matchesSize && iCurrentUnit < unitsSize) {

                if (debug >= 3) println "** M $iCurrentMatch < $matchesSize && U $iCurrentUnit < $unitsSize"

                Unite unit = allUnites[iCurrentUnit]

                Match match = matches[iCurrentMatch]

                if (debug >= 3) println ""+unit.getDeb()+"->"+unit.getFin()+"        "+match.getStart()+"->"+match.getEnd()

                if (unit.getFin() < match.getStart()) {

                        if (debug >= 3) "println next unit"

                                iCurrentUnit++

                } else if (unit.getDeb() > match.getEnd()) {

                        if (debug >= 3) "println next match"

                                iCurrentMatch++

                        selectedUnits = []

                        selectedUnitsPerMatch[iCurrentMatch] = selectedUnits

                } else {

                        if (debug >= 3) println "iCurrentUnit=$iCurrentUnit        iCurrentMatch=$iCurrentMatch"

                        if (strict_inclusion) {

                                if (debug >= 3) println "m.start ${match.getStart()} <= u.deb ${unit.getDeb()} && u.fin ${unit.getFin()} <= m.end ${match.getEnd()}"

                                if (match.getStart() <= unit.getDeb() && unit.getFin() <= match.getEnd()) {

                                        selectedUnits << unit

                                }

                        } else {

                                selectedUnits << unit

                        }

                        iCurrentUnit++

                }

        }

        return selectedUnitsPerMatch

}

static def filterUniteByInclusion(def debug, def allUnites, def matches, boolean strict_inclusion, int position) {

        def selectedUnitsPerMatch = groupByMatch(debug, allUnites, matches, strict_inclusion);

        //println "selectedUnitsPerMatch size="+selectedUnitsPerMatch.size()

        def selectedUnits = []

        if (position != 0) {

                if (position > 0) position--

                for (def m : selectedUnitsPerMatch.keySet()) {

                        if (selectedUnitsPerMatch[m].size() > position && selectedUnitsPerMatch[m].size() > 0) {

                                def units = selectedUnitsPerMatch[m]

                                //println "$m -> "+units.collect() {it -> it.getDeb()}

                                units = units.sort() { a, b -> a.getDeb() <=> b.getDeb() ?: a.getFin() <=> b.getFin() }

                                //println "$m -> "+units.collect() {it -> it.getDeb()}

                                selectedUnits << units[position]

                                if (debug >=3) println "dist select: "+units[position].getDeb()

                        }

                }

        } else {

                for (def m : selectedUnitsPerMatch.keySet()) selectedUnits.addAll(selectedUnitsPerMatch[m])

        }

        return selectedUnits

}

static def findAllUnitesInElements(def debug, def elements, String URSQL) {

        def params = getFilterParameters(URSQL)

        return findAllUnitesInElements(debug, elements, params[0], params[1], params[2], params[3])

}

static def findAllUnitesInElements(def debug, def elements, String typeRegex, String propName, boolean eq, String valueRegex) {

        def allElements = []

        for (Element element : elements) {

                allElements.addAll(filterElements(debug, element.getUnitesSousjacentes(), typeRegex, propName, eq, valueRegex));

        }

        return allElements;

}

/**

 * group all units without selection

 *

 * @param elements

 * @return

 */

static def groupAllUnitesInElements(def debug, def elements) {

        return groupAllUnitesInElements(debug, elements, "","","")

}

/**

 * group all units with URSQL selection

 *

 * @param elements

 * @param URSQL

 * @return

 */

static def groupAllUnitesInElements(def debug, def elements, String URSQL) {

        def params = getFilterParameters(URSQL)

        return groupAllUnitesInElements(debug, elements, params[0], params[1], params[2], params[3])

}

static def groupAllUnitesInElements(def debug, def elements, String typeRegex, String propName, boolean eq, String valueRegex) {

        def allElements = [:]

        for (Element element : elements) {

                allElements[element] = filterElements(debug, element.getUnitesSousjacentes(), typeRegex, propName, eq, valueRegex);

        }

        return allElements;

}

static def getFilterParameters(String URSQL) {

        String type = "";

        String prop = "";

        String value = "";

        int atidx = URSQL.indexOf("@");

        int equal_start_idx = URSQL.indexOf("=");

        int equal_end_idx = equal_start_idx

        int differentidx = URSQL.indexOf("!=");

        boolean eq = differentidx < 0 || differentidx != equal_start_idx-1

        if (!eq) {

                equal_start_idx--

        }

        if (atidx >= 0 && equal_start_idx >= 0 && atidx < equal_start_idx) { // TYPE@PROP=VALUE

                type = URSQL.substring(0, atidx)

                prop = URSQL.substring(atidx+1, equal_start_idx)

                value = URSQL.substring(equal_end_idx+1)

        } else if (atidx >= 0) { // TYPE@PROP

                type = URSQL.substring(0, atidx)

                prop = URSQL.substring(atidx+1)

        } else if (equal_start_idx >= 0) { // TYPE=VALUE -> not well formed

                type = URSQL.substring(0, equal_start_idx)

                value = URSQL.substring(equal_end_idx+1)

        } else { // TYPE

                type = URSQL;

        }

        //        println(["'"+type+"'", "'"+prop+"'", "'"+value+"'"])

        return [type, prop, eq, value]

}

static def filterElements(def debug, def allElements, String URSQL) {

        def params = getFilterParameters(URSQL)

        return filterElements(debug, allElements, params[0], params[1], params[2], params[3])

}

static def filterElements(def debug, def allElements, String typeRegex, String propName, boolean eq, String valueRegex) {

        if (debug >= 3) println "filtering "+allElements.size()+" elements with typeRegex='$typeRegex' propName='$propName' and valueRegex='$valueRegex'"

        if (typeRegex != null && typeRegex.length() > 0) {

                def filteredElements = []

                def matcher = /$typeRegex/

                for (Element element : allElements) {

                        if (element.getType() ==~ matcher) {

                                filteredElements << element

                        }

                }

                allElements = filteredElements;

        }

        if (debug >= 3) println " type step result: "+allElements.size()

        if (propName != null && propName.length() > 0) {

                def filteredElements = []

                if (valueRegex != null && valueRegex.length() > 0) {  // select only elements with the prop&value

                        def matcher = /$valueRegex/

                        for (Element element : allElements) {

                                def value = element.getProp(propName)

                                if (value ==~ matcher) {

                                        if (eq)        filteredElements << element

                                } else {

                                        if (!eq) filteredElements << element

                                }

                        }

                } else { // select only elements with the prop

                        for (Element element : allElements) {

                                if (element.getProps().containsKey(propName)) {

                                        filteredElements << element

                                }

                        }

                }

                allElements = filteredElements;

        }

        if (debug >= 3) println " prop&value step result: "+allElements.size()

        return allElements;

}

static def getCQL(String name, def unites) {

        return getCQL(name, unites, false, true)

}

/**

 *

 * @param name

 * @param unites

 * @param onePosition to return 1 token per patch

 * @return

 */

static def getCQL(String name, def unites, boolean onePosition, boolean limitNumberOfUnit) {

        //println "GETCQL of $name"

        def letters = "abcdefghijklmnopqrstu"//vwxyz0123456789"

        def MAXCQLQUERYSIZE = 1200 // 1150 // 1200 in fact

        HashSet<Integer> sizes = new HashSet<>()

        for (Unite unite : unites) {

                int size = unite.getFin() - unite.getDeb()+1

                if (size > letters.length()) size = letters.length()-1

                sizes.add(size)

        }

        int n = 0

        String totalleftquery = ""

        String totalrightquery = ""

        unites.sort() { it.getDeb() }

        def declaredsizes = []

        for (Unite unite : unites) {

                int size = unite.getFin() - unite.getDeb() + 1

                if (size < 0) {

                        println sprintf("** Warning: incoherent unit %s [%d, %d], size = "+size, unite.getProps(),unite.getDeb(), unite.getFin())

                        continue

                }

                if (onePosition) size = 1 // hack only the 1st position is needed for the Progression

                if (size > letters.length()) size = letters.length()-1

                String letter = ""+letters.charAt(size-1)

                String rightquery = letter+"="+unite.getDeb()

                String leftquery = ""

                if (!declaredsizes.contains(size)) {

                        declaredsizes << size

                        if (size == 1)

                                leftquery = letter+":[]"

                        else if (size == 2)

                                leftquery = letter+":[][]"

                        else if (size == 3)

                                leftquery = letter+":[][][]"             // [][][][]

                        else

                                leftquery = letter+":[][]{"+(size-1)+"}" // [][]{4}

                }

                if ((totalleftquery.length() + totalrightquery.length() + 2

                + leftquery.length() + rightquery.length()) >= MAXCQLQUERYSIZE) {

                        System.out.println("** $name : trop d'éléments pour la requête. Seuls les "+n+" premiers éléments sur ${unites.size()} seront affichés dans le graphique de progression.")

                        break

                }

                if (n > 0) {

                        if (leftquery.length() > 0) totalleftquery += "|"

                        totalrightquery += "|"

                }

                if (leftquery.length() > 0) totalleftquery += leftquery

                totalrightquery += rightquery

                n += 1

        }

        String query = totalleftquery+"::"+totalrightquery

        //println query

        return query

}