Laboratoire ICAR » Plateforme TXM

// Copyright © 2021 ENS de Lyon, CNRS, University of Franche-Comté

// Licensed under the terms of the GNU General Public License (http://www.gnu.org/licenses)

// @author sheiden

// @author mdecorde

// STANDARD DECLARATIONS

package org.txm.macro.projets.dth

import org.kohsuke.args4j.*

import groovy.transform.Field

import org.txm.rcp.swt.widget.parameters.*

import org.txm.rcp.editors.concordances.*

import org.txm.searchengine.cqp.corpus.CQPCorpus

import org.txm.searchengine.cqp.corpus.MainCorpus

import org.txm.functions.concordances.*

import org.txm.annotation.urs.*

import org.txm.concordance.core.functions.Concordance

import org.txm.concordance.rcp.editors.ConcordanceEditor

import visuAnalec.elements.Unite

import visuAnalec.vue.Vue

import org.txm.searchengine.cqp.CQPSearchEngine

import org.apache.commons.lang.StringUtils

import org.txm.macro.cqp.CQPUtils

import org.txm.searchengine.cqp.ICqiClient

import org.txm.utils.i18n.LangFormater

move_start = 0

move_end = 0

create_only_if_new = true

// Biblical References CQL Query

query = /[word="[1-2]\.?"]? @[word="Mat\.|Acts|Gen\.|Cor\.|chap\.|Deut\.|Exod\.|Sam\.|Rom\.|Luke|Matth\.|Iohn|Joh\.|Kings|Act\.|John|Prov\.|Tim\.|Heb\.|Mark|Pet\.|Psal\.|Gal\.|Tit\.|Isaiah|Levit\.|Ezek\.|Math\.|Col\.|Math|Numb\.|Numbers|Zach\.|Chron\.|Eph\.|Epist\.|Isay|Luk\.|Marke|Dan\.|Ephes\.|Esay|Ioh\.|Isai|Jo\.|Job|Josh\.|Mar\.|Proverbs|Psalm|Vers\.|Colos\.|Ierem\.|Iosh\.|Jer\.|Jerem\.|Judges|Micah|Rev\.|Rom|Thess\.|Titus|Ac\.|Act|Apoc\.|Apocalypse|Baruch|Chro\.|Coll\.|Deuteronomy|Eccles\.|Ecclesiastes|Ecclus\.|Esdras|Exo\.|Exod|Ezektel|Gen|Haggai|Iames|Ier\.|Iob\.|Ioel|Ioshua|Isa|Isa\.|Iud\.|Iude|Iudg\.|Joel|Joshua|Jud\.|Judg\.|Matt\.|Matthew|Num\.|Numb|Psa\.|Ruth|Timothy|Zach"] [pos="p"]? [word="[0-9].*"] ([word="[0-9].*|v.*|\p{P}" & word!="\)|]|:|\("])*/

bibRefs = "BibRefs"

bibBooks = [

"Ac.":"Acts",

"Act":"Acts",

"Act.":"Acts",

"Acts":"Acts",

"Apoc.":"Revelation",

"Apocalypse":"Revelation",

"Baruch":"Baruch",

"chap.":"/chap./",

"Chro.":"Chronicles",

"Chron.":"Chronicles",

"Col.":"Colossians",

"Coll.":"Colossians",

"Colos.":"Colossians",

"Cor.":"Corinthians",

"Dan.":"Daniel",

"Deut.":"Deuteronomy",

"Deuteronomy":"Deuteronomy",

"Eccles.":"Ecclesiastes",

"Ecclesiastes":"Ecclesiastes",

"Ecclus.":"Ecclesiasticus",

"Eph.":"Ephesians",

"Ephes.":"Ephesians",

"Epist.":"/Epist./",

"Esay":"Isaiah",

"Esdras":"Esdras",

"Exo.":"Exodus",

"Exod":"Exodus",

"Exod.":"Exodus",

"Ezek.":"Ezekiel",

"Gal.":"Galatians",

"Gen":"Genesis",

"Gen.":"Genesis",

"Haggai":"Haggai",

"Heb.":"Hebrews",

"Iames":"James",

"Ier.":"Jeremiah",

"Ierem.":"Jeremiah",

"Ioel":"Joel",

"Ioh.":"John",

"Iohn":"John",

"Iosh.":"Joshua",

"Ioshua":"Joshua",

"Isa":"Isaiah",

"Isa.":"Isaiah",

"Isai":"Isaiah",

"Isaiah":"Isaiah",

"Isay":"Isaiah",

"Iud.":"Jude",

"Iude":"Jude",

"Iudg.":"Judges",

"Jer.":"Jeremiah",

"Jerem.":"Jeremiah",

"Jo.":"John",

"Jo.":"Job",

"Job":"Job",

"Joel":"Joel",

"Joh.":"John",

"John":"John",

"Josh.":"Joshua",

"Joshua":"Joshua",

"Jud.":"Jude",

"Judg.":"Judges",

"Judges":"Judges",

"Kings":"Kings",

"Levit.":"Leviticus",

"Luk.":"Luke",

"Luke":"Luke",

"Mar.":"Mark",

"Mark":"Mark",

"Marke":"Mark",

"Mat.":"Matthew",

"Math":"Matthew",

"Math.":"Matthew",

"Matt.":"Matthew",

"Matth.":"Matthew",

"Matthew":"Matthew",

"Micah":"Micah",

"Num.":"Numbers",

"Numb":"Numbers",

"Numb.":"Numbers",

"Numbers":"Numbers",

"Pet.":"Peter",

"Prov.":"Proverbs",

"Proverbs":"Proverbs",

"Psa.":"Psalms",

"Psal.":"Psalms",

"Psalm":"Psalms",

"Rev.":"Revelation",

"Rom":"Romans",

"Rom.":"Romans",

"Ruth":"Ruth",

"Sam.":"Samuel",

"Thess.":"Thessalonians",

"Tim.":"Timothy",

"Timothy":"Timothy",

"Tit.":"Titus",

"Titus":"Titus",

"Vers.":"/Vers./",

"Zach":"Zechariah",

"Zach.":"Zechariah",

]

bibAbbr = [

"Ac.":"Acts",

"Act":"Acts",

"Act.":"Acts",

"Acts":"Acts",

"Apoc.":"Rev.",

"Apocalypse":"Rev.",

"Baruch":"Bar.",

"chap.":"/chap./",

"Chro.":"Chr.",

"Chron.":"Chr.",

"Col.":"Col.",

"Coll.":"Col.",

"Colos.":"Col.",

"Cor.":"Cor.",

"Dan.":"Dan.",

"Deut.":"Deut.",

"Deuteronomy":"Deut.",

"Eccles.":"Eccles.",

"Ecclesiastes":"Eccles.",

"Ecclus.":"Ecclus.",

"Eph.":"Eph.",

"Ephes.":"Eph.",

"Epist.":"/Epist./",

"Esay":"Isa.",

"Esdras":"Esdras",

"Exo.":"Exod.",

"Exod":"Exod.",

"Exod.":"Exod.",

"Ezek.":"Ezek.",

"Gal.":"Gal.",

"Gen":"Gen.",

"Gen.":"Gen.",

"Haggai":"Hag.",

"Heb.":"Heb.",

"Iames":"Jas.",

"Ier.":"Jer.",

"Ierem.":"Jer.",

"Ioel":"Joel",

"Ioh.":"John",

"Iohn":"John",

"Iosh.":"Josh.",

"Ioshua":"Josh.",

"Isa":"Isa.",

"Isa.":"Isa.",

"Isai":"Isa.",

"Isaiah":"Isa.",

"Isay":"Isa.",

"Iud.":"Jude",

"Iude":"Jude",

"Iudg.":"Judg.",

"Jer.":"Jer.",

"Jerem.":"Jer.",

"Jo.":"Kgs",

"Jo.":"Job",

"Job":"Job",

"Joel":"Joel",

"Joh.":"John",

"John":"John",

"Josh.":"Josh.",

"Joshua":"Josh.",

"Jud.":"Jude",

"Judg.":"Judg.",

"Judges":"Judg.",

"Kings":"Kgs",

"Levit.":"Lev.",

"Luk.":"Luke",

"Luke":"Luke",

"Mar.":"Mark",

"Mark":"Mark",

"Marke":"Mark",

"Mat.":"Matt.",

"Math":"Matt.",

"Math.":"Matt.",

"Matt.":"Matt.",

"Matth.":"Matt.",

"Matthew":"Matt.",

"Micah":"Mic.",

"Num.":"Num.",

"Numb":"Num.",

"Numb.":"Num.",

"Numbers":"Num.",

"Pet.":"Pet.",

"Prov.":"Prov.",

"Proverbs":"Prov.",

"Psa.":"Ps.",

"Psal.":"Ps.",

"Psalm":"Ps.",

"Rev.":"Rev.",

"Rom":"Rom.",

"Rom.":"Rom.",

"Ruth":"Ruth",

"Sam.":"Sam.",

"Thess.":"Thess.",

"Tim.":"Tim.",

"Timothy":"Tim.",

"Tit.":"Titus",

"Titus":"Titus",

"Vers.":"/Vers./",

"Zach":"Zech.",

"Zach.":"Zech.",

]

/*

println "bibAbbr lengths :"

println "len\tF"

bibAbbr.collect {  it.value }.sort().unique().collect {  it.length() }.sort().countBy { it }.each { println sprintf("%1d\t%2d", it.key, it.value) }

(5..8).each { max ->

	abbrs = bibAbbr.collect {  it.value }.sort().unique().findAll { it.length() == max }

	println max+" : "+abbrs

}

*/

// check for a corpus selection

utils = new CQPUtils()

corpusEngine = CQPSearchEngine.getCqiClient()

corpora = utils.getCorpora(this)

scriptName = this.class.getSimpleName()

if ((corpora == null) || corpora.size() > 1) {

	println "** $scriptName: please select a corpus in the Corpus view or provide a corpus name. Aborting."

	return false

}

corpus = corpora[0].getMainCorpus()

corpusName = corpus.getName()

wordProperty = corpus.getProperty("word")

if (!URSCorpora.isAnnotationStructureReady(corpus)) {

	println "** URS Annotation Structure of "+corpusName+" is not ready. Aborting."

	return

}

// check for corpus annotation structure unit types

def analecCorpus = URSCorpora.getCorpus(corpus)

// Composite Biblical Reference : number, book, chapters_verses_list, chapters_verses_form

def crType = "Composite Biblical Reference"

if (!analecCorpus.getStructure().getUnites().contains(crType)) {

	analecCorpus.getStructure().ajouterType(Unite.class, crType)

	analecCorpus.getStructure().ajouterProp(Unite.class, crType, "reference_form")

	analecCorpus.getStructure().ajouterProp(Unite.class, crType, "reference_id")

	analecCorpus.getStructure().ajouterProp(Unite.class, crType, "number")

	analecCorpus.getStructure().ajouterProp(Unite.class, crType, "book")

	analecCorpus.getStructure().ajouterProp(Unite.class, crType, "chapters_verses_form")

	analecCorpus.getStructure().ajouterProp(Unite.class, crType, "books_chapters_verses_list")

}

// Biblical Reference : number, book, chapter, verse

def rType = "Biblical Reference"

if (!analecCorpus.getStructure().getUnites().contains(rType)) {

	analecCorpus.getStructure().ajouterType(Unite.class, rType)

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "number")

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "book")

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "chapter")

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "verse")

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "raw_verses")

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "cr_id")

	analecCorpus.getStructure().ajouterProp(Unite.class, rType, "type")

}

// check for corpus annotation structure input form

def vue = URSCorpora.getVue(corpus)

if (!vue.getTypesAVoir(Unite.class).contains(crType)) {

	vue.ajouterType(Unite.class, crType)

	vue.ajouterProp(Unite.class, crType, "reference_id")

	vue.ajouterProp(Unite.class, crType, "reference_form")

	vue.ajouterProp(Unite.class, crType, "books_chapters_verses_list")

	vue.ajouterProp(Unite.class, crType, "chapters_verses_form")

	vue.ajouterProp(Unite.class, crType, "book")

	vue.ajouterProp(Unite.class, crType, "number")

}

if (!vue.getTypesAVoir(Unite.class).contains(rType)) {

	vue.ajouterType(Unite.class, rType)

	vue.ajouterProp(Unite.class, rType, "type")

	vue.ajouterProp(Unite.class, rType, "cr_id")

	vue.ajouterProp(Unite.class, rType, "raw_verses")

	vue.ajouterProp(Unite.class, rType, "verse")

	vue.ajouterProp(Unite.class, rType, "chapter")

	vue.ajouterProp(Unite.class, rType, "book")

	vue.ajouterProp(Unite.class, rType, "number")

}

// manage cqp matching strategy

cqp_matching_strategy = System.getProperty("cqp_matching_strategy")

if (!(cqp_matching_strategy == 'longest')) {

	println "Changing MatchingStrategy from '"+cqp_matching_strategy+"' to 'longest'"

	corpusEngine.query("set MatchingStrategy longest;")

	System.setProperty("cqp_matching_strategy", 'longest')

}

corpusEngine.cqpQuery(corpusName, bibRefs, query)

nmatches = corpusEngine.subCorpusSize("$corpusName:$bibRefs")

if (nmatches == 0) {

	println "** No references found. Aborting."

	return

}

println "Found "+nmatches+" biblical references."

println "Removing all <"+rType+"> units..."

analecCorpus.getUnites(rType).toArray(new Unite[0]).each { analecCorpus.supUnite(it) }

nRUnits = 0

def ScanChapsVers (book, num, chapsVers) {

	def lookingForChap = true

	def restIsVerse = false

	def CVList = []

	def maxVerse = 0

	def currentChap = 0

	def bookName

	if (num.length() > 0) {

		num += " "

	}

	if (bibAbbr.containsKey(book)) {

		bookName = num+bibAbbr[book]

	} else {

		println "** unkown book name: "+book

		bookName = num+"?"+book+"?"

	}

	chapsVers.each {

/*

15. vers. 22 , 23 , 24.			15.22,15.23,15.24		chap-VERS-verses

8. vers. 9.				8.9				chap-VERS-verse

4. v. 14				4.14				chap-V-verse

14. ver. 34.				14.34				chap-VER-verse

3. ver. 11 , 12 ,			3.11,3.12			chap-VER-verses

9. 9.					9.9				chap-verse

2. 13. 14 .				2.13,2.14			chap-verses

4. 41. 5. 26.				4.41,5.26			chap-verse-chap-verse

9. 13. 9. 21. 10. 12. 12. 19.		9.13,9.21,10.12,12.19		chaps-verses

9. 13. 9. 21. 10. 12. 12. 19. 2		9.13,9.21,10.12,12.19		chaps-verses-num

3. 11 8.				3.11,8				chap-verse-chap

5. 3 , 4 , & 5.				5.3,5.4,5.5			chap-verses-&-verse

5. 5. 7. 13 , 15. 27. 6. & 30. 25.	5.5,5.7,5.13,5.15,5.27,6,30.25	chap-verses-&-chap-verse

15.					15				chap

*/

		if (it != "," && it != "&" && it != "." && it != "[") {

			// remove trailing '.'

			it = (it ==~ /.*\./) ? it.substring(0, it.length()-1) : it

			if (restIsVerse) {

				if (it ==~ /[0-9].*/) {

					maxVerse = (it as Integer)

					CVList << [bookName, currentChap, (it as Integer)]

				}

			} else if (lookingForChap) {

				currentChap = (it as Integer)

				lookingForChap = false

			} else if (it ==~ /v.*/) {

				restIsVerse = true

			} else if ((it as Integer) > maxVerse) {

				maxVerse = (it as Integer)

				CVList << [bookName, currentChap, (it as Integer)]

			} else {

				currentChap = (it as Integer)

				maxVerse = 0

			}

		}

	}

	if (maxVerse == 0) {

		CVList << [bookName, currentChap, 0]

	}

//	return CVList.toString()

	return CVList

}

def starts = corpusEngine.dumpSubCorpus("$corpusName:$bibRefs", ICqiClient.CQI_CONST_FIELD_MATCH,    0, nmatches-1)

def targets = corpusEngine.dumpSubCorpus("$corpusName:$bibRefs", ICqiClient.CQI_CONST_FIELD_TARGET, 0, nmatches-1)

def ends   = corpusEngine.dumpSubCorpus("$corpusName:$bibRefs", ICqiClient.CQI_CONST_FIELD_MATCHEND, 0, nmatches-1)

[starts, targets, ends].transpose().each {

//[starts, targets, ends].transpose().collect {

		iUnit = 0

		if (it[0] < it[1]) { // number+book+chaps-verses

			def refForm = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[0]..it[2]) as int[]).join(' ')

			def number = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[0]..(it[1]-1)) as int[]).join(' ')

			number = (number ==~ /.*\./) ? number.substring(0, number.length()-1) : number

			def book = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[1]..it[1]) as int[]).join(' ')

			def chapsVersForms = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[1]+1)..it[2] as int[]).join(' ')

			def chapsVersFormsList = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[1]+1)..it[2] as int[]) as String[]

			def chapsVersList = ScanChapsVers(book, number, chapsVersFormsList)

			chapsVersList.reverse().each { b, c, v ->

				def properties = [:]

				properties["number"] = number

				properties["book"] = b

				properties["chapter"] = c as String

				properties["verse"] = v as String

				properties["raw_verses"] = LangFormater.format(chapsVersForms, corpus.getLang())

				properties["cr_id"] = 0 as String

				properties["type"] = "word"

				analecCorpus.addUniteSaisie(rType, it[0], it[2], properties)

				nRUnits++

			}

//		 	return refForm+'\t"'+refForm.replaceAll(/( )/,'"$1"')+'"\t'+number+'\t'+book+'\t'+chapsVersForms+'\t'+chapsVersList

//		 	return chapsVersList as String[]

		} else { // book+chaps-verses

			def refForm = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[0]..it[2]) as int[]).join(' ')

			def book = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[0]..it[0]) as int[]).join(' ')

			def chapsVersForms = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[0]+1)..it[2] as int[]).join(' ')

			def chapsVersFormsList = corpusEngine.cpos2Str("$corpusName.$wordProperty", (it[0]+1)..it[2] as int[]) as String[]

			def chapsVersList = ScanChapsVers(book, "", chapsVersFormsList)

			chapsVersList.reverse().each { b, c, v ->

				def properties = [:]

				properties["number"] = ""

				properties["book"] = b

				properties["chapter"] = c as String

				properties["raw_verses"] = LangFormater.format(chapsVersForms, corpus.getLang())

				properties["verse"] = v as String

				properties["cr_id"] = 0 as String

				properties["type"] = "word"

				analecCorpus.addUniteSaisie(rType, it[0], it[2], properties)

				nRUnits++

			}

//		 	return refForm+'\t"'+refForm.replaceAll(/( )/,'"$1"')+'"\t'+'\t'+book+'\t'+chapsVersForms+'\t'+chapsVersList

//		 	return chapsVersList as String[]

		}

	}

//	}.countBy { it }.sort { a,b -> -a.value <=> -b.value ?: a.key <=> b.key }.each {

//	}.flatten().countBy { it }.sort { a,b -> -a.value <=> -b.value ?: a.key <=> b.key }.each {

// 			println it.key+"\t"+it.value

//		}

if (!(cqp_matching_strategy == 'longest')) {

	println "Changing MatchingStrategy back to '"+cqp_matching_strategy+"'"

	corpusEngine.query("set MatchingStrategy "+cqp_matching_strategy+";")

	System.setProperty("cqp_matching_strategy", cqp_matching_strategy)

}

println "$nRUnits <$rType> units created."

print "Saving units... "

URSCorpora.saveCorpus(analecCorpus)

println "Done."

if (nRUnits > 0) corpus.setIsModified(true)