Laboratoire ICAR » Plateforme TXM

package org.txm.scripts.importer.xtz

import java.io.File;

import java.util.ArrayList;

import org.txm.objects.BaseParameters

import org.w3c.dom.Element

import org.txm.scripts.importer.*

import org.txm.utils.ConsoleProgressBar

import org.txm.utils.BundleUtils;

import org.txm.utils.io.FileCopy;

import org.txm.utils.i18n.*

import org.txm.importer.xtz.*

import javax.xml.stream.*

import org.txm.objects.*

import org.txm.importer.ApplyXsl2

class XTZPager extends Pager {

        Project project;

        Element corpusElem;

        String lang;

        String page_element;

        String wordTag;

        int wordsPerPage;

        File cssDirectory, jsDirectory, imagesDirectory;

        public XTZPager(ImportModule module) {

                super(module, "default");

                project = module.getProject()

                lang = project.getLang();

                wordsPerPage = project.getEditionDefinition("default").getWordsPerPage()

                page_element = project.getEditionDefinition("default").getPageElement()

                wordTag = project.getTokenizerWordElement()

                cssDirectory = new File(module.getSourceDirectory(), "css")

                jsDirectory = new File(module.getSourceDirectory(), "js")

                imagesDirectory = new File(module.getSourceDirectory(), "images")

        }

        public void process(ArrayList<File> files) {

                super.process(files);

                if (files == null) {

                        files = inputDirectory.listFiles();

                        if (files != null) Collections.sort(files);

                }

                if (!doDefaultEditionStep()) return;

                if (!doFacsEditionStep()) return;

                if (!doPostEditionXSLStep()) return;

                isSuccessFul = true;

                println ""

        }

        public boolean doDefaultEditionStep() {

                boolean build_edition = project.getEditionDefinition("default").getBuildEdition()

                if (!build_edition) {

                        return true;

                }

                def second = 0

                println "-- Building 'default' edition of  ${files.size()} texts..."

                def css = ["css/txm.css", "css/${corpusname}.css"] // default CSS inclusion

                // scan existing css files that must be declared in each HTML page

                if (cssDirectory.exists()) {

                        def cssFiles = cssDirectory.listFiles();

                        if (cssFiles != null)

                                for (File cssFile : cssFiles) {

                                        if (cssFile.isFile() && !cssFile.isHidden() && cssFile.getName().endsWith(".css"))

                                                css << "css/"+cssFile.getName();

                                }

                }

                ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

                for (File txmFile : files) {

                        try {

                                cpb.tick()

                                String textname = txmFile.getName();

                                int i = textname.lastIndexOf(".");

                                if (i > 0) textname = textname.substring(0, i);

                                File firstHTMLPageFile = new File(outputDirectory, textname+"_1.html");

                                if (firstHTMLPageFile.exists() && firstHTMLPageFile.lastModified() >= txmFile.lastModified()) continue;

                                List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

                                List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

                                Text t = project.getText(textname);

                                if (t == null) {

                                        t = new Text(project);

                                }

                                t.setName(textname);

                                t.setSourceFile(txmFile)

                                t.setTXMFile(txmFile)

                                def ed = new XTZDefaultPagerStep(this, txmFile, textname, NoSpaceBefore, NoSpaceAfter, css);

                                if (!ed.process()) {

                                        println "Fail to build 'default' edition for text: $txmFile"

                                        continue;

                                }

                                Edition edition = t.getEdition("default")

                                if (edition == null) {

                                        edition = new Edition(t);

                                }

                                edition.setName("default");

                                edition.setIndex(outputDirectory.getAbsolutePath());

                                for (i = 0 ; i < ed.getPageFiles().size();) {

                                        File f = ed.getPageFiles().get(i);

                                        String wordid = "w_0";

                                if (i < ed.getIdx().size()) wordid = ed.getIdx().get(i);

                                        edition.addPage(""+(++i), wordid);

                                }

                        } catch(Exception e) {

                                println "Error: could not create $txmFile 'default' edition: "+e

                                e.printStackTrace()

                        }

                }

                // copy default TXM css file in the "facs" edition directory

                File csshtmlDirectory = new File(outputDirectory, "css")

                csshtmlDirectory.mkdirs()

                BundleUtils.copyFiles("org.txm.core", "res", "org/txm/css", "txm.css", csshtmlDirectory);

                // copy CSS files in the "default" edition directory

                if (cssDirectory.exists()) {

                        FileCopy.copyFiles(cssDirectory, csshtmlDirectory)

                }

                if (jsDirectory.exists()) {

                        File jshtmlDirectory = new File(outputDirectory, "js")

                        FileCopy.copyFiles(jsDirectory, jshtmlDirectory)

                }

                if (imagesDirectory.exists()) {

                        File imageshtmlDirectory = new File(outputDirectory, "images")

                        FileCopy.copyFiles(imagesDirectory, imageshtmlDirectory)

                }

                // save changes

                println ""

                return true;

        }

        public boolean doFacsEditionStep() {

                boolean mustBuildFacsEdition = project.getEditionDefinition("facs").getBuildEdition()

                if (!mustBuildFacsEdition) return true;

                String imageDirectoryPath = project.getEditionDefinition("facs").getImagesDirectory().trim();

                File imageDirectory = new File(imageDirectoryPath);

                if (!imageDirectoryPath.startsWith("http") && imageDirectoryPath.length()== 0 && !imageDirectory.exists() && !imageDirectory.isDirectory()) {

                        imageDirectory = null;

                }

                def second = 0

                println "-- Building 'facs' edition of ${files.size()} texts..."

                File newEditionDirectory = new File(htmlDirectory, "facs");

                newEditionDirectory.mkdir();

                ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

                for (File txmFile : files) {

                        cpb.tick()

                        String txtname = txmFile.getName();

                        int i = txtname.lastIndexOf(".");

                        if (i > 0) txtname = txtname.substring(0, i);

                        File firstHTMLPageFile = new File(newEditionDirectory, txtname+"_1.html");

                        if (firstHTMLPageFile.exists() && firstHTMLPageFile.lastModified() >= txmFile.lastModified()) continue;

                        List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

                        List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

                        Text t = project.getText(txtname);

                        if (t == null) {

                                t = new Text(project);

                        }

                        t.setName(txtname);

                        t.setSourceFile(txmFile)

                        t.setTXMFile(txmFile)

                        Edition edition = t.getEdition("facs")

                        if (edition == null) {

                                edition = new Edition(t);

                        }

                        edition.setName("facs");

                        edition.setIndex(outputDirectory.getAbsolutePath());

                        try {

                                def ed = new XTZFacsPagerStep(txmFile, newEditionDirectory, imageDirectory, txtname, corpusname, "pb", "facs", wordTag, debug);

                                if (!ed.process()) {

                                        println "Fail to build 'facs' edition for text: $txmFile"

                                        continue;

                                }

                                def pages = ed.getPageFiles()

                                for (i = 0 ; i < pages.size();) {

                                        File f = pages[i][0];

                                        String wordid = pages[i][1]

                                        //TODO replace '""+(++i)' with something that fetch/findout the page 'name'

                                        // TODO or move the Edition and Page corpus declaration in the XTZDefaultPagerStep

                                        edition.addPage(""+(++i), wordid);

                                        //println "add facs page: $f $wordid"

                                }

                        } catch (Exception e) {

                                println "Error while processing $txmFile text: "+e

                                e.printStackTrace();

                                return false;

                        }

                }

                if (!imageDirectoryPath.startsWith("http") && imageDirectory != null) { // copy files only if local

                        File editionImagesDirectory = new File(newEditionDirectory, "res/images/"+corpusname+"/facs");

                        editionImagesDirectory.mkdirs();

                        FileCopy.copyFiles(imageDirectory, editionImagesDirectory);

                }

                // copy SimpleViewer files in the "facs" edition directory

                File jshtmlDirectory = new File(newEditionDirectory, "js")

                jshtmlDirectory.mkdirs()

                BundleUtils.copyFiles("org.txm.core", "res", "org/txm/js", "viewer", jshtmlDirectory);

                BundleUtils.copyFiles("org.txm.core", "res", "org/txm/", "images", newEditionDirectory);

                // copy default TXM css file in the "facs" edition directory

                File csshtmlDirectory = new File(newEditionDirectory, "css")

                csshtmlDirectory.mkdirs()

                BundleUtils.copyFiles("org.txm.core", "res", "org/txm/css", "txm.css", csshtmlDirectory);

                // copy CSS/JS/Images sources files in the "facs" edition directory

                if (cssDirectory.exists()) {

                        FileCopy.copyFiles(cssDirectory, csshtmlDirectory)

                }

                if (jsDirectory.exists()) {

                        FileCopy.copyFiles(jsDirectory, jshtmlDirectory)

                }

                if (imagesDirectory.exists()) {

                        File imageshtmlDirectory = new File(newEditionDirectory, "images")

                        FileCopy.copyFiles(imagesDirectory, imageshtmlDirectory)

                }

                project.setDefaultEditionName("default,facs");

                println ""

                return true;

        }

        /**

         * read from $bindir/txm and write the result in $bindir/txm

         *

         */

        public boolean doPostEditionXSLStep() {

                File xslDirectory = new File(module.getSourceDirectory(), "xsl/4-edition")

                if (xslDirectory.exists()) {

                        // prepare XSL parameters

                        def xslParams = module.getProject().getXsltParameters();

                        String s = module.getProject().getEditionDefinition("default").getWordsPerPage()

                        if (s != null && s.length() > 0)

                        // shared XSL parameters

                        xslParams["number-words-per-page"] = Integer.parseInt(s);

                        xslParams["pagination-element"] = module.getProject().getEditionDefinition("default").getPageElement()

                        xslParams["import-xml-path"] = module.getProject().getProjectDirectory()

                        //println "XSL PARAMS: "+xslParams

                        def xslFiles = xslDirectory.listFiles()

                        xslFiles = xslFiles.sort() { f ->

                                try {

                                return Integer.parseInt(f.getName().substring(0, f.getName().indexOf("-")))

                                } catch(Exception e) {}

                                return -1;

                        }

                        def editionsCreated = [:]

                        for (File xslFile : xslFiles) {

                                if (xslFile.isDirectory() || xslFile.isHidden() || !xslFile.getName().endsWith(".xsl")) continue;

                                if (!xslFile.getName().matches("[1-9]{1,3}-.+")) continue;

                                String xslName = xslFile.getName().substring(2); // remove the "1-", "2-", etc.

                                int idx2 = xslName.indexOf(".")

                                if (idx2 > 0) xslName = xslName.substring(0, idx2)

                                else {

                                        println "$xslFile is not a '.xsl' file"

                                        continue;

                                }

                                int idx3 = xslName.indexOf("-")

                                if (idx3 < 0) {

                                        println "$xslFile file does not follow the '{Number}-{editionName}-{step}.xsl' name pattern"

                                        continue;

                                }

                                String pagerStep = xslName.substring(idx3 + 1);

                                String editionName = xslName.substring(0, idx3);

                                int idx = editionName.indexOf(".")

                                if (idx > 0) editionName = editionName.substring(0, idx);

                                println "-- Building '$editionName' XSL edition with step '$pagerStep'..."

                                File newEditionDirectory = new File(htmlDirectory, editionName);

                                xslParams["output-directory"] = newEditionDirectory.toURI().toString()

                                if (editionsCreated[editionName] == null) { // first XSL, replace an edition

                                        editionsCreated[editionName] = xslFile

                                        //if (!importModule.isUpdatingCorpus()) {

                                        //TODO: optimisation if update is enable,

                                        newEditionDirectory.deleteDir(); // delete previous edition if any

                                        //}

                                        newEditionDirectory.mkdir()

                                        boolean deleteOutputFiles = "pager" == pagerStep;

                                        if (ApplyXsl2.processImportSources(xslFile, inputDirectory, newEditionDirectory, xslParams, deleteOutputFiles)) {

                                                println ""

                                        } else {

                                                reason = "Fail to apply edition XSL: $xslFile"

                                                return false;

                                        }

                                        // copy CSS files in the newEditionDirector edition directory

                                        if (cssDirectory.exists()) {

                                                File csshtmlDirectory = new File(newEditionDirectory, "css")

                                                FileCopy.copyFiles(cssDirectory, csshtmlDirectory)

                                        }

                                        if (jsDirectory.exists()) {

                                                File jshtmlDirectory = new File(newEditionDirectory, "js")

                                                FileCopy.copyFiles(jsDirectory, jshtmlDirectory)

                                        }

                                        if (imagesDirectory.exists()) {

                                                File imageshtmlDirectory = new File(newEditionDirectory, "images")

                                                FileCopy.copyFiles(imagesDirectory, imageshtmlDirectory)

                                        }

                                } else { // N+1 XSL working with HTML files

                                        def htmlFiles = newEditionDirectory.listFiles()

                                        htmlFiles.sort()

                                        if (ApplyXsl2.processImportSources(xslFile, htmlFiles, xslParams)) {

                                                if ("pager".equals(pagerStep)) {

                                                        // delete the one page HTML files only if the XSL step is "pager"

                                                        for (File f : htmlFiles) f.delete();

                                                }

                                                //        println ""

                                        } else {

                                                reason = "Fail to apply edition XSL: $xslFile"

                                                return false;

                                        }

                                }

                        }

                        // UPDATE import.xml: for each XML-TXM file, we must retrieve the first word ID from the XSL output files

                        //println "retrieve word ids from $inputDirectory"

                        println "-- Fetching page word IDs..."

                        ConsoleProgressBar cpb = new ConsoleProgressBar(editionsCreated.keySet().size())

                        for (String editionName : editionsCreated.keySet()) {

                                cpb.tick()

                                File newEditionDirectory = new File(htmlDirectory, editionName);

                                File xslFile = editionsCreated[editionName]

                                for (File txmFile : inputDirectory.listFiles()) {

                                        if (txmFile.isDirectory()) continue;

                                        String textName = txmFile.getName()

                                        int idx4 = textName.indexOf(".")

                                        if (idx4 > 0) textName = textName.substring(0, idx4);

                                        getFirstWordIDs(textName, editionName, newEditionDirectory, xslFile, txmFile);

                                }

                        }

                        println ""

                }

                return true;

        }

        private void getFirstWordIDs(String textName, String editionName, File newEditionDirectory, File xslFile, File txmFile) {

                //                println "call getFirstWordIDs textName=$textName editionName=$editionName dir=$newEditionDirectory xsl=$xslFile"

                Text t = project.getText(textName);

                if (t == null) {

                        t = new Text(project);

                }

                t.setName(textName);

                t.setSourceFile(txmFile)

                t.setTXMFile(txmFile)

                Edition edition = t.getEdition(editionName)

                if (edition == null) {

                        edition = new Edition(t);

                }

                edition.setName(editionName);

                edition.setIndex(outputDirectory.getAbsolutePath());

                LinkedHashMap<File, String> words = new LinkedHashMap<File, String>()

                def files = []

                newEditionDirectory.eachFile() {it -> if (it.isFile()) files << it}

                files.sort() { f1, f2 ->

                        String s1 = f1.getName()

                        String s2 = f2.getName()

                        int n1 = Integer.parseInt(s1.substring(s1.lastIndexOf("_")+1, s1.lastIndexOf(".")))

                        int n2 = Integer.parseInt(s2.substring(s2.lastIndexOf("_")+1, s2.lastIndexOf(".")))

                        return n1 - n2;

                }

                for (File f : files) {

                        String pagename = f.getName();

                        if (pagename.startsWith(textName+"_")) { // this is a page

                                String firstWordID = getMetaContent(f);

                                pagename = pagename.substring((textName+"_").length(), pagename.indexOf(".html")) // !!!!

                                edition.addPage(pagename, firstWordID)

                        }

                }

        }

        public static String getMetaContent(File f) {

                def inputData = f.toURI().toURL().openStream();

                def factory = XMLInputFactory.newInstance();

                factory.setProperty("javax.xml.stream.supportDTD", false); // ignore the DTD declared in doctype

                def parser = factory.createXMLStreamReader(inputData);

                String META = "meta"

                String BODY = "body"

                String NAME = "name"

                String DESCRIPTION = "txm:first-word-id"

                String CONTENT = "content"

                String content = "";

                String desc = "";

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {

                        if (event == XMLStreamConstants.START_ELEMENT) { // start elem

                                if (META.equals(parser.getLocalName())) { // ana elem

                                        desc = "";

                                        // fetch attribute values

                                        for (int i = 0 ; i < parser.getAttributeCount(); i++) { // scan attributes

                                                if (NAME.equals(parser.getAttributeLocalName(i))) { // found @name

                                                        desc = parser.getAttributeValue(i)

                                                } else if (CONTENT.equals(parser.getAttributeLocalName(i))) { // found @content

                                                        content = parser.getAttributeValue(i)

                                                }

                                        }

                                        if (DESCRIPTION.equals(desc)) { // stop now

                                                break;

                                        }

                                } else if (BODY.equals(parser.getLocalName())) { // no need to go further, meta@name="description" not found :(

                                        content = "";

                                        break;

                                }

                        }

                }

                if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                return content;

        }

        public static void main(def args) {

                println "RESULT: "+getMetaContent(new File("/home/mdecorde/TXM/corpora/QGRAALXTZ/HTML/QGRAALXTZ/default", "qgraal_cm_test201510_page_160_2.html"))

        }

}