Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

// This file is part of the TXM platform.

//

// The TXM platform is free software: you can redistribute it and/or modif y

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// The TXM platform is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

//

//

//

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$

//

package org.txm.scripts.importer.transcriber;

import java.io.File;

import org.txm.importer.*

import org.txm.importer.scripts.xmltxm.*;

import org.txm.*;

import org.txm.core.engines.*;

import org.txm.objects.*;

import org.txm.utils.i18n.*;

import org.txm.utils.*;

import org.txm.scripts.importer.*;

import org.txm.metadatas.*;

import org.txm.utils.io.FileCopy;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

//PARAMETERS

boolean removeInterviewer = false;//if true the transcription of speakers (en1 and enq2) defined in metadatas file will be ignored

boolean includeComments = false;

boolean ignoreTranscriberMetadata = false;

int csvHeaderNumber = 1;

int maxlines = 200;

String userDir = System.getProperty("user.home");

def MONITOR;

Project project;

try {project=projectBinding;MONITOR=monitor} catch (Exception)

{        }

if (project == null) { println "no project set. Aborting"; return; }

String corpusname = project.getName();

String basename = corpusname

String rootDir = project.getSrcdir();

String lang = project.getLang()

String model = lang

String encoding = project.getEncoding()

boolean annotate = project.getAnnotate()

String xsl = project.getFrontXSL();

def xslParams = project.getXsltParameters();

int wordsPerPage = project.getEditionDefinition("default").getWordsPerPage()

String page_element = project.getEditionDefinition("default").getPageElement()

boolean build_edition = project.getEditionDefinition("default").getBuildEdition()

boolean update = project.getDoUpdate()

File srcDir = new File(rootDir);

File binDir = project.getProjectDirectory();

binDir.mkdirs();

if (!binDir.exists()) {

        println "Could not create binDir "+binDir

        return;

}

File txmDir = new File(binDir,"txm/$corpusname");

if (!update) txmDir.deleteDir();

txmDir.mkdirs();

//get metadata values from CSV

Metadatas metadatas; // text metadata

File allMetadataFile = Metadatas.findMetadataFile(srcDir);

println "Trying to read metadata values from: "+allMetadataFile

if (allMetadataFile.exists()) {

        File copy = new File(binDir, allMetadataFile.getName())

        if (!FileCopy.copy(allMetadataFile, copy)) {

                println "Error: could not create a copy of metadata file "+allMetadataFile.getAbsoluteFile();

                return;

        }

        metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(),

                        Toolbox.getMetadataColumnSeparator(),

                        Toolbox.getMetadataTextSeparator(), 1)

}

else {

        println "no metadata file: "+allMetadataFile

}

final HashMap<String, String> textordersInfo = new HashMap<String, String>();

if (metadatas != null) {

        for (String t : metadatas.keySet()) {

                def ti = metadatas.get(t)

                for (org.txm.metadatas.Entry e : ti) {

                        if ("textorder".equals(e.getId())) {

                                String k = ""+t+".xml" // the sort test will use the xml-txm file names

                                textordersInfo[k] = e.value

                        }

                }

        }

}

File propertyFile = new File(srcDir, "import.properties")//default

Properties props = new Properties();

String[] metadatasToKeep;

if (propertyFile.exists() && propertyFile.canRead()) {

        FileInputStream input = new FileInputStream(propertyFile);

        props.load(input);

        input.close();

        if (props.getProperty("removeInterviewer") != null)

                removeInterviewer = Boolean.parseBoolean(props.get("removeInterviewer").toString());

        if (props.getProperty("ignoreTranscriberMetadata") != null)

                ignoreTranscriberMetadata = Boolean.parseBoolean(props.get("ignoreTranscriberMetadata").toString());

        if (props.getProperty("metadataList") != null)

                metadatasToKeep = props.get("metadataList").toString().split("|");

        if (props.getProperty("csvHeaderNumber") != null)

                csvHeaderNumber = props.get("csvHeaderNumber").toString().split("|");

        //if (props.getProperty("includeComments") != null)

        //        includeComments = props.get("includeComments").toString();

        println "import properties: "

        println " removeInterviewer: "+removeInterviewer

        println " ignoreTranscriberMetadata: "+ignoreTranscriberMetadata

        println " metadataToKeep: "+metadatasToKeep

        println " ignored csvHeaderSize: "+csvHeaderNumber

        //println " includeComments: "+includeComments

}

try {

        if (!update) {

                // Apply XSL

                if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

                if (MONITOR != null) MONITOR.worked(1, "XSL")

                if (xsl != null && xsl.trim().length() > 0) {

                        if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src")))

                                srcDir = new File(binDir, "src");

                        println ""

                }

                // select only trs files

                String ext = "trs";

                ArrayList<File> trsfiles = srcDir.listFiles(); //find all trs files

                if (trsfiles  == null) {

                        println ("No files in "+srcDir.getAbsolutePath())

                        return false;

                }

                for (int i = 0 ; i < trsfiles.size() ; i++) {

                        File f = trsfiles.get(i);

                        if (!f.getName().endsWith(ext) || !f.canRead() || f.isHidden()) {

                                trsfiles.remove(i)

                                i--;

                        }

                }

                if (trsfiles.size() == 0) {

                        println ("No transcription file (*.trs) found in "+srcDir.getAbsolutePath()+". Aborting.")

                        return false;

                }

                if (MONITOR != null) MONITOR.worked(1, "IMPORTER")

                if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

                println "-- IMPORTER"

                def imp = new importer(trsfiles, binDir, txmDir, metadatas, lang) //put result in the txm folder of binDir

                if (!imp.run()) {

                        println "Failed to prepare files - Aborting";

                        return;

                }

                if (MONITOR != null) MONITOR.worked(20)

                println "-- Xml Validation"

                if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

                for (File infile : txmDir.listFiles()) {

                        if (!ValidateXml.test(infile)) {

                                println "$infile : Validation failed";

                                infile.delete();

                        }

                }

                if (MONITOR != null) MONITOR.worked(5)

                if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

                println "-- Remove interviewer: "+removeInterviewer

                if (removeInterviewer)         {

                        if (metadatas == null) {

                                println "Can't remove interviewer without a metadata.csv file defining who are the interviewers."

                        } else {

                                println "Removing some speakers in "+txmDir.listFiles().length+" file(s)"

                                for (File infile : txmDir.listFiles()) {

                                        String filename = infile.getName();

                                        int idx = filename.indexOf(".xml");

                                        if (idx > 0)

                                                filename = filename.substring(0, idx);

                                        ArrayList<Pair<String, String>> metas = metadatas.get(filename)

                                        //println "filename=$filename metas= $metas"

                                        for (Pair p : metas) {

                                                if (p.getFirst().startsWith("enq")) {

                                                        new RemoveSpeaker(infile, infile, p.getFirst())

                                                }

                                        }

                                }

                        }

                }

                if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

                if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")

                boolean annotationSuccess = false;

                if (annotate) {

                        println "-- ANNOTATE - Running NLP tools"

                        def engine = Toolbox.getEngineManager(EngineType.ANNOTATION).getEngine("TreeTagger")

                        if (engine.processDirectory(txmDir, binDir, ["lang":model])) {

                                annotationSuccess = true;

                        }

                }

        } // end of importer and annotate steps

        xmltxmFiles = new ArrayList<File>(Arrays.asList(txmDir.listFiles()));

        if (metadatas != null && metadatas.getPropertyNames().contains("textorder")) {

                Collections.sort(xmltxmFiles, new Comparator<File>() {

                                        public int compare(File f1, File f2) {

                                                String o1 = textordersInfo[f1.getName()];

                                                String o2 = textordersInfo[f2.getName()];

                                                if (o1 == null && o2 == null) {

                                                        return f1.compareTo(f2);

                                                } else if (o1 == null) {

                                                        return 1

                                                } else if (o2 == null) {

                                                        return -1

                                                } else {

                                                        int c = o1.compareTo(o2);

                                                        if (c == 0) return f1.compareTo(f2);

                                                        else return c;

                                                }

                                        }

                                });

        } else {

                Collections.sort(xmltxmFiles);

        }

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        if (MONITOR != null) MONITOR.worked(25, "COMPILING")

        println "--COMPILING - Building Search Engine indexes"

        def comp = new compiler()

        if(debug) comp.setDebug();

        comp.removeInterviewers(removeInterviewer);

        comp.setIgnoreTranscriberMetadata(ignoreTranscriberMetadata);

        if (!comp.run(project, xmltxmFiles, corpusname, "default", binDir)) {

                println "Failed to compile files";

                return;

        }

        if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

        File htmlDir = new File(binDir,"HTML/$corpusname");

        htmlDir.deleteDir()

        htmlDir.mkdirs();

        if (build_edition) {

                if (MONITOR != null) MONITOR.worked(20, "EDITION")

                println "-- EDITION - Building editions"

                def second = 0

                println "Paginating "+xmltxmFiles.size()+" texts"

                ConsoleProgressBar cpb = new ConsoleProgressBar(xmltxmFiles.size());

                for (File txmFile : xmltxmFiles) {

                        cpb.tick()

                        String txtname = txmFile.getName();

                        int i = txtname.lastIndexOf(".");

                        if(i > 0) txtname = txtname.substring(0, i);

                        List<String> NoSpaceBefore = LangFormater.getNoSpaceBefore(lang);

                        List<String> NoSpaceAfter = LangFormater.getNoSpaceAfter(lang);

                        Text t = project.getText(txtname)

                        if (t == null) {

                                t = new Text(project);

                                t.setName(txtname);

                        }

                        t.setSourceFile(txmFile)

                        t.setTXMFile(txmFile)

                        Edition edition = t.getEdition("default")

                        if (edition != null) {

                                edition.delete();

                                edition = null;

                        }

                        def ed = new pager(txmFile, htmlDir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element, metadatas);

                        edition = t.getEdition("default")

                        edition = new Edition(t);

                        edition.setName("default");

                        edition.setIndex(htmlDir.getAbsolutePath());

                        for (i = 0 ; i < ed.getPageFiles().size();) {

                                File f = ed.getPageFiles().get(i);

                                String wordid = "w_0";

                                if (i < ed.getIdx().size()) wordid = ed.getIdx().get(i);

                                edition.addPage(""+(++i), wordid);

                        }

                }

                cpb.done()

                //copy transcriber.css

                File cssfile = new File(Toolbox.getTxmHomePath(), "css/transcriber.css")

                File cssTXMFile = new File(Toolbox.getTxmHomePath(), "css/txm.css")

                if (cssfile.exists() && htmlDir.exists()) {

                        FileCopy.copy(cssfile, new File(htmlDir, "onepage/transcriber.css"));

                        FileCopy.copy(cssfile, new File(htmlDir, "default/txm.css"));

                        FileCopy.copy(cssfile, new File(htmlDir, "default/transcriber.css"));

                }

                //copy media files

                println "Copying media files if any (mp3, wav, mp4 or avi) "+xmltxmFiles.size()+" texts"

                cpb = new ConsoleProgressBar(xmltxmFiles.size());

                for (File txmFile : xmltxmFiles) {

                        cpb.tick()

                        String txtname = txmFile.getName();

                        int i = txtname.lastIndexOf(".");

                        if(i > 0) txtname = txtname.substring(0, i);

                        File mediaFile = new File(project.getSrcdir(), txtname + ".mp3")

                        if (!mediaFile.exists()) mediaFile = new File(project.getSrcdir(), txtname + ".wav")

                        if (!mediaFile.exists()) mediaFile = new File(project.getSrcdir(), txtname + ".mp4")

                        if (!mediaFile.exists()) mediaFile = new File(project.getSrcdir(), txtname + ".avi")

                        if (mediaFile.exists()) {

                                File copy = new File(binDir, "media/"+mediaFile.getName())

                                copy.getParentFile().mkdirs()

                                FileCopy.copy(mediaFile, copy);

                        }

                }

                cpb.done()

        }

}

catch (Exception e){org.txm.utils.logger.Log.printStackTrace(e);}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

readyToLoad = project.save();