Laboratoire ICAR » Plateforme TXM

package org.txm.importer.xtz;

import java.io.File;

import java.io.FileFilter;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import org.eclipse.core.runtime.IProgressMonitor;

import org.txm.utils.xml.DomUtils;

import org.txm.metadatas.Metadatas

import org.txm.objects.BaseParameters

import org.txm.utils.io.FileCopy;

import org.txm.*

import org.w3c.dom.Element

public class XTZImport extends ImportModule {

        public XTZImport(File importParametersFile) {

                super(importParametersFile);

        }

        public XTZImport(BaseParameters importParameters) {

                super(importParameters);

        }

        @Override

        public void init(BaseParameters p) {

                super.init(p);

                importer = new XTZImporter(this)

                compiler = new XTZCompiler(this)

                annotater = new TTAnnotater(this);

                pager = new XTZPager(this)

        }

        @Override

        protected ArrayList<File> getTXMFilesOrder() {

                //System.out.println("XTZ FILES ORDER");

                if (importer == null) {

                        println "no importer step, using default text order"

                        return super.getTXMFilesOrder();

                }

                Metadatas metadata = importer.getMetadata();

                if (metadata == null) {

                        println "no metadata, using default text order"

                        return super.getTXMFilesOrder();

                }

                File txmDirectory = new File(binaryDirectory, "txm/"+corpusName);

                ArrayList<File> files = new ArrayList<File>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {

                        @Override

                        public boolean accept(File file) {

                                return file.isFile() && file.getName().endsWith(".xml");

                        }

                })));

                final HashMap<File, String> textorder = new HashMap<File, String>();

                for (File f : files) {

                        HashMap<String, String> m = metadata.getTextMetadata(f);

                        if (m != null && m.containsKey("textorder")) textorder[f] = m["textorder"];

                }

                println "Sorting texts using 'textorder' metadata values: "+textorder

                Collections.sort(files, new Comparator<File>() {

                        public int compare(File f1, File f2) {

                                String o1 = textorder[f1];

                                String o2 = textorder[f2];

                                if (o1 == null && o2 == null) {

                                        return f1.getName().compareTo(f2.getName());

                                } else if (o1 == null) {

                                        return 1

                                } else if (o2 == null) {

                                        return -1

                                } else {

                                        int c = o1.compareTo(o2);

                                        if (c == 0) return f1.getName().compareTo(f2.getName());

                                        else return c;

                                }

                        }

                });

                //println files

                return files;

        }

        public void start() throws InterruptedException {

                super.start();

                if (isSuccessful) {

                        //declare a local KR

                        //TODO find out how the annotation plugin may hook the import steps

//                        List<String> krnames = importParameters.getKnowledgeRepositoryNames();

//                        if (krnames.size() == 0) {

//                                importParameters.createKnowledgeRepositoryElement("DEFAULT"); // set a default KR shared by all XTZ corpus

//                        } else if (krnames.size() == 1 && krnames.get(0).equals("DEFAULT")) {

//                                // nothing to do

//                        } else {

//                                println("Corpus is using custom Knowledge repositories: "+importParameters.getKnowledgeRepositoryNames());

//                        }

                        //copy sub directories

                        if (isUpdatingCorpus()) {

                        } else {

                                def dirToCopy = ["xsl", "css", "dtd"]

                                println "--- Copying subdirectories $dirToCopy"

                                for (String dir : dirToCopy) {

                                        File origDirectory = new File(this.sourceDirectory, dir)

                                        if (origDirectory.exists()) {

                                                print "."

                                                File copyDirectory = new File(this.binaryDirectory, dir)

                                                FileCopy.copyFiles(origDirectory, copyDirectory)

                                        }

                                }

                                println ""

                        }

                }

        }

        public static void main(String[] args) {

                long start = System.currentTimeMillis()

                String userDir = System.getProperty("user.home")

                Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

                //Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows

                Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));

                //Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows

                Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));

                Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

                Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");

                Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");

                //Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows

                Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(userDir, "TXM"));

                //File importParametersFile = new File("/home/mdecorde/xml/brown-for-xtz/import.xml");

                //                File importParametersFile = new File("/home/mdecorde/xml/qgraalc/qgraal_cw/import.xml"); // new import

                //File importParametersFile = new File("/home/mdecorde/TXM/corpora/QGRAALXTZ/import.xml"); // corpus update

                //                File importParametersFile = new File("/home/mdecorde/xml/annotation/import.xml"); // new import

                //File importParametersFile = new File("/home/mdecorde/xml/baiptest/import.xml"); // new import

                //File importParametersFile = new File("/home/mdecorde/TXM/corpora/BAIP/import.xml"); // corpus update

                File importParametersFile = new File("/home/mdecorde/xml/xtzsmall/import.xml");

                BaseParameters b = new BaseParameters(importParametersFile);

                b.getKeyValueParameters().put(ImportKeys.CLEAN, "false")

                b.getKeyValueParameters().put(ImportKeys.MULTITHREAD, "false")

                b.getKeyValueParameters().put(ImportKeys.DEBUG, "true")

                b.getKeyValueParameters().put(ImportKeys.UPDATECORPUS, "false")

                XTZImport i = new XTZImport(b);

                i.start();

                println "Done: "+i.isSuccessful

                long end = System.currentTimeMillis()

                println "TIME: "+(end-start)

        }

}