Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

//

//

// $LastChangedDate: 2015-12-17 12:11:39 +0100 (jeu. 17 déc. 2015) $

// $LastChangedRevision: 3087 $

// $LastChangedBy: mdecorde $

//

package org.txm.scripts.importer

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Result;

import javax.xml.transform.Source;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import org.txm.utils.io.IOUtils

import org.w3c.dom.Document;

import org.w3c.dom.Element;

import org.w3c.dom.NodeList;

import org.xml.sax.SAXException;

import javax.xml.stream.*;

import java.io.File;

import java.net.URL;

// TODO: Auto-generated Javadoc

/**

 * Extract w tags from a tei file

 * not finished.

 *

 * @author mdecorde

 */

class WExtract

{

        /**

         * Process.

         *

         * @param infile the infile

         * @param outfile the outfile

         * @param max the max

         * @return the java.lang. object

         */

        public process(File infile, File outfile, int max)

        {

                println "Process "+infile.getName()+", keep $max words"

                int count = this.countW(infile);

                if(count < max)

                {

                        println "can't extract $max words, the file "+infile.getName()+" contains only $count words"

                        return;

                }

                String ms = "#ms_K"

                int tier = max/3;

                int from1 = 0

                int to1 = tier;

                int from2 = (count/2) - (tier/2);

                int to2 =(count/2) + (tier/2);

                int from3 = count -tier;

                int to3= count-1;

                boolean isSic = false;

                boolean isW = false;

                boolean isText = false;

                boolean printW = true;

                int wcount=0;

                println " count : "+count

                println "  get from "+from1+" to "+to1

                println "  get from "+from2+" to "+to2

                println "  get from "+from3+" to "+to3

                String localname;

                String prefix;

                InputStream inputData = infile.toURI().toURL().openStream();

                XMLInputFactory inputfactory = XMLInputFactory.newInstance();

                XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);

                XMLOutputFactory factory = XMLOutputFactory.newInstance();

                FileOutputStream output = new FileOutputStream(outfile);

                XMLStreamWriter writer = factory.createXMLStreamWriter(output, "UTF-8");

                writer.writeStartDocument("utf-8", "1.0");

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())

                {

                        if(isText)

                        {

                                if((wcount >= from1 && wcount <= to1 )||

                                (wcount >= from2 && wcount <= to2) ||

                                (wcount >= from3 && wcount <= to3))

                                        printW = true;

                                else

                                        printW = false;

                        }

                        else

                                printW = true;

                        switch (event)

                        {

                                case XMLStreamConstants.START_ELEMENT:

                                        localname = parser.getLocalName();

                                        prefix = parser.getPrefix();

                                /*

                                 if(localname == "supplied")

                                 if(parser.getAttributeValue(null,"source") != null)

                                 ms = parser.getAttributeValue(null,"source")

                                 if(localname == "sic")

                                 {

                                 isSic= true;

                                 }

                                 */

                                        if(localname == "text")

                                                isText = true;

                                        if(localname == "w")

                                        {

                                                isW= true;

                                                wcount++;

                                                if(isText)

                                                {

                                                        if((wcount >= from1 && wcount <= to1 )||

                                                        (wcount >= from2 && wcount <= to2) ||

                                                        (wcount >= from3 && wcount <= to3))

                                                                printW = true;

                                                        else

                                                                printW = false;

                                                }

                                                else

                                                        printW = true;

                                        }

                                /*if(!isSic)

                                 if(localname != "choice" && localname != "corr" && localname != "sic" && localname != "supplied" && localname != "seg")

                                 {*/

                                        if(localname == "w")

                                        {

                                                if(printW)

                                                {

                                                        if(prefix != null && prefix.length() > 0)

                                                                writer.writeStartElement(prefix+":"+localname);

                                                        else

                                                                writer.writeStartElement(localname);

                                                        for(int i= 0 ; i < parser.getAttributeCount() ;i++ )

                                                        {

                                                                if(parser.getAttributePrefix(i)!= "")

                                                                        writer.writeAttribute(parser.getAttributePrefix(i)+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));

                                                                else

                                                                        writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));

                                                        }

                                                        //writer.writeAttribute("srcmf:src", ms);

                                                }

                                        }

                                        else

                                        {

                                                if(prefix != null && prefix.length() > 0)

                                                        writer.writeStartElement(prefix+":"+localname);

                                                else

                                                        writer.writeStartElement(localname);

                                                if(localname == "teiHeader")

                                                {

                                                        writer.writeAttribute("xmlns:me", "http://www.menota.org/ns/1.0");

                                                        writer.writeAttribute("xmlns:bfm", "http://bfm.ens-lsh.fr/ns/1.0");

                                                        //writer.writeAttribute("xmlns:srcmf", "https://listes.cru.fr/wiki/srcmf/index");

                                                }

                                                if(localname == "TEI")

                                                {

                                                        writer.writeAttribute("xmlns","http://www.tei-c.org/ns/1.0");

                                                }

                                                for(int i= 0 ; i < parser.getAttributeCount() ;i++ )

                                                {

                                                        if(parser.getAttributePrefix(i)!= "")

                                                                writer.writeAttribute(parser.getAttributePrefix(i)+":"+parser.getAttributeLocalName(i), parser.getAttributeValue(i));

                                                        else

                                                                writer.writeAttribute(parser.getAttributeLocalName(i), parser.getAttributeValue(i));

                                                }

                                        }

                                //}

                                        break;

                                case XMLStreamConstants.END_ELEMENT:

                                        localname =parser.getLocalName()

                                /*if(localname == "sic")

                                 isSic= false;

                                 if(localname == "w")

                                 isW= false;

                                 if(localname == "supplied" && ms != "#ms_K")

                                 ms = "#ms_K";

                                 if(!isSic)

                                 if(localname != "choice" && localname != "corr" && localname != "sic" && localname != "supplied" && localname != "seg")

                                 {*/

                                        if(localname == "w")

                                        {

                                                if(printW)

                                                {

                                                        writer.writeEndElement();

                                                        writer.writeComment("\n");

                                                }

                                        }

                                        else

                                        {

                                                writer.writeEndElement();

                                                writer.writeCharacters("\n");

                                        }

                                //        }

                                        break;

                                case XMLStreamConstants.CHARACTERS:

                                //if(!isSic)

                                        if(isW)

                                        {

                                                if(printW)

                                                {

                                                        writer.writeCharacters(parser.getText().trim());

                                                }

                                        }

                                        else

                                                writer.writeCharacters(parser.getText().trim());

                                        break;

                        }

                }

                writer.flush();

                writer.close();

                output.close()

                if (parser != null) parser.close();

                if (inputData != null) inputData.close();

        }

        /**

         * Count w.

         *

         * @param infile the infile

         * @return the int

         */

        public int countW(File infile)

        {

                InputStream inputData = infile.toURI().toURL().openStream();

                XMLInputFactory inputfactory = XMLInputFactory.newInstance();

                XMLStreamReader parser = inputfactory.createXMLStreamReader(inputData);

                int count = 0;

                for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next())

                {

                        switch (event)

                        {

                                case XMLStreamConstants.START_ELEMENT:

                                        if(parser.getLocalName() == "w")

                                                count++;

                        }

                }

                if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                return count;

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                String userDir = System.getProperty("user.home");

                File directory = new File(userDir+"/xml/extract/");

                File outdir = new File(userDir+"/xml/extract/","results");

                outdir.mkdir();

                File maxfile = new File(userDir+"/xml/extract/maxfile");

                /*

                 * maxfile format:

                 *

                 * filename1.xml        45000

                 * filename2.xml        22500

                 * filename3.xml        45000

                 */

                HashMap<File, Integer> maxperfile = new HashMap<File, Integer>();

                maxfile.eachLine{it->

                        String[] split = it.split("\t");

                        if(split.length == 2)

                        {

                                try

                                {

                                String filename = it.split("\t")[0];

                                int max = Integer.parseInt(it.split("\t")[1])

                                maxperfile.put(filename, max);

                                }catch(Exception e ){}

                        }

                }

                println maxperfile;

                def files = directory.listFiles(IOUtils.HIDDENFILE_FILTER);

                for (File infile : files) {

                        if (maxperfile.containsKey(infile.getName())) {

                                File outfile = new File(outdir, infile.getName());

                                int max= maxperfile.get(infile.getName());

                                new WExtract().process(infile, outfile, max)

                        }

                }

        }

}