Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

// 

// 

// 

// $LastChangedDate:$

// $LastChangedRevision:$

// $LastChangedBy:$ 

//

package org.txm.scripts.importer

import java.util.Collections;

import java.io.File;

import org.txm.utils.io.FileCopy;

import org.txm.utils.*;

import java.io.File;

import java.util.HashMap;

import java.util.ArrayList;

import java.io.File;

import javax.xml.stream.*;

import java.net.URL;

// TODO: Auto-generated Javadoc

/**

 * The Class HTMLIndexer.

 */

class HTMLIndexer {

        /** The idxprefix. */

        static String idxprefix = "IDX-";

        /** The index. */

        HashMap<String, ArrayList<String>> index = new HashMap<String, ArrayList<String>>();

        /**

         * Process html dir.

         *

         * @param htmlDirectory the html directory

         * @return true, if successful

         */

        private boolean processHTMLDir(File htmlDirectory)

        {

                ArrayList<File> htmlfiles = DeleteDir.scanDirectory(htmlDirectory, true)

                Collections.sort(htmlfiles);

                for(File htmlFile : htmlfiles)//get all indexes

                {

                        if(htmlFile.getName().endsWith(".html"))

                                processHTMLFile(htmlFile);

                }

                ArrayList<String> tokens = new ArrayList<String>(index.keySet());

                Collections.sort(tokens);

                //fix doubles like étiquette&étiquettes

                for(int i = 0 ; i < tokens.size() ; i++)

                {

                        String t1 = tokens.get(i);

                        String t2 = tokens.get(i+1);

                        if(t1.equals(t2.substring(0, t2.length() -1)))

                        {

                                tokens.remove(i+1);

                                index.get(t1).addAll(index.get(t2));

                                //i--;

                        }

                }

                tokens = new ArrayList<String>(index.keySet());

                for(String token : tokens)

                {

                        println("Token: "+token);

                        println(index.get(token));

                }

        }

        /**

         * Process html file.

         *

         * @param htmlFile the html file

         * @return true, if successful

         */

        private boolean processHTMLFile(File htmlFile)

        {

                String lasttoken;

                String page;

                def inputData = null;

                def factory = null;

                try

                {

                        URL url = htmlFile.toURI().toURL();

                        println "process html file "+url;

                        inputData = url.openStream();

                        factory = XMLInputFactory.newInstance();

                        //factory.setXMLResolver resolver

                        //factory.setProperty(factory.IS_VALIDATING, false)

                        factory.setProperty("javax.xml.stream.supportDTD", false);

                        factory.setProperty("javax.xml.stream.isReplacingEntityReferences", false);

                        XMLStreamReader parser = factory.createXMLStreamReader(inputData);

                        for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) 

                        {

                                //println "parse"

                                switch (event) 

                                {

                                        case XMLStreamConstants.START_ELEMENT:

                                                //println "elem "+parser.getLocalName()

                                                if(parser.getLocalName() == "div")

                                                        if(parser.getAttributeValue(null, "id") != null && parser.getAttributeValue(null, "id").startsWith("index-body"))

                                                                page = parser.getAttributeValue(null, "id")

                                                String id = parser.getAttributeValue(null, "id");

                                                if(id != null && id.startsWith(idxprefix))

                                                {

                                                        if(!index.containsKey(lasttoken))

                                                                index.put(lasttoken, new ArrayList<String>());

                                                        index.get(lasttoken).add(htmlFile.getName()+"#"+id)

                                                }

                                                break;

                                        case XMLStreamConstants.CHARACTERS:

                                                String text = parser.getText().trim();

                                                if(text.length() > 0)

                                                {

                                                        def texts = text.split(" ");

                                                        lasttoken = texts[texts.size()-1];

                                                        if(lasttoken.endsWith("."))

                                                                lasttoken = lasttoken.substring(0, lasttoken.length() -1)

                                                }

                                }

                        }

                }

                catch(Exception e){println("File "+htmlFile+"\n"+e); 

                        if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                return false;}

                if (parser != null) parser.close();

                if (inputData != null) inputData.close();

                return true;

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args)

        {

                File htmlDirectory = new File("/home/mdecorde/xml/html")

                new HTMLIndexer().processHTMLDir(htmlDirectory);

        }

}