Laboratoire ICAR » Plateforme TXM

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

// 

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

// 

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

// 

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

// This file is part of the TXM platform.

//

// The TXM platform is free software: you can redistribute it and/or modif y

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// The TXM platform is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with the TXM platform.  If not, see <http://www.gnu.org/licenses/>.

// 

// 

// 

// $LastChangedDate: 2017-06-26 16:53:47 +0200 (lun. 26 juin 2017) $

// $LastChangedRevision: 3451 $

// $LastChangedBy: mdecorde $ 

//

package org.txm.scripts.importer;

import java.io.BufferedWriter;

import java.io.File;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLStreamReader

import org.txm.utils.io.IOUtils;

// TODO: Auto-generated Javadoc

/**

 * remove all ctrl char and replace \t by ' '.

 *

 * @param infile : the file to process

 * @param encoding : the encoding of the file

 */

class CleanFile {

        public CleanFile(File infile, String encoding){

                Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(infile),encoding));

                File outfile = File.createTempFile("filecleaner","123456", infile.getParentFile());

                Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile), "UTF-8"));

                String line = reader.readLine();

                while(line != null) {

                        writer.write(clean(line) +"\n")

                        line = reader.readLine();

                }

                writer.close();

                reader.close();

                infile.delete(); // erase old file

                outfile.renameTo(infile); // rename the copy

        }

        def static ctrl_reg = /[\p{Cntrl}&&[^\n\t]]/ // all ctrl but \n

        def static surrogate = "[^\\p{P}&&[^\\u0001-\\uFFFF]]"

        def static surrogate_reg = /$surrogate/

        def static surrogate_punct = "[\\p{P}&&[^\\u0001-\\uFFFF]]"

        def static surrogate_punct_reg = /$surrogate_punct/

        def static SURROGATE = "_"

        def static SURROGATE_PUNCT = "☒"

        def static EMPTY = ""

        def static SPACE = " "

        def static TAB = "\t"

        /**

         * Remove ctrl and surrogates chars from a String. except tabs

         *

         * @param str the str

         * @return the cleaned string

         */

        public static String cleanAllButTabs(String str) {

                str = str.replaceAll(ctrl_reg, EMPTY); // remove all ctrl chars

                str = str.replaceAll(surrogate_reg, SURROGATE); // replace all surrogate chars

                str = str.replaceAll(surrogate_punct_reg, SURROGATE_PUNCT); // replace all surrogate punct chars

                return str;

        }

        /**

         * Remove ctrl and surrogates chars from a String.

         *

         * @param str the str

         * @return the cleaned string

         */

        public static String clean(String str) {

                str = str.replace(TAB, SPACE); // replace \t by ' '

                str = str.replaceAll(ctrl_reg, EMPTY); // remove all ctrl chars

                str = str.replaceAll(surrogate_reg, SURROGATE); // replace all surrogate chars

                str = str.replaceAll(surrogate_punct_reg, SURROGATE_PUNCT); // replace all surrogate punct chars

                return str;

        }

        public static void removeSurrogateFromXmlFile(File file, File outputFile) {

                FileReader reader = new FileReader( file );

                XMLStreamReader xmlStreamReader = XMLInputFactory.newInstance().createXMLStreamReader( reader );

                String fileEncoding = xmlStreamReader.getEncoding();

                xmlStreamReader.close();

                reader.close();

                if (fileEncoding == null) fileEncoding = "UTF-8"; //$NON-NLS-1$

                String text = IOUtils.getText(file, fileEncoding)

                text = CleanFile.clean(text);

                IOUtils.write(outputFile, text)

        }

        /**

         * The main method.

         *

         * @param args the arguments

         */

        public static void main(String[] args){

                File infile = new File("C:\\Documents and Settings\\H\\xml\\cleaner\\test.txt")                

                String encoding = "CP1252";

                new CleanFile(infile, encoding);

        }

}