Laboratoire ICAR » Plateforme TXM

import org.w3c.tidy.Tidy

//	Tidy tidy = new Tidy(); // obtain a new Tidy instance

//	tidy.setXHTML(true); // set desired config options using tidy setters

//	tidy.setInputEncoding("UTF-8")

//	tidy.setOutputEncoding("UTF-8")

//	tidy.setShowErrors(0)

//	tidy.setShowWarnings(false)

package org.txm.macro.misc

import groovy.xml.QName

import java.nio.charset.Charset

import java.text.DecimalFormat

import org.txm.utils.xml.DomUtils;

import org.txm.importer.ValidateXml;

import groovy.util.XmlParser

@Field @Option(name="rootDir", usage="The directory contains the 'orig' directory which contains the html files", widget="Folder", required=false, def="/path")

File rootDir = new File("");

@Field @Option(name="encoding", usage="HTML encoding", widget="String", required=false, def="iso-8859-1")

String encoding = "iso-8859-1" // HTML files encoding

@Field @Option(name="debug", usage="show debug messages. Values = true, false", widget="String", required=false, def="false")

def debug = "true" // set true to debug the script

debug = ("true" == debug)

String corpusName = rootDir.getName()

File srcDir = new File(rootDir, "orig");

File outDir = new File(rootDir, "xhtml");

File outDir2 = new File(rootDir, corpusName);

File rejected = new File(rootDir, "duplicates");

File tmpDir = new File(rootDir, "tmp");

if (!srcDir.exists()) {

	println "STOP, srcDir does not exists $srcDir"

	return;

}

outDir.deleteDir()

outDir.mkdir()

outDir2.deleteDir()

outDir2.mkdir()

rejected.deleteDir()

rejected.mkdir()

tmpDir.deleteDir()

tmpDir.mkdir()

def allTags = new HashSet<String>();

def allStyles = new HashSet<String>();

def allClasses = new HashSet<String>();

def newPrefix = "Numéro de document : "

int itext = 1;

def formater = new DecimalFormat("0000");

int LIMITDIFF = 10

def metadatas = ["DocPublicationName", "DocHeader"]

def files = []

srcDir.eachFileMatch(~/.*\.html/){ htmlFile -> files << htmlFile}

files = files.sort()

def done = new HashSet<String>();

def ignored = []

def allTitles = [:]

def dones = [:]

def ignoreds = []

def getText(def node) {

	//if (debug) println "node: "+node

	String s = " ";

	if (node instanceof String) {

		s += " "+node

	} else {

		for (def c : node.children()) {

			s += " "+getText(c)

		}

	}

	//println " "+s.replace("\n", " ").trim();

	return " "+s.replace("\n", " ").trim();

}

println "Nb of HTML files: "+files.size()

for (File htmlFile : files) {

	println "Processing file $htmlFile"

	File tmpHTML = new File(tmpDir, htmlFile.getName())

	tmpHTML.withWriter("UTF-8") { writer -> 

		String txt = htmlFile.getText(encoding)

		txt = txt.replaceAll("<p></p>", " ");

		txt = txt.replaceAll("<p> </p>", " ");

		txt = txt.replaceAll("<br>", "<br> ");

		writer.write(txt)

	}

	String name = htmlFile.getName()

	name = name.substring(0, name.lastIndexOf("."));

	File xhtmlFile = new File(outDir, name+".xhtml")

	xhtmlFile.withWriter("UTF-8") { out ->

		def doc = org.jsoup.Jsoup.connect(tmpHTML.toURI().toURL().toString());

		println "current charset: "+doc.charset()

		doc.charset(Charset.forName("UTF-8"))

		println "current charset: "+doc.charset()

		out.println(doc.outerHtml​())

	}

	if (ValidateXml.test(xhtmlFile)) {

		def root = new XmlParser().parse(xhtmlFile)

		def tables = root.body.table.tbody.tr.td

		if (tables.size() == 0) tables = root.body.table.tr.td

		//println "Nb of txt : "+tables.size()

		for (def text : tables) {

			String sign = ""

			if (! text instanceof groovy.util.Node) { println "NOT NODE: "+text; continue; }

			//println "TEXT "

			//text.setName("text")

			boolean endOfText = false;

			def textMetadatas = [:]

			for (String metadata : metadatas) {

				textMetadatas[metadata] = ""

			}

			for (def subtable : text.table) text.remove(subtable)	

			for (def p : text.table.p) p.addChild(" ")		

			for (def child : text.span) {

				if ("color:red; font-weight:bold".equals(child.@style)) {

					//text.remove(child)

					if (debug) "Found bold: "+child

					child.replaceNode { node -> w(expFound: "y", child.text())}

				}

			}

			def startIgnoringText = false

			def tmp =""

			def ichar = 0

			String title = "";

			def ignoredText = ""

			def children = text.children()

			for (int idx = 0 ; idx < children.size() ; idx++) {

				def child  = children[idx]

				if (debug) println "child: $child"

				if (startIgnoringText) {

					if (debug) println "Ignoring text : "+ignoredText

					if (child instanceof String) ignoredText += child

					else ignoredText += child.text()

					def t = text.children().remove(idx);

					//if (tmp.length() > 0) println "removing : "+t

					idx--

					continue; // next child

				}

				if (child instanceof String) {

					//println " "+child

					ichar += child.length()

				} else {

					ichar += child.text().length()

					//		allTags.add(child.name().getLocalPart())

					//		allClasses.add(child.@class)

					//		allStyles.add(child.@style)

					def nn = child.name()

					try {nn = nn.getLocalPart()} catch(Exception e) {}

					switch (nn) {

						case "br": break;

						case "span":

						if (debug) println "Found span $child"

							String classV = child.@class

							String style = child.@style

							if (classV != null) {

								if (metadatas.contains(classV)) {

									textMetadatas[classV] = (textMetadatas[classV] + " " + child.text()).trim()

									text.remove(child);

									idx--

									//println "METADATA: "+classV + " = "+child.text().trim().length();

								} else if ("TitreArticleVisu" == classV) {

									title += getText(child)//.text().trim().replaceAll("\n", " ")+ " "

									child.name = new QName(child.name().getNamespaceURI(), "head", child.name().getPrefix())

								} else {

									println "UNKNOWED CLASS: "+classV + " = "+child.text().trim().length();

								}

							} else if (style != null) {

								if ("color:red; font-weight:bold".equals(style)) {

									//child.replaceNode { node -> w(expFound: "test")	}

									//println "KEYWORD: "+child.text().trim();

								} else {

									println "UNKNOWED STYLE: '"+style+"' = "+child.text().trim();

								}

							} else {

								println "UNKNOWED SPAN: "+child.text().trim();

							}

							break;

						case "a": break

						case "w": break;

						case "b":

							startIgnoringText = true;

							tmp = child.text()

							//if (tmp.length() > 0) println "START REMOVING FROM : "+tmp

							text.remove(child);

							idx--

							break;

						case "i": break;

						case "font":

							if (debug) println "Found font $child"

							String style = child.@style

							if ("font-style:italic;" == style) {

								if (debug) println "ITALIC: "+getText(child).trim();

								child.replaceNode { node -> i(getText(child))}

							} else if ("font-weight:bold;") {

								if (debug) println "BOLD: "+getText(child).trim();

								child.replaceNode { node -> b(getText(child))}

							} else {

								println "FSTYLE: '"+style+"' = "+getText(child).trim();

							}

							break;

						default: println child.name()

					}

				}

			}

			//rename td to text

			text.name = new QName(text.name().getNamespaceURI(), "text", text.name().getPrefix())

			//Write metadatas

			for( String metadata : metadatas) {

				text.attributes().put(metadata, textMetadatas[metadata])

				//sign+= " "+textMetadatas[metadata].trim()

			}

			// get document number

			ignoredText = ignoredText.replaceAll("\n", " ")

			int iNo = ignoredText.indexOf(newPrefix);

			//println ignoredText

			if (iNo >= 0) {

				String no = ignoredText.substring(iNo+newPrefix.length()).trim()

				text.attributes().put("idnews", no)

				//sign += " "+no

				text.attributes().put("date", no.substring(5,9)+"-"+no.substring(9,11)+"-"+no.substring(11,13))

				text.attributes().put("yyyymmdd", no.substring(5,13))

				text.attributes().put("yyyymm", no.substring(5,11))

				text.attributes().put("yyyy", no.substring(5,9))

				text.attributes().put("mm", no.substring(9,11))

				text.attributes().put("dd", no.substring(11,13))

			}

			//sign += " "+ichar

			sign += " "+title

			if (allTitles[title] == null) allTitles[title] = ichar

			if (Math.abs(allTitles[title] - ichar) > LIMITDIFF) {

				sign += " "+ichar

			}

			File xmlFile;

			if (done.contains(sign)) {

				ignored << sign

				xmlFile = new File(rejected, name+"_"+formater.format((itext++))+".xml")

				ignoreds << xmlFile.getName()

			} else {

				done << sign;

				xmlFile = new File(outDir2, name+"_"+formater.format((itext++))+".xml")

				dones[sign] = xmlFile

			}

			def writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(xmlFile) , "UTF-8")); //$NON-NLS-1$

				writer.println "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"

				new XmlNodePrinter(writer).print(text)

		}

	}

}

if (ignored.size() > 0) {

	File ignoredFile = new File (rejected, "ignored.txt");

	ignoredFile.withWriter("UTF-8") { writer ->

		writer.println "TOTAL: "+ignored.size()

		for (int i = 0 ; i < ignored.size() ; i++) {

			def sign = ignored[i]

			writer.println "\n**DUPLICATE\n "

			writer.println "keeped="+dones[sign];

			writer.println "rejected="+ignoreds[i];

			writer.println "SIGN="+sign

			writer.println "\n"

		}

	}

	println "TOTAL IGNORED: "+ignored.size()

}

println "TOTAL TEXT: $itext"

if (!debug) {

	outDir.deleteDir()

	tmpDir.deleteDir()

}