/ - Diff - Plateforme TXM - Forge du Centre Blaise Pascal

     import java.util.HashMap;
     import org.txm.scripts.importer.*;
     import org.txm.utils.*;
     import org.txm.utils.io.*
     import org.txm.objects.Project;
     import org.txm.metadatas.*;
-...
     	/** The outdir. */
     	File txmDir;
     	File binDir;
     	public boolean doTokenizeStep = true;
     	/** The trsfiles. */
     	ArrayList<String> trsfiles;
-...
     		this.metadatas = metadatas;
     		this.lang = lang;
     		this.project = project;
     		this.doTokenizeStep = project.getDoTokenizerStep()
+    	}
     	/**
-...
     		cpb.done()
     		if (metadatas != null) {
     			if (metadatas.getHeadersList().size() > 0) {
     				println "Injecting metadata "+metadatas.getHeadersList()+" in "+trsfiles.size()+" files"
-...
     		println "Tokenizing entities "+files.length+" files"
     		cpb = new ConsoleProgressBar(files.length)
     		for (File pfile : files) {
     			cpb.tick()
     			TokenizeEntities tokenizer = new TokenizeEntities(pfile.toURI().toURL());
     			File outfile = File.createTempFile("tok", ".xml", pfile.getParentFile());
-...
     		cpb.done()
     		//TOKENIZE
     		println "Tokenizing "+files.length+" files from $txmDir"
     		File tokenizedDir = new File(binDir, "tokenized")
     		tokenizedDir.deleteDir() // delete previous outputed files
     		tokenizedDir.mkdir()
     		cpb = new ConsoleProgressBar(files.length)
     		for (File pfile : files) {
     			cpb.tick()
     			String filename = pfile.getName().substring(0, pfile.getName().length()-4)
     			File tfile = new File(tokenizedDir, pfile.getName())
     			try {
     				TranscriberTokenizer tokenizer = new TranscriberTokenizer(pfile, tfile, lang)
     				tokenizer.setRetokenize("true" == project.getTokenizerParameter("doRetokenizeStep", "false"))
     				tokenizer.setDoBuildWordIDs("true" == project.getTokenizerParameter("doBuildWordIds", "true"))
     				if (!tokenizer.process()) {
     					println("Failed to tokenize "+pfile)
     		if (!doTokenizeStep) {
     			println "No tokenization do to."
     			cpb = new ConsoleProgressBar(files.length)
     			for (File f : files) {
     				File outfile = new File(tokenizedDir, f.getName());
     				FileCopy.copy(f, outfile);
     				cpb.tick()
+    			}
     			cpb.done()
     		} else {
     			println "Tokenizing "+files.length+" files from $txmDir"
     			cpb = new ConsoleProgressBar(files.length)
     			for (File pfile : files) {
     				cpb.tick()
     				String filename = pfile.getName().substring(0, pfile.getName().length()-4)
     				File tfile = new File(tokenizedDir, pfile.getName())
     				try {
     					TranscriberTokenizer tokenizer = new TranscriberTokenizer(pfile, tfile, lang)
     					tokenizer.setRetokenize("true" == project.getTokenizerParameter("doRetokenizeStep", "false"))
     					tokenizer.setDoBuildWordIDs("true" == project.getTokenizerParameter("doBuildWordIds", "true"))
     					if (!tokenizer.process()) {
     						println("Failed to tokenize "+pfile)
+    					}
     				} catch (Exception e) {
     					println "Error tokenizer: "+pfile
     					org.txm.utils.logger.Log.printStackTrace(e);
     					return false;
+    				}
     			} catch (Exception e) {
     				println "Error tokenizer: "+pfile
     				org.txm.utils.logger.Log.printStackTrace(e);
     				return false;
+    			}
     			cpb.done()
+    		}
     		cpb.done()
     		//TRANSFORM INTO XML-TEI-TXM
     		files = tokenizedDir.listFiles()
     		println("Building ${files.length} XML-TXM file"+(files.length > 1?"s":""))
     		cpb = new ConsoleProgressBar(files.length)
     		for (File tfile : files) {
     			cpb.tick()
     			String filename = tfile.getName().substring(0, tfile.getName().length()-4)
     			File xmlfile = new File(txmDir, tfile.getName())

     		if (MONITOR != null) MONITOR.worked(20)
     		println "-- Xml Validation"
     		if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
     		if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
     		ConsoleProgressBar cpb = new ConsoleProgressBar(txmDir.listFiles().length)
     		for (File infile : txmDir.listFiles()) {
     			if (!ValidateXml.test(infile)) {
     				println "$infile : Validation failed";
     				infile.delete();
+    			}
+    		}
+    			}
     			cpb.tick()
+    		}
     		cpb.done()
     		if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }
     		if (MONITOR != null) MONITOR.worked(20, "ANNOTATE")
     		boolean annotationSuccess = false;
     		if (annotate) {
     			println "-- ANNOTATE - Running NLP tools"

     					if (correspType.containsKey(type)) { // check if txm:ana
     						String corresptype = correspType.get(type);
     						String ref = correspRef.get(type);
     						anabalises.add([
     							"#"+ref,
     							"#"+corresptype,
     							value
     						]);
     						anabalises.add(["#"+ref, "#"+corresptype, value]);
     					} else if (type == ID) { // keep id attribute
     						String wordid = value
     						if (wordid.startsWith("w")) {

Laboratoire ICAR » Plateforme TXM

Révision 3345