/ - Diff - Plateforme TXM - Forge du Centre Blaise Pascal

Révision 2473

     package org.txm.importer.xtz;
     import java.io.File;
     import java.io.FileFilter;
     import java.util.ArrayList;
     import java.util.Arrays;
     import java.util.Collections;
     import java.util.List;
     import java.util.logging.Level;
-...
     import org.txm.utils.logger.Log;
     public class ImportModule {
     	public Project project;
     	public String corpusVersionProduced;
     	public File sourceDirectory;
     	public File binaryDirectory;
     	public Importer importer;
     	public Annotater annotater;
     	public Compiler compiler;
     	public Pager pager;
     	/**
     	 * set the variable to false to stop the import process at next step
     	 */
     	public boolean isSuccessful = true;
     	public String reason = "none";
     	public boolean debug = false;
     	public boolean multithread = false;
     	public boolean updateCorpus = false;
     	public String corpusName;
     	IProgressMonitor monitor;
     	public void setMonitor(IProgressMonitor monitor) {
     		this.monitor = monitor;
+    	}
     	public boolean isMultiThread() {
     		return multithread;
+    	}
     	public boolean isDebugging() {
     		return debug;
+    	}
     	public ImportModule(Project p) {
     		init(p);
+    	}
     	public boolean isUpdatingCorpus() {
     		return updateCorpus;
+    	}
     	public void init(Project p) {
     		this.project = p;
     		corpusName = project.getName();
     		//this.debug = "true".equals(project.getKeyValueParameters().get(ImportKeys.DEBUG));
     		// this.debug = "true".equals(project.getKeyValueParameters().get(ImportKeys.DEBUG));
     		if (Log.getLevel().intValue() < Level.INFO.intValue()) {
     			debug = true;
+    		}
     		this.multithread = project.getDoMultiThread();
     		this.updateCorpus = project.getDoUpdate();
     		this.sourceDirectory = project.getSrcdir();
     		this.binaryDirectory = project.getProjectDirectory();
     		if (!updateCorpus) { // clean directories only if it's a new import
     		if (!updateCorpus) { // clean directories only if it's a new import
     			File txmDir = new File(binaryDirectory, "txm");
     			//DeleteDir.deleteDirectory(binaryDirectory);
     			// DeleteDir.deleteDirectory(binaryDirectory);
     			try {
     				p.getRCPProject().getFolder("HTML").delete(true, new LogMonitor("XTZ delete project content"));
     				p.getRCPProject().getFolder("cqp").delete(true, new LogMonitor("XTZ delete project content"));
-...
     				DeleteDir.deleteDirectory(new File(binaryDirectory, "registry"));
     				DeleteDir.deleteDirectory(new File(binaryDirectory, "tokenized"));
     				DeleteDir.deleteDirectory(new File(binaryDirectory, "txm"));
     			} catch (CoreException e) {
+    			}
     			catch (CoreException e) {
     				e.printStackTrace();
+    			}
     			txmDir.mkdirs();
+    		}
+    	}
     	public void start() throws InterruptedException {
     		binaryDirectory.mkdirs(); // ensure output exists
     		//System.out.println("ImportModule.start");
     		// System.out.println("ImportModule.start");
     		if (!updateCorpus) { // create XML-TXM files and annotate
     			System.out.println(TXMCoreMessages.creatingCorpus);
     			if (importer != null) {
     				//System.out.println("ImportModule.start: importer: "+importer);
     				// System.out.println("ImportModule.start: importer: "+importer);
     				if (monitor != null) monitor.subTask("-- IMPORTER - Reading source files");
     				System.out.println("-- IMPORTER - Reading source files");
     				importer.process();
     				//importer.checkFiles();
     				// importer.checkFiles();
     				isSuccessful = isSuccessful & importer.isSuccessFul();
     				if (!isSuccessful) {
     					System.out.println("Error while importing corpus during 'importer' step, reason="+importer.getReason());
     					System.out.println("Error while importing corpus during 'importer' step, reason=" + importer.getReason());
     					return;
+    				}
     			} else {
     				System.out.println("XML-TXM files already produced in "+new File(binaryDirectory, "txm/"+corpusName));
+    			}
     			else {
     				System.out.println("XML-TXM files already produced in " + new File(binaryDirectory, "txm/" + corpusName));
+    			}
     			//System.out.println("GET FILES ORDER");
     			// System.out.println("GET FILES ORDER");
     			final List<String> orderedTextIDs = getTXMFilesOrder();
     			//declare in the right order the new texts produced in the "txm" directory
     			// declare in the right order the new texts produced in the "txm" directory
     			for (File build : new File(binaryDirectory, "txm").listFiles()) {
     				if (!build.isDirectory()) continue;
     				for (String name : orderedTextIDs) {
     					File xmltxmFile = new File(build, name+".xml");
     					File xmltxmFile = new File(build, name + ".xml");
     					if (xmltxmFile.isDirectory()) continue;
     					if (xmltxmFile.isHidden()) continue;
     					if (project.getText(name) != null) {
     					if (project.getText(name) == null) { // if text does not exists create it
     						Text t = new Text(project);
     						t.setName(name);
     						t.setTXMFile(xmltxmFile);
-...
+    					}
+    				}
+    			}
     			boolean annotate = project.getAnnotate();
     			if (annotate && annotater != null) {
     				if (monitor != null) monitor.subTask("-- ANNOTATE - Running NLP tools");
-...
     				annotater.process();
     				isSuccessful = isSuccessful & annotater.isSuccessFul();
     				if (!isSuccessful) {
     					System.out.println("Error while importing corpus during 'annotate' step, reason="+annotater.getReason());
     					System.out.println("Error while importing corpus during 'annotate' step, reason=" + annotater.getReason());
     					return;
+    				}
     			} else {
     				//System.out.println("XML-TXM files already annotated.");
+    			}
     		} else { // updating the corpus
     			else {
     				// System.out.println("XML-TXM files already annotated.");
+    			}
+    		}
     		else { // updating the corpus
     			System.out.println(TXMCoreMessages.updatingCorpus);
     			// fixing Text XML-TXM configurations
     			for (Text text : project.getTexts()) {
     				File f = text.getXMLTXMFile();
     				if (f == null || !f.exists()) { // ensure the XML-TXM file path is set
     					f = new File(project.getProjectDirectory(), "txm/"+project.getName()+"/"+text.getName()+".xml");
     					f = new File(project.getProjectDirectory(), "txm/" + project.getName() + "/" + text.getName() + ".xml");
     					text.setTXMFile(f);
+    				}
+    			}
+    		}
     		// XML-TXM files are ready to be compiled
     		final List<String> orderedTextIDs = getTXMFilesOrder();
     		Thread Tcompiler = new Thread("XTZ Compiler - "+project.getSrcdir().getName()) {
     		Thread Tcompiler = new Thread("XTZ Compiler - " + project.getSrcdir().getName()) {
     			@Override
     			public void run() {
     				if (compiler != null) {
     					if (monitor != null) monitor.subTask("-- COMPILING - Building Search Engine indexes");
     					if (monitor != null) monitor.subTask("-- COMPILING - Building Search Engine indexes");
     					System.out.println("-- COMPILING - Building Search Engine indexes");
     					compiler.process(orderedTextIDs);
     					isSuccessful = isSuccessful & compiler.isSuccessFul();
     					if (!isSuccessful) {
     						System.out.println("Error while importing corpus during 'compiler' step, reason="+compiler.getReason());
     						System.out.println("Error while importing corpus during 'compiler' step, reason=" + compiler.getReason());
     						return;
+    					}
     				} else {
+    				}
     				else {
     					System.out.println("No CQP index created.");
+    				}
+    			}
     		};
     		Thread Tpager = new Thread("XTZ Pager - "+project.getSrcdir().getName()) {
     		Thread Tpager = new Thread("XTZ Pager - " + project.getSrcdir().getName()) {
     			@Override
     			public void run() {
     				if (pager != null) {
     					if (monitor != null) monitor.subTask("-- EDITION - Building editions");
-...
     					pager.process(orderedTextIDs);
     					isSuccessful = isSuccessful & pager.isSuccessFul();
     					if (!isSuccessful) {
     						System.out.println("Error while importing corpus during 'pager' step, reason="+pager.getReason());
     						System.out.println("Error while importing corpus during 'pager' step, reason=" + pager.getReason());
     						return;
+    					}
     				} else {
+    				}
     				else {
     					System.out.println("No edition produced.");
+    				}
+    			}
     		};
     		Tcompiler.start();
     		if (!multithread) { //  && !updateCorpus
     		if (!multithread) { // && !updateCorpus
     			Tcompiler.join(); // wait for the end if not multithreaded
     			if (!isSuccessful) { // don't call pager is compiler step failed
     				return;
+    			}
+    		}
     		Tpager.start();
     		if (multithread) Tcompiler.join(); // wait for both threads to end
     		Tpager.join();
     		if (isSuccessful) { // all done TODO remove this code when Text._compute() will be implemented
     			for (Text t : project.getTexts()) {
     				t.setDirty(false);
-...
     			project.setDoUpdate(false);
+    		}
+    	}
     	protected List<String> getTXMFilesOrder() {
     		//		//System.out.println("DEFAULT FILES ORDER");
     		//		File txmDirectory = new File(binaryDirectory, "txm/"+corpusName);
     		//		ArrayList<File> files = new ArrayList<File>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {
     		//			@Override
     		//			public boolean accept(File file) {
     		//				return file.isFile() && file.getName().endsWith(".xml");
     		//			}
     		//		})));
     		//
     		//		Collections.sort(files);
     		return project.getTextsID();
     		//		return files;
     		// //System.out.println("DEFAULT FILES ORDER");
     		File txmDirectory = new File(binaryDirectory, "txm/" + corpusName);
     		ArrayList<File> files = new ArrayList<>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {
     			@Override
     			public boolean accept(File file) {
     				return file.isFile() && file.getName().endsWith(".xml");
+    			}
     		})));
     		Collections.sort(files);
     		ArrayList<String> ids = new ArrayList<>();
     		for (File f : files) {
     			String name = f.getName();
     			ids.add(name.substring(0, name.length() - 4));
+    		}
     		return ids;
     		// return project.getTextsID();
     		// return files;
+    	}
     	public void end() {
     		File paramFile = new File(binaryDirectory, "import.xml");
     		try {
     			//DomUtils.save(project.root.getOwnerDocument(), paramFile);
     			// DomUtils.save(project.root.getOwnerDocument(), paramFile);
     			project.saveParameters(true);
     			isSuccessful = true;
     		} catch (Exception e) {
+    		}
     		catch (Exception e) {
     			// TODO Auto-generated catch block
     			e.printStackTrace();
     			isSuccessful = false;
+    		}
+    	}
     	public String getCorpusName() {
     		return corpusName;
+    	}
     	public String getReason() {
     		return reason;
+    	}
     	public boolean isSuccessFul() {
     		return isSuccessful;
+    	}
     	public Project getProject() {
     		return project;
+    	}
     	public File getSourceDirectory() {
     		return sourceDirectory;
+    	}
     	public File getBinaryDirectory() {
     		return binaryDirectory;
+    	}
     	public void process() throws InterruptedException {
     		start();
     		if (isSuccessful)
     			end();
+    	}
     	public static void main(String[] args) {
     		//		File projectFile = new File("/home/mdecorde/xml/brown/import.xml");
     		// File projectFile = new File("/home/mdecorde/xml/brown/import.xml");
     		//
     		//		ImportModule module = new ImportModule(projectFile);
     		//		System.out.println("Parameters: "+module.getParameters());
     		//		try {
     		//			module.start();
     		// ImportModule module = new ImportModule(projectFile);
     		// System.out.println("Parameters: "+module.getParameters());
     		// try {
     		// module.start();
     		//
     		//			if (module.isSuccessful) {
     		//				System.out.println("Import sucessful. reloading corpora...");
     		//			} else {
     		//				System.out.println("Import failed, reason = "+module.getReason());
     		//			}
     		//		} catch (Exception e) {
     		//			e.printStackTrace();
     		//		}
     		// if (module.isSuccessful) {
     		// System.out.println("Import sucessful. reloading corpora...");
     		// } else {
     		// System.out.println("Import failed, reason = "+module.getReason());
     		// }
     		// } catch (Exception e) {
     		// e.printStackTrace();
     		// }
+    	}
+    }

      * @author mdecorde
      */
     public class ValidateXml {
     	/**
     	 * @throws XMLStreamException test a String, not implemented.
+    	 *
-...
     			XMLInputFactory factory = XMLInputFactory.newInstance();
     			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
     			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+    			}
     			parser.close();
     			inputData.close();
     		} catch (IOException e) {
+    		}
     		catch (IOException e) {
     			// TODO Auto-generated catch block
     			org.txm.utils.logger.Log.printStackTrace(e);
     			return false;
+    		}
     		return true;
+    	}
     	/**
     	 * test a content stored in a String, not implemented.
+    	 *
-...
     	public static boolean test(String xmldata) {
     		return testAndGetError(xmldata).length() == 0;
+    	}
     	/**
     	 * test a content stored in a String, not implemented.
+    	 *
-...
     		try {
     			Node records = new XmlParser().parseText(xmldata);
     			return "";
     		} catch (Exception e) {
+    		}
     		catch (Exception e) {
     			System.out.println(e.getLocalizedMessage());
     			return e.toString();
+    		}
+    	}
     	/**
     	 * test a file.
+    	 *
-...
     			XMLInputFactory factory = XMLInputFactory.newInstance();
     			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
     			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
+    			}
     			parser.close();
     			inputData.close();
     			return true;
     		} catch (Exception e) {
+    		}
     		catch (Exception e) {
     			System.out.println(e.getLocalizedMessage());
     			return false;
+    		}
+    	}
     	/**
     	 * small test to verify a file is a XML-TEI file.
+    	 *
     	 * @param infile
     	 *            the file to test
     	 * @return false if the file is not valid
     	 */
     	public static boolean teiTest(File infile) {
     		if (infile.isDirectory()) {
     			System.out.println(NLS.bind(TXMCoreMessages.xmlValidationColonP0IsADirectory, infile));
     			return false;
+    		}
     		if (!infile.exists()) {
     			System.out.println(NLS.bind(TXMCoreMessages.xmlValidationColonP0DoesNotExists, infile));
     			return false;
+    		}
     		if (!infile.canRead()) {
     			System.out.println(NLS.bind(TXMCoreMessages.xmlValidationColonP0IsNotReadable, infile));
     			return false;
+    		}
     		try {
     			URL url = infile.toURI().toURL();
     			InputStream inputData = url.openStream();
     			XMLInputFactory factory = XMLInputFactory.newInstance();
     			XMLStreamReader parser = factory.createXMLStreamReader(inputData);
     			int n = 0;
     			int tei = -1;
     			int text = -1;
     			int teiHeader = -1;
     			for (int event = parser.next(); event != XMLStreamConstants.END_DOCUMENT; event = parser.next()) {
     				if (event == XMLStreamConstants.START_ELEMENT) {
     					String name = parser.getLocalName();
     					if ("TEI".equals(name)) {
     						tei = n++;
+    					}
     					else if ("text".equals(name)) {
     						text = n++;
+    					}
     					else if ("teiHeader".equals(name)) {
     						teiHeader = n++;
+    					}
+    				}
+    			}
     			parser.close();
     			inputData.close();
     			boolean test = n == 3 && tei < teiHeader && teiHeader < text;
     			if (test) {
     				return true;
+    			}
     			else {
     				if (n != 3) {
     					if (tei == -1) {
     						System.out.println(infile.getName() + ": No 'TEI' tag found");
+    					}
     					if (teiHeader == -1) {
     						System.out.println(infile.getName() + ": No 'teiHeader' tag found");
+    					}
     					if (text == -1) {
     						System.out.println(infile.getName() + ": No 'text' tag found");
+    					}
+    				}
     				else {
     					System.out.println(infile.getName() + ": Malformed TEI XML");
+    				}
     				return false;
+    			}
+    		}
     		catch (Exception e) {
     			System.out.println(e.getLocalizedMessage());
     			return false;
+    		}
+    	}
     	/**
     	 * The main method.
+    	 *
     	 * @param args
     	 *            the arguments
     	 */
     	public static void main(String[] args) {
     		if (ValidateXml.test(new File("~/TXM/corpora/discours/txm/01_DeGaulle.xml"))) //$NON-NLS-1$
     			System.out.println("OK"); //$NON-NLS-1$
     		else
     			System.out.println("ERROR"); //$NON-NLS-1$
     		for (File file : new File(System.getProperty("user.home"), "xml/teierrors").listFiles()) {
     			if (ValidateXml.teiTest(file)) {
     				System.out.println("OK: " + file); //$NON-NLS-1$
+    			}
     			else {
     				System.out.println("ERROR: " + file); //$NON-NLS-1$
+    			}
+    		}
+    	}
+    }
+    }

Formats disponibles : Unified diff

Laboratoire ICAR » Plateforme TXM

Révision 2473