Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/transcriberLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/transcriberLoader.groovy	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/transcriber/transcriberLoader.groovy	(revision 945)
@@ -122,7 +122,9 @@
 		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
 		return;
 	}
-	metadatas = new Metadatas(copy, Toolbox.getPreference(Toolbox.METADATA_ENCODING), Toolbox.getPreference(Toolbox.METADATA_COLSEPARATOR), Toolbox.getPreference(Toolbox.METADATA_TXTSEPARATOR), 1)
+	metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(), 
+		Toolbox.getMetadataColumnSeparator(), 
+		Toolbox.getMetadataTextSeparator(), 1)
 }
 else
 	println "no metadata file: "+allmetadatasfile
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/importer.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/importer.groovy	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/discours/importer.groovy	(revision 945)
@@ -98,7 +98,9 @@
 				println "Error: could not create a copy of metadata file "+csvfile.getAbsoluteFile();
 				return;
 			}
-			metadatas = new Metadatas(copy, Toolbox.getPreference(TBXPreferences.METADATA_ENCODING), Toolbox.getPreference(Toolbox.METADATA_COLSEPARATOR), Toolbox.getPreference(Toolbox.METADATA_TXTSEPARATOR), 1)
+			metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(), 
+		Toolbox.getMetadataColumnSeparator(), 
+		Toolbox.getMetadataTextSeparator(), 1)
 		} else {
 			println "No metadata file: "+csvfile
 			println "Aborting"
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/txt/txtLoader.groovy
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/txt/txtLoader.groovy	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/txt/txtLoader.groovy	(revision 945)
@@ -97,7 +97,9 @@
 		println "Error: could not create a copy of metadata file "+allmetadatasfile.getAbsoluteFile();
 		return;
 	}
-	metadatas = new Metadatas(copy, Toolbox.getPreference(Toolbox.METADATA_ENCODING), Toolbox.getPreference(Toolbox.METADATA_COLSEPARATOR), Toolbox.getPreference(Toolbox.METADATA_TXTSEPARATOR), 1)
+	metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(), 
+		Toolbox.getMetadataColumnSeparator(), 
+		Toolbox.getMetadataTextSeparator(), 1)
 } else {
 	println "No metadata file: "+allmetadatasfile
 }
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Importer.java	(revision 945)
@@ -1,24 +0,0 @@
-package org.txm.importer.xtz;
-
-import java.io.File;
-
-/**
- * Takes any form of source files
- * 
- * After this step, the XML-TXM files are created.
- * 
- * they are validated before continuing 
- * @author mdecorde
- *
- */
-public abstract class Importer extends ImportStep {
-
-	public Importer(ImportModule module) {
-		super(module);
-		inputDirectory = module.getSourceDirectory();
-		outputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
-		outputDirectory.mkdirs();
-	}
-
-	public abstract void checkFiles();
-}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportKeys.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportKeys.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportKeys.java	(revision 945)
@@ -1,16 +0,0 @@
-package org.txm.importer.xtz;
-
-public class ImportKeys {
-
-	public static final String CLEAN = "clean.directories";
-	public static final String TTMODEL = "annotate.model";
-	public static final String TTANNOTATE = "annotate.run";
-	public static final String LANG = "lang";
-	
-	public static final String MULTITHREAD = "multithread";
-	public static final String DEBUG = "debug";
-	public static final String UPDATECORPUS = "corpus.update";
-	
-	public static final String NORMALISEANAVALUES = "normalize.ana.values";
-	public static final String NORMALISEATTRIBUTEVALUES = "normalize.attribute.values";
-}
\ No newline at end of file
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportStep.java	(revision 945)
@@ -1,54 +0,0 @@
-package org.txm.importer.xtz;
-
-import java.io.File;
-import java.util.HashMap;
-
-/**
- * One of the step of an import module
- * 
- * @author mdecorde
- *
- */
-public abstract class ImportStep {
-
-	protected File inputDirectory, outputDirectory;
-	protected ImportModule module;
-	
-	protected HashMap<String, Object> stepProperties = new HashMap<String, Object>();
-	protected boolean isSuccessFul = false;
-	protected String reason = "not set.";
-	protected boolean stopAtFirstError = true;
-	protected boolean debug = true;
-	
-	public ImportStep(ImportModule module) {
-		this.module = module;
-		debug = module.debug;
-	}
-	
-	public File getInputDirectory() {
-		return inputDirectory;
-	}
-	
-	public File getOutputDirectory() {
-		return outputDirectory;
-	}
-	
-	public ImportModule getImportModule() {
-		return module;
-	}
-	
-	public boolean isSuccessFul() {
-		return isSuccessFul;
-	}
-	
-	public String getReason() {
-		return reason;
-	}
-	
-	/**
-	 * Called when a step is interrupted to clean streams and stuff
-	 */
-	public abstract void cancel();
-	
-	public abstract void process();
-}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Compiler.java	(revision 945)
@@ -1,61 +0,0 @@
-package org.txm.importer.xtz;
-
-import java.io.File;
-import java.util.ArrayList;
-
-import org.txm.utils.DeleteDir;
-
-/**
- * Takes XML-TXM files, build the CQP files and call cwb utils
- * 
- * @author mdecorde
- *
- */
-public class Compiler extends ImportStep {
-
-	protected File cqpDirectory, registryDirectory, dataDirectory;
-	protected ArrayList<File> files;
-	
-	/**
-	 * Creates the output directories
-	 * 
-	 * @param module
-	 */
-	public Compiler(ImportModule module) {
-		super(module);
-
-		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
-		cqpDirectory = new File(module.getBinaryDirectory(), "cqp");		
-		outputDirectory = new File(module.getBinaryDirectory(), "data");
-		registryDirectory = new File(module.getBinaryDirectory(), "registry");
-		dataDirectory = new File(outputDirectory, module.getCorpusName());
-
-		DeleteDir.deleteDirectory(outputDirectory);
-		outputDirectory.mkdirs();
-		
-		DeleteDir.deleteDirectory(dataDirectory);
-		dataDirectory.mkdirs();
-		
-		DeleteDir.deleteDirectory(registryDirectory);
-		registryDirectory.mkdirs();
-		
-		if (!module.isUpdatingCorpus()) {
-			DeleteDir.deleteDirectory(cqpDirectory);
-			cqpDirectory.mkdir();
-		}
-	}
-
-	@Override
-	public void cancel() {
-		// TODO Auto-generated method stub
-	}
-
-	@Override
-	public void process() {
-		process(null); // no default files order set
-	}
-	
-	public void process(ArrayList<File> files) {
-		this.files = files;
-	}
-}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Step.java	(revision 945)
@@ -1,12 +0,0 @@
-package org.txm.importer.xtz;
-
-public class Step {
-
-	public Step() {
-		// TODO Auto-generated constructor stub
-	}
-
-	public boolean process() {
-		return true;
-	}
-}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/ImportModule.java	(revision 945)
@@ -1,259 +0,0 @@
-package org.txm.importer.xtz;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.logging.Level;
-
-import org.txm.Toolbox;
-import org.txm.core.preferences.TBXPreferences;
-import org.eclipse.core.runtime.IProgressMonitor;
-import org.txm.utils.xml.DomUtils;
-import org.txm.objects.BaseParameters;
-import org.txm.utils.DeleteDir;
-import org.txm.utils.logger.Log;
-
-public class ImportModule {
-
-	public BaseParameters importParameters;
-	
-	public String corpusVersionProduced;
-
-	public File sourceDirectory;
-	public File binaryDirectory;
-
-	public Importer importer;
-	public Annotater annotater;
-	public Compiler compiler;
-	public Pager pager;
-
-	/**
-	 * set the variable to false to stop the import process at next step
-	 */
-	public boolean isSuccessful = true;
-	public String reason = "none";
-	public boolean debug = false;
-	public boolean multithread = false;
-	public boolean updateCorpus = false;
-	public String corpusName;
-
-	IProgressMonitor monitor;
-	
-	public void setMonitor(IProgressMonitor monitor) {
-		this.monitor = monitor;
-	}
-
-	
-	public boolean isMultiThread() {
-		return multithread;
-	}
-
-	public boolean isDebugging() {
-		return debug;
-	}
-
-	public ImportModule(File importParametersFile) {
-		try {
-			BaseParameters b = new BaseParameters(importParametersFile);
-			init(b);
-		} catch (Exception e) {
-			e.printStackTrace();
-		}
-	}
-
-	public ImportModule(BaseParameters p) {
-		init(p);
-	}
-	
-	public boolean isUpdatingCorpus() {
-		return updateCorpus;
-	}
-
-	protected void init(BaseParameters p) {
-		this.importParameters = p;
-		this.importParameters.load();
-		corpusName = importParameters.name;
-		//this.debug = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.DEBUG));
-		
-		if (Log.getLevel().intValue() < Level.WARNING.intValue()) {
-			debug = true;
-		}
-		this.multithread = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.MULTITHREAD));
-		this.updateCorpus = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.UPDATECORPUS));
-
-		
-		this.sourceDirectory = importParameters.paramFile.getParentFile();
-		this.binaryDirectory = new File(Toolbox.getTxmHomePath(), "corpora/"+corpusName.toUpperCase());
-
-		if (!updateCorpus) { // clean directories only if it's a new import
-			DeleteDir.deleteDirectory(binaryDirectory);
-			binaryDirectory.mkdir();
-			
-			File txmDir = new File(binaryDirectory, "txm");
-			txmDir.mkdir();
-		}
-	}
-
-	public void start() throws InterruptedException {
-
-		binaryDirectory.mkdirs(); // ensure output exists
-		//System.out.println("ImportModule.start");
-		if (!updateCorpus) { // create XML-TXM files and annotate
-			//System.out.println("ImportModule.start: not updating");
-			if (importer != null) {
-				//System.out.println("ImportModule.start: importer: "+importer);
-				if (monitor != null) System.out.println("-- IMPORTER - Reading source files");
-				importer.process();
-				//importer.checkFiles();
-				isSuccessful = isSuccessful & importer.isSuccessFul();
-				if (!isSuccessful) {
-					System.out.println("Error while importing corpus during 'importer' step, reason="+importer.getReason());
-					return;
-				}
-			} else {
-				System.out.println("XML-TXM files already produced in "+new File(binaryDirectory, "txm/"+corpusName));
-			}
-
-			boolean annotate = "true".equals(importParameters.getCorpusElement().getAttribute("annotate"));
-			if (annotate && annotater != null) {
-				if (monitor != null) System.out.println("-- ANNOTATE - Running NLP tools");
-				annotater.process();
-				isSuccessful = isSuccessful & annotater.isSuccessFul();
-				if (!isSuccessful) {
-					System.out.println("Error while importing corpus during 'annotate' step, reason="+annotater.getReason());
-					return;
-				}
-			} else {
-				//System.out.println("XML-TXM files already annotated.");
-			}
-		} else {
-			System.out.println("Updating corpus...");
-		}
-		
-		//System.out.println("GET FILES ORDER");
-		final ArrayList<File> files = getTXMFilesOrder();
-		
-		Thread Tcompiler = new Thread() {
-			public void run() {
-				if (compiler != null) {
-					if (monitor != null) System.out.println("-- COMPILING - Building Search Engine indexes");
-					compiler.process(files);
-					isSuccessful = isSuccessful & compiler.isSuccessFul();
-					if (!isSuccessful) {
-						System.out.println("Error while importing corpus during 'compiler' step, reason="+compiler.getReason());
-						return;
-					}
-				} else {
-					System.out.println("No CQP index created.");
-				}
-			}
-		};
-
-		Thread Tpager = new Thread() {
-			public void run() {
-
-				if (pager != null) {
-					if (monitor != null) System.out.println("-- EDITION - Building edition");
-					pager.process(files);
-					isSuccessful = isSuccessful & pager.isSuccessFul();
-					if (!isSuccessful) {
-						System.out.println("Error while importing corpus during 'pager' step, reason="+pager.getReason());
-						return;
-					}
-				} else {
-					System.out.println("No edition produced.");
-				}
-			}
-		};
-
-		Tcompiler.start();
-		if (!multithread) {
-			Tcompiler.join(); // wait for the end if not multithreaded
-			if (!isSuccessful) { // don't call pager is compiler step failed
-				return;
-			}
-		}
-		
-		Tpager.start();
-		if (multithread) Tcompiler.join(); // wait for both thread to end
-		Tpager.join();		
-	}
-	
-	protected ArrayList<File> getTXMFilesOrder() {
-		//System.out.println("DEFAULT FILES ORDER");
-		File txmDirectory = new File(binaryDirectory, "txm/"+corpusName);
-		ArrayList<File> files = new ArrayList<File>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {
-			@Override
-			public boolean accept(File file) {
-				return file.isFile() && file.getName().endsWith(".xml");
-			}
-		})));
-		
-		Collections.sort(files);
-		
-		return files;
-	}
-
-
-	public void end() {
-		File paramFile = new File(binaryDirectory, "import.xml");
-		try {
-			DomUtils.save(importParameters.root.getOwnerDocument(), paramFile);
-			isSuccessful = true;
-		} catch (Exception e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-			isSuccessful = false;
-		}
-	}
-	
-	public String getCorpusName() {
-		return corpusName;
-	}
-
-	public String getReason() {
-		return reason;
-	}
-
-	public boolean isSuccessFul() {
-		return isSuccessful;
-	}
-
-	public BaseParameters getParameters() {
-		return importParameters;
-	}
-
-	public File getSourceDirectory() {
-		return sourceDirectory;
-	}
-
-	public File getBinaryDirectory() {
-		return binaryDirectory;
-	}
-	
-	public void process() throws InterruptedException {
-		start();
-		if (isSuccessful)
-			end();
-	}
-
-	public static void main(String[] args) {
-		File importParametersFile = new File("/home/mdecorde/xml/brown/import.xml");
-
-		ImportModule module = new ImportModule(importParametersFile);
-		System.out.println("Parameters: "+module.getParameters());
-		try {
-			module.start();
-
-			if (module.isSuccessful) {
-				System.out.println("Import sucessful. reloading corpora...");
-			} else {
-				System.out.println("Import failed, reason = "+module.getReason());
-			}
-		} catch (Exception e) {
-			e.printStackTrace();
-		}
-	}
-}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Annotater.java	(revision 945)
@@ -1,21 +0,0 @@
-package org.txm.importer.xtz;
-
-import java.io.File;
-
-
-/**
- * 
- * Takes the XML-TXM files and wrap a TAL Tool to update the XML-TXM files
- * 
- * @author mdecorde
- *
- */
-public abstract class Annotater extends ImportStep {
-
-	public Annotater(ImportModule module) {
-		super(module);
-		
-		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.corpusName);
-		outputDirectory = new File(module.getBinaryDirectory(), "txm");
-	}
-}
Index: tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java
===================================================================
--- tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java	(revision 944)
+++ tmp/org.txm.groovy.core/src/groovy/org/txm/importer/xtz/Pager.java	(revision 945)
@@ -1,49 +0,0 @@
-package org.txm.importer.xtz;
-
-import java.io.File;
-import java.util.ArrayList;
-
-import org.txm.utils.DeleteDir;
-
-/**
- * Takes the XML-TXM files and build an edition
- * 
- * @author mdecorde
- *
- */
-public class Pager extends ImportStep {
-
-	protected File htmlDirectory;
-	protected String corpusname;
-	protected ArrayList<File> files;
-
-	public Pager(ImportModule module, String editionName) {
-		super(module);
-
-		corpusname = module.getCorpusName();
-
-		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
-		htmlDirectory = new File(module.getBinaryDirectory(), "HTML/"+corpusname);
-		outputDirectory  = new File(htmlDirectory, editionName);
-
-		if (!module.isUpdatingCorpus()) {
-			DeleteDir.deleteDirectory(outputDirectory);
-			outputDirectory.mkdirs();
-		}
-	}
-
-	@Override
-	public void cancel() {
-		// TODO Auto-generated method stub
-
-	}
-
-	@Override
-	public void process() {
-		process(null); // no default files order set
-	}
-	
-	public void process(ArrayList<File> files) {
-		this.files = files;
-	}
-}
Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportKeys.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportKeys.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportKeys.java	(revision 945)
@@ -0,0 +1,16 @@
+package org.txm.importer.xtz;
+
+public class ImportKeys {
+
+	public static final String CLEAN = "clean.directories";
+	public static final String TTMODEL = "annotate.model";
+	public static final String TTANNOTATE = "annotate.run";
+	public static final String LANG = "lang";
+	
+	public static final String MULTITHREAD = "multithread";
+	public static final String DEBUG = "debug";
+	public static final String UPDATECORPUS = "corpus.update";
+	
+	public static final String NORMALISEANAVALUES = "normalize.ana.values";
+	public static final String NORMALISEATTRIBUTEVALUES = "normalize.attribute.values";
+}
\ No newline at end of file

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportKeys.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportStep.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportStep.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportStep.java	(revision 945)
@@ -0,0 +1,54 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.util.HashMap;
+
+/**
+ * One of the step of an import module
+ * 
+ * @author mdecorde
+ *
+ */
+public abstract class ImportStep {
+
+	protected File inputDirectory, outputDirectory;
+	protected ImportModule module;
+	
+	protected HashMap<String, Object> stepProperties = new HashMap<String, Object>();
+	protected boolean isSuccessFul = false;
+	protected String reason = "not set.";
+	protected boolean stopAtFirstError = true;
+	protected boolean debug = true;
+	
+	public ImportStep(ImportModule module) {
+		this.module = module;
+		debug = module.debug;
+	}
+	
+	public File getInputDirectory() {
+		return inputDirectory;
+	}
+	
+	public File getOutputDirectory() {
+		return outputDirectory;
+	}
+	
+	public ImportModule getImportModule() {
+		return module;
+	}
+	
+	public boolean isSuccessFul() {
+		return isSuccessFul;
+	}
+	
+	public String getReason() {
+		return reason;
+	}
+	
+	/**
+	 * Called when a step is interrupted to clean streams and stuff
+	 */
+	public abstract void cancel();
+	
+	public abstract void process();
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportStep.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/Compiler.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/Compiler.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/Compiler.java	(revision 945)
@@ -0,0 +1,61 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.txm.utils.DeleteDir;
+
+/**
+ * Takes XML-TXM files, build the CQP files and call cwb utils
+ * 
+ * @author mdecorde
+ *
+ */
+public class Compiler extends ImportStep {
+
+	protected File cqpDirectory, registryDirectory, dataDirectory;
+	protected ArrayList<File> files;
+	
+	/**
+	 * Creates the output directories
+	 * 
+	 * @param module
+	 */
+	public Compiler(ImportModule module) {
+		super(module);
+
+		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
+		cqpDirectory = new File(module.getBinaryDirectory(), "cqp");		
+		outputDirectory = new File(module.getBinaryDirectory(), "data");
+		registryDirectory = new File(module.getBinaryDirectory(), "registry");
+		dataDirectory = new File(outputDirectory, module.getCorpusName());
+
+		DeleteDir.deleteDirectory(outputDirectory);
+		outputDirectory.mkdirs();
+		
+		DeleteDir.deleteDirectory(dataDirectory);
+		dataDirectory.mkdirs();
+		
+		DeleteDir.deleteDirectory(registryDirectory);
+		registryDirectory.mkdirs();
+		
+		if (!module.isUpdatingCorpus()) {
+			DeleteDir.deleteDirectory(cqpDirectory);
+			cqpDirectory.mkdir();
+		}
+	}
+
+	@Override
+	public void cancel() {
+		// TODO Auto-generated method stub
+	}
+
+	@Override
+	public void process() {
+		process(null); // no default files order set
+	}
+	
+	public void process(ArrayList<File> files) {
+		this.files = files;
+	}
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/Compiler.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/Annotater.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/Annotater.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/Annotater.java	(revision 945)
@@ -0,0 +1,21 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+
+
+/**
+ * 
+ * Takes the XML-TXM files and wrap a TAL Tool to update the XML-TXM files
+ * 
+ * @author mdecorde
+ *
+ */
+public abstract class Annotater extends ImportStep {
+
+	public Annotater(ImportModule module) {
+		super(module);
+		
+		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.corpusName);
+		outputDirectory = new File(module.getBinaryDirectory(), "txm");
+	}
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/Annotater.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/Step.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/Step.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/Step.java	(revision 945)
@@ -0,0 +1,12 @@
+package org.txm.importer.xtz;
+
+public class Step {
+
+	public Step() {
+		// TODO Auto-generated constructor stub
+	}
+
+	public boolean process() {
+		return true;
+	}
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/Step.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/Pager.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/Pager.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/Pager.java	(revision 945)
@@ -0,0 +1,49 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.txm.utils.DeleteDir;
+
+/**
+ * Takes the XML-TXM files and build an edition
+ * 
+ * @author mdecorde
+ *
+ */
+public class Pager extends ImportStep {
+
+	protected File htmlDirectory;
+	protected String corpusname;
+	protected ArrayList<File> files;
+
+	public Pager(ImportModule module, String editionName) {
+		super(module);
+
+		corpusname = module.getCorpusName();
+
+		inputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
+		htmlDirectory = new File(module.getBinaryDirectory(), "HTML/"+corpusname);
+		outputDirectory  = new File(htmlDirectory, editionName);
+
+		if (!module.isUpdatingCorpus()) {
+			DeleteDir.deleteDirectory(outputDirectory);
+			outputDirectory.mkdirs();
+		}
+	}
+
+	@Override
+	public void cancel() {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public void process() {
+		process(null); // no default files order set
+	}
+	
+	public void process(ArrayList<File> files) {
+		this.files = files;
+	}
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/Pager.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportModule.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportModule.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportModule.java	(revision 945)
@@ -0,0 +1,259 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.logging.Level;
+
+import org.txm.Toolbox;
+import org.txm.core.preferences.TBXPreferences;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.txm.utils.xml.DomUtils;
+import org.txm.objects.BaseParameters;
+import org.txm.utils.DeleteDir;
+import org.txm.utils.logger.Log;
+
+public class ImportModule {
+
+	public BaseParameters importParameters;
+	
+	public String corpusVersionProduced;
+
+	public File sourceDirectory;
+	public File binaryDirectory;
+
+	public Importer importer;
+	public Annotater annotater;
+	public Compiler compiler;
+	public Pager pager;
+
+	/**
+	 * set the variable to false to stop the import process at next step
+	 */
+	public boolean isSuccessful = true;
+	public String reason = "none";
+	public boolean debug = false;
+	public boolean multithread = false;
+	public boolean updateCorpus = false;
+	public String corpusName;
+
+	IProgressMonitor monitor;
+	
+	public void setMonitor(IProgressMonitor monitor) {
+		this.monitor = monitor;
+	}
+
+	
+	public boolean isMultiThread() {
+		return multithread;
+	}
+
+	public boolean isDebugging() {
+		return debug;
+	}
+
+	public ImportModule(File importParametersFile) {
+		try {
+			BaseParameters b = new BaseParameters(importParametersFile);
+			init(b);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	public ImportModule(BaseParameters p) {
+		init(p);
+	}
+	
+	public boolean isUpdatingCorpus() {
+		return updateCorpus;
+	}
+
+	protected void init(BaseParameters p) {
+		this.importParameters = p;
+		this.importParameters.load();
+		corpusName = importParameters.name;
+		//this.debug = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.DEBUG));
+		
+		if (Log.getLevel().intValue() < Level.WARNING.intValue()) {
+			debug = true;
+		}
+		this.multithread = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.MULTITHREAD));
+		this.updateCorpus = "true".equals(importParameters.getKeyValueParameters().get(ImportKeys.UPDATECORPUS));
+
+		
+		this.sourceDirectory = importParameters.paramFile.getParentFile();
+		this.binaryDirectory = new File(Toolbox.getTxmHomePath(), "corpora/"+corpusName.toUpperCase());
+
+		if (!updateCorpus) { // clean directories only if it's a new import
+			DeleteDir.deleteDirectory(binaryDirectory);
+			binaryDirectory.mkdir();
+			
+			File txmDir = new File(binaryDirectory, "txm");
+			txmDir.mkdir();
+		}
+	}
+
+	public void start() throws InterruptedException {
+
+		binaryDirectory.mkdirs(); // ensure output exists
+		//System.out.println("ImportModule.start");
+		if (!updateCorpus) { // create XML-TXM files and annotate
+			//System.out.println("ImportModule.start: not updating");
+			if (importer != null) {
+				//System.out.println("ImportModule.start: importer: "+importer);
+				if (monitor != null) System.out.println("-- IMPORTER - Reading source files");
+				importer.process();
+				//importer.checkFiles();
+				isSuccessful = isSuccessful & importer.isSuccessFul();
+				if (!isSuccessful) {
+					System.out.println("Error while importing corpus during 'importer' step, reason="+importer.getReason());
+					return;
+				}
+			} else {
+				System.out.println("XML-TXM files already produced in "+new File(binaryDirectory, "txm/"+corpusName));
+			}
+
+			boolean annotate = "true".equals(importParameters.getCorpusElement().getAttribute("annotate"));
+			if (annotate && annotater != null) {
+				if (monitor != null) System.out.println("-- ANNOTATE - Running NLP tools");
+				annotater.process();
+				isSuccessful = isSuccessful & annotater.isSuccessFul();
+				if (!isSuccessful) {
+					System.out.println("Error while importing corpus during 'annotate' step, reason="+annotater.getReason());
+					return;
+				}
+			} else {
+				//System.out.println("XML-TXM files already annotated.");
+			}
+		} else {
+			System.out.println("Updating corpus...");
+		}
+		
+		//System.out.println("GET FILES ORDER");
+		final ArrayList<File> files = getTXMFilesOrder();
+		
+		Thread Tcompiler = new Thread() {
+			public void run() {
+				if (compiler != null) {
+					if (monitor != null) System.out.println("-- COMPILING - Building Search Engine indexes");
+					compiler.process(files);
+					isSuccessful = isSuccessful & compiler.isSuccessFul();
+					if (!isSuccessful) {
+						System.out.println("Error while importing corpus during 'compiler' step, reason="+compiler.getReason());
+						return;
+					}
+				} else {
+					System.out.println("No CQP index created.");
+				}
+			}
+		};
+
+		Thread Tpager = new Thread() {
+			public void run() {
+
+				if (pager != null) {
+					if (monitor != null) System.out.println("-- EDITION - Building edition");
+					pager.process(files);
+					isSuccessful = isSuccessful & pager.isSuccessFul();
+					if (!isSuccessful) {
+						System.out.println("Error while importing corpus during 'pager' step, reason="+pager.getReason());
+						return;
+					}
+				} else {
+					System.out.println("No edition produced.");
+				}
+			}
+		};
+
+		Tcompiler.start();
+		if (!multithread) {
+			Tcompiler.join(); // wait for the end if not multithreaded
+			if (!isSuccessful) { // don't call pager is compiler step failed
+				return;
+			}
+		}
+		
+		Tpager.start();
+		if (multithread) Tcompiler.join(); // wait for both thread to end
+		Tpager.join();		
+	}
+	
+	protected ArrayList<File> getTXMFilesOrder() {
+		//System.out.println("DEFAULT FILES ORDER");
+		File txmDirectory = new File(binaryDirectory, "txm/"+corpusName);
+		ArrayList<File> files = new ArrayList<File>(Arrays.asList(txmDirectory.listFiles(new FileFilter() {
+			@Override
+			public boolean accept(File file) {
+				return file.isFile() && file.getName().endsWith(".xml");
+			}
+		})));
+		
+		Collections.sort(files);
+		
+		return files;
+	}
+
+
+	public void end() {
+		File paramFile = new File(binaryDirectory, "import.xml");
+		try {
+			DomUtils.save(importParameters.root.getOwnerDocument(), paramFile);
+			isSuccessful = true;
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			isSuccessful = false;
+		}
+	}
+	
+	public String getCorpusName() {
+		return corpusName;
+	}
+
+	public String getReason() {
+		return reason;
+	}
+
+	public boolean isSuccessFul() {
+		return isSuccessful;
+	}
+
+	public BaseParameters getParameters() {
+		return importParameters;
+	}
+
+	public File getSourceDirectory() {
+		return sourceDirectory;
+	}
+
+	public File getBinaryDirectory() {
+		return binaryDirectory;
+	}
+	
+	public void process() throws InterruptedException {
+		start();
+		if (isSuccessful)
+			end();
+	}
+
+	public static void main(String[] args) {
+		File importParametersFile = new File("/home/mdecorde/xml/brown/import.xml");
+
+		ImportModule module = new ImportModule(importParametersFile);
+		System.out.println("Parameters: "+module.getParameters());
+		try {
+			module.start();
+
+			if (module.isSuccessful) {
+				System.out.println("Import sucessful. reloading corpora...");
+			} else {
+				System.out.println("Import failed, reason = "+module.getReason());
+			}
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/ImportModule.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/importer/xtz/Importer.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/importer/xtz/Importer.java	(revision 0)
+++ tmp/org.txm.core/src/java/org/txm/importer/xtz/Importer.java	(revision 945)
@@ -0,0 +1,24 @@
+package org.txm.importer.xtz;
+
+import java.io.File;
+
+/**
+ * Takes any form of source files
+ * 
+ * After this step, the XML-TXM files are created.
+ * 
+ * they are validated before continuing 
+ * @author mdecorde
+ *
+ */
+public abstract class Importer extends ImportStep {
+
+	public Importer(ImportModule module) {
+		super(module);
+		inputDirectory = module.getSourceDirectory();
+		outputDirectory = new File(module.getBinaryDirectory(), "txm/"+module.getCorpusName());
+		outputDirectory.mkdirs();
+	}
+
+	public abstract void checkFiles();
+}

Property changes on: tmp/org.txm.core/src/java/org/txm/importer/xtz/Importer.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: tmp/org.txm.core/src/java/org/txm/metadatas/Metadatas.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/metadatas/Metadatas.java	(revision 944)
+++ tmp/org.txm.core/src/java/org/txm/metadatas/Metadatas.java	(revision 945)
@@ -235,6 +235,12 @@
 	 */
 	public static boolean convertCsvToXml(File csvfile, File xmlFile, String encoding, String separator, String txtseparator, int nbheaderline) throws Exception
 	{	
+		if (separator == null || separator.length() == 0) {
+			separator = "\t";
+		}
+		if (encoding == null || encoding.length() == 0) {
+			encoding = "UTF-8";
+		}
 		xmlFile.createNewFile();
 
 		if(!csvfile.exists())
Index: tmp/org.txm.core/src/java/org/txm/Toolbox.java
===================================================================
--- tmp/org.txm.core/src/java/org/txm/Toolbox.java	(revision 944)
+++ tmp/org.txm.core/src/java/org/txm/Toolbox.java	(revision 945)
@@ -84,7 +84,6 @@
 	private static boolean state = false;
 
 	public static Workspace workspace;
-
 	
 	/**
 	 * 
Index: tmp/org.txm.core/META-INF/MANIFEST.MF
===================================================================
--- tmp/org.txm.core/META-INF/MANIFEST.MF	(revision 944)
+++ tmp/org.txm.core/META-INF/MANIFEST.MF	(revision 945)
@@ -390,6 +390,7 @@
  org.txm.importer.filters,
  org.txm.importer.scripting,
  org.txm.importer.xmltxm,
+ org.txm.importer.xtz,
  org.txm.js,
  org.txm.js.viewer,
  org.txm.metadatas,
Index: tmp/org.txm.statsengine.r.core/src/org/txm/statsengine/r/core/StartRserve.java
===================================================================
--- tmp/org.txm.statsengine.r.core/src/org/txm/statsengine/r/core/StartRserve.java	(revision 944)
+++ tmp/org.txm.statsengine.r.core/src/org/txm/statsengine/r/core/StartRserve.java	(revision 945)
@@ -162,7 +162,7 @@
 			Thread.sleep(200);
 		} catch (InterruptedException ix) { }
 
-		int attempts = 20;
+		int attempts = 10;
 		while (attempts > 0) {
 			try {
 				System.out.print("."); //$NON-NLS-1$
@@ -171,7 +171,7 @@
 				return true;
 			} catch (Exception e2) {
 				try {
-					Thread.sleep(2000);
+					Thread.sleep(1500);
 				} catch (InterruptedException ix) { }
 			}
 
Index: tmp/org.txm.searchengine.cqp.core/src/org/txm/searchengine/cqp/corpus/Corpus.java
===================================================================
--- tmp/org.txm.searchengine.cqp.core/src/org/txm/searchengine/cqp/corpus/Corpus.java	(revision 944)
+++ tmp/org.txm.searchengine.cqp.core/src/org/txm/searchengine/cqp/corpus/Corpus.java	(revision 945)
@@ -1107,6 +1107,7 @@
 					this.getQualifiedCqpId(), queryResultId,
 					query.getQueryString());
 			queryResult = new QueryResult(queryResultId, queryResultName, this,	query);
+			
 			if (save) super.addQueryLog(query.toString(), new ArrayList<String>());
 		} catch (Exception e) {
 			org.txm.utils.logger.Log.printStackTrace(e);
