Laboratoire ICAR » Plateforme TXM

		// version += "beta";

		if ("fr".equals(locale)) {

		}

		else {

		}

	}

}

				}

			}

			catch (CqiClientException e) {

				Log.printStackTrace(e);

			}

		}

boolean removeInterviewer = false;//if true the transcription of speakers (en1 and enq2) defined in metadatas file will be ignored

boolean includeComments = false;

boolean ignoreTranscriberMetadata = false;

int maxlines = 200;

String userDir = System.getProperty("user.home");

}

File propertyFile = new File(srcDir, "import.properties")//default

Properties props = new Properties();

if (propertyFile.exists() && propertyFile.canRead()) {

	FileInputStream input = new FileInputStream(propertyFile);

	props.load(input);

		removeInterviewer = Boolean.parseBoolean(props.get("removeInterviewer").toString());

	if (props.getProperty("ignoreTranscriberMetadata") != null)

		ignoreTranscriberMetadata = Boolean.parseBoolean(props.get("ignoreTranscriberMetadata").toString());

	//if (props.getProperty("includeComments") != null)

	//	includeComments = props.get("includeComments").toString();

	println "import properties: "

	println " removeInterviewer: "+removeInterviewer

	println " ignoreTranscriberMetadata: "+ignoreTranscriberMetadata

	//println " includeComments: "+includeComments

}

import org.txm.scripts.filters.TagSentences.*

import org.txm.scripts.filters.Tokeniser.*

/**

 * The Class importer.

 */

	 * @param basename the basename

	 * @return true, if successful

	 */

		File stokenizedDir = new File(binDir,"stokenized");

		stokenizedDir.deleteDir();

		}

		println("Create simple XML files ("+srcfiles.size()+" files)")

		for (File srcfile : srcfiles) {

			if (srcfile.length() == 0) {

				println "Skipping empty file: "+srcfile

				continue;

			}

			File xmlfile;

			try {

				int point = srcfile.getName().indexOf(".");

				xmlfile.delete()

			}

		}

		println("Tokenizing "+txmDir.listFiles().length+" files")

		for (File pfile : txmDir.listFiles()) {

			File tfile = new File(tokenizedDir, pfile.getName());

			SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(pfile, tfile, lang);

				tfile.delete()

			}

		}

		// Tag sentences

		List<File> stokenfiles = tokenizedDir.listFiles()

		File stokenizeDir = new File(tokenizedDir.getParentFile(), "stokenized")

		stokenizeDir.mkdir();

		println("Tagging sentences of "+stokenfiles.size()+" files")

		for (File f : stokenfiles) {

			Sequence S = new Sequence();

			Filter F1 = new CutHeader();

			Filter F7 = new TagSentences();

			S = F1 = F7 = F11 = null;

			headerfile.delete();//remove the prepared file to clean

		}

		//TRANSFORM INTO XML-TEI-TXM

		println("Building xml-tei-txm ("+stokenizeDir.listFiles().length+" files)")

		for (File tfile : stokenizeDir.listFiles()) {

			File xmlfile = new File(txmDir, tfile.getName());

			def correspType = new HashMap<String,String>()

				new File(xmlfile.getParent(),xmlfile.getName()).delete()

			}

		}

		files = txmDir.listFiles()

		if (files == null || files.size() == 0) return false

import org.txm.*;

import org.txm.objects.*;

import org.txm.core.engines.*;

import org.txm.utils.i18n.*;

import org.txm.metadatas.*;

import org.txm.utils.io.FileCopy;

if (metadatas != null) {

	println "-- INJECTING METADATA - from csv file: "+allMetadataFile

	println("Injecting metadata: "+metadatas.getHeadersList()+" in texts of directory "+txmDir)

		File outfile = File.createTempFile("temp", ".xml", infile.getParentFile());

		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {

			}

		}

	}

}

	def second = 0

	println "Paginating text: "

	for (File txmFile : files) {

		String txtname = txmFile.getName();

		if (second) { print(", ") }

		if (second > 0 && (second++ % 5) == 0) println ""

		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element);

		Edition edition = new Edition(t);

			edition.addPage(""+(++i), wordid);

		}

	}

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (MONITOR != null) MONITOR.worked(20, "FINALIZING")

class importer {

	/** The validation. */

	/** The tokenize. */

	/** The sentence. */

	/**

	 * Do tokenize.

	 */

	public boolean run(File srcDir, File binDir, File txmDir, String basename, String ignoredElements, String lang)

	{

		//TEST EACH XML FILE IF VALID

		List<File> okfiles = []

		for (File f : txmDir.listFiles()) { // clean directory

			String name = f.getName()

			if (f.isHidden() || !name.toLowerCase().endsWith(".xml")) {

				if (!f.delete()) {

					println "WARNING: could not clean $txmDir directory: TXM could not delete $f"

				}

			}

		}

		files = txmDir.listFiles()

		if (files == null || files.size() == 0) {

			println "No XML file (*.xml) to process. Aborting"

		}

		files = files.sort()

			if (!f.isHidden() && f.canRead() && !f.getName().endsWith(".properties") && !f.getName().startsWith("metadata") && !f.isDirectory()) {

				if (!ValidateXml.test(f)) {

				} else {

				}

			}

		}

		okfiles.sort()

		println "Files to process: "+okfiles

		if (okfiles.size() == 0) {

			println "No file. Check if the file extensions are '.xml'"

		}

		// Fix surrogates

		File srcDirectory = new File(binDir, "src")

		srcDirectory.mkdir()

		for (File f : okfiles) {

			File outputFile = new File (srcDirectory, f.getName())

		}

		okfiles = srcDirectory.listFiles()

		okfiles.sort()

		List<File> tokenfiles;

		if (tokenize || sentence) {

			println "Tokenizing (words=$tokenize sentences=$sentence) "+okfiles.size()+" files"

			for (File f : okfiles) {

				SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)

					tokenizer.setOutSideTextTagsAndKeepContent(ignoredElements)

				if (!tokenizer.process()) {

					println("Failed to process "+f)

					outfile.delete()

				}

			}

			tokenfiles = new File(binDir, "tokenized").listFiles()

		} else {

		}

//		Change XML tag "text" to "textunit"

		}

		//TRANSFORM INTO XML-TEI-TXM

		println("Building XML-TXM ("+tokenfiles.size()+" files)")

		for (File f : tokenfiles) {

			//ArrayList<String> milestones = new ArrayList<String>();

			def correspType = new HashMap<String,String>()

			def correspRef = new HashMap<String,String>()

			//il faut lister les id de tous les respStmt

			//fait la correspondance entre le respId et le rapport d'execution de l'outil

			//fait la correspondance entre le respId et les attributs type de la propriété ana du w txm

			//pour construire les ref vers les taxonomies

			//associe un id d'item avec sa description et son URI

			//informations de respStmt

			//resps (respId <voir ci-dessus>, [description, person, date])

			//lance le traitement

			builder.setHeaderInfos(respId,resps, applications, taxonomiesUtilisees, itemsURI)

			//builder.setAddTEIHeader();

			if (!builder.process(new File(txmDir,txmfile))) {

			}

		}

		okfiles = txmDir.listFiles()

		return okfiles != null && okfiles.size() > 0;

// $LastChangedRevision: 3349 $

// $LastChangedBy: mdecorde $

//

import org.txm.utils.*

import org.w3c.dom.Element

try {project=projectBinding;MONITOR=monitor} catch (Exception)

{	}

if (project == null) { println "no project set. Aborting"; return; }

String basename = corpusname

String lang = project.getLang()

String model = lang

String encoding = project.getEncoding()

boolean annotate = project.getAnnotate()

int wordsPerPage = project.getEditionDefinition("default").getWordsPerPage()

String page_element = project.getEditionDefinition("default").getPageElement()

boolean build_edition = project.getEditionDefinition("default").getBuildEdition()

boolean doTokenizeStep = project.getDoTokenizerStep()

File binDir = project.getProjectDirectory()

if (!binDir.exists()) {

	return;

}

File propertyFile = new File(rootDir, "import.properties")//default

println "Trying to read import properties file: "+propertyFile

if (propertyFile.exists() && propertyFile.canRead()) {

	println " sort metadata: "+textSortAttribute

	println " normalize attributes: "+normalizeMetadata

	println " ignored elements: "+ignoredElements

	println " stop if a XML source is malformed: "+stopIfMalformed

}

// Apply XSL

if (xsl != null && xslParams != null && xsl.trim().length() > 0) {

	new File(binDir, "src").deleteDir() // remove old outputed files if any

	if (ApplyXsl2.processImportSources(new File(xsl), srcDir, new File(binDir, "src"), xslParams))

	// return; // error during process

	println ""

}

// copy xml+dtd files

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (srcfiles != null) {

	for (int i = 0 ; i < srcfiles.size() ; i++) {// check XML format, and copy file into the "txm" directory

		File f = srcfiles.get(i)

		if (f.isDirectory() || f.isHidden() || f.getName().equals("import.xml") || f.getName().matches("metadata\\.....?") || f.getName().endsWith(".properties")) {

		}

		if (f.getName().toLowerCase().endsWith(".xml") && ValidateXml.test(f)) {

		} else {

		}

	}

} else {

	println "The $srcDir source directory is empty. Aborting."

}

if (ignoredFiles.size() > 0) {

}

if (txmDir.listFiles() == null) {

	println "No txm file to process"

}

// filtering

 }*/

//get metadata values from CSV

println "Trying to read metadata from: "+allMetadataFile

if (allMetadataFile.exists()) {

	println " Metadata file found: "+allMetadataFile

	File copy = new File(binDir, allMetadataFile.getName())

	if (!FileCopy.copy(allMetadataFile, copy)) {

	}

	metadatas = new Metadatas(copy, Toolbox.getMetadataEncoding(), Toolbox.getMetadataColumnSeparator(), Toolbox.getMetadataTextSeparator(), 1)

} else {

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

println "-- IMPORTER - Reading source files"

imp.doValidation(true) // change this to not validate xml

imp.doTokenize(doTokenizeStep) // change this, to not tokenize xml

if (!imp.run( srcDir, binDir, txmDir, basename, ignoredElements, lang)) {

}

if (metadatas != null) {

	println("-- INJECTING METADATA - "+metadatas.getHeadersList()+" in texts of directory "+new File(binDir,"txm"))

	def files = txmDir.listFiles()

	ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

	for (File infile : files) {

		cpb.tick()

		if (!metadatas.injectMetadatasInXml(infile, outfile, "text", null)) {

			outfile.delete();

		} else {

	return;

}

boolean annotationSuccess = false;

if (annotate) {

	println "-- ANNOTATE - Running NLP tools"

	def engine = Toolbox.getEngineManager(EngineType.ANNOTATION).getEngine("TreeTagger")

	if (engine.processDirectory(txmDir, binDir, ["lang":model])) {

	}

}

def c = new compiler();

if(debug) c.setDebug();

//c.setCwbPath("~/TXM/cwb/bin");

c.setAnnotationSuccess(annotationSuccess)

if (!c.run(project, binDir, txmDir, corpusname, null, srcfiles, metadatas)) {

	return;

}

if (MONITOR != null && MONITOR.isCanceled()) { return MONITOR.done(); }

if (build_edition) {

	println "-- EDITION - Building edition"

	def second = 0

	println "Paginating "+filelist.size()+" texts"

	ConsoleProgressBar cpb = new ConsoleProgressBar(filelist.size());

	for (File txmFile : filelist) {

		cpb.tick()

		t.setSourceFile(txmFile)

		t.setTXMFile(txmFile)

		def ed = new pager(txmFile, outdir, txtname, NoSpaceBefore, NoSpaceAfter, wordsPerPage, basename, page_element);

		for (i = 0 ; i < ed.getPageFiles().size();) {

		}

	}

	cpb.done()

}

		float W = 1f + queryLabel.getSize().x / (float) queryLabel.getText().length();

		// System.out.println("W= "+W+" Maxs "+refMax+ " " +leftMax+" " +keywordMax+" "+rightMax);

		((FormData) sash.getLayoutData()).left = new FormAttachment(0, 25 + (int) (refMax * W));

		referenceColumn.setWidth(2 * (int) (refMax * W));

		getResultArea().layout(true, true);