Laboratoire ICAR » Plateforme TXM

 *

 */

public class ExportCorpusDialog extends Dialog {

	/**

	 * list of the default files to include in the corpus binary

	 */

	private List<String> defaultFiles = Arrays.asList("HTML", "txm", "data", "registry", "annotations", "analec", "temporary_annotations", "xsl", "css", ".settings", ".project");

	private File zipFile;

	private LinkedHashMap<File, Boolean> selectedFiles;

	private CQPCorpus corpus;

	private TableViewer treeViewer;

	private TableViewerColumn nameColumn;

	private Text textField;

	private Button selectButton;

	public final static String ID = ExportCorpusDialog.class.getName();

	/**

	 * 

	 * @param parentShell

		super(parentShell);

		this.corpus = corpus;

	}

	@Override

	protected void configureShell(Shell newShell) {

		super.configureShell(newShell);

		newShell.setText(NLS.bind("Exporting {0}...", corpus.getName()));

		newShell.setMinimumSize(400, 300);

	}

	@Override

	protected Control createDialogArea(Composite parent) {

		Composite comp = (Composite) super.createDialogArea(parent);

		GridLayout layout = (GridLayout) comp.getLayout();

		layout.numColumns = 3;

		Label l = new Label(comp, SWT.LEFT);

		l.setText("Output file");

		l.setLayoutData(new GridData(GridData.BEGINNING, GridData.CENTER, false, false));

		textField = new Text(comp, SWT.SINGLE | SWT.BORDER);

		textField.setLayoutData(new GridData(GridData.FILL, GridData.CENTER, true, false));

		String previousPath = TXMPreferences.getString("path", ExportCorpusDialog.class.getName());

		if (previousPath == null || previousPath.length() == 0) {

		}

		textField.setText(previousPath + "/" + corpus.getName() + "-" + Toolbox.dateformat.format(Calendar.getInstance().getTime()) + ".txm"); //$NON-NLS-1$

		selectButton = new Button(comp, SWT.PUSH);

		selectButton.setText("...");

		selectButton.addSelectionListener(new SelectionListener() {

			@Override

			public void widgetSelected(SelectionEvent e) {

				FileDialog dialog = new FileDialog(selectButton.getShell(), SWT.SAVE);

				String[] exts = { "*.txm", "*.zip" }; //$NON-NLS-1$ //$NON-NLS-2$

				dialog.setFilterExtensions(exts);

				String path = textField.getText();

					File p = new File(path);

					if (p.isDirectory()) {

						dialog.setFilterPath(textField.getText());

						dialog.setFilterPath(p.getParent());

						dialog.setFileName(p.getName());

					}

				}

				//

				path = dialog.open();

				if (path != null) {

					textField.setText(path);

					TXMPreferences.put(ExportCorpusDialog.class.getName(), "path", dir.getAbsolutePath());

				}

			}

			@Override

			public void widgetDefaultSelected(SelectionEvent e) {}

		});

		selectedFiles = new LinkedHashMap<>();

		Project project = corpus.getProject();

		File directory = project.getProjectDirectory();

		if (directory.exists()) {

			File[] files = directory.listFiles(new FileFilter() {

				@Override

				public boolean accept(File f) {

					return !f.getName().startsWith(".") && !f.isHidden() && f.isDirectory() && !defaultFiles.contains(f.getName());

				}

			});

			Arrays.sort(files, new Comparator<File>() {

				@Override

				public int compare(File arg0, File arg1) {

					if (arg0.isDirectory() && arg1.isDirectory()) {

			});

			for (File f : files) {

				if (!f.isHidden() && f.isDirectory()) {

					if (defaultFiles.contains(f.getName())) {

						selectedFiles.put(f, true);

					}

					}

				}

			}

			if (files.length > 0) {

				treeViewer = new TableViewer(comp, SWT.CHECK | SWT.BORDER | SWT.V_SCROLL | SWT.H_SCROLL);

				treeViewer.setContentProvider(new ArrayContentProvider());

				treeViewer.getTable().addListener(SWT.Selection, event -> {

					if (event.detail == SWT.CHECK) {

						TableItem item = (TableItem) event.item;

						selectedFiles.put((File) event.item.getData(), item.getChecked());

					}

				});

				treeViewer.getTable().setLayoutData(new GridData(GridData.FILL, GridData.FILL, true, true, 3, 1));

				treeViewer.getTable().setHeaderVisible(true);

				treeViewer.getTable().setLinesVisible(true);

				nameColumn = new TableViewerColumn(treeViewer, SWT.NONE);

				nameColumn.getColumn().pack();

				nameColumn.setLabelProvider(new CellLabelProvider() {

					@Override

					public void update(ViewerCell cell) {

						Object element = cell.getElement();

						if (element instanceof File) {

							File f = (File) element;

							cell.setText(f.getName());

						}

					}

				});

				treeViewer.setInput(files);

				for (int i = 0; i < files.length; i++) {

					treeViewer.getTable().getItem(i).setChecked(selectedFiles.get(files[i]));

				}

				Button selectAllButton = new Button(comp, SWT.PUSH);

				selectAllButton.setText("Select All");

				selectAllButton.addSelectionListener(new SelectionListener() {

					@Override

					public void widgetSelected(SelectionEvent e) {

						massSelect(true);

					}

					@Override

					public void widgetDefaultSelected(SelectionEvent e) {}

				});

				Button clearAllButton = new Button(comp, SWT.PUSH);

				clearAllButton.setText("Clear selection");

				clearAllButton.addSelectionListener(new SelectionListener() {

					@Override

					public void widgetSelected(SelectionEvent e) {

						massSelect(false);

					}

					@Override

					public void widgetDefaultSelected(SelectionEvent e) {}

				});

			}

		}

		return comp;

	}

	protected void massSelect(boolean b) {

		for (int i = 0; i < treeViewer.getTable().getItems().length; i++) {

			treeViewer.getTable().getItem(i).setChecked(b);

			selectedFiles.put(f, b);

		}

	}

	@Override

	protected boolean isResizable() {

		return true;

	}

	@Override

	protected void buttonPressed(int buttonId) {

		if (buttonId == Dialog.OK) {

		}

		super.buttonPressed(buttonId);

	}

	public File getZipFile() {

		return zipFile;

	}

	public LinkedHashMap<File, Boolean> getFileSelection() {

		return selectedFiles;

	}

	public HashSet<String> getIgnoreNames() {

		HashSet<String> ignore = new HashSet<>();

		for (File f : selectedFiles.keySet()) {

            class="org.txm.ruby.core.RubyScriptEngine"

            name="Ruby">

      </ScriptEngine>

   </extension>

</plugin>

 */

class Annotate {

	boolean cancelNow = false;

	/** The report file. */

	File reportFile;//contains the txm:application tag content

	/** The resp person. */

	String respPerson;

	/** The resp id. */

	String respId;

	/** The resp desc. */

	String respDesc;

	/** The resp date. */

	String respDate;

	/** The resp when. */

	String respWhen;

	/** The app ident. */

	String appIdent;

	/** The app version. */

	String appVersion;

	/** The distributor. */

	String distributor;

	/** The publi stmt. */

	String publiStmt;

	/** The source stmt. */

	String sourceStmt;

	/** The types. */

	def types;

	/** The types title. */

	def typesTITLE;

	/** The types desc. */

	def typesDesc;

	/** The types tagset. */

	def typesTAGSET;

	/** The types web. */

	def typesWEB;

	/** The idform. */

	String idform;

	/** The debug. */

	boolean debug = false;

	File modelsDirectory;

	public Annotate() {

		modelsDirectory = new File(TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.MODELS_PATH)); // default models directory is set in the Toolbox

	}

	/**

	 * Sets the debug.

	 */

	public void setDebug() {

		debug = true;

	}

	String id;

	/**

	 * Inits the tt outfile infos.

	{

		initTTOutfileInfos(rootDirFile, modelfile, modelfilename, null);

	}

	/**

	 * Inits the tt outfile infos.

	 *

		String[] split = id.split("\\.");

		if (split.length > 0) id = split[0];

		if (id.equals("??")) id = "xx"

		reportFile = new File(rootDirFile,"NLPToolsParameters.xml");

		respPerson = System.getProperty("user.name");

		respId = "txm";

		respDesc = "NLP annotation tool";

		respDate = DateFormat.getDateInstance(DateFormat.SHORT, Locale.UK).format(new Date());

		respWhen = DateFormat.getDateInstance(DateFormat.FULL, Locale.UK).format(new Date());

		appIdent = "TreeTagger";

		appVersion = "3.2";

		distributor = "";

		publiStmt = """""";

		sourceStmt = """""";

		if (properties != null && properties.length == 2) {

		} else {

		}

		//TODO: the tagset, website and description should be referenced in the model catalog

		if(modelfile.getName() == "rgaqcj.par") {

		} else {

		}

		idform ="w";

	}

	/**

	 * Apply tt.

	 *

	 * @return true, if successful

	 */

	public boolean applyTT(File ttsrcfile, File ttoutfile, File modelfile) {

	}

	/**

	 * Apply tt.

	 *

		try {

			File infile = ttsrcfile;

			File outfile = ttoutfile;

			def tt = new TreeTagger(TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.INSTALL_PATH)+"/bin/", options);

			tt.settoken();

			tt.setlemma();

		}

		return true;

	}

	/**

	 * Write standoff file.

	 *

		transfo.setInfos(distributor,  publiStmt, sourceStmt);

		return transfo.process( ttoutfile, posfile, encoding );

	}

	/**

	 * Run step by step : build TT src files, run TT, build xml-standoff files, inject standoff annotations

	 *

			println "TREETAGGER INSTALL PATH : "+TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.INSTALL_PATH);

			println "TREETAGGER MODELS PATH : "+TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.MODELS_PATH)

		}

		//test if the Toolbox know TreeTagger

		if (!new File(TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.INSTALL_PATH)+"/bin/").exists()) {

			println("Could not find TreeTagger binaries in "+TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.INSTALL_PATH)+"/bin/")

			langAll = new LangDetector(binDir).getLang();

			println "General lang $langAll"

		}

		//cleaning

		new File(binDir, "annotations").deleteDir();

		new File(binDir, "annotations").mkdir();

		new File(binDir, "treetagger").deleteDir();

		new File(binDir, "treetagger").mkdir();

		ArrayList<String> milestones = [];

		//BUILD TT FILE READY TO BE TAGGED

		List<File> files = txmDir.listFiles(IOUtils.HIDDENFILE_FILTER)

		println("Building TT source files ("+files.size()+") from directory "+txmDir)

		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

		for (File f : files) {

			if (!ttsrcbuilder.process(resultfile, null))

				System.out.println("Failed to build tt src file of "+srcfile);

		}

		if (cancelNow) return;

		File modelDirectory = new File(TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.MODELS_PATH));

		if (!modelDirectory.exists()) {

			println "Skipping ANNOTATE: TreeTagger language model file directory not found: "+modelDirectory.getAbsolutePath();

		}

		println("")

		//Convert encoding if needed

		//APPLY TREETAGGER

		files = new File(binDir, "treetagger").listFiles(IOUtils.HIDDENFILE_FILTER)

		println("Applying $modelfilename TreeTagger model on dir: "+new File(binDir, "treetagger")+ " ("+files.size()+" files)")

			modelfile = new File(modelsDirectory, tmpModelFileName);

			if (debug)

				println "model file : "+modelfile;

			File

			if (!modelfile.exists()) {

				println "Skipping ANNOTATE: '$modelfile' TreeTagger language model file not found."

				if(System.getProperty("os.name").startsWith("Windows") || System.getProperty("os.name").startsWith("Mac"))

				println "Skipping ANNOTATE: impossible to access the '$modelfile' TreeTagger language model file."

				return false;

			}

			//			if (modelfile.getName().equals("sp.par")) {//UTF >> Latin1

			//				if(debug)

			//					println "fix encoding for model "+modelfile

			//				new EncodingConverter(f, "UTF-8", "ISO-8859-1")

			//			}

			cpb.tick()

			File infile = f;

			File outfile = new File(f.getParent(),f.getName()+"-out.tt");

			if (!applyTT(infile, outfile, modelfile)) {

				return false;

			}

			//			//Reconvert encoding if needed

			//			if (modelfile.getName().equals("sp.par")) {

			//				if(debug)

			//			}

		}

		println("")

		if (cancelNow) return;

		//BUILD STAND-OFF FILES

		if (modelfile == null) {

			println "no model applied"

			return false;

		}

		initTTOutfileInfos(binDir, modelfile, modelfilename);

		files = new File(binDir, "treetagger").listFiles(IOUtils.HIDDENFILE_FILTER)// now contains the result files of TT

		println "Building stdoff files ("+files.size()+") from dir:"+new File(binDir, "treetagger")+" to "+new File(binDir, "annotations");

				println("Failed to build standoff file of "+ttrezfile);

		}

		println("")

		if (cancelNow) return;

		//INJECT ANNOTATIONS

		List<File> interpfiles = new File(binDir, "annotations").listFiles(IOUtils.HIDDENFILE_FILTER);

		List<File> txmfiles = txmDir.listFiles(IOUtils.HIDDENFILE_FILTER);

				return false;

			}

			builder = null;

			//println "renaming files..."

			if (!(srcfile.delete() && temp.renameTo(srcfile)))

				println "Warning can't rename file "+temp+" to "+srcfile

		println("")

		return true;

	}

	public void setModelsDirectory(File modelsDirectory) {

		this.modelsDirectory = modelsDirectory;

	}

	/**

	 * Run file by file. Allow to have one different lang per file. Default behavior add new word properties

	 *

	{

		return run(binDir, txmDir, langs, false, new String[0], new String[0]);

	}

	/**

	 * Run file by file. Allow to have one different lang per file

	 *

			println("Path to TreeTagger is wrong "+TreeTaggerPreferences.getInstance().getString(TreeTaggerPreferences.INSTALL_PATH)+"/bin/")

			return true;

		}

		List<File> listfiles = txmDir.listFiles(IOUtils.HIDDENFILE_FILTER);

		//cleaning

		File annotDir = new File(binDir,"annotations");

		annotDir.deleteDir();

		File treetaggerDir = new File(binDir,"treetagger");

		treetaggerDir.deleteDir();

		treetaggerDir.mkdir();

		int cores = Runtime.getRuntime().availableProcessors()

		int coresToUse = Math.max(1.0, cores * 0.7)

		ExecutorService pool = Executors.newFixedThreadPool(coresToUse)

		def files = txmDir.listFiles(IOUtils.HIDDENFILE_FILTER)

		ConsoleProgressBar cpb = new ConsoleProgressBar(files.size())

		for (File teiFile : files) {

			int counter = 1;

			ThreadFile t = new ThreadFile("TT_"+counter++, teiFile) {

						public void run() {

							if (cancelNow) return;

							if (langs.get(f.getName()) == null) {

								println "Error: no lang defined for file $f"

								return;

							}

							String lang = langs.get(f.getName());

							run(f, lang, binDir, txmDir, replace, properties, options, annotDir, ptreetaggerDir, treetaggerDir)

							cpb.tick();

						}

					};

			pool.execute(t)

		}

		pool.shutdown()

		pool.awaitTermination(10, TimeUnit.HOURS)

		println ""

		return true;

	}

	public boolean run(File f, String lang, File binDir, File txmDir) {

		File annotDir = new File(binDir,"annotations");

		annotDir.mkdir();

		File ptreetaggerDir = new File(binDir,"ptreetagger");

		ptreetaggerDir.mkdir();

		File treetaggerDir = new File(binDir,"treetagger");

		treetaggerDir.mkdir();

		return run(f, lang, binDir, txmDir, false, new String[0], new String[0], annotDir, ptreetaggerDir, treetaggerDir)

	}

	public boolean run(File f, String lang, File binDir, File txmDir, boolean replace, String[] properties, String[] options, File annotDir, File ptreetaggerDir, File treetaggerDir) {

		File modelfile = new File(modelsDirectory, lang+".par");

		if (!"??".equals(lang) && !modelfile.exists()) {

			println "Error: No Modelfile available for lang "+modelfile+". Continue import process ";

		File ttsrcfile = new File(ptreetaggerDir, f.getName()+"-src.tt");

		File ttrezfile = new File(treetaggerDir, f.getName()+"-out.tt");

		//println ("TT with $model "+f+"+"+annotfile+" > "+ttsrcfile+" > "+ttrezfile);

		//BUILD TT FILE READY TO BE TAGGED

		def builder = new BuildTTSrc(f.toURL());

		builder.process(ttsrcfile, null);

		String tmpModelFileName = modelfile.getName()

		if (tmpModelFileName.startsWith("??")) {

			if (f.length() > LangDetector.MINIMALSIZE) {

		//Apply TT

		applyTT(ttsrcfile, ttrezfile, modelfile, options);

		//CREATE STANDOFF FILES

		initTTOutfileInfos(binDir, modelfile, lang, properties);

		writeStandoffFile(ttrezfile, annotfile)

		//INJECT ANNOTATIONS

		File tmpFile = new File(txmDir, "temp_"+f.getName())

		builder = new AnnotationInjection(f.toURL(), annotfile.toURL(), replace);

		builder.process(tmpFile);

		if (!(f.delete() && tmpFile.renameTo(f))) println "Warning can't rename file "+tmpFile+" to "+f

		return f.exists();

	}

	public void setCancelNow() {

		cancelNow = true;

	}

	public class ThreadFile extends Thread {

		File f;

		public ThreadFile(String name, File f) {