Index: TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CompressCQPIndexes.java
===================================================================
--- TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CompressCQPIndexes.java	(revision 0)
+++ TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CompressCQPIndexes.java	(revision 3463)
@@ -0,0 +1,166 @@
+// Copyright © 2010-2022 ENS de Lyon., University of Franche-Comté
+// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of
+// Lyon 2, University of Franche-Comté, University of Nice
+// Sophia Antipolis, University of Paris 3.
+// 
+// The TXM platform is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation,
+// either version 2 of the License, or (at your option) any
+// later version.
+// 
+// The TXM platform is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE. See the GNU General Public License for more
+// details.
+// 
+// You should have received a copy of the GNU General
+// Public License along with the TXM platform. If not, see
+// http://www.gnu.org/licenses.
+// 
+// 
+// 
+// $LastChangedDate: 2016-05-24 10:43:03 +0200 (Tue, 24 May 2016) $
+// $LastChangedRevision: 3216 $
+// $LastChangedBy: mdecorde $ 
+//
+package org.txm.importer.cwb;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.txm.core.messages.TXMCoreMessages;
+import org.txm.searchengine.cqp.core.messages.CQPSearchEngineCoreMessages;
+import org.txm.utils.OSDetector;
+import org.txm.utils.io.IOUtils;
+import org.txm.utils.logger.Log;
+
+/**
+ * The Class CompressCQPIndexes.
+ */
+public class CompressCQPIndexes {
+	
+	
+	public static boolean compressAll(File cqpToolsDirectory, File registryfile, String corpusid, File dataDirectory, boolean txm081fix)
+			throws IOException, InterruptedException {
+		//Runtime.getRuntime().
+		
+		if (!registryfile.exists()) {
+			System.out.println("No registry file found: "+registryfile);
+			return false;
+		}
+		
+		if (!dataDirectory.exists()) {
+			System.out.println("No data directory found: "+dataDirectory);
+			return false;
+		}
+		
+		File huff = new File(cqpToolsDirectory, "cwb-huffcode");
+		if (OSDetector.isFamilyWindows()) {
+			huff = new File(cqpToolsDirectory, "cwb-huffcode.exe");
+		}
+		
+		File rdxcompressor = new File(cqpToolsDirectory, "cwb-compress-rdx");
+		if (OSDetector.isFamilyWindows()) {
+			rdxcompressor = new File(cqpToolsDirectory, "cwb-compress-rdx.exe");
+		}
+		
+		if (!huff.exists()) {
+			System.out.println("No huff executable found: "+huff);
+			return false;
+		}
+		
+		if (!rdxcompressor.exists()) {
+			System.out.println("No rdxcompressor executable found: "+rdxcompressor);
+			return false;
+		}
+		
+		ArrayList<String> args = new ArrayList<>(Arrays.asList(huff.getAbsolutePath(), "-T", "-r", registryfile.getParent()));
+		
+//		ReadRegistryFile rrf = new ReadRegistryFile(registryfile);
+//		rrf.read();
+//		for (String p : rrf.pAttributes) {
+//			args.add("-P");
+//			args.add(p);
+//		}
+		args.add("-A");
+		args.add(corpusid);
+		
+		ProcessBuilder processBuilder = new ProcessBuilder(args);
+		processBuilder.redirectErrorStream(true);
+		Process process = processBuilder.start();
+		String messages = CwbProcess.getOutputMessages(process);
+		System.out.println(messages);
+		if (process.exitValue() != 0) {
+			System.out.println("Error while compressing with huff");
+			return false;
+		}
+		
+		ArrayList<String> args2 = new ArrayList<>(Arrays.asList(rdxcompressor.getAbsolutePath(), "-T", "-r", registryfile.getParent()));
+		
+//		for (String p : rrf.pAttributes) {
+//			File f = new File(dataDirectory, p+".corpus");
+//			if (f.length() > 0) {
+//				args2.add("-P");
+//				args2.add(p);
+//			}
+//		}
+		args2.add("-A");
+		args2.add(corpusid);
+		
+		processBuilder = new ProcessBuilder(args2);
+		processBuilder.redirectErrorStream(true);
+		process = processBuilder.start();
+		messages = CwbProcess.getOutputMessages(process);
+		System.out.println(messages);
+		if (process.exitValue() != 0) {
+			System.out.println("Error while compressing rdx files");
+			return false;
+		}
+		
+		// remove .corpus .corpus.rdx and corpus.rev files
+		int s = 0;
+		int a = 0;
+		for (File f : dataDirectory.listFiles()) {
+			if (f.getName().endsWith(".corpus") || f.getName().endsWith(".corpus.rdx") ||f.getName().endsWith(".corpus.rev")) {
+				s += f.length();
+				f.delete();
+				if (txm081fix) f.createNewFile();
+			}
+			if (f.getName().matches(".+(\\.hcd|\\.huf|\\.huf\\.syn|\\.crc|\\.crx)")) {
+				a += f.length();
+			}
+		}
+		
+		System.out.println("cleared: "+s);
+		System.out.println("created: "+a);
+		System.out.println("diff="+(s-a));
+		return true;
+	}
+
+	/**
+	 * The main method.
+	 *
+	 * @param args the arguments
+	 */
+	public static void main(String[] args) {
+		try {
+			String userdir = System.getProperty("user.home");
+			File tools = new File(userdir, "workspace-cpp/CWB-lib/src/builds/linux-64"); //$NON-NLS-1$
+			File registry = new File(userdir, "runtime-rcpapplication.product/corpora/NOV13-P1/registry/nov13-p1"); //$NON-NLS-1$
+			File data = new File(userdir, "runtime-rcpapplication.product/corpora/NOV13-P1/data/NOV13-P1"); //$NON-NLS-1$
+			CompressCQPIndexes.compressAll(tools, registry, "NOV13-P1", data, true);
+
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+}
\ No newline at end of file

Property changes on: TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CompressCQPIndexes.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

Index: TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/ReadRegistryFile.java
===================================================================
--- TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/ReadRegistryFile.java	(revision 3462)
+++ TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/ReadRegistryFile.java	(revision 3463)
@@ -130,7 +130,7 @@
 		ArrayList<String> errors = new ArrayList<>();
 		
 		// test p-attributes
-		String[] exts = { ".corpus", ".lexicon", ".corpus.cnt", ".corpus.rdx", ".corpus.rev", ".lexicon.idx", ".lexicon.srt" };
+		String[] exts = { ".lexicon", ".corpus.cnt", ".lexicon.idx", ".lexicon.srt" };
 		for (String p : pAttributes) {
 			for (String ext : exts) {
 				File f = new File(dataDirectory, p + ext);
@@ -141,6 +141,25 @@
 			}
 		}
 		
+		// test p-attributes with optional compression files
+		String[] extsCompressed = { ".corpus	.hcd	.huf	.huf.syn", ".corpus.rdx	.crc	.crx", ".corpus.rev	.crc	.crx"};
+		for (String p : pAttributes) {
+			for (String extsTabulated : extsCompressed) {
+				String[] exts2 = extsTabulated.split("\t");
+				String ext = exts2[0];
+				File f = new File(dataDirectory, p + ext);
+				if (!f.exists()) {
+					// System.out.println("MISSING: " + f.exists() + " " + f.getAbsolutePath());
+					for (int i = 1 ; i < exts2.length ; i++) {
+						f = new File(dataDirectory, p + exts2[i]);
+						if (!f.exists()) {
+							errors.add(f.getName());
+						}
+					}
+				}
+			}
+		}
+		
 		String[] sexts = { ".rng" };
 		String[] spexts = { ".avs", ".avx", ".rng" };
 		for (String s : sattrs.keySet()) {
Index: TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CwbProcess.java
===================================================================
--- TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CwbProcess.java	(revision 3462)
+++ TXM/trunk/org.txm.searchengine.cqp.core/src/org/txm/importer/cwb/CwbProcess.java	(revision 3463)
@@ -164,6 +164,32 @@
 		return true;
 	}
 	
+	public static String getOutputMessages(Process process) throws InterruptedException {
+		final InputStream is = process.getInputStream();
+		StringBuffer buffer = new StringBuffer();
+		Thread t = new Thread() {
+			@Override
+			public void run() {
+				try {
+					InputStreamReader isr = new InputStreamReader(is);
+					BufferedReader br = new BufferedReader(isr);
+					String line;
+					while ((line = br.readLine()) != null) {
+						buffer.append(line+"\n");
+					}
+					isr.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		};
+		t.start();
+		t.join();
+		
+		return buffer.toString();
+		
+	}
+	
 	public void endProcess() throws InterruptedException {
 		this.process.waitFor();
 
Index: TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/macro/projects/nov13/CreateTheOtherTurns.groovy
===================================================================
--- TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/macro/projects/nov13/CreateTheOtherTurns.groovy	(revision 3462)
+++ TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/macro/projects/nov13/CreateTheOtherTurns.groovy	(revision 3463)
@@ -132,7 +132,7 @@
 						writer.writeEndElement() // w
 					}
 					
-					def startOtherReg = /^\*([^*])?([^\p{Zs}]+.*)$/
+					def startOtherReg = /^([^*])?\*([^\p{Zs}]+.*)$/
 					def endOtherReg = /^(.*[^\p{Zs}]+)\*([^*])?$/
 					def startAndEndOtherReg = /^([^*])?\*(.*[^\p{Zs}]+)\*([^*])?$/
 					String previousOtherStarting = "<none>"
