Index: tmp/org.txm.libs.msoffice/src/org/txm/libs/msoffice/ReadExcel.java
===================================================================
--- tmp/org.txm.libs.msoffice/src/org/txm/libs/msoffice/ReadExcel.java	(revision 2356)
+++ tmp/org.txm.libs.msoffice/src/org/txm/libs/msoffice/ReadExcel.java	(revision 2357)
@@ -2,17 +2,14 @@
 
 import java.io.BufferedOutputStream;
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 
 import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
@@ -22,10 +19,10 @@
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.ss.usermodel.WorkbookFactory;
-import org.apache.poi.ss.util.SheetUtil;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 
 public class ReadExcel {
-
+	
 	Workbook wb = null;
 	Sheet ws = null;
 	String[] header = null;
@@ -35,23 +32,36 @@
 	private File tableFile;
 	
 	public ReadExcel(File tableFile, String sheetName) throws EncryptedDocumentException, InvalidFormatException, IOException {
-		wb = WorkbookFactory.create(tableFile);
+		
 		this.tableFile = tableFile;
 		
-		if (sheetName == null || sheetName.length() == 0) {
-			ws = wb.getSheetAt(0);
-		} else {
-			ws = wb.getSheet(sheetName);
-			if (ws == null) {
+		if (tableFile.exists() && tableFile.length() > 0) {
+			wb = WorkbookFactory.create(tableFile);
+			if (sheetName == null || sheetName.length() == 0) {
 				ws = wb.getSheetAt(0);
+			} else {
+				ws = wb.getSheet(sheetName);
+				if (ws == null) {
+					ws = wb.getSheetAt(0);
+				}
 			}
+			
+			nRows = ws.getPhysicalNumberOfRows();
+		} else {
+			wb = new XSSFWorkbook();
+			ws = wb.createSheet();
 		}
-		
-		nRows = ws.getPhysicalNumberOfRows();
 	}
 	
-	private void removeLines(HashMap<String, String> rules) {
+	/**
+	 * one rule (regex test) per column
+	 * all lines are processed
+	 * 
+	 * @param rules String: column name to test -> regex to match
+	 */
+	private boolean removeLines(HashMap<String, String> rules) {
 		Row headers = ws.getRow(0);
+		if (headers == null) return false;
 		int colMax = headers.getLastCellNum();
 		
 		ArrayList<Integer> columnIdxToTest = new ArrayList<Integer>();
@@ -67,56 +77,169 @@
 			}
 		}
 		
+		if (columnIdxToTest.size() == 0) {
+			System.out.println("Error: no column found with name="+rules.keySet());
+			return false;
+		}
+		
+		ArrayList<Integer> removed = new ArrayList<Integer>();
 		for (int rowIndex = 0 ; rowIndex < nRows ; rowIndex++) {
 			Row row = ws.getRow(rowIndex);
-			
+			if (row == null) continue;
 			for (int i = 0 ; i < columnIdxToTest.size() ; i++) {
 				int colIndex = columnIdxToTest.get(i);
 				
 				Cell cell = row.getCell(colIndex);
 				if (cell != null) {
 					String value = cellToString(cell).trim();
-					if (columnsTest.get(i).equals(value)) {
-						cell.setCellValue(rules.get("TODOOOOOOOOOOOOOOOOO"));
+					if (value.matches(columnsTest.get(i))) {
+						removed.add(0, rowIndex);
+						ws.removeRow(row);
+						break; // next row
 					}
 				}
 			}
 		}
+		
+		// TODO very slow when there is a lot of lines
+		for (int rowIndex : removed) {
+			ws.shiftRows(rowIndex+1, nRows, -1);
+			nRows--;
+		}
+		
+		return true;
 	}
 	
-	private void renameColumns(HashMap<String, String> rules) {
+	/**
+	 * extract lines and columns following rules.
+	 * 
+	 * don't forget to save the ReadeExcel object
+	 * 
+	 * @param excel2
+	 * @param lineRules
+	 * @param columnsSelection
+	 * @return
+	 * @throws EncryptedDocumentException
+	 * @throws InvalidFormatException
+	 * @throws IOException
+	 */
+	public boolean extractTo(ReadExcel excel2, HashMap<String, String> lineRules, List<String> columnsSelection) throws EncryptedDocumentException, InvalidFormatException, IOException {
+		Sheet ws2 = excel2.ws;
+		if (!extractTo(ws2, lineRules, columnsSelection)) {
+			return false;
+		}
+		
+		return true;
+	}
+	
+	/**
+	 * one rule (regex test) per column
+	 * all lines are processed
+	 * 
+	 * @param lineRules String: column name to test -> regex to match
+	 * @throws IOException 
+	 * @throws InvalidFormatException 
+	 * @throws EncryptedDocumentException 
+	 */
+	public boolean extractTo(Sheet ws2, HashMap<String, String> lineRules, List<String> columnsSelection) throws EncryptedDocumentException, InvalidFormatException, IOException {
 		Row headers = ws.getRow(0);
+		if (headers == null) return false;
+		
 		int colMax = headers.getLastCellNum();
+		
+		ArrayList<Integer> columnIdxToTest = new ArrayList<Integer>();
+		ArrayList<Integer> columnIdxToWrite = new ArrayList<Integer>();
+		ArrayList<String> columns = new ArrayList<String>();
+		ArrayList<String> columnsTest = new ArrayList<String>();
 		for (int colIndex = 0 ; colIndex < colMax ; colIndex++) {
 			Cell cell = headers.getCell(colIndex);
 			if (cell != null) {
 				String value = cellToString(cell).trim();
-				if (rules.containsKey(value)) {
-					cell.setCellValue(rules.get(value));
+				columns.add(value);
+				if (lineRules.containsKey(value)) {
+					columnIdxToTest.add(colIndex);
+					columnsTest.add(lineRules.get(value));
 				}
+				
+				if (columnsSelection.contains(value)) {
+					columnIdxToWrite.add(colIndex);
+				}
 			}
 		}
+		
+		if (columnIdxToTest.size() == 0) {
+			System.out.println("Error: no column found with name="+lineRules.keySet());
+			return false;
+		}
+		
+		if (columnIdxToWrite.size() != columnsSelection.size()) {
+			System.out.println("Error: not all columns found="+columnIdxToWrite+" of "+columnsSelection);
+			ArrayList<String> notFound = new ArrayList<String>(columnsSelection);
+			notFound.removeAll(columns);
+			System.out.println("NOT FOUND: "+notFound);
+			return false;
+		}
+		
+		Row row2 = ws2.createRow(0);
+		int nCell = 0;
+		for (int iCol : columnIdxToWrite) {
+			Cell cell2 = row2.createCell(nCell);
+			cell2.setCellValue(cellToString(headers.getCell(iCol)).trim());
+			nCell++;
+		}
+		
+		int nRowWritten = 1;
+		for (int rowIndex = 0 ; rowIndex < nRows ; rowIndex++) {
+			Row row = ws.getRow(rowIndex);
+			if (row == null) continue;
+			for (int i = 0 ; i < columnIdxToTest.size() ; i++) {
+				int colIndex = columnIdxToTest.get(i);
+				
+				Cell cell = row.getCell(colIndex);
+				if (cell != null) {
+					String value = cellToString(cell).trim();
+					if (value.matches(columnsTest.get(i))) {
+						
+						// write new line
+						
+						row2 = ws2.createRow(nRowWritten);
+						nCell = 0;
+						for (int iCol : columnIdxToWrite) {
+							Cell cell2 = row2.createCell(nCell);
+							cell2.setCellValue(cellToString(row.getCell(iCol)));
+							nCell++;
+						}
+						nRowWritten++;
+						
+						break; // next row
+					}
+				}
+			}
+		}
+		
+		return true;
 	}
 	
-	public void keepColumns(Collection<String> columnsToKeep) {
+	/**
+	 * 
+	 * @param rules String:old column name ->  String:new column name
+	 */
+	private boolean renameColumns(HashMap<String, String> rules) {
 		Row headers = ws.getRow(0);
+		if (headers == null) return false;
+		
 		int colMax = headers.getLastCellNum();
-		ArrayList<Integer> columnsToDelete = new ArrayList<Integer>();
 		for (int colIndex = 0 ; colIndex < colMax ; colIndex++) {
 			Cell cell = headers.getCell(colIndex);
 			if (cell != null) {
 				String value = cellToString(cell).trim();
-				if (columnsToKeep.contains(value)) {
-					columnsToDelete.add(colIndex);
+				if (rules.containsKey(value)) {
+					String s = rules.get(value);
+					cell.setCellValue(s);
 				}
 			}
 		}
-		
-		Collections.sort(columnsToDelete);
-
-		for (int i = columnsToDelete.size() -1 ; i >= 0 ; i--) {
-			SheetUtility.deleteColumn(ws, i);
-		}
+		return true;
 	}
 	
 	public boolean save() throws IOException {
@@ -136,10 +259,10 @@
 			header = new String[0];
 			return header != null;
 		}
-
+		
 		Row firstRow = ws.getRow(0);
 		int colMax = firstRow.getLastCellNum();
-
+		
 		header = new String[colMax];
 		for (int it = 0 ; it < colMax; it++) {
 			header[it] = firstRow.getCell(it).getStringCellValue();
@@ -199,7 +322,7 @@
 	public void close() throws IOException {
 		if (wb != null) wb.close();
 	}
-
+	
 	/**
 	 * 
 	 * @param inputFile
@@ -207,14 +330,14 @@
 	 * @return list of lines (line = list of cells) with values converted to String - the file is completely read
 	 */
 	public static ArrayList<ArrayList<String>> toTable(File inputFile, String sheetName) {
-
+		
 		ArrayList<ArrayList<String>> data = new ArrayList<ArrayList<String>>();
-
+		
 		if (!inputFile.canRead()) {
 			System.out.println("** Excel2XML: '"+inputFile.getName()+"' file not readable. Aborting.");
 			return data;
 		}
-
+		
 		try {
 			Workbook wb = WorkbookFactory.create(inputFile);
 			Sheet ws;
@@ -226,23 +349,23 @@
 					ws = wb.getSheetAt(0);
 				}
 			}
-
+			
 			if (ws == null) {
 				System.out.println("** Excel2XML: no sheet found. Aborting.");
 				return data;
 			}
-
+			
 			int nRows = ws.getPhysicalNumberOfRows();
 			if (nRows == 0) return null;
-
+			
 			Row firstRow = ws.getRow(0);
 			int colMax = firstRow.getLastCellNum();
-
+			
 			ArrayList<String> headers = new ArrayList<String>();
 			for (int it = 0 ; it < colMax; it++) {
 				headers.add(firstRow.getCell(it).getStringCellValue());
 			}
-
+			
 			for (int rowIndex = 0 ; rowIndex < nRows ; rowIndex++) {
 				Row row = ws.getRow(rowIndex);
 				ArrayList<String> dataLine = new ArrayList<String>();
@@ -263,66 +386,93 @@
 			e.printStackTrace();
 			return null;
 		}
-
+		
 		return data;
 	}
-
+	
 	/**
 	 * 
-	 * @param cell
+	 * @param cell. If null returns ""
 	 * @return always a String starting with '#' if an error occurs 
 	 */
 	public static String cellToString(Cell cell) {
+		if (cell == null) return "";
+		
 		switch (cell.getCellTypeEnum()) {
-		case FORMULA:
-			return "#formulla";
-		case NUMERIC:
-			if (DateUtil.isCellDateFormatted(cell)) {
-                SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
-                return dateFormat.format(cell.getDateCellValue());
-            } else {
-                Double value = cell.getNumericCellValue();
-                return value.toString();
-            }
-		case STRING:
-			return cell.getStringCellValue();
-		case BLANK:
-			return "";
-		case BOOLEAN:
-			return ""+cell.getBooleanCellValue();	
-		case ERROR:
-			return "#error";
-		default:
-			return "#typeerror";
+			case FORMULA:
+				return "#formulla";
+			case NUMERIC:
+				if (DateUtil.isCellDateFormatted(cell)) {
+					SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+					return dateFormat.format(cell.getDateCellValue());
+				} else {
+					Double value = cell.getNumericCellValue();
+					return value.toString();
+				}
+			case STRING:
+				return cell.getStringCellValue();
+			case BLANK:
+				return "";
+			case BOOLEAN:
+				return ""+cell.getBooleanCellValue();	
+			case ERROR:
+				return "#error";
+			default:
+				return "#typeerror";
 		}
 	}
-
+	
 	public static void main(String[] args) throws Exception {
-//		ArrayList<ArrayList<String>> data = toTable(new File("/home/mdecorde/xml/ruscorpora1m-test/metadata.xlsx"), null);
-//		if (data.size() == 0) {
-//			System.out.println("no data.");
-//		} else {
-//			System.out.println(data);
-//		}
+		//		ArrayList<ArrayList<String>> data = toTable(new File("/home/mdecorde/xml/ruscorpora1m-test/metadata.xlsx"), null);
+		//		if (data.size() == 0) {
+		//			System.out.println("no data.");
+		//		} else {
+		//			System.out.println(data);
+		//		}
 		
-		File tableFile = new File("/home/mdecorde/TEMP/ANTRACT/AF/mini trs fixed/metadata.xlsx");
-		File table2File = new File("/home/mdecorde/TEMP/ANTRACT/AF/mini trs fixed/metadata2.xlsx");
+		File tableFile = new File("/home/mdecorde/TEMP/ANTRACT/AF/all.xlsx");
+		File table2File = null;
 		
 		System.out.println("open...");
 		ReadExcel excel = new ReadExcel(tableFile, null);
-		System.out.println("keeping...");
-		excel.keepColumns(Arrays.asList("Date de diffusion", "Durée", "Identifiant de la notice", "Nom fichier segmenté (info)", "Notes du titre", "Titre propre"));
-		System.out.println("renaming...");
+		
+		HashMap<String, String> lineRules = new HashMap<String, String>(); // line tests to select line to keep
+		List<String> columnsSelection; // lsit of columns to keep
+		
+
+//		//emissions
+//		table2File = new File("/home/mdecorde/TEMP/ANTRACT/AF/emissions.xlsx");
+//		columnsSelection = Arrays.asList(
+//				"Identifiant de la notice", "Titre propre", "Notes du titre", "Date de diffusion", "Durée", "Nom fichier segmenté (info)", "antract_video",
+//				"antract_debut","antract_fin","antract_duree","antract_tc_type","antract_tc_date");
+//		lineRules.put("Type de notice", "Notice sommaire");
+		
+		//sujets
+		table2File = new File("/home/mdecorde/TEMP/ANTRACT/AF/sujets.xlsx");
+		columnsSelection = Arrays.asList(
+				"Identifiant de la notice", "Titre propre", "Notes du titre", "Lien notice principale",
+				"Date de diffusion", "Type de date", "Durée", "Genre", "Langue VO / VE", "Nature de production", "Producteurs (Aff.)", "Thématique",
+				"Nom fichier segmenté (info)", "antract_video", "antract_debut","antract_fin","antract_duree","antract_tc_type","antract_tc_date",
+				"Résumé", "Séquences", "Descripteurs (Aff. Lig.)", "Générique (Aff. Lig.)"
+				);
+		lineRules.put("Type de notice", "Notice sujet");
+		
+		table2File.delete();
+		ReadExcel excel2 = new ReadExcel(table2File, null);
+		if (!excel.extractTo(excel2, lineRules, columnsSelection)) {
+			System.out.println("FAIL");
+			return;
+		}
+		
+		System.out.println("renaming column...");
 		HashMap<String, String> rules = new HashMap<String, String>();
-		rules.put("Date de diffusion", "date_diffusion");
-		rules.put("Durée", "duree");
-		rules.put("Identifiant de la notice", "");
-		rules.put("Nom fichier segmenté (info)", "fichier_segmente");
-		rules.put("Notes du titre", "subtitle");
-		rules.put("Titre propre", "title");
-		excel.renameColumns(rules);
-		System.out.println("saving...");
-		excel.saveAs(table2File);
+		rules.put("Identifiant de la notice", "id");
+		excel2.renameColumns(rules);
+		
+		System.out.println("saving&closing...");
+		excel2.save();
+		excel2.close();
+		excel.close();
 		System.out.println("done.");
 	}
 }
