Révision 2258

tmp/org.txm.utils/META-INF/MANIFEST.MF (revision 2258)
8 8
 org.eclipse.core.runtime;visibility:=reexport,
9 9
 org.eclipse.core.net;bundle-version="1.2.200";visibility:=reexport,
10 10
 org.txm.libs.colt;bundle-version="1.2.0";visibility:=reexport,
11
 org.txm.libs.itext;bundle-version="2.1.5";visibility:=reexport
11
 org.txm.libs.itext;bundle-version="2.1.5";visibility:=reexport,
12
 org.txm.libs.msoffice;bundle-version="1.0.0";visibility:=reexport,
13
 org.txm.libs.office;bundle-version="0.8.2";visibility:=reexport
12 14
Bundle-RequiredExecutionEnvironment: JavaSE-1.7
13 15
Export-Package: org.apache.commons.cli,
14 16
 org.apache.commons.io;uses:="org.apache.commons.io.filefilter",
tmp/org.txm.utils/src/org/txm/utils/CsvReader.java (revision 2258)
66 66
 * stream.
67 67
 */
68 68
public class CsvReader {
69
	
69

  
70 70
	/** The input stream. */
71 71
	private Reader inputStream = null;
72 72

  
......
905 905

  
906 906
									if (userSettings.SafetySwitch
907 907
											&& dataBuffer.Position
908
													- dataBuffer.ColumnStart
909
													+ columnBuffer.Position > 100000) {
908
											- dataBuffer.ColumnStart
909
											+ columnBuffer.Position > 100000) {
910 910
										close();
911 911

  
912 912
										throw new IOException(
913 913
												"Maximum column length of 100,000 exceeded in column " //$NON-NLS-1$
914
														+ NumberFormat
915
																.getIntegerInstance()
916
																.format(
917
																		columnsCount)
918
														+ " in record " //$NON-NLS-1$
919
														+ NumberFormat
920
																.getIntegerInstance()
921
																.format(
922
																		currentRecord)
923
														+ ". Set the SafetySwitch property to false" //$NON-NLS-1$
924
														+ " if you're expecting column lengths greater than 100,000 characters to" //$NON-NLS-1$
925
														+ " avoid this error."); //$NON-NLS-1$
914
												+ NumberFormat
915
												.getIntegerInstance()
916
												.format(
917
														columnsCount)
918
												+ " in record " //$NON-NLS-1$
919
												+ NumberFormat
920
												.getIntegerInstance()
921
												.format(
922
														currentRecord)
923
												+ ". Set the SafetySwitch property to false" //$NON-NLS-1$
924
												+ " if you're expecting column lengths greater than 100,000 characters to" //$NON-NLS-1$
925
												+ " avoid this error."); //$NON-NLS-1$
926 926
									}
927 927
								}
928 928
							} // end else
......
1158 1158

  
1159 1159
									if (userSettings.SafetySwitch
1160 1160
											&& dataBuffer.Position
1161
													- dataBuffer.ColumnStart
1162
													+ columnBuffer.Position > 100000) {
1161
											- dataBuffer.ColumnStart
1162
											+ columnBuffer.Position > 100000) {
1163 1163
										close();
1164 1164

  
1165 1165
										throw new IOException(
1166 1166
												"Maximum column length of 100,000 exceeded in column " //$NON-NLS-1$
1167
														+ NumberFormat
1168
																.getIntegerInstance()
1169
																.format(
1170
																		columnsCount)
1171
														+ " in record " //$NON-NLS-1$
1172
														+ NumberFormat
1173
																.getIntegerInstance()
1174
																.format(
1175
																		currentRecord)
1176
														+ ". Set the SafetySwitch property to false" //$NON-NLS-1$
1177
														+ " if you're expecting column lengths greater than 100,000 characters to" //$NON-NLS-1$
1178
														+ " avoid this error."); //$NON-NLS-1$
1167
												+ NumberFormat
1168
												.getIntegerInstance()
1169
												.format(
1170
														columnsCount)
1171
												+ " in record " //$NON-NLS-1$
1172
												+ NumberFormat
1173
												.getIntegerInstance()
1174
												.format(
1175
														currentRecord)
1176
												+ ". Set the SafetySwitch property to false" //$NON-NLS-1$
1177
												+ " if you're expecting column lengths greater than 100,000 characters to" //$NON-NLS-1$
1178
												+ " avoid this error."); //$NON-NLS-1$
1179 1179
									}
1180 1180
								}
1181 1181
							} // end else
......
1203 1203
				if (rawBuffer.Position == 0) {
1204 1204
					rawRecord = new String(dataBuffer.Buffer,
1205 1205
							dataBuffer.LineStart, dataBuffer.Position
1206
									- dataBuffer.LineStart - 1);
1206
							- dataBuffer.LineStart - 1);
1207 1207
				} else {
1208 1208
					rawRecord = new String(rawBuffer.Buffer, 0,
1209 1209
							rawBuffer.Position)
1210 1210
							+ new String(dataBuffer.Buffer,
1211 1211
									dataBuffer.LineStart, dataBuffer.Position
1212
											- dataBuffer.LineStart - 1);
1212
									- dataBuffer.LineStart - 1);
1213 1213
				}
1214 1214
			} else {
1215 1215
				// for hasMoreData to ever be false, all data would have had to
......
1261 1261

  
1262 1262
			System.arraycopy(dataBuffer.Buffer, dataBuffer.LineStart,
1263 1263
					rawBuffer.Buffer, rawBuffer.Position, dataBuffer.Count
1264
							- dataBuffer.LineStart);
1264
					- dataBuffer.LineStart);
1265 1265

  
1266 1266
			rawBuffer.Position += dataBuffer.Count - dataBuffer.LineStart;
1267 1267
		}
......
1387 1387

  
1388 1388
					currentValue = new String(dataBuffer.Buffer,
1389 1389
							dataBuffer.ColumnStart, lastLetter
1390
									- dataBuffer.ColumnStart + 1);
1390
							- dataBuffer.ColumnStart + 1);
1391 1391
				}
1392 1392
			} else {
1393 1393
				updateCurrentValue();
......
1415 1415

  
1416 1416
			throw new IOException(
1417 1417
					"Maximum column count of 100,000 exceeded in record " //$NON-NLS-1$
1418
							+ NumberFormat.getIntegerInstance().format(
1419
									currentRecord)
1420
							+ ". Set the SafetySwitch property to false" //$NON-NLS-1$
1421
							+ " if you're expecting more than 100,000 columns per record to" //$NON-NLS-1$
1422
							+ " avoid this error."); //$NON-NLS-1$
1418
					+ NumberFormat.getIntegerInstance().format(
1419
							currentRecord)
1420
					+ ". Set the SafetySwitch property to false" //$NON-NLS-1$
1421
					+ " if you're expecting more than 100,000 columns per record to" //$NON-NLS-1$
1422
					+ " avoid this error."); //$NON-NLS-1$
1423 1423
		}
1424 1424

  
1425 1425
		// check to see if our current holder array for
......
1687 1687
	 * The Class ComplexEscape.
1688 1688
	 */
1689 1689
	private class ComplexEscape {
1690
		
1690

  
1691 1691
		/** The Constant UNICODE. */
1692 1692
		private static final int UNICODE = 1;
1693 1693

  
......
1725 1725
	 * The Class DataBuffer.
1726 1726
	 */
1727 1727
	private class DataBuffer {
1728
		
1728

  
1729 1729
		/** The Buffer. */
1730 1730
		public char[] Buffer;
1731 1731

  
......
1766 1766
	 * The Class ColumnBuffer.
1767 1767
	 */
1768 1768
	private class ColumnBuffer {
1769
		
1769

  
1770 1770
		/** The Buffer. */
1771 1771
		public char[] Buffer;
1772 1772

  
......
1786 1786
	 * The Class RawRecordBuffer.
1787 1787
	 */
1788 1788
	private class RawRecordBuffer {
1789
		
1789

  
1790 1790
		/** The Buffer. */
1791 1791
		public char[] Buffer;
1792 1792

  
......
1798 1798
		 */
1799 1799
		public RawRecordBuffer() {
1800 1800
			Buffer = new char[StaticSettings.INITIAL_COLUMN_BUFFER_SIZE
1801
					* StaticSettings.INITIAL_COLUMN_COUNT];
1801
			                  * StaticSettings.INITIAL_COLUMN_COUNT];
1802 1802
			Position = 0;
1803 1803
		}
1804 1804
	}
......
1807 1807
	 * The Class Letters.
1808 1808
	 */
1809 1809
	private class Letters {
1810
		
1810

  
1811 1811
		/** The Constant LF. */
1812 1812
		public static final char LF = '\n';
1813 1813

  
......
1916 1916
	 * The Class HeadersHolder.
1917 1917
	 */
1918 1918
	private class HeadersHolder {
1919
		
1919

  
1920 1920
		/** The Headers. */
1921 1921
		public String[] Headers;
1922 1922

  
......
1955 1955
		/** The Constant INITIAL_COLUMN_BUFFER_SIZE. */
1956 1956
		public static final int INITIAL_COLUMN_BUFFER_SIZE = 50;
1957 1957
	}
1958

  
1959
	protected HashMap<String, String> getRecord() throws IOException {
1960
		HashMap<String, String> record = new HashMap<String, String>();
1961
		for (String h : getHeaders()) {
1962
			record.put(h, get(h));
1963
		}
1964

  
1965
		return record;
1966
	}
1958 1967
}
tmp/org.txm.utils/src/org/txm/utils/TableReader.java (revision 2258)
1
package org.txm.utils;
2

  
3
import java.io.File;
4
import java.io.IOException;
5
import java.nio.charset.Charset;
6
import java.util.Arrays;
7
import java.util.HashMap;
8

  
9
import org.txm.libs.msoffice.ReadExcel;
10
import org.txm.libs.office.ReadODS;
11

  
12
/**
13
 * Read a table from a TSV, XLSX or ODS file
14
 * 
15
 * @author mdecorde
16
 *
17
 */
18
public class TableReader {
19
	
20
	File tableFile;
21
	String mode = "tsv";
22
	
23
	HashMap<String, String> record;
24
	CsvReader tsvReader = null;
25
	ReadExcel excelReader = null;
26
	ReadODS odsReader;
27
	
28
	public TableReader(File tableFile) throws Exception {
29
		this.tableFile = tableFile;
30
		
31
		if (tableFile.getName().toLowerCase().endsWith(".xlsx")) {
32
			mode = "excel";
33
			excelReader = new ReadExcel(tableFile, null);
34
		} else if (tableFile.getName().toLowerCase().endsWith(".ods")) {
35
			mode = "ods";
36
			odsReader = new ReadODS(tableFile, null);
37
		} else if (tableFile.getName().toLowerCase().endsWith(".tsv")) {
38
			tsvReader = new CsvReader(tableFile.getAbsolutePath(), "\t".charAt(0), Charset.forName("UTF-8"));
39
			mode = "tsv";
40
		}
41
	}
42
	
43
	public String getMode() {
44
		return mode;
45
	}
46
	
47
	public boolean readHeaders() throws IOException {
48
		if (tsvReader != null) {
49
			return tsvReader.readHeaders();
50
		} else if (excelReader != null) {
51
			return excelReader.readHeaders();
52
		}
53
		
54
		return false;
55
	}
56
	
57
	/**
58
	 * read the next record
59
	 * @return true if there is another record
60
	 * @throws IOException
61
	 */
62
	public boolean readRecord() throws IOException {
63
		if (tsvReader != null) {
64
			if (tsvReader.readRecord()) {
65
				record = tsvReader.getRecord();
66
				return true;
67
			} else {
68
				return false;
69
			}
70
		} else if (excelReader != null) {
71
			if (excelReader.readRecord()) {
72
				record = excelReader.getRecord();
73
				return true;
74
			} else {
75
				return false;
76
			}
77
		}
78
		
79
		return false;
80
	}
81
	
82
	public HashMap<String, String> getRecord() {
83
		return record;
84
	}
85

  
86
	public String[] getHeaders() throws IOException {
87
		if (tsvReader != null) {
88
			return tsvReader.getHeaders();
89
		} else if (excelReader != null) {
90
			return excelReader.getHeaders();
91
		}
92
		
93
		return null;
94
	}
95
	
96
	String EMPTY = "";
97
	public String get(String h) throws IOException {
98
		String s = getRecord().get(h);
99
		if (s == null) return EMPTY;
100
		
101
		return s;
102
	}
103
	
104
	public void close() throws IOException {
105
		if (tsvReader != null) {
106
			tsvReader.close();
107
		} else if (excelReader != null) {
108
			excelReader.close();
109
		}
110
	}
111
	
112
	public static void main(String[] args) {
113
		try {
114
			System.out.println("Loading...");
115
			TableReader reader = new TableReader(new File("/home/mdecorde/TEMP/ANTRACT/AF/ANTRACT_AF_Notices_3oct19.xlsx"));
116
			System.out.println("Table loaded");
117
			
118
			System.out.println("Reading headers...");
119
			reader.readHeaders();
120
			System.out.println(Arrays.toString(reader.getHeaders()));
121
			
122
			System.out.println("Reading record...");
123
			reader.readRecord();
124
			System.out.println(reader.getRecord());
125
			
126
			System.out.println("Reading record...");
127
			reader.readRecord();
128
			System.out.println(reader.getRecord());
129
		} catch (Exception e) {
130
			// TODO Auto-generated catch block
131
			e.printStackTrace();
132
		}
133
	}
134
}
0 135

  
tmp/org.txm.libs.msoffice/src/org/txm/libs/msoffice/ReadExcel.java (revision 2258)
1 1
package org.txm.libs.msoffice;
2 2

  
3 3
import java.io.File;
4
import java.io.IOException;
5
import java.text.SimpleDateFormat;
4 6
import java.util.ArrayList;
7
import java.util.HashMap;
5 8

  
9
import org.apache.poi.EncryptedDocumentException;
10
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
6 11
import org.apache.poi.ss.usermodel.Cell;
12
import org.apache.poi.ss.usermodel.DateUtil;
7 13
import org.apache.poi.ss.usermodel.Row;
8 14
import org.apache.poi.ss.usermodel.Sheet;
9 15
import org.apache.poi.ss.usermodel.Workbook;
......
11 17

  
12 18
public class ReadExcel {
13 19

  
20
	Workbook wb = null;
21
	Sheet ws = null;
22
	String[] header = null;
23
	HashMap<String, String> record = null;
24
	int iRow = 0;
25
	int nRows;
26
	
27
	public ReadExcel(File tableFile, String sheetName) throws EncryptedDocumentException, InvalidFormatException, IOException {
28
		wb = WorkbookFactory.create(tableFile);
29
		
30
		if (sheetName == null || sheetName.length() == 0) {
31
			ws = wb.getSheetAt(0);
32
		} else {
33
			ws = wb.getSheet(sheetName);
34
			if (ws == null) {
35
				ws = wb.getSheetAt(0);
36
			}
37
		}
38
		
39
		nRows = ws.getPhysicalNumberOfRows();
40
	}
41
	
42
	public boolean readHeaders() {
43
		
44
		if (nRows == 0) {
45
			header = new String[0];
46
			return header != null;
47
		}
48

  
49
		Row firstRow = ws.getRow(0);
50
		int colMax = firstRow.getLastCellNum();
51

  
52
		header = new String[colMax];
53
		for (int it = 0 ; it < colMax; it++) {
54
			header[it] = firstRow.getCell(it).getStringCellValue();
55
		}
56
		
57
		if (iRow == 0) iRow = 1; // skip first line
58
		return true;
59
	}
60
	
61
	public boolean readRecord() {
62
		if (iRow < nRows) {
63
			_getRecord();
64
			iRow++; // for next 
65
			return true;
66
		} else { // end of file
67
			record = null;
68
			return false;
69
		}
70
	}
71
	
72
	public String[] getHeaders() {
73
		return header;
74
	}
75
	
76
	public HashMap<String, String> getRecord() {
77
		return record;
78
	}
79
	
80
	public String get(String h) {
81
		if (record == null) return null;
82
		
83
		return record.get(h);
84
	}
85
	
86
	protected void _getRecord() {
87
		record = new HashMap<String, String>();
88
		Row row = ws.getRow(iRow);
89
		
90
		for (int colIndex = 0 ; colIndex < header.length ; colIndex++) {
91
			String col = header[colIndex];
92
			Cell cell = row.getCell(colIndex);
93
			if (cell != null) {
94
				String value = cellToString(cell).trim();
95
				if (value == null) {
96
					record.put(header[colIndex], "");
97
				} else {
98
					record.put(header[colIndex], value);
99
				}
100
			} else {
101
				record.put(header[colIndex], "");
102
			}
103
		}
104
		
105
		for (String k : record.keySet()) if (record.get(k) == null) System.out.println("ERROR null value with "+k);
106
	}
107
	
108
	public void close() throws IOException {
109
		if (wb != null) wb.close();
110
	}
111

  
14 112
	/**
15 113
	 * 
16 114
	 * @param inputFile
17 115
	 * @param sheetName
18
	 * @return list of lines (line = list of cells) with values converted to String
116
	 * @return list of lines (line = list of cells) with values converted to String - the file is completely read
19 117
	 */
20 118
	public static ArrayList<ArrayList<String>> toTable(File inputFile, String sheetName) {
21 119

  
......
88 186
		case FORMULA:
89 187
			return "#formulla";
90 188
		case NUMERIC:
91
			return ""+cell.getNumericCellValue();
189
			if (DateUtil.isCellDateFormatted(cell)) {
190
                SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
191
                return dateFormat.format(cell.getDateCellValue());
192
            } else {
193
                Double value = cell.getNumericCellValue();
194
                return value.toString();
195
            }
92 196
		case STRING:
93 197
			return cell.getStringCellValue();
94 198
		case BLANK:
......
100 204
		default:
101 205
			return "#typeerror";
102 206
		}
103

  
104 207
	}
105 208

  
106 209
	public static void main(String[] args) {
tmp/org.txm.libs.office/src/org/txm/libs/office/ReadODS.java (revision 2258)
13 13

  
14 14
public class ReadODS {
15 15
	
16
	SpreadsheetDocument spreadsheet = null;
17
	Table table = null;
18
	String[] headers;
19
	
20
	public ReadODS(File tableFile, String sheetname) throws Exception {
21
		spreadsheet = SpreadsheetDocument.loadDocument(tableFile);
22

  
23
		// the first access to the document causes a very long process
24
		int sheetCount = spreadsheet.getSheetCount();
25
		if (sheetCount < 1) {
26
			System.out.println("** ReadODS: no sheet found in file. Aborting.");
27
			return;
28
		}
29
		
30
		Table table = null;
31
		if (sheetname != null) {
32
			table = spreadsheet.getSheetByName(sheetname);
33
		}
34
		if (table == null) {
35
			table = spreadsheet.getSheetByIndex(0);
36
		}
37
		
38
		//table.getRowByIndex(0);
39
	}
40
	
41
	public boolean readHeaders() {
42
		Row row = table.getRowByIndex(0);
43
		int ncolumns = row.getCellCount();
44
		
45
		headers = new String[ncolumns];
46
		for (int j = 0 ; j < ncolumns ; j++) {
47
			String cell = row.getCellByIndex(j).getDisplayText();
48
			if (cell == null) {
49
				headers[j] = "";
50
			} else {
51
				headers[j] = cell;
52
			}
53
		}
54
		
55
		return headers.length > 0;
56
	}
57
	
58
	public String[] getHeaders() {
59
		return headers;
60
	}
61
	
62
	public boolean readRecord() {
63
		return false;
64
	}
65
	
66
	public HashMap<String, String> getRecord() {
67
		return null;
68
	}
69

  
16 70
	public static ArrayList<HashMap<String, String>> toHashMap(File inputFile, String sheetname) throws Exception {
17 71
		SpreadsheetDocument spreadsheet = SpreadsheetDocument.loadDocument(inputFile);
18 72

  

Formats disponibles : Unified diff