Index: TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/xml/importer.groovy
===================================================================
--- TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/xml/importer.groovy	(revision 3343)
+++ TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/xml/importer.groovy	(revision 3344)
@@ -164,8 +164,8 @@
 				File infile = f
 				File outfile = new File(binDir, "tokenized/"+f.getName())
 				SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, TokenizerClasses.newTokenizerClasses(project.getPreferencesScope(), lang))
-				boolean retokenize = "true" == project.getTokenizerParameter("doRetokenizeStep", "false")
-				tokenizer.setRetokenize(retokenize)
+				tokenizer.setRetokenize("true" == project.getTokenizerParameter("doRetokenizeStep", "false"))
+				tokenizer.setDoBuildWordIDs("true" == project.getTokenizerParameter("doBuildWordIds", "true"))
 				if (ignoredElements != null && ignoredElements.trim().length() > 0) {
 					tokenizer.setOutSideTextTagsAndKeepContent(ignoredElements)
 				}
Index: TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/transcriber/importer.groovy
===================================================================
--- TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/transcriber/importer.groovy	(revision 3343)
+++ TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/transcriber/importer.groovy	(revision 3344)
@@ -188,6 +188,7 @@
 			try {
 				TranscriberTokenizer tokenizer = new TranscriberTokenizer(pfile, tfile, lang)
 				tokenizer.setRetokenize("true" == project.getTokenizerParameter("doRetokenizeStep", "false"))
+				tokenizer.setDoBuildWordIDs("true" == project.getTokenizerParameter("doBuildWordIds", "true"))
 				if (!tokenizer.process()) {
 					println("Failed to tokenize "+pfile)
 				}
Index: TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/xtz/XTZImporter.groovy
===================================================================
--- TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/xtz/XTZImporter.groovy	(revision 3343)
+++ TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/importer/xtz/XTZImporter.groovy	(revision 3344)
@@ -340,7 +340,6 @@
 		new File(module.getBinaryDirectory(),"tokenized").deleteDir()
 		new File(module.getBinaryDirectory(),"tokenized").mkdir()
 		
-		boolean retokenize = "true" == module.getProject().getTokenizerParameter("doRetokenizeStep", "false")
 		String outSideTextTagsRegex = "";
 		String outSideTextTagsAndKeepContentRegex = "";
 		String noteRegex = "";
@@ -398,7 +397,8 @@
 						tokenizer.setStringTokenizer(stringTokenizer)
 					}
 				}
-				tokenizer.setRetokenize(retokenize)
+				tokenizer.setRetokenize("true" == project.getTokenizerParameter("doRetokenizeStep", "false"))
+				tokenizer.setDoBuildWordIDs("true" == project.getTokenizerParameter("doBuildWordIds", "true"))
 				if (outSideTextTagsRegex != null && outSideTextTagsRegex.trim().length() > 0) {
 					tokenizer.setOutSideTextTags(outSideTextTagsRegex)
 				}
Index: TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/filters/Tokeniser/SimpleTokenizerXml.groovy
===================================================================
--- TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/filters/Tokeniser/SimpleTokenizerXml.groovy	(revision 3343)
+++ TXM/trunk/org.txm.groovy.core/src/groovy/org/txm/scripts/filters/Tokeniser/SimpleTokenizerXml.groovy	(revision 3344)
@@ -61,6 +61,8 @@
 	
 	StringTokenizer stringTokenizer;
 	boolean retokenize = false
+	boolean doBuildWordIDs = true
+	ArrayList noIdWrittenErrors = new ArrayList()
 	LinkedHashMap<String, String>retokenizedWordProperties = new LinkedHashMap()
 	
 	/** The word_tags. */
@@ -183,7 +185,7 @@
 		if (event == XMLStreamConstants.START_ELEMENT ) {
 			
 			localname = parser.getLocalName();
-			if (wordid != null) {
+			if (wordid != null) { // normalizing word elements
 				localname = word_element_to_create;
 			}
 			
@@ -210,30 +212,22 @@
 				}
 			}
 			
-			String attrprefix, attname;
-			boolean hasId = false;
-			//boolean hasType = false
-			boolean hasN = false
-			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
-				attname = parser.getAttributeLocalName(i);
-				attrprefix = parser.getAttributePrefix(i);
-				if ("id".equals(attname)) hasId = true;
-				//if ("type".equals(attname)) hasType = true;
-				if ("n".equals(attname)) hasN = true;
-				
-				if (attrprefix != null && attrprefix.length() > 0) {
-					writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))
-				} else {
-					writer.writeAttribute(attname, parser.getAttributeValue(i))
+			if (word_element_to_create.equals(localname)) { // the localname has been normalized to word_element_to_create
+				writeWordAttributes(true)
+			} else {
+				String attrprefix, attname;
+				for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+					attname = parser.getAttributeLocalName(i);
+					attrprefix = parser.getAttributePrefix(i);
+					
+					if (attrprefix != null && attrprefix.length() > 0) {
+						writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))
+					} else {
+						writer.writeAttribute(attname, parser.getAttributeValue(i))
+					}
 				}
 			}
 			
-			if (wordid != null && !hasId && localname == word_element_to_create) {
-				writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
-			}
-			if (!hasN && localname == word_element_to_create) {
-				writer.writeAttribute("n", ""+wordcount);
-			}
 			if (!reg_word_tags.matcher(localname).matches()) {
 				writer.writeCharacters("\n");
 			}
@@ -433,6 +427,10 @@
 				previousEvent = event;
 			}
 			
+			if (noIdWrittenErrors.size() > 0) {
+				println "Missing word IDs in source XML: "+noIdWrittenErrors.join(", ")
+			}
+			
 			parser.close()
 			writer.close();
 			output.close();
@@ -453,6 +451,10 @@
 		this.retokenize = retokenize
 	}
 	
+	public void setDoBuildWordIDs(boolean doBuildWordIDs) {
+		this.doBuildWordIDs = doBuildWordIDs
+	}
+	
 	/**
 	 * Set the element and content to ignore
 	 * 
@@ -517,7 +519,7 @@
 			for (def word : words) {
 				wordcount++;
 				writer.writeStartElement(word_element_to_create);
-				writeWordAttributes();// id
+				writeWordAttributes(false);// id
 				writer.writeCharacters(word);
 				writer.writeEndElement();
 				writer.writeCharacters("\n");
@@ -533,18 +535,52 @@
 	 *
 	 * @return the java.lang. object
 	 */
-	protected writeWordAttributes() {
-		writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
-		writer.writeAttribute("n",""+wordcount);
+	protected writeWordAttributes(boolean fromParser) {
+		boolean idWritten = doBuildWordIDs;
+		if (doBuildWordIDs) {
+			writer.writeAttribute("id", "w_"+filename+"_"+wordcount);
+			writer.writeAttribute("n",""+wordcount);
+		}
+		
+		// rewrite the properties of the retokenized word
 		for (String attr : retokenizedWordProperties.keySet()) {
-			if ("id" == attr) {
-				writer.writeAttribute("old-id", retokenizedWordProperties[attr]);
-			} else if ("n" == attr) {
-				writer.writeAttribute("old-n", retokenizedWordProperties[attr]);
+			if (doBuildWordIDs && "id" == attr) {
+				writer.writeAttribute("foreign-id", retokenizedWordProperties[attr]);
+			} else if (doBuildWordIDs && "n" == attr) {
+				writer.writeAttribute("foreign-n", retokenizedWordProperties[attr]);
 			} else {
 				writer.writeAttribute(attr, retokenizedWordProperties[attr]);
+				if ("id" == attr) idWritten = true;
 			}
 		}
+		if (fromParser) {
+			String attrprefix, attname;
+			for (int i = 0 ; i < parser.getAttributeCount() ; i++) {
+				attname = parser.getAttributeLocalName(i);
+				attrprefix = parser.getAttributePrefix(i);
+				
+				if (doBuildWordIDs && "id" == attname) {
+					writer.writeAttribute("foreign-id", parser.getAttributeValue(i));
+				} else if (doBuildWordIDs && "n" == attname) {
+					writer.writeAttribute("foreign-n", parser.getAttributeValue(i));
+				} else {
+					if (attrprefix != null && attrprefix.length() > 0) {
+						writer.writeAttribute(attrprefix+":"+attname, parser.getAttributeValue(i))
+					} else {
+						writer.writeAttribute(attname, parser.getAttributeValue(i))
+					}
+					if ("id" == attname) idWritten = true;
+				}
+			}
+		}
+		if (!idWritten && noIdWrittenErrors.size() < 20) {
+			noIdWrittenErrors << parser.getLocation().toString();
+			if (noIdWrittenErrors.size() == 20) {
+				noIdWrittenErrors << "...";
+			}
+		}
+		
+		
 	}
 	
 	public void setStartTag(String tag)
Index: TXM/trunk/org.txm.rcp/src/main/java/org/txm/rcp/editors/imports/sections/TokenizerSection.java
===================================================================
--- TXM/trunk/org.txm.rcp/src/main/java/org/txm/rcp/editors/imports/sections/TokenizerSection.java	(revision 3343)
+++ TXM/trunk/org.txm.rcp/src/main/java/org/txm/rcp/editors/imports/sections/TokenizerSection.java	(revision 3344)
@@ -34,6 +34,8 @@
 	
 	Button doRetokenizeStepButton;
 	
+	Button doBuildWordIDsButton;
+	
 	Text textElisions;
 	
 	Text textPuncts;
@@ -111,6 +113,13 @@
 		gdata.colspan = 3;
 		doRetokenizeStepButton.setLayoutData(gdata);
 		
+		doBuildWordIDsButton = toolkit.createButton(sectionClient, "Build word identifiers", SWT.CHECK);
+		doBuildWordIDsButton.setToolTipText(TXMUIMessages.performWordSegmentationWithinWord);
+		doBuildWordIDsButton.setSelection(true);
+		doBuildWordIDsButton.setVisible(moduleParams.get(ImportModuleCustomization.ADVANCEDTOKENIZER));
+		gdata = new TableWrapData(TableWrapData.LEFT, TableWrapData.FILL);
+		gdata.colspan = 3;
+		doBuildWordIDsButton.setLayoutData(gdata);
 		
 		Label l0 = toolkit.createLabel(sectionClient, TXMUIMessages.separatorCharacters, SWT.WRAP);
 		gdata = new TableWrapData(TableWrapData.LEFT, TableWrapData.FILL);
@@ -303,6 +312,7 @@
 	protected void updateFieldsEnableState() {
 		boolean enabled = doTokenizeStepButton.getSelection();
 		doRetokenizeStepButton.setEnabled(enabled);
+		doBuildWordIDsButton.setEnabled(enabled);
 		textWhiteSpaces.setEnabled(enabled);
 		textPuncts.setEnabled(enabled);
 		textElisions.setEnabled(enabled);
@@ -329,6 +339,8 @@
 		doTokenizeStepButton.setSelection(doTokenizeStep);
 		boolean doRetokenizeStep = "true".equals(project.getTokenizerParameter("doRetokenizeStep", "false")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 		doRetokenizeStepButton.setSelection(doRetokenizeStep);
+		boolean doBuildWordIds = "true".equals(project.getTokenizerParameter("doBuildWordIds", "true")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+		doBuildWordIDsButton.setSelection(doBuildWordIds);
 		updateFieldsEnableState();
 		
 		// HashMap<String, TTest> additionalTests = project.getTokenizerAdditionalTests();
@@ -351,6 +363,7 @@
 		project.addTokenizerParameter("word_tags", wordElementText.getText()); //$NON-NLS-1$
 		project.addTokenizerParameter("doTokenizeStep", "" + doTokenizeStepButton.getSelection()); //$NON-NLS-1$ //$NON-NLS-2$
 		project.addTokenizerParameter("doRetokenizeStep", "" + doRetokenizeStepButton.getSelection()); //$NON-NLS-1$ //$NON-NLS-2$
+		project.addTokenizerParameter("doBuildWordIds", "" + doBuildWordIDsButton.getSelection()); //$NON-NLS-1$ //$NON-NLS-2$
 		project.addTokenizerParameter("onlyThoseTests", "false"); //$NON-NLS-1$ //$NON-NLS-2$
 		// project.setTokenizerAdditionalTests(additionalTokenizerTests); //$NON-NLS-1$
 		
