Laboratoire ICAR » Plateforme TXM

		textPunctsStrong.setText(project.getTokenizerParameter("punct_strong", TokenizerClasses.punct_strong));

		textElisions.setText(project.getTokenizerParameter("regElision", TokenizerClasses.regElision));

		wordElementText.setText(wtagElement);

		boolean doTokenizeStep = project.getDoTokenizerStep();

		project.addTokenizerParameter("regPunct", textPuncts.getText()); //$NON-NLS-1$

		project.addTokenizerParameter("punct_strong", textPunctsStrong.getText()); //$NON-NLS-1$

		project.addTokenizerParameter("regElision", textElisions.getText()); //$NON-NLS-1$

		project.addTokenizerParameter("doTokenizeStep", ""+doTokenizeStepButton.getSelection()); //$NON-NLS-1$

		project.addTokenizerParameter("onlyThoseTests", "false"); //$NON-NLS-1$

		//project.setTokenizerAdditionalTests(additionalTokenizerTests); //$NON-NLS-1$

		//		}

		if (!rcpProject.getName().equals(pName)) { // project directory and project name must stay in sync

			IPath destination = new Path(this.getName());

			}

		}

		ImportEngines engines = Toolbox.getImportEngines();

	public String getTokenizerWordElement() {

		IEclipsePreferences params = this.getPreferencesScope().getNode("Tokenizer");

	}

	public boolean getDoTokenizerStep() {

			switch (event) {

				case XMLStreamConstants.START_ELEMENT:

					localname = parser.getLocalName()

						milestonesLength << mileStoneDistance;

						mileStoneDistance = 0

						milestonesCounter++;

						mileStoneDistance++

						start = true

					}

					break;

		milestonesLength[milestonesCounter] = mileStoneDistance;

		parser.close()

		milestonesCounter = 0;

	}

	public void processStartElement() {

		super.processStartElement();

			mileStoneDistance = 0

			mileStoneID = parser.getAttributeValue(null, "id")

			if (mileStoneID == null) mileStoneID = parser.getAttributeValue("xml", "id");

			if (mileStoneID == null) mileStoneID = "0";

			milestonesCounter++;

			// println "end of $milestonesCounter len="+milestonesLength[milestonesCounter]+" dist="+mileStoneDistance

			writer.writeAttribute(msEndAttributeName, Integer.toString((milestonesLength[milestonesCounter] - mileStoneDistance - 1)))

			writer.writeAttribute(msStartAttributeName, Integer.toString(mileStoneDistance))

			writer.writeAttribute(msIdAttributeName, mileStoneID)

			mileStoneDistance++

			start = true

		}

	}

	public static void main(String[] args) {

		println "Sucess: "+msp.process(outputFile)

	}

}

	/** The word_tags. */

	public static String word_tags = "w";

	/** The intraword_tags. */

	public static String intraword_tags = "c|ex|caesura";

		for (TTest test : tests) {

			System.out.println(" "+test+"");

		}

	}

	public static boolean isValid() {

			if (debug) System.out.println(" Tokenizer parametrized with "+key+"="+value+"");

				tag_all = value;

				enclitics = value;

				encliticsFR = value;

				div_tags = value;

				q_tags = value;

				extraword1_tags = value;

				corr_tags_no_seg = value;

				word_tags = value;

				intraword_tags = value;

				punct_quotes = value;

				punct_strong1 = value;

				punct_strong2 = value;

				punct_paren_open1 = value;

				punct_paren_open2 = value;

				punct_paren_close1 = value;

				punct_paren_close2 = value;

				punct_weak = value;

				entity = value;

				seg_tags = value;

				regPunct = value;

				regElision = value;

				whitespaces = value;

				tmp_strong_punct = value; // this is temporary

			else

				System.out.println("MISSING TOKENIZER KEY: "+key);

		String tmp_strong_punct = null;

		IEclipsePreferences params = projectScope.getNode("Tokenizer");

		try {

				//					Element param = (Element) params.item(i);

				//					String key = param.getAttribute("name");

				//					if (value == null || value.length() == 0) value = param.getTextContent();

				if (value.length() == 0) value = null;

				if (debug) System.out.println(" Tokenizer parametrized with "+key+"="+value+"");

					tag_all = value;

					enclitics = value;

					encliticsFR = value;

					div_tags = value;

					q_tags = value;

					extraword1_tags = value;

					corr_tags_no_seg = value;

					word_tags = value;

					intraword_tags = value;

					punct_quotes = value;

					punct_strong1 = value;

					punct_strong2 = value;

					punct_paren_open1 = value;

					punct_paren_open2 = value;

					punct_paren_close1 = value;

					punct_paren_close2 = value;

					punct_weak = value;

					entity = value;

					seg_tags = value;

					regPunct = value;

					regElision = value;

					whitespaces = value;

					tmp_strong_punct = value; // this is temporary

			}

			//recombine

			Log.info("\nEnd of Groovy import script: " + ExecTimer.stop()); //$NON-NLS-1$

			monitor.worked(90);

			Object ready = binding.getVariable("readyToLoad"); //$NON-NLS-1$

			if (ready != null && ready instanceof Boolean) {

					return false

				} else {

					if (inputFile.delete()) {

						outputFile.renameTo(inputFile)

					} else {

						println "Fail to encode $milestone in ${inputFile}: could not replace the file."

				File infile = f;

				File outfile = new File(module.getBinaryDirectory(),"tokenized/"+f.getName());

				SimpleTokenizerXml tokenizer = new SimpleTokenizerXml(infile, outfile, lang)

				if (outSideTextTagsRegex != null && outSideTextTagsRegex.trim().length() > 0) {

					tokenizer.setOutSideTextTags(outSideTextTagsRegex)

		corpusname = module.getProject().getName();

		regPath = module.getBinaryDirectory().getAbsolutePath() + "/registry/"+corpusname.toLowerCase()

		doNormalizeAttributeValues = "true".equals(module.getProject().getPreferencesScope().getNode("import").get(TBXPreferences.NORMALISEATTRIBUTEVALUES, "false"))

		doNormalizeAnaValues = "true".equals(module.getProject().getPreferencesScope().getNode("import").get(TBXPreferences.NORMALISEANAVALUES, "false"))

	/** The word_tags. */

	String word_tags = TokenizerClasses.word_tags;

	Pattern reg_word_tags;

	/** The intraword_tags. */

	String intraword_tags = TokenizerClasses.intraword_tags;

		word_tags = TokenizerClasses.word_tags;

		reg_word_tags = Pattern.compile(word_tags);

		intraword_tags = TokenizerClasses.intraword_tags;

		punct_strong = TokenizerClasses.punct_strong;

		punct_all = TokenizerClasses.punct_all;