Laboratoire ICAR » Plateforme TXM

Require-Bundle: org.txm.index.rcp;bundle-version="1.0.0";visibility:=reexport,

 org.txm.wordcloud.core;bundle-version="1.0.0";visibility:=reexport,

 org.txm.chartsengine.rcp;visibility:=reexport

Require-Bundle: org.txm.textsbalance.core;bundle-version="1.0.0";visibility:=reexport,

 org.txm.chartsengine.rcp;bundle-version="1.0.0";visibility:=reexport

import org.txm.importer.PersonalNamespaceContext;

import org.txm.importer.PersonalNamespaceContext;

import org.txm.importer.PersonalNamespaceContext;

	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins">

		<accessrules>

			<accessrule kind="accessible" pattern="**"/>

		</accessrules>

	</classpathentry>

Require-Bundle: org.txm.concordance.rcp;visibility:=reexport,

 org.txm.para.core;visibility:=reexport

      <import plugin="org.txm.searchengine.cqp.core" version="1.1.0" match="greaterOrEqual"/>

      <import plugin="org.txm.chartsengine.jfreechart.core" version="1.0.0" match="greaterOrEqual"/>

      <import plugin="org.txm.chartsengine.r.core" version="1.0.0" match="greaterOrEqual"/>

Require-Bundle: org.txm.rcp;visibility:=reexport,

 org.txm.searchengine.cqp.rcp;bundle-version="1.0.0"

groovy.compiler.level=-1

groovy.script.filters=**/*.dsld,y,**/*.gradle,n

 rsearch,org.txm.s

package org.txm.importer.tigersearch;

import java.io.File;

import java.util.ArrayList;

import ims.tiger.index.writer.*

import ims.tiger.system.*

import org.txm.Toolbox;

import org.txm.importer.ApplyXsl2;

import org.txm.importer.xtz.*

import org.txm.objects.BaseParameters

import org.txm.utils.BundleUtils;

import org.txm.utils.io.FileCopy;

import org.apache.log4j.BasicConfigurator;

class TSImport extends XTZImport {

	public TSImport(BaseParameters params) {

		super(params);

	}

	@Override

	public void init(BaseParameters p) {

		super.init(p);

		importer = new TSImporter(this); // only to build metadata

		compiler = new XTZCompiler(this)

		annotater = null; // no annotater step to do

		pager = new XTZPager(this)

	}

	/**

	 * Do a XTZ Import then build the TIGERSearch indexes in the binary corpus "tiger" directory

	 */

	@Override

	public void start() throws InterruptedException {		

		File tigerSrcDir = sourceDirectory

		def xmlFiles = []

		sourceDirectory.listFiles(new FileFilter() {

			boolean accept(File file) {

				if (file.isDirectory()) return false;

				if (file.isHidden()) return false;

				if (file.getName().equals("import.xml")) return false;

				if (!file.getName().endsWith(".xml")) return false;

				xmlFiles << file

			}

		});

		xmlFiles.remove(new File(sourceDirectory, "import.xml"))

		if (xmlFiles.size() == 0) {

			println "Error no XML file found in $sourceDirectory"

			isSuccessful = false;

			return;

		}

		File master = xmlFiles[0];

		println "Main TIGER XML file found: $master"

		File tsXSLFile = new File(Toolbox.getTXMHOMEPATH(), "xsl/ts.xsl");

		BundleUtils.copyFiles("TIGERSearchRCP", "src", "org/txm/importer/tigersearch", "ts.xsl", tsXSLFile.getParentFile());

		File xmltxmSrcDir = new File(binaryDirectory, "src"); // output directory of the TS XSL transformation

		xmltxmSrcDir.mkdirs();

		FileCopy.copy(master, new File(xmltxmSrcDir, master.getName()));

		if (!ApplyXsl2.processImportSources(tsXSLFile, xmltxmSrcDir, xmltxmSrcDir)) {

			println "Error while applying TS XSL file to $tigerSrcDir"

			isSuccessful = false;

			return;

		}

		File[] files = xmltxmSrcDir.listFiles();

		if (files == null || files.length == 0) {

			println "Error while applying TS XSL file to $xmltxmSrcDir is empty"

			isSuccessful = false;

			return;

		}

		sourceDirectory = xmltxmSrcDir; // hop

		File txmDir = new File(binaryDirectory, "txm/"+corpusName); 

		txmDir.mkdirs();

		FileCopy.copyFiles(sourceDirectory, txmDir) // the compiler step will use these files

		super.start(); // call the usual XTZ import 

		if (isSuccessful) {

			File tigerDir = new File(binaryDirectory, "tiger");

			tigerDir.mkdir();

				File logprop = new File(tigerDir, "tigersearch.logprop");

				logprop.withWriter("UTF-8") { writer ->

					writer.write("""# Default log configuration of the TIGERSearch suite

log4j.rootLogger=WARN,Logfile

log4j.logger.ims.tiger.gui.tigersearch.TIGERSearch=WARNING

log4j.appender.Logfile=org.apache.log4j.RollingFileAppender

log4j.appender.Logfile.File=\${user.home}/tigersearch/tigersearch.log

log4j.appender.Logfile.MaxFileSize=500KB

log4j.appender.Logfile.MaxBackupIndex=1

log4j.appender.Logfile.layout=org.apache.log4j.PatternLayout

log4j.appender.Logfile.layout.ConversionPattern=%5r %-5p [%t] %c{2} - %m%n""")	

				}

				BasicConfigurator.configure();

				String uri = master.getAbsolutePath();

				File tigerBinDir = new File(tigerDir, corpusName)

				tigerBinDir.mkdir()

				try {

					IndexBuilderErrorHandler handler = new SimpleErrorHandler(tigerBinDir.getAbsolutePath());

					XMLIndexing indexing = new XMLIndexing(corpusName, uri, tigerBinDir.getAbsolutePath(), handler,false);

					indexing.startIndexing();

				}

				catch (Exception e) { System.out.println(e.getMessage()); }

			}

	}

}

<?xml version="1.0" encoding="UTF-8"?>

  <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

 <!-- ==================================================================

      XML Schema for the TIGER-XML format

      http://www.ims.uni-stuttgart.de/projekte/TIGER/public/TigerXML.xsd

      ==================================================================

      TIGER Project, Wolfgang Lezius

      IMS, University of Stuttgart, 04/01/2003

      ================================================================== -->

  <!-- ======================================================

       INCLUDES DECLARATION OF THE HEADER

       ====================================================== -->

  <xs:include schemaLocation="TigerXMLHeader.xsd"/>

  <!-- ======================================================

       INCLUDES DECLARATION OF SUBCORPORA AND SENTENCES

       ====================================================== -->

  <xs:include schemaLocation="TigerXMLSubcorpus.xsd"/>

  <!-- ======================================================

       DECLARATION OF THE CORPUS DOCUMENT

       ====================================================== -->

  <!-- declaration of the root element: corpus -->

  <xs:element name="corpus">

        <xs:complexType>

      <xs:sequence>

        <xs:choice>           

           <!-- header of the document is optional -->

           <xs:element name="head" type="headType" minOccurs="0" maxOccurs="1" />

        <xs:/choice>

        <xs:element name="body" type="bodyType" minOccurs="1" maxOccurs="1" />

      <xs:/sequence>

      <!-- corpus ID -->

      <xs:attribute name="id" type="idType" use="required" />

      <!-- optional attribute: TigerXML version; used by TIGERSearch only -->

      <xs:attribute name="version" type="xsd:string" use="optional" />

    <xs:/complexType>

    <xs:/element>

  <!-- declaration of the body type -->

  <xs:complexType name="bodyType">

    <xs:choice minOccurs="1" maxOccurs="unbounded">

      <xs:element name="subcorpus" type="subcorpusType" minOccurs="1" maxOccurs="1"/>

      <xs:element name="s" type="sentenceType" minOccurs="1" maxOccurs="1"/>

    <xs:/choice>

  <xs:/complexType>

<xs:/schema>

// Copyright © 2010-2013 ENS de Lyon.

// Copyright © 2007-2010 ENS de Lyon, CNRS, INRP, University of

// Lyon 2, University of Franche-Comté, University of Nice

// Sophia Antipolis, University of Paris 3.

//

// The TXM platform is free software: you can redistribute it

// and/or modify it under the terms of the GNU General Public

// License as published by the Free Software Foundation,

// either version 2 of the License, or (at your option) any

// later version.

//

// The TXM platform is distributed in the hope that it will be

// useful, but WITHOUT ANY WARRANTY; without even the implied

// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

// PURPOSE. See the GNU General Public License for more

// details.

//

// You should have received a copy of the GNU General

// Public License along with the TXM platform. If not, see

// http://www.gnu.org/licenses.

//

// $LastChangedDate: 2015-06-03 15:04:53 +0200 (mer., 03 juin 2015) $

// $LastChangedRevision: 2984 $

// $LastChangedBy: mdecorde $

//

package org.txm.importer.tigersearch;

import javax.xml.stream.XMLStreamReader;

import org.txm.sw.RemoveTag;

import org.txm.importer.ApplyXsl2;

import org.txm.importer.ValidateXml;

import org.txm.objects.*;

import org.txm.tokenizer.TokenizerClasses;

import org.txm.utils.*;

import org.txm.*;

import org.txm.scripts.teitxm.*;

import org.txm.utils.i18n.*;

import org.txm.metadatas.*;

import javax.xml.stream.*;

import org.w3c.dom.Element

import org.txm.utils.xml.DomUtils;

import org.txm.importer.xtz.*

String userDir = System.getProperty("user.home");

def MONITOR;

boolean debug = org.txm.utils.logger.Log.isPrintingErrors();

BaseParameters params;

try {params = paramsBinding;MONITOR=monitor} catch (Exception)

{	println "DEV MODE";//exception means we debug

	debug = true

	params = new BaseParameters(new File(userDir, "xml/roland/import.xml"))

	params.load()

	if (!org.txm.Toolbox.isInitialized()) {

		TokenizerClasses.loadFromNode(params.getTokenizerElement(params.getCorpusElement()));

		Toolbox.setParam(Toolbox.INSTALL_DIR,new File("/usr/lib/TXM"));

		//Toolbox.setParam(Toolbox.INSTALL_DIR,new File("C:\\Program Files\\TXM"));//For Windows

		Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File(userDir,"treetagger"));

		//Toolbox.setParam(Toolbox.TREETAGGER_INSTALL_PATH,new File("C:\\Program Files\\treetagger"));//for Windows

		Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File(userDir,"treetagger/models"));

		Toolbox.setParam(Toolbox.METADATA_ENCODING, "UTF-8");

		Toolbox.setParam(Toolbox.METADATA_COLSEPARATOR, ",");

		Toolbox.setParam(Toolbox.METADATA_TXTSEPARATOR, "\"");

		//Toolbox.setParam(Toolbox.TREETAGGER_MODELS_PATH,new File("C:\\Program Files\\treetagger\\models"));//for Windows

		Toolbox.setParam(Toolbox.USER_TXM_HOME, new File(System.getProperty("user.home"), "TXM"));

	}

}

if (params == null) { println "no parameters. Aborting"; return; }

//params.getKeyValueParameters().put(ImportKeys.CLEAN, "false")

//params.getKeyValueParameters().put(ImportKeys.MULTITHREAD, "false")

//params.getKeyValueParameters().put(ImportKeys.DEBUG, "false")

//params.getKeyValueParameters().put(ImportKeys.UPDATECORPUS, "false")

TSImport i = new TSImport(params);

i.process();

readyToLoad = i.isSuccessful

<?xml version="1.0"?>

<xsl:stylesheet version="1.0"

                xmlns:xsl="http://www.w3.org/1999/XSL/Transform"

                xmlns:tei="http://www.tei-c.org/ns/1.0"

                xmlns:xd="http://www.pnp-software.com/XSLTdoc"

                xmlns:edate="http://exslt.org/dates-and-times"

                exclude-result-prefixes="edate xd">

  <xd:doc type="stylesheet">

    <xd:short>

      Feuille de transformation du format TIGER-XML vers le format XML-TXM

    </xd:short>

    <xd:detail>

      This stylesheet is free software; you can redistribute it and/or

      modify it under the terms of the GNU Lesser General Public

      License as published by the Free Software Foundation; either

      version 3 of the License, or (at your option) any later version.

      This stylesheet is distributed in the hope that it will be useful,

      but WITHOUT ANY WARRANTY; without even the implied warranty of

      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

      Lesser General Public License for more details.

      You should have received a copy of GNU Lesser Public License with

      this stylesheet. If not, see http://www.gnu.org/licenses/lgpl.html

    </xd:detail>

    <xd:author>Matthieu Decorde, matthieu.decorde AT ens-lyon.fr</xd:author>

    <xd:author>Serge Heiden, slh AT ens-lyon.fr</xd:author>

    <xd:author>Alexey Lavrentev, alexei.lavrentev AT ens-lyon.fr></xd:author>

    <xd:copyright>2016, ENS de Lyon/CNRS (UMR IHRIM Cactus)</xd:copyright>

  </xd:doc>

  <xsl:output 

   method="xml"

   encoding="UTF-8"

   indent="yes" />

  <xsl:template match="corpus">

    <corpus>

    	<xsl:choose>

    		<xsl:when test="subcorpus">

    			<xsl:apply-templates select="subcorpus"/>

    		</xsl:when>

    		<xsl:otherwise>

    			<text>

    				<xsl:apply-templates select="s"/>

    			</text>			

    		</xsl:otherwise>

    	</xsl:choose>

    	<xsl:apply-templates/>

    </corpus>

  </xsl:template>

  <xsl:template match="subcorpus">

    <text>

	<xsl:attribute name="name"><xsl:value-of select="@name"/></xsl:attribute>

	<xsl:apply-templates select="s"/>

    </text>

  </xsl:template>

  <xsl:template match="s">

	<p>

	  <s>

	    <xsl:attribute name="id"><xsl:value-of select="@id"/></xsl:attribute>

	      <xsl:apply-templates select="graph/terminals/t" />

	  </s>

	</p>

  </xsl:template>

  <xsl:template match="t">

		<w>

			<xsl:for-each select="@*[not(name()='word')]">

				<xsl:copy/>

			</xsl:for-each>

	        <xsl:value-of select="@word"/>

		</w>

  </xsl:template>

</xsl:stylesheet>

package org.txm.importer.tigersearch

import org.txm.Toolbox

import org.txm.importer.xtz.ImportModule;

import org.txm.importer.xtz.XTZImporter

import org.txm.metadatas.Metadatas

import org.txm.utils.io.FileCopy

/**

 * Only build the Metadatas object since all XML-TXM files already exists.

 * Metadatas is used to build text order.

 * 

 * 

 * @author mdecorde

 *

 */

class TSImporter extends XTZImporter {

	public TSImporter(ImportModule module) {

		super(module);

	}

	@Override

	public void process() {

		File binDir = module.getBinaryDirectory();

		//prepare metadata if any

		File allmetadatafile = new File(inputDirectory, "metadata.csv");

		println allmetadatafile

		if (allmetadatafile.exists()) {

			File copy = new File(binDir, "metadata.csv")

			if (!FileCopy.copy(allmetadatafile, copy)) {

				println "Error: could not create a copy of the metadata file "+allmetadatafile.getAbsoluteFile();

				return;

			}

			this.metadata = new Metadatas(copy,

					Toolbox.getPreference(Toolbox.METADATA_ENCODING),

					Toolbox.getPreference(Toolbox.METADATA_COLSEPARATOR),

					Toolbox.getPreference(Toolbox.METADATA_TXTSEPARATOR), 1)

		}

		isSuccessFul = true;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

  <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

 <!-- ===========================================================================

      XML Schema for the subcorpus part of the TIGER-XML format

      http://www.ims.uni-stuttgart.de/projekte/TIGER/public/TigerXMLSubcorpus.xsd

      ===========================================================================

      TIGER Project, Wolfgang Lezius

      IMS, University of Stuttgart, 04/01/2003

      =========================================================================== -->

  <!-- ======================================================

       DECLARATION OF SUBCORPORA AND SENTENCES

       ====================================================== -->

  <!-- declaration of the subcorpus element -->

  <xs:element name="subcorpus" type="subcorpusType"/>

  <!-- declaration of the subcorpus type -->

  <xs:complexType name="subcorpusType">

    <!-- A subcorpus may comprise another subcorpora or sentences -->

    <xs:choice minOccurs="0" maxOccurs="unbounded">

      <xs:element name="subcorpus" type="subcorpusType" minOccurs="1" maxOccurs="1"/>

      <xs:element name="s" type="sentenceType" minOccurs="1" maxOccurs="1"/>

    <xs:/choice>

    <!-- required: subcorpus name -->

    <xs:attribute name="name" type="xsd:string" use="required"/>

    <!-- optional: reference to external subcorpus file 

         A subcorpus of a TigerXML corpus can also be stored in separate file. 

         This attribute points to the external subcorpus file. The pointer is

         an URI. Examples: file:relative.xml or file:/path/to/absolute.xml 

         Note: If there is a pointer to an external file, the subcorpus

               element must be empty. -->

    <xs:attribute name="external" type="xsd:anyURI"/>  

  <xs:/complexType>

  <!-- declaration of the sentence type -->

  <xs:complexType name="sentenceType">

    <xs:sequence>

      <xs:element name="graph" type="graphType" minOccurs="0" maxOccurs="1"/>

      <xs:element name="matches" type="matchesType" minOccurs="0" maxOccurs="1"/>

    <xs:/sequence>

    <xs:attribute name="id" type="idType" use="required"/>

  <xs:/complexType>

  <!-- declaration of the graph type -->

  <xs:complexType name="graphType">

    <xs:sequence>

      <xs:element name="terminals" type="terminalsType" minOccurs="1" maxOccurs="1"/>

      <xs:element name="nonterminals" type="nonterminalsType" minOccurs="1" maxOccurs="1"/>

    <xs:/sequence>

    <xs:attribute name="root" type="idrefType" use="required"/>

    <!-- indicated that the exported sentence is discontinuous -->

    <xs:attribute name="discontinuous" type="xsd:boolean" default="false" use="optional"/>

  <xs:/complexType>

  <!-- declaration of the terminals type -->

  <xs:complexType name="terminalsType">

    <xs:sequence>

      <xs:element name="t" type="tType" minOccurs="1" maxOccurs="unbounded"/>

    <xs:/sequence>

  <xs:/complexType>

  <!-- declaration of the t element -->

  <xs:complexType name="tType">

    <!-- secondary edges possible -->

    <xs:sequence>

      <xs:element name="secedge" type="secedgeType" minOccurs="0" maxOccurs="unbounded"/>

    <xs:/sequence>

    <xs:attribute name="id" type="idType" use="required"/>    

    <xs:attributeGroup ref="tfeatureAttributes"/>

  <xs:/complexType>

  <!-- declaration of the nonterminals type -->

  <xs:complexType name="nonterminalsType">

    <xs:sequence>

      <xs:element name="nt" type="ntType" minOccurs="0" maxOccurs="unbounded"/>

    <xs:/sequence>

  <xs:/complexType>

  <!-- declaration of the nt element -->

  <xs:complexType name="ntType">

    <!-- edge and secondary edges possible -->

    <xs:sequence>

      <xs:element name="edge" type="edgeType" minOccurs="0" maxOccurs="unbounded"/>

      <xs:element name="secedge" type="secedgeType" minOccurs="0" maxOccurs="unbounded"/>

    <xs:/sequence>

    <xs:attribute name="id" type="idType" use="required"/>    

    <xs:attributeGroup ref="ntfeatureAttributes"/>

  <xs:/complexType>

  <!-- declaration of the edge type -->

  <xs:complexType name="edgeType">

    <xs:attribute name="idref" type="idrefType" use="required"/>    

    <xs:attributeGroup ref="edgelabelAttribute"/>

  <xs:/complexType>

  <!-- declaration of the secondary edge type -->

  <xs:complexType name="secedgeType">

    <xs:attribute name="idref" type="idrefType" use="required"/>    

    <xs:attributeGroup ref="secedgelabelAttribute"/>

  <xs:/complexType>

  <!-- declaration of the matches type -->

  <xs:complexType name="matchesType">

    <xs:sequence>

      <xs:element name="match" type="matchType" minOccurs="1" maxOccurs="unbounded"/>

    <xs:/sequence>

  <xs:/complexType>

  <!-- declaration of the match type -->

  <xs:complexType name="matchType">

    <xs:sequence>

      <xs:element name="variable" type="varType" minOccurs="1" maxOccurs="unbounded"/>

    <xs:/sequence>

    <xs:attribute name="subgraph" type="idrefType" use="required"/>    

  <xs:/complexType>

  <!-- declaration of the variable type -->

  <xs:complexType name="varType">

    <xs:attribute name="name" type="xsd:string" use="required"/>    

    <xs:attribute name="idref" type="idrefType" use="required"/>    

  <xs:/complexType>

  <!-- ======================================================

       SENTENCE DECLARATIONS THAT SHOULD BE REFINED

       ====================================================== -->

  <!-- declaration of the TERMINAL FEATURE ATTRIBUTES;

       this group is unrestricted, but should be refined by a 

       specialised, corpus-dependent schema -->

  <xs:attributeGroup name="tfeatureAttributes">

    <xs:anyAttribute processContents="skip"/>

  <xs:/attributeGroup>

  <!-- declaration of the NONTERMINAL FEATURE ATTRIBUTES;

       this group is unrestricted, but should be refined by a 

       specialised, corpus-dependent schema -->

  <xs:attributeGroup name="ntfeatureAttributes">

    <xs:anyAttribute processContents="skip"/>

  <xs:/attributeGroup>

  <!-- declaration of the EDGE-LABEL ATTRIBUTE;

       the label attribute is optional which should be refined by a 

       specialised, corpus-dependent schema -->

  <xs:attributeGroup name="edgelabelAttribute">

    <xs:attribute name="label" type="xsd:string" use="optional"/>    

  <xs:/attributeGroup>

  <!-- declaration of the SECONDARY-EDGE-LABEL ATTRIBUTE;

       the label attribute is optional which should be refined by a 

       specialised, corpus-dependent schema -->

  <xs:attributeGroup name="secedgelabelAttribute">

    <xs:attribute name="label" type="xsd:string" use="optional"/>    

  <xs:/attributeGroup>

  <!-- ======================================================

       ID and IDREF TYPE DECLARATIONS

       ====================================================== -->

  <!-- Even though XML Schema are a W3C Recommendation, schema

       support of XML parsers is still restricted. Using some

       parsers you might have problems with the ID and IDREF

       attributes in combination with an "anyAttribute"

       declaration. In this case, just modify the base type 

       of the following two declarations to "xsd:string".  -->

  <!-- declaration of idType -->

  <xs:simpleType name="idType">

    <xs:restriction base="xsd:ID"/>

  <xs:/simpleType>

  <!-- declaration of idrefType -->

  <xs:simpleType name="idrefType">

    <xs:restriction base="xsd:IDREF"/>

  <xs:/simpleType>

<xs:/schema>

<html>

<body>

<p>TIGERSearch import module. This is a prototype that can only manage SRCMF TIGERSearch sources</p>

</body>

</html>

<?xml version="1.0" encoding="UTF-8"?>

  <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

 <!-- =======================================================================

      XML SubSchema for the header part of the TIGER-XML format

      http://www.ims.uni-stuttgart.de/projekte/TIGER/publicTigerXMLHeader.xsd

      =======================================================================

      TIGER Project, Wolfgang Lezius 

      IMS, University of Stuttgart, 04/01/2003

      ======================================================================= -->

  <!-- ======================================================

       DECLARATION OF THE HEADER

       ====================================================== -->

  <!-- declaration of the head element -->

  <xs:element name="head" type="headType"/>

  <!-- declaration of the header type -->

  <xs:complexType name="headType">

     <xs:sequence>

        <xs:element name="meta" type="metaType" minOccurs="0" maxOccurs="1"/>

        <xs:element name="annotation" type="annotationType" minOccurs="0" maxOccurs="1"/>

     <xs:/sequence>    

     <!-- optional: reference to external header file 

          The header of a TigerXML corpus can also be stored in separate file. 

          This attribute points to the external header file. The pointer is

          an URI. Examples: file:relative.xml or file:/path/to/absolute.xml

          Note: If there is a pointer to an external file, the head

                element must be empty. -->

     <xs:attribute name="external" type="xsd:anyURI"/>  

  <xs:/complexType>

  <!-- declaration of the meta information type -->

  <xs:complexType name="metaType">

    <xs:sequence>

      <xs:element name="name" type="xsd:string" minOccurs="0" maxOccurs="1"/>

      <xs:element name="author" type="xsd:string" minOccurs="0" maxOccurs="1"/>

      <xs:element name="date" type="xsd:string" minOccurs="0" maxOccurs="1"/>

      <xs:element name="description" type="xsd:string" minOccurs="0" maxOccurs="1"/>

      <xs:element name="format" type="xsd:string" minOccurs="0" maxOccurs="1"/>

      <xs:element name="history" type="xsd:string" minOccurs="0" maxOccurs="1"/>

    <xs:/sequence>    

  <xs:/complexType>

  <!-- declaration of the annotation type -->

  <xs:complexType name="annotationType">

    <xs:sequence>

      <xs:element name="feature" type="featureType" minOccurs="1" maxOccurs="unbounded"/>

      <xs:element name="edgelabel" type="edgelabelType" minOccurs="0" maxOccurs="1"/>

      <xs:element name="secedgelabel" type="edgelabelType" minOccurs="0" maxOccurs="1"/>

    <xs:/sequence>

  <xs:/complexType>

  <!-- declaration of the feature type -->

  <xs:complexType name="featureType">

    <xs:sequence>

       <xs:element name="value" type="featurevalueType" minOccurs="0" maxOccurs="unbounded"/>

    <xs:/sequence>

    <xs:attribute name="name" type="featurenameType" use="required"/>

    <xs:attribute name="domain" use="required">

       <xs:simpleType>

         <xs:restriction base="xsd:string">

           <xs:enumeration value="T"/>     <!-- feature for terminal nodes -->

           <xs:enumeration value="NT"/>    <!-- feature for nonterminal nodes -->

           <xs:enumeration value="FREC"/>  <!-- feature for both -->

         <xs:/restriction>

       <xs:/simpleType>

    <xs:/attribute>

  <xs:/complexType>

  <!-- declaration of the (secondary) edge label type -->

  <xs:complexType name="edgelabelType">

    <xs:sequence>

       <xs:element name="value" type="featurevalueType" minOccurs="0" maxOccurs="unbounded"/>

    <xs:/sequence>

  <xs:/complexType>

  <!-- declaration of the feature value type -->

  <xs:complexType name="featurevalueType">

    <xs:simpleContent>   <!-- element content: documentation of the feature value -->

      <xs:extension base="xsd:string">

        <xs:attribute name="name" type="xsd:string"/>

      <xs:/extension>

    <xs:/simpleContent>

  <xs:/complexType>

  <!-- ======================================================

       HEADER DECLARATIONS THAT SHOULD BE REFINED

       ====================================================== -->

  <!-- declaration of the FEATURE NAMES used in the corpus header;

       this type is unrestricted, but should be refined by a 

       specialised, corpus-dependent schema -->

  <xs:simpleType name="featurenameType">

    <xs:restriction base="xsd:string">

      <xs:minLength value="1"/>

      <xs:maxLength value="20"/>

      <xs:whiteSpace value="preserve"/>

    <xs:/restriction>

  <xs:/simpleType>

<xs:/schema>

<html>

<body>

<p>TIGERSearch import module. This is a prototype that can only manage SRCMF TIGERSearch sources</p>

</body>

</html>

#! /usr/bin/groovy

package org.txm.importer.srcmf;

/*

 * To change this template, choose Tools | Templates

 * and open the template in the editor.

 */

// Set up globals

// def masterFile = new File('/home/tomr/Documents/Work/lyon12/srcmf/groovy/SrcmfImport/aucassin_surface/master_pos.xml')

// def outputFile = new File('/home/tomr/Documents/Work/lyon12/srcmf/groovy/SrcmfImport/aucassin_surface/master_pos2.xml')

// def headerFile = new File('/home/tomr/Documents/Work/SRCMF/srcmf_ts/header_srcmf.xml')

// def feats = [nt:['cat', 'type', 'coord'], t:['pos', 'form', 'q']]

// def firstFeat = [t:'word', nt:'cat']

// Command-line entry point

def cli = new CliBuilder(

    usage:'MasterReheader.groovy [options] master_file.xml header_file.xml'

)

cli.h(longOpt:'help', 'Prints this message.')

cli.o(longOpt:'output', args:1, argName:'outputfile.xml', 'Output to given file.')

cli.nt(longOpt:'ntfeats', args:1, argName:'ntfeats', 'NT features for which to provide value node.')

cli.t(longOpt:'tfeats', args:1, argName:'tfeats', 'T features for which  to provide value node.')

cli.nt1(longOpt:'ntfeat1st', args:1, argName:'1st ntfeat', 'First NT feature listed in header.')

cli.t1(longOpt:'tfeat1st', args:1, argName:'1st tfeat', 'First T feature listed in header.')

options = cli.parse(args)

if (options.arguments().size() == 2) {

    def masterFile = new File(options.arguments()[0])

    def headerFile = new File(options.arguments()[1])

    def masterFolder = masterFile.getCanonicalFile().getParent()

    def outputFile = null

    if (options.o) {

        outputFile = new File(options.o)

    } else {

        outputFile = new File(masterFolder, 'MasterReheader_out.xml')

    }

    def ntfirst = 'cat'

    if (options.nt1) {

        ntfirst = options.nt1

    }

    def tfirst = 'word'

    if (options.t1) {

        tfirst = options.t1

    }

    script(

        masterFile, headerFile, outputFile, 

        ['nt':options.nts, 't':options.ts],

        ['nt':options.nt1, 't':options.t1]

    )    

} else {

    println 'Incorrect number of command line arguments... exiting'

    println cli.usage()

}

def script(

    File masterFile, File headerFile, File outputFile, HashMap feats, HashMap firstFeat

) {

    // Load master and header files

    def master = new XmlParser().parse(masterFile)

    def header = new XmlParser().parse(headerFile)

    def masterFolder = masterFile.getCanonicalFile().getParent()

    // Set up locals

    def attrVal = [nt:[:], t:[:]]

    // Scan subcorpus files and build attribute lists.

    master.body.subcorpus.each { 

        def subcorpusFile = new File (masterFolder, it.'@external'[5..-1])

        def subcorpus = new XmlParser().parse(subcorpusFile)

        // Closure for t & nt nodes processing.

        def getvals = { node, type ->

            node.attributes().each { mEntry ->

                if (! attrVal[type].keySet().contains(mEntry.getKey())) {

                    attrVal[type][mEntry.getKey()] = new HashSet()

                }

                attrVal[type][mEntry.getKey()].add(mEntry.getValue())

            }

        }

        subcorpus.s.graph.terminals.t.each { getvals.call(it, 't') }

        subcorpus.s.graph.nonterminals.nt.each { getvals.call(it, 'nt') }

    }

    // Id isn't an attribute in the header.

    attrVal['t'].remove('id')

    attrVal['nt'].remove('id')

    // Remove old feature nodes in master file

    def oldFeatureNodes = master.head.annotation.feature

    while (oldFeatureNodes) {

        node = oldFeatureNodes.pop()

        node.parent().remove(node)

    }

    assert (! master.head.annotation.feature)

    // Check firstFeat was relevant

    ['t', 'nt'].each { type ->

        if (! (attrVal[type].keySet().contains(firstFeat[type]))) {

            firstFeat[type] = attrVal[type].keySet().sort()[0]

        }

    }

    assert attrVal['t'].keySet().contains(firstFeat['t'])

    assert attrVal['nt'].keySet().contains(firstFeat['nt'])

    def featList = [:]

    ['t', 'nt'].each { type ->

        featList[type] = [firstFeat[type]]

        featList[type].addAll(attrVal[type].keySet().findAll { it != firstFeat[type] })

    }

    // Add new feature and value nodes

    ['t', 'nt'].each { type ->

        featList[type].each { feat ->

            def fNode = new Node(master.head.annotation[0], 'feature', 

                ['domain':type.toUpperCase(), 'name':feat]

            )

            // Add value node if the node value is given in 'feats'

            if (feats[type].contains(feat)) {

                attrVal[type][feat].each { value ->

                    assert header.'**'.feature

                    assert header.'**'.feature[0].'@name'

                    assert header.'**'.feature[0].'@domain'

                    assert ['NT', 'T'].contains(header.'**'.feature[0].'@domain')

                    def hFNode = header.'**'.feature.find {

                        it.'@name' == feat && (

                            it.'@domain' == type.toUpperCase() || it.'@domain' == 'FREC'

                        )

                    }

                    def vText = '[unknown]'

                    if (hFNode && hFNode.value.find { it.'@name' == value }) {

                        vText = hFNode.value.find { it.'@name' ==  value }.text()

                    }

                    new Node(fNode, 'value', ['name':value], vText)

                }

            }

        }    

    }

    // Save to output_file

    outputFile.withWriter { writer ->

        writer << groovy.xml.XmlUtil.serialize(master)

    }

}

/*

 * Calls all import scripts

 */

/**

 *

 * @author tmr

 */

// Command line form of import statements:

// import MasterReheader

// import PunctInjectImport

// import SubcorpusDataInject

// import TigerSubcorpus2Main

// import java.util.logging.FileHandler

// import javax.xml.parsers.DocumentBuilderFactory

// TXM package statement

package org.txm.importer.srcmf

import java.util.logging.*;

// Command line entry point

def cli = new CliBuilder(

    usage:'SrcmfImport.groovy [options] tiger_master.xml xml_txm.xml header_file.xml'

)

cli.h(longOpt:'help', 'Prints this message.')

options = cli.parse(args)

if (options.arguments().size() != 3) {

    println 'Incorrect number of command line arguments... exiting'

    println cli.usage()

    System.exit(2)

}

def tigerFile = new File(options.arguments()[0])

def txmFile = new File(options.arguments()[1])

def headerFile = new File(options.arguments()[2])

def tigerXmlAll = doAllButPnc(

    tigerFile, 

    txmFile, 

    headerFile,

    txmFile.getAbsoluteFile().getParentFile().getParentFile()

)

doPnc(tigerXmlAll, txmFile)

tigerXmlAll.delete()

def doAllButPnc(File tigerFile, File txmFile, File headerFile, File binDir) {

    // Run pos injection script

    File txmSrcDir = txmFile.getAbsoluteFile().getParentFile()

    File tigerDir = new File(binDir, "tiger")

    tigerDir.mkdir()

    File masterpos = new File(tigerDir, "master_pos.xml")

    File xmltxm = txmSrcDir.listFiles()[0]

    File logFile = new File(binDir, "tiger.log")

    def sdi = new SubcorpusDataInject(

        xmltxm, 

        new FileHandler(logFile.getAbsolutePath()), "vers"

    )

    sdi.processMaster(tigerFile, masterpos)

    // Run reheader script

    def reheader = new MasterReheader()

    File tmp = File.createTempFile("tmp", ".xml",tigerDir)

    def feats = ['nt':['cat', 'type', 'coord'], 't':['pos', 'form', 'q']]

    def firstFeat = ['nt':'cat', 't':'word']

    reheader.script(masterpos, headerFile, tmp, feats, firstFeat)

    if (!tmp.exists()) {

    	println "Error: reheader failed"

    }

    masterpos.delete()

    tmp.renameTo(masterpos)

    // Run merge master & subcorpus script

    def tigerXmlAll = new File(masterpos.getParentFile(), "TigerAll.xml")

    def mergescript = new TigerSubcorpus2Main()

    mergescript.script(masterpos, tigerXmlAll)

    return tigerXmlAll

}

def doPnc(File tigerXmlAll, File txmFile) {

    injector = new PunctInjectImport(tigerXmlAll, txmFile)

    injector.outputFile = new File(tigerXmlAll.getParentFile(), "TigerPnc.xml")

    injector.process()

}

#! /usr/bin/groovy

package org.txm.importer.srcmf;

/*