Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/mwl-lookup.perl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/mwl-lookup.perl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/mwl-lookup.perl	(revision 1683)
@@ -0,0 +1,97 @@
+#!/usr/local/bin/perl
+
+use Getopt::Std;
+getopt('dhf:');
+
+# This perl script recognizes multi word units in the input stream
+# and puts them on one line. Input must have one-word-per-line format.
+# The multi word units are listed in the parameter file with POS tags.
+# Each line contains one multi word unit where the individual words
+# are separated by blanks followed by a tab character and the blank-
+# separated list of POS tags.
+# Author: Helmut Schmid, IMS, Uni Stuttgart
+
+if (!defined($opt_f) || defined($opt_h)) {
+  $0 =~ s/.*\///;
+  printf "\nUsage: $0 [-d del] -f mwl-file ...files...\n";
+  print "\nOptions:\n";
+  print "-d del : Use del as delimiter rather than a blank\n\n";
+  die
+}
+
+if (!open(FILE, $opt_f)) {
+  die "\nCan't open mwl file: ",$opt_f,"\n";
+}
+if (defined($opt_d)) {
+  $del = $opt_d;
+} else {
+  $del = " ";
+}
+
+$N=1;
+while (<FILE>) {
+  chomp();
+  next if /^$/;
+  @G = split("\t");
+  @F = split(/\s+/,$G[0]);
+  $state = 0;
+  for($i=0; $i<=$#F; $i++) {
+    if (!exists($arc{$state,$F[$i]})) {
+      $arc{$state,$F[$i]} = $N++;
+    }
+    $state = $arc{$state,$F[$i]};
+   }
+  $final{$state} = $G[1];
+}
+close(FILE);
+
+
+$last = $match = $last_match = 0;
+$state = 0;
+
+for (;;) {
+  if ($match == $last) {
+    if (!($token[$last] = <>)) {
+      if ($last_match > 0) {
+	print $token[0];
+	for ($i=1; $i<=$last_match; $i++) {
+	  print $del,$token[$i];
+	}
+	print "\n";
+      } else {
+	$i=0;
+      }
+      for (; $i<$last; $i++) {
+	print $token[$i],"\n";
+      }
+      last;
+    }
+    chomp($token[$last++]);
+  }
+  if (($s = $arc{$state, $token[$match]}) ||
+      ($s = $arc{$state, lc($token[$match])}) ||
+      ($s = $arc{$state, ucfirst(lc($token[$match]))})) {
+    if (exists($final{$s})) {
+      $last_match = $match;
+      $last_tag = $final{$s};
+    }
+    $state = $s;
+    $match++;
+  } else {
+    if ($last_match > 0) {
+      print $token[0];
+      for($i=1; $i<=$last_match; $i++) {
+	print $del,$token[$i];
+      }
+      print "\t$last_tag\n";
+    } else {
+      print $token[0],"\n";
+    }
+    for($i=0,$k=$last_match+1; $k<$last; ) {
+      $token[$i++] = $token[$k++];
+    }
+    $last = $last - $last_match - 1;
+    $last_match = $match = 0;
+    $state = 0;
+  }
+}
Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output-german.perl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output-german.perl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output-german.perl	(revision 1683)
@@ -0,0 +1,125 @@
+#!/usr/bin/perl
+
+use Getopt::Std;
+getopts('t');
+
+print doc_start();
+
+$n = 0;
+while (<>) {
+  s/.-SBAR$/O/;
+
+  if (/^(.*)-(.*)\t(.*)\/(.*)$/) {
+    $token[$n] = $1;
+    $tag[$n] = $2;
+    $chunk[$n] = $4;
+    if ($chunk[$n] =~ /^(.*)-(.*)$/) {
+      $flag[$n] = $1;
+      $chunk[$n] = $2;
+    }
+    else {
+      undef $flag[$n];
+      undef $chunk[$n];
+    }
+    print_sentence()  if $token[$n] eq '.';
+    $n++;
+  } 
+
+  else {
+    $markup[$n] .= $_;
+  }
+}
+
+print_sentence();
+print doc_end();
+
+
+sub print_sentence {
+  my($i,$chunk);
+
+  for( $i=0; $i<=$n; $i++ ) {
+
+    if ($flag[$i] eq 'I' && $chunk ne $chunk[$i]) {
+      $flag[$i] = 'B';
+    }
+
+    if ($flag[$i] eq 'B') {
+      if (defined $chunk) {
+	$cetags[$i-1] = end_tag($chunk);
+      }
+      $chunk = $chunk[$i];
+      $cbtags[$i] .= start_tag($chunk[$i]);
+    }
+
+    # German chunker uses E-flags for PCs
+    elsif ($flag[$i] eq 'E') {
+      if ($chunk[$i] eq $chunk) {
+	$cetags[$i] = end_tag($chunk);
+	undef $chunk;
+      }
+      elsif ($chunk[$i] eq "PC" && $chunk eq "NC") {
+	for( $k=$i-1; $k>=0; $k-- ) {
+	  if ($chunk[$k] eq "NC") {
+	    $chunk[$k] = "PC";
+	  }
+	  if ($flag[$k] ne "I") {
+	    last;
+	  }
+	}
+	$cbtags[$k] = start_tag($chunk[$i]);
+	$cetags[$i] = end_tag($chunk[$i]);
+	undef $chunk;
+	undef $inPC;
+      }
+    }
+
+    elsif ($flag[$i] ne 'I' && defined $chunk) {
+      $cetags[$i-1] = end_tag($chunk);
+      undef $chunk;
+    }
+  }
+
+  for( $i=0; $i<=$n; $i++ ) {
+    print $markup[$i];
+    print $cbtags[$i];
+    print token_and_tag($token[$i],$tag[$i]) if defined $token[$i];
+    print $cetags[$i];
+  }
+
+  undef @token;
+  undef @tag;
+  undef @chunk;
+  undef @cbtags;
+  undef @cetags;
+  undef @flag;
+  undef @markup;
+  $n = 0;
+}
+
+sub doc_start {
+  return '' unless defined $opt_t;
+  return "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"yes\"?>\n<corpus>\n";
+}
+
+sub doc_end {
+  return '' unless defined $opt_t;
+  return "</corpus>\n";
+}
+
+sub start_tag {
+  my $t=shift;
+  return "<$t>\n" unless defined $opt_t;
+  return "  <phrase cat=\"$t\">\n";
+}
+
+sub end_tag {
+  my $t=shift;
+  return "</$t>\n" unless defined $opt_t;
+  return "  </phrase>\n";
+}
+
+sub token_and_tag {
+  my ($token,$tag)=@_;
+  return "$token\t$tag\n" unless defined $opt_t;
+  return "    <token word=\"$token\" pos=\"$tag\"/>\n";
+}
Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/utf8-tokenize.perl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/utf8-tokenize.perl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/utf8-tokenize.perl	(revision 1683)
@@ -0,0 +1,205 @@
+#!/usr/bin/perl
+
+########################################################################
+#                                                                      #
+#  tokenization script for tagger preprocessing                        #
+#  Author: Helmut Schmid, IMS, University of Stuttgart                 #
+#          Serge Sharoff, University of Leeds                          #
+#  Description:                                                        #
+#  - splits input text into tokens (one token per line)                #
+#  - cuts off punctuation, parentheses etc.                            #
+#  - disambiguates periods                                             #
+#  - preserves SGML markup                                             #
+#                                                                      #
+########################################################################
+
+use Getopt::Std;
+use utf8;
+use Encode;
+
+getopts('hgfeiza:');
+
+# Modify the following lines in order to adapt the tokenizer to other
+# types of text and/or languages
+
+# characters which have to be cut off at the beginning of a word
+my $PChar='[Â¿Â¡{(\\`"â€šâ€žâ€ â€¡â€¹â€˜â€™â€œâ€�â€¢â€“â€”â€º'."'";
+
+# characters which have to be cut off at the end of a word
+my $FChar=']}\'\`\"),;:\!\?\%â€šâ€žâ€¦â€ â€¡â€°â€¹â€˜â€™â€œâ€�â€¢â€“â€”â€º';
+
+# character sequences which have to be cut off at the beginning of a word
+my $PClitic='';
+
+# character sequences which have to be cut off at the end of a word
+my $FClitic;
+
+if (defined($opt_e)) {
+  # English
+  $FClitic = '\'(s|re|ve|d|m|em|ll)|n\'t';
+}
+if (defined($opt_i)) {
+  # Italian
+  $PClitic = '[dD][ae]ll\'|[nN]ell\'|[Aa]ll\'|[lLDd]\'|[Ss]ull\'|[Qq]uest\'|[Uu]n\'|[Ss]enz\'|[Tt]utt\'';
+}
+if (defined($opt_f)) {
+  # French
+  $PClitic = '[dcjlmnstDCJLNMST]\'|[Qq]u\'|[Jj]usqu\'|[Ll]orsqu\'';
+  $FClitic = '-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mmes?|-m\'|-moi|-nous|-on|-toi|-tu|-t\'|-vous|-en|-y|-ci|-l';
+}
+if (defined($opt_z)) {
+  # Galician
+  $FClitic = '-la|-las|-lo|-los|-nos';
+}
+
+
+### NO MODIFICATIONS REQUIRED BEYOND THIS LINE #########################
+
+if (defined($opt_h)) {
+  die "
+Usage: utf8-tokenize.perl [ options ] ...files...
+
+Options:
+-e : English text 
+-f : French text
+-i : Italian text
+-a <file>: <file> contains a list of words which are either abbreviations or
+           words which should not be further split.
+";
+}
+
+# Read the list of abbreviations and words
+if (defined($opt_a)) {
+  die "Can't read: $opt_a: $!\n"  unless (open(FILE, $opt_a));
+  while (<FILE>) {
+      $_ = decode('utf8',$_);
+      s/^[ \t\r\n]+//;
+      s/[ \t\r\n]+$//;
+      next if (/^\#/ || /^\s$/);    # ignore comments
+      $Token{$_} = 1;
+  }
+  close FILE;
+}
+
+#SS: main loop; 
+my $first_line = 1;
+while (<>) {
+  $_ = decode('utf8',$_);
+  # delete optional byte order markers (BOM)
+  if ($first_line) {
+      undef $first_line;
+      s/^\x{FEFF}//;
+  }
+
+  # replace newlines and tab characters with blanks
+  tr/\n\t/  /;
+
+  # replace blanks within SGML tags
+  while (s/(<[^<> ]*) ([^<>]*>)/$1\377$2/g) {
+  }
+  ;
+  #Separ: Ã¿Ã¾
+
+  # replace whitespace with a special character
+  tr/ /\376/;
+
+  # restore SGML tags
+  tr/\377\376/ \377/;
+
+  # prepare SGML-Tags for tokenization
+  s/(<[^<>]*>)/\377$1\377/g;
+  s/^\377//;
+  s/\377$//;
+  s/\377\377\377*/\377/g;
+
+  @S = split("\377");
+  for ( $i=0; $i<=$#S; $i++) {
+    $_ = $S[$i];
+  
+    if (/^<.*>$/) {
+      # SGML tag
+      print encode('utf8',"$_\n");
+    } else {
+      # add a blank at the beginning and the end of each segment
+      $_ = ' '.$_.' ';
+      # insert missing blanks after punctuation
+      s/(\.\.\.)/ ... /g;
+      s/([;\!\?])([^ ])/$1 $2/g;
+      s/([.,:])([^ 0-9.])/$1 $2/g;
+    
+      @F = split;
+      for ( $j=0; $j<=$#F; $j++) {
+	my $suffix="";
+	$_ = $F[$j];
+	# separate punctuation and parentheses from words
+	do {
+	  $finished = 1;
+	  # cut off preceding punctuation
+	  if (s/^([$PChar])(.)/$2/) {
+	    print encode('utf8',"$1\n");
+	    $finished = 0;
+	  }
+	  # cut off trailing punctuation
+	  if (s/(.)([$FChar])$/$1/) {
+	    $suffix = "$2\n$suffix";
+	    $finished = 0;
+	  }
+	  # cut off trailing periods if punctuation precedes
+	  if (s/([$FChar])\.$//) { 
+	    $suffix = ".\n$suffix";
+	    if ($_ eq "") {
+	      $_ = $1;
+	    } else {
+	      $suffix = "$1\n$suffix";
+	    }
+	    $finished = 0; 
+	  }
+	} while (!$finished);
+                
+	# handle explicitly listed tokens
+	if (defined($Token{$_})) {
+	  print encode('utf8',"$_\n$suffix");
+	  next;
+	}
+                
+	# abbreviations of the form A. or U.S.A.
+	if (/^([A-Za-z-]\.)+$/) {
+	  print encode('utf8',"$_\n$suffix");
+	  next;
+	}
+                 
+
+	# disambiguate periods
+	if (/^(..*)\.$/ && $_ ne "..." && !($opt_g && /^[0-9]+\.$/)) {
+	  $_ = $1;
+	  $suffix = ".\n$suffix";
+	  if (defined($Token{$_})) {
+ 	    print encode('utf8',"$_\n$suffix");
+	    next;
+	  }
+	}
+                 
+	# cut off clitics
+	while (s/^(--)(.)/$2/) {
+ 	    print encode('utf8',"$1\n");
+	}
+	if ($PClitic ne '') {
+	  while (s/^($PClitic)(.)/$2/) {
+ 	    print encode('utf8',"$1\n");
+	  }
+	}
+
+	while (s/(.)(--)$/$1/) {
+	    $suffix = "$2\n$suffix";
+	}
+	if ($FClitic ne '') {
+	  while (s/(.)($FClitic)$/$1/) {
+	    $suffix = "$2\n$suffix";
+	  }
+	}
+                 
+	print encode('utf8',"$_\n$suffix");
+      }
+    }
+  }
+}

Property changes on: tmp/org.txm.treetagger.core.win32/res/win32/cmd/utf8-tokenize.perl
___________________________________________________________________
Added: svn:executable
   + *

Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output.perl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output.perl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output.perl	(revision 1683)
@@ -0,0 +1,123 @@
+#!/usr/bin/perl
+
+use Getopt::Std;
+getopts('t');
+
+print doc_start();
+
+$n = 0;
+while (<>) {
+  s/.-SBAR$/O/;
+
+  if (/^(.*)-(.*)\t(.*)\/(.*)$/) {
+    $token[$n] = $1;
+    $tag[$n] = $2;
+    $chunk[$n] = $4;
+    if ($chunk[$n] =~ /^(.*)-(.*)$/) {
+      $flag[$n] = $1;
+      $chunk[$n] = $2;
+    }
+    else {
+      undef $flag[$n];
+      undef $chunk[$n];
+    }
+    print_sentence()  if $token[$n] eq '.';
+    $n++;
+  } 
+
+  else {
+    $markup[$n] .= $_;
+  }
+}
+
+print_sentence();
+print doc_end();
+
+
+sub print_sentence {
+  my($i,$chunk);
+
+  for( $i=0; $i<=$n; $i++ ) {
+
+    if ($flag[$i] eq 'I' && $chunk ne $chunk[$i]) {
+      $flag[$i] = 'B';
+    }
+
+    if ($flag[$i] ne '' && $token[$i] eq '.') {
+      delete $flag[$i];
+      $chunk[$i] = '0';
+    }
+
+    if ($flag[$i] eq 'B') {
+      if (defined $chunk) {
+	if (($chunk eq 'PC' && $chunk[$i] eq 'NC') ||
+	    ($chunk eq 'PP' && $chunk[$i] eq 'NP'))
+	  {
+	    $inPC = $chunk;
+	  }
+	else {
+	  $cetags[$i-1] = end_tag($chunk);
+	  if (defined $inPC) {
+	    $cetags[$i-1] .= end_tag($inPC);
+	    undef $inPC;
+	  }
+	}
+      }
+      $chunk = $chunk[$i];
+      $cbtags[$i] .= start_tag($chunk[$i]);
+    }
+
+    elsif ($flag[$i] ne 'I' && defined $chunk) {
+      $cetags[$i-1] = end_tag($chunk);
+      undef $chunk;
+      if (defined $inPC) {
+	$cetags[$i-1] .= end_tag($inPC);
+	undef $inPC;
+      }
+    }
+  }
+
+  for( $i=0; $i<=$n; $i++ ) {
+    print $markup[$i];
+    print $cbtags[$i];
+    print token_and_tag($token[$i],$tag[$i]) if defined $token[$i];
+    print $cetags[$i];
+  }
+
+  undef @token;
+  undef @tag;
+  undef @chunk;
+  undef @cbtags;
+  undef @cetags;
+  undef @flag;
+  undef @markup;
+  $n = 0;
+}
+
+sub doc_start {
+  return '' unless defined $opt_t;
+  return "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"yes\"?>\n<corpus>\n";
+}
+
+sub doc_end {
+  return '' unless defined $opt_t;
+  return "</corpus>\n";
+}
+
+sub start_tag {
+  my $t=shift;
+  return "<$t>\n" unless defined $opt_t;
+  return "  <phrase cat=\"$t\">\n";
+}
+
+sub end_tag {
+  my $t=shift;
+  return "</$t>\n" unless defined $opt_t;
+  return "  </phrase>\n";
+}
+
+sub token_and_tag {
+  my ($token,$tag)=@_;
+  return "$token\t$tag\n" unless defined $opt_t;
+  return "    <token word=\"$token\" pos=\"$tag\"/>\n";
+}
Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/mwl-lookup-greek.perl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/mwl-lookup-greek.perl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/mwl-lookup-greek.perl	(revision 1683)
@@ -0,0 +1,81 @@
+#!/usr/bin/perl
+
+$month{"Éáíïõáñßïõ"} = 1;
+$month{"Öåâñïõáñßïõ"} = 1;
+$month{"Ìáñôßïõ"} = 1;
+$month{"Áðñéëßïõ"} = 1;
+$month{"ÌáÀïõ"} = 1;
+$month{"Ìáúïõ"} = 1;
+$month{"Ìáßïõ"} = 1;
+$month{"Éïõíßïõ"} = 1;
+$month{"Éïõëßïõ"} = 1;
+$month{"Áõãïýóôïõ"} = 1;
+$month{"Óåðôåìâñßïõ"} = 1;
+$month{"Ïêôùâñßïõ"} = 1;
+$month{"Íïåìâñßïõ"} = 1;
+$month{"Äåêåìâñßïõ"} = 1;
+
+$month{"ÉáíïõÜñéï"} = 1;
+$month{"ÖåâñïõÜñéï"} = 1;
+$month{"ÌÜñôéï"} = 1;
+$month{"Áðñßëéï"} = 1;
+$month{"ÌÜéï"} = 1;
+$month{"Éïýíéï"} = 1;
+$month{"Éïýëéï"} = 1;
+$month{"Áõãïýóôï"} = 1;
+$month{"Áýãïõóôï"} = 1;
+$month{"ÓåðôÝìâñéï"} = 1;
+$month{"Ïêôþâñéï"} = 1;
+$month{"ÍïÝìâñéï"} = 1;
+$month{"ÄåêÝìâñéï"} = 1;
+
+while (<>) {
+    chomp;
+    if ($_ eq '') {
+	print_sentence();
+    }
+    else {
+	push @token, $_;
+    }
+}
+print_sentence();
+
+sub print_sentence {
+    for( $i=0; $i<=$#token; $i++ ) {
+	if (exists $month{$token[$i]}) {
+	    $start = $end = $i;
+	    if ($token[$start-1] =~ /^[1-9][0-9]?([çáï]ò?)?(-[1-9][0-9]?([çáï]ò?)?)?$/){
+		$start--;
+	    }
+	    if ($token[$start-1] eq 'ÐÝìðôç') {
+		$start--;
+	    }
+	    if ($token[$end+1] eq 'ôïõ') {
+		$end++;
+	    }
+	    if ($token[$end+1] =~ /^(1[0-9][0-9][0-9]|20[0-9][0-9]|'[0-9][0-9])$/) {
+		$end++;
+	    }
+	    for( $k=$start; $k<$end; $k++) {
+		$join[$k] = 1;
+	    }
+	}
+	elsif (($token[$i] eq "áð'" && $token[$i+1] eq 'üëá') ||
+	       ($token[$i] eq 'åí' && 
+		($token[$i+1] eq 'ëüãù' || $token[$i+1] eq 'üøåé')))
+	{
+	    $join[$i] = 1;
+	}
+	       
+    }
+    for( $i=0; $i<=$#token; $i++ ) {
+	if ($join[$i] == 1) {
+	    print "$token[$i] "
+	}
+	else {
+	    print "$token[$i]\n"
+	}
+    }
+    undef @token;
+    undef @join;
+}
Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/tokenize.pl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/tokenize.pl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/tokenize.pl	(revision 1683)
@@ -0,0 +1,197 @@
+#!/usr/bin/perl
+
+########################################################################
+#                                                                      #
+#  tokenization script for tagger preprocessing                        #
+#  Author: Helmut Schmid, IMS, University of Stuttgart                 #
+#          Serge Sharoff, University of Leeds                          #
+#  Description:                                                        #
+#  - splits input text into tokens (one token per line)                #
+#  - cuts off punctuation, parentheses etc.                            #
+#  - disambiguates periods                                             #
+#  - preserves SGML markup                                             #
+#                                                                      #
+########################################################################
+
+use Getopt::Std;
+
+getopts('hfeia:u');
+
+use utf8;
+if (defined $opt_u) {
+    use open ':utf8';
+    binmode(STDIN,":utf8");
+    binmode(STDOUT,":utf8");
+}
+
+# Modify the following lines in order to adapt the tokenizer to other
+# types of text and/or languages
+
+# characters which have to be cut off at the beginning of a word
+my $PChar='[Â¿Â¡{(\\`"â€šâ€žâ€ â€¡â€¹â€˜â€™â€œâ€�â€¢â€“â€”â€º';
+
+# characters which have to be cut off at the end of a word
+my $FChar=']}\'\`\"),;:\!\?\%â€šâ€žâ€¦â€ â€¡â€°â€¹â€˜â€™â€œâ€�â€¢â€“â€”â€º';
+
+# character sequences which have to be cut off at the beginning of a word
+my $PClitic='';
+
+# character sequences which have to be cut off at the end of a word
+my $FClitic;
+
+if (defined($opt_e)) {
+  # English
+  $FClitic = '\'(s|re|ve|d|m|em|ll)|n\'t';
+}
+if (defined($opt_i)) {
+  # Italian
+  $PClitic = '[dD][ae]ll\'|[nN]ell\'|[Aa]ll\'|[lLDd]\'|[Ss]ull\'|[Qq]uest\'|[Uu]n\'|[Ss]enz\'|[Tt]utt\'';
+}
+if (defined($opt_f)) {
+  # French
+  $PClitic = '[dcjlmnstDCJLNMST]\'|[Qq]u\'|[Jj]usqu\'|[Ll]orsqu\'';
+  $FClitic = '-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mmes?|-m\'|-moi|-nous|-on|-toi|-tu|-t\'|-vous|-en|-y|-ci|-l';
+}
+
+
+### NO MODIFICATIONS REQUIRED BEYOND THIS LINE #########################
+
+if (defined($opt_h)) {
+  die "
+Usage: tokenize.perl [ options ] ...files...
+
+Options:
+-u : use UTF8 encoding
+-e : English text 
+-f : French text
+-i : Italian text
+-a <file>: <file> contains a list of words which are either abbreviations or
+           words which should not be further split.
+";
+}
+
+# Read the list of abbreviations and words
+if (defined($opt_a)) {
+  die "Can't read: $opt_a: $!\n"  unless (open(FILE, $opt_a));
+  while (<FILE>) {
+    s/^[ \t\r\n]+//;
+    s/[ \t\r\n]+$//;
+    next if (/^\#/ || /^\s$/);    # ignore comments
+    $Token{$_} = 1;
+  }
+  close FILE;
+}
+
+#SS: main loop; 
+my $first_line = 1;
+while (<>) {
+  # delete optional byte order markers (BOM)
+  if ($first_line) {
+      undef $first_line;
+      s/^\x{FEFF}//;
+  }
+
+  # replace newlines and tab characters with blanks
+  tr/\n\t/  /;
+
+  # replace blanks within SGML tags
+  while (s/(<[^<> ]*) ([^<>]*>)/$1\377$2/g) {
+  }
+  ;
+  #Separ: Ã¿Ã¾
+
+  # replace whitespace with a special character
+  tr/ /\376/;
+
+  # restore SGML tags
+  tr/\377\376/ \377/;
+
+  # prepare SGML-Tags for tokenization
+  s/(<[^<>]*>)/\377$1\377/g;
+  s/^\377//;
+  s/\377$//;
+  s/\377\377\377*/\377/g;
+
+  @S = split("\377");
+  for ( $i=0; $i<=$#S; $i++) {
+    $_ = $S[$i];
+  
+    if (/^<.*>$/) {
+      # SGML tag
+      print $_,"\n";
+    } else {
+      # add a blank at the beginning and the end of each segment
+      $_ = ' '.$_.' ';
+      # insert missing blanks after punctuation
+      s/(\.\.\.)/ ... /g;
+      s/([;\!\?])([^ ])/$1 $2/g;
+      s/([.,:])([^ 0-9.])/$1 $2/g;
+    
+      @F = split;
+      for ( $j=0; $j<=$#F; $j++) {
+	my $suffix="";
+	$_ = $F[$j];
+	# separate punctuation and parentheses from words
+	do {
+	  $finished = 1;
+	  # cut off preceding punctuation
+	  if (s/^([$PChar])(.)/$2/) {
+	    print $1,"\n";
+	    $finished = 0;
+	  }
+	  # cut off trailing punctuation
+	  if (s/(.)([$FChar])$/$1/) {
+	    $suffix = "$2\n$suffix";
+	    $finished = 0;
+	  }
+	  # cut off trailing periods if punctuation precedes
+	  if (s/([$FChar])\.$//) { 
+	    $suffix = ".\n$suffix";
+	    if ($_ eq "") {
+	      $_ = $1;
+	    } else {
+	      $suffix = "$1\n$suffix";
+	    }
+	    $finished = 0; 
+	  }
+	} while (!$finished);
+                
+	# handle explicitly listed tokens
+	if (defined($Token{$_})) {
+	  print "$_\n$suffix";
+	  next;
+	}
+                
+	# abbreviations of the form A. or U.S.A.
+	if (/^([A-Za-z-]\.)+$/) {
+	  print "$_\n$suffix";
+	  next;
+	}
+                 
+	# disambiguate periods
+	if (/^(..*)\.$/ && $_ ne "..." && !/^[0-9]+\.$/) {
+	  $_ = $1;
+	  $suffix = ".\n$suffix";
+	  if (defined($Token{$_})) {
+	    print "$_\n$suffix";
+	    next;
+	  }
+	}
+                 
+	# cut off clitics
+	if ($PClitic ne '') {
+	  while (s/^($PClitic)(.)/$2/) {
+	    print $1,"\n";
+	  }
+	}
+	if ($FClitic ne '') {
+	  while (s/(.)($FClitic)$/$1/) {
+	    $suffix = "$2\n$suffix";
+	  }
+	}
+                 
+	print "$_\n$suffix";
+      }
+    }
+  }
+}

Property changes on: tmp/org.txm.treetagger.core.win32/res/win32/cmd/tokenize.pl
___________________________________________________________________
Added: svn:executable
   + *

Index: tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output-french.perl
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output-french.perl	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/cmd/filter-chunker-output-french.perl	(revision 1683)
@@ -0,0 +1,229 @@
+#!/usr/bin/perl
+
+###################################################################
+###                                                             ###
+###      File: filter-chunker-output.perl                       ###
+###    Author: Michel Genereux                                  ###
+###            (indicated modifications by Dennis Spohr (DS))   ###
+###   Purpose: Filter chunker output and create XML-like markup ###
+###   Created: Mon Feb 19 2007                                  ###
+###                                                             ###
+###################################################################
+
+use Getopt::Std;
+getopts('t');
+
+$| = 1;
+
+print doc_start();
+
+### DS start: end-of-sentence marker
+$eos = '[.?!;]';
+$push = 1;
+###
+
+$n = 0;
+
+while (<>) {
+  s/.-SBAR$/O/;
+  s/I-PC$/0/  if (/I-PC$/ && !$inside_pp);
+
+  ### DS start: process lemma column
+  if (($token[$n],$tag[$n],$tag,$chunk[$n],$x,$lemma[$n]) = $_ =~ /^(.*)-(.*)\t(.*)\/(.*)(\t(.*))?$/) {
+  ### DS end
+
+    ### DS start: chunking error; some SENTs have e.g. I-NP although
+    ###           they mark the end of a sentence; lead to omission 
+    ###           of closing tags
+    $push = 0;
+    $chunk[$n] = 0 if ($tag eq 'SENT' && $chunk[$n] =~ /^(I|B)-/);
+    ### DS end
+
+    if ($chunk[$n] =~ /^(.*)-(.*)$/) {
+      $flag[$n] = $1;
+      $chunk[$n] = $2;
+    } else {
+      undef $flag[$n];
+      undef $chunk[$n];
+    }
+
+    ### DS start: performance boost: set $n to 0 after printing
+    ###           sentence; otherwise $n and arrays get too big 
+    ###           and cause slowdown
+    if ($token[$n] =~ /^$eos\s*$/ && $chunk[$n] == 0 && $tag[$n] eq 'SENT') {
+      print_sentence(0);
+      $n = 0;
+      $start_markup = "";
+    } else {
+      $n++;
+    }
+    ### DS end
+
+  ### DS start: keep markup already present in input data and insert
+  ###           chunker markup correctly; if an element starting before
+  ###           the sentence is closed before the sentence is closed
+  ###           (e.g. headlines without sentence end markers), then
+  ###           the sentence should also be closed, e.g. avoid cases like
+  ###           <HEADLINE><s>Les résultats de jeudi</HEADLINE></s>
+  } elsif (/^<([^\/]*?)(( |~).*)?>/ && $push) {
+      push(@tag_stack,$1);
+      $start_markup .= "$&\n";
+  } elsif (/^<\/(.*?)>/ && $1 eq $tag_stack[$#tag_stack]) {
+      $end_markup = "$&\n";
+      print_sentence(1);
+      $n = 0;
+      $push = 1;
+      $start_markup = "";
+      $end_markup = "";
+      pop(@tag_stack);
+  ### DS end
+
+  } else {
+    $markup[$n] .= $_;
+  }
+}
+
+print_sentence(1);
+print doc_end();
+
+
+sub print_sentence {
+
+  ### DS start: indicate whether print_sentence is forced by
+  ###           closing input markup
+  my $forced = shift;
+  ### DS end
+
+  my($i,$chunk);
+
+  for( $i=0; $i<=$n; $i++ ) {
+    if ($flag[$i] eq 'I' && $chunk ne $chunk[$i]) {
+      $flag[$i] = 'B';
+    }
+    if ($flag[$i] eq 'B') {
+      if (defined $chunk) {
+	$cetags[$i-1] .= end_tag($chunk);
+      }
+      if ($chunk[$i] eq 'PC') {
+	for( $k=$i+1; $k<=$n; $k++ ) {
+	  last if ($flag[$k] eq 'B');
+	}
+	for( $k++; $k<=$n; $k++ ) {
+	  last if ($flag[$k] ne 'I');
+	}
+	if ($k <= $n && $flag[$k] eq 'E' && $chunk[$k] eq 'PC') {
+	  $markup[$k+1] .= end_tag('PC');
+	  undef $flag[$k];
+	  undef $chunk[$k];
+
+        ### DS start: $k may be greater than $n; add closing PC tag
+        ###           to $markup[$n]; otherwise closing tags are
+        ###           omitted
+	} elsif ($k > $n && ($forced || $token[$n] =~ /^$eos\s*$/)) {
+          $markup[$n] .= end_tag('PC');
+        ### DS end
+
+        } else {
+	  $markup[$k] .= end_tag('PC');
+	}
+	undef $chunk;
+      }
+      else {
+	$chunk = $chunk[$i];
+      }
+      $cbtags[$i] .= start_tag($chunk[$i]);
+    }
+    elsif ($flag[$i] eq 'E') {
+      if ($chunk[$i] eq $chunk) {
+	$cetags[$i] .= end_tag($chunk);
+	undef $chunk;
+      }
+      elsif ($chunk[$i] eq 'PC') {
+	$cetags[$i-1] .= end_tag($chunk) if defined $chunk;
+	$cetags[$i] .= end_tag("PC");
+	my $k;
+	for( $k=$i; $k>=0; $k-- ) {
+	  if ($flag[$k] eq 'B') {
+	    $cbtags[$k] = start_tag("PC").$cbtags[$k];
+	    last;
+	  }
+	}
+	undef $chunk;
+      }
+      else {
+	die;
+      }
+    }
+    elsif ($flag[$i] ne 'I' && defined $chunk) {
+      $cetags[$i-1] .= end_tag($chunk);
+      undef $chunk;
+    }
+  }
+
+  $printed = 0;# start_tag("s");
+
+  ### DS start: print opening tags of input markup before sentence
+  print $start_markup;
+  print start_tag("s") if $n > 0;
+  ### DS end
+
+  for( $i=0; $i<=$n; $i++ ) {
+    print $markup[$i];
+    #unless ($printed) {
+    #  print start_tag("s");
+    #  $printed = 1;
+    #}
+    print $cbtags[$i];
+
+    ### DS start: slightly renamed sub and added lemma parameter
+    print token_and_tag_and_lemma($token[$i],$tag[$i],$lemma[$i]) if defined $token[$i];
+    ### DS end
+
+    print $cetags[$i];
+  }
+
+  ### DS start: print closing "s" tag and closing input markup if 
+  ###           print_sentence had been forced
+  print end_tag("s") if $n>0;
+  print $end_markup if $forced;
+  ### DS end
+
+  undef @token;
+  undef @tag;
+  undef @chunk;
+  undef @cbtags;
+  undef @cetags;
+  undef @flag;
+  undef @markup;
+}
+
+sub doc_start {
+  return '' unless defined $opt_t;
+  return "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"yes\"?>\n<corpus>\n";
+}
+
+sub doc_end {
+  return '' unless defined $opt_t;
+  return "</corpus>\n";
+}
+
+sub start_tag {
+  my $t=shift;
+  return "<$t>\n" unless defined $opt_t;
+  return "  <phrase cat=\"$t\">\n";
+}
+
+sub end_tag {
+  my $t=shift;
+  return "</$t>\n" unless defined $opt_t;
+  return "  </phrase>\n";
+}
+
+### DS start: also process and output lemma parameter
+sub token_and_tag_and_lemma {
+  my ($token,$tag,$lemma)=@_;
+  return "$token\t$tag\t$lemma\n" unless defined $opt_t;
+  return "    <token word=\"$token\" lemma=\"$lemma\" pos=\"$tag\"/>\n";
+}
+### DS end
+
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/italian-abbreviations
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/italian-abbreviations	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/italian-abbreviations	(revision 1683)
@@ -0,0 +1,78 @@
+All.
+Art.
+B.
+Cap.
+Cfr.
+D.
+Doc.
+E.
+Em.
+F.
+G.
+Gen.
+H.
+Ind.
+J.
+K.
+Ltd.
+M.
+Med.
+N.
+On.
+P.
+Pag.
+Ph.
+Prof.
+Q.
+R.
+Racc.
+Rec.
+Rep.
+Sig.
+St.
+U.
+W.
+Z.
+art.
+artt.
+c.
+ca.
+cap.
+cfr.
+cit.
+cm.
+def.
+doc.
+ecc.
+em.
+emm.
+es.
+etc.
+f.
+g.
+id.
+imp.
+m.
+n.
+nn.
+ob.
+on.
+onn.
+op.
+p.
+pag.
+pagg.
+par.
+pp.
+prof.
+q.
+r.
+reg.
+riv.
+s.
+sec.
+segg.
+sig.
+u.
+v.
+vol.
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/french-abbreviations
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/french-abbreviations	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/french-abbreviations	(revision 1683)
@@ -0,0 +1,97 @@
+rendez-vous
+d'abord
+d'accord
+d'ailleurs
+d'aprÃ¨s
+d'autant
+d'Âœuvre
+d'oeuvre
+c'est-Ã -dire
+moi-mÃªme
+toi-mÃªme
+lui-mÃªme
+elle-mÃªme
+nous-mÃªmes
+vous-mÃªmes
+eux-mÃªmes
+elles-mÃªmes
+par-ci
+par-lÃ 
+Rendez-vous
+D'abord
+D'accord
+D'ailleurs
+D'aprÃ¨s
+D'autant
+D'Âœuvre
+D'oeuvre
+C'est-Ã -dire
+Moi-mÃªme
+Toi-mÃªme
+Lui-mÃªme
+Elle-mÃªme
+Nous-mÃªmes
+Vous-mÃªmes
+Eux-mÃªmes
+Elles-mÃªmes
+Par-ci
+Par-lÃ 
+Arg.
+Aus.
+Aut.
+Cag.
+Cl.
+Corp.
+Corresp.
+E.
+Esp.
+Etc.
+Fr.
+Fra.
+G.
+H.
+I.
+Inc.
+Incr.
+Inscr.
+It.
+Ita.
+Jap.
+K.
+P.
+R.
+RÃ©g.
+RÃ©n.
+Tch.
+TÃ©l.
+V.
+W.
+Yam.
+abst.
+adj.
+adm.
+app.
+av.
+b.
+cf.
+corp.
+diss.
+div.
+dr.
+etc.
+expr.
+g.
+id.
+inscr.
+maj.
+p.
+r.
+rad.
+ratt.
+sc.
+suffr.
+tÃ©l.
+vot.
+Ã©col.
+Ã©d.
+Ã©tiq.
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/spanish-abbreviations
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/spanish-abbreviations	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/spanish-abbreviations	(revision 1683)
@@ -0,0 +1,5 @@
+Ref.
+Vol.
+etc.
+App.
+Rec.
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/dutch-abbreviations
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/dutch-abbreviations	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/dutch-abbreviations	(revision 1683)
@@ -0,0 +1,103 @@
+'t
+'s
+'n
+2bis.
+3bis.
+7bis.
+AR.
+Actualit.
+Afd.
+Antw.
+Arbh.
+Art.
+B.St.
+B.s.
+Besl.W.
+Bull.
+Bull.Bel.
+Cass.
+Cf.
+Com.I.B.
+D.t/V.I.
+Dhr.
+Doc.
+Dr.
+Fisc.
+Fr.
+Gec.
+II.
+III.
+J.-L.M.
+NR.
+NRS.
+Nat.
+No.
+Nr.
+Onderafd.
+PAR.
+Par.
+RECHTSFAK.
+RKW.
+TELEF.
+Volksvert.
+Vr.
+a.
+adv.-gen.
+afd.
+aj.
+al.
+arb.
+art.
+artt.
+b.
+b.v.
+b.w.
+bijv.
+blz.
+bv.
+c.q.
+cf.
+cfr.
+concl.
+d.
+d.d.
+d.i.
+d.w.z.
+dd.
+doc.
+e.
+e.d.
+e.v.
+enz.
+f.
+fr.
+g.w.
+gepubl.
+i.p.v.
+i.v.m.
+j.t.t.
+jl.
+k.b.
+kol.
+m.b.t.
+m.i.
+max.
+n.a.v.
+nl.
+nr.
+nrs.
+o.a.
+o.b.s.i.
+o.m.
+opm.
+p.
+par.
+pct.
+pp.
+ref.
+resp.
+respekt.
+t.a.v.
+t.o.v.
+vb.
+w.
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/english-abbreviations
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/english-abbreviations	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/english-abbreviations	(revision 1683)
@@ -0,0 +1,117 @@
+Adm.
+Ala.
+Ariz.
+Ark.
+Aug.
+Ave.
+Bancorp.
+Bhd.
+Brig.
+Bros.
+CO.
+CORP.
+COS.
+Ca.
+Calif.
+Canada-U.S.
+Canadian-U.S.
+Capt.
+Cia.
+Cie.
+Co.
+Col.
+Colo.
+Conn.
+Corp.
+Cos.
+D-Mass.
+Dec.
+Del.
+Dept.
+Dr.
+Drs.
+Etc.
+Feb.
+Fla.
+Ft.
+Ga.
+Gen.
+Gov.
+Hon.
+INC.
+Ill.
+Inc.
+Ind.
+Jan.
+Japan-U.S.
+Jr.
+Kan.
+Korean-U.S.
+Ky.
+La.
+Lt.
+Ltd.
+Maj.
+Mass.
+Md.
+Messrs.
+Mfg.
+Mich.
+Minn.
+Miss.
+Mo.
+Mr.
+Mrs.
+Ms.
+Neb.
+Nev.
+No.
+Nos.
+Nov.
+Oct.
+Okla.
+Ont.
+Ore.
+Pa.
+Ph.
+Prof.
+Prop.
+Pty.
+Rep.
+Reps.
+Rev.
+S.p.A.
+Sen.
+Sens.
+Sept.
+Sgt.
+Sino-U.S.
+Sr.
+St.
+Ste.
+Tenn.
+Tex.
+U.S.-U.K.
+U.S.-U.S.S.R.
+Va.
+Vt.
+W.Va.
+Wash.
+Wis.
+Wyo.
+a.k.a.
+a.m.
+anti-U.S.
+cap.
+days.
+etc.
+ft.
+i.e.
+non-U.S.
+p.m.
+president-U.S.
+s.r.l.
+v.
+v.B.
+v.w.
+vs.
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/german-abbreviations
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/german-abbreviations	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/german-abbreviations	(revision 1683)
@@ -0,0 +1,1300 @@
+A.
+A.-G.
+A.G.
+ADN-Korr.
+AT-Mot.
+Abb.
+Abess.
+Abl.
+Ablief.-Gew.
+Abm.
+Abs.
+Abt.
+Abtlg.
+Agl.
+Agt.
+Akt.-Ges.
+Aktbr.
+Alg.
+Alleininh.
+Allg.
+Altwageneint.
+Alu-Felg.
+Alum.
+Am.
+Amp.
+Anf.
+Anfr.
+Anfrag.
+Ang.
+Angb.
+Angeb.
+Angl.
+AnhÃ¤ngerkuppl.
+Anl.
+Anleih.
+Ann.-Exp.
+Ann.-Exped.
+Ant.
+Anten.
+Anz.
+Anz.-Exp.
+Anz.-Verm.
+Anzahlg.
+Anzhlg.
+Apoth.
+App.
+Appartem.
+April-Lief.
+Argent.
+Atl.
+Aufb.
+Aufst.
+Aug.
+Augsb.
+Ausg.
+Ausgl.
+Ausk.
+Ausl.
+Ausl.-Akt.
+Auslandsanl.
+Auslandsb.
+Ausst.
+Ausstattg.
+Austral.
+Ausz.
+Aut.
+Autom.
+Automat.
+Automin.
+B.
+B.P.
+BGBl.
+Bahnhofstr.
+Balk.
+Bau-Ing.
+Bauges.
+Bauj.
+Bay.
+Bayer.
+Bb.
+Bd.
+Bed.
+Beding.
+Ber.
+Beratg.
+Bereif.
+Bergb.
+Bergstr.
+Bernh.
+Bes.
+Besichtig.
+Bestzust.
+Beteil.
+Beteilig.
+Betr.
+Bett.
+Bew.
+Bewerb.
+Bewerbg.
+Bez.
+Bgl.
+Bhf.
+Bierbr.
+Bildzuschr.
+Bilf.
+Bj.
+Bk.
+Bkz.
+Bl.
+Bln.
+Boch.
+Bod.
+Bor.
+Bov.
+Br.
+Brem.
+Brh.
+Brok.
+Brsg.
+Bu.
+Bung.
+Burgstr.
+Bw.
+Bwsp.
+Bz.
+BÃ¤d.
+C.
+C.G.
+Cabr.
+Can.
+Cap.
+Cav.
+Cbr.
+Cem.
+Centralb.
+Cert.
+Ch.
+Charlottenstr.
+Chem.
+Chem.-Ing.
+Chevr.
+Chr.
+Christophstr.
+Cie.
+Co.
+Colorvergl.
+Commerzb.
+Conc.
+Cons.
+Corneliusstr.
+Corp.
+Cp.
+Cpt.
+Cz.
+D.
+DG.
+DM.
+DUB-Schulth.
+DW.
+Dahlb.
+Dawes-Anl.
+Dept.
+Dev.
+Dez.
+Di.
+Dipl.
+Dipl.-Ing.
+Dipl.-Kfm.
+Dir.
+Direktionsw.
+Div.
+Do.
+Doll.
+Don.
+Dorfk.
+Dpf.
+Dr.
+Dr.-Ing.
+Dreij.
+Drog.
+Dt.
+Du.
+Dyckerh.
+Dyn.
+DÃ¤n.
+DÃ¼sseld.
+E.
+E.h.
+Einf.
+Einh.
+Einr.
+Eint.
+Eintr.
+Einw.
+Einwohn.
+Einz.
+Einzelzi.
+Eisenb.
+El.
+Elektr.
+Em.
+Endpr.
+Engl.
+Ent.
+Entsch.
+Entw.
+Erdgesch.
+Erf.
+Erfahr.
+Erstzul.
+Erzgeb.
+Esc.
+Eterna.
+Etg.-Hs.
+Eur.
+Ew.
+Exp.
+Expl.
+F.
+FS.
+Fa.
+Fabr.
+Fabrikat.
+Fachm.
+Fachricht.
+Fahrz.
+Fam.
+Fb.
+Fd.
+Fds.
+Feb.
+Febr.
+Febr.-Abl.
+Febr.-MÃ¤rz-Abl.
+Feldstr.
+Fensterh.
+Ferd.
+Ferdinandstr.
+Fernschr.
+Ferr.
+Feuervers.
+Ffm.
+Fil.
+Fin.
+Finanzier.
+Finanzierg.
+FinanzierungsmÃ¶glichk.
+Finnl.
+Ford.
+Fortschr.
+Fr.
+Frankf.
+Franz.
+Franziskanerstr.
+Freiverk.
+Frhr.
+Fried.
+Friedr.
+Friedrich-Ebert-Str.
+Friedrichstr.
+Frl.
+FrÃ¼hst.
+FÃ¼hrersch.
+G.
+G.M.B.H.
+G.m.b.H.
+Gar.
+Garag.
+Gart.
+Geb.
+Gebr.
+Gehaltsanspr.
+Gehmin.
+Gelsenk.
+Gem.
+Gen.
+Ges.
+Ges.-Gew.
+Ges.-Verschuld.
+GeschÃ¤ftshs.
+Giroz.
+Gladb.
+Gleisanschl.
+GmbH.
+Goldschm.
+Gr.
+Graf-Adolf-Str.
+Graf-Recke-Str.
+Grdst.
+GroÃŸbr.
+Grundst.
+Grundstck.
+GÃ¼.
+GÃ¼rtelr.
+H.
+H.-J.
+Ha.
+Hall.
+Halogenl.
+Halteg.
+Hamb.
+Hambg.
+Han.
+Hand.
+Handelsges.
+Handelsvertr.
+Hann.
+Hardenbergstr.
+Hartst.
+Hauptstr.
+Hausprosp.
+Hbf.
+Hbg.
+Hd.
+Hdbk.
+Hdl.
+Hdt.
+Hecksch.
+Heerstr.
+Hein.
+Heinr.
+Heizg.
+Herm.
+Herst.
+Highv.
+Hindenburgstr.
+Hochs.
+Hofbr.
+Hoffm.
+Hold.
+Holst.
+Holstenbr.
+Hp.
+Hrsg.
+Hs.
+Hsm.
+Htt.
+Hw.
+Hyp.
+Hypbk.
+Hypo.
+Hypoth.
+Hzg.
+HÃ¼tt.
+I.
+I.E.
+IG.
+Imm.
+Immob.
+Immobil.
+Imp.
+Inc.
+Ind.
+Ind.-Akt.
+Ind.-Obligat.
+Industrie-Obl.
+Ing.
+Inh.
+Inl.
+Instr.
+Int.
+Interess.
+Interessent.
+Intern.
+Inv.
+Invent.
+Invest.
+Inzahlungn.
+Isestr.
+J.
+Jag.
+Jahrh.
+Jan.
+Jan.-Abl.
+Jan.-Febr.-Abl.
+Jan.-Lief.
+Jan./Febr.
+Jap.
+Joh.
+Jos.
+Jr.
+Juli-Abl.
+K.
+K.-Marx-St.
+K.G.
+K.o.
+KG.
+KHB.
+Kaderabt.
+Kaiser-Wilhelm-Str.
+Kaiserstr.
+Kam.
+Kampnag.
+Kap.
+Kap.-Anl.
+Kassel-Ha.
+Kaufm.
+Kaufpr.
+Kaufprs.
+Kenn-Nr.
+Kenntn.
+Kfm.
+Kg.
+Kl.
+Klimaanl.
+Klosterstr.
+KlÃ¶.
+KlÃ¶ckn.
+Komf.
+Kon.
+Kopenh.
+Kopfst.
+Kr.
+Kraftw.
+Krdbk.
+Krs.
+Kt.
+Kunstm.
+KÃ¼.
+L.
+L.-Gr.
+Lad.
+Landhs.
+Landstr.
+Lb.
+Lbd.
+Lbk.
+Ldb.
+Ldbk.
+Ldkr.
+Ldkrb.
+Ldsch.
+Ldw.
+Leb.
+Lebensl.
+Lebensvers.
+Led.
+Lederp.
+Lein.
+Leipz.
+Leitg.
+Lessingstr.
+Lfg.
+Lg.
+Li.
+Lic.
+Lichtb.
+Lief.
+Liegew.
+Lim.
+Lit.
+LohnhÃ¼ttenpr.
+Lomb.
+Ltd.
+Ltg.
+Ltr.
+Lufth.
+LÃ¼b.
+M.
+MWSt.
+Mad.
+Mag.
+Mannesm.
+Marktstr.
+Masch.
+Mech.
+Merc.
+Merced.
+Met.
+Metallges.
+Mg.
+Mi.
+Mill.
+Min.
+Mio.
+Mo.
+Mod.
+Moltkestr.
+Mon.
+Mont.
+Morg.
+Mot.
+Mr.
+Mrd.
+Mrg.
+Mrs.
+Mte.
+Mu.
+MwSt.
+Mwst.
+MÃ¼.
+MÃ¼nsterstr.
+N.
+N.Y.
+ND-Korr.
+NSU-Niederl.
+Nachf.
+Nachl.
+Nachn.
+Nachs.
+Nam.
+Nat.
+Nb.
+Nbg.
+Ndb.
+Nebell.
+Nebenr.
+Neckarw.
+Neckerm.
+Nederl.
+Nettopr.
+Neub.
+Neupr.
+NeuprÃ¤g.
+Nh.
+Nied.
+Niederl.
+Nieders.
+Niedersachs.
+Niveauregl.
+No.
+Nov.
+Nr.
+Nutzfl.
+Nutzl.
+NÃ¤h.
+O.
+O.-L.
+Obb.
+Obligat.
+Off.
+Okt.
+Oldb.
+Opt.
+Optionssch.
+Org.-Spr.
+Ost.
+Ostb.
+Ostfr.
+P.
+Pac.
+Pap.
+Parkpl.
+Pat.
+Pens.
+Pens.-Pr.
+Pers.
+Pes.
+Petroleum-Comp.
+Pf.
+Pfandbr.
+Pfb.
+Pfbr.
+Pfdbr.
+Pfg.
+Pfi.
+Ph.
+Pkt.
+Pkw.
+Pl.
+Porz.
+Pos.
+Postf.
+Poststr.
+Pr.
+Prinzenstr.
+Priv.
+Prof.
+Prop.
+Prosp.
+Prov.
+Proz.
+Prs.
+R.
+Ra.
+Rd.
+Recklingh.
+Red.
+Ref.
+Reg.
+Reg.-Bez.
+Rent.
+Rest.
+Restaur.
+Rh.
+Rheinbod.
+Rheinmet.
+Rhld.
+Rob.
+Rud.
+Ruhrgeb.
+RÃ¼ckn.
+RÃ¼ckst.
+RÃ¼tgersw.
+S.
+S.A.
+SL-Hardt.
+Sa.
+Sa.-Nr.
+Sachsenstr.
+Sal.
+Salzdetf.
+Sbd.
+Sch.
+Schauenburgerstr.
+Schbd.
+Schiebed.
+Schiff.
+Schiffsb.
+Schl.
+Schlafpl.
+Schlafzim.
+Schlesw.-Holst.
+Schuldverschreib.
+Schw.
+Sd.
+Sec.
+Sek.
+Sel.
+Sem.
+Sep.
+Sept.
+Servol.
+Servolenk.
+Sicherheitsg.
+Siegl.
+Siem.
+Silb.
+Sinf.
+So.
+Sonderinst.
+Sort.
+Sp.
+SpVg.
+Spaldingstr.
+Span.
+Spfr.
+Spinn.
+Spr.
+Sprachkenntn.
+Spvg.
+Spvgg.
+SpÃ¤tv.
+St.
+StK.
+Stahlschiebed.
+Stahlw.
+Stck.
+Std.
+Steink.
+Stell.
+Stellg.
+Stellv.
+Stg.
+Stk.
+Str.
+SÃ¼ddeutschld.
+SÃ¼dw.
+T.
+T.E.A.M.
+TO.
+Tabl.
+Tagungsr.
+Tankst.
+Tauernkrw.
+Teilh.
+Teilz.
+Tel.
+Tel.-Nr.
+Tel.-Sa.-Nr.
+Telefon-Sa.-Nr.
+Telegr.
+Teleph.
+Terr.
+Tg.
+Th.
+Thyss.
+ThÃ¼r.
+Tit.
+Tob.
+Toil.
+Tr.
+Ts.
+Tx.
+TÃ¤tigk.
+U-Bhf.
+U.
+U.S.
+UBM.
+UP.
+Ums.
+Un.
+Unterlag.
+Untern.
+Urt.
+V.
+VWD.
+Vb.
+Vbk.
+Veloursp.
+Ver.
+Verbindlichk.
+Vereinsb.
+Verg.
+Verg.-Gr.
+Verk.
+Verl.
+Verm.
+Vers.
+Versich.
+Versicherung.
+Vertr.
+Verw.
+Vict.
+Virg.
+Vogtl.
+Volkskorr.
+Vollp.
+Vollpens.
+VorfÃ¼hrw.
+VorfÃ¼hrwg.
+Vorst.
+Vorverk.
+Vorw.
+Vorz.
+Vulk.
+Vw.
+Vz.
+W.
+Wag.
+Wandelanl.
+Wandelschuldverschreibung.
+Wass.
+Wasserversorg.
+Wb.
+Welt-Agt.
+Welt-Agtr.
+Welt-Verl.
+Werkst.
+Werkzeugmasch.
+Wertpapier-Kenn-Nr.
+West.
+Westerw.
+Westf.
+Westfalia-Werkzeugco.
+Wf.
+Wfl.
+Whg.
+Wilh.
+Wilhelmstr.
+Wintergart.
+Wintersh.
+Wirkungskr.
+Wk.
+Woch.
+Wohn.
+Wohnfl.
+Wohng.
+Wohnh.
+Wollk.
+Wuppertal-E.
+Ww.
+Wwe.
+Wz.
+WÃ¼rtt.
+Y.
+Young-Anl.
+Z.
+Zahlg.
+Zellst.
+Zem.
+Zentr.
+Zentralhzg.
+Zentralverr.
+Zeugn.
+Zeugnisabschr.
+Zhzg.
+Zi.
+Ziff.
+Zim.
+Zimm.
+Zst.
+Ztrlhzg.
+Zubeh.
+Zuf.
+Zuschr.
+Zust.
+Zw.
+Zyl.
+ZÃ¤hringerstr.
+a.
+a.D.
+a.G.
+a.M.
+a.d.
+a.i.
+abgeschl.
+abgeschloss.
+abgest.
+absol.
+abzug.
+abzugeb.
+all.
+alleinst.
+allererst.
+allg.
+allgem.
+amerik.
+amtl.
+and.
+anerk.
+angeb.
+angen.
+ao.
+arr.
+aufhalts.
+ausf.
+ausfÃ¼hrl.
+ausgeb.
+ausgef.
+ausgest.
+ausgez.
+ausl.
+auslÃ¤nd.
+ausschlieÃŸl.
+aut.
+autom.
+automat.
+b.
+bed.
+bedeut.
+bef.
+bek.
+bes.
+beschr.
+besond.
+best.
+bevorz.
+bez.
+bish.
+bl.
+blaumet.
+br.
+brit.
+brosch.
+bzw.
+c.
+ca.
+can.
+cand.
+cd.
+charakterv.
+chem.
+chines.
+cl.
+d.
+d.J.
+d.M.
+d.h.
+dav.
+dch.
+demokr.
+demokrat.
+dergl.
+desgl.
+dgl.
+dir.
+div.
+dl.
+do.
+dpa.
+dpd.
+dto.
+dtsch.
+dÃ¤n.
+e.
+e.G.m.b.H.
+e.V.
+e.h.
+ect.
+ege.
+ehem.
+eig.
+ein.
+eingeb.
+eingef.
+einger.
+eingericht.
+einh.
+einschl.
+einschlieÃŸl.
+el.
+eleg.
+elektr.
+elfenb.
+em.
+ems.
+engl.
+entspr.
+entsprech.
+erb.
+erbet.
+erdenkl.
+erf.
+erfahr.
+erfdl.
+erfolgr.
+erford.
+erforderl.
+erfordl.
+erschl.
+erstkl.
+erstklass.
+erstklassig.
+erw.
+erworb.
+etc.
+europ.
+ev.
+evgl.
+evt.
+evtl.
+ewe.
+ex.
+exkl.
+f.
+fa.
+fabrikn.
+festverzinsl.
+ff.
+ffrs.
+fl.
+flieÃŸ.
+ford.
+fortlfd.
+fr.
+franz.
+franzÃ¶s.
+frÃ¼h.
+fÃ¼hr.
+g.
+ganzjÃ¤hr.
+gar.
+geb.
+gebild.
+gebr.
+geeig.
+geeign.
+gefl.
+geg.
+gegenÃ¼b.
+gegr.
+gel.
+geleg.
+gem.
+gen.
+generalÃ¼berh.
+gepfl.
+gepr.
+ger.
+ges.
+gesch.
+geschl.
+geschloss.
+gest.
+gesund.
+getr.
+gewerbl.
+gez.
+ggf.
+gil.
+gl.
+goldmet.
+gr.
+grad.
+groÃŸ.
+grundbuchl.
+grÃ¶ÃŸ.
+grÃ¼n.
+grÃ¼ndl.
+grÃ¼nmet.
+gt.
+gut.
+gutbÃ¼rgerl.
+gz.
+gÃ¼nst.
+h.
+h.c.
+ha.
+habil.
+halb.
+handgeschr.
+handgeschrieb.
+handschriftl.
+heizb.
+heiÃŸluftgetr.
+herrl.
+hfl.
+hgk.
+hi.
+hl.
+holl.
+hollÃ¤nd.
+hÃ¶.
+hÃ¶chst.
+i.
+i.R.
+i.V.
+inbegr.
+inbegriff.
+incl.
+inkl.
+inl.
+inlÃ¤nd.
+innerh.
+insbes.
+insges.
+int.
+interess.
+intern.
+internat.
+ital.
+j.
+jed.
+jegl.
+jg.
+jr.
+jun.
+jung.
+jur.
+jÃ¤hr.
+jÃ¤hrl.
+jÃ¼ng.
+k.
+k.o.
+kan.
+kanad.
+kas.
+kath.
+kauf.
+kaufm.
+kbr.
+kem.
+kfm.
+kl.
+komf.
+komfort.
+kompl.
+kons.
+kostenl.
+kultiv.
+kurzfr.
+kurzfrist.
+l.
+landschaftl.
+langfr.
+langj.
+langjÃ¤hr.
+lb.
+leb.
+led.
+leit.
+lfd.
+lfr.
+lg.
+lib.
+lieferb.
+lni.
+lt.
+lust.
+m.
+m.b.H.
+mass.
+max.
+mbH.
+med.
+mehrj.
+meld.
+met'bl.
+mgl.
+mh.
+min.
+mind.
+mittl.
+mm.
+mod.
+monatl.
+mp.
+mtl.
+musikal.
+mÃ¤nnl.
+mÃ¶bl.
+mÃ¶gl.
+n.
+n.P.
+n.n.
+nachm.
+neuw.
+neuwert.
+neuzeitl.
+nom.
+nordd.
+nordeurop.
+norw.
+not.
+nÃ¤h.
+nÃ¶rdl.
+o.
+o.Ã¤.
+oHG.
+od.
+op.
+ortsgeb.
+p.
+p.a.
+pass.
+pcz.
+perf.
+pers.
+persÃ¶nl.
+pharmazeut.
+phil.
+pk.
+pm.
+pol.
+pp.
+ppa.
+prakt.
+preisg.
+preisgÃ¼nst.
+preisw.
+priv.
+proz.
+pt.
+r.
+raff.
+rd.
+ref.
+reichl.
+reizv.
+rep.
+reprÃ¤sent.
+rer.
+rhein.
+rl.
+romant.
+ruh.
+ruhig.
+russ.
+rustic.
+rustik.
+s.
+schl.
+schlk.
+schriftl.
+schw.
+se.
+seel.
+selbst.
+selbstÃ¤nd.
+sen.
+sep.
+seriÃ¶s.
+sfr.
+sfrs.
+sh.
+sid.
+silb.
+silbergrÃ¼nmet.
+silbermet.
+sitz.
+sof.
+sog.
+sonn.
+sonnengetr.
+sow.
+span.
+spez.
+st.
+staatl.
+stellv.
+stellvertr.
+stgs.
+su.
+sympath.
+sÃ¤mtl.
+sÃ¼dd.
+sÃ¼dl.
+t.
+ta.
+tar.
+tarifbest.
+techn.
+teilw.
+tel.
+tgl.
+tie.
+tot.
+tra.
+tÃ¤gl.
+tÃ¼cht.
+tÃ¼r.
+u.
+u.a.
+u.a.m.
+u.Ã¤.
+uff.
+uhl.
+umstÃ¤ndeh.
+umstÃ¤ndehalb.
+unabh.
+unfallfr.
+ung.
+ungek.
+ungekÃ¼nd.
+unmittelb.
+unt.
+unv.
+unverb.
+unverbindl.
+usw.
+v.
+v.H.
+v.d.H.
+v.l.n.r.
+v.r.n.l.
+verantw.
+verantwortl.
+vereid.
+verh.
+verhandlungsgew.
+verk.
+verkauf.
+verkehrsg.
+verkehrsgÃ¼nst.
+verkf.
+verkÃ¤ufl.
+verm.
+vermiet.
+verp.
+verpacht.
+versch.
+verschied.
+vertr.
+vertraul.
+verw.
+verÃ¤nd.
+ves.
+vgl.
+viels.
+vk.
+vorh.
+vorhand.
+vorhd.
+vorm.
+w.
+waldr.
+weibl.
+werd.
+westf.
+westl.
+wfm.
+wg.
+wirtschaftl.
+wissenschaftl.
+wÃ¤rmed.
+z.
+z.B.
+z.T.
+z.Z.
+z.Zt.
+zentr.
+ztr.
+zugel.
+zugelass.
+zur.
+zus.
+zusÃ¤tzl.
+zuverl.
+zuzÃ¼gl.
+zw.
+zz.
+zzgl.
+zzt.
+Ã–lheizg.
+Ã–lhz.
+Ã–lhzg.
+Ã–lzentralhzg.
+Ã–sterr.
+Ãœbern.
+Ã¤.
+Ã¤hnl.
+Ã¤rztl.
+Ã¶ff.
+Ã¶ffentl.
+Ã¶sterr.
+Ã¼.
+Ã¼b.
+Ã¼bern.
+Ã¼bl.
Index: tmp/org.txm.treetagger.core.win32/res/win32/lib/spanish-mwls
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/lib/spanish-mwls	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/lib/spanish-mwls	(revision 1683)
@@ -0,0 +1,293 @@
+A diferencia de
+A diferencia del
+A fin de
+A lo largo de
+A medida que
+A menudo
+A partir de
+A pesar de
+A tÃ­tulo de
+A veces
+Abu Dhabi
+Ahora bien
+Al contrario
+Al mismo tiempo
+Antigua y Barbuda
+AsÃ­ pues
+Burkina Faso
+Cabo Verde
+Con relaciÃ³n a
+Con respecto a
+Con respecto al
+Costa Rica
+Cote d'Ivoire
+Dado que
+De acuerdo con
+De conformidad con
+De hecho
+De ordinario
+Debido a
+Dentro de
+Dentro del
+DespuÃ©s de
+DespuÃ©s del
+El Salvador
+Emiratos Ã�rabes Unidos
+En cambio
+En caso de que
+En consecuencia
+En cuanto a
+En efecto
+En especial
+En funciÃ³n de
+En general
+En lo que
+En lugar de
+En particular
+En principio
+En relaciÃ³n con
+En todo caso
+En vista de
+Entre tanto
+Es decir
+Estados Unidos
+Estados Unidos de AmÃ©rica
+Esto es
+Fuera de
+Gracias a
+Ha de
+HabrÃ¡ que
+Hasta ahora
+Hay que
+Islas VÃ­rgenes
+Joint Committee Technical
+Junto con
+Mar del Plata
+Mientras que
+Mientras tanto
+Naciones Unidas
+No obstante
+Nueva Caledonia
+Nueva Delhi
+Nueva York
+Nueva Zelandia
+Papua Nueva Guinea
+PaÃ­ses Bajos
+Pese a
+Por consiguiente
+Por ejemplo
+Por el contrario
+Por el momento
+Por encima de
+Por encima del
+Por lo general
+Por lo tanto
+Por medio de
+Por medio del
+Por regla general
+Por tanto
+Por Ãºltimo
+Puerto Rico
+Puesto que
+Ras Al Khaimah
+Reino Unido
+Reino Unido de Gran BretaÃ±a
+Saint Kitts
+San Marino
+San Pedro
+San Vicente
+Santa LucÃ­a
+Santo TomÃ©
+Si bien
+Siempre que
+Sierra Leona
+Sin embargo
+Sri Lanka
+Tal vez
+Tan pronto como
+Tienen que
+Umm al Qaiwain
+Una vez que
+UniÃ³n de RepÃºblicas Socialistas SoviÃ©ticas
+VÃ­rgenes BritÃ¡nicas
+a cambio de
+a excepciÃ³n de
+a fin de
+a fondo
+a la derecha
+a la izquierda de
+a la izquierda del
+a la que
+a la vez
+a lo largo de
+a lo largo del
+a medida que
+a menos que
+a menudo
+a nivel de
+a partir de
+a partir del
+a pesar de
+a primera vista
+a priori
+a propÃ³sito
+a punto de
+a razÃ³n de
+a saber
+a travÃ©s de
+a travÃ©s del
+a tÃ­tulo de
+a veces
+acerca de
+ahora bien
+al cabo de
+al cabo del
+al contrario que
+al dÃ­a
+al fin
+al lado
+al menos
+al mismo tiempo
+al pie de
+al respecto
+asÃ­ como
+asÃ­ que
+cada vez que
+como mÃ­nimo
+con detalle
+con ocasiÃ³n de
+con relaciÃ³n a
+con respecto a
+con respecto al
+conforme a
+conforme al
+dado que
+de acuerdo con
+de antemano
+de conformidad con
+de forma que
+de hecho
+de manera que
+de modo que
+de momento
+de nuevo
+de ordinario
+de todas formas
+debajo de
+debajo del
+debe de
+debido a
+debido a que
+debido al
+del orden de
+delante de
+dentro de
+dentro del
+desde el momento en que
+despuÃ©s de
+despuÃ©s del
+en absoluto
+en breve
+en cambio
+en caso de que
+en cierta medida
+en comÃºn
+en conformidad con
+en consecuencia
+en cuanto a
+en cuanto al
+en cuestiÃ³n
+en definitiva
+en detalle
+en efecto
+en especial
+en funciÃ³n de
+en funciÃ³n del
+en general
+en lo que
+en lugar de
+en lugar del
+en materia de
+en menos de
+en parte
+en particular
+en principio
+en realidad
+en relaciÃ³n con
+en tanto
+en teorÃ­a
+en todo caso
+en torno al
+en vez de
+en vez del
+en virtud de
+en virtud del
+en vista de
+encima de
+es decir
+esto es
+frente a
+fuera de
+fuera del
+gracias a
+ha de
+habrÃ¡ de
+habrÃ¡ que
+habrÃ¡n de
+habrÃ­a que
+han de
+hay que
+haya de
+haya que
+hayan de
+hoy dÃ­a
+in situ
+junto con
+mientras que
+no mÃ¡s
+no obstante
+o sea
+por ahora
+por cierto
+por consecuencia
+por consiguiente
+por cuanto
+por ejemplo
+por el momento
+por encima de
+por encima del
+por ende
+por extensiÃ³n
+por lo general
+por lo menos
+por lo tanto
+por medio de
+por medio del
+por parte de
+por parte del
+por quÃ©
+por separado
+por supuesto
+por tanto
+por un lado
+puesto que
+respecto de
+respecto del
+si bien
+siempre que
+siempre y cuando
+sin distinciÃ³n
+sin embargo
+sobre todo
+tal vez
+tan pronto como
+tanto por ciento
+tendrÃ¡ que
+tendrÃ¡n que
+tener que
+tenga que
+tengan que
+tiene que
+tienen que
+una vez que
+ya que
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-italian.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-italian.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-italian.bat	(revision 1683)
@@ -0,0 +1,24 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\italian-utf8.par -token -lemma -sgml -no-unknown
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -i -a %LIB%\italian-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -i -a %LIB%\italian-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% 
+goto end
+
+:label2
+echo.
+echo Usage: tag-italian file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-french.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-french.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-french.bat	(revision 1683)
@@ -0,0 +1,26 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\french-utf8.par -quiet -token -lemma -sgml -no-unknown
+set CHUNKOPT=%LIB%\french-chunker-utf8.par -token -sgml -eps 0.00000001 -hyphen-heuristics
+set OPT=-nae "if ($#F==0){print}else{print \"$F[0]-$F[1]\n\"}"
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -f -a %LIB%\french-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% | perl %OPT% | %BIN%\tree-tagger %CHUNKOPT% | perl %CMD%\filter-chunker-output.perl | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -f -a %LIB%\french-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% | perl %OPT% | %BIN%\tree-tagger %CHUNKOPT% | perl %CMD%\filter-chunker-output-french.perl | %BIN%\tree-tagger %TAGOPT%
+goto end
+
+:label2
+echo.
+echo Usage: chunk-french file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/train-tree-tagger.exe
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: tmp/org.txm.treetagger.core.win32/res/win32/bin/train-tree-tagger.exe
___________________________________________________________________
Added: svn:executable
   + *
Added: svn:mime-type
   + application/octet-stream

Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-french.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-french.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-french.bat	(revision 1683)
@@ -0,0 +1,24 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\french-utf8.par -token -lemma -sgml -no-unknown
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -f -a %LIB%\french-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -f -a %LIB%\french-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% 
+goto end
+
+:label2
+echo.
+echo Usage: tag-french file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-spanish.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-spanish.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-spanish.bat	(revision 1683)
@@ -0,0 +1,24 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\spanish-utf8.par -token -lemma -sgml -no-unknown
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -a %LIB%\spanish-abbreviations "%~1" | perl %CMD%\mwl-lookup.perl  -f %LIB%\spanish-mwls | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -a %LIB%\spanish-abbreviations "%~1" | perl %CMD%\mwl-lookup.perl  -f %LIB%\spanish-mwls | %BIN%\tree-tagger %TAGOPT%
+goto end
+
+:label2
+echo.
+echo Usage: tag-spanish file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-english.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-english.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-english.bat	(revision 1683)
@@ -0,0 +1,26 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\english-utf8.par -quiet -token -lemma -sgml -no-unknown
+set CHUNKOPT=%LIB%\english-chunker-utf8.par -token -sgml -eps 0.00000001 -hyphen-heuristics
+set OPT=-nae "if ($#F==0){print}else{print \"$F[0]-$F[1]\n\"}"
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -e -a %LIB%\english-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% | perl %OPT% | %BIN%\tree-tagger %CHUNKOPT% | perl %CMD%\filter-chunker-output.perl | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -e -a %LIB%\english-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% | perl %OPT% | %BIN%\tree-tagger %CHUNKOPT% | perl %CMD%\filter-chunker-output.perl | %BIN%\tree-tagger %TAGOPT% 
+goto end
+
+:label2
+echo.
+echo Usage: chunk-english file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-german.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-german.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/chunk-german.bat	(revision 1683)
@@ -0,0 +1,26 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\german-utf8.par -quiet -token -lemma -sgml -no-unknown
+set CHUNKOPT=%LIB%\german-chunker-utf8.par -token -sgml -eps 0.00000001 -hyphen-heuristics
+set OPT=-nae "if ($#F==0){print}else{print \"$F[0]-$F[1]\n\"}"
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -a %LIB%\german-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% | perl %OPT% | %BIN%\tree-tagger %CHUNKOPT% | perl %CMD%\filter-chunker-output.perl | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -a %LIB%\german-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% | perl %OPT% | %BIN%\tree-tagger %CHUNKOPT% | perl %CMD%\filter-chunker-output-german.perl | %BIN%\tree-tagger %TAGOPT%
+goto end
+
+:label2
+echo.
+echo Usage: chunk-german file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-dutch.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-dutch.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-dutch.bat	(revision 1683)
@@ -0,0 +1,24 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\dutch-utf8.par -token -lemma -sgml -no-unknown
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -a %LIB%\dutch-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -a %LIB%\dutch-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% 
+goto end
+
+:label2
+echo.
+echo Usage: tag-dutch file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tree-tagger-flush.exe
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: tmp/org.txm.treetagger.core.win32/res/win32/bin/tree-tagger-flush.exe
___________________________________________________________________
Added: svn:executable
   + *
Added: svn:mime-type
   + application/octet-stream

Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tree-tagger.exe
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: tmp/org.txm.treetagger.core.win32/res/win32/bin/tree-tagger.exe
___________________________________________________________________
Added: svn:executable
   + *
Added: svn:mime-type
   + application/octet-stream

Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-english.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-english.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-english.bat	(revision 1683)
@@ -0,0 +1,24 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\english-utf8.par -token -lemma -sgml -no-unknown
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -e -a %LIB%\english-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -e -a %LIB%\english-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% 
+goto end
+
+:label2
+echo.
+echo Usage: tag-english file {file}
+echo.
+
+:end
Index: tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-german.bat
===================================================================
--- tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-german.bat	(revision 0)
+++ tmp/org.txm.treetagger.core.win32/res/win32/bin/tag-german.bat	(revision 1683)
@@ -0,0 +1,24 @@
+@echo off
+
+set TAGDIR=C:\TreeTagger
+
+set BIN=%TAGDIR%\bin
+set CMD=%TAGDIR%\cmd
+set LIB=%TAGDIR%\lib
+set TAGOPT=%LIB%\german-utf8.par -token -lemma -sgml -no-unknown
+
+if "%2"=="" goto label1
+perl %CMD%\utf8-tokenize.perl -a %LIB%\german-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% > "%~2"
+goto end
+
+:label1
+if "%1"=="" goto label2
+perl %CMD%\utf8-tokenize.perl -a %LIB%\german-abbreviations "%~1" | %BIN%\tree-tagger %TAGOPT% 
+goto end
+
+:label2
+echo.
+echo Usage: tag-german file {file}
+echo.
+
+:end
