initial checkin

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@2 ec762483-ff6d-05da-a07a-a48fb63a330f

1 files changed, 23 insertions, 0 deletions

diff --git a/extools/test_data/make_len_cats.pl b/extools/test_data/make_len_cats.pl
new file mode 100755
index 00000000..25ef75fa
--- /dev/null
+++ b/extools/test_data/make_len_cats.pl
@@ -0,0 +1,23 @@
+#!/usr/bin/perl -w
+use strict;
+
+my $max_len = 15;
+my @cat_names = qw( NULL SHORT SHORT MID MID MID LONG LONG LONG LONG LONG VLONG VLONG VLONG VLONG VLONG );
+
+while(<>) {
+  chomp;
+  my @words = split /\s+/;
+  my $len = scalar @words;
+  my @spans;
+  for (my $i =0; $i < $len; $i++) {
+    for (my $k = 1; $k <= $max_len; $k++) {
+      my $j = $i + $k;
+      next if ($j > $len);
+      my $cat = $cat_names[$k];
+      die unless $cat;
+      push @spans, "$i-$j:$cat";
+    }
+  }
+  print "@spans\n";
+}
+