word aligner cleanup, new features

1 files changed, 27 insertions, 0 deletions

diff --git a/word-aligner/support/classify.pl b/word-aligner/support/classify.pl
new file mode 100755
index 00000000..893c7b22
--- /dev/null
+++ b/word-aligner/support/classify.pl
@@ -0,0 +1,27 @@
+#!/usr/bin/perl -w
+use strict;
+
+die "Usage: $0 classes.txt corpus.txt" unless scalar @ARGV == 2;
+
+my ($class, $text) = @ARGV;
+open C, "<$class" or die "Can't read $class: $!";
+open T, "<$text" or die "Can't read $text: $!";
+
+my %dict = ();
+my $cc = 0;
+while(<C>) {
+  chomp;
+  my ($word, $cat) = split /\s+/;
+  die "'$word' '$cat'" unless (defined $word && defined $cat);
+  $dict{$word} = $cat;
+  $cc++;
+}
+close C;
+print STDERR "Loaded classes for $cc words\n";
+
+while(<T>) {
+  chomp;
+  my @cats = map { $dict{$_} or die "Undefined class for $_"; } split /\s+/;
+  print "@cats\n";
+}
+