summaryrefslogtreecommitdiff
path: root/word-aligner/support
diff options
context:
space:
mode:
authorredpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-11-14 23:28:04 +0000
committerredpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-11-14 23:28:04 +0000
commit49fb41843a2ad81e3ef2b65e5b9264809aac1847 (patch)
tree49511a3358d20829278dfd8511546eb69ea9d46c /word-aligner/support
parent0c4066c71258b8a60b219f015b69634dc564af98 (diff)
few fixes
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@722 ec762483-ff6d-05da-a07a-a48fb63a330f
Diffstat (limited to 'word-aligner/support')
-rwxr-xr-xword-aligner/support/classify.pl12
-rwxr-xr-xword-aligner/support/make_lex_grammar.pl2
2 files changed, 11 insertions, 3 deletions
diff --git a/word-aligner/support/classify.pl b/word-aligner/support/classify.pl
index 893c7b22..e9e79063 100755
--- a/word-aligner/support/classify.pl
+++ b/word-aligner/support/classify.pl
@@ -1,7 +1,10 @@
#!/usr/bin/perl -w
use strict;
-die "Usage: $0 classes.txt corpus.txt" unless scalar @ARGV == 2;
+die "Usage: $0 [-unk] classes.txt corpus.txt\n Applies a vocabulary map to a corpus\n" unless scalar @ARGV == 2 || (scalar @ARGV == 3 && $ARGV[0] eq '-unk');
+
+my $unk = $ARGV[0] eq '-unk';
+shift @ARGV if $unk;
my ($class, $text) = @ARGV;
open C, "<$class" or die "Can't read $class: $!";
@@ -19,9 +22,14 @@ while(<C>) {
close C;
print STDERR "Loaded classes for $cc words\n";
+my @cats;
while(<T>) {
chomp;
- my @cats = map { $dict{$_} or die "Undefined class for $_"; } split /\s+/;
+ if ($unk) {
+ @cats = map { $dict{$_} or "UNK" } split /\s+/;
+ } else {
+ @cats = map { $dict{$_} or die "Undefined class for $_"; } split /\s+/;
+ }
print "@cats\n";
}
diff --git a/word-aligner/support/make_lex_grammar.pl b/word-aligner/support/make_lex_grammar.pl
index 2cc1be52..0d034abc 100755
--- a/word-aligner/support/make_lex_grammar.pl
+++ b/word-aligner/support/make_lex_grammar.pl
@@ -147,7 +147,7 @@ while(<EF>) {
}
}
-print STDERR "Loading Giza output...\n";
+#print STDERR "Loading Giza output...\n";
my %model4;
#while(<M4>) {
# my $en = <M4>; chomp $en;