From 49fb41843a2ad81e3ef2b65e5b9264809aac1847 Mon Sep 17 00:00:00 2001 From: redpony Date: Sun, 14 Nov 2010 23:28:04 +0000 Subject: few fixes git-svn-id: https://ws10smt.googlecode.com/svn/trunk@722 ec762483-ff6d-05da-a07a-a48fb63a330f --- word-aligner/support/classify.pl | 12 ++++++++++-- word-aligner/support/make_lex_grammar.pl | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'word-aligner/support') diff --git a/word-aligner/support/classify.pl b/word-aligner/support/classify.pl index 893c7b22..e9e79063 100755 --- a/word-aligner/support/classify.pl +++ b/word-aligner/support/classify.pl @@ -1,7 +1,10 @@ #!/usr/bin/perl -w use strict; -die "Usage: $0 classes.txt corpus.txt" unless scalar @ARGV == 2; +die "Usage: $0 [-unk] classes.txt corpus.txt\n Applies a vocabulary map to a corpus\n" unless scalar @ARGV == 2 || (scalar @ARGV == 3 && $ARGV[0] eq '-unk'); + +my $unk = $ARGV[0] eq '-unk'; +shift @ARGV if $unk; my ($class, $text) = @ARGV; open C, "<$class" or die "Can't read $class: $!"; @@ -19,9 +22,14 @@ while() { close C; print STDERR "Loaded classes for $cc words\n"; +my @cats; while() { chomp; - my @cats = map { $dict{$_} or die "Undefined class for $_"; } split /\s+/; + if ($unk) { + @cats = map { $dict{$_} or "UNK" } split /\s+/; + } else { + @cats = map { $dict{$_} or die "Undefined class for $_"; } split /\s+/; + } print "@cats\n"; } diff --git a/word-aligner/support/make_lex_grammar.pl b/word-aligner/support/make_lex_grammar.pl index 2cc1be52..0d034abc 100755 --- a/word-aligner/support/make_lex_grammar.pl +++ b/word-aligner/support/make_lex_grammar.pl @@ -147,7 +147,7 @@ while() { } } -print STDERR "Loading Giza output...\n"; +#print STDERR "Loading Giza output...\n"; my %model4; #while() { # my $en = ; chomp $en; -- cgit v1.2.3