From 9e73380e46ada871ad114f4fd4238b8f2263ef27 Mon Sep 17 00:00:00 2001 From: redpony Date: Thu, 28 Oct 2010 21:10:12 +0000 Subject: psg for lex trans git-svn-id: https://ws10smt.googlecode.com/svn/trunk@699 ec762483-ff6d-05da-a07a-a48fb63a330f --- .../support/generate_per_sentence_grammars.pl | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'word-aligner/support') diff --git a/word-aligner/support/generate_per_sentence_grammars.pl b/word-aligner/support/generate_per_sentence_grammars.pl index 80243419..8779ac9c 100755 --- a/word-aligner/support/generate_per_sentence_grammars.pl +++ b/word-aligner/support/generate_per_sentence_grammars.pl @@ -2,7 +2,7 @@ use strict; use utf8; -die "Usage: $0 f.voc corpus.f-e grammar.f-e.gz\n" unless scalar @ARGV == 3; +die "Usage: $0 f.voc corpus.f-e grammar.f-e.gz [OUT]filtered.f-e.gz [OUT]per_sentence_grammar.f-e [OUT]train.f-e.sgml\n" unless scalar @ARGV == 6; my $MAX_INMEM = 2500; @@ -10,7 +10,14 @@ open FV,"<$ARGV[0]" or die "Can't read $ARGV[0]: $!"; open C,"<$ARGV[1]" or die "Can't read $ARGV[1]: $!"; open G,"gunzip -c $ARGV[2]|" or die "Can't read $ARGV[2]: $!"; +open FILT,"|gzip -c > $ARGV[3]" or die "Can't write $ARGV[3]: $!"; +open PSG,">$ARGV[4]" or die "Can't write $ARGV[4]: $!"; +open OTRAIN,">$ARGV[5]" or die "Can't write $ARGV[5]: $!"; + +binmode FILT, ":utf8"; +binmode PSG, ":utf8"; binmode STDOUT, ":utf8"; +binmode STDERR, ":utf8"; binmode FV, ":utf8"; binmode C, ":utf8"; binmode G, ":utf8"; @@ -35,7 +42,7 @@ while() { chomp; my ($f, $e, $feats) = split / \|\|\| /; if ($most_freq{$f}) { - print "$_\n"; + print FILT "$_\n"; $memrc++; } else { $loadrc++; @@ -47,20 +54,19 @@ while() { push @$r, "$e ||| $feats"; } } +close FILT; close G; print STDERR " mem rc: $memrc\n"; print STDERR " load rc: $loadrc\n"; my $id = 0; -open O, ">ps.grammar" or die; -binmode(O,":utf8"); while() { chomp; my ($f,$e) = split / \|\|\| /; my @fwords = split /\s+/, $f; my $tot = 0; my %used; - my $fpos = tell(O); + my $fpos = tell(PSG); for my $f (@fwords) { next if $most_freq{$f}; next if $used{$f}; @@ -69,15 +75,15 @@ while() { my $num = scalar @$r; $tot += $num; for my $rule (@$r) { - print O "$f ||| $rule\n"; + print PSG "$f ||| $rule\n"; } $used{$f} = 1; } - print O "###EOS###\n"; - print STDERR " $_ \n"; - #print STDERR "id=$id POS=$fpos\n"; + print PSG "###EOS###\n"; + print OTRAIN " $_ \n"; $id++; - last if $id == 10; } +close PSG; +close OTRAIN; +print STDERR "Done.\n"; -close O; -- cgit v1.2.3