diff options
| author | Chris Dyer <cdyer@allegro.clab.cs.cmu.edu> | 2013-01-21 16:53:05 -0500 |
|---|---|---|
| committer | Chris Dyer <cdyer@allegro.clab.cs.cmu.edu> | 2013-01-21 16:53:05 -0500 |
| commit | e1f71a6ce868d116f04082b697a8d61afcd625f1 (patch) | |
| tree | 0b026f8f44b5193ff6d9561990e6eada19f7e863 /corpus | |
| parent | 7cac43b858f3b681555bf0578f54b1f822c43207 (diff) | |
a little bit of cleanup
Diffstat (limited to 'corpus')
| -rwxr-xr-x | corpus/filter-length.pl | 4 | ||||
| -rwxr-xr-x | corpus/paste-files.pl | 8 | ||||
| -rwxr-xr-x | corpus/support/tokenizer.pl | 1 |
3 files changed, 6 insertions, 7 deletions
diff --git a/corpus/filter-length.pl b/corpus/filter-length.pl index 38837f14..2e257cda 100755 --- a/corpus/filter-length.pl +++ b/corpus/filter-length.pl @@ -44,11 +44,11 @@ while(<F>) { $lines++; if ($lines % 100000 == 0) { print STDERR " [$lines]\n"; } elsif ($lines % 2500 == 0) { print STDERR "."; } - my ($sf, $se, @d) = split / \|\|\| /; + my ($sf, $se, @d) = split /\s*\|\|\|\s*/; if (scalar @d != 0 or !defined $se) { $bad_format++; if ($bad_format > 100 && ($bad_format / $lines) > 0.02) { - die "Corpus appears to be incorretly formatted, example: $_"; + die "$bad_format / $lines : Corpus appears to be incorretly formatted, example: $_"; } next; } diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl index 0b788386..4cb424ad 100755 --- a/corpus/paste-files.pl +++ b/corpus/paste-files.pl @@ -41,9 +41,9 @@ while(1) { } warn "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/; $r =~ s/\|\|\|/ /g; - $r =~ s/ +//g; - $r =~ s/^ //; - $r =~ s/ $//; + $r =~ s/\s+/ /g; + $r =~ s/^ +//; + $r =~ s/ +$//; $anum++; push @line, $r; } @@ -56,5 +56,5 @@ for (my $i = 1; $i < scalar @fhs; $i++) { my $r = <$fh>; die "Mismatched number of lines.\n" if defined $r; } -print STDERR "Bad lines containing ||| were $bad\n"; +print STDERR "Number of lines containing ||| was: $bad\n" if $bad > 0; diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl index e9c3a37d..b5190858 100755 --- a/corpus/support/tokenizer.pl +++ b/corpus/support/tokenizer.pl @@ -147,7 +147,6 @@ while(<STDIN>){ print STDOUT " $new_line\n"; } -print STDERR "\n"; ######################################################################## |