From bae5fe99037ae7e101953ad0df118127191c711c Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Tue, 15 Jan 2013 01:20:00 -0500
Subject: corpus files

---
 corpus/paste-files.pl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'corpus/paste-files.pl')

diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 24c70599..0b788386 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -17,6 +17,7 @@ for my $file (@ARGV) {
 binmode(STDOUT,":utf8");
 binmode(STDERR,":utf8");
 
+my $bad = 0;
 my $lc = 0;
 my $done = 0;
 my $fl = 0;
@@ -34,7 +35,15 @@ while(1) {
       last;
     }
     chomp $r;
-    die "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/;
+    if ($r =~ /\|\|\|/) {
+      $r = '';
+      $bad++;
+    }
+    warn "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/;
+    $r =~ s/\|\|\|/ /g;
+    $r =~ s/  +//g;
+    $r =~ s/^ //;
+    $r =~ s/ $//;
     $anum++;
     push @line, $r;
   }
@@ -47,4 +56,5 @@ for (my $i = 1; $i < scalar @fhs; $i++) {
   my $r = <$fh>;
   die "Mismatched number of lines.\n" if defined $r;
 }
+print STDERR "Bad lines containing ||| were $bad\n";
 
-- 
cgit v1.2.3


From e1f71a6ce868d116f04082b697a8d61afcd625f1 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Mon, 21 Jan 2013 16:53:05 -0500
Subject: a little bit of cleanup

---
 corpus/filter-length.pl     | 4 ++--
 corpus/paste-files.pl       | 8 ++++----
 corpus/support/tokenizer.pl | 1 -
 3 files changed, 6 insertions(+), 7 deletions(-)

(limited to 'corpus/paste-files.pl')

diff --git a/corpus/filter-length.pl b/corpus/filter-length.pl
index 38837f14..2e257cda 100755
--- a/corpus/filter-length.pl
+++ b/corpus/filter-length.pl
@@ -44,11 +44,11 @@ while(<F>) {
   $lines++;
   if ($lines % 100000 == 0) { print STDERR " [$lines]\n"; }
   elsif ($lines % 2500 == 0) { print STDERR "."; }
-  my ($sf, $se, @d) = split / \|\|\| /;
+  my ($sf, $se, @d) = split /\s*\|\|\|\s*/;
   if (scalar @d != 0 or !defined $se) {
     $bad_format++;
     if ($bad_format > 100 && ($bad_format / $lines) > 0.02) {
-      die "Corpus appears to be incorretly formatted, example: $_";
+      die "$bad_format / $lines : Corpus appears to be incorretly formatted, example: $_";
     }
     next;
   }
diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 0b788386..4cb424ad 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -41,9 +41,9 @@ while(1) {
     }
     warn "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/;
     $r =~ s/\|\|\|/ /g;
-    $r =~ s/  +//g;
-    $r =~ s/^ //;
-    $r =~ s/ $//;
+    $r =~ s/\s+/ /g;
+    $r =~ s/^ +//;
+    $r =~ s/ +$//;
     $anum++;
     push @line, $r;
   }
@@ -56,5 +56,5 @@ for (my $i = 1; $i < scalar @fhs; $i++) {
   my $r = <$fh>;
   die "Mismatched number of lines.\n" if defined $r;
 }
-print STDERR "Bad lines containing ||| were $bad\n";
+print STDERR "Number of lines containing ||| was: $bad\n" if $bad > 0;
 
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index e9c3a37d..b5190858 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -147,7 +147,6 @@ while(<STDIN>){
 
     print STDOUT " $new_line\n";
 }
-print STDERR "\n";
 
 ########################################################################
    
-- 
cgit v1.2.3