From bae5fe99037ae7e101953ad0df118127191c711c Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Tue, 15 Jan 2013 01:20:00 -0500
Subject: corpus files

---
 corpus/paste-files.pl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'corpus/paste-files.pl')

diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 24c70599..0b788386 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -17,6 +17,7 @@ for my $file (@ARGV) {
 binmode(STDOUT,":utf8");
 binmode(STDERR,":utf8");
 
+my $bad = 0;
 my $lc = 0;
 my $done = 0;
 my $fl = 0;
@@ -34,7 +35,15 @@ while(1) {
       last;
     }
     chomp $r;
-    die "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/;
+    if ($r =~ /\|\|\|/) {
+      $r = '';
+      $bad++;
+    }
+    warn "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/;
+    $r =~ s/\|\|\|/ /g;
+    $r =~ s/  +//g;
+    $r =~ s/^ //;
+    $r =~ s/ $//;
     $anum++;
     push @line, $r;
   }
@@ -47,4 +56,5 @@ for (my $i = 1; $i < scalar @fhs; $i++) {
   my $r = <$fh>;
   die "Mismatched number of lines.\n" if defined $r;
 }
+print STDERR "Bad lines containing ||| were $bad\n";
 
-- 
cgit v1.2.3


From e1f71a6ce868d116f04082b697a8d61afcd625f1 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Mon, 21 Jan 2013 16:53:05 -0500
Subject: a little bit of cleanup

---
 corpus/filter-length.pl     | 4 ++--
 corpus/paste-files.pl       | 8 ++++----
 corpus/support/tokenizer.pl | 1 -
 3 files changed, 6 insertions(+), 7 deletions(-)

(limited to 'corpus/paste-files.pl')

diff --git a/corpus/filter-length.pl b/corpus/filter-length.pl
index 38837f14..2e257cda 100755
--- a/corpus/filter-length.pl
+++ b/corpus/filter-length.pl
@@ -44,11 +44,11 @@ while(<F>) {
   $lines++;
   if ($lines % 100000 == 0) { print STDERR " [$lines]\n"; }
   elsif ($lines % 2500 == 0) { print STDERR "."; }
-  my ($sf, $se, @d) = split / \|\|\| /;
+  my ($sf, $se, @d) = split /\s*\|\|\|\s*/;
   if (scalar @d != 0 or !defined $se) {
     $bad_format++;
     if ($bad_format > 100 && ($bad_format / $lines) > 0.02) {
-      die "Corpus appears to be incorretly formatted, example: $_";
+      die "$bad_format / $lines : Corpus appears to be incorretly formatted, example: $_";
     }
     next;
   }
diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 0b788386..4cb424ad 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -41,9 +41,9 @@ while(1) {
     }
     warn "$ARGV[$anum]:$lc contains a ||| symbol - please remove.\n" if $r =~ /\|\|\|/;
     $r =~ s/\|\|\|/ /g;
-    $r =~ s/  +//g;
-    $r =~ s/^ //;
-    $r =~ s/ $//;
+    $r =~ s/\s+/ /g;
+    $r =~ s/^ +//;
+    $r =~ s/ +$//;
     $anum++;
     push @line, $r;
   }
@@ -56,5 +56,5 @@ for (my $i = 1; $i < scalar @fhs; $i++) {
   my $r = <$fh>;
   die "Mismatched number of lines.\n" if defined $r;
 }
-print STDERR "Bad lines containing ||| were $bad\n";
+print STDERR "Number of lines containing ||| was: $bad\n" if $bad > 0;
 
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index e9c3a37d..b5190858 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -147,7 +147,6 @@ while(<STDIN>){
 
     print STDOUT " $new_line\n";
 }
-print STDERR "\n";
 
 ########################################################################
    
-- 
cgit v1.2.3


From 3a162d28033d1b9d5241e31f32978dba4eba6296 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Fri, 8 Mar 2013 22:44:49 -0500
Subject: few preproc fixes

---
 corpus/paste-files.pl        | 1 +
 corpus/support/quote-norm.pl | 2 ++
 corpus/support/token_list    | 2 ++
 3 files changed, 5 insertions(+)

(limited to 'corpus/paste-files.pl')

diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 4cb424ad..ef2cd937 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -34,6 +34,7 @@ while(1) {
       $done = 1;
       last;
     }
+    $r =~ s/\r//g;
     chomp $r;
     if ($r =~ /\|\|\|/) {
       $r = '';
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index d2980092..b104e73c 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -11,6 +11,8 @@ while(<STDIN>) {
   s/&\s*squot\s*;/'/gi;
   s/&\s*quot\s*;/"/gi;
   s/&\s*amp\s*;/&/gi;
+  s/&\s*nbsp\s*;/&/gi;
+  s/&\s*#\s*160\s*;/ /gi;
   s/ (\d\d): (\d\d)/ $1:$2/g;
   s/[\x{20a0}]\x{20ac}]/ EUR /g;
   s/[\x{00A3}]/ GBP /g;
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d470cb22..366cd7ff 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -37,6 +37,8 @@ tel.
 10.
 
 ##################### abbreviation: words that contain period.
+EE.UU.
+ee.uu.
 U.A.E
 Ala.
 Ph.D.
-- 
cgit v1.2.3