add case preservation to compound splitter

1 files changed, 19 insertions, 0 deletions

diff --git a/compound-split/compound-split.pl b/compound-split/compound-split.pl
index 7f45d345..0f359504 100755
--- a/compound-split/compound-split.pl
+++ b/compound-split/compound-split.pl
@@ -12,12 +12,14 @@ my $BEAM = 2.1;
 my $OUTPUT = 'plf';
 my $HELP;
 my $VERBOSE;
+my $PRESERVE_CASE;
 
 GetOptions("decoder=s" => \$CDEC,
            "language=s" => \$LANG,
            "beam=f" => \$BEAM,
            "output=s" => \$OUTPUT,
            "verbose" => \$VERBOSE,
+           "preserve_case" => \$PRESERVE_CASE,
            "help" => \$HELP
           ) or usage();
 
@@ -60,15 +62,18 @@ while(<STDIN>) {
   my @words = split /\s+/;
   my @res = ();
   my @todo = ();
+  my @casings = ();
   for (my $i=0; $i < scalar @words; $i++) {
     my $word = lc $words[$i];
     if (length($word)<6 || $word =~ /^[,\-0-9\.]+$/ || $word =~ /[@.\-\/:]/) {
+      push @casings, 0;
       if ($IS_PLF) {
         push @res, "(('" . escape($word) . "',0,1),),";
       } else {
         push @res, $word;
       }
     } else {
+      push @casings, guess_casing($words[$i]);
       push @res, undef;
       push @todo, $word;
     }
@@ -84,6 +89,7 @@ while(<STDIN>) {
         unless ($IS_PLF) {
           $seg =~ s/^# //o;
         }
+        if ($PRESERVE_CASE && $casings[$i]) { $seg = recase_words($seg); }
         $res[$i] = $seg;
       }
     }
@@ -100,6 +106,12 @@ while(<STDIN>) {
 close IN;
 close OUT;
 
+sub recase_words {
+  my $word = shift;
+  $word =~ s/\b(\w)/\u$1/g;
+  return $word;
+}
+
 sub escape {
   $_ = shift;
   s/\\/\\\\/g;
@@ -107,6 +119,11 @@ sub escape {
   return $_;
 }
 
+sub guess_casing {
+  my $word = shift @_;
+  if (lc($word) eq $word) { return 0; } else { return 1; }
+}
+
 sub usage {
   print <<EOT;
 
@@ -118,6 +135,8 @@ Usage: $0 [OPTIONS] < file.txt
     --beam NUM            Beam threshold, used with PLF output
                             (probably between 1.5 and 5.0)
     --output plf|1best    Output format, 1best or plf (lattice)
+    --preserve_case       Preserve the casing of the input word
+                            (model will be scored lowercase)
     --verbose             Show verbose decoder output
 
 EOT