Adding morphology-segmentation stuff. Changes include: local-gi-pipeline (--morf arg), eval-pipeline (--oov-grammar, --lmorder)

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@382 ec762483-ff6d-05da-a07a-a48fb63a330f

2 files changed, 19 insertions, 7 deletions

diff --git a/gi/pipeline/evaluation-pipeline.pl b/gi/pipeline/evaluation-pipeline.pl
index 13fe07cf..e940a5b9 100755
--- a/gi/pipeline/evaluation-pipeline.pl
+++ b/gi/pipeline/evaluation-pipeline.pl
@@ -123,16 +123,20 @@ my $dataDir = '/export/ws10smt/data';
 my @features;
 my $bkoffgram;
 my $gluegram;
+my $oovgram;
 my $usefork;
+my $lmorder = 3;
 if (GetOptions(
         "backoff-grammar=s" => \$bkoffgram,
         "glue-grammar=s" => \$gluegram,
+        "oov-grammar=s" => \$oovgram,
         "data=s" => \$dataDir,
         "pmem=s" => \$PMEM,
         "features=s@" => \@features,
         "use-fork" => \$usefork,
         "jobs=i" => \$JOBS,
         "out-dir=s" => \$outdir,
+        "lmorder=i" => \$lmorder,
 ) == 0 || @ARGV!=2 || $help) {
         print_help();
         exit;
@@ -214,7 +218,6 @@ my $testini = mydircat($outdir, "cdec-test.ini");
 write_cdec_ini($testini, $testgrammar);
 
 
-
 # VEST
 print STDERR "\nMINIMUM ERROR TRAINING\n";
 my $tuned_weights = mydircat($outdir, 'weights.tuned');
@@ -294,17 +297,18 @@ sub write_cdec_ini {
   my ($filename, $grammar_path) = (@_);
   open CDECINI, ">$filename" or die "Can't write $filename: $!";
   my $glue = ($gluegram ? "$glue_grmr" : "$datadir/glue/glue.scfg.gz");
+  my $oov = ($oovgram ? "$oovgram" : "$datadir/oov.scfg.gz");
   print CDECINI <<EOT;
 formalism=scfg
 cubepruning_pop_limit=100
 add_pass_through_rules=true
 scfg_extra_glue_grammar=$glue
-grammar=$datadir/oov.scfg.gz
+grammar=$oov
 grammar=$grammar_path
 scfg_default_nt=OOV
 scfg_no_hiero_glue_grammar=true
 feature_function=WordPenalty
-feature_function=LanguageModel -o 3 $LANG_MODEL
+feature_function=LanguageModel -o $lmorder $LANG_MODEL
 EOT
   print CDECINI "grammar=$bkoff_grmr\n" if $bkoffgram;
   close CDECINI;
diff --git a/gi/pipeline/local-gi-pipeline.pl b/gi/pipeline/local-gi-pipeline.pl
index 13a2b421..e31167a2 100755
--- a/gi/pipeline/local-gi-pipeline.pl
+++ b/gi/pipeline/local-gi-pipeline.pl
@@ -20,7 +20,7 @@ my $CONTEXT_SIZE = 1;
 my $BIDIR = 0;
 my $TOPICS_CONFIG = "pyp-topics.conf";
 my $LANGUAGE = "target";
-my $LABEL_THRESHOLD = 0;
+my $LABEL_THRESHOLD = "0";
 my $PRESERVE_PHRASES;
 
 my $MODEL = "pyp";
@@ -28,6 +28,7 @@ my $NUM_ITERS = 100;
 my $PR_SCALE_P = 0;
 my $PR_SCALE_C = 0;
 my $PR_FLAGS = "";
+my $MORFMARK = "";
 
 my $EXTOOLS = "$SCRIPT_DIR/../../extools";
 die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS;
@@ -50,9 +51,10 @@ my $REMOVE_TAGS_CORPUS = "$SCRIPT_DIR/scripts/remove-tags-from-corpus.pl";
 my $REMOVE_TAGS_CONTEXT = "$SCRIPT_DIR/scripts/remove-tags-from-contexts.pl";
 my $EXTRACTOR = "$EXTOOLS/extractor";
 my $TOPIC_TRAIN = "$PYPTOOLS/pyp-contexts-train";
+my $MORF_DOC_FILTER = "$SCRIPT_DIR/../morf-segmentation/filter_docs.pl";
 
 assert_exec($PATCH_CORPUS, $SORT_KEYS, $REDUCER, $EXTRACTOR,
-            $S2L, $C2D, $TOPIC_TRAIN, $SPLIT, $REMOVE_TAGS_CONTEXT, $REMOVE_TAGS_CORPUS);
+            $S2L, $C2D, $TOPIC_TRAIN, $SPLIT, $REMOVE_TAGS_CONTEXT, $REMOVE_TAGS_CORPUS, $MORF_DOC_FILTER);
 
 my $BACKOFF_GRAMMAR;
 my $DEFAULT_CAT;
@@ -82,6 +84,7 @@ usage() unless &GetOptions('base_phrase_max_size=i' => \$BASE_PHRASE_MAX_SIZE,
                            'language=s' => \$LANGUAGE,
                            'get_name_only' => \$NAME_SHORTCUT,
                            'preserve_phrases' => \$PRESERVE_PHRASES,
+                           'morf=s' => \$MORFMARK,
                           );
 if ($NAME_SHORTCUT) {
   $NUM_TOPICS = $NUM_TOPICS_FINE;
@@ -216,7 +219,7 @@ sub cluster_dir {
 }
 
 sub labeled_dir {
-  if (lc($MODEL) eq "pyp" && $LABEL_THRESHOLD != 0) {
+  if (lc($MODEL) eq "pyp" && $LABEL_THRESHOLD ne "0") {
     return cluster_dir() . "_lt$LABEL_THRESHOLD";
   } else {
     return cluster_dir();
@@ -272,8 +275,13 @@ sub extract_context {
      $ccopt = "-c 0";
      $postsort = "" unless ($PRESERVE_PHRASES);
    }
+
    my $presort = ($PRESERVE_PHRASES ? "| $REMOVE_TAGS_CONTEXT --phrase=tok --context=tag " : "");
 
+   if ($MORFMARK ne "") { 
+     $presort = $presort . "| $MORF_DOC_FILTER \"$MORFMARK\" "; 
+   }
+
    my $cmd = "$EXTRACTOR -i $CORPUS_CLUSTER $ccopt -L $BASE_PHRASE_MAX_SIZE -C -S $CONTEXT_SIZE --phrase_language $LANGUAGE --context_language $LANGUAGE $presort | $SORT_KEYS $postsort | $GZIP > $OUT_CONTEXTS";
    safesystem($cmd) or die "Failed to extract contexts.";
   }
@@ -351,7 +359,7 @@ sub label_spans_with_topics {
     safesystem("$ZCAT $IN_CLUSTERS > $CLUSTER_DIR/clusters.txt") or die "Failed to unzip";
     safesystem("$EXTRACTOR --base_phrase_spans -i $CORPUS_CLUSTER -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -S $CONTEXT_SIZE | $S2L $CLUSTER_DIR/clusters.txt $CONTEXT_SIZE $LABEL_THRESHOLD $extra > $OUT_SPANS") or die "Failed to label spans";
     unlink("$CLUSTER_DIR/clusters.txt") or warn "Failed to remove $CLUSTER_DIR/clusters.txt";
-    safesystem("paste -d ' ' $CORPUS_LEX $OUT_SPANS > $LABELED_DIR/corpus.src_trg_al_label") or die "Couldn't paste";
+    safesystem("paste -d ' ' $CORPUS_LEX $OUT_SPANS | sed 's/ *||| *\$//'  > $LABELED_DIR/corpus.src_trg_al_label") or die "Couldn't paste";
   }
 }