Adding morphology-segmentation stuff. Changes include: local-gi-pipeline (--morf arg), eval-pipeline (--oov-grammar, --lmorder)

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@382 ec762483-ff6d-05da-a07a-a48fb63a330f

1 files changed, 12 insertions, 4 deletions

diff --git a/gi/pipeline/local-gi-pipeline.pl b/gi/pipeline/local-gi-pipeline.pl
index 13a2b421..e31167a2 100755
--- a/gi/pipeline/local-gi-pipeline.pl
+++ b/gi/pipeline/local-gi-pipeline.pl
@@ -20,7 +20,7 @@ my $CONTEXT_SIZE = 1;
 my $BIDIR = 0;
 my $TOPICS_CONFIG = "pyp-topics.conf";
 my $LANGUAGE = "target";
-my $LABEL_THRESHOLD = 0;
+my $LABEL_THRESHOLD = "0";
 my $PRESERVE_PHRASES;
 
 my $MODEL = "pyp";
@@ -28,6 +28,7 @@ my $NUM_ITERS = 100;
 my $PR_SCALE_P = 0;
 my $PR_SCALE_C = 0;
 my $PR_FLAGS = "";
+my $MORFMARK = "";
 
 my $EXTOOLS = "$SCRIPT_DIR/../../extools";
 die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS;
@@ -50,9 +51,10 @@ my $REMOVE_TAGS_CORPUS = "$SCRIPT_DIR/scripts/remove-tags-from-corpus.pl";
 my $REMOVE_TAGS_CONTEXT = "$SCRIPT_DIR/scripts/remove-tags-from-contexts.pl";
 my $EXTRACTOR = "$EXTOOLS/extractor";
 my $TOPIC_TRAIN = "$PYPTOOLS/pyp-contexts-train";
+my $MORF_DOC_FILTER = "$SCRIPT_DIR/../morf-segmentation/filter_docs.pl";
 
 assert_exec($PATCH_CORPUS, $SORT_KEYS, $REDUCER, $EXTRACTOR,
-            $S2L, $C2D, $TOPIC_TRAIN, $SPLIT, $REMOVE_TAGS_CONTEXT, $REMOVE_TAGS_CORPUS);
+            $S2L, $C2D, $TOPIC_TRAIN, $SPLIT, $REMOVE_TAGS_CONTEXT, $REMOVE_TAGS_CORPUS, $MORF_DOC_FILTER);
 
 my $BACKOFF_GRAMMAR;
 my $DEFAULT_CAT;
@@ -82,6 +84,7 @@ usage() unless &GetOptions('base_phrase_max_size=i' => \$BASE_PHRASE_MAX_SIZE,
                            'language=s' => \$LANGUAGE,
                            'get_name_only' => \$NAME_SHORTCUT,
                            'preserve_phrases' => \$PRESERVE_PHRASES,
+                           'morf=s' => \$MORFMARK,
                           );
 if ($NAME_SHORTCUT) {
   $NUM_TOPICS = $NUM_TOPICS_FINE;
@@ -216,7 +219,7 @@ sub cluster_dir {
 }
 
 sub labeled_dir {
-  if (lc($MODEL) eq "pyp" && $LABEL_THRESHOLD != 0) {
+  if (lc($MODEL) eq "pyp" && $LABEL_THRESHOLD ne "0") {
     return cluster_dir() . "_lt$LABEL_THRESHOLD";
   } else {
     return cluster_dir();
@@ -272,8 +275,13 @@ sub extract_context {
      $ccopt = "-c 0";
      $postsort = "" unless ($PRESERVE_PHRASES);
    }
+
    my $presort = ($PRESERVE_PHRASES ? "| $REMOVE_TAGS_CONTEXT --phrase=tok --context=tag " : "");
 
+   if ($MORFMARK ne "") { 
+     $presort = $presort . "| $MORF_DOC_FILTER \"$MORFMARK\" "; 
+   }
+
    my $cmd = "$EXTRACTOR -i $CORPUS_CLUSTER $ccopt -L $BASE_PHRASE_MAX_SIZE -C -S $CONTEXT_SIZE --phrase_language $LANGUAGE --context_language $LANGUAGE $presort | $SORT_KEYS $postsort | $GZIP > $OUT_CONTEXTS";
    safesystem($cmd) or die "Failed to extract contexts.";
   }
@@ -351,7 +359,7 @@ sub label_spans_with_topics {
     safesystem("$ZCAT $IN_CLUSTERS > $CLUSTER_DIR/clusters.txt") or die "Failed to unzip";
     safesystem("$EXTRACTOR --base_phrase_spans -i $CORPUS_CLUSTER -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -S $CONTEXT_SIZE | $S2L $CLUSTER_DIR/clusters.txt $CONTEXT_SIZE $LABEL_THRESHOLD $extra > $OUT_SPANS") or die "Failed to label spans";
     unlink("$CLUSTER_DIR/clusters.txt") or warn "Failed to remove $CLUSTER_DIR/clusters.txt";
-    safesystem("paste -d ' ' $CORPUS_LEX $OUT_SPANS > $LABELED_DIR/corpus.src_trg_al_label") or die "Couldn't paste";
+    safesystem("paste -d ' ' $CORPUS_LEX $OUT_SPANS | sed 's/ *||| *\$//'  > $LABELED_DIR/corpus.src_trg_al_label") or die "Couldn't paste";
   }
 }