diff options
Diffstat (limited to 'gi/morf-segmentation/morf-pipeline.pl')
| -rwxr-xr-x | gi/morf-segmentation/morf-pipeline.pl | 466 |
1 files changed, 466 insertions, 0 deletions
diff --git a/gi/morf-segmentation/morf-pipeline.pl b/gi/morf-segmentation/morf-pipeline.pl new file mode 100755 index 00000000..da40eb57 --- /dev/null +++ b/gi/morf-segmentation/morf-pipeline.pl @@ -0,0 +1,466 @@ +#!/usr/bin/perl -w +use strict; +use File::Copy; + +#WARNING.. THIS SCRIPT IS CURRENTLY SOMEWHAT BROKEN. AND UGLY. + +my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path cwd /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } + +use Getopt::Long "GetOptions"; + +my $GZIP = 'gzip'; +my $ZCAT = 'gunzip -c'; +my $SED = 'sed -e'; + +my $MORF_TRAIN = "$SCRIPT_DIR/morftrain.sh"; +my $MORF_SEGMENT = "$SCRIPT_DIR/morfsegment.py"; + +my $LINESTRIPPER = "$SCRIPT_DIR/linestripper.py"; +my $ALIGNER = "/export/ws10smt/software/berkeleyaligner/berkeleyaligner.jar"; +#java -d64 -Xmx10g -jar $ALIGNER ++word-align.conf >> aligner.log +assert_exec($MORF_TRAIN, $LINESTRIPPER, $MORF_SEGMENT, $ALIGNER); + +my $OUTPUT = './morfwork'; +my $PPL_SRC = 50; +my $PPL_TRG = 50; +my $MARKER = "#"; +my $MAX_WORDS = 40; +my $SENTENCES;# = 100000; +my $SPLIT_TYPE = ""; +my $NAME_SHORTCUT; + +usage() unless &GetOptions('max_words=i' => \$MAX_WORDS, + 'output=s' => \$OUTPUT, + 'ppl_src=i' => \$PPL_SRC, + 'ppl_trg=i' => \$PPL_TRG, + 'sentences=i' => \$SENTENCES, + 'marker=s' => \$MARKER, + 'split=s' => \$SPLIT_TYPE, + 'get_name_only' => \$NAME_SHORTCUT, + ); +#if ($NAME_SHORTCUT) { +# print STDERR labeled_dir(); +# exit 0; +#} + +usage() unless scalar @ARGV >= 2; + +my %CORPUS; # for (src,trg) it has (orig, name, filtered, final) + +$CORPUS{'src'}{'orig'} = $ARGV[0]; +open F, "<$CORPUS{'src'}{'orig'}" or die "Can't read $CORPUS{'src'}{'orig'}: $!"; close F; +$CORPUS{'src'}{'name'} = get_basename($CORPUS{'src'}{'orig'}); + +$CORPUS{'trg'}{'orig'} = $ARGV[1]; +open F, "<$CORPUS{'trg'}{'orig'}" or die "Can't read $CORPUS{'trg'}{'orig'}: $!"; close F; +$CORPUS{'trg'}{'name'} = get_basename($CORPUS{'trg'}{'orig'}); + + +my %DEV; # for (src,trg) has (orig, final.split final.unsplit + + +#my %TEST_SRC; #original, final + #trg has original, final.split final.recombined + +my $TEST_SRC; +my $TEST_TRG; + +my $TEST_SRC_name; +my $TEST_TRG_name; + +if (@ARGV >= 4) { + $DEV{'src'}{'orig'} = $ARGV[2]; + open F, "<$DEV{'src'}{'orig'}" or die "Can't read $DEV{'src'}{'orig'}: $!"; close F; + $DEV{'src'}{'name'} = get_basename($DEV{'src'}{'orig'}); + $DEV{'trg'}{'orig'} = $ARGV[3]; + open F, "<$DEV{'trg'}{'orig'}" or die "Can't read $DEV{'trg'}{'orig'}: $!"; close F; + $DEV{'trg'}{'name'} = get_basename($DEV{'trg'}{'orig'}); +} +if (@ARGV >= 6) { + $TEST_SRC = $ARGV[4]; + open F, "<$TEST_SRC" or die "Can't read $TEST_SRC: $!"; close F; + $TEST_SRC_name = get_basename($TEST_SRC); + $TEST_TRG = $ARGV[5]; + open F, "<$TEST_TRG" or die "Can't read $TEST_TRG: $!"; close F; + $TEST_TRG_name = get_basename($TEST_TRG); +} + +my $SPLIT_SRC; #use these to check whether that part is being split +my $SPLIT_TRG; + +#OUTPUT WILL GO IN THESE +my $CORPUS_DIR = $OUTPUT . '/' . corpus_dir(); #subsampled corpus +my $MODEL_SRC_DIR = $OUTPUT . '/' . model_dir("src"); #splitting.. +my $MODEL_TRG_DIR = $OUTPUT . '/' . model_dir("trg"); # .. models +my $PROCESSED_DIR = $OUTPUT . '/' . processed_dir(); #segmented copora+alignments +my $ALIGNMENT_DIR = $PROCESSED_DIR . '/alignments'; + +$CORPUS{'src'}{'filtered'} = $CORPUS_DIR . "/$CORPUS{'src'}{'name'}"; +$CORPUS{'trg'}{'filtered'} = $CORPUS_DIR . "/$CORPUS{'trg'}{'name'}"; + +print STDERR "Output: $OUTPUT\n"; +print STDERR "Corpus: $CORPUS_DIR\n"; +print STDERR "Model-src: $MODEL_SRC_DIR\n"; +print STDERR "Model-trg: $MODEL_TRG_DIR\n"; +print STDERR "Finaldir: $PROCESSED_DIR\n"; + +safemkdir($OUTPUT) or die "Couldn't create output directory $OUTPUT: $!"; +safemkdir($CORPUS_DIR) or die "Couldn't create output directory $CORPUS_DIR: $!"; +filter_corpus(); + +safemkdir($PROCESSED_DIR); +safemkdir($ALIGNMENT_DIR); + +if ($SPLIT_SRC) { + safemkdir($MODEL_SRC_DIR) or die "Couldn't create output directory $MODEL_SRC_DIR: $!"; + learn_segmentation("src"); + apply_segmentation("src"); +} +if ($SPLIT_TRG) { + safemkdir($MODEL_TRG_DIR) or die "Couldn't create output directory $MODEL_TRG_DIR: $!"; + learn_segmentation("trg"); + apply_segmentation("trg"); +} + +#copy corpora if they haven't been put in place by splitting operations +if (! -f "$PROCESSED_DIR/$CORPUS{'src'}{'name'}") { + copy($CORPUS{'src'}{'filtered'}, "$PROCESSED_DIR/$CORPUS{'src'}{'name'}") or die "Copy failed: $!"; +} +if (! -f "$PROCESSED_DIR/$CORPUS{'trg'}{'name'}") { + copy($CORPUS{'trg'}{'filtered'}, "$PROCESSED_DIR/$CORPUS{'trg'}{'name'}") or die "Copy failed: $!"; +} +if ($DEV{'src'}{'orig'} && ! -f "$PROCESSED_DIR/$DEV{'src}{'name'}") { + copy( +} +if ($TEST_SRC) { ifne_copy($TEST_SRC, "$PROCESSED_DIR/$TEST_SRC_name"); } +if ($TEST_TRG) { ifne_copy("$TEST_TRG.unsplit", "$PROCESSED_DIR/$TEST_TRG_name.unsplit"); } + + + +do_align(); + +system("date"); +print STDERR "All done. You now need to train a language model (if target split), preprocess the test data and put various things where the eval scripts can find them\n\n". + +sub ifne_copy { + my ($src, $dest) = @_; + if (! -f $dest) { + copy($src, $dest) or die "Copy failed: $!"; + } +} + +} + +sub filter_corpus { + print STDERR "\n!!!FILTERING TRAINING COPRUS!!!\n"; + if ( -f $CORPUS{'src'}{'filtered'} && -f $CORPUS{'trg'}{'filtered'}) { + print STDERR "$CORPUS{'src'}{'filtered'} and $CORPUS{'trg'}{'filtered'} exist, reusing...\n"; + return; + } + my $args = "$CORPUS{'src'}{'orig'} $CORPUS{'trg'}{'orig'} $MAX_WORDS"; + if ($SENTENCES) { $args = $args . " $SENTENCES"; } + safesystem("$LINESTRIPPER $args 1> $CORPUS{'src'}{'filtered'} 2> $CORPUS{'trg'}{'filtered'}") or die "Failed to filter training corpus for length."; +} + +sub learn_segmentation +{ + my $WHICH = shift; + my $corpus; my $dev; my $test; my $moddir; my $ppl; + + if ($WHICH eq "src") { + print STDERR "\n!!!LEARNING SEGMENTATION MODEL (SOURCE LANGUAGE)!!!\n"; + $corpus = $CORPUS{'src'}{'filtered'}; + $dev = $DEV{'src'}{'orig'}; + $test = $TEST_SRC; + $moddir = $MODEL_SRC_DIR; + $ppl = $PPL_SRC; + } else { + print STDERR "\n!!!LEARNING SEGMENTATION MODEL (TARGET LANGUAGE)!!!\n"; + $corpus = $CORPUS{'trg'}{'filtered'}; + $dev = $DEV{'trg'}{'orig'}; + $test = $TEST_TRG; + $moddir = $MODEL_TRG_DIR; + $ppl = $PPL_TRG; + } + system("date"); + my $cmd = "cat $corpus"; + if ($dev) { $cmd = "$cmd $dev"; } + if ($test) { $cmd = "$cmd $test"; } + my $tmpfile = "$CORPUS_DIR/all.tmp.gz"; + safesystem("$cmd | $GZIP > $tmpfile") or die "Failed to concatenate data for model learning.."; + learn_segmentation_any($tmpfile, $moddir, $ppl); + safesystem("rm $tmpfile"); +} + +sub learn_segmentation_any { + my($INPUT_FILE, $SEGOUT_DIR, $PPL) = @_; + my $SEG_FILE = $SEGOUT_DIR . "/segmentation.ready"; + if ( -f $SEG_FILE) { + print STDERR "$SEG_FILE exists, reusing...\n"; + return; + } + my $cmd = "$MORF_TRAIN $INPUT_FILE $SEGOUT_DIR $PPL \"$MARKER\""; + safesystem($cmd) or die "Failed to learn segmentation model"; +} + +sub do_align { + print STDERR "\n!!!WORD ALIGNMENT!!!\n"; + system("date"); + + my $ALIGNMENTS = "$ALIGNMENT_DIR/training.align"; + if ( -f $ALIGNMENTS ) { + print STDERR "$ALIGNMENTS exists, reusing...\n"; + return; + } + my $conf_file = "$ALIGNMENT_DIR/word-align.conf"; + + #decorate training files with identifiers to stop the aligner from training on dev and test too + #since they are in same directory + safesystem("cd $PROCESSED_DIR && ln -s $CORPUS{'src'}{'name'} corpus.src") or die "Failed to symlink: $!"; + safesystem("cd $PROCESSED_DIR && ln -s $CORPUS{'trg'}{'name'} corpus.trg") or die "Failed to symlink: $!"; + + write_wconf($conf_file, $PROCESSED_DIR); + safesystem("java -d64 -Xmx24g -jar $ALIGNER ++$conf_file > $ALIGNMENT_DIR/aligner.log") or die "Failed to run word alignment."; + +} + +sub apply_segmentation { + my $WHICH = shift; + my $moddir; + my $datfile; + if ($WHICH eq "src") { + print STDERR "\n!!!APPLYING SEGMENTATION MODEL (SOURCE LANGUAGE)!!!\n"; + apply_segmentation_any($MODEL_SRC_DIR, $CORPUS{'src'}{'filtered'}, "$PROCESSED_DIR/$CORPUS{'src'}{'name'}"); + if ($DEV{'src'}{'orig'}) { + apply_segmentation_any($MODEL_SRC_DIR, $DEV{'src'}{'orig'}, "$PROCESSED_DIR/$DEV{'src'}{'name'}"); + } + if ($TEST_SRC) { + apply_segmentation_any($MODEL_SRC_DIR, $TEST_SRC, "$PROCESSED_DIR/$TEST_SRC_name"); + } + } else { + print STDERR "\n!!!APPLYING SEGMENTATION MODEL (TARGET LANGUAGE)!!!\n"; + apply_segmentation_any($MODEL_TRG_DIR, $CORPUS{'trg'}{'filtered'}, "$PROCESSED_DIR/$CORPUS{'trg'}{'name'}"); + if ($DEV{'trg'}{'orig'}) { + $DEV{'trg'}{'final'} = "$PROCESSED_DIR/$DEV{'trg'}{'name'}"; + apply_segmentation_any($MODEL_TRG_DIR, $DEV{'trg'}{'orig'}, $DEV{'trg'}{'final'}); + } + if ($TEST_TRG) { + apply_segmentation_any($MODEL_TRG_DIR, $TEST_TRG, "$PROCESSED_DIR/$TEST_TRG_name.split"); + copy($TEST_TRG, "$PROCESSED_DIR/$TEST_TRG_name.unsplit") or die "Could not copy unsegmented test set"; + } + } + if ($WHICH eq "src" || $WHICH eq "trg") { + write_eval_sh("$PROCESSED_DIR/eval-devtest.sh"); + } +} + +sub apply_segmentation_any { + my($moddir, $datfile, $outfile) = @_; + if ( -f $outfile) { + print STDERR "$outfile exists, reusing...\n"; + return; + } + + my $args = "$moddir/inputvocab.gz $moddir/segmentation.ready \"$MARKER\""; + safesystem("cat $datfile | $MORF_SEGMENT $args &> $outfile") or die "Could not segment $datfile"; +} + +sub beautify_numlines { + return ($SENTENCES ? $SENTENCES : "_all"); +} + +sub corpus_dir { + return "s" . beautify_numlines() . ".w" . $MAX_WORDS; +} + +sub model_dir { + my $lang = shift; + if ($lang eq "src") { + return corpus_dir() . ".PPL" . $PPL_SRC . ".src"; + } elsif ($lang eq "trg") { + return corpus_dir() . ".PPL" . $PPL_TRG . ".trg"; + } else { + return "PPLundef"; + } +} + +sub split_name { + #parses SPLIT_TYPE, which can have the following values + # t|s|ts|st (last 2 are equiv) + # or is undefined when no splitting is done + my $name = ""; + + if ($SPLIT_TYPE) { + $SPLIT_SRC = lc($SPLIT_TYPE) =~ /s/; + $SPLIT_TRG = lc($SPLIT_TYPE) =~ /t/; + $name = $name . ($SPLIT_SRC ? $PPL_SRC : "0"); + $name = $name . "_" . ($SPLIT_TRG ? $PPL_TRG : "0"); + } else { + #no splitting + $name = "0"; + } + + return "sp_" . $name; + +} + +sub processed_dir { + return corpus_dir() . "." . split_name; +} + +sub usage { + print <<EOT; + +Usage: $0 [OPTIONS] corpus.src corpus.trg dev.src dev.trg test.src test.trg + +Learns a segmentation model and splits up corpora as necessary. Word alignments are trained on a specified subset of the training corpus. + +EOT + exit 1; +}; + +sub safemkdir { + my $dir = shift; + if (-d $dir) { return 1; } + return mkdir($dir); +} + +sub assert_exec { + my @files = @_; + for my $file (@files) { + die "Can't find $file - did you run make?\n" unless -e $file; + die "Can't execute $file" unless -e $file; + } +}; +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "ERROR: Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + +sub get_basename +{ + my $x = shift; + $x = `basename $x`; + $x =~ s/\n//; + return $x; +} + +sub write_wconf { + my ($filename, $train_dir) = @_; + open WCONF, ">$filename" or die "Can't write $filename: $!"; + +#TODO CHANGE ITERATIONS BELOW!!! + print WCONF <<EOT; +## ---------------------- +## This is an example training script for the Berkeley +## word aligner. In this configuration it uses two HMM +## alignment models trained jointly and then decoded +## using the competitive thresholding heuristic. + +########################################## +# Training: Defines the training regimen +########################################## +forwardModels MODEL1 HMM +reverseModels MODEL1 HMM +mode JOINT JOINT +iters 1 1 + +############################################### +# Execution: Controls output and program flow +############################################### +execDir $ALIGNMENT_DIR +create +overwriteExecDir +saveParams true +numThreads 1 +msPerLine 10000 +alignTraining + +################# +# Language/Data +################# +foreignSuffix src +englishSuffix trg + +# Choose the training sources, which can either be directories or files that list files/directories +trainSources $train_dir/ +#trainSources $train_dir/sources +testSources +sentences MAX + +################# +# 1-best output +################# +competitiveThresholding + +EOT + close WCONF; +} + +sub write_eval_sh +{ + my ($filename) = @_; + open EVALFILE, ">$filename" or die "Can't write $filename: $!"; + + print EVALFILE <<EOT; +#!/bin/bash +d=`dirname \$0` + +EVAL_MAIN=/export/ws10smt/data/eval.sh +EOT + + if ($SPLIT_TRG) { + print EVALFILE <<EOT; +echo "OUTPUT EVALUATION" +echo "-----------------" +\$EVAL_MAIN "\$1" \$d/$TEST_TRG_name.split + +echo "RECOMBINED OUTPUT EVALUATION" +echo "----------------------------" +marker="$MARKER" +cat "\$1" | sed -e "s/\$marker \$marker//g" -e "s/\$marker//g" > "\$1.recombined" + +\$EVAL_MAIN "\$1.recombined" \$d/$TEST_TRG_name.unsplit +EOT + + } else { + print EVALFILE <<EOT; +#!/bin/bash +d=`dirname \$0` + +EVAL_MAIN=/export/ws10smt/data/eval.sh + +echo "ARTIFICIAL SPLIT EVALUATION" +echo "--------------------------" + +MARKER="$MARKER" +#split the output translation +cat "\$1" | $MORFSEGMENT $MODEL_TRG_DIR/inputvocab.gz $MODEL_TRG_DIR/segmentation.ready "\$MARKER" > "\$1.split" + +\$EVAL_MAIN "i\$1.split" \$d/$TEST_TRG_name.split + + +echo "DIRECT EVALUATION" +echo "--------------------------" +\$EVAL_MAIN "\$1" \$d/$TEST_TRG_name.unsplit + +EOT + } + close EVALFILE; + +} + |