summaryrefslogtreecommitdiff
path: root/gi/morf-segmentation/morftrain.sh
diff options
context:
space:
mode:
authorChris Dyer <cdyer@cab.ark.cs.cmu.edu>2012-10-02 00:19:43 -0400
committerChris Dyer <cdyer@cab.ark.cs.cmu.edu>2012-10-02 00:19:43 -0400
commite26434979adc33bd949566ba7bf02dff64e80a3e (patch)
treed1c72495e3af6301bd28e7e66c42de0c7a944d1f /gi/morf-segmentation/morftrain.sh
parent0870d4a1f5e14cc7daf553b180d599f09f6614a2 (diff)
cdec cleanup, remove bayesian stuff, parsing stuff
Diffstat (limited to 'gi/morf-segmentation/morftrain.sh')
-rwxr-xr-xgi/morf-segmentation/morftrain.sh110
1 files changed, 0 insertions, 110 deletions
diff --git a/gi/morf-segmentation/morftrain.sh b/gi/morf-segmentation/morftrain.sh
deleted file mode 100755
index 9004922f..00000000
--- a/gi/morf-segmentation/morftrain.sh
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/bin/bash
-
-if [[ $# -lt 3 ]]; then
- echo "Trains a morfessor model and places the result in writedir"
- echo
- echo "Usage: `basename $0` corpus_input_file writedir [PPL] [marker] [lines]"
- echo -e "\tcorpus_input_file contains a sentence per line."
- exit 1
-fi
-
-MORFESSOR_DIR="/export/ws10smt/software/morfessor_catmap0.9.2"
-SCRIPT_DIR=$(dirname `readlink -f $0`)
-
-MORFBINDIR="$MORFESSOR_DIR/bin"
-MORFMAKEFILE_TRAIN="$MORFESSOR_DIR/train/Makefile"
-VOCABEXT="$SCRIPT_DIR/vocabextractor.sh"
-
-MARKER="#"
-
-if [[ ! -f $VOCABEXT ]]; then
- echo "$VOCABEXT doesn't exist!"
- exit 1
-fi
-if [[ ! -f $MORFMAKEFILE_TRAIN ]]; then
- echo "$MORFMAKEFILE_TRAIN doesn't exist!"
- exit 1
-fi
-
-
-CORPUS="$1"
-WRITETODIR=$2
-
-if [[ ! -f $CORPUS ]]; then
- echo "$CORPUS doesn't exist!"
- exit 1
-fi
-
-PPL=10
-LINES=0
-if [[ $# -gt 2 ]]; then
- PPL=$3
-fi
-if [[ $# -gt 3 ]]; then
- MARKER="$4"
-fi
-if [[ $# -gt 4 ]]; then
- LINES=$5
-fi
-
-mkdir -p $WRITETODIR
-
-#extract vocabulary to train on
-echo "Extracting vocabulary..."
-if [[ -f $WRITETODIR/inputvocab.gz ]]; then
- echo " ....$WRITETODIR/inputvocab.gz exists, reusing."
-else
- if [[ $LINES -gt 0 ]]; then
- $VOCABEXT $CORPUS $LINES | gzip > $WRITETODIR/inputvocab.gz
- else
- $VOCABEXT $CORPUS | gzip > $WRITETODIR/inputvocab.gz
- fi
-fi
-
-
-#train it
-echo "Training morf model..."
-if [[ -f $WRITETODIR/segmentation.final.gz ]]; then
- echo " ....$WRITETODIR/segmentation.final.gz exists, reusing.."
-else
- OLDPWD=`pwd`
- cd $WRITETODIR
-
- #put the training Makefile in place, with appropriate modifications
- sed -e "s/^GZIPPEDINPUTDATA = .*$/GZIPPEDINPUTDATA = inputvocab.gz/" \
- -e "s/^PPLTHRESH = .*$/PPLTHRESH = $PPL/" \
- -e "s;^BINDIR = .*$;BINDIR = $MORFBINDIR;" \
- $MORFMAKEFILE_TRAIN > ./Makefile
-
- date
- make > ./trainmorf.log 2>&1
- cd $OLDPWD
-
-
- echo "Post processing..."
- #remove comments, counts and morph types
- #mark morphs
-
- if [[ ! -f $WRITETODIR/segmentation.final.gz ]]; then
- echo "Failed to learn segmentation model: $WRITETODIR/segmentation.final.gz not written"
- exit 1
- fi
-
- zcat $WRITETODIR/segmentation.final.gz | \
- awk '$1 !~ /^#/ {print}' | \
- cut -d ' ' --complement -f 1 | \
- sed -e "s/\/...//g" -e "s/ + /$MARKER $MARKER/g" \
- > $WRITETODIR/segmentation.ready
-
- if [[ ! -f $WRITETODIR/segmentation.ready ]]; then
- echo "Failed to learn segmentation model: $WRITETODIR/segmentation.final.gz not written"
- exit 1
- fi
-
-
-
- echo "Done training."
- date
-fi
-echo "Segmentation model is $WRITETODIR/segmentation.ready."
-