From 22b3bffd201fe43cea4c4639bcefc3c0e4746518 Mon Sep 17 00:00:00 2001
From: redpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>
Date: Thu, 22 Jul 2010 20:52:22 +0000
Subject: add additional filtering step

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@368 ec762483-ff6d-05da-a07a-a48fb63a330f
---
 gi/pipeline/scripts/refilter.pl    | 37 +++++++++++++++++++++++++++++++++++++
 gi/pipeline/scripts/rekey.pl       |  8 ++++++++
 gi/pipeline/scripts/sort-by-key.sh |  2 +-
 3 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100755 gi/pipeline/scripts/refilter.pl
 create mode 100755 gi/pipeline/scripts/rekey.pl

(limited to 'gi/pipeline/scripts')

diff --git a/gi/pipeline/scripts/refilter.pl b/gi/pipeline/scripts/refilter.pl
new file mode 100755
index 00000000..11a36ebe
--- /dev/null
+++ b/gi/pipeline/scripts/refilter.pl
@@ -0,0 +1,37 @@
+#!/usr/bin/perl -w
+use strict;
+
+my $NUM_TRANSLATIONS = 30;
+my $pk = '';
+my %dict;
+while(<>) {
+  s/^(.+)\t//;
+  my $key = $1;
+  if ($key ne $pk) {
+    if ($pk) {
+      emit_dict();
+    }
+    %dict = ();
+    $pk = $key;
+  }
+  my ($lhs, $f, $e, $s) = split / \|\|\| /;
+  my $score = 0;
+  if ($s =~ /XEF=([^ ]+)/) {
+    $score += $1;
+  } else { die; }
+  if ($s =~ /GenerativeProb=([^ ]+)/) {
+    $score += ($1 / 10);
+  } else { die; }
+  $dict{"$lhs ||| $f ||| $e ||| $s"} = $score;
+}
+emit_dict();
+
+sub emit_dict {
+  my $cc = 0;
+  for my $k (sort { $dict{$a} <=> $dict{$b} } keys %dict) {
+    print "$k";
+    $cc++;
+    if ($cc >= $NUM_TRANSLATIONS) { last; }
+  }
+}
+
diff --git a/gi/pipeline/scripts/rekey.pl b/gi/pipeline/scripts/rekey.pl
new file mode 100755
index 00000000..31eb86b8
--- /dev/null
+++ b/gi/pipeline/scripts/rekey.pl
@@ -0,0 +1,8 @@
+#!/usr/bin/perl
+
+while(<>) {
+  my ($lhs, $f, $e, $s) = split / \|\|\| /;
+  $f =~ s/\[X[0-9]+\]/\[X\]/g;
+  print "$f\t$_";
+}
+
diff --git a/gi/pipeline/scripts/sort-by-key.sh b/gi/pipeline/scripts/sort-by-key.sh
index 948dd4df..7ae33e03 100755
--- a/gi/pipeline/scripts/sort-by-key.sh
+++ b/gi/pipeline/scripts/sort-by-key.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 
 export LANG=C
-sort -t $'\t' -k 1
+sort -t $'\t' -k 1 -T /tmp -S 6000000000
 
-- 
cgit v1.2.3