From 7cc92b65a3185aa242088d830e166e495674efc9 Mon Sep 17 00:00:00 2001
From: redpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>
Date: Tue, 22 Jun 2010 05:12:27 +0000
Subject: initial checkin

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@2 ec762483-ff6d-05da-a07a-a48fb63a330f
---
 tests/run-system-tests.pl | 121 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100755 tests/run-system-tests.pl

(limited to 'tests/run-system-tests.pl')

diff --git a/tests/run-system-tests.pl b/tests/run-system-tests.pl
new file mode 100755
index 00000000..8555ef78
--- /dev/null
+++ b/tests/run-system-tests.pl
@@ -0,0 +1,121 @@
+#!/usr/bin/perl -w
+use strict;
+my $script_dir; BEGIN { use Cwd qw/ abs_path cwd /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
+
+use IPC::Run3;
+use File::Temp qw ( tempdir );
+my $TEMP_DIR = tempdir( CLEANUP => 1 );
+
+#my $cwd = cwd();
+#die "Sanity failed: $cwd" unless -d $cwd;
+my $DECODER = "$script_dir/../decoder/cdec";
+my $FILTER = "$script_dir/tools/filter-stderr.pl";
+my $COMPARE_STATS = "$script_dir/tools/compare-statistics.pl";
+
+die "Can't find $DECODER" unless -f $DECODER;
+die "Can't execute $DECODER" unless -x $DECODER;
+die "Can't find $FILTER" unless -f $FILTER;
+die "Can't execute $FILTER" unless -x $FILTER;
+die "Can't find $COMPARE_STATS" unless -f $COMPARE_STATS;
+die "Can't execute $COMPARE_STATS" unless -x $COMPARE_STATS;
+
+my $TEST_DIR = "$script_dir/system_tests";
+opendir DIR, $TEST_DIR or die "Can't open $TEST_DIR: $!";
+#my @test_dirs = grep { /^\./ && −d "$some_dir/$_" } readdir(DIR);
+my @tests = grep { !/^\./ && -d "$TEST_DIR/$_" } readdir(DIR);
+closedir DIR;
+
+print STDERR " DECODER: $DECODER\n";
+print STDERR "   TESTS: @tests\n";
+print STDERR "TEMP DIR: $TEMP_DIR\n";
+
+my $FAIL = 0;
+my $PASS = 0;
+for my $test (@tests) {
+  print "TEST: $test\n";
+  chdir "$TEST_DIR/$test" or die "Can't chdir to $TEST_DIR/$test: $!";
+  my $CMD = "$DECODER";
+  unless (-f 'gold.statistics') {
+    print "  missing gold.statistics -- SKIPPING\n";
+    $FAIL++;
+    next;
+  }
+  unless (-f 'gold.stdout') {
+    print "  missing gold.stdout -- SKIPPING\n";
+    $FAIL++;
+    next;
+  }
+  if (-f 'cdec.ini') {
+    $CMD .= ' -c cdec.ini';
+  }
+  if (-f 'weights') {
+    $CMD .= ' -w weights';
+  }
+  if (-f 'input.txt') {
+    $CMD .= ' -i input.txt';
+  }
+
+  run3 $CMD, \undef, "$TEMP_DIR/stdout", "$TEMP_DIR/stderr";
+  if ($? != 0) {
+    print STDERR "  non-zero exit! command: $CMD\n";
+    $FAIL++;
+  } else {
+    die unless -f "$TEMP_DIR/stdout";
+    my $failed = 0;
+    run3 "diff gold.stdout $TEMP_DIR/stdout";
+    if ($? != 0) {
+      print STDERR "  FAILED differences in output!\n";
+      $failed = 1;
+    }
+    die unless -f "$TEMP_DIR/stderr";
+    run3 "$FILTER", "$TEMP_DIR/stderr", "$TEMP_DIR/test.statistics";
+    if ($? != 0) {
+      print STDERR "  non-zero exit: $FILTER\n";
+      $FAIL++;
+      next;
+    }
+    my @lines;
+    run3 "$COMPARE_STATS gold.statistics", "$TEMP_DIR/test.statistics", \@lines;
+    if (scalar @lines != 1) {
+      print STDERR "  unexpected output: @lines\n";
+      $FAIL++;
+      next;
+    }
+    my $l = $lines[0]; chomp $l;
+    if ($l =~ /^(\d+) (\d+)$/) {
+      my $passes = $1;
+      my $total = $2;
+      my $pct = $passes * 100 / $total;
+      $pct = sprintf "%.2f", $pct;
+      
+      if ($total == $passes) {
+        if ($failed) {
+          print "    (decoder statistics match, though)\n";
+        } else {
+          print "  PASSED\n";
+        }
+      } else {
+        if ($failed) {
+          print "     ($pct of decoder search statistics match)\n";
+        } else {
+          print "  FAILED $pct of decoder search statistics match\n";
+        }
+      }
+    } else {
+      $failed = 1;
+      print STDERR "  bad format: $l\n";
+    }
+    if ($failed) { $FAIL++; } else { $PASS++; }
+  }
+}
+
+my $TOT = $PASS + $FAIL;
+print "\nSUMMARY: $PASS / $TOT TESTS PASSED\n";
+if ($FAIL != 0) {
+  print "  !!! THERE WERE FAILURES - DECODER IS ACTING SUSPICIOUSLY !!!\n\n";
+  exit 1;
+} else {
+  print "\n";
+  exit 0;
+}
+
-- 
cgit v1.2.3