Merge remote-tracking branch 'upstream/master'

14 files changed, 47 insertions, 0 deletions

diff --git a/tests/system_tests/cfg_rescore/README b/tests/system_tests/cfg_rescore/README
new file mode 100644
index 00000000..9605cbf3
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/README
@@ -0,0 +1,4 @@
+If the hypergraph serialization format changes, you will need to rerun
+the following command to regenerate the compiled CFG:
+
+cat input.cfg | ../../../training/utils/grammar_convert  > input.txt
diff --git a/tests/system_tests/cfg_rescore/cdec.ini b/tests/system_tests/cfg_rescore/cdec.ini
new file mode 100644
index 00000000..1a913f2d
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/cdec.ini
@@ -0,0 +1,2 @@
+formalism=rescore
+k_best=100
diff --git a/tests/system_tests/cfg_rescore/gold.statistics b/tests/system_tests/cfg_rescore/gold.statistics
new file mode 100644
index 00000000..7b05e2d8
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/gold.statistics
@@ -0,0 +1,3 @@
+-lm_nodes 8
+-lm_edges 10
+-lm_paths 4
diff --git a/tests/system_tests/cfg_rescore/gold.stdout b/tests/system_tests/cfg_rescore/gold.stdout
new file mode 100644
index 00000000..ccf99263
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/gold.stdout
@@ -0,0 +1,4 @@
+0 ||| the broccoli was eaten by John ||| Passive=1 Definite=1 ||| 2
+0 ||| John ate the broccoli ||| Active=1 Definite=1 ||| 1.1
+0 ||| broccoli was eaten by John ||| Passive=1 ||| 1
+0 ||| John ate broccoli ||| Active=1 ||| 0.1
diff --git a/tests/system_tests/cfg_rescore/input.cfg b/tests/system_tests/cfg_rescore/input.cfg
new file mode 100644
index 00000000..75602c75
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/input.cfg
@@ -0,0 +1,8 @@
+[S] ||| [NP1] [VP] ||| Active=1
+[S] ||| [NP2] [VPSV] by [NP1] ||| Passive=1
+[VP] ||| [V] [NP2]
+[V] ||| ate
+[VPSV] ||| was eaten
+[NP1] ||| John
+[NP2] ||| broccoli
+[NP2] ||| the broccoli ||| Definite=1
diff --git a/tests/system_tests/cfg_rescore/input.txt b/tests/system_tests/cfg_rescore/input.txt
new file mode 100644
index 00000000..2999a5fb
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/input.txt
@@ -0,0 +1 @@
+{"rules":[1,"[S] ||| [NP1] [VP] ||| [1] [2] ||| Active=1",2,"[S] ||| [NP2] [VPSV] by [NP1] ||| [1] [2] by [3] ||| Passive=1",3,"[VP] ||| [V] [NP2] ||| [1] [2]",4,"[V] ||| ate ||| ate",5,"[VPSV] ||| was eaten ||| was eaten",6,"[NP1] ||| John ||| John",7,"[NP2] ||| broccoli ||| broccoli",8,"[NP2] ||| the broccoli ||| the broccoli ||| Definite=1",9,"[Goal] ||| [X] ||| [1]"],"features":["PhraseModel_0","PhraseModel_1","PhraseModel_2","PhraseModel_3","PhraseModel_4","PhraseModel_5","PhraseModel_6","PhraseModel_7","PhraseModel_8","PhraseModel_9","PhraseModel_10","PhraseModel_11","PhraseModel_12","PhraseModel_13","PhraseModel_14","PhraseModel_15","PhraseModel_16","PhraseModel_17","PhraseModel_18","PhraseModel_19","PhraseModel_20","PhraseModel_21","PhraseModel_22","PhraseModel_23","PhraseModel_24","PhraseModel_25","PhraseModel_26","PhraseModel_27","PhraseModel_28","PhraseModel_29","PhraseModel_30","PhraseModel_31","PhraseModel_32","PhraseModel_33","PhraseModel_34","PhraseModel_35","PhraseModel_36","PhraseModel_37","PhraseModel_38","PhraseModel_39","PhraseModel_40","PhraseModel_41","PhraseModel_42","PhraseModel_43","PhraseModel_44","PhraseModel_45","PhraseModel_46","PhraseModel_47","PhraseModel_48","PhraseModel_49","PhraseModel_50","PhraseModel_51","PhraseModel_52","PhraseModel_53","PhraseModel_54","PhraseModel_55","PhraseModel_56","PhraseModel_57","PhraseModel_58","PhraseModel_59","PhraseModel_60","PhraseModel_61","PhraseModel_62","PhraseModel_63","PhraseModel_64","PhraseModel_65","PhraseModel_66","PhraseModel_67","PhraseModel_68","PhraseModel_69","PhraseModel_70","PhraseModel_71","PhraseModel_72","PhraseModel_73","PhraseModel_74","PhraseModel_75","PhraseModel_76","PhraseModel_77","PhraseModel_78","PhraseModel_79","PhraseModel_80","PhraseModel_81","PhraseModel_82","PhraseModel_83","PhraseModel_84","PhraseModel_85","PhraseModel_86","PhraseModel_87","PhraseModel_88","PhraseModel_89","PhraseModel_90","PhraseModel_91","PhraseModel_92","PhraseModel_93","PhraseModel_94","PhraseModel_95","PhraseModel_96","PhraseModel_97","PhraseModel_98","PhraseModel_99","Active","Passive","Definite"],"edges":[{"tail":[],"spans":[-1,-1,-1,-1],"feats":[],"rule":6}],"node":{"in_edges":[0],"cat":"NP1","node_hash":"0000000000000006"},"edges":[{"tail":[],"spans":[-1,-1,-1,-1],"feats":[],"rule":4}],"node":{"in_edges":[1],"cat":"V","node_hash":"0000000000000004"},"edges":[{"tail":[],"spans":[-1,-1,-1,-1],"feats":[],"rule":7},{"tail":[],"spans":[-1,-1,-1,-1],"feats":[102,1],"rule":8}],"node":{"in_edges":[2,3],"cat":"NP2","node_hash":"0000000000000008"},"edges":[{"tail":[1,2],"spans":[-1,-1,-1,-1],"feats":[],"rule":3}],"node":{"in_edges":[4],"cat":"VP","node_hash":"0000000000000003"},"edges":[{"tail":[],"spans":[-1,-1,-1,-1],"feats":[],"rule":5}],"node":{"in_edges":[5],"cat":"VPSV","node_hash":"0000000000000005"},"edges":[{"tail":[0,3],"spans":[-1,-1,-1,-1],"feats":[100,1],"rule":1},{"tail":[2,4,0],"spans":[-1,-1,-1,-1],"feats":[101,1],"rule":2}],"node":{"in_edges":[6,7],"cat":"S","node_hash":"0000000000000002"},"edges":[{"tail":[5],"spans":[-1,-1,-1,-1],"feats":[],"rule":9}],"node":{"in_edges":[8],"cat":"Goal","node_hash":"000000000000003D"}}
diff --git a/tests/system_tests/cfg_rescore/weights b/tests/system_tests/cfg_rescore/weights
new file mode 100644
index 00000000..bd3bb1af
--- /dev/null
+++ b/tests/system_tests/cfg_rescore/weights
@@ -0,0 +1,3 @@
+Active 0.1
+Passive 1
+Definite 1
diff --git a/tests/system_tests/t2s/cdec.ini b/tests/system_tests/t2s/cdec.ini
new file mode 100644
index 00000000..ad83438f
--- /dev/null
+++ b/tests/system_tests/t2s/cdec.ini
@@ -0,0 +1,2 @@
+formalism=t2s
+grammar=grammar.t2s
diff --git a/tests/system_tests/t2s/gold.statistics b/tests/system_tests/t2s/gold.statistics
new file mode 100644
index 00000000..5778a24c
--- /dev/null
+++ b/tests/system_tests/t2s/gold.statistics
@@ -0,0 +1,3 @@
+-lm_nodes 7
+-lm_edges 9
+-lm_paths 4
diff --git a/tests/system_tests/t2s/gold.stdout b/tests/system_tests/t2s/gold.stdout
new file mode 100644
index 00000000..afb11818
--- /dev/null
+++ b/tests/system_tests/t2s/gold.stdout
@@ -0,0 +1 @@
+qiangshou bei jingfang jibi .
diff --git a/tests/system_tests/t2s/grammar.t2s b/tests/system_tests/t2s/grammar.t2s
new file mode 100644
index 00000000..2e6cf68c
--- /dev/null
+++ b/tests/system_tests/t2s/grammar.t2s
@@ -0,0 +1,8 @@
+(S [NP-C] [VP] (PUNC .)) ||| [1] [2] . ||| R1=1
+(NP-C (DT the) (NN gunman)) ||| qiangshou ||| R2=1
+(NP-C (DT the) [NN]) ||| [1] ||| R2a=1
+(NN gunman) ||| qiangshou ||| R2b=1
+(VP (VBD was) (VP-C [VBN] (PP (IN by) [NP-C]))) ||| bei [2] [1] ||| R3=1
+(NP-C (DT the) (NN police)) ||| jingfang ||| R4=1
+(VBN killed) ||| jibi ||| R5=1
+(VBN killed) ||| killed' ||| R6=1
diff --git a/tests/system_tests/t2s/input.txt b/tests/system_tests/t2s/input.txt
new file mode 100644
index 00000000..b8fe314e
--- /dev/null
+++ b/tests/system_tests/t2s/input.txt
@@ -0,0 +1 @@
+(S (NP-C (DT the) (NN gunman)) (VP (VBD was) (VP-C (VBN killed) (PP (IN by) (NP-C (DT the) (NN police))))) (PUNC .))
diff --git a/tests/system_tests/t2s/weights b/tests/system_tests/t2s/weights
new file mode 100644
index 00000000..4980db45
--- /dev/null
+++ b/tests/system_tests/t2s/weights
@@ -0,0 +1,6 @@
+R1 1
+R2a 1
+R2b 1
+R3 1
+R5 1
+R4 1
diff --git a/tests/tools/filter-stderr.pl b/tests/tools/filter-stderr.pl
index 4a762324..54fe9210 100755
--- a/tests/tools/filter-stderr.pl
+++ b/tests/tools/filter-stderr.pl
@@ -13,6 +13,7 @@ if (/Init.*\s+Viterbi:\s+($REAL)/) {
   # -LM Viterbi: australia is have diplomatic relations with north korea one of the few countries .
   print "-lm_trans $1\n";
 }
+if (/NODES NOT UNIQUELY IDENTIFIED/) { print "NODES_NOT_UNIQUE 1\n"; }
 #Constr. forest (nodes/edges): 111/305
 #Constr. forest       (paths): 9899
 if (/Constr\. forest\s+\(nodes\/edges\): (\d+)\/(\d+)/) { print "constr_nodes $1\nconstr_edges $2\n"; }