1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
#include "lattice.h"
#include "tdict.h"
#include "agrammar.h"
#include "bottom_up_parser.h"
#include "hg.h"
#include "hg_intersect.h"
#include "../utils/ParamsArray.h"
using namespace std;
typedef aTextGrammar aGrammar;
aGrammar * load_grammar(string & grammar_filename){
cerr<<"start_load_grammar "<<grammar_filename<<endl;
aGrammar * test = new aGrammar(grammar_filename);
return test;
}
Lattice convertSentenceToLattice(const string & str){
std::vector<WordID> vID;
TD::ConvertSentence(str , &vID);
Lattice lsentence;
lsentence.resize(vID.size());
for (int i=0; i<vID.size(); i++){
lsentence[i].push_back( LatticeArc(vID[i], 0.0, 1) );
}
// if(!lsentence.IsSentence())
// cout<<"not a sentence"<<endl;
return lsentence;
}
bool parseSentencePair(const string & goal_sym, const string & src, const string & tgt, GrammarPtr & g, Hypergraph &hg){
Lattice lsource = convertSentenceToLattice(src);
//parse the source sentence by the grammar
vector<GrammarPtr> grammars(1, g);
ExhaustiveBottomUpParser parser = ExhaustiveBottomUpParser(goal_sym, grammars);
if (!parser.Parse(lsource, &hg)){
cerr<<"source sentence does not parse by the grammar!"<<endl;
return false;
}
//intersect the hg with the target sentence
Lattice ltarget = convertSentenceToLattice(tgt);
//forest.PrintGraphviz();
return HG::Intersect(ltarget, & hg);
}
int main(int argc, char** argv){
ParamsArray params(argc, argv);
params.setDescription("scfg models");
params.addConstraint("grammar_file", "grammar file ", true); // optional
params.addConstraint("input_file", "parallel input file", true); //optional
if (!params.runConstraints("scfg")) {
return 0;
}
cerr<<"get parametters\n\n\n";
string input_file = params.asString("input_file", "parallel_corpora");
string grammar_file = params.asString("grammar_file", "./grammar.pr");
string src = "el gato .";
string tgt = "the cat .";
string goal_sym = "X";
srand(123);
/*load grammar*/
aGrammar * agrammar = load_grammar(grammar_file);
agrammar->SetGoalNT(goal_sym);
cout<<"before split nonterminal"<<endl;
GrammarPtr g( agrammar);
Hypergraph hg;
if (! parseSentencePair(goal_sym, src, tgt, g, hg) ){
cerr<<"target sentence is not parsed by the grammar!\n";
return 1;
}
hg.PrintGraphviz();
if (! parseSentencePair(goal_sym, src, tgt, g, hg) ){
cerr<<"target sentence is not parsed by the grammar!\n";
return 1;
}
hg.PrintGraphviz();
//hg.clear();
if (1==1) return 1;
agrammar->PrintAllRules();
/*split grammar*/
cout<<"split NTs\n";
cerr<<"first of all write all nonterminals"<<endl;
// agrammar->printAllNonterminals();
agrammar->setMaxSplit(2);
agrammar->splitNonterminal(4);
cout<<"after split nonterminal"<<endl;
agrammar->PrintAllRules();
Hypergraph hg1;
if (! parseSentencePair(goal_sym, src, tgt, g, hg1) ){
cerr<<"target sentence is not parsed by the grammar!\n";
return 1;
}
hg1.PrintGraphviz();
agrammar->splitNonterminal(15);
cout<<"after split nonterminal"<<TD::Convert(15)<<endl;
agrammar->PrintAllRules();
/*load training corpus*/
/*for each sentence pair in training corpus*/
// forest.PrintGraphviz();
/*calculate expected count*/
}
|
