diff options
Diffstat (limited to 'decoder/rule_lexer.l')
| -rw-r--r-- | decoder/rule_lexer.l | 316 |
1 files changed, 0 insertions, 316 deletions
diff --git a/decoder/rule_lexer.l b/decoder/rule_lexer.l deleted file mode 100644 index 083a5bb1..00000000 --- a/decoder/rule_lexer.l +++ /dev/null @@ -1,316 +0,0 @@ -%option nounput -%{ -#include "rule_lexer.h" - -#include <string> -#include <iostream> -#include <sstream> -#include <cstring> -#include <cassert> -#include <stack> -#include "tdict.h" -#include "fdict.h" -#include "trule.h" -#include "verbose.h" - -int lex_line = 0; -std::istream* scfglex_stream = NULL; -RuleLexer::RuleCallback rule_callback = NULL; -void* rule_callback_extra = NULL; -std::vector<int> scfglex_phrase_fnames; - -#undef YY_INPUT -#define YY_INPUT(buf, result, max_size) (result = scfglex_stream->read(buf, max_size).gcount()) - -#define YY_SKIP_YYWRAP 1 -int num_rules = 0; -int yywrap() { return 1; } -bool fl = true; -#define MAX_TOKEN_SIZE 255 -std::string scfglex_tmp_token(MAX_TOKEN_SIZE, '\0'); - -#define MAX_RULE_SIZE 200 -WordID scfglex_src_rhs[MAX_RULE_SIZE]; -WordID scfglex_trg_rhs[MAX_RULE_SIZE]; -int scfglex_src_rhs_size; -int scfglex_trg_rhs_size; -WordID scfglex_lhs; -int scfglex_src_arity; -int scfglex_trg_arity; - -#define MAX_FEATS 100 -int scfglex_feat_ids[MAX_FEATS]; -double scfglex_feat_vals[MAX_FEATS]; -int scfglex_num_feats; - -#define MAX_ARITY 200 -int scfglex_nt_sanity[MAX_ARITY]; -int scfglex_src_nts[MAX_ARITY]; -// float scfglex_nt_size_means[MAX_ARITY]; -// float scfglex_nt_size_vars[MAX_ARITY]; -std::stack<TRulePtr> ctf_rule_stack; -unsigned int ctf_level = 0; - -#define MAX_ALS 200 -AlignmentPoint scfglex_als[MAX_ALS]; -int scfglex_num_als; - -void sanity_check_trg_symbol(WordID nt, int index) { - if (scfglex_src_nts[index-1] != nt) { - std::cerr << "Target symbol with index " << index << " is of type " << TD::Convert(nt*-1) - << " but corresponding source is of type " - << TD::Convert(scfglex_src_nts[index-1] * -1) << std::endl; - abort(); - } -} - -void sanity_check_trg_index(int index) { - if (index > scfglex_src_arity) { - std::cerr << "Target index " << index << " exceeds source arity " << scfglex_src_arity << std::endl; - abort(); - } - int& flag = scfglex_nt_sanity[index - 1]; - if (flag) { - std::cerr << "Target index " << index << " used multiple times!" << std::endl; - abort(); - } - flag = 1; -} - -void scfglex_reset() { - scfglex_src_arity = 0; - scfglex_trg_arity = 0; - scfglex_num_feats = 0; - scfglex_src_rhs_size = 0; - scfglex_trg_rhs_size = 0; - scfglex_num_als = 0; -} - -void check_and_update_ctf_stack(const TRulePtr& rp) { - if (ctf_level > ctf_rule_stack.size()){ - std::cerr << "Found rule at projection level " << ctf_level << " but previous rule was at level " - << ctf_rule_stack.size()-1 << " (cannot exceed previous level by more than one; line " << lex_line << ")" << std::endl; - abort(); - } - while (ctf_rule_stack.size() > ctf_level) - ctf_rule_stack.pop(); - // ensure that rule has the same signature as parent (coarse) rule. Rules may *only* - // differ by the rhs nonterminals, not terminals or permutation of nonterminals. - if (ctf_rule_stack.size() > 0) { - TRulePtr& coarse_rp = ctf_rule_stack.top(); - if (rp->f_.size() != coarse_rp->f_.size() || rp->e_ != coarse_rp->e_) { - std::cerr << "Rule " << (rp->AsString()) << " is not a projection of " << - (coarse_rp->AsString()) << std::endl; - abort(); - } - for (int i=0; i<rp->f_.size(); ++i) { - if (((rp->f_[i]<0) != (coarse_rp->f_[i]<0)) || - ((rp->f_[i]>0) && (rp->f_[i] != coarse_rp->f_[i]))) { - std::cerr << "Rule " << (rp->AsString()) << " is not a projection of " << - (coarse_rp->AsString()) << std::endl; - abort(); - } - } - } -} - -%} - -REAL [\-+]?[0-9]+(\.[0-9]*([eE][-+]*[0-9]+)?)?|inf|[\-+]inf -NT [^\t \[\],]+ - -%x LHS_END SRC TRG FEATS FEATVAL ALIGNS -%% - -<INITIAL>[ \t] { - ctf_level++; - }; - -<INITIAL>\[{NT}\] { - scfglex_tmp_token.assign(yytext + 1, yyleng - 2); - scfglex_lhs = -TD::Convert(scfglex_tmp_token); - // std::cerr << scfglex_tmp_token << "\n"; - BEGIN(LHS_END); - } - -<SRC>\[{NT}\] { - scfglex_tmp_token.assign(yytext + 1, yyleng - 2); - scfglex_src_nts[scfglex_src_arity] = scfglex_src_rhs[scfglex_src_rhs_size] = -TD::Convert(scfglex_tmp_token); - ++scfglex_src_arity; - ++scfglex_src_rhs_size; - } - -<SRC>\[{NT},[1-9][0-9]?\] { - int index = yytext[yyleng - 2] - '0'; - if (yytext[yyleng - 3] == ',') { - scfglex_tmp_token.assign(yytext + 1, yyleng - 4); - } else { - scfglex_tmp_token.assign(yytext + 1, yyleng - 5); - index += 10 * (yytext[yyleng - 3] - '0'); - } - if ((scfglex_src_arity+1) != index) { - std::cerr << "Src indices must go in order: expected " << scfglex_src_arity << " but got " << index << std::endl; - abort(); - } - scfglex_src_nts[scfglex_src_arity] = scfglex_src_rhs[scfglex_src_rhs_size] = -TD::Convert(scfglex_tmp_token); - ++scfglex_src_rhs_size; - ++scfglex_src_arity; - } - -<TRG>\[{NT},[1-9][0-9]?\] { - int index = yytext[yyleng - 2] - '0'; - if (yytext[yyleng - 3] == ',') { - scfglex_tmp_token.assign(yytext + 1, yyleng - 4); - } else { - scfglex_tmp_token.assign(yytext + 1, yyleng - 5); - index += 10 * (yytext[yyleng - 3] - '0'); - } - ++scfglex_trg_arity; - // std::cerr << "TRG INDEX: " << index << std::endl; - sanity_check_trg_symbol(-TD::Convert(scfglex_tmp_token), index); - sanity_check_trg_index(index); - scfglex_trg_rhs[scfglex_trg_rhs_size] = 1 - index; - ++scfglex_trg_rhs_size; -} - -<TRG>\[[1-9][0-9]?\] { - int index = yytext[yyleng - 2] - '0'; - if (yyleng == 4) { - index += 10 * (yytext[yyleng - 3] - '0'); - } - ++scfglex_trg_arity; - sanity_check_trg_index(index); - scfglex_trg_rhs[scfglex_trg_rhs_size] = 1 - index; - ++scfglex_trg_rhs_size; -} - -<LHS_END>[ \t] { ; } -<LHS_END>\|\|\| { - scfglex_reset(); - BEGIN(SRC); - } -<INITIAL,LHS_END>. { - std::cerr << "Line " << lex_line << ": unexpected input in LHS: " << yytext << std::endl; - abort(); - } - -<SRC>\|\|\| { - memset(scfglex_nt_sanity, 0, scfglex_src_arity * sizeof(int)); - BEGIN(TRG); - } -<SRC>[^ \t]+ { - scfglex_tmp_token.assign(yytext, yyleng); - scfglex_src_rhs[scfglex_src_rhs_size] = TD::Convert(scfglex_tmp_token); - ++scfglex_src_rhs_size; - } -<SRC>[ \t]+ { ; } - -<TRG>\|\|\| { - BEGIN(FEATS); - } -<TRG>[^ \t]+ { - scfglex_tmp_token.assign(yytext, yyleng); - scfglex_trg_rhs[scfglex_trg_rhs_size] = TD::Convert(scfglex_tmp_token); - ++scfglex_trg_rhs_size; - } -<TRG>[ \t]+ { ; } - -<TRG,FEATS,ALIGNS>\n { - if (scfglex_src_arity != scfglex_trg_arity) { - std::cerr << "Line " << lex_line << ": LHS and RHS arity mismatch!\n"; - abort(); - } - // const bool ignore_grammar_features = false; - // if (ignore_grammar_features) scfglex_num_feats = 0; - TRulePtr rp(new TRule(scfglex_lhs, scfglex_src_rhs, scfglex_src_rhs_size, scfglex_trg_rhs, scfglex_trg_rhs_size, scfglex_feat_ids, scfglex_feat_vals, scfglex_num_feats, scfglex_src_arity, scfglex_als, scfglex_num_als)); - check_and_update_ctf_stack(rp); - TRulePtr coarse_rp = ((ctf_level == 0) ? TRulePtr() : ctf_rule_stack.top()); - rule_callback(rp, ctf_level, coarse_rp, rule_callback_extra); - ctf_rule_stack.push(rp); - // std::cerr << rp->AsString() << std::endl; - num_rules++; - lex_line++; - if (!SILENT) { - if (num_rules % 50000 == 0) { std::cerr << '.' << std::flush; fl = true; } - if (num_rules % 2000000 == 0) { std::cerr << " [" << num_rules << "]\n"; fl = false; } - } - ctf_level = 0; - BEGIN(INITIAL); - } - -<FEATS>[ \t;] { ; } -<FEATS>[^ \t=;]+= { - scfglex_tmp_token.assign(yytext, yyleng - 1); - const int fid = FD::Convert(scfglex_tmp_token); - if (fid < 1) { - std::cerr << "\nUNWEIGHED FEATURE " << scfglex_tmp_token << std::endl; - abort(); - } - scfglex_feat_ids[scfglex_num_feats] = fid; - BEGIN(FEATVAL); - } -<FEATS>\|\|\| { - BEGIN(ALIGNS); - } -<FEATVAL>{REAL} { - scfglex_feat_vals[scfglex_num_feats] = strtod(yytext, NULL); - ++scfglex_num_feats; - BEGIN(FEATS); - } -<FEATVAL>. { - std::cerr << "Line " << lex_line << ": unexpected input in feature value: " << yytext << std::endl; - abort(); - } -<FEATS>{REAL} { - scfglex_feat_ids[scfglex_num_feats] = scfglex_phrase_fnames[scfglex_num_feats]; - scfglex_feat_vals[scfglex_num_feats] = strtod(yytext, NULL); - ++scfglex_num_feats; - } -<FEATS>. { - std::cerr << "Line " << lex_line << " unexpected input in features: " << yytext << std::endl; - abort(); - } -<ALIGNS>[0-9]+-[0-9]+ { - int i = 0; - int a = 0; - int b = 0; - while (i < yyleng) { - char c = yytext[i]; - if (c == '-') break; - a *= 10; - a += c - '0'; - ++i; - } - ++i; - while (i < yyleng) { - b *= 10; - b += yytext[i] - '0'; - ++i; - } - scfglex_als[scfglex_num_als++]=AlignmentPoint(a,b); - } -<ALIGNS>[ \t] ; -<ALIGNS>. { - std::cerr << "Line " << lex_line << ": unexpected input in alignment: " << yytext << std::endl; - abort(); - } -%% - -#include "filelib.h" - -void RuleLexer::ReadRules(std::istream* in, RuleLexer::RuleCallback func, void* extra) { - if (scfglex_phrase_fnames.empty()) { - scfglex_phrase_fnames.resize(100); - for (int i = 0; i < scfglex_phrase_fnames.size(); ++i) { - std::ostringstream os; - os << "PhraseModel_" << i; - scfglex_phrase_fnames[i] = FD::Convert(os.str()); - } - } - lex_line = 1; - scfglex_stream = in; - rule_callback_extra = extra, - rule_callback = func; - yylex(); -} - |