diff options
author | Kenneth Heafield <github@kheafield.com> | 2012-10-22 12:07:20 +0100 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2012-10-22 12:07:20 +0100 |
commit | ac586bc9b156b4ae687cd5961ba1fe7b20ec57d6 (patch) | |
tree | 052473b46d7fa18d51f897cdb9e7c93a7186dafd /gi/pyp-topics/src/contexts_lexer.l | |
parent | 97b85c082b3e55c28a8b0c0eb762483ac84a1577 (diff) | |
parent | ad6d4a1b2519896f2b16a282699ce4e64041fab8 (diff) |
Merge remote branch 'upstream/master'
Conflicts:
Jamroot
bjam
decoder/Jamfile
decoder/cdec.cc
dpmert/Jamfile
jam-files/sanity.jam
klm/lm/Jamfile
klm/util/Jamfile
mira/Jamfile
Diffstat (limited to 'gi/pyp-topics/src/contexts_lexer.l')
-rw-r--r-- | gi/pyp-topics/src/contexts_lexer.l | 113 |
1 files changed, 0 insertions, 113 deletions
diff --git a/gi/pyp-topics/src/contexts_lexer.l b/gi/pyp-topics/src/contexts_lexer.l deleted file mode 100644 index 64cd7ca3..00000000 --- a/gi/pyp-topics/src/contexts_lexer.l +++ /dev/null @@ -1,113 +0,0 @@ -%{ -#include "contexts_lexer.h" - -#include <string> -#include <iostream> -#include <sstream> -#include <cstring> -#include <cassert> -#include <algorithm> - -int lex_line = 0; -std::istream* contextslex_stream = NULL; -ContextsLexer::ContextsCallback contexts_callback = NULL; -void* contexts_callback_extra = NULL; - -#undef YY_INPUT -#define YY_INPUT(buf, result, max_size) (result = contextslex_stream->read(buf, max_size).gcount()) - -#define YY_SKIP_YYWRAP 1 -int num_phrases = 0; -int yywrap() { return 1; } - -#define MAX_TOKEN_SIZE 255 -std::string contextslex_tmp_token(MAX_TOKEN_SIZE, '\0'); -ContextsLexer::PhraseContextsType current_contexts; - -#define MAX_CONTEXT_SIZE 255 -//std::string tmp_context[MAX_CONTEXT_SIZE]; -ContextsLexer::Context tmp_context; - - -void contextslex_reset() { - current_contexts.phrase.clear(); - current_contexts.contexts.clear(); - current_contexts.counts.clear(); - tmp_context.clear(); -} - -%} - -INT [\-+]?[0-9]+|inf|[\-+]inf - -%x CONTEXT COUNT COUNT_END -%% - -<INITIAL>[^\t]+ { - contextslex_reset(); - current_contexts.phrase.assign(yytext, yyleng); - BEGIN(CONTEXT); - } -<INITIAL>\t { - ; - } - -<INITIAL,CONTEXT,COUNT>\n { - std::cerr << "ERROR: contexts_lexer.l: unexpected newline while trying to read phrase|context|count." << std::endl; - abort(); - } - -<CONTEXT>\|\|\| { - current_contexts.contexts.push_back(tmp_context); - tmp_context.clear(); - BEGIN(COUNT); - } -<CONTEXT>[^ \t]+ { - contextslex_tmp_token.assign(yytext, yyleng); - tmp_context.push_back(contextslex_tmp_token); - } -<CONTEXT>[ \t]+ { ; } - -<COUNT>[ \t]+ { ; } -<COUNT>C={INT} { - current_contexts.counts.push_back(std::make_pair(current_contexts.counts.size(), atoi(yytext+2))); - BEGIN(COUNT_END); - } -<COUNT>. { - std::cerr << "ERROR: contexts_lexer.l: unexpected content while reading count." << std::endl; - abort(); - } - -<COUNT_END>[ \t]+ { ; } -<COUNT_END>\|\|\| { - BEGIN(CONTEXT); - } -<COUNT_END>\n { - //std::cerr << "READ:" << current_contexts.phrase << " with " << current_contexts.contexts.size() - // << " contexts, and " << current_contexts.counts.size() << " counts." << std::endl; - std::sort(current_contexts.counts.rbegin(), current_contexts.counts.rend()); - - contexts_callback(current_contexts, contexts_callback_extra); - current_contexts.phrase.clear(); - current_contexts.contexts.clear(); - current_contexts.counts.clear(); - BEGIN(INITIAL); - } -<COUNT_END>. { - contextslex_tmp_token.assign(yytext, yyleng); - std::cerr << "ERROR: contexts_lexer.l: unexpected content while looking for ||| closing count." << std::endl; - abort(); - } - -%% - -#include "filelib.h" - -void ContextsLexer::ReadContexts(std::istream* in, ContextsLexer::ContextsCallback func, void* extra) { - lex_line = 1; - contextslex_stream = in; - contexts_callback_extra = extra, - contexts_callback = func; - yylex(); -} - |