summaryrefslogtreecommitdiff
path: root/gi/pyp-topics/src/contexts_lexer.l
diff options
context:
space:
mode:
authorKenneth Heafield <github@kheafield.com>2012-10-22 12:07:20 +0100
committerKenneth Heafield <github@kheafield.com>2012-10-22 12:07:20 +0100
commitac586bc9b156b4ae687cd5961ba1fe7b20ec57d6 (patch)
tree052473b46d7fa18d51f897cdb9e7c93a7186dafd /gi/pyp-topics/src/contexts_lexer.l
parent97b85c082b3e55c28a8b0c0eb762483ac84a1577 (diff)
parentad6d4a1b2519896f2b16a282699ce4e64041fab8 (diff)
Merge remote branch 'upstream/master'
Conflicts: Jamroot bjam decoder/Jamfile decoder/cdec.cc dpmert/Jamfile jam-files/sanity.jam klm/lm/Jamfile klm/util/Jamfile mira/Jamfile
Diffstat (limited to 'gi/pyp-topics/src/contexts_lexer.l')
-rw-r--r--gi/pyp-topics/src/contexts_lexer.l113
1 files changed, 0 insertions, 113 deletions
diff --git a/gi/pyp-topics/src/contexts_lexer.l b/gi/pyp-topics/src/contexts_lexer.l
deleted file mode 100644
index 64cd7ca3..00000000
--- a/gi/pyp-topics/src/contexts_lexer.l
+++ /dev/null
@@ -1,113 +0,0 @@
-%{
-#include "contexts_lexer.h"
-
-#include <string>
-#include <iostream>
-#include <sstream>
-#include <cstring>
-#include <cassert>
-#include <algorithm>
-
-int lex_line = 0;
-std::istream* contextslex_stream = NULL;
-ContextsLexer::ContextsCallback contexts_callback = NULL;
-void* contexts_callback_extra = NULL;
-
-#undef YY_INPUT
-#define YY_INPUT(buf, result, max_size) (result = contextslex_stream->read(buf, max_size).gcount())
-
-#define YY_SKIP_YYWRAP 1
-int num_phrases = 0;
-int yywrap() { return 1; }
-
-#define MAX_TOKEN_SIZE 255
-std::string contextslex_tmp_token(MAX_TOKEN_SIZE, '\0');
-ContextsLexer::PhraseContextsType current_contexts;
-
-#define MAX_CONTEXT_SIZE 255
-//std::string tmp_context[MAX_CONTEXT_SIZE];
-ContextsLexer::Context tmp_context;
-
-
-void contextslex_reset() {
- current_contexts.phrase.clear();
- current_contexts.contexts.clear();
- current_contexts.counts.clear();
- tmp_context.clear();
-}
-
-%}
-
-INT [\-+]?[0-9]+|inf|[\-+]inf
-
-%x CONTEXT COUNT COUNT_END
-%%
-
-<INITIAL>[^\t]+ {
- contextslex_reset();
- current_contexts.phrase.assign(yytext, yyleng);
- BEGIN(CONTEXT);
- }
-<INITIAL>\t {
- ;
- }
-
-<INITIAL,CONTEXT,COUNT>\n {
- std::cerr << "ERROR: contexts_lexer.l: unexpected newline while trying to read phrase|context|count." << std::endl;
- abort();
- }
-
-<CONTEXT>\|\|\| {
- current_contexts.contexts.push_back(tmp_context);
- tmp_context.clear();
- BEGIN(COUNT);
- }
-<CONTEXT>[^ \t]+ {
- contextslex_tmp_token.assign(yytext, yyleng);
- tmp_context.push_back(contextslex_tmp_token);
- }
-<CONTEXT>[ \t]+ { ; }
-
-<COUNT>[ \t]+ { ; }
-<COUNT>C={INT} {
- current_contexts.counts.push_back(std::make_pair(current_contexts.counts.size(), atoi(yytext+2)));
- BEGIN(COUNT_END);
- }
-<COUNT>. {
- std::cerr << "ERROR: contexts_lexer.l: unexpected content while reading count." << std::endl;
- abort();
- }
-
-<COUNT_END>[ \t]+ { ; }
-<COUNT_END>\|\|\| {
- BEGIN(CONTEXT);
- }
-<COUNT_END>\n {
- //std::cerr << "READ:" << current_contexts.phrase << " with " << current_contexts.contexts.size()
- // << " contexts, and " << current_contexts.counts.size() << " counts." << std::endl;
- std::sort(current_contexts.counts.rbegin(), current_contexts.counts.rend());
-
- contexts_callback(current_contexts, contexts_callback_extra);
- current_contexts.phrase.clear();
- current_contexts.contexts.clear();
- current_contexts.counts.clear();
- BEGIN(INITIAL);
- }
-<COUNT_END>. {
- contextslex_tmp_token.assign(yytext, yyleng);
- std::cerr << "ERROR: contexts_lexer.l: unexpected content while looking for ||| closing count." << std::endl;
- abort();
- }
-
-%%
-
-#include "filelib.h"
-
-void ContextsLexer::ReadContexts(std::istream* in, ContextsLexer::ContextsCallback func, void* extra) {
- lex_line = 1;
- contextslex_stream = in;
- contexts_callback_extra = extra,
- contexts_callback = func;
- yylex();
-}
-