From 69b0bf8d618338c82fda17878defff77fb35a69f Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 13 May 2012 16:18:43 -0700 Subject: fast creg training code for univariate linear and logistic regression --- utils/Makefile.am | 4 ++ utils/fast_sparse_vector.h | 16 ++++- utils/json_feature_map_lexer.h | 15 +++++ utils/json_feature_map_lexer.ll | 132 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 utils/json_feature_map_lexer.h create mode 100644 utils/json_feature_map_lexer.ll (limited to 'utils') diff --git a/utils/Makefile.am b/utils/Makefile.am index 46650c75..b7da0f06 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -33,12 +33,16 @@ libutils_a_SOURCES = \ sparse_vector.cc \ timing_stats.cc \ verbose.cc \ + json_feature_map_lexer.cc \ weights.cc if HAVE_CMPH libutils_a_SOURCES += perfect_hash.cc endif +json_feature_map_lexer.cc: json_feature_map_lexer.ll + $(LEX) -s -8 -CF -o$@ $< + phmt_SOURCES = phmt.cc ts_SOURCES = ts.cc m_test_SOURCES = m_test.cc diff --git a/utils/fast_sparse_vector.h b/utils/fast_sparse_vector.h index af832950..68caa704 100644 --- a/utils/fast_sparse_vector.h +++ b/utils/fast_sparse_vector.h @@ -7,8 +7,6 @@ // important: indexes are integers // important: iterators may return elements in any order -#include "config.h" - #include #include #include @@ -16,8 +14,9 @@ #include #include -#include +#include "config.h" +#include #if HAVE_BOOST_ARCHIVE_TEXT_OARCHIVE_HPP #include #endif @@ -119,6 +118,17 @@ class FastSparseVector { std::memcpy(this, &other, sizeof(FastSparseVector)); if (is_remote_) data_.rbmap = new std::map(*data_.rbmap); } + FastSparseVector(std::pair* first, std::pair* last) { + const ptrdiff_t n = last - first; + if (n <= LOCAL_MAX) { + is_remote_ = false; + local_size_ = n; + std::memcpy(data_.local, first, sizeof(std::pair) * n); + } else { + is_remote_ = true; + data_.rbmap = new std::map(first, last); + } + } void erase(int k) { if (is_remote_) { data_.rbmap->erase(k); diff --git a/utils/json_feature_map_lexer.h b/utils/json_feature_map_lexer.h new file mode 100644 index 00000000..3324aa29 --- /dev/null +++ b/utils/json_feature_map_lexer.h @@ -0,0 +1,15 @@ +#ifndef _RULE_LEXER_H_ +#define _RULE_LEXER_H_ + +#include +#include + +#include "sparse_vector.h" + +struct JSONFeatureMapLexer { + typedef void (*FeatureMapCallback)(const std::string& id, const SparseVector& fmap, void* extra); + static void ReadRules(std::istream* in, FeatureMapCallback func, void* extra); +}; + +#endif + diff --git a/utils/json_feature_map_lexer.ll b/utils/json_feature_map_lexer.ll new file mode 100644 index 00000000..372b52f5 --- /dev/null +++ b/utils/json_feature_map_lexer.ll @@ -0,0 +1,132 @@ +%option nounput +%{ + +#include "json_feature_map_lexer.h" +#include "fdict.h" +#include "fast_sparse_vector.h" + +#define YY_DECL int json_fmap_yylex (void) +#undef YY_INPUT +#define YY_INPUT(buf, result, max_size) (result = jfmap_stream->read(buf, max_size).gcount()) +#define YY_SKIP_YYWRAP 1 +int yywrap() { return 1; } + +JSONFeatureMapLexer::FeatureMapCallback json_fmap_callback = NULL; +void* json_fmap_callback_extra = NULL; +std::istream* jfmap_stream = NULL; +bool fl = true; +unsigned spos = 0; +char featname[16000]; +#define MAX_FEATS 20000 +std::pair featmap[MAX_FEATS]; +unsigned curfeat = 0; +std::string instid; + +inline unsigned unicode_escape_to_utf8(uint16_t w1, uint16_t w2, char* putf8) { + uint32_t cp; + if((w1 & 0xfc00) == 0xd800) { + if((w2 & 0xfc00) == 0xdc00) { + cp = 0x10000 + (((static_cast(w1) & 0x3ff) << 10) | (w2 & 0x3ff)); + } else { + abort(); + } + } else { + cp = w1; + } + + + if(cp < 0x80) { + putf8[0] = static_cast(cp); + return 1; + } else if(cp < 0x0800) { + putf8[0] = 0xc0 | ((cp >> 6) & 0x1f); + putf8[1] = 0x80 | (cp & 0x3f); + return 2; + } else if(cp < 0x10000) { + putf8[0] = 0xe0 | ((cp >> 6) & 0x0f); + putf8[1] = 0x80 | ((cp >> 6) & 0x3f); + putf8[2] = 0x80 | (cp & 0x3f); + return 3; + } else if(cp < 0x1fffff) { + putf8[0] = 0xf0 | ((cp >> 18) & 0x07); + putf8[1] = 0x80 | ((cp >> 12) & 0x3f); + putf8[2] = 0x80 | ((cp >> 6) & 0x3f); + putf8[3] = 0x80 | (cp & 0x3f); + return 4; + } else { + abort(); + } + return 0; +} + +%} + +ID [A-Za-z_0-9]+ +HEX_D [a-fA-F0-9] +INT [-]?[0-9]+ +DOUBLE {INT}((\.[0-9]+)?([eE][-+]?[0-9]+)?) +WS [ \t\r\n] +LCB [{] +RCB [}] +UNESCAPED_CH [^\"\\\b\n\r\f\t] + +%x JSON PREVAL STRING JSONVAL POSTVAL DOUBLE +%% + +{ID} { instid = yytext; BEGIN(JSON); } + +{WS}*{LCB}{WS}* { BEGIN(PREVAL); } + +\" { BEGIN(STRING); spos=0; } + +\" { featname[spos] = 0; + featmap[curfeat].first = FD::Convert(featname); + BEGIN(JSONVAL); + } +{UNESCAPED_CH} { featname[spos++] = yytext[0]; } +\\\" { featname[spos++] = '"'; } +\\\\ { featname[spos++] = '\\'; } +\\\/ { featname[spos++] = '/'; } +\\b { } +\\f { } +\\n { } +\\r { } +\\t { } +\\u{HEX_D}{HEX_D}{HEX_D}{HEX_D} { abort(); + } + +{WS}*:{WS}* { BEGIN(DOUBLE); } +{DOUBLE} { featmap[curfeat++].second = strtod(yytext, 0); + BEGIN(POSTVAL); } + +{WS}*,{WS}* { BEGIN(PREVAL); } +{WS}*{RCB}\n* { + const SparseVector x(&featmap[0], &featmap[curfeat]); + json_fmap_callback(instid, x, json_fmap_callback_extra); + curfeat = 0; + BEGIN(INITIAL); + } + +. { std::cerr << "bad input: " << yytext << std::endl; abort(); } + +%% + +void JSONFeatureMapLexer::ReadRules(std::istream* in, FeatureMapCallback func, void* extra) { + json_fmap_callback = func; + json_fmap_callback_extra = extra; + jfmap_stream = in; + json_fmap_yylex(); +} + +#if 0 +void cb(const std::string& id, const SparseVector& fmap, void* extra) { + (void) extra; + static int cc = 0; + cc++; +} + +int main() { + JSONFeatureMapLexer::ReadRules(&std::cin, cb, NULL); +} +#endif + -- cgit v1.2.3