blob: c2f3713040709cd301ff4809c32e0da9a4c5b5c9 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#ifndef _CORPUS_HH
#define _CORPUS_HH
#include <vector>
#include <string>
#include <map>
#include <boost/shared_ptr.hpp>
#include <boost/ptr_container/ptr_vector.hpp>
////////////////////////////////////////////////////////////////
// Corpus
////////////////////////////////////////////////////////////////
typedef int Term;
typedef std::vector<Term> Document;
typedef std::vector<Term> Terms;
class Corpus {
public:
typedef boost::ptr_vector<Document>::const_iterator const_iterator;
public:
Corpus();
virtual ~Corpus() {}
unsigned read(const std::string &filename);
const_iterator begin() const { return m_documents.begin(); }
const_iterator end() const { return m_documents.end(); }
const Document& at(size_t i) const { return m_documents.at(i); }
int num_documents() const { return m_documents.size(); }
int num_terms() const { return m_num_terms; }
int num_types() const { return m_num_types; }
protected:
int m_num_terms, m_num_types;
boost::ptr_vector<Document> m_documents;
};
typedef int DocumentId;
struct DocumentTerm {
DocumentTerm(DocumentId d, Term t) : term(t), doc(d) {}
Term term;
DocumentId doc;
};
typedef std::vector<DocumentTerm> DocumentTerms;
class TestCorpus {
public:
typedef boost::ptr_vector<DocumentTerms>::const_iterator const_iterator;
public:
TestCorpus();
~TestCorpus() {}
void read(const std::string &filename);
const_iterator begin() const { return m_lines.begin(); }
const_iterator end() const { return m_lines.end(); }
int num_instances() const { return m_lines.size(); }
protected:
boost::ptr_vector<DocumentTerms> m_lines;
};
class TermBackoff {
public:
typedef std::vector<Term> dictionary_type;
typedef dictionary_type::const_iterator const_iterator;
const static int NullBackoff=-1;
public:
TermBackoff() { order(1); }
~TermBackoff() {}
void read(const std::string &filename);
const_iterator begin() const { return m_dict.begin(); }
const_iterator end() const { return m_dict.end(); }
const Term& operator[](const Term& t) const {
assert(t < static_cast<int>(m_dict.size()));
return m_dict[t];
}
Term& operator[](const Term& t) {
if (t >= static_cast<int>(m_dict.size()))
m_dict.resize(t+1, -1);
return m_dict[t];
}
bool has_backoff(const Term& t) {
return t >= 0 && t < static_cast<int>(m_dict.size()) && m_dict[t] >= 0;
}
int order() const { return m_backoff_order; }
void order(int o) {
if (o >= (int)m_terms_at_order.size())
m_terms_at_order.resize(o, 0);
m_backoff_order = o;
}
// int levels() const { return m_terms_at_order.size(); }
bool is_null(const Term& term) const { return term < 0; }
int terms_at_level(int level) const {
assert (level < (int)m_terms_at_order.size());
return m_terms_at_order.at(level);
}
int& terms_at_level(int level) {
assert (level < (int)m_terms_at_order.size());
return m_terms_at_order.at(level);
}
int size() const { return m_dict.size(); }
protected:
dictionary_type m_dict;
int m_backoff_order;
std::vector<int> m_terms_at_order;
};
typedef boost::shared_ptr<TermBackoff> TermBackoffPtr;
#endif // _CORPUS_HH
|
