blob: 243f7e2c299ef36e2ce9393513b337d8fd155ac6 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
#ifndef _CORPUS_HH
#define _CORPUS_HH
#include <vector>
#include <string>
#include <map>
#include <boost/shared_ptr.hpp>
#include <boost/ptr_container/ptr_vector.hpp>
////////////////////////////////////////////////////////////////
// Corpus
////////////////////////////////////////////////////////////////
typedef int Term;
typedef std::vector<Term> Document;
typedef std::vector<Term> Terms;
class Corpus {
public:
typedef boost::ptr_vector<Document>::const_iterator const_iterator;
public:
Corpus();
~Corpus() {}
unsigned read(const std::string &filename);
const_iterator begin() const { return m_documents.begin(); }
const_iterator end() const { return m_documents.end(); }
const Document& at(size_t i) const { return m_documents.at(i); }
int num_documents() const { return m_documents.size(); }
int num_terms() const { return m_num_terms; }
int num_types() const { return m_num_types; }
protected:
int m_num_terms, m_num_types;
boost::ptr_vector<Document> m_documents;
};
typedef int DocumentId;
struct DocumentTerm {
DocumentTerm(DocumentId d, Term t) : term(t), doc(d) {}
Term term;
DocumentId doc;
};
typedef std::vector<DocumentTerm> DocumentTerms;
class TestCorpus {
public:
typedef boost::ptr_vector<DocumentTerms>::const_iterator const_iterator;
public:
TestCorpus();
~TestCorpus() {}
void read(const std::string &filename);
const_iterator begin() const { return m_lines.begin(); }
const_iterator end() const { return m_lines.end(); }
int num_instances() const { return m_lines.size(); }
protected:
boost::ptr_vector<DocumentTerms> m_lines;
};
class TermBackoff {
public:
typedef std::vector<Term> dictionary_type;
typedef dictionary_type::const_iterator const_iterator;
public:
TermBackoff() : m_backoff_order(-1) {}
~TermBackoff() {}
void read(const std::string &filename);
const_iterator begin() const { return m_dict.begin(); }
const_iterator end() const { return m_dict.end(); }
const Term& operator[](const Term& t) const {
assert(t < static_cast<int>(m_dict.size()));
return m_dict[t];
}
int order() const { return m_backoff_order; }
// int levels() const { return m_terms_at_order.size(); }
bool is_null(const Term& term) const { return term < 0; }
int terms_at_level(int level) const {
assert (level < (int)m_terms_at_order.size());
return m_terms_at_order[level];
}
int size() const { return m_dict.size(); }
protected:
dictionary_type m_dict;
int m_backoff_order;
std::vector<int> m_terms_at_order;
};
typedef boost::shared_ptr<TermBackoff> TermBackoffPtr;
#endif // _CORPUS_HH
|
