1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
#include <iostream>
#include <vector>
#include <map>
#include <string>
#include "timer.h"
#include "crp.h"
#include "ccrp.h"
#include "sampler.h"
#include "tdict.h"
const size_t MAX_DOC_LEN_CHARS = 10000000;
using namespace std;
void ShowTopWordsForTopic(const map<WordID, int>& counts) {
multimap<int, WordID> ms;
for (map<WordID,int>::const_iterator it = counts.begin(); it != counts.end(); ++it)
ms.insert(make_pair(it->second, it->first));
int cc = 0;
for (multimap<int, WordID>::reverse_iterator it = ms.rbegin(); it != ms.rend(); ++it) {
cerr << it->first << ':' << TD::Convert(it->second) << " ";
++cc;
if (cc==20) break;
}
cerr << endl;
}
void tc() {
MT19937 rng;
CCRP<string> crp(0.1, 5);
double un = 0.25;
int tt = 0;
tt += crp.increment("hi", un, &rng);
tt += crp.increment("foo", un, &rng);
tt += crp.increment("bar", un, &rng);
tt += crp.increment("bar", un, &rng);
tt += crp.increment("bar", un, &rng);
tt += crp.increment("bar", un, &rng);
tt += crp.increment("bar", un, &rng);
tt += crp.increment("bar", un, &rng);
tt += crp.increment("bar", un, &rng);
cout << "tt=" << tt << endl;
cout << crp << endl;
cout << " P(bar)=" << crp.prob("bar", un) << endl;
cout << " P(hi)=" << crp.prob("hi", un) << endl;
cout << " P(baz)=" << crp.prob("baz", un) << endl;
cout << " P(foo)=" << crp.prob("foo", un) << endl;
double x = crp.prob("bar", un) + crp.prob("hi", un) + crp.prob("baz", un) + crp.prob("foo", un);
cout << " tot=" << x << endl;
tt += crp.decrement("hi", &rng);
tt += crp.decrement("bar", &rng);
cout << crp << endl;
tt += crp.decrement("bar", &rng);
cout << crp << endl;
cout << "tt=" << tt << endl;
cout << crp.log_crp_prob() << endl;
}
int main(int argc, char** argv) {
tc();
if (argc != 3) {
cerr << "Usage: " << argv[0] << " num-classes num-samples\n";
return 1;
}
const int num_classes = atoi(argv[1]);
const int num_iterations = atoi(argv[2]);
const int burnin_size = num_iterations * 0.9;
if (num_classes < 2) {
cerr << "Must request more than 1 class\n";
return 1;
}
if (num_iterations < 5) {
cerr << "Must request more than 5 iterations\n";
return 1;
}
cerr << "CLASSES: " << num_classes << endl;
char* buf = new char[MAX_DOC_LEN_CHARS];
vector<vector<int> > wji; // w[j][i] - observed word i of doc j
vector<vector<int> > zji; // z[j][i] - topic assignment for word i of doc j
cerr << "READING DOCUMENTS\n";
while(cin) {
cin.getline(buf, MAX_DOC_LEN_CHARS);
if (buf[0] == 0) continue;
wji.push_back(vector<WordID>());
TD::ConvertSentence(buf, &wji.back());
}
cerr << "READ " << wji.size() << " DOCUMENTS\n";
MT19937 rng;
cerr << "INITIALIZING RANDOM TOPIC ASSIGNMENTS\n";
zji.resize(wji.size());
double disc = 0.05;
double beta = 10.0;
double alpha = 50.0;
double uniform_topic = 1.0 / num_classes;
double uniform_word = 1.0 / TD::NumWords();
vector<CCRP<int> > dr(zji.size(), CCRP<int>(disc, beta)); // dr[i] describes the probability of using a topic in document i
vector<CCRP<int> > wr(num_classes, CCRP<int>(disc, alpha)); // wr[k] describes the probability of generating a word in topic k
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
zj.resize(num_words);
for (int i = 0; i < num_words; ++i) {
int random_topic = rng.next() * num_classes;
if (random_topic == num_classes) { --random_topic; }
zj[i] = random_topic;
const int word = wj[i];
dr[j].increment(random_topic, uniform_topic, &rng);
wr[random_topic].increment(word, uniform_word, &rng);
}
}
cerr << "SAMPLING\n";
vector<map<WordID, int> > t2w(num_classes);
Timer timer;
SampleSet<double> ss;
ss.resize(num_classes);
double total_time = 0;
for (int iter = 0; iter < num_iterations; ++iter) {
cerr << '.';
if (iter && iter % 10 == 0) {
total_time += timer.Elapsed();
timer.Reset();
double llh = 0;
for (int j = 0; j < dr.size(); ++j)
llh += dr[j].log_crp_prob();
for (int j = 0; j < wr.size(); ++j)
llh += wr[j].log_crp_prob();
cerr << " [LLH=" << llh << " I=" << iter << "]\n";
}
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
for (int i = 0; i < num_words; ++i) {
const int word = wj[i];
const int cur_topic = zj[i];
dr[j].decrement(cur_topic, &rng);
wr[cur_topic].decrement(word, &rng);
for (int k = 0; k < num_classes; ++k) {
ss[k]= dr[j].prob(k, uniform_topic) * wr[k].prob(word, uniform_word);
}
const int new_topic = rng.SelectSample(ss);
dr[j].increment(new_topic, uniform_topic, &rng);
wr[new_topic].increment(word, uniform_word, &rng);
zj[i] = new_topic;
if (iter > burnin_size) {
++t2w[cur_topic][word];
}
}
}
}
for (int i = 0; i < num_classes; ++i) {
cerr << "---------------------------------\n";
ShowTopWordsForTopic(t2w[i]);
}
cerr << "-------------\n";
#if 0
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
zj.resize(num_words);
for (int i = 0; i < num_words; ++i) {
cerr << TD::Convert(wji[j][i]) << '(' << zj[i] << ") ";
}
cerr << endl;
}
#endif
return 0;
}
|
