blob: 4dd4789c3ac7e2c88ddc52c31367608fc1d89b44 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
#include <iostream>
#include <vector>
#include <map>
#include "timer.h"
#include "crp.h"
#include "sampler.h"
#include "tdict.h"
Dict TD::dict_;
std::string TD::empty = "";
std::string TD::space = " ";
using namespace std;
void ShowTopWords(const map<WordID, int>& counts) {
multimap<int, WordID> ms;
for (map<WordID,int>::const_iterator it = counts.begin(); it != counts.end(); ++it)
ms.insert(make_pair(it->second, it->first));
int cc = 0;
for (multimap<int, WordID>::reverse_iterator it = ms.rbegin(); it != ms.rend(); ++it) {
cerr << it->first << ':' << TD::Convert(it->second) << " ";
++cc;
if (cc==12) break;
}
cerr << endl;
}
int main(int argc, char** argv) {
if (argc != 3) {
cerr << "Usage: " << argv[0] << " num-classes num-samples\n";
return 1;
}
const int num_classes = atoi(argv[1]);
const int num_iterations = atoi(argv[2]);
const int burnin_size = num_iterations * 0.666;
if (num_classes < 2) {
cerr << "Must request more than 1 class\n";
return 1;
}
if (num_iterations < 5) {
cerr << "Must request more than 5 iterations\n";
return 1;
}
cerr << "CLASSES: " << num_classes << endl;
char* buf = new char[800000];
vector<vector<int> > wji; // w[j][i] - observed word i of doc j
vector<vector<int> > zji; // z[j][i] - topic assignment for word i of doc j
cerr << "READING DOCUMENTS\n";
while(cin) {
cin.getline(buf, 800000);
if (buf[0] == 0) continue;
wji.push_back(vector<WordID>());
TD::ConvertSentence(buf, &wji.back());
}
cerr << "READ " << wji.size() << " DOCUMENTS\n";
MT19937 rng;
cerr << "INITIALIZING RANDOM TOPIC ASSIGNMENTS\n";
zji.resize(wji.size());
double beta = 0.01;
double alpha = 0.001;
vector<CRP<int> > dr(zji.size(), CRP<int>(beta)); // dr[i] describes the probability of using a topic in document i
vector<CRP<int> > wr(num_classes, CRP<int>(alpha)); // wr[k] describes the probability of generating a word in topic k
int random_topic = rng.next() * num_classes;
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
zj.resize(num_words);
for (int i = 0; i < num_words; ++i) {
if (random_topic == num_classes) { --random_topic; }
zj[i] = random_topic;
const int word = wj[i];
dr[j].increment(random_topic);
wr[random_topic].increment(word);
}
}
cerr << "SAMPLING\n";
vector<map<WordID, int> > t2w(num_classes);
bool needline = false;
Timer timer;
SampleSet ss;
ss.resize(num_classes);
double total_time = 0;
for (int iter = 0; iter < num_iterations; ++iter) {
if (iter && iter % 10 == 0) {
total_time += timer.Elapsed();
timer.Reset();
cerr << '.'; needline=true;
prob_t lh = prob_t::One();
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
for (int i = 0; i < num_words; ++i) {
const int word = wj[i];
const int cur_topic = zj[i];
lh *= dr[j].prob(cur_topic);
lh *= wr[cur_topic].prob(word);
if (iter > burnin_size) {
++t2w[cur_topic][word];
}
}
}
if (iter && iter % 200 == 0) { cerr << " [ITER=" << iter << " SEC/SAMPLE=" << (total_time / 200) << " LLH=" << log(lh) << "]\n"; needline=false; total_time=0; }
//cerr << "ITERATION " << iter << " LOG LIKELIHOOD: " << log(lh) << endl;
}
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
for (int i = 0; i < num_words; ++i) {
const int word = wj[i];
const int cur_topic = zj[i];
dr[j].decrement(cur_topic);
wr[cur_topic].decrement(word);
for (int k = 0; k < num_classes; ++k) {
ss[k]= dr[j].prob(k) * wr[k].prob(word);
}
const int new_topic = rng.SelectSample(ss);
dr[j].increment(new_topic);
wr[new_topic].increment(word);
zj[i] = new_topic;
}
}
}
if (needline) cerr << endl;
#if 0
for (int j = 0; j < zji.size(); ++j) {
const size_t num_words = wji[j].size();
vector<int>& zj = zji[j];
const vector<int>& wj = wji[j];
zj.resize(num_words);
for (int i = 0; i < num_words; ++i) {
cout << TD::Convert(wj[i]) << '(' << zj[i] << ") ";
}
cout << endl;
}
#endif
for (int i = 0; i < num_classes; ++i) {
cerr << "---------------------------------\n";
ShowTopWords(t2w[i]);
}
return 0;
}
|
