summaryrefslogtreecommitdiff
path: root/extractor
diff options
context:
space:
mode:
authorPaul Baltescu <pauldb89@gmail.com>2013-03-06 11:43:16 +0000
committerPaul Baltescu <pauldb89@gmail.com>2013-03-06 11:43:16 +0000
commit245e705ac7b6fca80a7aa4898f5467053ec2c585 (patch)
tree3034bcadfd9ed3778f1c89f5ea883270a3f64194 /extractor
parent6717daa034bfcfa429275f64d84607ae1568d488 (diff)
Fix memory leak in trie.
Diffstat (limited to 'extractor')
-rw-r--r--extractor/matchings_trie.cc3
-rw-r--r--extractor/mocks/mock_precomputation.h1
-rw-r--r--extractor/precomputation.cc19
-rw-r--r--extractor/precomputation.h2
-rw-r--r--extractor/precomputation_test.cc34
-rw-r--r--extractor/run_extractor.cc15
6 files changed, 10 insertions, 64 deletions
diff --git a/extractor/matchings_trie.cc b/extractor/matchings_trie.cc
index 921ec582..8ea795db 100644
--- a/extractor/matchings_trie.cc
+++ b/extractor/matchings_trie.cc
@@ -14,6 +14,9 @@ void MatchingsTrie::ResetTree(shared_ptr<TrieNode> root) {
for (auto child: root->children) {
ResetTree(child.second);
}
+ if (root->suffix_link != NULL) {
+ root->suffix_link.reset();
+ }
root.reset();
}
}
diff --git a/extractor/mocks/mock_precomputation.h b/extractor/mocks/mock_precomputation.h
index 987bdb2f..9bc72235 100644
--- a/extractor/mocks/mock_precomputation.h
+++ b/extractor/mocks/mock_precomputation.h
@@ -4,6 +4,5 @@
class MockPrecomputation : public Precomputation {
public:
- MOCK_CONST_METHOD0(GetInvertedIndex, const Index&());
MOCK_CONST_METHOD0(GetCollocations, const Index&());
};
diff --git a/extractor/precomputation.cc b/extractor/precomputation.cc
index 8a76beb1..189ac42c 100644
--- a/extractor/precomputation.cc
+++ b/extractor/precomputation.cc
@@ -41,7 +41,6 @@ Precomputation::Precomputation(
for (int j = 1; j <= max_frequent_phrase_len && i + j <= data.size(); ++j) {
pattern.push_back(data[i + j - 1]);
if (frequent_patterns_set.count(pattern)) {
- inverted_index[pattern].push_back(i);
int is_super_frequent = super_frequent_patterns_set.count(pattern);
matchings.push_back(make_tuple(i, j, is_super_frequent));
} else {
@@ -158,19 +157,7 @@ void Precomputation::WriteBinary(const fs::path& filepath) const {
FILE* file = fopen(filepath.string().c_str(), "w");
// TODO(pauldb): Refactor this code.
- int size = inverted_index.size();
- fwrite(&size, sizeof(int), 1, file);
- for (auto entry: inverted_index) {
- size = entry.first.size();
- fwrite(&size, sizeof(int), 1, file);
- fwrite(entry.first.data(), sizeof(int), size, file);
-
- size = entry.second.size();
- fwrite(&size, sizeof(int), 1, file);
- fwrite(entry.second.data(), sizeof(int), size, file);
- }
-
- size = collocations.size();
+ int size = collocations.size();
fwrite(&size, sizeof(int), 1, file);
for (auto entry: collocations) {
size = entry.first.size();
@@ -183,10 +170,6 @@ void Precomputation::WriteBinary(const fs::path& filepath) const {
}
}
-const Index& Precomputation::GetInvertedIndex() const {
- return inverted_index;
-}
-
const Index& Precomputation::GetCollocations() const {
return collocations;
}
diff --git a/extractor/precomputation.h b/extractor/precomputation.h
index 28426bfa..3d44c2a6 100644
--- a/extractor/precomputation.h
+++ b/extractor/precomputation.h
@@ -30,7 +30,6 @@ class Precomputation {
void WriteBinary(const fs::path& filepath) const;
- virtual const Index& GetInvertedIndex() const;
virtual const Index& GetCollocations() const;
static int NON_TERMINAL;
@@ -49,7 +48,6 @@ class Precomputation {
void AddStartPositions(vector<int>& positions, int pos1, int pos2);
void AddStartPositions(vector<int>& positions, int pos1, int pos2, int pos3);
- Index inverted_index;
Index collocations;
};
diff --git a/extractor/precomputation_test.cc b/extractor/precomputation_test.cc
index 9edb29db..6b77b9c0 100644
--- a/extractor/precomputation_test.cc
+++ b/extractor/precomputation_test.cc
@@ -35,40 +35,6 @@ class PrecomputationTest : public Test {
shared_ptr<MockSuffixArray> suffix_array;
};
-TEST_F(PrecomputationTest, TestInvertedIndex) {
- Precomputation precomputation(suffix_array, 100, 3, 10, 5, 1, 4, 2);
- Index inverted_index = precomputation.GetInvertedIndex();
-
- EXPECT_EQ(8, inverted_index.size());
- vector<int> key = {2};
- vector<int> expected_value = {1, 5, 8, 11};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {3};
- expected_value = {2, 6, 9};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {4};
- expected_value = {0, 10};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {5};
- expected_value = {3, 7};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {4, 2};
- expected_value = {0, 10};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {2, 3};
- expected_value = {1, 5, 8};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {3, 5};
- expected_value = {2, 6};
- EXPECT_EQ(expected_value, inverted_index[key]);
- key = {2, 3, 5};
- expected_value = {1, 5};
- EXPECT_EQ(expected_value, inverted_index[key]);
-
- key = {2, 4};
- EXPECT_EQ(0, inverted_index.count(key));
-}
-
TEST_F(PrecomputationTest, TestCollocations) {
Precomputation precomputation(suffix_array, 3, 3, 10, 5, 1, 4, 2);
Index collocations = precomputation.GetCollocations();
diff --git a/extractor/run_extractor.cc b/extractor/run_extractor.cc
index 36dbd7a0..5255737d 100644
--- a/extractor/run_extractor.cc
+++ b/extractor/run_extractor.cc
@@ -31,14 +31,6 @@ namespace fs = boost::filesystem;
namespace po = boost::program_options;
using namespace std;
-void my_pause() {
- cerr << "pausing..." << endl;
- for (int i = 0; i < 10000000; ++i) {
- cerr << endl;
- }
- cerr << "end pause" << endl;
-}
-
int main(int argc, char** argv) {
// TODO(pauldb): Also take arguments from config file.
po::options_description desc("Command line options");
@@ -122,7 +114,7 @@ int main(int argc, char** argv) {
cerr << "Reading alignment took "
<< GetDuration(start_time, stop_time) << " seconds" << endl;
- cerr << "Precomputating collocations..." << endl;
+ cerr << "Precomputing collocations..." << endl;
start_time = Clock::now();
shared_ptr<Precomputation> precomputation = make_shared<Precomputation>(
source_suffix_array,
@@ -150,6 +142,8 @@ int main(int argc, char** argv) {
<< GetDuration(preprocess_start_time, preprocess_stop_time)
<< " seconds" << endl;
+ cerr << "creating grammar extractor" << endl;
+
Clock::time_point extraction_start_time = Clock::now();
vector<shared_ptr<Feature> > features = {
make_shared<TargetGivenSourceCoherent>(),
@@ -176,6 +170,9 @@ int main(int argc, char** argv) {
vm["max_samples"].as<int>(),
vm["tight_phrases"].as<bool>());
+ // Release extra memory used by the initial precomputation.
+ precomputation.reset();
+
int grammar_id = 0;
fs::path grammar_path = vm["grammars"].as<string>();
if (!fs::is_directory(grammar_path)) {