summaryrefslogtreecommitdiff
path: root/utils
diff options
context:
space:
mode:
authorPatrick Simianer <p@simianer.de>2015-02-26 14:24:41 +0100
committerPatrick Simianer <p@simianer.de>2015-02-26 14:24:41 +0100
commit29ddfafb0dea599965e6a881c25b396a6db2f40f (patch)
tree5755ec058361776657041ba088062b086eea6d68 /utils
parent4223261682388944fe1b1cf31b9d51d88f9ad53b (diff)
parent03989754cb2511431e1df6001fca41b3806ad461 (diff)
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'utils')
-rw-r--r--utils/Makefile.am6
-rw-r--r--utils/dedup_corpus.cc21
-rw-r--r--utils/hash.h3
3 files changed, 29 insertions, 1 deletions
diff --git a/utils/Makefile.am b/utils/Makefile.am
index dd74ddc0..c858ac7e 100644
--- a/utils/Makefile.am
+++ b/utils/Makefile.am
@@ -1,4 +1,4 @@
-bin_PROGRAMS = reconstruct_weights atools
+bin_PROGRAMS = reconstruct_weights atools dedup_corpus
noinst_PROGRAMS = \
ts \
@@ -98,6 +98,10 @@ atools_SOURCES = atools.cc
atools_LDADD = libutils.a
atools_LDFLAGS = $(STATIC_FLAGS)
+dedup_corpus_SOURCES = dedup_corpus.cc
+dedup_corpus_LDADD = libutils.a
+dedup_corpus_LDFLAGS = $(STATIC_FLAGS)
+
phmt_SOURCES = phmt.cc
phmt_LDADD = libutils.a $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS)
ts_SOURCES = ts.cc
diff --git a/utils/dedup_corpus.cc b/utils/dedup_corpus.cc
new file mode 100644
index 00000000..818a6ffa
--- /dev/null
+++ b/utils/dedup_corpus.cc
@@ -0,0 +1,21 @@
+#include <iostream>
+#include "hash.h"
+
+using namespace std;
+
+#define INITIAL_SIZE 20000000
+
+int main(int argc, char **argv) {
+ if (argc != 1) {
+ cerr << "Usage: " << argv[0] << " < file.txt\n";
+ return 1;
+ }
+ SPARSE_HASH_SET<uint64_t> seen(INITIAL_SIZE);
+ string line;
+ while(getline(cin, line)) {
+ uint64_t h = cdec::MurmurHash3_64(&line[0], line.size(), 17);
+ if (seen.insert(h).second)
+ cout << line << '\n';
+ }
+}
+
diff --git a/utils/hash.h b/utils/hash.h
index 24d2b6ad..7de4db6d 100644
--- a/utils/hash.h
+++ b/utils/hash.h
@@ -13,7 +13,9 @@
# include <sparsehash/dense_hash_map>
# include <sparsehash/dense_hash_set>
# include <sparsehash/sparse_hash_map>
+# include <sparsehash/sparse_hash_set>
# define SPARSE_HASH_MAP google::sparse_hash_map
+# define SPARSE_HASH_SET google::sparse_hash_set
# define HASH_MAP google::dense_hash_map
# define HASH_SET google::dense_hash_set
# define HASH_MAP_DELETED(h,deleted) do { (h).set_deleted_key(deleted); } while(0)
@@ -29,6 +31,7 @@
namespace std { using std::tr1::unordered_map; using std::tr1::unordered_set; }
#endif
# define SPARSE_HASH_MAP std::unordered_map
+# define SPARSE_HASH_SET std::unordered_set
# define HASH_MAP std::unordered_map
# define HASH_SET std::unordered_set
# define HASH_MAP_DELETED(h,deleted)