summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorgraehl <graehl@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-07-24 21:17:51 +0000
committergraehl <graehl@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-07-24 21:17:51 +0000
commit1fbdaa4d49acf90d2124aef8810d723ddbb0dad5 (patch)
treee3871052691166849e88ff159a64dafff9373e9b
parent01a4f22830758857df1f991b8377553afba60c59 (diff)
sparse_vector non-const [], consistently use default ctor, operator bool instead of explicit 0
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@398 ec762483-ff6d-05da-a07a-a48fb63a330f
-rwxr-xr-xdecoder/feature_vector.h14
-rwxr-xr-xdecoder/have_64_bits.h17
-rwxr-xr-xdecoder/murmur_hash.h187
3 files changed, 218 insertions, 0 deletions
diff --git a/decoder/feature_vector.h b/decoder/feature_vector.h
new file mode 100755
index 00000000..1b272506
--- /dev/null
+++ b/decoder/feature_vector.h
@@ -0,0 +1,14 @@
+#ifndef _FEATURE_VECTOR_H_
+#define _FEATURE_VECTOR_H_
+
+#include <vector>
+#include "sparse_vector.h"
+#include "fdict.h"
+
+typedef double Featval;
+typedef SparseVectorList<Featval> FeatureVectorList;
+typedef SparseVector<Featval> FeatureVector;
+typedef SparseVector<Featval> WeightVector;
+typedef std::vector<Featval> DenseWeightVector;
+
+#endif
diff --git a/decoder/have_64_bits.h b/decoder/have_64_bits.h
new file mode 100755
index 00000000..d1e6064f
--- /dev/null
+++ b/decoder/have_64_bits.h
@@ -0,0 +1,17 @@
+#ifndef HAVE_64_BITS_H
+#define HAVE_64_BITS_H
+
+#include <stdint.h>
+
+#undef HAVE_64_BITS
+
+#if INTPTR_MAX == INT32_MAX
+# define HAVE_64_BITS 0
+#elif INTPTR_MAX >= INT64_MAX
+# define HAVE_64_BITS 1
+#else
+# error "couldn't tell if HAVE_64_BITS from INTPTR_MAX INT32_MAX INT64_MAX"
+#endif
+
+
+#endif
diff --git a/decoder/murmur_hash.h b/decoder/murmur_hash.h
new file mode 100755
index 00000000..8ddad339
--- /dev/null
+++ b/decoder/murmur_hash.h
@@ -0,0 +1,187 @@
+#ifndef _MURMUR_HASH_H_
+#define _MURMUR_HASH_H_
+
+//NOTE: quite fast, nice collision properties, but endian dependent hash values
+
+#include <stdint.h>
+
+// MurmurHash2, by Austin Appleby
+
+static const uint32_t DEFAULT_SEED=2654435769U;
+
+#if sizeof(void *)==8
+typedef uint64_t MurmurInt;
+//MurmurInt MurmurHash(void const *key, int len, uint32_t seed=DEFAULT_SEED);
+
+inline uint64_t MurmurHash64( const void * key, int len, unsigned int seed=DEFAULT_SEED )
+{
+ const uint64_t m = 0xc6a4a7935bd1e995;
+ const int r = 47;
+
+ uint64_t h = seed ^ (len * m);
+
+ const uint64_t * data = (const uint64_t *)key;
+ const uint64_t * end = data + (len/8);
+
+ while(data != end)
+ {
+ uint64_t k = *data++;
+
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+
+ h ^= k;
+ h *= m;
+ }
+
+ const unsigned char * data2 = (const unsigned char*)data;
+
+ switch(len & 7)
+ {
+ case 7: h ^= uint64_t(data2[6]) << 48;
+ case 6: h ^= uint64_t(data2[5]) << 40;
+ case 5: h ^= uint64_t(data2[4]) << 32;
+ case 4: h ^= uint64_t(data2[3]) << 24;
+ case 3: h ^= uint64_t(data2[2]) << 16;
+ case 2: h ^= uint64_t(data2[1]) << 8;
+ case 1: h ^= uint64_t(data2[0]);
+ h *= m;
+ };
+
+ h ^= h >> r;
+ h *= m;
+ h ^= h >> r;
+
+ return h;
+}
+
+inline uint32_t MurmurHash2(void const *key, int len, uint32_t seed=DEFAULT_SEED)
+{
+ return (uint32_t) MurmurHash64(key,len,seed);
+}
+
+typedef uint64_t MurmurInt;
+inline MurmurInt MurmurHash(void const *key, int len, uint32_t seed=DEFAULT_SEED)
+{
+ return MurmurHash64(key,len,seed);
+}
+
+#else // 32-bit
+typedef uint32_t MurmurInt;
+
+
+// Note - This code makes a few assumptions about how your machine behaves -
+// 1. We can read a 4-byte value from any address without crashing
+// 2. sizeof(int) == 4
+inline uint32_t MurmurHash2 ( const void * key, int len, uint32_t seed=DEFAULT_SEED)
+{
+ // 'm' and 'r' are mixing constants generated offline.
+ // They're not really 'magic', they just happen to work well.
+
+ const uint32_t m = 0x5bd1e995;
+ const int r = 24;
+
+ // Initialize the hash to a 'random' value
+
+ uint32_t h = seed ^ len;
+
+ // Mix 4 bytes at a time into the hash
+
+ const unsigned char * data = (const unsigned char *)key;
+
+ while(len >= 4)
+ {
+ uint32_t k = *(uint32_t *)data;
+
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+
+ h *= m;
+ h ^= k;
+
+ data += 4;
+ len -= 4;
+ }
+
+ // Handle the last few bytes of the input array
+
+ switch(len)
+ {
+ case 3: h ^= data[2] << 16;
+ case 2: h ^= data[1] << 8;
+ case 1: h ^= data[0];
+ h *= m;
+ };
+
+ // Do a few final mixes of the hash to ensure the last few
+ // bytes are well-incorporated.
+
+ h ^= h >> 13;
+ h *= m;
+ h ^= h >> 15;
+
+ return h;
+}
+
+inline MurmurInt MurmurHash ( const void * key, int len, uint32_t seed=DEFAULT_SEED) {
+ return MurmurHash32(key,len,seed);
+}
+
+// 64-bit hash for 32-bit platforms
+
+inline uint64_t MurmurHash64 ( const void * key, int len, uint32_t seed=DEFAULT_SEED)
+{
+ const uint32_t m = 0x5bd1e995;
+ const int r = 24;
+
+ uint32_t h1 = seed ^ len;
+ uint32_t h2 = 0;
+
+ const uint32_t * data = (const uint32_t *)key;
+
+ while(len >= 8)
+ {
+ uint32_t k1 = *data++;
+ k1 *= m; k1 ^= k1 >> r; k1 *= m;
+ h1 *= m; h1 ^= k1;
+ len -= 4;
+
+ uint32_t k2 = *data++;
+ k2 *= m; k2 ^= k2 >> r; k2 *= m;
+ h2 *= m; h2 ^= k2;
+ len -= 4;
+ }
+
+ if(len >= 4)
+ {
+ uint32_t k1 = *data++;
+ k1 *= m; k1 ^= k1 >> r; k1 *= m;
+ h1 *= m; h1 ^= k1;
+ len -= 4;
+ }
+
+ switch(len)
+ {
+ case 3: h2 ^= ((unsigned char*)data)[2] << 16;
+ case 2: h2 ^= ((unsigned char*)data)[1] << 8;
+ case 1: h2 ^= ((unsigned char*)data)[0];
+ h2 *= m;
+ };
+
+ h1 ^= h2 >> 18; h1 *= m;
+ h2 ^= h1 >> 22; h2 *= m;
+ h1 ^= h2 >> 17; h1 *= m;
+ h2 ^= h1 >> 19; h2 *= m;
+
+ uint64_t h = h1;
+
+ h = (h << 32) | h2;
+
+ return h;
+}
+
+#endif
+
+#endif