summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--training/mpi_batch_optimize.cc35
-rw-r--r--utils/filelib.cc19
-rw-r--r--utils/filelib.h5
-rw-r--r--utils/stringlib.cc363
-rw-r--r--utils/stringlib.h3
-rw-r--r--utils/ts.cc6
6 files changed, 427 insertions, 4 deletions
diff --git a/training/mpi_batch_optimize.cc b/training/mpi_batch_optimize.cc
index cc5953f6..0ba8c530 100644
--- a/training/mpi_batch_optimize.cc
+++ b/training/mpi_batch_optimize.cc
@@ -22,6 +22,7 @@ namespace mpi = boost::mpi;
#include "ff_register.h"
#include "decoder.h"
#include "filelib.h"
+#include "stringlib.h"
#include "optimize.h"
#include "fdict.h"
#include "weights.h"
@@ -42,6 +43,7 @@ bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
("correction_buffers,M", po::value<int>()->default_value(10), "Number of gradients for LBFGS to maintain in memory")
("gaussian_prior,p","Use a Gaussian prior on the weights")
("means,u", po::value<string>(), "File containing the means for Gaussian prior")
+ ("per_sentence_grammar_scratch,P", po::value<string>(), "(Optional) location of scratch space to copy per-sentence grammars for fast access, useful if a RAM disk is available")
("sigma_squared", po::value<double>()->default_value(1.0), "Sigma squared term for spherical Gaussian prior");
po::options_description clo("Command line options");
clo.add_options()
@@ -186,6 +188,36 @@ struct VectorPlus : public binary_function<vector<T>, vector<T>, vector<T> > {
}
};
+void MovePerSentenceGrammars(const string& root, int size, int rank, vector<string>* c) {
+ if (!DirectoryExists(root)) {
+ cerr << "Can't find scratch space at " << root << endl;
+ abort();
+ }
+ ostringstream os;
+ os << root << "/psg." << size << "_of_" << rank;
+ const string path = os.str();
+ MkDirP(path);
+ string sent;
+ map<string, string> attr;
+ for (unsigned i = 0; i < c->size(); ++i) {
+ sent = (*c)[i];
+ attr.clear();
+ ProcessAndStripSGML(&sent, &attr);
+ map<string, string>::iterator it = attr.find("grammar");
+ if (it != attr.end()) {
+ string src_file = it->second;
+ bool is_gzipped = (src_file.size() > 3) && (src_file.rfind(".gz") == (src_file.size() - 3));
+ string new_name = path + "/" + md5(sent);
+ if (is_gzipped) new_name += ".gz";
+ CopyFile(src_file, new_name);
+ it->second = new_name;
+ }
+ ostringstream ns;
+ ns << SGMLOpenSegTag(attr) << ' ' << sent << " </seg>";
+ (*c)[i] = ns.str();
+ }
+}
+
int main(int argc, char** argv) {
#ifdef HAVE_MPI
mpi::environment env(argc, argv);
@@ -257,6 +289,9 @@ int main(int argc, char** argv) {
ReadTrainingCorpus(conf["training_data"].as<string>(), rank, size, &corpus);
assert(corpus.size() > 0);
+ if (conf.count("per_sentence_grammar_scratch"))
+ MovePerSentenceGrammars(conf["per_sentence_grammar_scratch"].as<string>(), rank, size, &corpus);
+
TrainingObserver observer;
while (!converged) {
observer.Reset();
diff --git a/utils/filelib.cc b/utils/filelib.cc
index a0969b1a..d206fc19 100644
--- a/utils/filelib.cc
+++ b/utils/filelib.cc
@@ -2,6 +2,12 @@
#include <unistd.h>
#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <cstdlib>
+#include <cstdio>
+#include <sys/stat.h>
+#include <sys/types.h>
using namespace std;
@@ -32,3 +38,16 @@ void MkDirP(const string& dir) {
}
}
+#if 0
+void CopyFile(const string& inf, const string& outf) {
+ WriteFile w(outf);
+ CopyFile(inf,*w);
+}
+#else
+void CopyFile(const string& inf, const string& outf) {
+ ofstream of(outf.c_str(), fstream::trunc|fstream::binary);
+ ifstream in(inf.c_str(), fstream::binary);
+ of << in.rdbuf();
+}
+#endif
+
diff --git a/utils/filelib.h b/utils/filelib.h
index a8622246..bb6e7415 100644
--- a/utils/filelib.h
+++ b/utils/filelib.h
@@ -113,9 +113,6 @@ inline void CopyFile(std::string const& inf,std::ostream &out) {
CopyFile(*r,out);
}
-inline void CopyFile(std::string const& inf,std::string const& outf) {
- WriteFile w(outf);
- CopyFile(inf,*w);
-}
+void CopyFile(std::string const& inf,std::string const& outf);
#endif
diff --git a/utils/stringlib.cc b/utils/stringlib.cc
index ade02ca9..3a56965c 100644
--- a/utils/stringlib.cc
+++ b/utils/stringlib.cc
@@ -90,3 +90,366 @@ void ProcessAndStripSGML(string* pline, map<string, string>* out) {
}
}
+string SGMLOpenSegTag(const map<string, string>& attr) {
+ ostringstream os;
+ os << "<seg";
+ for (map<string,string>::const_iterator it = attr.begin(); it != attr.end(); ++it)
+ os << ' ' << it->first << '=' << '"' << it->second << '"';
+ os << '>';
+ return os.str();
+}
+
+class MD5 {
+public:
+ typedef unsigned int size_type; // must be 32bit
+
+ MD5();
+ MD5(const std::string& text);
+ void update(const unsigned char *buf, size_type length);
+ void update(const char *buf, size_type length);
+ MD5& finalize();
+ std::string hexdigest() const;
+
+private:
+ void init();
+ typedef unsigned char uint1; // 8bit
+ typedef unsigned int uint4; // 32bit
+ enum {blocksize = 64}; // VC6 won't eat a const static int here
+
+ void transform(const uint1 block[blocksize]);
+ static void decode(uint4 output[], const uint1 input[], size_type len);
+ static void encode(uint1 output[], const uint4 input[], size_type len);
+
+ bool finalized;
+ uint1 buffer[blocksize]; // bytes that didn't fit in last 64 byte chunk
+ uint4 count[2]; // 64bit counter for number of bits (lo, hi)
+ uint4 state[4]; // digest so far
+ uint1 digest[16]; // the result
+
+ // low level logic operations
+ static inline uint4 F(uint4 x, uint4 y, uint4 z);
+ static inline uint4 G(uint4 x, uint4 y, uint4 z);
+ static inline uint4 H(uint4 x, uint4 y, uint4 z);
+ static inline uint4 I(uint4 x, uint4 y, uint4 z);
+ static inline uint4 rotate_left(uint4 x, int n);
+ static inline void FF(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac);
+ static inline void GG(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac);
+ static inline void HH(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac);
+ static inline void II(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac);
+};
+
+// Constants for MD5Transform routine.
+#define S11 7
+#define S12 12
+#define S13 17
+#define S14 22
+#define S21 5
+#define S22 9
+#define S23 14
+#define S24 20
+#define S31 4
+#define S32 11
+#define S33 16
+#define S34 23
+#define S41 6
+#define S42 10
+#define S43 15
+#define S44 21
+
+///////////////////////////////////////////////
+
+// F, G, H and I are basic MD5 functions.
+inline MD5::uint4 MD5::F(uint4 x, uint4 y, uint4 z) {
+ return (x&y) | (~x&z);
+}
+
+inline MD5::uint4 MD5::G(uint4 x, uint4 y, uint4 z) {
+ return (x&z) | (y&~z);
+}
+
+inline MD5::uint4 MD5::H(uint4 x, uint4 y, uint4 z) {
+ return x^y^z;
+}
+
+inline MD5::uint4 MD5::I(uint4 x, uint4 y, uint4 z) {
+ return y ^ (x | ~z);
+}
+
+// rotate_left rotates x left n bits.
+inline MD5::uint4 MD5::rotate_left(uint4 x, int n) {
+ return (x << n) | (x >> (32-n));
+}
+
+// FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
+// Rotation is separate from addition to prevent recomputation.
+inline void MD5::FF(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) {
+ a = rotate_left(a+ F(b,c,d) + x + ac, s) + b;
+}
+
+inline void MD5::GG(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) {
+ a = rotate_left(a + G(b,c,d) + x + ac, s) + b;
+}
+
+inline void MD5::HH(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) {
+ a = rotate_left(a + H(b,c,d) + x + ac, s) + b;
+}
+
+inline void MD5::II(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) {
+ a = rotate_left(a + I(b,c,d) + x + ac, s) + b;
+}
+
+//////////////////////////////////////////////
+
+// default ctor, just initailize
+MD5::MD5()
+{
+ init();
+}
+
+//////////////////////////////////////////////
+
+// nifty shortcut ctor, compute MD5 for string and finalize it right away
+MD5::MD5(const std::string &text)
+{
+ init();
+ update(text.c_str(), text.length());
+ finalize();
+}
+
+//////////////////////////////
+
+void MD5::init()
+{
+ finalized=false;
+
+ count[0] = 0;
+ count[1] = 0;
+
+ // load magic initialization constants.
+ state[0] = 0x67452301;
+ state[1] = 0xefcdab89;
+ state[2] = 0x98badcfe;
+ state[3] = 0x10325476;
+}
+
+//////////////////////////////
+
+// decodes input (unsigned char) into output (uint4). Assumes len is a multiple of 4.
+void MD5::decode(uint4 output[], const uint1 input[], size_type len)
+{
+ for (unsigned int i = 0, j = 0; j < len; i++, j += 4)
+ output[i] = ((uint4)input[j]) | (((uint4)input[j+1]) << 8) |
+ (((uint4)input[j+2]) << 16) | (((uint4)input[j+3]) << 24);
+}
+
+//////////////////////////////
+
+// encodes input (uint4) into output (unsigned char). Assumes len is
+// a multiple of 4.
+void MD5::encode(uint1 output[], const uint4 input[], size_type len)
+{
+ for (size_type i = 0, j = 0; j < len; i++, j += 4) {
+ output[j] = input[i] & 0xff;
+ output[j+1] = (input[i] >> 8) & 0xff;
+ output[j+2] = (input[i] >> 16) & 0xff;
+ output[j+3] = (input[i] >> 24) & 0xff;
+ }
+}
+
+//////////////////////////////
+
+// apply MD5 algo on a block
+void MD5::transform(const uint1 block[blocksize])
+{
+ uint4 a = state[0], b = state[1], c = state[2], d = state[3], x[16];
+ decode (x, block, blocksize);
+
+ /* Round 1 */
+ FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
+ FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
+ FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
+ FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
+ FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
+ FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
+ FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
+ FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
+ FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
+ FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
+ FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
+ FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
+ FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
+ FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
+ FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
+ FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
+
+ /* Round 2 */
+ GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
+ GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
+ GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
+ GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
+ GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
+ GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */
+ GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
+ GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
+ GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
+ GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
+ GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
+ GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */
+ GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
+ GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */
+ GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */
+ GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */
+
+ /* Round 3 */
+ HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
+ HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
+ HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
+ HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
+ HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
+ HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
+ HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
+ HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
+ HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
+ HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
+ HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
+ HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */
+ HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
+ HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
+ HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
+ HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
+
+ /* Round 4 */
+ II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
+ II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
+ II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
+ II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
+ II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
+ II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
+ II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
+ II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
+ II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
+ II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
+ II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
+ II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
+ II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
+ II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
+ II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
+ II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
+
+ state[0] += a;
+ state[1] += b;
+ state[2] += c;
+ state[3] += d;
+
+ // Zeroize sensitive information.
+ memset(x, 0, sizeof x);
+}
+
+//////////////////////////////
+
+// MD5 block update operation. Continues an MD5 message-digest
+// operation, processing another message block
+void MD5::update(const unsigned char input[], size_type length)
+{
+ // compute number of bytes mod 64
+ size_type index = count[0] / 8 % blocksize;
+
+ // Update number of bits
+ if ((count[0] += (length << 3)) < (length << 3))
+ count[1]++;
+ count[1] += (length >> 29);
+
+ // number of bytes we need to fill in buffer
+ size_type firstpart = 64 - index;
+
+ size_type i;
+
+ // transform as many times as possible.
+ if (length >= firstpart)
+ {
+ // fill buffer first, transform
+ memcpy(&buffer[index], input, firstpart);
+ transform(buffer);
+
+ // transform chunks of blocksize (64 bytes)
+ for (i = firstpart; i + blocksize <= length; i += blocksize)
+ transform(&input[i]);
+
+ index = 0;
+ }
+ else
+ i = 0;
+
+ // buffer remaining input
+ memcpy(&buffer[index], &input[i], length-i);
+}
+
+//////////////////////////////
+
+// for convenience provide a verson with signed char
+void MD5::update(const char input[], size_type length)
+{
+ update((const unsigned char*)input, length);
+}
+
+//////////////////////////////
+
+// MD5 finalization. Ends an MD5 message-digest operation, writing the
+// the message digest and zeroizing the context.
+MD5& MD5::finalize()
+{
+ static unsigned char padding[64] = {
+ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ };
+
+ if (!finalized) {
+ // Save number of bits
+ unsigned char bits[8];
+ encode(bits, count, 8);
+
+ // pad out to 56 mod 64.
+ size_type index = count[0] / 8 % 64;
+ size_type padLen = (index < 56) ? (56 - index) : (120 - index);
+ update(padding, padLen);
+
+ // Append length (before padding)
+ update(bits, 8);
+
+ // Store state in digest
+ encode(digest, state, 16);
+
+ // Zeroize sensitive information.
+ memset(buffer, 0, sizeof buffer);
+ memset(count, 0, sizeof count);
+
+ finalized=true;
+ }
+
+ return *this;
+}
+
+//////////////////////////////
+
+// return hex representation of digest as string
+std::string MD5::hexdigest() const
+{
+ if (!finalized)
+ return "";
+
+ char buf[33];
+ for (int i=0; i<16; i++)
+ sprintf(buf+i*2, "%02x", digest[i]);
+ buf[32]=0;
+
+ return std::string(buf);
+}
+
+//////////////////////////////
+
+std::string md5(const std::string& str) {
+ MD5 md5 = MD5(str);
+ return md5.hexdigest();
+}
+
diff --git a/utils/stringlib.h b/utils/stringlib.h
index 8022bb88..cafbdac3 100644
--- a/utils/stringlib.h
+++ b/utils/stringlib.h
@@ -249,6 +249,7 @@ inline void SplitCommandAndParam(const std::string& in, std::string* cmd, std::s
}
void ProcessAndStripSGML(std::string* line, std::map<std::string, std::string>* out);
+std::string SGMLOpenSegTag(const std::map<std::string, std::string>& attr);
// given the first character of a UTF8 block, find out how wide it is
// see http://en.wikipedia.org/wiki/UTF-8 for more info
@@ -260,4 +261,6 @@ inline unsigned int UTF8Len(unsigned char x) {
else return 0;
}
+std::string md5(const std::string& in);
+
#endif
diff --git a/utils/ts.cc b/utils/ts.cc
index 3694e076..bf4f8f69 100644
--- a/utils/ts.cc
+++ b/utils/ts.cc
@@ -7,6 +7,7 @@
#include "prob.h"
#include "sparse_vector.h"
#include "fast_sparse_vector.h"
+#include "stringlib.h"
using namespace std;
@@ -79,6 +80,11 @@ int main() {
y -= y;
}
cerr << "Counted " << c << " times\n";
+
+ cerr << md5("this is a test") << endl;
+ cerr << md5("some other ||| string is") << endl;
+ map<string,string> x; x["id"] = "12"; x["grammar"] = "/path/to/grammar.gz";
+ cerr << SGMLOpenSegTag(x) << endl;
return 0;
}