summaryrefslogtreecommitdiff
path: root/extools
diff options
context:
space:
mode:
Diffstat (limited to 'extools')
-rw-r--r--extools/extract.cc26
-rw-r--r--extools/extract.h4
-rw-r--r--extools/sentence_pair.cc87
-rw-r--r--extools/sentence_pair.h8
-rw-r--r--extools/striped_grammar.cc2
5 files changed, 89 insertions, 38 deletions
diff --git a/extools/extract.cc b/extools/extract.cc
index 567348f4..44cd51af 100644
--- a/extools/extract.cc
+++ b/extools/extract.cc
@@ -5,6 +5,7 @@
#include <utility>
#include <tr1/unordered_map>
#include <set>
+#include <boost/tuple/tuple_comparison.hpp>
#include <boost/functional/hash.hpp>
@@ -15,6 +16,7 @@
using namespace std;
using namespace tr1;
+using namespace boost;
namespace {
inline bool IsWhitespace(char c) { return c == ' ' || c == '\t'; }
@@ -114,27 +116,37 @@ void Extract::LoosenPhraseBounds(const AnnotatedParallelSentence& sentence,
}
}
+template <typename K, typename V>
+void
+lookup_and_append(const map<K, V> &dict, const K &key, V &output)
+{
+ typename map<K, V>::const_iterator found = dict.find(key);
+ if (found != dict.end())
+ copy(found->second.begin(), found->second.end(), back_inserter(output));
+}
+
// this uses the TARGET span (i,j) to annotate phrases, will copy
// phrases if there is more than one annotation.
// TODO: support source annotation
void Extract::AnnotatePhrasesWithCategoryTypes(const WordID default_cat,
- const Array2D<vector<WordID> >& types,
+ const map< tuple<short,short,short,short>, vector<WordID> > &types,
vector<ParallelSpan>* phrases) {
const int num_unannotated_phrases = phrases->size();
// have to use num_unannotated_phrases since we may grow the vector
for (int i = 0; i < num_unannotated_phrases; ++i) {
ParallelSpan& phrase = (*phrases)[i];
- const vector<WordID>* pcats = &types(phrase.j1, phrase.j2);
- if (pcats->empty() && default_cat != 0) {
- static vector<WordID> s_default(1, default_cat);
- pcats = &s_default;
+ vector<WordID> cats;
+ lookup_and_append(types, make_tuple(phrase.i1, phrase.i2, phrase.j1, phrase.j2), cats);
+ lookup_and_append(types, make_tuple((short)-1, (short)-1, phrase.j1, phrase.j2), cats);
+ lookup_and_append(types, make_tuple(phrase.i1, phrase.i2, (short)-1, (short)-1), cats);
+ if (cats.empty() && default_cat != 0) {
+ cats = vector<WordID>(1, default_cat);
}
- if (pcats->empty()) {
+ if (cats.empty()) {
cerr << "ERROR span " << phrase.i1 << "," << phrase.i2 << "-"
<< phrase.j1 << "," << phrase.j2 << " has no type. "
"Did you forget --default_category?\n";
}
- const vector<WordID>& cats = *pcats;
phrase.cat = cats[0];
for (int ci = 1; ci < cats.size(); ++ci) {
ParallelSpan new_phrase = phrase;
diff --git a/extools/extract.h b/extools/extract.h
index 76292bed..e9ea5e65 100644
--- a/extools/extract.h
+++ b/extools/extract.h
@@ -4,6 +4,7 @@
#include <iostream>
#include <utility>
#include <vector>
+#include <boost/tuple/tuple.hpp>
#include "array2d.h"
#include "wordid.h"
#include "sparse_vector.h"
@@ -74,9 +75,8 @@ struct Extract {
// this uses the TARGET span (i,j) to annotate phrases, will copy
// phrases if there is more than one annotation.
- // TODO: support source annotation
static void AnnotatePhrasesWithCategoryTypes(const WordID default_cat,
- const Array2D<std::vector<WordID> >& types,
+ const std::map< boost::tuple<short,short,short,short>, std::vector<WordID> > &types,
std::vector<ParallelSpan>* phrases);
// use the Chiang (2007) extraction logic to extract consistent subphrases
diff --git a/extools/sentence_pair.cc b/extools/sentence_pair.cc
index 02df3349..b2881737 100644
--- a/extools/sentence_pair.cc
+++ b/extools/sentence_pair.cc
@@ -6,12 +6,14 @@
#include <vector>
#include <utility>
#include <set>
+#include <boost/tuple/tuple_comparison.hpp>
#include "tdict.h"
#include "wordid.h"
#include "array2d.h"
using namespace std;
+using namespace boost;
namespace {
inline bool IsWhitespace(char c) { return c == ' ' || c == '\t'; }
@@ -38,7 +40,6 @@ void AnnotatedParallelSentence::AllocateForAlignment() {
f_aligned.resize(f_len, 0);
e_aligned.resize(e_len, 0);
aligns_by_fword.resize(f_len);
- span_types.resize(e_len, e_len+1);
}
// read an alignment point of the form X-Y where X and Y are strings
@@ -48,44 +49,76 @@ int AnnotatedParallelSentence::ReadAlignmentPoint(const char* buf,
const int start,
const int end,
const bool permit_col,
- short* a,
- short* b) {
+ short* a, short* b, short* c, short* d) {
if (end - start < 3) {
- cerr << "Alignment point badly formed: " << string(buf, start, end-start) << endl << buf << endl;
+ cerr << "Alignment point badly formed 1: " << string(buf, start, end-start) << endl << buf << endl;
exit(1);
}
- int c = start;
+ int ch = start;
*a = 0;
- while(c < end && buf[c] != '-') {
- if (buf[c] < '0' || buf[c] > '9') {
- cerr << "Alignment point badly formed: " << string(buf, start, end-start) << endl << buf << endl;
+ while(ch < end && buf[ch] != '-') {
+ if (buf[ch] < '0' || buf[ch] > '9') {
+ cerr << "Alignment point badly formed 2: " << string(buf, start, end-start) << endl << buf << endl;
exit(1);
}
(*a) *= 10;
- (*a) += buf[c] - '0';
- ++c;
+ (*a) += buf[ch] - '0';
+ ++ch;
}
- ++c;
- if (c >= end) {
- cerr << "Alignment point badly formed: " << string(buf, start, end-start) << endl << buf << endl;
+ ++ch;
+ if (ch >= end) {
+ cerr << "Alignment point badly formed 3: " << string(buf, start, end-start) << endl << buf << endl;
exit(1);
}
(*b) = 0;
- while(c < end && (!permit_col || (permit_col && buf[c] != ':'))) {
- if (buf[c] < '0' || buf[c] > '9') {
- cerr << "Alignment point badly formed: " << string(buf, start, end-start) << endl << buf << endl;
+ while(ch < end && (c == 0 && (!permit_col || (permit_col && buf[ch] != ':')) || c != 0 && buf[ch] != '-')) {
+ if (buf[ch] < '0' || buf[ch] > '9') {
+ cerr << "Alignment point badly formed 4: " << string(buf, start, end-start) << endl << buf << endl;
exit(1);
}
(*b) *= 10;
- (*b) += buf[c] - '0';
- ++c;
+ (*b) += buf[ch] - '0';
+ ++ch;
}
- return c;
+ if (c != 0)
+ {
+ ++ch;
+ if (ch >= end) {
+ cerr << "Alignment point badly formed 5: " << string(buf, start, end-start) << endl << buf << endl;
+ exit(1);
+ }
+ (*c) = 0;
+ while(ch < end && buf[ch] != '-') {
+ if (buf[ch] < '0' || buf[ch] > '9') {
+ cerr << "Alignment point badly formed 6: " << string(buf, start, end-start) << endl << buf << endl;
+ exit(1);
+ }
+ (*c) *= 10;
+ (*c) += buf[ch] - '0';
+ ++ch;
+ }
+ ++ch;
+ if (ch >= end) {
+ cerr << "Alignment point badly formed 7: " << string(buf, start, end-start) << endl << buf << endl;
+ exit(1);
+ }
+ (*d) = 0;
+ while(ch < end && (!permit_col || (permit_col && buf[ch] != ':'))) {
+ if (buf[ch] < '0' || buf[ch] > '9') {
+ cerr << "Alignment point badly formed 8: " << string(buf, start, end-start) << endl << buf << endl;
+ exit(1);
+ }
+ (*d) *= 10;
+ (*d) += buf[ch] - '0';
+ ++ch;
+ }
+ }
+ return ch;
}
void AnnotatedParallelSentence::ParseAlignmentPoint(const char* buf, int start, int end) {
short a, b;
- ReadAlignmentPoint(buf, start, end, false, &a, &b);
+ ReadAlignmentPoint(buf, start, end, false, &a, &b, 0, 0);
if (a >= f_len || b >= e_len) {
cerr << "(" << a << ',' << b << ") is out of bounds. INPUT=\n" << buf << endl;
exit(1);
@@ -98,18 +131,22 @@ void AnnotatedParallelSentence::ParseAlignmentPoint(const char* buf, int start,
}
void AnnotatedParallelSentence::ParseSpanLabel(const char* buf, int start, int end) {
- short a,b;
- int c = ReadAlignmentPoint(buf, start, end, true, &a, &b) + 1;
- if (buf[c-1] != ':' || c >= end) {
+ short a,b,c,d;
+ int ch = ReadAlignmentPoint(buf, start, end, true, &a, &b, &c, &d) + 1;
+ if (buf[ch-1] != ':' || ch >= end) {
cerr << "Span badly formed: " << string(buf, start, end-start) << endl << buf << endl;
exit(1);
}
- if (a >= e_len || b > e_len) {
+ if (a >= f_len || b > f_len) {
cerr << "(" << a << ',' << b << ") is out of bounds in labeled span. INPUT=\n" << buf << endl;
exit(1);
}
+ if (c >= e_len || d > e_len) {
+ cerr << "(" << c << ',' << d << ") is out of bounds in labeled span. INPUT=\n" << buf << endl;
+ exit(1);
+ }
// cerr << a << " " << b << " " << string(buf,c,end-c) << endl;
- span_types(a,b).push_back(-TD::Convert(string(buf, c, end-c)));
+ span_types[make_tuple(a,b,c,d)].push_back(-TD::Convert(string(buf, ch, end-ch)));
}
// INPUT FORMAT
diff --git a/extools/sentence_pair.h b/extools/sentence_pair.h
index d78be359..b5a7ca93 100644
--- a/extools/sentence_pair.h
+++ b/extools/sentence_pair.h
@@ -1,8 +1,10 @@
#ifndef _SENTENCE_PAIR_H_
#define _SENTENCE_PAIR_H_
+#include <map>
#include <utility>
#include <vector>
+#include <boost/tuple/tuple.hpp>
#include "wordid.h"
#include "array2d.h"
@@ -22,12 +24,12 @@ struct AnnotatedParallelSentence {
std::vector<std::vector<std::pair<short, short> > > aligns_by_fword;
// span type information
- Array2D<std::vector<WordID> > span_types; // span_types(i,j) is the list of category
- // types for a span (i,j) in the TARGET language.
+ std::map< boost::tuple<short,short,short,short>, std::vector<WordID> > span_types;
+ // span_types(i,j,k,l) is the list of category span (i,j) in source and (k,l) in the target language.
int f_len, e_len;
- static int ReadAlignmentPoint(const char* buf, int start, int end, bool permit_col, short* a, short* b);
+ static int ReadAlignmentPoint(const char* buf, int start, int end, bool permit_col, short* a, short* b, short* c, short* d);
private:
void Reset();
diff --git a/extools/striped_grammar.cc b/extools/striped_grammar.cc
index accf44eb..785f4bbe 100644
--- a/extools/striped_grammar.cc
+++ b/extools/striped_grammar.cc
@@ -33,7 +33,7 @@ void RuleStatistics::ParseRuleStatistics(const char* buf, int start, int end) {
while(ptr < end && buf[ptr] != ',' && !IsWhitespace(buf[ptr])) { ++ptr; }
if (ptr > vstart) {
short a, b;
- AnnotatedParallelSentence::ReadAlignmentPoint(buf, vstart, ptr, false, &a, &b);
+ AnnotatedParallelSentence::ReadAlignmentPoint(buf, vstart, ptr, false, &a, &b, 0, 0);
aligns.push_back(make_pair(a,b));
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-01 22:18:39 +0000