summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWu, Ke <wuke@cs.umd.edu>2014-12-06 12:18:18 -0500
committerWu, Ke <wuke@cs.umd.edu>2014-12-06 12:18:18 -0500
commita4d7d7ed27c87d56c925f2b56643d6cc9aee026f (patch)
tree8196045b05c08a73005adfcecb5b3b9d3c46200b
parent2f0d1f17aea094dda3644b5b583c0e06bb02eee0 (diff)
parentf74345e54b9f5f5f894d81df61e85fddc53a520a (diff)
Merge branch 'const_reorder' into softsyn
-rw-r--r--.gitignore4
-rw-r--r--configure.ac1
-rw-r--r--decoder/Makefile.am1
-rw-r--r--decoder/ff_const_reorder.cc6
-rw-r--r--decoder/ff_const_reorder_common.h1441
-rw-r--r--training/Makefile.am4
-rw-r--r--training/const_reorder/Makefile.am8
-rw-r--r--training/const_reorder/argument_reorder_model.cc (renamed from utils/argument_reorder_model.cc)6
-rw-r--r--training/const_reorder/constituent_reorder_model.cc (renamed from utils/constituent_reorder_model.cc)7
-rw-r--r--utils/Makefile.am12
-rw-r--r--utils/alignment.h200
-rw-r--r--utils/argument_reorder_model.h246
-rw-r--r--utils/srl_sentence.h211
-rw-r--r--utils/tree.h699
-rw-r--r--utils/tsuruoka_maxent.h153
15 files changed, 1464 insertions, 1535 deletions
diff --git a/.gitignore b/.gitignore
index c75328dc..fccfcce7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -181,10 +181,10 @@ training/mr_reduce_to_weights
training/optimize_test
training/plftools
training/test_ngram
-utils/argument_reorder_model_trainer
+training/const_reorder/argument_reorder_model_trainer
+training/const_reorder/const_reorder_model_trainer
utils/atools
utils/bin/
-utils/const_reorder_model_trainer
utils/crp_test
utils/dict_test
utils/logval_test
diff --git a/configure.ac b/configure.ac
index 0b1f04f0..aaad9a78 100644
--- a/configure.ac
+++ b/configure.ac
@@ -235,6 +235,7 @@ AC_CONFIG_FILES([training/minrisk/Makefile])
AC_CONFIG_FILES([training/mira/Makefile])
AC_CONFIG_FILES([training/latent_svm/Makefile])
AC_CONFIG_FILES([training/dtrain/Makefile])
+AC_CONFIG_FILES([training/const_reorder/Makefile])
# external feature function example code
AC_CONFIG_FILES([example_extff/Makefile])
diff --git a/decoder/Makefile.am b/decoder/Makefile.am
index 2f293b15..727e5af5 100644
--- a/decoder/Makefile.am
+++ b/decoder/Makefile.am
@@ -45,6 +45,7 @@ libcdec_a_SOURCES = \
ff_basic.h \
ff_bleu.h \
ff_charset.h \
+ ff_const_reorder_common.h \
ff_const_reorder.h \
ff_context.h \
ff_csplit.h \
diff --git a/decoder/ff_const_reorder.cc b/decoder/ff_const_reorder.cc
index 8da551c6..e4fb4725 100644
--- a/decoder/ff_const_reorder.cc
+++ b/decoder/ff_const_reorder.cc
@@ -4,11 +4,8 @@
#include "stringlib.h"
#include "hg.h"
#include "sentence_metadata.h"
-#include "tree.h"
-#include "srl_sentence.h"
-#include "tsuruoka_maxent.h"
#include "hash.h"
-#include "argument_reorder_model.h"
+#include "ff_const_reorder_common.h"
#include <sstream>
#include <string>
@@ -16,6 +13,7 @@
#include <stdio.h>
using namespace std;
+using namespace const_reorder;
typedef HASH_MAP<std::string, vector<double> > MapClassifier;
diff --git a/decoder/ff_const_reorder_common.h b/decoder/ff_const_reorder_common.h
new file mode 100644
index 00000000..7c111de3
--- /dev/null
+++ b/decoder/ff_const_reorder_common.h
@@ -0,0 +1,1441 @@
+#ifndef _FF_CONST_REORDER_COMMON_H
+#define _FF_CONST_REORDER_COMMON_H
+
+#include <string>
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <string>
+#include <sstream>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "maxent.h"
+#include "stringlib.h"
+
+namespace const_reorder {
+
+struct STreeItem {
+ STreeItem(const char *pszTerm) {
+ m_pszTerm = new char[strlen(pszTerm) + 1];
+ strcpy(m_pszTerm, pszTerm);
+
+ m_ptParent = NULL;
+ m_iBegin = -1;
+ m_iEnd = -1;
+ m_iHeadChild = -1;
+ m_iHeadWord = -1;
+ m_iBrotherIndex = -1;
+ }
+ ~STreeItem() {
+ delete[] m_pszTerm;
+ for (size_t i = 0; i < m_vecChildren.size(); i++) delete m_vecChildren[i];
+ }
+ int fnAppend(STreeItem *ptChild) {
+ m_vecChildren.push_back(ptChild);
+ ptChild->m_iBrotherIndex = m_vecChildren.size() - 1;
+ ptChild->m_ptParent = this;
+ return m_vecChildren.size() - 1;
+ }
+ int fnGetChildrenNum() { return m_vecChildren.size(); }
+
+ bool fnIsPreTerminal(void) {
+ int I;
+ if (this == NULL || m_vecChildren.size() == 0) return false;
+
+ for (I = 0; I < m_vecChildren.size(); I++)
+ if (m_vecChildren[I]->m_vecChildren.size() > 0) return false;
+
+ return true;
+ }
+
+ public:
+ char *m_pszTerm;
+
+ std::vector<STreeItem *> m_vecChildren; // children items
+ STreeItem *m_ptParent; // the parent item
+
+ int m_iBegin;
+ int m_iEnd; // the node span words[m_iBegin, m_iEnd]
+ int m_iHeadChild; // the index of its head child
+ int m_iHeadWord; // the index of its head word
+ int m_iBrotherIndex; // the index in his brothers
+};
+
+struct SGetHeadWord {
+ typedef std::vector<std::string> CVectorStr;
+ SGetHeadWord() {}
+ ~SGetHeadWord() {}
+ int fnGetHeadWord(char *pszCFGLeft, CVectorStr vectRight) {
+ // 0 indicating from right to left while 1 indicating from left to right
+ char szaHeadLists[201] = "0";
+
+ /* //head rules for Egnlish
+ if( strcmp( pszCFGLeft, "ADJP" ) == 0 )
+ strcpy( szaHeadLists, "0NNS 0QP 0NN 0$ 0ADVP 0JJ 0VBN 0VBG 0ADJP
+ 0JJR 0NP 0JJS 0DT 0FW 0RBR 0RBS 0SBAR 0RB 0" );
+ else if( strcmp( pszCFGLeft, "ADVP" ) == 0 )
+ strcpy( szaHeadLists, "1RB 1RBR 1RBS 1FW 1ADVP 1TO 1CD 1JJR 1JJ 1IN
+ 1NP 1JJS 1NN 1" );
+ else if( strcmp( pszCFGLeft, "CONJP" ) == 0 )
+ strcpy( szaHeadLists, "1CC 1RB 1IN 1" );
+ else if( strcmp( pszCFGLeft, "FRAG" ) == 0 )
+ strcpy( szaHeadLists, "1" );
+ else if( strcmp( pszCFGLeft, "INTJ" ) == 0 )
+ strcpy( szaHeadLists, "0" );
+ else if( strcmp( pszCFGLeft, "LST" ) == 0 )
+ strcpy( szaHeadLists, "1LS 1: 1CLN 1" );
+ else if( strcmp( pszCFGLeft, "NAC" ) == 0 )
+ strcpy( szaHeadLists, "0NN 0NNS 0NNP 0NNPS 0NP 0NAC 0EX 0$ 0CD 0QP
+ 0PRP 0VBG 0JJ 0JJS 0JJR 0ADJP 0FW 0" );
+ else if( strcmp( pszCFGLeft, "PP" ) == 0 )
+ strcpy( szaHeadLists, "1IN 1TO 1VBG 1VBN 1RP 1FW 1" );
+ else if( strcmp( pszCFGLeft, "PRN" ) == 0 )
+ strcpy( szaHeadLists, "1" );
+ else if( strcmp( pszCFGLeft, "PRT" ) == 0 )
+ strcpy( szaHeadLists, "1RP 1" );
+ else if( strcmp( pszCFGLeft, "QP" ) == 0 )
+ strcpy( szaHeadLists, "0$ 0IN 0NNS 0NN 0JJ 0RB 0DT 0CD 0NCD 0QP 0JJR
+ 0JJS 0" );
+ else if( strcmp( pszCFGLeft, "RRC" ) == 0 )
+ strcpy( szaHeadLists, "1VP 1NP 1ADVP 1ADJP 1PP 1" );
+ else if( strcmp( pszCFGLeft, "S" ) == 0 )
+ strcpy( szaHeadLists, "0TO 0IN 0VP 0S 0SBAR 0ADJP 0UCP 0NP 0" );
+ else if( strcmp( pszCFGLeft, "SBAR" ) == 0 )
+ strcpy( szaHeadLists, "0WHNP 0WHPP 0WHADVP 0WHADJP 0IN 0DT 0S 0SQ
+ 0SINV 0SBAR 0FRAG 0" );
+ else if( strcmp( pszCFGLeft, "SBARQ" ) == 0 )
+ strcpy( szaHeadLists, "0SQ 0S 0SINV 0SBARQ 0FRAG 0" );
+ else if( strcmp( pszCFGLeft, "SINV" ) == 0 )
+ strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0S 0SINV 0ADJP 0NP
+ 0" );
+ else if( strcmp( pszCFGLeft, "SQ" ) == 0 )
+ strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0SQ 0" );
+ else if( strcmp( pszCFGLeft, "UCP" ) == 0 )
+ strcpy( szaHeadLists, "1" );
+ else if( strcmp( pszCFGLeft, "VP" ) == 0 )
+ strcpy( szaHeadLists, "0TO 0VBD 0VBN 0MD 0VBZ 0VB 0VBG 0VBP 0VP
+ 0ADJP 0NN 0NNS 0NP 0" );
+ else if( strcmp( pszCFGLeft, "WHADJP" ) == 0 )
+ strcpy( szaHeadLists, "0CC 0WRB 0JJ 0ADJP 0" );
+ else if( strcmp( pszCFGLeft, "WHADVP" ) == 0 )
+ strcpy( szaHeadLists, "1CC 1WRB 1" );
+ else if( strcmp( pszCFGLeft, "WHNP" ) == 0 )
+ strcpy( szaHeadLists, "0WDT 0WP 0WP$ 0WHADJP 0WHPP 0WHNP 0" );
+ else if( strcmp( pszCFGLeft, "WHPP" ) == 0 )
+ strcpy( szaHeadLists, "1IN 1TO FW 1" );
+ else if( strcmp( pszCFGLeft, "NP" ) == 0 )
+ strcpy( szaHeadLists, "0NN NNP NNS NNPS NX POS JJR 0NP 0$ ADJP PRN
+ 0CD 0JJ JJS RB QP 0" );
+ */
+
+ if (strcmp(pszCFGLeft, "ADJP") == 0)
+ strcpy(szaHeadLists, "0ADJP JJ 0AD NN CS 0");
+ else if (strcmp(pszCFGLeft, "ADVP") == 0)
+ strcpy(szaHeadLists, "0ADVP AD 0");
+ else if (strcmp(pszCFGLeft, "CLP") == 0)
+ strcpy(szaHeadLists, "0CLP M 0");
+ else if (strcmp(pszCFGLeft, "CP") == 0)
+ strcpy(szaHeadLists, "0DEC SP 1ADVP CS 0CP IP 0");
+ else if (strcmp(pszCFGLeft, "DNP") == 0)
+ strcpy(szaHeadLists, "0DNP DEG 0DEC 0");
+ else if (strcmp(pszCFGLeft, "DVP") == 0)
+ strcpy(szaHeadLists, "0DVP DEV 0");
+ else if (strcmp(pszCFGLeft, "DP") == 0)
+ strcpy(szaHeadLists, "1DP DT 1");
+ else if (strcmp(pszCFGLeft, "FRAG") == 0)
+ strcpy(szaHeadLists, "0VV NR NN 0");
+ else if (strcmp(pszCFGLeft, "INTJ") == 0)
+ strcpy(szaHeadLists, "0INTJ IJ 0");
+ else if (strcmp(pszCFGLeft, "LST") == 0)
+ strcpy(szaHeadLists, "1LST CD OD 1");
+ else if (strcmp(pszCFGLeft, "IP") == 0)
+ strcpy(szaHeadLists, "0IP VP 0VV 0");
+ // strcpy( szaHeadLists, "0VP 0VV 1IP 0" );
+ else if (strcmp(pszCFGLeft, "LCP") == 0)
+ strcpy(szaHeadLists, "0LCP LC 0");
+ else if (strcmp(pszCFGLeft, "NP") == 0)
+ strcpy(szaHeadLists, "0NP NN NT NR QP 0");
+ else if (strcmp(pszCFGLeft, "PP") == 0)
+ strcpy(szaHeadLists, "1PP P 1");
+ else if (strcmp(pszCFGLeft, "PRN") == 0)
+ strcpy(szaHeadLists, "0 NP IP VP NT NR NN 0");
+ else if (strcmp(pszCFGLeft, "QP") == 0)
+ strcpy(szaHeadLists, "0QP CLP CD OD 0");
+ else if (strcmp(pszCFGLeft, "VP") == 0)
+ strcpy(szaHeadLists, "1VP VA VC VE VV BA LB VCD VSB VRD VNV VCP 1");
+ else if (strcmp(pszCFGLeft, "VCD") == 0)
+ strcpy(szaHeadLists, "0VCD VV VA VC VE 0");
+ if (strcmp(pszCFGLeft, "VRD") == 0)
+ strcpy(szaHeadLists, "0VRD VV VA VC VE 0");
+ else if (strcmp(pszCFGLeft, "VSB") == 0)
+ strcpy(szaHeadLists, "0VSB VV VA VC VE 0");
+ else if (strcmp(pszCFGLeft, "VCP") == 0)
+ strcpy(szaHeadLists, "0VCP VV VA VC VE 0");
+ else if (strcmp(pszCFGLeft, "VNV") == 0)
+ strcpy(szaHeadLists, "0VNV VV VA VC VE 0");
+ else if (strcmp(pszCFGLeft, "VPT") == 0)
+ strcpy(szaHeadLists, "0VNV VV VA VC VE 0");
+ else if (strcmp(pszCFGLeft, "UCP") == 0)
+ strcpy(szaHeadLists, "0");
+ else if (strcmp(pszCFGLeft, "WHNP") == 0)
+ strcpy(szaHeadLists, "0WHNP NP NN NT NR QP 0");
+ else if (strcmp(pszCFGLeft, "WHPP") == 0)
+ strcpy(szaHeadLists, "1WHPP PP P 1");
+
+ /* //head rules for GENIA corpus
+ if( strcmp( pszCFGLeft, "ADJP" ) == 0 )
+ strcpy( szaHeadLists, "0NNS 0QP 0NN 0$ 0ADVP 0JJ 0VBN 0VBG 0ADJP
+ 0JJR 0NP 0JJS 0DT 0FW 0RBR 0RBS 0SBAR 0RB 0" );
+ else if( strcmp( pszCFGLeft, "ADVP" ) == 0 )
+ strcpy( szaHeadLists, "1RB 1RBR 1RBS 1FW 1ADVP 1TO 1CD 1JJR 1JJ 1IN
+ 1NP 1JJS 1NN 1" );
+ else if( strcmp( pszCFGLeft, "CONJP" ) == 0 )
+ strcpy( szaHeadLists, "1CC 1RB 1IN 1" );
+ else if( strcmp( pszCFGLeft, "FRAG" ) == 0 )
+ strcpy( szaHeadLists, "1" );
+ else if( strcmp( pszCFGLeft, "INTJ" ) == 0 )
+ strcpy( szaHeadLists, "0" );
+ else if( strcmp( pszCFGLeft, "LST" ) == 0 )
+ strcpy( szaHeadLists, "1LS 1: 1CLN 1" );
+ else if( strcmp( pszCFGLeft, "NAC" ) == 0 )
+ strcpy( szaHeadLists, "0NN 0NNS 0NNP 0NNPS 0NP 0NAC 0EX 0$ 0CD 0QP
+ 0PRP 0VBG 0JJ 0JJS 0JJR 0ADJP 0FW 0" );
+ else if( strcmp( pszCFGLeft, "PP" ) == 0 )
+ strcpy( szaHeadLists, "1IN 1TO 1VBG 1VBN 1RP 1FW 1" );
+ else if( strcmp( pszCFGLeft, "PRN" ) == 0 )
+ strcpy( szaHeadLists, "1" );
+ else if( strcmp( pszCFGLeft, "PRT" ) == 0 )
+ strcpy( szaHeadLists, "1RP 1" );
+ else if( strcmp( pszCFGLeft, "QP" ) == 0 )
+ strcpy( szaHeadLists, "0$ 0IN 0NNS 0NN 0JJ 0RB 0DT 0CD 0NCD 0QP 0JJR
+ 0JJS 0" );
+ else if( strcmp( pszCFGLeft, "RRC" ) == 0 )
+ strcpy( szaHeadLists, "1VP 1NP 1ADVP 1ADJP 1PP 1" );
+ else if( strcmp( pszCFGLeft, "S" ) == 0 )
+ strcpy( szaHeadLists, "0TO 0IN 0VP 0S 0SBAR 0ADJP 0UCP 0NP 0" );
+ else if( strcmp( pszCFGLeft, "SBAR" ) == 0 )
+ strcpy( szaHeadLists, "0WHNP 0WHPP 0WHADVP 0WHADJP 0IN 0DT 0S 0SQ
+ 0SINV 0SBAR 0FRAG 0" );
+ else if( strcmp( pszCFGLeft, "SBARQ" ) == 0 )
+ strcpy( szaHeadLists, "0SQ 0S 0SINV 0SBARQ 0FRAG 0" );
+ else if( strcmp( pszCFGLeft, "SINV" ) == 0 )
+ strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0S 0SINV 0ADJP 0NP
+ 0" );
+ else if( strcmp( pszCFGLeft, "SQ" ) == 0 )
+ strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0SQ 0" );
+ else if( strcmp( pszCFGLeft, "UCP" ) == 0 )
+ strcpy( szaHeadLists, "1" );
+ else if( strcmp( pszCFGLeft, "VP" ) == 0 )
+ strcpy( szaHeadLists, "0TO 0VBD 0VBN 0MD 0VBZ 0VB 0VBG 0VBP 0VP
+ 0ADJP 0NN 0NNS 0NP 0" );
+ else if( strcmp( pszCFGLeft, "WHADJP" ) == 0 )
+ strcpy( szaHeadLists, "0CC 0WRB 0JJ 0ADJP 0" );
+ else if( strcmp( pszCFGLeft, "WHADVP" ) == 0 )
+ strcpy( szaHeadLists, "1CC 1WRB 1" );
+ else if( strcmp( pszCFGLeft, "WHNP" ) == 0 )
+ strcpy( szaHeadLists, "0WDT 0WP 0WP$ 0WHADJP 0WHPP 0WHNP 0" );
+ else if( strcmp( pszCFGLeft, "WHPP" ) == 0 )
+ strcpy( szaHeadLists, "1IN 1TO FW 1" );
+ else if( strcmp( pszCFGLeft, "NP" ) == 0 )
+ strcpy( szaHeadLists, "0NN NNP NNS NNPS NX POS JJR 0NP 0$ ADJP PRN
+ 0CD 0JJ JJS RB QP 0" );
+ */
+
+ return fnMyOwnHeadWordRule(szaHeadLists, vectRight);
+ }
+
+ private:
+ int fnMyOwnHeadWordRule(char *pszaHeadLists, CVectorStr vectRight) {
+ char szHeadList[201], *p;
+ char szTerm[101];
+ int J;
+
+ p = pszaHeadLists;
+
+ int iCountRight;
+
+ iCountRight = vectRight.size();
+
+ szHeadList[0] = '\0';
+ while (1) {
+ szTerm[0] = '\0';
+ sscanf(p, "%s", szTerm);
+ if (strlen(szHeadList) == 0) {
+ if (strcmp(szTerm, "0") == 0) {
+ return iCountRight - 1;
+ }
+ if (strcmp(szTerm, "1") == 0) {
+ return 0;
+ }
+
+ sprintf(szHeadList, "%c %s ", szTerm[0], szTerm + 1);
+ p = strstr(p, szTerm);
+ p += strlen(szTerm);
+ } else {
+ if ((szTerm[0] == '0') || (szTerm[0] == '1')) {
+ if (szHeadList[0] == '0') {
+ for (J = iCountRight - 1; J >= 0; J--) {
+ sprintf(szTerm, " %s ", vectRight.at(J).c_str());
+ if (strstr(szHeadList, szTerm) != NULL) return J;
+ }
+ } else {
+ for (J = 0; J < iCountRight; J++) {
+ sprintf(szTerm, " %s ", vectRight.at(J).c_str());
+ if (strstr(szHeadList, szTerm) != NULL) return J;
+ }
+ }
+
+ szHeadList[0] = '\0';
+ } else {
+ strcat(szHeadList, szTerm);
+ strcat(szHeadList, " ");
+
+ p = strstr(p, szTerm);
+ p += strlen(szTerm);
+ }
+ }
+ }
+
+ return 0;
+ }
+};
+
+struct SParsedTree {
+ SParsedTree() { m_ptRoot = NULL; }
+ ~SParsedTree() {
+ if (m_ptRoot != NULL) delete m_ptRoot;
+ }
+ static SParsedTree *fnConvertFromString(const char *pszStr) {
+ if (strcmp(pszStr, "(())") == 0) return NULL;
+ SParsedTree *pTree = new SParsedTree();
+
+ std::vector<std::string> vecSyn;
+ fnReadSyntactic(pszStr, vecSyn);
+
+ int iLeft = 1, iRight = 1; //# left/right parenthesis
+
+ STreeItem *pcurrent;
+
+ pTree->m_ptRoot = new STreeItem(vecSyn[1].c_str());
+
+ pcurrent = pTree->m_ptRoot;
+
+ for (size_t i = 2; i < vecSyn.size() - 1; i++) {
+ if (strcmp(vecSyn[i].c_str(), "(") == 0)
+ iLeft++;
+ else if (strcmp(vecSyn[i].c_str(), ")") == 0) {
+ iRight++;
+ if (pcurrent == NULL) {
+ // error
+ fprintf(stderr, "ERROR in ConvertFromString\n");
+ fprintf(stderr, "%s\n", pszStr);
+ return NULL;
+ }
+ pcurrent = pcurrent->m_ptParent;
+ } else {
+ STreeItem *ptNewItem = new STreeItem(vecSyn[i].c_str());
+ pcurrent->fnAppend(ptNewItem);
+ pcurrent = ptNewItem;
+
+ if (strcmp(vecSyn[i - 1].c_str(), "(") != 0 &&
+ strcmp(vecSyn[i - 1].c_str(), ")") != 0) {
+ pTree->m_vecTerminals.push_back(ptNewItem);
+ pcurrent = pcurrent->m_ptParent;
+ }
+ }
+ }
+
+ if (iLeft != iRight) {
+ // error
+ fprintf(stderr, "the left and right parentheses are not matched!");
+ fprintf(stderr, "ERROR in ConvertFromString\n");
+ fprintf(stderr, "%s\n", pszStr);
+ return NULL;
+ }
+
+ return pTree;
+ }
+
+ int fnGetNumWord() { return m_vecTerminals.size(); }
+
+ void fnSetSpanInfo() {
+ int iNextNum = 0;
+ fnSuffixTraverseSetSpanInfo(m_ptRoot, iNextNum);
+ }
+
+ void fnSetHeadWord() {
+ for (size_t i = 0; i < m_vecTerminals.size(); i++)
+ m_vecTerminals[i]->m_iHeadWord = i;
+ SGetHeadWord *pGetHeadWord = new SGetHeadWord();
+ fnSuffixTraverseSetHeadWord(m_ptRoot, pGetHeadWord);
+ delete pGetHeadWord;
+ }
+
+ STreeItem *fnFindNodeForSpan(int iLeft, int iRight, bool bLowest) {
+ STreeItem *pTreeItem = m_vecTerminals[iLeft];
+
+ while (pTreeItem->m_iEnd < iRight) {
+ pTreeItem = pTreeItem->m_ptParent;
+ if (pTreeItem == NULL) break;
+ }
+ if (pTreeItem == NULL) return NULL;
+ if (pTreeItem->m_iEnd > iRight) return NULL;
+
+ assert(pTreeItem->m_iEnd == iRight);
+ if (bLowest) return pTreeItem;
+
+ while (pTreeItem->m_ptParent != NULL &&
+ pTreeItem->m_ptParent->fnGetChildrenNum() == 1)
+ pTreeItem = pTreeItem->m_ptParent;
+
+ return pTreeItem;
+ }
+
+ private:
+ void fnSuffixTraverseSetSpanInfo(STreeItem *ptItem, int &iNextNum) {
+ int I;
+ int iNumChildren = ptItem->fnGetChildrenNum();
+ for (I = 0; I < iNumChildren; I++)
+ fnSuffixTraverseSetSpanInfo(ptItem->m_vecChildren[I], iNextNum);
+
+ if (I == 0) {
+ ptItem->m_iBegin = iNextNum;
+ ptItem->m_iEnd = iNextNum++;
+ } else {
+ ptItem->m_iBegin = ptItem->m_vecChildren[0]->m_iBegin;
+ ptItem->m_iEnd = ptItem->m_vecChildren[I - 1]->m_iEnd;
+ }
+ }
+
+ void fnSuffixTraverseSetHeadWord(STreeItem *ptItem,
+ SGetHeadWord *pGetHeadWord) {
+ int I, iHeadchild;
+
+ if (ptItem->m_vecChildren.size() == 0) return;
+
+ for (I = 0; I < ptItem->m_vecChildren.size(); I++)
+ fnSuffixTraverseSetHeadWord(ptItem->m_vecChildren[I], pGetHeadWord);
+
+ std::vector<std::string> vecRight;
+
+ if (ptItem->m_vecChildren.size() == 1)
+ iHeadchild = 0;
+ else {
+ for (I = 0; I < ptItem->m_vecChildren.size(); I++)
+ vecRight.push_back(std::string(ptItem->m_vecChildren[I]->m_pszTerm));
+
+ iHeadchild = pGetHeadWord->fnGetHeadWord(ptItem->m_pszTerm, vecRight);
+ }
+
+ ptItem->m_iHeadChild = iHeadchild;
+ ptItem->m_iHeadWord = ptItem->m_vecChildren[iHeadchild]->m_iHeadWord;
+ }
+
+ static void fnReadSyntactic(const char *pszSyn,
+ std::vector<std::string> &vec) {
+ char *p;
+ int I;
+
+ int iLeftNum, iRightNum;
+ char *pszTmp, *pszTerm;
+ pszTmp = new char[strlen(pszSyn)];
+ pszTerm = new char[strlen(pszSyn)];
+ pszTmp[0] = pszTerm[0] = '\0';
+
+ vec.clear();
+
+ char *pszLine;
+ pszLine = new char[strlen(pszSyn) + 1];
+ strcpy(pszLine, pszSyn);
+
+ char *pszLine2;
+
+ while (1) {
+ while ((strlen(pszLine) > 0) && (pszLine[strlen(pszLine) - 1] > 0) &&
+ (pszLine[strlen(pszLine) - 1] <= ' '))
+ pszLine[strlen(pszLine) - 1] = '\0';
+
+ if (strlen(pszLine) == 0) break;
+
+ // printf( "%s\n", pszLine );
+ pszLine2 = pszLine;
+ while (pszLine2[0] <= ' ') pszLine2++;
+ if (pszLine2[0] == '<') continue;
+
+ sscanf(pszLine2 + 1, "%s", pszTmp);
+
+ if (pszLine2[0] == '(') {
+ iLeftNum = 0;
+ iRightNum = 0;
+ }
+
+ p = pszLine2;
+ while (1) {
+ pszTerm[0] = '\0';
+ sscanf(p, "%s", pszTerm);
+
+ if (strlen(pszTerm) == 0) break;
+ p = strstr(p, pszTerm);
+ p += strlen(pszTerm);
+
+ if ((pszTerm[0] == '(') || (pszTerm[strlen(pszTerm) - 1] == ')')) {
+ if (pszTerm[0] == '(') {
+ vec.push_back(std::string("("));
+ iLeftNum++;
+
+ I = 1;
+ while (pszTerm[I] == '(' && pszTerm[I] != '\0') {
+ vec.push_back(std::string("("));
+ iLeftNum++;
+
+ I++;
+ }
+
+ if (strlen(pszTerm) > 1) vec.push_back(std::string(pszTerm + I));
+ } else {
+ char *pTmp;
+ pTmp = pszTerm + strlen(pszTerm) - 1;
+ while ((pTmp[0] == ')') && (pTmp >= pszTerm)) pTmp--;
+ pTmp[1] = '\0';
+
+ if (strlen(pszTerm) > 0) vec.push_back(std::string(pszTerm));
+ pTmp += 2;
+
+ for (I = 0; I <= (int)strlen(pTmp); I++) {
+ vec.push_back(std::string(")"));
+ iRightNum++;
+ }
+ }
+ } else {
+ char *q;
+ q = strchr(pszTerm, ')');
+ if (q != NULL) {
+ q[0] = '\0';
+ if (pszTerm[0] != '\0') vec.push_back(std::string(pszTerm));
+ vec.push_back(std::string(")"));
+ iRightNum++;
+
+ q++;
+ while (q[0] == ')') {
+ vec.push_back(std::string(")"));
+ q++;
+ iRightNum++;
+ }
+
+ while (q[0] == '(') {
+ vec.push_back(std::string("("));
+ q++;
+ iLeftNum++;
+ }
+
+ if (q[0] != '\0') vec.push_back(std::string(q));
+ } else
+ vec.push_back(std::string(pszTerm));
+ }
+ }
+
+ if (iLeftNum != iRightNum) {
+ fprintf(stderr, "%s\n", pszSyn);
+ assert(iLeftNum == iRightNum);
+ }
+ /*if ( iLeftNum != iRightNum ) {
+ printf( "ERROR: left( and right ) is not matched, %d ( and %d
+ )\n", iLeftNum, iRightNum );
+ return;
+ }*/
+
+ if (vec.size() >= 2 && strcmp(vec[1].c_str(), "(") == 0) {
+ //( (IP..) )
+ std::vector<std::string>::iterator it;
+ it = vec.begin();
+ it++;
+ vec.insert(it, std::string("ROOT"));
+ }
+
+ break;
+ }
+
+ delete[] pszLine;
+ delete[] pszTmp;
+ delete[] pszTerm;
+ }
+
+ public:
+ STreeItem *m_ptRoot;
+ std::vector<STreeItem *> m_vecTerminals; // the leaf nodes
+};
+
+struct SParseReader {
+ SParseReader(const char *pszParse_Fname, bool bFlattened = false)
+ : m_bFlattened(bFlattened) {
+ m_fpIn = fopen(pszParse_Fname, "r");
+ assert(m_fpIn != NULL);
+ }
+ ~SParseReader() {
+ if (m_fpIn != NULL) fclose(m_fpIn);
+ }
+
+ SParsedTree *fnReadNextParseTree() {
+ SParsedTree *pTree = NULL;
+ char *pszLine = new char[100001];
+ int iLen;
+
+ while (fnReadNextSentence(pszLine, &iLen) == true) {
+ if (iLen == 0) continue;
+
+ pTree = SParsedTree::fnConvertFromString(pszLine);
+ if (pTree == NULL) break;
+ if (m_bFlattened)
+ fnPostProcessingFlattenedParse(pTree);
+ else {
+ pTree->fnSetSpanInfo();
+ pTree->fnSetHeadWord();
+ }
+ break;
+ }
+
+ delete[] pszLine;
+ return pTree;
+ }
+
+ SParsedTree *fnReadNextParseTreeWithProb(double *pProb) {
+ SParsedTree *pTree = NULL;
+ char *pszLine = new char[100001];
+ int iLen;
+
+ while (fnReadNextSentence(pszLine, &iLen) == true) {
+ if (iLen == 0) continue;
+
+ char *p = strchr(pszLine, ' ');
+ assert(p != NULL);
+ p[0] = '\0';
+ p++;
+ if (pProb) (*pProb) = atof(pszLine);
+
+ pTree = SParsedTree::fnConvertFromString(p);
+ if (m_bFlattened)
+ fnPostProcessingFlattenedParse(pTree);
+ else {
+ pTree->fnSetSpanInfo();
+ pTree->fnSetHeadWord();
+ }
+ break;
+ }
+
+ delete[] pszLine;
+ return pTree;
+ }
+
+ private:
+ /*
+ * since to the parse tree is a flattened tree, use the head mark to identify
+ * head info.
+ * the head node will be marked as "*XP*"
+ */
+ void fnSetParseTreeHeadInfo(SParsedTree *pTree) {
+ for (size_t i = 0; i < pTree->m_vecTerminals.size(); i++)
+ pTree->m_vecTerminals[i]->m_iHeadWord = i;
+ fnSuffixTraverseSetHeadWord(pTree->m_ptRoot);
+ }
+
+ void fnSuffixTraverseSetHeadWord(STreeItem *pTreeItem) {
+ if (pTreeItem->m_vecChildren.size() == 0) return;
+
+ for (size_t i = 0; i < pTreeItem->m_vecChildren.size(); i++)
+ fnSuffixTraverseSetHeadWord(pTreeItem->m_vecChildren[i]);
+
+ std::vector<std::string> vecRight;
+
+ int iHeadchild;
+
+ if (pTreeItem->fnIsPreTerminal()) {
+ iHeadchild = 0;
+ } else {
+ size_t i;
+ for (i = 0; i < pTreeItem->m_vecChildren.size(); i++) {
+ char *p = pTreeItem->m_vecChildren[i]->m_pszTerm;
+ if (p[0] == '*' && p[strlen(p) - 1] == '*') {
+ iHeadchild = i;
+ p[strlen(p) - 1] = '\0';
+ std::string str = p + 1;
+ strcpy(p, str.c_str()); // erase the "*..*"
+ break;
+ }
+ }
+ assert(i < pTreeItem->m_vecChildren.size());
+ }
+
+ pTreeItem->m_iHeadChild = iHeadchild;
+ pTreeItem->m_iHeadWord = pTreeItem->m_vecChildren[iHeadchild]->m_iHeadWord;
+ }
+ void fnPostProcessingFlattenedParse(SParsedTree *pTree) {
+ pTree->fnSetSpanInfo();
+ fnSetParseTreeHeadInfo(pTree);
+ }
+ bool fnReadNextSentence(char *pszLine, int *piLength) {
+ if (feof(m_fpIn) == true) return false;
+
+ int iLen;
+
+ pszLine[0] = '\0';
+
+ fgets(pszLine, 10001, m_fpIn);
+ iLen = strlen(pszLine);
+ while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
+ pszLine[iLen - 1] = '\0';
+ iLen--;
+ }
+
+ if (piLength != NULL) (*piLength) = iLen;
+
+ return true;
+ }
+
+ private:
+ FILE *m_fpIn;
+ const bool m_bFlattened;
+};
+
+/*
+ * Note:
+ * m_vec_s_align.size() may not be equal to the length of source side
+ *sentence
+ * due to the last words may not be aligned
+ *
+ */
+struct SAlignment {
+ typedef std::vector<int> SingleAlign;
+ SAlignment(const char* pszAlign) { fnInitializeAlignment(pszAlign); }
+ ~SAlignment() {}
+
+ bool fnIsAligned(int i, bool s) const {
+ const std::vector<SingleAlign>* palign;
+ if (s == true)
+ palign = &m_vec_s_align;
+ else
+ palign = &m_vec_t_align;
+ if ((*palign)[i].size() == 0) return false;
+ return true;
+ }
+
+ /*
+ * return true if [b, e] is aligned phrases on source side (if s==true) or on
+ * the target side (if s==false);
+ * return false, otherwise.
+ */
+ bool fnIsAlignedPhrase(int b, int e, bool s, int* pob, int* poe) const {
+ int ob, oe; //[b, e] on the other side
+ if (s == true)
+ fnGetLeftRightMost(b, e, m_vec_s_align, ob, oe);
+ else
+ fnGetLeftRightMost(b, e, m_vec_t_align, ob, oe);
+
+ if (ob == -1) {
+ if (pob != NULL) (*pob) = -1;
+ if (poe != NULL) (*poe) = -1;
+ return false; // no aligned word among [b, e]
+ }
+ if (pob != NULL) (*pob) = ob;
+ if (poe != NULL) (*poe) = oe;
+
+ int bb, be; //[b, e] back given [ob, oe] on the other side
+ if (s == true)
+ fnGetLeftRightMost(ob, oe, m_vec_t_align, bb, be);
+ else
+ fnGetLeftRightMost(ob, oe, m_vec_s_align, bb, be);
+
+ if (bb < b || be > e) return false;
+ return true;
+ }
+
+ bool fnIsAlignedTightPhrase(int b, int e, bool s, int* pob, int* poe) const {
+ const std::vector<SingleAlign>* palign;
+ if (s == true)
+ palign = &m_vec_s_align;
+ else
+ palign = &m_vec_t_align;
+
+ if ((*palign).size() <= e || (*palign)[b].size() == 0 ||
+ (*palign)[e].size() == 0)
+ return false;
+
+ return fnIsAlignedPhrase(b, e, s, pob, poe);
+ }
+
+ void fnGetLeftRightMost(int b, int e, bool s, int& ob, int& oe) const {
+ if (s == true)
+ fnGetLeftRightMost(b, e, m_vec_s_align, ob, oe);
+ else
+ fnGetLeftRightMost(b, e, m_vec_t_align, ob, oe);
+ }
+
+ /*
+ * look the translation of source[b, e] is continuous or not
+ * 1) return "Unaligned": if the source[b, e] is translated silently;
+ * 2) return "Con't": if none of target words in target[.., ..] is exclusively
+ * aligned to any word outside source[b, e]
+ * 3) return "Discon't": otherwise;
+ */
+ std::string fnIsContinuous(int b, int e) const {
+ int ob, oe;
+ fnGetLeftRightMost(b, e, true, ob, oe);
+ if (ob == -1) return "Unaligned";
+
+ for (int i = ob; i <= oe; i++) {
+ if (!fnIsAligned(i, false)) continue;
+ const SingleAlign& a = m_vec_t_align[i];
+ int j;
+ for (j = 0; j < a.size(); j++)
+ if (a[j] >= b && a[j] <= e) break;
+ if (j == a.size()) return "Discon't";
+ }
+ return "Con't";
+ }
+
+ const SingleAlign* fnGetSingleWordAlign(int i, bool s) const {
+ if (s == true) {
+ if (i >= m_vec_s_align.size()) return NULL;
+ return &(m_vec_s_align[i]);
+ } else {
+ if (i >= m_vec_t_align.size()) return NULL;
+ return &(m_vec_t_align[i]);
+ }
+ }
+
+ private:
+ void fnGetLeftRightMost(int b, int e, const std::vector<SingleAlign>& align,
+ int& ob, int& oe) const {
+ ob = oe = -1;
+ for (int i = b; i <= e && i < align.size(); i++) {
+ if (align[i].size() > 0) {
+ if (align[i][0] < ob || ob == -1) ob = align[i][0];
+ if (oe < align[i][align[i].size() - 1])
+ oe = align[i][align[i].size() - 1];
+ }
+ }
+ }
+ void fnInitializeAlignment(const char* pszAlign) {
+ m_vec_s_align.clear();
+ m_vec_t_align.clear();
+
+ std::vector<std::string> terms = SplitOnWhitespace(std::string(pszAlign));
+ int si, ti;
+ for (size_t i = 0; i < terms.size(); i++) {
+ sscanf(terms[i].c_str(), "%d-%d", &si, &ti);
+
+ while (m_vec_s_align.size() <= si) {
+ SingleAlign sa;
+ m_vec_s_align.push_back(sa);
+ }
+ while (m_vec_t_align.size() <= ti) {
+ SingleAlign sa;
+ m_vec_t_align.push_back(sa);
+ }
+
+ m_vec_s_align[si].push_back(ti);
+ m_vec_t_align[ti].push_back(si);
+ }
+
+ // sort
+ for (size_t i = 0; i < m_vec_s_align.size(); i++) {
+ std::sort(m_vec_s_align[i].begin(), m_vec_s_align[i].end());
+ }
+ for (size_t i = 0; i < m_vec_t_align.size(); i++) {
+ std::sort(m_vec_t_align[i].begin(), m_vec_t_align[i].end());
+ }
+ }
+
+ private:
+ std::vector<SingleAlign> m_vec_s_align; // source side words' alignment
+ std::vector<SingleAlign> m_vec_t_align; // target side words' alignment
+};
+
+struct SAlignmentReader {
+ SAlignmentReader(const char* pszFname) {
+ m_fpIn = fopen(pszFname, "r");
+ assert(m_fpIn != NULL);
+ }
+ ~SAlignmentReader() {
+ if (m_fpIn != NULL) fclose(m_fpIn);
+ }
+ SAlignment* fnReadNextAlignment() {
+ if (feof(m_fpIn) == true) return NULL;
+ char* pszLine = new char[100001];
+ pszLine[0] = '\0';
+ fgets(pszLine, 10001, m_fpIn);
+ int iLen = strlen(pszLine);
+ if (iLen == 0) return NULL;
+ while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
+ pszLine[iLen - 1] = '\0';
+ iLen--;
+ }
+ SAlignment* pAlign = new SAlignment(pszLine);
+ delete[] pszLine;
+ return pAlign;
+ }
+
+ private:
+ FILE* m_fpIn;
+};
+
+struct SArgument {
+ SArgument(const char* pszRole, int iBegin, int iEnd, float fProb) {
+ m_pszRole = new char[strlen(pszRole) + 1];
+ strcpy(m_pszRole, pszRole);
+ m_iBegin = iBegin;
+ m_iEnd = iEnd;
+ m_fProb = fProb;
+ m_pTreeItem = NULL;
+ }
+ ~SArgument() { delete[] m_pszRole; }
+
+ void fnSetTreeItem(STreeItem* pTreeItem) {
+ m_pTreeItem = pTreeItem;
+ if (m_pTreeItem != NULL && m_pTreeItem->m_iBegin != -1) {
+ assert(m_pTreeItem->m_iBegin == m_iBegin);
+ assert(m_pTreeItem->m_iEnd == m_iEnd);
+ }
+ }
+
+ char* m_pszRole; // argument rule, e.g., ARG0, ARGM-TMP
+ int m_iBegin;
+ int m_iEnd; // the span of the argument, [m_iBegin, m_iEnd]
+ float m_fProb; // the probability of this role,
+ STreeItem* m_pTreeItem;
+};
+
+struct SPredicate {
+ SPredicate(const char* pszLemma, int iPosition) {
+ if (pszLemma != NULL) {
+ m_pszLemma = new char[strlen(pszLemma) + 1];
+ strcpy(m_pszLemma, pszLemma);
+ } else
+ m_pszLemma = NULL;
+ m_iPosition = iPosition;
+ }
+ ~SPredicate() {
+ if (m_pszLemma != NULL) delete[] m_pszLemma;
+ for (size_t i = 0; i < m_vecArgt.size(); i++) delete m_vecArgt[i];
+ }
+ int fnAppend(const char* pszRole, int iBegin, int iEnd) {
+ SArgument* pArgt = new SArgument(pszRole, iBegin, iEnd, 1.0);
+ return fnAppend(pArgt);
+ }
+ int fnAppend(SArgument* pArgt) {
+ m_vecArgt.push_back(pArgt);
+ int iPosition = m_vecArgt.size() - 1;
+ return iPosition;
+ }
+
+ char* m_pszLemma; // lemma of the predicate, for Chinese, it's always as same
+ // as the predicate itself
+ int m_iPosition; // the position in sentence
+ std::vector<SArgument*> m_vecArgt; // arguments associated to the predicate
+};
+
+struct SSrlSentence {
+ SSrlSentence() { m_pTree = NULL; }
+ ~SSrlSentence() {
+ if (m_pTree != NULL) delete m_pTree;
+
+ for (size_t i = 0; i < m_vecPred.size(); i++) delete m_vecPred[i];
+ }
+ int fnAppend(const char* pszLemma, int iPosition) {
+ SPredicate* pPred = new SPredicate(pszLemma, iPosition);
+ return fnAppend(pPred);
+ }
+ int fnAppend(SPredicate* pPred) {
+ m_vecPred.push_back(pPred);
+ int iPosition = m_vecPred.size() - 1;
+ return iPosition;
+ }
+ int GetPredicateNum() { return m_vecPred.size(); }
+
+ SParsedTree* m_pTree;
+ std::vector<SPredicate*> m_vecPred;
+};
+
+struct SSrlSentenceReader {
+ SSrlSentenceReader(const char* pszSrlFname) {
+ m_fpIn = fopen(pszSrlFname, "r");
+ assert(m_fpIn != NULL);
+ }
+ ~SSrlSentenceReader() {
+ if (m_fpIn != NULL) fclose(m_fpIn);
+ }
+
+ inline void fnReplaceAll(std::string& str, const std::string& from,
+ const std::string& to) {
+ size_t start_pos = 0;
+ while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
+ str.replace(start_pos, from.length(), to);
+ start_pos += to.length(); // In case 'to' contains 'from', like replacing
+ // 'x' with 'yx'
+ }
+ }
+
+ // TODO: here only considers flat predicate-argument structure
+ // i.e., no overlap among them
+ SSrlSentence* fnReadNextSrlSentence() {
+ std::vector<std::vector<std::string> > vecContent;
+ if (fnReadNextContent(vecContent) == false) return NULL;
+
+ SSrlSentence* pSrlSentence = new SSrlSentence();
+ int iSize = vecContent.size();
+ // put together syntactic text
+ std::ostringstream ostr;
+ for (int i = 0; i < iSize; i++) {
+ std::string strSynSeg =
+ vecContent[i][5]; // the 5th column is the syntactic segment
+ size_t iPosition = strSynSeg.find_first_of('*');
+ assert(iPosition != std::string::npos);
+ std::ostringstream ostrTmp;
+ ostrTmp << "(" << vecContent[i][2] << " " << vecContent[i][0]
+ << ")"; // the 2th column is POS-tag, and the 0th column is word
+ strSynSeg.replace(iPosition, 1, ostrTmp.str());
+ fnReplaceAll(strSynSeg, "(", " (");
+ ostr << strSynSeg;
+ }
+ std::string strSyn = ostr.str();
+ pSrlSentence->m_pTree = SParsedTree::fnConvertFromString(strSyn.c_str());
+ pSrlSentence->m_pTree->fnSetHeadWord();
+ pSrlSentence->m_pTree->fnSetSpanInfo();
+
+ // read predicate-argument structure
+ int iNumPred = vecContent[0].size() - 8;
+ for (int i = 0; i < iNumPred; i++) {
+ std::vector<std::string> vecRole;
+ std::vector<int> vecBegin;
+ std::vector<int> vecEnd;
+ int iPred = -1;
+ for (int j = 0; j < iSize; j++) {
+ const char* p = vecContent[j][i + 8].c_str();
+ const char* q;
+ if (p[0] == '(') {
+ // starting position of an argument(or predicate)
+ vecBegin.push_back(j);
+ q = strchr(p, '*');
+ assert(q != NULL);
+ vecRole.push_back(vecContent[j][i + 8].substr(1, q - p - 1));
+ if (vecRole.back().compare("V") == 0) {
+ assert(iPred == -1);
+ iPred = vecRole.size() - 1;
+ }
+ }
+ if (p[strlen(p) - 1] == ')') {
+ // end position of an argument(or predicate)
+ vecEnd.push_back(j);
+ assert(vecBegin.size() == vecEnd.size());
+ }
+ }
+ assert(iPred != -1);
+ SPredicate* pPred = new SPredicate(
+ pSrlSentence->m_pTree->m_vecTerminals[vecBegin[iPred]]->m_pszTerm,
+ vecBegin[iPred]);
+ pSrlSentence->fnAppend(pPred);
+ for (size_t j = 0; j < vecBegin.size(); j++) {
+ if (j == iPred) continue;
+ pPred->fnAppend(vecRole[j].c_str(), vecBegin[j], vecEnd[j]);
+ pPred->m_vecArgt.back()->fnSetTreeItem(
+ pSrlSentence->m_pTree->fnFindNodeForSpan(vecBegin[j], vecEnd[j],
+ false));
+ }
+ }
+ return pSrlSentence;
+ }
+
+ private:
+ bool fnReadNextContent(std::vector<std::vector<std::string> >& vecContent) {
+ vecContent.clear();
+ if (feof(m_fpIn) == true) return false;
+ char* pszLine;
+ pszLine = new char[100001];
+ pszLine[0] = '\0';
+ int iLen;
+ while (!feof(m_fpIn)) {
+ fgets(pszLine, 10001, m_fpIn);
+ iLen = strlen(pszLine);
+ while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
+ pszLine[iLen - 1] = '\0';
+ iLen--;
+ }
+ if (iLen == 0) break; // end of this sentence
+
+ std::vector<std::string> terms = SplitOnWhitespace(std::string(pszLine));
+ assert(terms.size() > 7);
+ vecContent.push_back(terms);
+ }
+ delete[] pszLine;
+ return true;
+ }
+
+ private:
+ FILE* m_fpIn;
+};
+
+typedef std::unordered_map<std::string, int> Map;
+typedef std::unordered_map<std::string, int>::iterator Iterator;
+
+struct Tsuruoka_Maxent {
+ Tsuruoka_Maxent(const char* pszModelFName) {
+ if (pszModelFName != NULL) {
+ m_pModel = new ME_Model();
+ m_pModel->load_from_file(pszModelFName);
+ } else
+ m_pModel = NULL;
+ }
+
+ ~Tsuruoka_Maxent() {
+ if (m_pModel != NULL) delete m_pModel;
+ }
+
+ void fnTrain(const char* pszInstanceFName, const char* pszAlgorithm,
+ const char* pszModelFName, int /*iNumIteration*/) {
+ assert(strcmp(pszAlgorithm, "l1") == 0 || strcmp(pszAlgorithm, "l2") == 0 ||
+ strcmp(pszAlgorithm, "sgd") == 0 ||
+ strcmp(pszAlgorithm, "SGD") == 0);
+ FILE* fpIn = fopen(pszInstanceFName, "r");
+
+ ME_Model* pModel = new ME_Model();
+
+ char* pszLine = new char[100001];
+ int iNumInstances = 0;
+ int iLen;
+ while (!feof(fpIn)) {
+ pszLine[0] = '\0';
+ fgets(pszLine, 20000, fpIn);
+ if (strlen(pszLine) == 0) {
+ continue;
+ }
+
+ iLen = strlen(pszLine);
+ while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
+ pszLine[iLen - 1] = '\0';
+ iLen--;
+ }
+
+ iNumInstances++;
+
+ ME_Sample* pmes = new ME_Sample();
+
+ char* p = strrchr(pszLine, ' ');
+ assert(p != NULL);
+ p[0] = '\0';
+ p++;
+ std::vector<std::string> vecContext;
+ SplitOnWhitespace(std::string(pszLine), &vecContext);
+
+ pmes->label = std::string(p);
+ for (size_t i = 0; i < vecContext.size(); i++)
+ pmes->add_feature(vecContext[i]);
+ pModel->add_training_sample((*pmes));
+ if (iNumInstances % 100000 == 0)
+ fprintf(stdout, "......Reading #Instances: %1d\n", iNumInstances);
+ delete pmes;
+ }
+ fprintf(stdout, "......Reading #Instances: %1d\n", iNumInstances);
+ fclose(fpIn);
+
+ if (strcmp(pszAlgorithm, "l1") == 0)
+ pModel->use_l1_regularizer(1.0);
+ else if (strcmp(pszAlgorithm, "l2") == 0)
+ pModel->use_l2_regularizer(1.0);
+ else
+ pModel->use_SGD();
+
+ pModel->train();
+ pModel->save_to_file(pszModelFName);
+
+ delete pModel;
+ fprintf(stdout, "......Finished Training\n");
+ fprintf(stdout, "......Model saved as %s\n", pszModelFName);
+ delete[] pszLine;
+ }
+
+ double fnEval(const char* pszContext, const char* pszOutcome) const {
+ std::vector<std::string> vecContext;
+ ME_Sample* pmes = new ME_Sample();
+ SplitOnWhitespace(std::string(pszContext), &vecContext);
+
+ for (size_t i = 0; i < vecContext.size(); i++)
+ pmes->add_feature(vecContext[i]);
+ std::vector<double> vecProb = m_pModel->classify(*pmes);
+ delete pmes;
+ int iLableID = m_pModel->get_class_id(pszOutcome);
+ return vecProb[iLableID];
+ }
+ void fnEval(const char* pszContext,
+ std::vector<std::pair<std::string, double> >& vecOutput) const {
+ std::vector<std::string> vecContext;
+ ME_Sample* pmes = new ME_Sample();
+ SplitOnWhitespace(std::string(pszContext), &vecContext);
+
+ vecOutput.clear();
+
+ for (size_t i = 0; i < vecContext.size(); i++)
+ pmes->add_feature(vecContext[i]);
+ std::vector<double> vecProb = m_pModel->classify(*pmes);
+
+ for (size_t i = 0; i < vecProb.size(); i++) {
+ std::string label = m_pModel->get_class_label(i);
+ vecOutput.push_back(make_pair(label, vecProb[i]));
+ }
+ delete pmes;
+ }
+ void fnEval(const char* pszContext, std::vector<double>& vecOutput) const {
+ std::vector<std::string> vecContext;
+ ME_Sample* pmes = new ME_Sample();
+ SplitOnWhitespace(std::string(pszContext), &vecContext);
+
+ vecOutput.clear();
+
+ for (size_t i = 0; i < vecContext.size(); i++)
+ pmes->add_feature(vecContext[i]);
+ std::vector<double> vecProb = m_pModel->classify(*pmes);
+
+ for (size_t i = 0; i < vecProb.size(); i++) {
+ std::string label = m_pModel->get_class_label(i);
+ vecOutput.push_back(vecProb[i]);
+ }
+ delete pmes;
+ }
+ int fnGetClassId(const std::string& strLabel) const {
+ return m_pModel->get_class_id(strLabel);
+ }
+
+ private:
+ ME_Model* m_pModel;
+};
+
+// an argument item or a predicate item (the verb itself)
+struct SSRLItem {
+ SSRLItem(const STreeItem *tree_item, std::string role)
+ : tree_item_(tree_item), role_(role) {}
+ ~SSRLItem() {}
+ const STreeItem *tree_item_;
+ const std::string role_;
+};
+
+struct SPredicateItem {
+ SPredicateItem(const SParsedTree *tree, const SPredicate *pred)
+ : pred_(pred) {
+ vec_items_.reserve(pred->m_vecArgt.size() + 1);
+ for (int i = 0; i < pred->m_vecArgt.size(); i++) {
+ vec_items_.push_back(
+ new SSRLItem(pred->m_vecArgt[i]->m_pTreeItem,
+ std::string(pred->m_vecArgt[i]->m_pszRole)));
+ }
+ vec_items_.push_back(
+ new SSRLItem(tree->m_vecTerminals[pred->m_iPosition]->m_ptParent,
+ std::string("Pred")));
+ sort(vec_items_.begin(), vec_items_.end(), SortFunction);
+
+ begin_ = vec_items_[0]->tree_item_->m_iBegin;
+ end_ = vec_items_[vec_items_.size() - 1]->tree_item_->m_iEnd;
+ }
+
+ ~SPredicateItem() { vec_items_.clear(); }
+
+ static bool SortFunction(SSRLItem *i, SSRLItem *j) {
+ return (i->tree_item_->m_iBegin < j->tree_item_->m_iBegin);
+ }
+
+ std::vector<SSRLItem *> vec_items_;
+ int begin_;
+ int end_;
+ const SPredicate *pred_;
+};
+
+struct SArgumentReorderModel {
+ public:
+ static std::string fnGetBlockOutcome(int iBegin, int iEnd,
+ SAlignment *pAlign) {
+ return pAlign->fnIsContinuous(iBegin, iEnd);
+ }
+ static void fnGetReorderType(SPredicateItem *pPredItem, SAlignment *pAlign,
+ std::vector<std::string> &vecStrLeftReorder,
+ std::vector<std::string> &vecStrRightReorder) {
+ std::vector<int> vecLeft, vecRight;
+ for (int i = 0; i < pPredItem->vec_items_.size(); i++) {
+ const STreeItem *pCon1 = pPredItem->vec_items_[i]->tree_item_;
+ int iLeft1, iRight1;
+ pAlign->fnGetLeftRightMost(pCon1->m_iBegin, pCon1->m_iEnd, true, iLeft1,
+ iRight1);
+ vecLeft.push_back(iLeft1);
+ vecRight.push_back(iRight1);
+ }
+ std::vector<int> vecLeftPosition;
+ fnGetRelativePosition(vecLeft, vecLeftPosition);
+ std::vector<int> vecRightPosition;
+ fnGetRelativePosition(vecRight, vecRightPosition);
+
+ vecStrLeftReorder.clear();
+ vecStrRightReorder.clear();
+ for (int i = 1; i < vecLeftPosition.size(); i++) {
+ std::string strOutcome;
+ fnGetOutcome(vecLeftPosition[i - 1], vecLeftPosition[i], strOutcome);
+ vecStrLeftReorder.push_back(strOutcome);
+ fnGetOutcome(vecRightPosition[i - 1], vecRightPosition[i], strOutcome);
+ vecStrRightReorder.push_back(strOutcome);
+ }
+ }
+
+ /*
+ * features:
+ * f1: (left_label, right_label, parent_label)
+ * f2: (left_label, right_label, parent_label, other_right_sibling_label)
+ * f3: (left_label, right_label, parent_label, other_left_sibling_label)
+ * f4: (left_label, right_label, left_head_pos)
+ * f5: (left_label, right_label, left_head_word)
+ * f6: (left_label, right_label, right_head_pos)
+ * f7: (left_label, right_label, right_head_word)
+ * f8: (left_label, right_label, left_chunk_status)
+ * f9: (left_label, right_label, right_chunk_status)
+ * f10: (left_label, parent_label)
+ * f11: (right_label, parent_label)
+ *
+ * f1: (left_role, right_role, predicate_term)
+ * f2: (left_role, right_role, predicate_term, other_right_role)
+ * f3: (left_role, right_role, predicate_term, other_left_role)
+ * f4: (left_role, right_role, left_head_pos)
+ * f5: (left_role, right_role, left_head_word)
+ * f6: (left_role, right_role, left_syntactic_label)
+ * f7: (left_role, right_role, right_head_pos)
+ * f8: (left_role, right_role, right_head_word)
+ * f8: (left_role, right_role, right_syntactic_label)
+ * f8: (left_role, right_role, left_chunk_status)
+ * f9: (left_role, right_role, right_chunk_status)
+ * f10: (left_role, right_role, left_chunk_status)
+ * f11: (left_role, right_role, right_chunk_status)
+ * f12: (left_label, parent_label)
+ * f13: (right_label, parent_label)
+ */
+ static void fnGenerateFeature(const SParsedTree *pTree,
+ const SPredicate *pPred,
+ const SPredicateItem *pPredItem, int iPos,
+ const std::string &strBlock1,
+ const std::string &strBlock2,
+ std::ostringstream &ostr) {
+ SSRLItem *pSRLItem1 = pPredItem->vec_items_[iPos - 1];
+ SSRLItem *pSRLItem2 = pPredItem->vec_items_[iPos];
+ const STreeItem *pCon1 = pSRLItem1->tree_item_;
+ const STreeItem *pCon2 = pSRLItem2->tree_item_;
+
+ std::string left_role = pSRLItem1->role_;
+ std::string right_role = pSRLItem2->role_;
+
+ std::string predicate_term =
+ pTree->m_vecTerminals[pPred->m_iPosition]->m_pszTerm;
+
+ std::vector<std::string> vec_other_right_sibling;
+ for (int i = iPos + 1; i < pPredItem->vec_items_.size(); i++)
+ vec_other_right_sibling.push_back(
+ std::string(pPredItem->vec_items_[i]->role_));
+ if (vec_other_right_sibling.size() == 0)
+ vec_other_right_sibling.push_back(std::string("NULL"));
+
+ std::vector<std::string> vec_other_left_sibling;
+ for (int i = 0; i < iPos - 1; i++)
+ vec_other_right_sibling.push_back(
+ std::string(pPredItem->vec_items_[i]->role_));
+ if (vec_other_left_sibling.size() == 0)
+ vec_other_left_sibling.push_back(std::string("NULL"));
+
+ // generate features
+ // f1
+ ostr << "f1=" << left_role << "_" << right_role << "_" << predicate_term;
+ ostr << "f1=" << left_role << "_" << right_role;
+
+ // f2
+ for (int i = 0; i < vec_other_right_sibling.size(); i++) {
+ ostr << " f2=" << left_role << "_" << right_role << "_" << predicate_term
+ << "_" << vec_other_right_sibling[i];
+ ostr << " f2=" << left_role << "_" << right_role << "_"
+ << vec_other_right_sibling[i];
+ }
+ // f3
+ for (int i = 0; i < vec_other_left_sibling.size(); i++) {
+ ostr << " f3=" << left_role << "_" << right_role << "_" << predicate_term
+ << "_" << vec_other_left_sibling[i];
+ ostr << " f3=" << left_role << "_" << right_role << "_"
+ << vec_other_left_sibling[i];
+ }
+ // f4
+ ostr << " f4=" << left_role << "_" << right_role << "_"
+ << pTree->m_vecTerminals[pCon1->m_iHeadWord]->m_ptParent->m_pszTerm;
+ // f5
+ ostr << " f5=" << left_role << "_" << right_role << "_"
+ << pTree->m_vecTerminals[pCon1->m_iHeadWord]->m_pszTerm;
+ // f6
+ ostr << " f6=" << left_role << "_" << right_role << "_" << pCon2->m_pszTerm;
+ // f7
+ ostr << " f7=" << left_role << "_" << right_role << "_"
+ << pTree->m_vecTerminals[pCon2->m_iHeadWord]->m_ptParent->m_pszTerm;
+ // f8
+ ostr << " f8=" << left_role << "_" << right_role << "_"
+ << pTree->m_vecTerminals[pCon2->m_iHeadWord]->m_pszTerm;
+ // f9
+ ostr << " f9=" << left_role << "_" << right_role << "_" << pCon2->m_pszTerm;
+ // f10
+ ostr << " f10=" << left_role << "_" << right_role << "_" << strBlock1;
+ // f11
+ ostr << " f11=" << left_role << "_" << right_role << "_" << strBlock2;
+ // f12
+ ostr << " f12=" << left_role << "_" << predicate_term;
+ ostr << " f12=" << left_role;
+ // f13
+ ostr << " f13=" << right_role << "_" << predicate_term;
+ ostr << " f13=" << right_role;
+ }
+
+ private:
+ static void fnGetOutcome(int i1, int i2, std::string &strOutcome) {
+ assert(i1 != i2);
+ if (i1 < i2) {
+ if (i2 > i1 + 1)
+ strOutcome = std::string("DM");
+ else
+ strOutcome = std::string("M");
+ } else {
+ if (i1 > i2 + 1)
+ strOutcome = std::string("DS");
+ else
+ strOutcome = std::string("S");
+ }
+ }
+
+ static void fnGetRelativePosition(const std::vector<int> &vecLeft,
+ std::vector<int> &vecPosition) {
+ vecPosition.clear();
+
+ std::vector<float> vec;
+ for (int i = 0; i < vecLeft.size(); i++) {
+ if (vecLeft[i] == -1) {
+ if (i == 0)
+ vec.push_back(-1);
+ else
+ vec.push_back(vecLeft[i - 1] + 0.1);
+ } else
+ vec.push_back(vecLeft[i]);
+ }
+
+ for (int i = 0; i < vecLeft.size(); i++) {
+ int count = 0;
+
+ for (int j = 0; j < vecLeft.size(); j++) {
+ if (j == i) continue;
+ if (vec[j] < vec[i]) {
+ count++;
+ } else if (vec[j] == vec[i] && j < i) {
+ count++;
+ }
+ }
+ vecPosition.push_back(count);
+ }
+ }
+};
+} // namespace const_reorder
+
+#endif // _FF_CONST_REORDER_COMMON_H
diff --git a/training/Makefile.am b/training/Makefile.am
index 8ef3c939..2812a9be 100644
--- a/training/Makefile.am
+++ b/training/Makefile.am
@@ -8,5 +8,5 @@ SUBDIRS = \
dtrain \
latent_svm \
mira \
- rampion
-
+ rampion \
+ const_reorder
diff --git a/training/const_reorder/Makefile.am b/training/const_reorder/Makefile.am
new file mode 100644
index 00000000..2e81e588
--- /dev/null
+++ b/training/const_reorder/Makefile.am
@@ -0,0 +1,8 @@
+bin_PROGRAMS = const_reorder_model_trainer argument_reorder_model_trainer
+
+AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder
+
+const_reorder_model_trainer_SOURCES = constituent_reorder_model.cc
+const_reorder_model_trainer_LDADD = ../../utils/libutils.a
+argument_reorder_model_trainer_SOURCES = argument_reorder_model.cc
+argument_reorder_model_trainer_LDADD = ../../utils/libutils.a
diff --git a/utils/argument_reorder_model.cc b/training/const_reorder/argument_reorder_model.cc
index c4e90cba..54402436 100644
--- a/utils/argument_reorder_model.cc
+++ b/training/const_reorder/argument_reorder_model.cc
@@ -12,12 +12,12 @@
#include <string>
#include <vector>
-#include "filelib.h"
+#include "utils/filelib.h"
-#include "argument_reorder_model.h"
-#include "tsuruoka_maxent.h"
+#include "decoder/ff_const_reorder_common.h"
using namespace std;
+using namespace const_reorder;
inline void fnPreparingTrainingdata(const char* pszFName, int iCutoff,
const char* pszNewFName) {
diff --git a/utils/constituent_reorder_model.cc b/training/const_reorder/constituent_reorder_model.cc
index bdb7c5d1..6bec3f0b 100644
--- a/utils/constituent_reorder_model.cc
+++ b/training/const_reorder/constituent_reorder_model.cc
@@ -10,13 +10,12 @@
#include <boost/program_options.hpp>
-#include "filelib.h"
+#include "utils/filelib.h"
-#include "alignment.h"
-#include "tree.h"
-#include "tsuruoka_maxent.h"
+#include "decoder/ff_const_reorder_common.h"
using namespace std;
+using namespace const_reorder;
typedef std::unordered_map<std::string, int> Map;
typedef std::unordered_map<std::string, int>::iterator Iterator;
diff --git a/utils/Makefile.am b/utils/Makefile.am
index 53967561..fabb4454 100644
--- a/utils/Makefile.am
+++ b/utils/Makefile.am
@@ -1,4 +1,4 @@
-bin_PROGRAMS = reconstruct_weights atools const_reorder_model_trainer argument_reorder_model_trainer
+bin_PROGRAMS = reconstruct_weights atools
noinst_PROGRAMS = \
ts \
@@ -20,9 +20,7 @@ noinst_LIBRARIES = libutils.a
libutils_a_SOURCES = \
test_data \
alias_sampler.h \
- alignment.h \
alignment_io.h \
- argument_reorder_model.h \
array2d.h \
b64tools.h \
batched_append.h \
@@ -61,15 +59,12 @@ libutils_a_SOURCES = \
show.h \
small_vector.h \
sparse_vector.h \
- srl_sentence.h \
star.h \
static_utoa.h \
stringlib.h \
string_piece.hh \
tdict.h \
timing_stats.h \
- tree.h \
- tsuruoka_maxent.h \
utoa.h \
value_array.h \
verbose.h \
@@ -91,7 +86,6 @@ libutils_a_SOURCES = \
filelib.cc \
stringlib.cc \
string_piece.cc \
- synutils.h \
sparse_vector.cc \
timing_stats.cc \
verbose.cc \
@@ -106,10 +100,6 @@ reconstruct_weights_LDADD = libutils.a
atools_SOURCES = atools.cc
atools_LDADD = libutils.a
atools_LDFLAGS = $(STATIC_FLAGS)
-const_reorder_model_trainer_SOURCES = constituent_reorder_model.cc
-const_reorder_model_trainer_LDADD = libutils.a
-argument_reorder_model_trainer_SOURCES = argument_reorder_model.cc
-argument_reorder_model_trainer_LDADD = libutils.a
phmt_SOURCES = phmt.cc
phmt_LDADD = libutils.a $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS)
diff --git a/utils/alignment.h b/utils/alignment.h
deleted file mode 100644
index 456577ca..00000000
--- a/utils/alignment.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * alignment.h
- *
- * Created on: May 23, 2013
- * Author: lijunhui
- */
-
-#ifndef ALIGNMENT_H_
-#define ALIGNMENT_H_
-
-#include <string>
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "stringlib.h"
-
-/*
- * Note:
- * m_vec_s_align.size() may not be equal to the length of source side
- *sentence
- * due to the last words may not be aligned
- *
- */
-struct SAlignment {
- typedef std::vector<int> SingleAlign;
- SAlignment(const char* pszAlign) { fnInitializeAlignment(pszAlign); }
- ~SAlignment() {}
-
- bool fnIsAligned(int i, bool s) const {
- const std::vector<SingleAlign>* palign;
- if (s == true)
- palign = &m_vec_s_align;
- else
- palign = &m_vec_t_align;
- if ((*palign)[i].size() == 0) return false;
- return true;
- }
-
- /*
- * return true if [b, e] is aligned phrases on source side (if s==true) or on
- * the target side (if s==false);
- * return false, otherwise.
- */
- bool fnIsAlignedPhrase(int b, int e, bool s, int* pob, int* poe) const {
- int ob, oe; //[b, e] on the other side
- if (s == true)
- fnGetLeftRightMost(b, e, m_vec_s_align, ob, oe);
- else
- fnGetLeftRightMost(b, e, m_vec_t_align, ob, oe);
-
- if (ob == -1) {
- if (pob != NULL) (*pob) = -1;
- if (poe != NULL) (*poe) = -1;
- return false; // no aligned word among [b, e]
- }
- if (pob != NULL) (*pob) = ob;
- if (poe != NULL) (*poe) = oe;
-
- int bb, be; //[b, e] back given [ob, oe] on the other side
- if (s == true)
- fnGetLeftRightMost(ob, oe, m_vec_t_align, bb, be);
- else
- fnGetLeftRightMost(ob, oe, m_vec_s_align, bb, be);
-
- if (bb < b || be > e) return false;
- return true;
- }
-
- bool fnIsAlignedTightPhrase(int b, int e, bool s, int* pob, int* poe) const {
- const std::vector<SingleAlign>* palign;
- if (s == true)
- palign = &m_vec_s_align;
- else
- palign = &m_vec_t_align;
-
- if ((*palign).size() <= e || (*palign)[b].size() == 0 ||
- (*palign)[e].size() == 0)
- return false;
-
- return fnIsAlignedPhrase(b, e, s, pob, poe);
- }
-
- void fnGetLeftRightMost(int b, int e, bool s, int& ob, int& oe) const {
- if (s == true)
- fnGetLeftRightMost(b, e, m_vec_s_align, ob, oe);
- else
- fnGetLeftRightMost(b, e, m_vec_t_align, ob, oe);
- }
-
- /*
- * look the translation of source[b, e] is continuous or not
- * 1) return "Unaligned": if the source[b, e] is translated silently;
- * 2) return "Con't": if none of target words in target[.., ..] is exclusively
- * aligned to any word outside source[b, e]
- * 3) return "Discon't": otherwise;
- */
- std::string fnIsContinuous(int b, int e) const {
- int ob, oe;
- fnGetLeftRightMost(b, e, true, ob, oe);
- if (ob == -1) return "Unaligned";
-
- for (int i = ob; i <= oe; i++) {
- if (!fnIsAligned(i, false)) continue;
- const SingleAlign& a = m_vec_t_align[i];
- int j;
- for (j = 0; j < a.size(); j++)
- if (a[j] >= b && a[j] <= e) break;
- if (j == a.size()) return "Discon't";
- }
- return "Con't";
- }
-
- const SingleAlign* fnGetSingleWordAlign(int i, bool s) const {
- if (s == true) {
- if (i >= m_vec_s_align.size()) return NULL;
- return &(m_vec_s_align[i]);
- } else {
- if (i >= m_vec_t_align.size()) return NULL;
- return &(m_vec_t_align[i]);
- }
- }
-
- private:
- void fnGetLeftRightMost(int b, int e, const std::vector<SingleAlign>& align,
- int& ob, int& oe) const {
- ob = oe = -1;
- for (int i = b; i <= e && i < align.size(); i++) {
- if (align[i].size() > 0) {
- if (align[i][0] < ob || ob == -1) ob = align[i][0];
- if (oe < align[i][align[i].size() - 1])
- oe = align[i][align[i].size() - 1];
- }
- }
- }
- void fnInitializeAlignment(const char* pszAlign) {
- m_vec_s_align.clear();
- m_vec_t_align.clear();
-
- std::vector<std::string> terms = SplitOnWhitespace(std::string(pszAlign));
- int si, ti;
- for (size_t i = 0; i < terms.size(); i++) {
- sscanf(terms[i].c_str(), "%d-%d", &si, &ti);
-
- while (m_vec_s_align.size() <= si) {
- SingleAlign sa;
- m_vec_s_align.push_back(sa);
- }
- while (m_vec_t_align.size() <= ti) {
- SingleAlign sa;
- m_vec_t_align.push_back(sa);
- }
-
- m_vec_s_align[si].push_back(ti);
- m_vec_t_align[ti].push_back(si);
- }
-
- // sort
- for (size_t i = 0; i < m_vec_s_align.size(); i++) {
- std::sort(m_vec_s_align[i].begin(), m_vec_s_align[i].end());
- }
- for (size_t i = 0; i < m_vec_t_align.size(); i++) {
- std::sort(m_vec_t_align[i].begin(), m_vec_t_align[i].end());
- }
- }
-
- private:
- std::vector<SingleAlign> m_vec_s_align; // source side words' alignment
- std::vector<SingleAlign> m_vec_t_align; // target side words' alignment
-};
-
-struct SAlignmentReader {
- SAlignmentReader(const char* pszFname) {
- m_fpIn = fopen(pszFname, "r");
- assert(m_fpIn != NULL);
- }
- ~SAlignmentReader() {
- if (m_fpIn != NULL) fclose(m_fpIn);
- }
- SAlignment* fnReadNextAlignment() {
- if (feof(m_fpIn) == true) return NULL;
- char* pszLine = new char[100001];
- pszLine[0] = '\0';
- fgets(pszLine, 10001, m_fpIn);
- int iLen = strlen(pszLine);
- if (iLen == 0) return NULL;
- while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
- pszLine[iLen - 1] = '\0';
- iLen--;
- }
- SAlignment* pAlign = new SAlignment(pszLine);
- delete[] pszLine;
- return pAlign;
- }
-
- private:
- FILE* m_fpIn;
-};
-
-#endif /* ALIGNMENT_H_ */
diff --git a/utils/argument_reorder_model.h b/utils/argument_reorder_model.h
deleted file mode 100644
index 077fa5ba..00000000
--- a/utils/argument_reorder_model.h
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * argument_reorder_model.h
- *
- * Created on: Dec 15, 2013
- * Author: lijunhui
- */
-
-#ifndef ARGUMENT_REORDER_MODEL_H_
-#define ARGUMENT_REORDER_MODEL_H_
-
-#include <string>
-#include <vector>
-
-#include "alignment.h"
-#include "tree.h"
-#include "srl_sentence.h"
-
-// an argument item or a predicate item (the verb itself)
-struct SSRLItem {
- SSRLItem(const STreeItem *tree_item, std::string role)
- : tree_item_(tree_item), role_(role) {}
- ~SSRLItem() {}
- const STreeItem *tree_item_;
- const std::string role_;
-};
-
-struct SPredicateItem {
- SPredicateItem(const SParsedTree *tree, const SPredicate *pred)
- : pred_(pred) {
- vec_items_.reserve(pred->m_vecArgt.size() + 1);
- for (int i = 0; i < pred->m_vecArgt.size(); i++) {
- vec_items_.push_back(
- new SSRLItem(pred->m_vecArgt[i]->m_pTreeItem,
- std::string(pred->m_vecArgt[i]->m_pszRole)));
- }
- vec_items_.push_back(
- new SSRLItem(tree->m_vecTerminals[pred->m_iPosition]->m_ptParent,
- std::string("Pred")));
- sort(vec_items_.begin(), vec_items_.end(), SortFunction);
-
- begin_ = vec_items_[0]->tree_item_->m_iBegin;
- end_ = vec_items_[vec_items_.size() - 1]->tree_item_->m_iEnd;
- }
-
- ~SPredicateItem() { vec_items_.clear(); }
-
- static bool SortFunction(SSRLItem *i, SSRLItem *j) {
- return (i->tree_item_->m_iBegin < j->tree_item_->m_iBegin);
- }
-
- std::vector<SSRLItem *> vec_items_;
- int begin_;
- int end_;
- const SPredicate *pred_;
-};
-
-struct SArgumentReorderModel {
- public:
- static std::string fnGetBlockOutcome(int iBegin, int iEnd,
- SAlignment *pAlign) {
- return pAlign->fnIsContinuous(iBegin, iEnd);
- }
- static void fnGetReorderType(SPredicateItem *pPredItem, SAlignment *pAlign,
- std::vector<std::string> &vecStrLeftReorder,
- std::vector<std::string> &vecStrRightReorder) {
- std::vector<int> vecLeft, vecRight;
- for (int i = 0; i < pPredItem->vec_items_.size(); i++) {
- const STreeItem *pCon1 = pPredItem->vec_items_[i]->tree_item_;
- int iLeft1, iRight1;
- pAlign->fnGetLeftRightMost(pCon1->m_iBegin, pCon1->m_iEnd, true, iLeft1,
- iRight1);
- vecLeft.push_back(iLeft1);
- vecRight.push_back(iRight1);
- }
- std::vector<int> vecLeftPosition;
- fnGetRelativePosition(vecLeft, vecLeftPosition);
- std::vector<int> vecRightPosition;
- fnGetRelativePosition(vecRight, vecRightPosition);
-
- vecStrLeftReorder.clear();
- vecStrRightReorder.clear();
- for (int i = 1; i < vecLeftPosition.size(); i++) {
- std::string strOutcome;
- fnGetOutcome(vecLeftPosition[i - 1], vecLeftPosition[i], strOutcome);
- vecStrLeftReorder.push_back(strOutcome);
- fnGetOutcome(vecRightPosition[i - 1], vecRightPosition[i], strOutcome);
- vecStrRightReorder.push_back(strOutcome);
- }
- }
-
- /*
- * features:
- * f1: (left_label, right_label, parent_label)
- * f2: (left_label, right_label, parent_label, other_right_sibling_label)
- * f3: (left_label, right_label, parent_label, other_left_sibling_label)
- * f4: (left_label, right_label, left_head_pos)
- * f5: (left_label, right_label, left_head_word)
- * f6: (left_label, right_label, right_head_pos)
- * f7: (left_label, right_label, right_head_word)
- * f8: (left_label, right_label, left_chunk_status)
- * f9: (left_label, right_label, right_chunk_status)
- * f10: (left_label, parent_label)
- * f11: (right_label, parent_label)
- *
- * f1: (left_role, right_role, predicate_term)
- * f2: (left_role, right_role, predicate_term, other_right_role)
- * f3: (left_role, right_role, predicate_term, other_left_role)
- * f4: (left_role, right_role, left_head_pos)
- * f5: (left_role, right_role, left_head_word)
- * f6: (left_role, right_role, left_syntactic_label)
- * f7: (left_role, right_role, right_head_pos)
- * f8: (left_role, right_role, right_head_word)
- * f8: (left_role, right_role, right_syntactic_label)
- * f8: (left_role, right_role, left_chunk_status)
- * f9: (left_role, right_role, right_chunk_status)
- * f10: (left_role, right_role, left_chunk_status)
- * f11: (left_role, right_role, right_chunk_status)
- * f12: (left_label, parent_label)
- * f13: (right_label, parent_label)
- */
- static void fnGenerateFeature(const SParsedTree *pTree,
- const SPredicate *pPred,
- const SPredicateItem *pPredItem, int iPos,
- const std::string &strBlock1,
- const std::string &strBlock2,
- std::ostringstream &ostr) {
- SSRLItem *pSRLItem1 = pPredItem->vec_items_[iPos - 1];
- SSRLItem *pSRLItem2 = pPredItem->vec_items_[iPos];
- const STreeItem *pCon1 = pSRLItem1->tree_item_;
- const STreeItem *pCon2 = pSRLItem2->tree_item_;
-
- std::string left_role = pSRLItem1->role_;
- std::string right_role = pSRLItem2->role_;
-
- std::string predicate_term =
- pTree->m_vecTerminals[pPred->m_iPosition]->m_pszTerm;
-
- std::vector<std::string> vec_other_right_sibling;
- for (int i = iPos + 1; i < pPredItem->vec_items_.size(); i++)
- vec_other_right_sibling.push_back(
- std::string(pPredItem->vec_items_[i]->role_));
- if (vec_other_right_sibling.size() == 0)
- vec_other_right_sibling.push_back(std::string("NULL"));
-
- std::vector<std::string> vec_other_left_sibling;
- for (int i = 0; i < iPos - 1; i++)
- vec_other_right_sibling.push_back(
- std::string(pPredItem->vec_items_[i]->role_));
- if (vec_other_left_sibling.size() == 0)
- vec_other_left_sibling.push_back(std::string("NULL"));
-
- // generate features
- // f1
- ostr << "f1=" << left_role << "_" << right_role << "_" << predicate_term;
- ostr << "f1=" << left_role << "_" << right_role;
-
- // f2
- for (int i = 0; i < vec_other_right_sibling.size(); i++) {
- ostr << " f2=" << left_role << "_" << right_role << "_" << predicate_term
- << "_" << vec_other_right_sibling[i];
- ostr << " f2=" << left_role << "_" << right_role << "_"
- << vec_other_right_sibling[i];
- }
- // f3
- for (int i = 0; i < vec_other_left_sibling.size(); i++) {
- ostr << " f3=" << left_role << "_" << right_role << "_" << predicate_term
- << "_" << vec_other_left_sibling[i];
- ostr << " f3=" << left_role << "_" << right_role << "_"
- << vec_other_left_sibling[i];
- }
- // f4
- ostr << " f4=" << left_role << "_" << right_role << "_"
- << pTree->m_vecTerminals[pCon1->m_iHeadWord]->m_ptParent->m_pszTerm;
- // f5
- ostr << " f5=" << left_role << "_" << right_role << "_"
- << pTree->m_vecTerminals[pCon1->m_iHeadWord]->m_pszTerm;
- // f6
- ostr << " f6=" << left_role << "_" << right_role << "_" << pCon2->m_pszTerm;
- // f7
- ostr << " f7=" << left_role << "_" << right_role << "_"
- << pTree->m_vecTerminals[pCon2->m_iHeadWord]->m_ptParent->m_pszTerm;
- // f8
- ostr << " f8=" << left_role << "_" << right_role << "_"
- << pTree->m_vecTerminals[pCon2->m_iHeadWord]->m_pszTerm;
- // f9
- ostr << " f9=" << left_role << "_" << right_role << "_" << pCon2->m_pszTerm;
- // f10
- ostr << " f10=" << left_role << "_" << right_role << "_" << strBlock1;
- // f11
- ostr << " f11=" << left_role << "_" << right_role << "_" << strBlock2;
- // f12
- ostr << " f12=" << left_role << "_" << predicate_term;
- ostr << " f12=" << left_role;
- // f13
- ostr << " f13=" << right_role << "_" << predicate_term;
- ostr << " f13=" << right_role;
- }
-
- private:
- static void fnGetOutcome(int i1, int i2, std::string &strOutcome) {
- assert(i1 != i2);
- if (i1 < i2) {
- if (i2 > i1 + 1)
- strOutcome = std::string("DM");
- else
- strOutcome = std::string("M");
- } else {
- if (i1 > i2 + 1)
- strOutcome = std::string("DS");
- else
- strOutcome = std::string("S");
- }
- }
-
- static void fnGetRelativePosition(const std::vector<int> &vecLeft,
- std::vector<int> &vecPosition) {
- vecPosition.clear();
-
- std::vector<float> vec;
- for (int i = 0; i < vecLeft.size(); i++) {
- if (vecLeft[i] == -1) {
- if (i == 0)
- vec.push_back(-1);
- else
- vec.push_back(vecLeft[i - 1] + 0.1);
- } else
- vec.push_back(vecLeft[i]);
- }
-
- for (int i = 0; i < vecLeft.size(); i++) {
- int count = 0;
-
- for (int j = 0; j < vecLeft.size(); j++) {
- if (j == i) continue;
- if (vec[j] < vec[i]) {
- count++;
- } else if (vec[j] == vec[i] && j < i) {
- count++;
- }
- }
- vecPosition.push_back(count);
- }
- }
-};
-
-#endif /* ARGUMENT_REORDER_MODEL_H_ */
diff --git a/utils/srl_sentence.h b/utils/srl_sentence.h
deleted file mode 100644
index 9d509600..00000000
--- a/utils/srl_sentence.h
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * srl_sentence.h
- *
- * Created on: May 26, 2013
- * Author: junhuili
- */
-
-#ifndef SRL_SENTENCE_H_
-#define SRL_SENTENCE_H_
-
-#include <sstream>
-#include <vector>
-
-#include "tree.h"
-#include "stringlib.h"
-
-struct SArgument {
- SArgument(const char* pszRole, int iBegin, int iEnd, float fProb) {
- m_pszRole = new char[strlen(pszRole) + 1];
- strcpy(m_pszRole, pszRole);
- m_iBegin = iBegin;
- m_iEnd = iEnd;
- m_fProb = fProb;
- m_pTreeItem = NULL;
- }
- ~SArgument() { delete[] m_pszRole; }
-
- void fnSetTreeItem(STreeItem* pTreeItem) {
- m_pTreeItem = pTreeItem;
- if (m_pTreeItem != NULL && m_pTreeItem->m_iBegin != -1) {
- assert(m_pTreeItem->m_iBegin == m_iBegin);
- assert(m_pTreeItem->m_iEnd == m_iEnd);
- }
- }
-
- char* m_pszRole; // argument rule, e.g., ARG0, ARGM-TMP
- int m_iBegin;
- int m_iEnd; // the span of the argument, [m_iBegin, m_iEnd]
- float m_fProb; // the probability of this role,
- STreeItem* m_pTreeItem;
-};
-
-struct SPredicate {
- SPredicate(const char* pszLemma, int iPosition) {
- if (pszLemma != NULL) {
- m_pszLemma = new char[strlen(pszLemma) + 1];
- strcpy(m_pszLemma, pszLemma);
- } else
- m_pszLemma = NULL;
- m_iPosition = iPosition;
- }
- ~SPredicate() {
- if (m_pszLemma != NULL) delete[] m_pszLemma;
- for (size_t i = 0; i < m_vecArgt.size(); i++) delete m_vecArgt[i];
- }
- int fnAppend(const char* pszRole, int iBegin, int iEnd) {
- SArgument* pArgt = new SArgument(pszRole, iBegin, iEnd, 1.0);
- return fnAppend(pArgt);
- }
- int fnAppend(SArgument* pArgt) {
- m_vecArgt.push_back(pArgt);
- int iPosition = m_vecArgt.size() - 1;
- return iPosition;
- }
-
- char* m_pszLemma; // lemma of the predicate, for Chinese, it's always as same
- // as the predicate itself
- int m_iPosition; // the position in sentence
- std::vector<SArgument*> m_vecArgt; // arguments associated to the predicate
-};
-
-struct SSrlSentence {
- SSrlSentence() { m_pTree = NULL; }
- ~SSrlSentence() {
- if (m_pTree != NULL) delete m_pTree;
-
- for (size_t i = 0; i < m_vecPred.size(); i++) delete m_vecPred[i];
- }
- int fnAppend(const char* pszLemma, int iPosition) {
- SPredicate* pPred = new SPredicate(pszLemma, iPosition);
- return fnAppend(pPred);
- }
- int fnAppend(SPredicate* pPred) {
- m_vecPred.push_back(pPred);
- int iPosition = m_vecPred.size() - 1;
- return iPosition;
- }
- int GetPredicateNum() { return m_vecPred.size(); }
-
- SParsedTree* m_pTree;
- std::vector<SPredicate*> m_vecPred;
-};
-
-struct SSrlSentenceReader {
- SSrlSentenceReader(const char* pszSrlFname) {
- m_fpIn = fopen(pszSrlFname, "r");
- assert(m_fpIn != NULL);
- }
- ~SSrlSentenceReader() {
- if (m_fpIn != NULL) fclose(m_fpIn);
- }
-
- inline void fnReplaceAll(std::string& str, const std::string& from,
- const std::string& to) {
- size_t start_pos = 0;
- while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
- str.replace(start_pos, from.length(), to);
- start_pos += to.length(); // In case 'to' contains 'from', like replacing
- // 'x' with 'yx'
- }
- }
-
- // TODO: here only considers flat predicate-argument structure
- // i.e., no overlap among them
- SSrlSentence* fnReadNextSrlSentence() {
- std::vector<std::vector<std::string> > vecContent;
- if (fnReadNextContent(vecContent) == false) return NULL;
-
- SSrlSentence* pSrlSentence = new SSrlSentence();
- int iSize = vecContent.size();
- // put together syntactic text
- std::ostringstream ostr;
- for (int i = 0; i < iSize; i++) {
- std::string strSynSeg =
- vecContent[i][5]; // the 5th column is the syntactic segment
- size_t iPosition = strSynSeg.find_first_of('*');
- assert(iPosition != std::string::npos);
- std::ostringstream ostrTmp;
- ostrTmp << "(" << vecContent[i][2] << " " << vecContent[i][0]
- << ")"; // the 2th column is POS-tag, and the 0th column is word
- strSynSeg.replace(iPosition, 1, ostrTmp.str());
- fnReplaceAll(strSynSeg, "(", " (");
- ostr << strSynSeg;
- }
- std::string strSyn = ostr.str();
- pSrlSentence->m_pTree = SParsedTree::fnConvertFromString(strSyn.c_str());
- pSrlSentence->m_pTree->fnSetHeadWord();
- pSrlSentence->m_pTree->fnSetSpanInfo();
-
- // read predicate-argument structure
- int iNumPred = vecContent[0].size() - 8;
- for (int i = 0; i < iNumPred; i++) {
- std::vector<std::string> vecRole;
- std::vector<int> vecBegin;
- std::vector<int> vecEnd;
- int iPred = -1;
- for (int j = 0; j < iSize; j++) {
- const char* p = vecContent[j][i + 8].c_str();
- const char* q;
- if (p[0] == '(') {
- // starting position of an argument(or predicate)
- vecBegin.push_back(j);
- q = strchr(p, '*');
- assert(q != NULL);
- vecRole.push_back(vecContent[j][i + 8].substr(1, q - p - 1));
- if (vecRole.back().compare("V") == 0) {
- assert(iPred == -1);
- iPred = vecRole.size() - 1;
- }
- }
- if (p[strlen(p) - 1] == ')') {
- // end position of an argument(or predicate)
- vecEnd.push_back(j);
- assert(vecBegin.size() == vecEnd.size());
- }
- }
- assert(iPred != -1);
- SPredicate* pPred = new SPredicate(
- pSrlSentence->m_pTree->m_vecTerminals[vecBegin[iPred]]->m_pszTerm,
- vecBegin[iPred]);
- pSrlSentence->fnAppend(pPred);
- for (size_t j = 0; j < vecBegin.size(); j++) {
- if (j == iPred) continue;
- pPred->fnAppend(vecRole[j].c_str(), vecBegin[j], vecEnd[j]);
- pPred->m_vecArgt.back()->fnSetTreeItem(
- pSrlSentence->m_pTree->fnFindNodeForSpan(vecBegin[j], vecEnd[j],
- false));
- }
- }
- return pSrlSentence;
- }
-
- private:
- bool fnReadNextContent(std::vector<std::vector<std::string> >& vecContent) {
- vecContent.clear();
- if (feof(m_fpIn) == true) return false;
- char* pszLine;
- pszLine = new char[100001];
- pszLine[0] = '\0';
- int iLen;
- while (!feof(m_fpIn)) {
- fgets(pszLine, 10001, m_fpIn);
- iLen = strlen(pszLine);
- while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
- pszLine[iLen - 1] = '\0';
- iLen--;
- }
- if (iLen == 0) break; // end of this sentence
-
- std::vector<std::string> terms = SplitOnWhitespace(std::string(pszLine));
- assert(terms.size() > 7);
- vecContent.push_back(terms);
- }
- delete[] pszLine;
- return true;
- }
-
- private:
- FILE* m_fpIn;
-};
-#endif /* SRL_SENTENCE_H_ */
diff --git a/utils/tree.h b/utils/tree.h
deleted file mode 100644
index 6c3406d6..00000000
--- a/utils/tree.h
+++ /dev/null
@@ -1,699 +0,0 @@
-/*
- * tree.h
- *
- * Created on: May 23, 2013
- * Author: lijunhui
- */
-
-#ifndef TREE_H_
-#define TREE_H_
-
-#include <assert.h>
-#include <stdio.h>
-
-#include <string>
-#include <vector>
-
-struct STreeItem {
- STreeItem(const char *pszTerm) {
- m_pszTerm = new char[strlen(pszTerm) + 1];
- strcpy(m_pszTerm, pszTerm);
-
- m_ptParent = NULL;
- m_iBegin = -1;
- m_iEnd = -1;
- m_iHeadChild = -1;
- m_iHeadWord = -1;
- m_iBrotherIndex = -1;
- }
- ~STreeItem() {
- delete[] m_pszTerm;
- for (size_t i = 0; i < m_vecChildren.size(); i++) delete m_vecChildren[i];
- }
- int fnAppend(STreeItem *ptChild) {
- m_vecChildren.push_back(ptChild);
- ptChild->m_iBrotherIndex = m_vecChildren.size() - 1;
- ptChild->m_ptParent = this;
- return m_vecChildren.size() - 1;
- }
- int fnGetChildrenNum() { return m_vecChildren.size(); }
-
- bool fnIsPreTerminal(void) {
- int I;
- if (this == NULL || m_vecChildren.size() == 0) return false;
-
- for (I = 0; I < m_vecChildren.size(); I++)
- if (m_vecChildren[I]->m_vecChildren.size() > 0) return false;
-
- return true;
- }
-
- public:
- char *m_pszTerm;
-
- std::vector<STreeItem *> m_vecChildren; // children items
- STreeItem *m_ptParent; // the parent item
-
- int m_iBegin;
- int m_iEnd; // the node span words[m_iBegin, m_iEnd]
- int m_iHeadChild; // the index of its head child
- int m_iHeadWord; // the index of its head word
- int m_iBrotherIndex; // the index in his brothers
-};
-
-struct SGetHeadWord {
- typedef std::vector<std::string> CVectorStr;
- SGetHeadWord() {}
- ~SGetHeadWord() {}
- int fnGetHeadWord(char *pszCFGLeft, CVectorStr vectRight) {
- // 0 indicating from right to left while 1 indicating from left to right
- char szaHeadLists[201] = "0";
-
- /* //head rules for Egnlish
- if( strcmp( pszCFGLeft, "ADJP" ) == 0 )
- strcpy( szaHeadLists, "0NNS 0QP 0NN 0$ 0ADVP 0JJ 0VBN 0VBG 0ADJP
- 0JJR 0NP 0JJS 0DT 0FW 0RBR 0RBS 0SBAR 0RB 0" );
- else if( strcmp( pszCFGLeft, "ADVP" ) == 0 )
- strcpy( szaHeadLists, "1RB 1RBR 1RBS 1FW 1ADVP 1TO 1CD 1JJR 1JJ 1IN
- 1NP 1JJS 1NN 1" );
- else if( strcmp( pszCFGLeft, "CONJP" ) == 0 )
- strcpy( szaHeadLists, "1CC 1RB 1IN 1" );
- else if( strcmp( pszCFGLeft, "FRAG" ) == 0 )
- strcpy( szaHeadLists, "1" );
- else if( strcmp( pszCFGLeft, "INTJ" ) == 0 )
- strcpy( szaHeadLists, "0" );
- else if( strcmp( pszCFGLeft, "LST" ) == 0 )
- strcpy( szaHeadLists, "1LS 1: 1CLN 1" );
- else if( strcmp( pszCFGLeft, "NAC" ) == 0 )
- strcpy( szaHeadLists, "0NN 0NNS 0NNP 0NNPS 0NP 0NAC 0EX 0$ 0CD 0QP
- 0PRP 0VBG 0JJ 0JJS 0JJR 0ADJP 0FW 0" );
- else if( strcmp( pszCFGLeft, "PP" ) == 0 )
- strcpy( szaHeadLists, "1IN 1TO 1VBG 1VBN 1RP 1FW 1" );
- else if( strcmp( pszCFGLeft, "PRN" ) == 0 )
- strcpy( szaHeadLists, "1" );
- else if( strcmp( pszCFGLeft, "PRT" ) == 0 )
- strcpy( szaHeadLists, "1RP 1" );
- else if( strcmp( pszCFGLeft, "QP" ) == 0 )
- strcpy( szaHeadLists, "0$ 0IN 0NNS 0NN 0JJ 0RB 0DT 0CD 0NCD 0QP 0JJR
- 0JJS 0" );
- else if( strcmp( pszCFGLeft, "RRC" ) == 0 )
- strcpy( szaHeadLists, "1VP 1NP 1ADVP 1ADJP 1PP 1" );
- else if( strcmp( pszCFGLeft, "S" ) == 0 )
- strcpy( szaHeadLists, "0TO 0IN 0VP 0S 0SBAR 0ADJP 0UCP 0NP 0" );
- else if( strcmp( pszCFGLeft, "SBAR" ) == 0 )
- strcpy( szaHeadLists, "0WHNP 0WHPP 0WHADVP 0WHADJP 0IN 0DT 0S 0SQ
- 0SINV 0SBAR 0FRAG 0" );
- else if( strcmp( pszCFGLeft, "SBARQ" ) == 0 )
- strcpy( szaHeadLists, "0SQ 0S 0SINV 0SBARQ 0FRAG 0" );
- else if( strcmp( pszCFGLeft, "SINV" ) == 0 )
- strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0S 0SINV 0ADJP 0NP
- 0" );
- else if( strcmp( pszCFGLeft, "SQ" ) == 0 )
- strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0SQ 0" );
- else if( strcmp( pszCFGLeft, "UCP" ) == 0 )
- strcpy( szaHeadLists, "1" );
- else if( strcmp( pszCFGLeft, "VP" ) == 0 )
- strcpy( szaHeadLists, "0TO 0VBD 0VBN 0MD 0VBZ 0VB 0VBG 0VBP 0VP
- 0ADJP 0NN 0NNS 0NP 0" );
- else if( strcmp( pszCFGLeft, "WHADJP" ) == 0 )
- strcpy( szaHeadLists, "0CC 0WRB 0JJ 0ADJP 0" );
- else if( strcmp( pszCFGLeft, "WHADVP" ) == 0 )
- strcpy( szaHeadLists, "1CC 1WRB 1" );
- else if( strcmp( pszCFGLeft, "WHNP" ) == 0 )
- strcpy( szaHeadLists, "0WDT 0WP 0WP$ 0WHADJP 0WHPP 0WHNP 0" );
- else if( strcmp( pszCFGLeft, "WHPP" ) == 0 )
- strcpy( szaHeadLists, "1IN 1TO FW 1" );
- else if( strcmp( pszCFGLeft, "NP" ) == 0 )
- strcpy( szaHeadLists, "0NN NNP NNS NNPS NX POS JJR 0NP 0$ ADJP PRN
- 0CD 0JJ JJS RB QP 0" );
- */
-
- if (strcmp(pszCFGLeft, "ADJP") == 0)
- strcpy(szaHeadLists, "0ADJP JJ 0AD NN CS 0");
- else if (strcmp(pszCFGLeft, "ADVP") == 0)
- strcpy(szaHeadLists, "0ADVP AD 0");
- else if (strcmp(pszCFGLeft, "CLP") == 0)
- strcpy(szaHeadLists, "0CLP M 0");
- else if (strcmp(pszCFGLeft, "CP") == 0)
- strcpy(szaHeadLists, "0DEC SP 1ADVP CS 0CP IP 0");
- else if (strcmp(pszCFGLeft, "DNP") == 0)
- strcpy(szaHeadLists, "0DNP DEG 0DEC 0");
- else if (strcmp(pszCFGLeft, "DVP") == 0)
- strcpy(szaHeadLists, "0DVP DEV 0");
- else if (strcmp(pszCFGLeft, "DP") == 0)
- strcpy(szaHeadLists, "1DP DT 1");
- else if (strcmp(pszCFGLeft, "FRAG") == 0)
- strcpy(szaHeadLists, "0VV NR NN 0");
- else if (strcmp(pszCFGLeft, "INTJ") == 0)
- strcpy(szaHeadLists, "0INTJ IJ 0");
- else if (strcmp(pszCFGLeft, "LST") == 0)
- strcpy(szaHeadLists, "1LST CD OD 1");
- else if (strcmp(pszCFGLeft, "IP") == 0)
- strcpy(szaHeadLists, "0IP VP 0VV 0");
- // strcpy( szaHeadLists, "0VP 0VV 1IP 0" );
- else if (strcmp(pszCFGLeft, "LCP") == 0)
- strcpy(szaHeadLists, "0LCP LC 0");
- else if (strcmp(pszCFGLeft, "NP") == 0)
- strcpy(szaHeadLists, "0NP NN NT NR QP 0");
- else if (strcmp(pszCFGLeft, "PP") == 0)
- strcpy(szaHeadLists, "1PP P 1");
- else if (strcmp(pszCFGLeft, "PRN") == 0)
- strcpy(szaHeadLists, "0 NP IP VP NT NR NN 0");
- else if (strcmp(pszCFGLeft, "QP") == 0)
- strcpy(szaHeadLists, "0QP CLP CD OD 0");
- else if (strcmp(pszCFGLeft, "VP") == 0)
- strcpy(szaHeadLists, "1VP VA VC VE VV BA LB VCD VSB VRD VNV VCP 1");
- else if (strcmp(pszCFGLeft, "VCD") == 0)
- strcpy(szaHeadLists, "0VCD VV VA VC VE 0");
- if (strcmp(pszCFGLeft, "VRD") == 0)
- strcpy(szaHeadLists, "0VRD VV VA VC VE 0");
- else if (strcmp(pszCFGLeft, "VSB") == 0)
- strcpy(szaHeadLists, "0VSB VV VA VC VE 0");
- else if (strcmp(pszCFGLeft, "VCP") == 0)
- strcpy(szaHeadLists, "0VCP VV VA VC VE 0");
- else if (strcmp(pszCFGLeft, "VNV") == 0)
- strcpy(szaHeadLists, "0VNV VV VA VC VE 0");
- else if (strcmp(pszCFGLeft, "VPT") == 0)
- strcpy(szaHeadLists, "0VNV VV VA VC VE 0");
- else if (strcmp(pszCFGLeft, "UCP") == 0)
- strcpy(szaHeadLists, "0");
- else if (strcmp(pszCFGLeft, "WHNP") == 0)
- strcpy(szaHeadLists, "0WHNP NP NN NT NR QP 0");
- else if (strcmp(pszCFGLeft, "WHPP") == 0)
- strcpy(szaHeadLists, "1WHPP PP P 1");
-
- /* //head rules for GENIA corpus
- if( strcmp( pszCFGLeft, "ADJP" ) == 0 )
- strcpy( szaHeadLists, "0NNS 0QP 0NN 0$ 0ADVP 0JJ 0VBN 0VBG 0ADJP
- 0JJR 0NP 0JJS 0DT 0FW 0RBR 0RBS 0SBAR 0RB 0" );
- else if( strcmp( pszCFGLeft, "ADVP" ) == 0 )
- strcpy( szaHeadLists, "1RB 1RBR 1RBS 1FW 1ADVP 1TO 1CD 1JJR 1JJ 1IN
- 1NP 1JJS 1NN 1" );
- else if( strcmp( pszCFGLeft, "CONJP" ) == 0 )
- strcpy( szaHeadLists, "1CC 1RB 1IN 1" );
- else if( strcmp( pszCFGLeft, "FRAG" ) == 0 )
- strcpy( szaHeadLists, "1" );
- else if( strcmp( pszCFGLeft, "INTJ" ) == 0 )
- strcpy( szaHeadLists, "0" );
- else if( strcmp( pszCFGLeft, "LST" ) == 0 )
- strcpy( szaHeadLists, "1LS 1: 1CLN 1" );
- else if( strcmp( pszCFGLeft, "NAC" ) == 0 )
- strcpy( szaHeadLists, "0NN 0NNS 0NNP 0NNPS 0NP 0NAC 0EX 0$ 0CD 0QP
- 0PRP 0VBG 0JJ 0JJS 0JJR 0ADJP 0FW 0" );
- else if( strcmp( pszCFGLeft, "PP" ) == 0 )
- strcpy( szaHeadLists, "1IN 1TO 1VBG 1VBN 1RP 1FW 1" );
- else if( strcmp( pszCFGLeft, "PRN" ) == 0 )
- strcpy( szaHeadLists, "1" );
- else if( strcmp( pszCFGLeft, "PRT" ) == 0 )
- strcpy( szaHeadLists, "1RP 1" );
- else if( strcmp( pszCFGLeft, "QP" ) == 0 )
- strcpy( szaHeadLists, "0$ 0IN 0NNS 0NN 0JJ 0RB 0DT 0CD 0NCD 0QP 0JJR
- 0JJS 0" );
- else if( strcmp( pszCFGLeft, "RRC" ) == 0 )
- strcpy( szaHeadLists, "1VP 1NP 1ADVP 1ADJP 1PP 1" );
- else if( strcmp( pszCFGLeft, "S" ) == 0 )
- strcpy( szaHeadLists, "0TO 0IN 0VP 0S 0SBAR 0ADJP 0UCP 0NP 0" );
- else if( strcmp( pszCFGLeft, "SBAR" ) == 0 )
- strcpy( szaHeadLists, "0WHNP 0WHPP 0WHADVP 0WHADJP 0IN 0DT 0S 0SQ
- 0SINV 0SBAR 0FRAG 0" );
- else if( strcmp( pszCFGLeft, "SBARQ" ) == 0 )
- strcpy( szaHeadLists, "0SQ 0S 0SINV 0SBARQ 0FRAG 0" );
- else if( strcmp( pszCFGLeft, "SINV" ) == 0 )
- strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0S 0SINV 0ADJP 0NP
- 0" );
- else if( strcmp( pszCFGLeft, "SQ" ) == 0 )
- strcpy( szaHeadLists, "0VBZ 0VBD 0VBP 0VB 0MD 0VP 0SQ 0" );
- else if( strcmp( pszCFGLeft, "UCP" ) == 0 )
- strcpy( szaHeadLists, "1" );
- else if( strcmp( pszCFGLeft, "VP" ) == 0 )
- strcpy( szaHeadLists, "0TO 0VBD 0VBN 0MD 0VBZ 0VB 0VBG 0VBP 0VP
- 0ADJP 0NN 0NNS 0NP 0" );
- else if( strcmp( pszCFGLeft, "WHADJP" ) == 0 )
- strcpy( szaHeadLists, "0CC 0WRB 0JJ 0ADJP 0" );
- else if( strcmp( pszCFGLeft, "WHADVP" ) == 0 )
- strcpy( szaHeadLists, "1CC 1WRB 1" );
- else if( strcmp( pszCFGLeft, "WHNP" ) == 0 )
- strcpy( szaHeadLists, "0WDT 0WP 0WP$ 0WHADJP 0WHPP 0WHNP 0" );
- else if( strcmp( pszCFGLeft, "WHPP" ) == 0 )
- strcpy( szaHeadLists, "1IN 1TO FW 1" );
- else if( strcmp( pszCFGLeft, "NP" ) == 0 )
- strcpy( szaHeadLists, "0NN NNP NNS NNPS NX POS JJR 0NP 0$ ADJP PRN
- 0CD 0JJ JJS RB QP 0" );
- */
-
- return fnMyOwnHeadWordRule(szaHeadLists, vectRight);
- }
-
- private:
- int fnMyOwnHeadWordRule(char *pszaHeadLists, CVectorStr vectRight) {
- char szHeadList[201], *p;
- char szTerm[101];
- int J;
-
- p = pszaHeadLists;
-
- int iCountRight;
-
- iCountRight = vectRight.size();
-
- szHeadList[0] = '\0';
- while (1) {
- szTerm[0] = '\0';
- sscanf(p, "%s", szTerm);
- if (strlen(szHeadList) == 0) {
- if (strcmp(szTerm, "0") == 0) {
- return iCountRight - 1;
- }
- if (strcmp(szTerm, "1") == 0) {
- return 0;
- }
-
- sprintf(szHeadList, "%c %s ", szTerm[0], szTerm + 1);
- p = strstr(p, szTerm);
- p += strlen(szTerm);
- } else {
- if ((szTerm[0] == '0') || (szTerm[0] == '1')) {
- if (szHeadList[0] == '0') {
- for (J = iCountRight - 1; J >= 0; J--) {
- sprintf(szTerm, " %s ", vectRight.at(J).c_str());
- if (strstr(szHeadList, szTerm) != NULL) return J;
- }
- } else {
- for (J = 0; J < iCountRight; J++) {
- sprintf(szTerm, " %s ", vectRight.at(J).c_str());
- if (strstr(szHeadList, szTerm) != NULL) return J;
- }
- }
-
- szHeadList[0] = '\0';
- } else {
- strcat(szHeadList, szTerm);
- strcat(szHeadList, " ");
-
- p = strstr(p, szTerm);
- p += strlen(szTerm);
- }
- }
- }
-
- return 0;
- }
-};
-
-struct SParsedTree {
- SParsedTree() { m_ptRoot = NULL; }
- ~SParsedTree() {
- if (m_ptRoot != NULL) delete m_ptRoot;
- }
- static SParsedTree *fnConvertFromString(const char *pszStr) {
- if (strcmp(pszStr, "(())") == 0) return NULL;
- SParsedTree *pTree = new SParsedTree();
-
- std::vector<std::string> vecSyn;
- fnReadSyntactic(pszStr, vecSyn);
-
- int iLeft = 1, iRight = 1; //# left/right parenthesis
-
- STreeItem *pcurrent;
-
- pTree->m_ptRoot = new STreeItem(vecSyn[1].c_str());
-
- pcurrent = pTree->m_ptRoot;
-
- for (size_t i = 2; i < vecSyn.size() - 1; i++) {
- if (strcmp(vecSyn[i].c_str(), "(") == 0)
- iLeft++;
- else if (strcmp(vecSyn[i].c_str(), ")") == 0) {
- iRight++;
- if (pcurrent == NULL) {
- // error
- fprintf(stderr, "ERROR in ConvertFromString\n");
- fprintf(stderr, "%s\n", pszStr);
- return NULL;
- }
- pcurrent = pcurrent->m_ptParent;
- } else {
- STreeItem *ptNewItem = new STreeItem(vecSyn[i].c_str());
- pcurrent->fnAppend(ptNewItem);
- pcurrent = ptNewItem;
-
- if (strcmp(vecSyn[i - 1].c_str(), "(") != 0 &&
- strcmp(vecSyn[i - 1].c_str(), ")") != 0) {
- pTree->m_vecTerminals.push_back(ptNewItem);
- pcurrent = pcurrent->m_ptParent;
- }
- }
- }
-
- if (iLeft != iRight) {
- // error
- fprintf(stderr, "the left and right parentheses are not matched!");
- fprintf(stderr, "ERROR in ConvertFromString\n");
- fprintf(stderr, "%s\n", pszStr);
- return NULL;
- }
-
- return pTree;
- }
-
- int fnGetNumWord() { return m_vecTerminals.size(); }
-
- void fnSetSpanInfo() {
- int iNextNum = 0;
- fnSuffixTraverseSetSpanInfo(m_ptRoot, iNextNum);
- }
-
- void fnSetHeadWord() {
- for (size_t i = 0; i < m_vecTerminals.size(); i++)
- m_vecTerminals[i]->m_iHeadWord = i;
- SGetHeadWord *pGetHeadWord = new SGetHeadWord();
- fnSuffixTraverseSetHeadWord(m_ptRoot, pGetHeadWord);
- delete pGetHeadWord;
- }
-
- STreeItem *fnFindNodeForSpan(int iLeft, int iRight, bool bLowest) {
- STreeItem *pTreeItem = m_vecTerminals[iLeft];
-
- while (pTreeItem->m_iEnd < iRight) {
- pTreeItem = pTreeItem->m_ptParent;
- if (pTreeItem == NULL) break;
- }
- if (pTreeItem == NULL) return NULL;
- if (pTreeItem->m_iEnd > iRight) return NULL;
-
- assert(pTreeItem->m_iEnd == iRight);
- if (bLowest) return pTreeItem;
-
- while (pTreeItem->m_ptParent != NULL &&
- pTreeItem->m_ptParent->fnGetChildrenNum() == 1)
- pTreeItem = pTreeItem->m_ptParent;
-
- return pTreeItem;
- }
-
- private:
- void fnSuffixTraverseSetSpanInfo(STreeItem *ptItem, int &iNextNum) {
- int I;
- int iNumChildren = ptItem->fnGetChildrenNum();
- for (I = 0; I < iNumChildren; I++)
- fnSuffixTraverseSetSpanInfo(ptItem->m_vecChildren[I], iNextNum);
-
- if (I == 0) {
- ptItem->m_iBegin = iNextNum;
- ptItem->m_iEnd = iNextNum++;
- } else {
- ptItem->m_iBegin = ptItem->m_vecChildren[0]->m_iBegin;
- ptItem->m_iEnd = ptItem->m_vecChildren[I - 1]->m_iEnd;
- }
- }
-
- void fnSuffixTraverseSetHeadWord(STreeItem *ptItem,
- SGetHeadWord *pGetHeadWord) {
- int I, iHeadchild;
-
- if (ptItem->m_vecChildren.size() == 0) return;
-
- for (I = 0; I < ptItem->m_vecChildren.size(); I++)
- fnSuffixTraverseSetHeadWord(ptItem->m_vecChildren[I], pGetHeadWord);
-
- std::vector<std::string> vecRight;
-
- if (ptItem->m_vecChildren.size() == 1)
- iHeadchild = 0;
- else {
- for (I = 0; I < ptItem->m_vecChildren.size(); I++)
- vecRight.push_back(std::string(ptItem->m_vecChildren[I]->m_pszTerm));
-
- iHeadchild = pGetHeadWord->fnGetHeadWord(ptItem->m_pszTerm, vecRight);
- }
-
- ptItem->m_iHeadChild = iHeadchild;
- ptItem->m_iHeadWord = ptItem->m_vecChildren[iHeadchild]->m_iHeadWord;
- }
-
- static void fnReadSyntactic(const char *pszSyn,
- std::vector<std::string> &vec) {
- char *p;
- int I;
-
- int iLeftNum, iRightNum;
- char *pszTmp, *pszTerm;
- pszTmp = new char[strlen(pszSyn)];
- pszTerm = new char[strlen(pszSyn)];
- pszTmp[0] = pszTerm[0] = '\0';
-
- vec.clear();
-
- char *pszLine;
- pszLine = new char[strlen(pszSyn) + 1];
- strcpy(pszLine, pszSyn);
-
- char *pszLine2;
-
- while (1) {
- while ((strlen(pszLine) > 0) && (pszLine[strlen(pszLine) - 1] > 0) &&
- (pszLine[strlen(pszLine) - 1] <= ' '))
- pszLine[strlen(pszLine) - 1] = '\0';
-
- if (strlen(pszLine) == 0) break;
-
- // printf( "%s\n", pszLine );
- pszLine2 = pszLine;
- while (pszLine2[0] <= ' ') pszLine2++;
- if (pszLine2[0] == '<') continue;
-
- sscanf(pszLine2 + 1, "%s", pszTmp);
-
- if (pszLine2[0] == '(') {
- iLeftNum = 0;
- iRightNum = 0;
- }
-
- p = pszLine2;
- while (1) {
- pszTerm[0] = '\0';
- sscanf(p, "%s", pszTerm);
-
- if (strlen(pszTerm) == 0) break;
- p = strstr(p, pszTerm);
- p += strlen(pszTerm);
-
- if ((pszTerm[0] == '(') || (pszTerm[strlen(pszTerm) - 1] == ')')) {
- if (pszTerm[0] == '(') {
- vec.push_back(std::string("("));
- iLeftNum++;
-
- I = 1;
- while (pszTerm[I] == '(' && pszTerm[I] != '\0') {
- vec.push_back(std::string("("));
- iLeftNum++;
-
- I++;
- }
-
- if (strlen(pszTerm) > 1) vec.push_back(std::string(pszTerm + I));
- } else {
- char *pTmp;
- pTmp = pszTerm + strlen(pszTerm) - 1;
- while ((pTmp[0] == ')') && (pTmp >= pszTerm)) pTmp--;
- pTmp[1] = '\0';
-
- if (strlen(pszTerm) > 0) vec.push_back(std::string(pszTerm));
- pTmp += 2;
-
- for (I = 0; I <= (int)strlen(pTmp); I++) {
- vec.push_back(std::string(")"));
- iRightNum++;
- }
- }
- } else {
- char *q;
- q = strchr(pszTerm, ')');
- if (q != NULL) {
- q[0] = '\0';
- if (pszTerm[0] != '\0') vec.push_back(std::string(pszTerm));
- vec.push_back(std::string(")"));
- iRightNum++;
-
- q++;
- while (q[0] == ')') {
- vec.push_back(std::string(")"));
- q++;
- iRightNum++;
- }
-
- while (q[0] == '(') {
- vec.push_back(std::string("("));
- q++;
- iLeftNum++;
- }
-
- if (q[0] != '\0') vec.push_back(std::string(q));
- } else
- vec.push_back(std::string(pszTerm));
- }
- }
-
- if (iLeftNum != iRightNum) {
- fprintf(stderr, "%s\n", pszSyn);
- assert(iLeftNum == iRightNum);
- }
- /*if ( iLeftNum != iRightNum ) {
- printf( "ERROR: left( and right ) is not matched, %d ( and %d
- )\n", iLeftNum, iRightNum );
- return;
- }*/
-
- if (vec.size() >= 2 && strcmp(vec[1].c_str(), "(") == 0) {
- //( (IP..) )
- std::vector<std::string>::iterator it;
- it = vec.begin();
- it++;
- vec.insert(it, std::string("ROOT"));
- }
-
- break;
- }
-
- delete[] pszLine;
- delete[] pszTmp;
- delete[] pszTerm;
- }
-
- public:
- STreeItem *m_ptRoot;
- std::vector<STreeItem *> m_vecTerminals; // the leaf nodes
-};
-
-struct SParseReader {
- SParseReader(const char *pszParse_Fname, bool bFlattened = false)
- : m_bFlattened(bFlattened) {
- m_fpIn = fopen(pszParse_Fname, "r");
- assert(m_fpIn != NULL);
- }
- ~SParseReader() {
- if (m_fpIn != NULL) fclose(m_fpIn);
- }
-
- SParsedTree *fnReadNextParseTree() {
- SParsedTree *pTree = NULL;
- char *pszLine = new char[100001];
- int iLen;
-
- while (fnReadNextSentence(pszLine, &iLen) == true) {
- if (iLen == 0) continue;
-
- pTree = SParsedTree::fnConvertFromString(pszLine);
- if (pTree == NULL) break;
- if (m_bFlattened)
- fnPostProcessingFlattenedParse(pTree);
- else {
- pTree->fnSetSpanInfo();
- pTree->fnSetHeadWord();
- }
- break;
- }
-
- delete[] pszLine;
- return pTree;
- }
-
- SParsedTree *fnReadNextParseTreeWithProb(double *pProb) {
- SParsedTree *pTree = NULL;
- char *pszLine = new char[100001];
- int iLen;
-
- while (fnReadNextSentence(pszLine, &iLen) == true) {
- if (iLen == 0) continue;
-
- char *p = strchr(pszLine, ' ');
- assert(p != NULL);
- p[0] = '\0';
- p++;
- if (pProb) (*pProb) = atof(pszLine);
-
- pTree = SParsedTree::fnConvertFromString(p);
- if (m_bFlattened)
- fnPostProcessingFlattenedParse(pTree);
- else {
- pTree->fnSetSpanInfo();
- pTree->fnSetHeadWord();
- }
- break;
- }
-
- delete[] pszLine;
- return pTree;
- }
-
- private:
- /*
- * since to the parse tree is a flattened tree, use the head mark to identify
- * head info.
- * the head node will be marked as "*XP*"
- */
- void fnSetParseTreeHeadInfo(SParsedTree *pTree) {
- for (size_t i = 0; i < pTree->m_vecTerminals.size(); i++)
- pTree->m_vecTerminals[i]->m_iHeadWord = i;
- fnSuffixTraverseSetHeadWord(pTree->m_ptRoot);
- }
-
- void fnSuffixTraverseSetHeadWord(STreeItem *pTreeItem) {
- if (pTreeItem->m_vecChildren.size() == 0) return;
-
- for (size_t i = 0; i < pTreeItem->m_vecChildren.size(); i++)
- fnSuffixTraverseSetHeadWord(pTreeItem->m_vecChildren[i]);
-
- std::vector<std::string> vecRight;
-
- int iHeadchild;
-
- if (pTreeItem->fnIsPreTerminal()) {
- iHeadchild = 0;
- } else {
- size_t i;
- for (i = 0; i < pTreeItem->m_vecChildren.size(); i++) {
- char *p = pTreeItem->m_vecChildren[i]->m_pszTerm;
- if (p[0] == '*' && p[strlen(p) - 1] == '*') {
- iHeadchild = i;
- p[strlen(p) - 1] = '\0';
- std::string str = p + 1;
- strcpy(p, str.c_str()); // erase the "*..*"
- break;
- }
- }
- assert(i < pTreeItem->m_vecChildren.size());
- }
-
- pTreeItem->m_iHeadChild = iHeadchild;
- pTreeItem->m_iHeadWord = pTreeItem->m_vecChildren[iHeadchild]->m_iHeadWord;
- }
- void fnPostProcessingFlattenedParse(SParsedTree *pTree) {
- pTree->fnSetSpanInfo();
- fnSetParseTreeHeadInfo(pTree);
- }
- bool fnReadNextSentence(char *pszLine, int *piLength) {
- if (feof(m_fpIn) == true) return false;
-
- int iLen;
-
- pszLine[0] = '\0';
-
- fgets(pszLine, 10001, m_fpIn);
- iLen = strlen(pszLine);
- while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
- pszLine[iLen - 1] = '\0';
- iLen--;
- }
-
- if (piLength != NULL) (*piLength) = iLen;
-
- return true;
- }
-
- private:
- FILE *m_fpIn;
- const bool m_bFlattened;
-};
-
-#endif /* TREE_H_ */
diff --git a/utils/tsuruoka_maxent.h b/utils/tsuruoka_maxent.h
deleted file mode 100644
index 82da44ff..00000000
--- a/utils/tsuruoka_maxent.h
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * tsuruoka_maxent.h
- *
- */
-
-#ifndef TSURUOKA_MAXENT_H_
-#define TSURUOKA_MAXENT_H_
-
-#include <assert.h>
-#include <string.h>
-#include <string>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include "stringlib.h"
-#include "maxent.h"
-
-typedef std::unordered_map<std::string, int> Map;
-typedef std::unordered_map<std::string, int>::iterator Iterator;
-
-struct Tsuruoka_Maxent {
- Tsuruoka_Maxent(const char* pszModelFName) {
- if (pszModelFName != NULL) {
- m_pModel = new ME_Model();
- m_pModel->load_from_file(pszModelFName);
- } else
- m_pModel = NULL;
- }
-
- ~Tsuruoka_Maxent() {
- if (m_pModel != NULL) delete m_pModel;
- }
-
- void fnTrain(const char* pszInstanceFName, const char* pszAlgorithm,
- const char* pszModelFName, int /*iNumIteration*/) {
- assert(strcmp(pszAlgorithm, "l1") == 0 || strcmp(pszAlgorithm, "l2") == 0 ||
- strcmp(pszAlgorithm, "sgd") == 0 ||
- strcmp(pszAlgorithm, "SGD") == 0);
- FILE* fpIn = fopen(pszInstanceFName, "r");
-
- ME_Model* pModel = new ME_Model();
-
- char* pszLine = new char[100001];
- int iNumInstances = 0;
- int iLen;
- while (!feof(fpIn)) {
- pszLine[0] = '\0';
- fgets(pszLine, 20000, fpIn);
- if (strlen(pszLine) == 0) {
- continue;
- }
-
- iLen = strlen(pszLine);
- while (iLen > 0 && pszLine[iLen - 1] > 0 && pszLine[iLen - 1] < 33) {
- pszLine[iLen - 1] = '\0';
- iLen--;
- }
-
- iNumInstances++;
-
- ME_Sample* pmes = new ME_Sample();
-
- char* p = strrchr(pszLine, ' ');
- assert(p != NULL);
- p[0] = '\0';
- p++;
- std::vector<std::string> vecContext;
- SplitOnWhitespace(std::string(pszLine), &vecContext);
-
- pmes->label = std::string(p);
- for (size_t i = 0; i < vecContext.size(); i++)
- pmes->add_feature(vecContext[i]);
- pModel->add_training_sample((*pmes));
- if (iNumInstances % 100000 == 0)
- fprintf(stdout, "......Reading #Instances: %1d\n", iNumInstances);
- delete pmes;
- }
- fprintf(stdout, "......Reading #Instances: %1d\n", iNumInstances);
- fclose(fpIn);
-
- if (strcmp(pszAlgorithm, "l1") == 0)
- pModel->use_l1_regularizer(1.0);
- else if (strcmp(pszAlgorithm, "l2") == 0)
- pModel->use_l2_regularizer(1.0);
- else
- pModel->use_SGD();
-
- pModel->train();
- pModel->save_to_file(pszModelFName);
-
- delete pModel;
- fprintf(stdout, "......Finished Training\n");
- fprintf(stdout, "......Model saved as %s\n", pszModelFName);
- delete[] pszLine;
- }
-
- double fnEval(const char* pszContext, const char* pszOutcome) const {
- std::vector<std::string> vecContext;
- ME_Sample* pmes = new ME_Sample();
- SplitOnWhitespace(std::string(pszContext), &vecContext);
-
- for (size_t i = 0; i < vecContext.size(); i++)
- pmes->add_feature(vecContext[i]);
- std::vector<double> vecProb = m_pModel->classify(*pmes);
- delete pmes;
- int iLableID = m_pModel->get_class_id(pszOutcome);
- return vecProb[iLableID];
- }
- void fnEval(const char* pszContext,
- std::vector<std::pair<std::string, double> >& vecOutput) const {
- std::vector<std::string> vecContext;
- ME_Sample* pmes = new ME_Sample();
- SplitOnWhitespace(std::string(pszContext), &vecContext);
-
- vecOutput.clear();
-
- for (size_t i = 0; i < vecContext.size(); i++)
- pmes->add_feature(vecContext[i]);
- std::vector<double> vecProb = m_pModel->classify(*pmes);
-
- for (size_t i = 0; i < vecProb.size(); i++) {
- std::string label = m_pModel->get_class_label(i);
- vecOutput.push_back(make_pair(label, vecProb[i]));
- }
- delete pmes;
- }
- void fnEval(const char* pszContext, std::vector<double>& vecOutput) const {
- std::vector<std::string> vecContext;
- ME_Sample* pmes = new ME_Sample();
- SplitOnWhitespace(std::string(pszContext), &vecContext);
-
- vecOutput.clear();
-
- for (size_t i = 0; i < vecContext.size(); i++)
- pmes->add_feature(vecContext[i]);
- std::vector<double> vecProb = m_pModel->classify(*pmes);
-
- for (size_t i = 0; i < vecProb.size(); i++) {
- std::string label = m_pModel->get_class_label(i);
- vecOutput.push_back(vecProb[i]);
- }
- delete pmes;
- }
- int fnGetClassId(const std::string& strLabel) const {
- return m_pModel->get_class_id(strLabel);
- }
-
- private:
- ME_Model* m_pModel;
-};
-
-#endif /* TSURUOKA_MAXENT_H_ */