Reversed out broken thresholding

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@324 ec762483-ff6d-05da-a07a-a48fb63a330f

3 files changed, 92 insertions, 233 deletions

diff --git a/gi/posterior-regularisation/prjava/src/phrase/Corpus.java b/gi/posterior-regularisation/prjava/src/phrase/Corpus.java
index 2afc18dc..ad092cc6 100644
--- a/gi/posterior-regularisation/prjava/src/phrase/Corpus.java
+++ b/gi/posterior-regularisation/prjava/src/phrase/Corpus.java
@@ -18,9 +18,6 @@ public class Corpus
 	public int splitSentinel;
 	public int phraseSentinel;
 	public int rareSentinel;
-	private boolean[] rareWords;
-	private boolean[] rarePhrases;
-	private boolean[] rareContexts;
 
 	public Corpus()
 	{
@@ -45,10 +42,6 @@ public class Corpus
 		{
 			return Corpus.this.getPhrase(phraseId);
 		}
-		public TIntArrayList getRawPhrase()
-		{
-			return Corpus.this.getRawPhrase(phraseId);
-		}
 		public String getPhraseString()
 		{
 			return Corpus.this.getPhraseString(phraseId);
@@ -61,10 +54,6 @@ public class Corpus
 		{
 			return Corpus.this.getContext(contextId);
 		}
-		public TIntArrayList getRawContext()
-		{
-			return Corpus.this.getRawContext(contextId);
-		}		
 		public String getContextString(boolean insertPhraseSentinel)
 		{
 			return Corpus.this.getContextString(contextId, insertPhraseSentinel);
@@ -145,35 +134,13 @@ public class Corpus
 	
 	public TIntArrayList getPhrase(int phraseId)
 	{
-		TIntArrayList phrase = phraseLexicon.lookup(phraseId);
-		if (rareWords != null)
-		{
-			boolean first = true;
-			for (int i = 0; i < phrase.size(); ++i)
-			{
-				if (rareWords[phrase.get(i)])
-				{
-					if (first)
-					{
-						phrase = (TIntArrayList) phrase.clone();
-						first = false;
-					}
-					phrase.set(i, rareSentinel);
-				}
-			}
-		}
-		return phrase;
-	}
-	
-	public TIntArrayList getRawPhrase(int phraseId)
-	{
 		return phraseLexicon.lookup(phraseId);
 	}
 	
 	public String getPhraseString(int phraseId)
 	{
 		StringBuffer b = new StringBuffer();
-		for (int tid: getRawPhrase(phraseId).toNativeArray())
+		for (int tid: getPhrase(phraseId).toNativeArray())
 		{
 			if (b.length() > 0)
 				b.append(" ");
@@ -184,35 +151,13 @@ public class Corpus
 	
 	public TIntArrayList getContext(int contextId)
 	{
-		TIntArrayList context = contextLexicon.lookup(contextId);
-		if (rareWords != null)
-		{
-			boolean first = true;
-			for (int i = 0; i < context.size(); ++i)
-			{
-				if (rareWords[context.get(i)])
-				{
-					if (first)
-					{
-						context = (TIntArrayList) context.clone();
-						first = false;
-					}
-					context.set(i, rareSentinel);
-				}
-			}
-		}
-		return context;
-	}
-	
-	public TIntArrayList getRawContext(int contextId)
-	{
 		return contextLexicon.lookup(contextId);
 	}
 	
 	public String getContextString(int contextId, boolean insertPhraseSentinel)
 	{
 		StringBuffer b = new StringBuffer();
-		TIntArrayList c = getRawContext(contextId);
+		TIntArrayList c = getContext(contextId);
 		for (int i = 0; i < c.size(); ++i)
 		{
 			if (i > 0) b.append(" ");
@@ -227,15 +172,14 @@ public class Corpus
 		return wordId == splitSentinel || wordId == phraseSentinel;
 	}
 	
-	static Corpus readFromFile(Reader in) throws IOException
-	{
-		Corpus c = new Corpus();
-		
+	List<Edge> readEdges(Reader in) throws IOException
+	{	
 		// read in line-by-line
 		BufferedReader bin = new BufferedReader(in);
 		String line;
 		Pattern separator = Pattern.compile(" \\|\\|\\| ");
-
+		
+		List<Edge> edges = new ArrayList<Edge>();
 		while ((line = bin.readLine()) != null)
 		{
 			// split into phrase and contexts
@@ -250,10 +194,8 @@ public class Corpus
 			st = new StringTokenizer(phraseToks, " ");
 			TIntArrayList ptoks = new TIntArrayList();
 			while (st.hasMoreTokens())
-				ptoks.add(c.wordLexicon.insert(st.nextToken()));
-			int phraseId = c.phraseLexicon.insert(ptoks);
-			if (phraseId == c.phraseToContext.size())
-				c.phraseToContext.add(new ArrayList<Edge>());
+				ptoks.add(wordLexicon.insert(st.nextToken()));
+			int phraseId = phraseLexicon.insert(ptoks);
 			
 			// process contexts
 			String[] parts = separator.split(rest);
@@ -261,34 +203,45 @@ public class Corpus
 			for (int i = 0; i < parts.length; i += 2)
 			{
 				// process pairs of strings - context and count
-				TIntArrayList ctx = new TIntArrayList();
 				String ctxString = parts[i];
 				String countString = parts[i + 1];
+
+				assert (countString.startsWith("C="));
+				int count = Integer.parseInt(countString.substring(2).trim());
+				
+				TIntArrayList ctx = new TIntArrayList();
 				StringTokenizer ctxStrtok = new StringTokenizer(ctxString, " ");
 				while (ctxStrtok.hasMoreTokens())
 				{
 					String token = ctxStrtok.nextToken();
-					//if (!token.equals("<PHRASE>"))
-						ctx.add(c.wordLexicon.insert(token));
+					ctx.add(wordLexicon.insert(token));
 				}
-				int contextId = c.contextLexicon.insert(ctx);
-				if (contextId == c.contextToPhrase.size())
-					c.contextToPhrase.add(new ArrayList<Edge>());
+				int contextId = contextLexicon.insert(ctx);
 
-				assert (countString.startsWith("C="));
-				Edge e = c.new Edge(phraseId, contextId, 
-						Integer.parseInt(countString.substring(2).trim()));
-				c.edges.add(e);
-				
-				// index the edge for fast phrase, context lookup
-				c.phraseToContext.get(phraseId).add(e);
-				c.contextToPhrase.get(contextId).add(e);
+				edges.add(new Edge(phraseId, contextId, count));
 			}
 		}
-		
-		return c;
+		return edges;
 	}
 	
+	static Corpus readFromFile(Reader in) throws IOException
+	{	
+		Corpus c = new Corpus();
+		c.edges = c.readEdges(in);
+		for (Edge edge: c.edges)
+		{
+			while (edge.getPhraseId() >= c.phraseToContext.size())
+				c.phraseToContext.add(new ArrayList<Edge>());
+			while (edge.getContextId() >= c.contextToPhrase.size())
+				c.contextToPhrase.add(new ArrayList<Edge>());
+			
+			// index the edge for fast phrase, context lookup
+			c.phraseToContext.get(edge.getPhraseId()).add(edge);
+			c.contextToPhrase.get(edge.getContextId()).add(edge);
+		}
+		return c;
+	}
+		
 	TIntArrayList phraseEdges(TIntArrayList phrase)
 	{
 		TIntArrayList r = new TIntArrayList(4);
@@ -307,86 +260,4 @@ public class Corpus
 		out.println("Corpus has " + edges.size() + " edges " + phraseLexicon.size() + " phrases " 
 				+ contextLexicon.size() + " contexts and " + wordLexicon.size() + " word types");
 	}
-
-	public void applyWordThreshold(int wordThreshold) 
-	{
-		int[] counts = new int[wordLexicon.size()];
-		for (Edge e: edges)
-		{
-			TIntArrayList phrase = e.getPhrase();
-			for (int i = 0; i < phrase.size(); ++i)
-				counts[phrase.get(i)] += e.getCount();
-			
-			TIntArrayList context = e.getContext();
-			for (int i = 0; i < context.size(); ++i)
-				counts[context.get(i)] += e.getCount();
-		}
-
-		int count = 0;
-		rareWords = new boolean[wordLexicon.size()];
-		for (int i = 0; i < wordLexicon.size(); ++i)
-		{
-			rareWords[i] = counts[i] < wordThreshold;
-			if (rareWords[i])
-				count++;
-		}
-		System.err.println("There are " + count + " rare words");
-	}
-	
-	public void applyPhraseThreshold(int threshold) 
-	{
-		rarePhrases = new boolean[phraseLexicon.size()];
-
-		int n = 0;
-		for (int i = 0; i < phraseLexicon.size(); ++i)
-		{
-			List<Edge> contexts = phraseToContext.get(i);
-			int count = 0;
-			for (Edge edge: contexts)
-			{
-				count += edge.getCount();
-				if (count >= threshold)
-					break;
-			}
-			
-			if (count < threshold)
-			{
-				rarePhrases[i] = true;
-				n++;
-			}
-		}
-		System.err.println("There are " + n + " rare phrases");
-	}
-	
-	public void applyContextThreshold(int threshold) 
-	{
-		rareContexts = new boolean[contextLexicon.size()];
-
-		int n = 0;
-		for (int i = 0; i < contextLexicon.size(); ++i)
-		{
-			List<Edge> phrases = contextToPhrase.get(i);
-			int count = 0;
-			for (Edge edge: phrases)
-			{
-				count += edge.getCount();
-				if (count >= threshold)
-					break;
-			}
-			
-			if (count < threshold)
-			{
-				rareContexts[i] = true;
-				n++;
-			}
-		}
-		System.err.println("There are " + n + " rare contexts");
-	}
-	
-	boolean isRare(Edge edge)
-	{
-		if (rarePhrases != null && rarePhrases[edge.getPhraseId()] == true) return true;
-		if (rareContexts != null && rareContexts[edge.getContextId()] == true) return true;
-		return false;
-	}
 }
\ No newline at end of file
diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java
index 9ee766d4..e7e4af32 100644
--- a/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java
+++ b/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java
@@ -23,7 +23,7 @@ import phrase.Corpus.Edge;
 public class PhraseCluster {

 	

 	public int K;

-	private int n_phrases, n_words, n_contexts, n_positions, edge_threshold;

+	private int n_phrases, n_words, n_contexts, n_positions;

 	public Corpus c;

 	public ExecutorService pool; 

 

@@ -44,7 +44,6 @@ public class PhraseCluster {
 		n_phrases=c.getNumPhrases();

 		n_contexts=c.getNumContexts();

 		n_positions=c.getNumContextPositions();

-		edge_threshold=0;

 

 		emit=new double [K][n_positions][n_words];

 		pi=new double[n_phrases][K];

@@ -76,7 +75,7 @@ public class PhraseCluster {
 		pool = Executors.newFixedThreadPool(threads);

 	}

 

-	public double EM(boolean skipBigPhrases)

+	public double EM(int phraseSizeLimit)

 	{

 		double [][][]exp_emit=new double [K][n_positions][n_words];

 		double [][]exp_pi=new double[n_phrases][K];

@@ -92,19 +91,17 @@ public class PhraseCluster {
 		//E

 		for(int phrase=0; phrase < n_phrases; phrase++)

 		{

-			if (skipBigPhrases && c.getPhrase(phrase).size() >= 2)

+			if (phraseSizeLimit >= 1 && c.getPhrase(phrase).size() > phraseSizeLimit)

 			{

 				System.arraycopy(pi[phrase], 0, exp_pi[phrase], 0, K);

 				continue;

-			}			

+			}	

 

 			List<Edge> contexts = c.getEdgesForPhrase(phrase);

 

 			for (int ctx=0; ctx<contexts.size(); ctx++)

 			{

 				Edge edge = contexts.get(ctx);

-				if (edge.getCount() < edge_threshold || c.isRare(edge)) 

-					continue;

 				

 				double p[]=posterior(edge);

 				double z = arr.F.l1norm(p);

@@ -138,10 +135,9 @@ public class PhraseCluster {
 		return loglikelihood;

 	}

 	

-	public double VBEM(double alphaEmit, double alphaPi, boolean skipBigPhrases)

+	public double VBEM(double alphaEmit, double alphaPi)

 	{

 		// FIXME: broken - needs to be done entirely in log-space

-		assert !skipBigPhrases : "FIXME: implement this!";

 		

 		double [][][]exp_emit = new double [K][n_positions][n_words];

 		double [][]exp_pi = new double[n_phrases][K];

@@ -240,21 +236,21 @@ public class PhraseCluster {
 		return kl;

 	}

 	

-	public double PREM(double scalePT, double scaleCT, boolean skipBigPhrases)

+	public double PREM(double scalePT, double scaleCT, int phraseSizeLimit)

 	{

 		if (scaleCT == 0)

 		{

 			if (pool != null)

-				return PREM_phrase_constraints_parallel(scalePT, skipBigPhrases);

+				return PREM_phrase_constraints_parallel(scalePT, phraseSizeLimit);

 			else

-				return PREM_phrase_constraints(scalePT, skipBigPhrases);

+				return PREM_phrase_constraints(scalePT, phraseSizeLimit);

 		}

-		else

-			return this.PREM_phrase_context_constraints(scalePT, scaleCT, skipBigPhrases);

+		else // FIXME: ignores phraseSizeLimit

+			return this.PREM_phrase_context_constraints(scalePT, scaleCT);

 	}

 

 	

-	public double PREM_phrase_constraints(double scalePT, boolean skipBigPhrases)

+	public double PREM_phrase_constraints(double scalePT, int phraseSizeLimit)

 	{

 		double [][][]exp_emit=new double[K][n_positions][n_words];

 		double [][]exp_pi=new double[n_phrases][K];

@@ -272,8 +268,9 @@ public class PhraseCluster {
 		int failures=0, iterations=0;

 		long start = System.currentTimeMillis();

 		//E

-		for(int phrase=0; phrase<n_phrases; phrase++){

-			if (skipBigPhrases && c.getPhrase(phrase).size() >= 2)

+		for(int phrase=0; phrase<n_phrases; phrase++)

+		{

+			if (phraseSizeLimit >= 1 && c.getPhrase(phrase).size() > phraseSizeLimit)

 			{

 				System.arraycopy(pi[phrase], 0, exp_pi[phrase], 0, K);

 				continue;

@@ -328,7 +325,7 @@ public class PhraseCluster {
 		return primal;

 	}

 

-	public double PREM_phrase_constraints_parallel(final double scalePT, boolean skipBigPhrases)

+	public double PREM_phrase_constraints_parallel(final double scalePT, int phraseSizeLimit)

 	{

 		assert(pool != null);

 		

@@ -338,12 +335,11 @@ public class PhraseCluster {
 		double [][][]exp_emit=new double [K][n_positions][n_words];

 		double [][]exp_pi=new double[n_phrases][K];

 		

-		if (skipBigPhrases)

-		{

-			for(double [][]i:exp_emit)

-				for(double []j:i)

-					Arrays.fill(j, 1e-100);

-		}

+		for(double [][]i:exp_emit)

+			for(double []j:i)

+				Arrays.fill(j, 1e-10);

+		for(double []j:pi)

+			Arrays.fill(j, 1e-10);

 		

 		double loglikelihood=0, kl=0, l1lmax=0, primal=0;

 		final AtomicInteger failures = new AtomicInteger(0);

@@ -356,12 +352,13 @@ public class PhraseCluster {
 

 		//E

 		for(int phrase=0;phrase<n_phrases;phrase++){

-			if (skipBigPhrases && c.getPhrase(phrase).size() >= 2)

+			if (phraseSizeLimit >= 1 && c.getPhrase(phrase).size() > phraseSizeLimit)

 			{

 				n -= 1;

 				System.arraycopy(pi[phrase], 0, exp_pi[phrase], 0, K);

 				continue;

 			}

+

 			final int p=phrase;

 			pool.execute(new Runnable() {

 				public void run() {

@@ -445,10 +442,8 @@ public class PhraseCluster {
 		return primal;

 	}

 	

-	public double PREM_phrase_context_constraints(double scalePT, double scaleCT, boolean skipBigPhrases)

+	public double PREM_phrase_context_constraints(double scalePT, double scaleCT)

 	{	

-		assert !skipBigPhrases : "Not supported yet - FIXME!"; //FIXME

-		

 		double[][][] exp_emit = new double [K][n_positions][n_words];

 		double[][] exp_pi = new double[n_phrases][K];

 

@@ -500,10 +495,14 @@ public class PhraseCluster {
 	 */

 	public double[] posterior(Corpus.Edge edge) 

 	{

-		double[] prob=Arrays.copyOf(pi[edge.getPhraseId()], K);

-		

-		//if (edge.getCount() < edge_threshold)

-			//System.out.println("Edge: " + edge + " probs for phrase " + Arrays.toString(prob));

+		double[] prob;

+		if (edge.getPhraseId() < n_phrases)

+			prob = Arrays.copyOf(pi[edge.getPhraseId()], K);

+		else

+		{

+			prob = new double[K];

+			Arrays.fill(prob, 1.0);

+		}

 		

 		TIntArrayList ctx = edge.getContext();

 		for(int tag=0;tag<K;tag++)

@@ -511,23 +510,17 @@ public class PhraseCluster {
 			for(int c=0;c<n_positions;c++)

 			{

 				int word = ctx.get(c);

-				//if (edge.getCount() < edge_threshold)

-					//System.out.println("\ttag: " + tag + " context word: " + word + " prob " + emit[tag][c][word]);

-

-				if (!this.c.isSentinel(word))

+				if (!this.c.isSentinel(word) && word < n_words)

 					prob[tag]*=emit[tag][c][word];

 			}

 		}

-

-		//if (edge.getCount() < edge_threshold)

-			//System.out.println("prob " + Arrays.toString(prob));

 		

 		return prob;

 	}

 	

-	public void displayPosterior(PrintStream ps)

+	public void displayPosterior(PrintStream ps, List<Edge> testing)

 	{	

-		for (Edge edge : c.getEdges())

+		for (Edge edge : testing)

 		{

 			double probs[] = posterior(edge);

 			arr.F.l1normalize(probs);

@@ -660,9 +653,4 @@ public class PhraseCluster {
 			emit[t][p][w] = v;

 		}

 	}

-

-	public void setEdgeThreshold(int edgeThreshold) 

-	{

-		this.edge_threshold = edgeThreshold;

-	}

 }

diff --git a/gi/posterior-regularisation/prjava/src/phrase/Trainer.java b/gi/posterior-regularisation/prjava/src/phrase/Trainer.java
index 7f0b1970..ec1a5804 100644
--- a/gi/posterior-regularisation/prjava/src/phrase/Trainer.java
+++ b/gi/posterior-regularisation/prjava/src/phrase/Trainer.java
@@ -7,8 +7,11 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.List;
 import java.util.Random;
 
+import phrase.Corpus.Edge;
+
 import arr.F;
 
 public class Trainer 
@@ -34,10 +37,6 @@ public class Trainer
         parser.accepts("agree");
         parser.accepts("no-parameter-cache");
         parser.accepts("skip-large-phrases").withRequiredArg().ofType(Integer.class).defaultsTo(5);
-        parser.accepts("rare-word").withRequiredArg().ofType(Integer.class).defaultsTo(10);
-        parser.accepts("rare-edge").withRequiredArg().ofType(Integer.class).defaultsTo(1);
-        parser.accepts("rare-phrase").withRequiredArg().ofType(Integer.class).defaultsTo(2);
-        parser.accepts("rare-context").withRequiredArg().ofType(Integer.class).defaultsTo(2);
         OptionSet options = parser.parse(args);
 
         if (options.has("help") || !options.has("in"))
@@ -61,10 +60,6 @@ public class Trainer
 		double alphaEmit = (vb) ? (Double) options.valueOf("alpha-emit") : 0;
 		double alphaPi = (vb) ? (Double) options.valueOf("alpha-pi") : 0;
 		int skip = (Integer) options.valueOf("skip-large-phrases");
-		int wordThreshold = (Integer) options.valueOf("rare-word");
-		int edgeThreshold = (Integer) options.valueOf("rare-edge");
-		int phraseThreshold = (Integer) options.valueOf("rare-phrase");
-		int contextThreshold = (Integer) options.valueOf("rare-context");
 		
 		if (options.has("seed"))
 			F.rng = new Random((Long) options.valueOf("seed"));
@@ -86,14 +81,7 @@ public class Trainer
 			e.printStackTrace();
 			System.exit(1);
 		}
-		
-		if (wordThreshold > 1)
-			corpus.applyWordThreshold(wordThreshold);
-		if (phraseThreshold > 1)
-			corpus.applyPhraseThreshold(phraseThreshold);
-		if (contextThreshold > 1)
-			corpus.applyContextThreshold(contextThreshold);
-		
+				
 		if (!options.has("agree"))
 			System.out.println("Running with " + tags + " tags " +
 					"for " + iterations + " iterations " +
@@ -127,7 +115,6 @@ public class Trainer
 					e.printStackTrace();
 				}
  			}
-			cluster.setEdgeThreshold(edgeThreshold);
  		}
 				
 		double last = 0;
@@ -138,20 +125,24 @@ public class Trainer
 				o = agree.EM();
 			else
 			{
+				if (i < skip)
+					System.out.println("Skipping phrases of length > " + (i+1));
+				
 				if (scale_phrase <= 0 && scale_context <= 0)
 				{
 					if (!vb)
-						o = cluster.EM(i < skip);
+						o = cluster.EM((i < skip) ? i+1 : 0);
 					else
-						o = cluster.VBEM(alphaEmit, alphaPi, i < skip);	
+						o = cluster.VBEM(alphaEmit, alphaPi);	
 				}
 				else
-					o = cluster.PREM(scale_phrase, scale_context, i < skip);
+					o = cluster.PREM(scale_phrase, scale_context, (i < skip) ? i+1 : 0);
 			}
 			
 			System.out.println("ITER: "+i+" objective: " + o);
 			
-			if (i != 0 && Math.abs((o - last) / o) < threshold)
+			// sometimes takes a few iterations to break the ties
+			if (i > 5 && Math.abs((o - last) / o) < threshold)
 			{
 				last = o;
 				break;
@@ -171,10 +162,19 @@ public class Trainer
 			File outfile = (File) options.valueOf("out");
 			try {
 				PrintStream ps = FileUtil.printstream(outfile);
-				cluster.displayPosterior(ps);
+				List<Edge> test;
+				if (!options.has("test"))
+					test = corpus.getEdges();
+				else
+				{
+					infile = (File) options.valueOf("test");
+					System.out.println("Reading testing concordance from " + infile);
+					test = corpus.readEdges(FileUtil.reader(infile));
+				}
+				cluster.displayPosterior(ps, test);
 				ps.close();
 			} catch (IOException e) {
-				System.err.println("Failed to open output file: " + outfile);
+				System.err.println("Failed to open either testing file or output file");
 				e.printStackTrace();
 				System.exit(1);
 			}