git-svn-id: https://ws10smt.googlecode.com/svn/trunk@173 ec762483-ff6d-05da-a07a-a48fb63a330f

3 files changed, 104 insertions, 44 deletions

diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java
index cd28c12e..731d03ac 100644
--- a/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java
+++ b/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java
@@ -1,11 +1,16 @@
 package phrase;

 

 import io.FileUtil;

+

+import java.io.FileOutputStream;

+import java.io.IOException;

+import java.io.OutputStream;

 import java.io.PrintStream;

 import java.util.Arrays;

 import java.util.concurrent.ExecutorService;

 import java.util.concurrent.Executors;

 import java.util.concurrent.LinkedBlockingQueue;

+import java.util.zip.GZIPOutputStream;

 

 public class PhraseCluster {

 	

@@ -26,28 +31,46 @@ public class PhraseCluster {
 	public static void main(String[] args) {

 		String input_fname = args[0];

 		int tags = Integer.parseInt(args[1]);

-		String outputDir = args[2];

+		String output_fname = args[2];

 		int iterations = Integer.parseInt(args[3]);

 		double scale = Double.parseDouble(args[4]);

 		int threads = Integer.parseInt(args[5]);

+		boolean runEM = Boolean.parseBoolean(args[6]);

 		

 		PhraseCorpus corpus = new PhraseCorpus(input_fname);

 		PhraseCluster cluster = new PhraseCluster(tags, corpus, scale, threads);

 		

-		PhraseObjective.ps = FileUtil.openOutFile(outputDir + "/phrase_stat.out");

+		//PhraseObjective.ps = FileUtil.openOutFile(outputDir + "/phrase_stat.out");

 		

+		double last = 0;

 		for(int i=0;i<iterations;i++){

-			double o = cluster.PREM();

-		    //double o = cluster.EM();

-			PhraseObjective.ps.println("ITER: "+i+" objective: " + o);

+			

+			double o;

+			if (runEM || i < 3) 

+				o = cluster.EM();

+			else

+				o = cluster.PREM();

+			//PhraseObjective.ps.

+			System.out.println("ITER: "+i+" objective: " + o);

+			last = o;

+		}

+		

+		if (runEM)

+		{

+			double l1lmax = cluster.posterior_l1lmax();

+			System.out.println("Final l1lmax term " + l1lmax + ", total PR objective " + (last - scale*l1lmax));

+			// nb. KL is 0 by definition

 		}

 		

-		PrintStream ps=io.FileUtil.openOutFile(outputDir + "/posterior.out");

+		PrintStream ps=io.FileUtil.openOutFile(output_fname);

 		cluster.displayPosterior(ps);

-		ps.println();

-		cluster.displayModelParam(ps);

 		ps.close();

-		PhraseObjective.ps.close();

+		

+		//PhraseObjective.ps.close();

+

+		//ps = io.FileUtil.openOutFile(outputDir + "/parameters.out");

+		//cluster.displayModelParam(ps);

+		//ps.close();

 		

 		cluster.finish();

 	}

@@ -61,7 +84,7 @@ public class PhraseCluster {
 		if (threads > 0)

 			pool = Executors.newFixedThreadPool(threads);

 		

-		emit=new double [K][PhraseCorpus.NUM_CONTEXT][n_words];

+		emit=new double [K][c.numContexts][n_words];

 		pi=new double[n_phrase][K];

 		

 		for(double [][]i:emit){

@@ -82,7 +105,7 @@ public class PhraseCluster {
 	}

 		

 	public double EM(){

-		double [][][]exp_emit=new double [K][PhraseCorpus.NUM_CONTEXT][n_words];

+		double [][][]exp_emit=new double [K][c.numContexts][n_words];

 		double [][]exp_pi=new double[n_phrase][K];

 		

 		double loglikelihood=0;

@@ -93,7 +116,9 @@ public class PhraseCluster {
 			for(int ctx=0;ctx<data.length;ctx++){

 				int context[]=data[ctx];

 				double p[]=posterior(phrase,context);

-				loglikelihood+=Math.log(arr.F.l1norm(p));

+				double z = arr.F.l1norm(p);

+				assert z > 0;

+				loglikelihood+=Math.log(z);

 				arr.F.l1normalize(p);

 				

 				int contextCnt=context[context.length-1];

@@ -132,7 +157,7 @@ public class PhraseCluster {
 		if (pool != null)

 			return PREMParallel();

 		

-		double [][][]exp_emit=new double [K][PhraseCorpus.NUM_CONTEXT][n_words];

+		double [][][]exp_emit=new double [K][c.numContexts][n_words];

 		double [][]exp_pi=new double[n_phrase][K];

 		

 		double loglikelihood=0;

@@ -142,7 +167,7 @@ public class PhraseCluster {
 			PhraseObjective po=new PhraseObjective(this,phrase);

 			po.optimizeWithProjectedGradientDescent();

 			double [][] q=po.posterior();

-			loglikelihood+=po.getValue();

+			loglikelihood+=po.llh;

 			primal+=po.primal();

 			for(int edge=0;edge<q.length;edge++){

 				int []context=c.data[phrase][edge];

@@ -184,7 +209,7 @@ public class PhraseCluster {
 		final LinkedBlockingQueue<PhraseObjective> expectations 

 			= new LinkedBlockingQueue<PhraseObjective>();

 		

-		double [][][]exp_emit=new double [K][PhraseCorpus.NUM_CONTEXT][n_words];

+		double [][][]exp_emit=new double [K][c.numContexts][n_words];

 		double [][]exp_pi=new double[n_phrase][K];

 		

 		double loglikelihood=0;

@@ -220,7 +245,7 @@ public class PhraseCluster {
 				int phrase = po.phrase;

 				//System.out.println("" + Thread.currentThread().getId() + " taken phrase " + phrase);

 				double [][] q=po.posterior();

-				loglikelihood+=po.getValue();

+				loglikelihood+=po.llh;

 				primal+=po.primal();

 				for(int edge=0;edge<q.length;edge++){

 					int []context=c.data[phrase][edge];

@@ -295,18 +320,19 @@ public class PhraseCluster {
 				// emit phrase

 				ps.print(c.phraseList[i]);

 				ps.print("\t");

-				ps.print(c.getContextString(e));

-				ps.print("||| C=" + e[e.length-1] + " |||");

-

+				ps.print(c.getContextString(e, true));

 				int t=arr.F.argmax(probs);

+				ps.println(" ||| C=" + t);

+

+				//ps.print("||| C=" + e[e.length-1] + " |||");

 				

-				ps.print(t+"||| [");

-				for(t=0;t<K;t++){

-					ps.print(probs[t]+", ");

-				}

+				//ps.print(t+"||| [");

+				//for(t=0;t<K;t++){

+				//	ps.print(probs[t]+", ");

+				//}

 				// for (int t = 0; t < numTags; ++t)

 				// System.out.print(" " + probs[t]);

-				ps.println("]");

+				//ps.println("]");

 			}

 		}

 	}

@@ -329,14 +355,14 @@ public class PhraseCluster {
 		ps.println("P(word|tag,position)");

 		for (int i = 0; i < K; ++i)

 		{

-			ps.println(i);

-			for(int position=0;position<PhraseCorpus.NUM_CONTEXT;position++){

-				ps.println(position);

+			for(int position=0;position<c.numContexts;position++){

+				ps.println("tag " + i + " position " + position);

 				for(int word=0;word<emit[i][position].length;word++){

-					if((word+1)%100==0){

-						ps.println();

-					}

-					ps.print(c.wordList[word]+"="+emit[i][position][word]+"\t");

+					//if((word+1)%100==0){

+					//	ps.println();

+					//}

+					if (emit[i][position][word] > 1e-10)

+						ps.print(c.wordList[word]+"="+emit[i][position][word]+"\t");

 				}

 				ps.println();

 			}

@@ -344,4 +370,26 @@ public class PhraseCluster {
 		}

 		

 	}

+	

+	double posterior_l1lmax()

+	{

+		double sum=0;

+		for(int phrase=0;phrase<c.data.length;phrase++)

+		{

+			int [][] data = c.data[phrase];

+			double [] maxes = new double[K];

+			for(int ctx=0;ctx<data.length;ctx++)

+			{

+				int context[]=data[ctx];

+				double p[]=posterior(phrase,context);

+				arr.F.l1normalize(p);

+

+				for(int tag=0;tag<K;tag++)

+					maxes[tag] = Math.max(maxes[tag], p[tag]);

+			}

+			for(int tag=0;tag<K;tag++)

+				sum += maxes[tag];

+		}

+		return sum;

+	}

 }

diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java
index 99545371..b8f1f24a 100644
--- a/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java
+++ b/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java
@@ -9,11 +9,9 @@ import java.util.HashMap;
 import java.util.Scanner;

 

 public class PhraseCorpus {

-

 	

 	public static String LEX_FILENAME="../pdata/lex.out";

 	public static String DATA_FILENAME="../pdata/btec.con";

-	public static int NUM_CONTEXT=4;

 	

 	public HashMap<String,Integer>wordLex;

 	public HashMap<String,Integer>phraseLex;

@@ -23,6 +21,7 @@ public class PhraseCorpus {
 	

 	//data[phrase][num context][position]

 	public int data[][][];

+	public int numContexts;

 	

 	public static void main(String[] args) {

 		// TODO Auto-generated method stub

@@ -40,6 +39,7 @@ public class PhraseCorpus {
 		

 		ArrayList<int[][]>dataList=new ArrayList<int[][]>();

 		String line=null;

+		numContexts = 0;

 		

 		while((line=readLine(r))!=null){

 			

@@ -54,7 +54,12 @@ public class PhraseCorpus {
 			for(int i=0;i<toks.length;i+=2){

 				String ctx=toks[i];

 				String words[]=ctx.split(" ");

-				int []context=new int [NUM_CONTEXT+1];

+				if (numContexts == 0)

+					numContexts = words.length - 1;

+				else

+					assert numContexts == words.length - 1;

+				

+				int []context=new int [numContexts+1];

 				int idx=0;

 				for(String word:words){

 					if(word.equals("<PHRASE>")){

@@ -68,9 +73,7 @@ public class PhraseCorpus {
 				String count=toks[i+1];

 				context[idx]=Integer.parseInt(count.trim().substring(2));

 				

-				

 				ctxList.add(context);

-				

 			}

 			

 			dataList.add(ctxList.toArray(new int [0][]));

@@ -157,13 +160,17 @@ public class PhraseCorpus {
 		return dict;

 	}

 	

-	public String getContextString(int context[])

+	public String getContextString(int context[], boolean addPhraseMarker)

 	{

 		StringBuffer b = new StringBuffer();

 		for (int i=0;i<context.length-1;i++)

 		{

 			if (b.length() > 0)

 				b.append(" ");

+

+			if (i == context.length/2)

+				b.append("<PHRASE> ");

+			

 			b.append(wordList[context[i]]);

 		}

 		return b.toString();

diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java
index 71c91b96..b7c62261 100644
--- a/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java
+++ b/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java
@@ -20,17 +20,17 @@ import optimization.util.MathUtils;
 public class PhraseObjective extends ProjectedObjective{

 

 	private static final double GRAD_DIFF = 0.002;

-	public static double INIT_STEP_SIZE=1;

-	public static double VAL_DIFF=0.001;

-	private double c1=0.0001;

-	private double c2=0.9;

+	public static double INIT_STEP_SIZE = 10;

+	public static double VAL_DIFF = 0.001; // FIXME needs to be tuned

+	//private double c1=0.0001; // wolf stuff

+	//private double c2=0.9;

 	

 	private PhraseCluster c;

 	

 	/**@brief

 	 *  for debugging purposes

 	 */

-	public static PrintStream ps;

+	//public static PrintStream ps;

 	

 	/**@brief current phrase being optimzed*/

 	public int phrase;

@@ -61,7 +61,7 @@ public class PhraseObjective extends ProjectedObjective{
 	/**@brief likelihood under p

 	 * 

 	 */

-	private double llh;

+	public double llh;

 	

 	public PhraseObjective(PhraseCluster cluster, int phraseIdx){

 		phrase=phraseIdx;

@@ -181,7 +181,7 @@ public class PhraseObjective extends ProjectedObjective{
 		boolean succed = optimizer.optimize(this,stats,compositeStop);

 //		System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1));

 		if(succed){

-			System.out.println("Ended optimization in " + optimizer.getCurrentIteration());

+			//System.out.println("Ended optimization in " + optimizer.getCurrentIteration());

 		}else{

 			System.out.println("Failed to optimize");

 		}

@@ -208,6 +208,10 @@ public class PhraseObjective extends ProjectedObjective{
 		double kl=-loglikelihood

 			+MathUtils.dotProduct(parameters, gradient);

 //		ps.print(", "+kl);

+		//System.out.println("llh " + llh);

+		//System.out.println("kl " + kl);

+		

+

 		l=l-kl;

 		double sum=0;

 		for(int tag=0;tag<c.K;tag++){

@@ -219,6 +223,7 @@ public class PhraseObjective extends ProjectedObjective{
 			}

 			sum+=max;

 		}

+		//System.out.println("l1lmax " + sum);

 //		ps.println(", "+sum);

 		l=l-c.scale*sum;

 		return l;