diff options
Diffstat (limited to 'example/output.stderr')
| -rw-r--r-- | example/output.stderr | 4106 |
1 files changed, 4106 insertions, 0 deletions
diff --git a/example/output.stderr b/example/output.stderr new file mode 100644 index 0000000..864d110 --- /dev/null +++ b/example/output.stderr @@ -0,0 +1,4106 @@ +CONFIGURATION + k=100 + input=train.in + references=train.en + gold=train.gold + gold_mrl=train.funql + init_weights=weights.init + cdec_ini=cdec.ini + stopwords_file=stopwords.en + output_weights=output-weights + debug=false + print_kbest=true + eta=0.01 + iterate=3 + stop_after=30 + scale_model=1.0 + normalize=false + skip_on_no_proper_gold=false + no_update=false + hope_fear_max=10000000000 + variant=rampion + help=false + k_given=true + input_given=true + references_given=true + gold_given=true + gold_mrl_given=true + init_weights_given=true + cdec_ini_given=true + stopwords_file_given=true + output_weights_given=true + print_kbest_given=true + eta_given=true + iterate_given=true + stop_after_given=true + variant_given=true + +================= + EXAMPLE: 0 + REFERENCE: give me the cities in virginia + GOLD MRL: answer(city(loc_2(stateid('virginia')))) +GOLD OUTPUT: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + +<<< KBEST +0 ||| give me the cities in virginia ||| ||| {:decoder=>-2.93469, :psb=>1.0} +1 ||| give me cities in virginia ||| ||| {:decoder=>-3.13281, :psb=>0.49473859088183875} +2 ||| give me the cities virginia ||| ||| {:decoder=>-3.2086, :psb=>0.6511126026643229} +3 ||| give me the towns in virginia ||| ||| {:decoder=>-3.25434, :psb=>0.48549177170732344} +4 ||| give me the cities of virginia ||| ||| {:decoder=>-3.30623, :psb=>0.6389431042462724} +5 ||| i give the cities in virginia ||| ||| {:decoder=>-3.3163, :psb=>0.6389431042462724} +6 ||| unto me the cities in virginia ||| ||| {:decoder=>-3.36245, :psb=>0.8034284189446518} +7 ||| give unto me the cities in virginia ||| ||| {:decoder=>-3.36825, :psb=>0.7034711503007025} +8 ||| me give the cities in virginia ||| ||| {:decoder=>-3.37352, :psb=>0.668740304976422} +9 ||| unto me the towns in virginia ||| ||| {:decoder=>-3.80009, :psb=>0.35930411196308426} +>>> + + [TOP1] + nrl: give me the cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: give me the cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: give me cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>0.7710654032817191, :psb=>0.49473859088183875}, #1 + +================= + EXAMPLE: 1 + REFERENCE: what are the high points of states surrounding mississippi + GOLD MRL: answer(high_point_1(state(next_to_2(stateid('mississippi'))))) +GOLD OUTPUT: [placeid('cheaha mountain'),placeid('clingmans dome'),placeid('driskill mountain'),placeid('magazine mountain')] +NO MT OUTPUT, skipping example + +================= + EXAMPLE: 2 + REFERENCE: name the rivers in arkansas + GOLD MRL: answer(river(loc_2(stateid('arkansas')))) +GOLD OUTPUT: [riverid(arkansas),riverid(mississippi),riverid(ouachita),riverid(red),riverid('st. francis'),riverid(white)] + +<<< KBEST +0 ||| call the rivers in arkansas ||| ||| {:decoder=>-2.07277, :psb=>0.7521206186172787} +1 ||| the call rivers in arkansas ||| ||| {:decoder=>-2.45543, :psb=>0.5318295896944989} +2 ||| the rivers call in arkansas ||| ||| {:decoder=>-2.49639, :psb=>0.4472135954999579} +3 ||| name the rivers in arkansas ||| ||| {:decoder=>-2.54568, :psb=>1.0} +4 ||| the rivers in arkansas call ||| ||| {:decoder=>-2.55167, :psb=>0.7521206186172787} +5 ||| call the rivers in the arkansas ||| ||| {:decoder=>-2.60767, :psb=>0.4272870063962341} +6 ||| call the rivers to arkansas ||| ||| {:decoder=>-2.62805, :psb=>0.3760603093086393} +7 ||| called the rivers in arkansas ||| ||| {:decoder=>-2.64447, :psb=>0.7521206186172787} +8 ||| call the rivers in nicky ||| ||| {:decoder=>-2.65825, :psb=>0.4949232003839765} +9 ||| call the rivers into arkansas ||| ||| {:decoder=>-2.68599, :psb=>0.3760603093086393} +10 ||| call the rivers in from ||| ||| {:decoder=>-2.68817, :psb=>0.4949232003839765} +11 ||| call the rivers in first ||| ||| {:decoder=>-2.69615, :psb=>0.4949232003839765} +12 ||| call rivers in the arkansas ||| ||| {:decoder=>-2.7041, :psb=>0.40410310093532464} +13 ||| do the rivers in arkansas ||| ||| {:decoder=>-2.71057, :psb=>0.7521206186172787} +14 ||| call the rivers in made ||| ||| {:decoder=>-2.71232, :psb=>0.4949232003839765} +>>> + + [TOP1] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787} + + [HOPE] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787}, #0 + + [FEAR] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787}, #0 + +================= + EXAMPLE: 3 + REFERENCE: can you tell me the capital of texas + GOLD MRL: answer(capital(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(austin,tx)] + +<<< KBEST +0 ||| you can tell me , what the capital of texas is ||| ||| {:decoder=>-2.30551, :psb=>0.3852971889399676} +1 ||| you can you tell me , what the capital of texas is ||| ||| {:decoder=>-2.374, :psb=>0.47987820666906633} +2 ||| can you tell me , what the capital of texas is ||| ||| {:decoder=>-2.39111, :psb=>0.5270027137504644} +3 ||| you can tell me , what is the capital of texas ||| ||| {:decoder=>-2.43333, :psb=>0.3852971889399676} +4 ||| you can tell me what the capital of texas is ||| ||| {:decoder=>-2.46734, :psb=>0.4272870063962341} +5 ||| you can you tell me , what is the capital of texas ||| ||| {:decoder=>-2.50182, :psb=>0.47987820666906633} +6 ||| can you tell me , what is the capital of texas ||| ||| {:decoder=>-2.51894, :psb=>0.5270027137504644} +7 ||| can you tell me what the capital of texas is ||| ||| {:decoder=>-2.5461, :psb=>0.5844356470407898} +8 ||| you can you tell me what the capital of texas is ||| ||| {:decoder=>-2.54851, :psb=>0.5270027137504644} +9 ||| you can you tell me , was the capital of texas is ||| ||| {:decoder=>-2.56709, :psb=>0.47987820666906633} +10 ||| you tell me , what the capital of texas is ||| ||| {:decoder=>-2.57737, :psb=>0.46478456293957165} +11 ||| you can tell me , what the capital from texas is ||| ||| {:decoder=>-2.60129, :psb=>0.2095549161132726} +12 ||| you can tell me , was the capital of texas is ||| ||| {:decoder=>-2.62677, :psb=>0.3852971889399676} +13 ||| you can you tell me , what the capital from texas is ||| ||| {:decoder=>-2.66978, :psb=>0.33932513407933634} +14 ||| you can i say , what the capital of texas is ||| ||| {:decoder=>-2.67341, :psb=>0.33910457813956707} +15 ||| you can you tell me , what the capital of texas , ||| ||| {:decoder=>-2.70216, :psb=>0.47987820666906633} +16 ||| you can me say , what the capital of texas is ||| ||| {:decoder=>-2.74144, :psb=>0.3524279555168876} +17 ||| you can tell me what the capital from texas is ||| ||| {:decoder=>-2.76312, :psb=>0.23239228146978586} +18 ||| you can tell me , what the capital city of texas is ||| ||| {:decoder=>-2.8331, :psb=>0.21200626759025185} +19 ||| you can you tell me what the capital from texas is ||| ||| {:decoder=>-2.8443, :psb=>0.3726471925966664} +20 ||| you can you tell me what the capital of texas , ||| ||| {:decoder=>-2.87667, :psb=>0.5270027137504644} +21 ||| you can you tell me , was the capital of texas , ||| ||| {:decoder=>-2.89525, :psb=>0.47987820666906633} +22 ||| you can you tell me , what the capital city of texas is ||| ||| {:decoder=>-2.90159, :psb=>0.33707998271946277} +23 ||| can you tell me , what the capital city of texas is ||| ||| {:decoder=>-2.9187, :psb=>0.3672056269893592} +24 ||| you can you tell me , what the capital city of texas , ||| ||| {:decoder=>-3.22975, :psb=>0.33707998271946277} +>>> + + [TOP1] + nrl: you can tell me , what the capital of texas is + mrl: answer(population_1(answer(capital(loc_2(stateid('texas')))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3852971889399676} + + [HOPE] + nrl: can you tell me , what the capital of texas is + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>0.9073833636284949, :psb=>0.5270027137504644}, #2 + + [FEAR] + nrl: you can tell me , what the capital of texas is + mrl: answer(population_1(answer(capital(loc_2(stateid('texas')))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3852971889399676}, #0 + +================= + EXAMPLE: 4 + REFERENCE: could you tell me what is the highest point in the state of oregon + GOLD MRL: answer(highest(place(loc_2(state(stateid('oregon')))))) +GOLD OUTPUT: [placeid('mount hood')] + +<<< KBEST +0 ||| could you tell me that the highest point in the state of oregon is ||| ||| {:decoder=>-3.9874, :psb=>0.7367565054628357} +1 ||| could you tell me what the highest point in the state of oregon is ||| ||| {:decoder=>-4.02165, :psb=>0.8142441551106129} +2 ||| could you tell me which the highest point in the state of oregon is ||| ||| {:decoder=>-4.0219, :psb=>0.7367565054628357} +3 ||| could you tell me which is the highest point in the state of oregon ||| ||| {:decoder=>-4.05843, :psb=>0.7992975447228272} +4 ||| could you tell me that the highest point in the state oregon is ||| ||| {:decoder=>-4.16269, :psb=>0.5940876484765517} +5 ||| you could tell me that the highest point in the state of oregon is ||| ||| {:decoder=>-4.17303, :psb=>0.633160218956179} +6 ||| you could tell me what the highest point in the state of oregon is ||| ||| {:decoder=>-4.1808, :psb=>0.684694991173804} +7 ||| could you tell me what the highest point in the state oregon is ||| ||| {:decoder=>-4.19694, :psb=>0.6734009436331136} +8 ||| could you tell me which the highest point in the state oregon is ||| ||| {:decoder=>-4.19719, :psb=>0.5940876484765517} +9 ||| you could tell me which the highest point in the state of oregon is ||| ||| {:decoder=>-4.21109, :psb=>0.633160218956179} +10 ||| could you tell me what is the highest point in the state of oregon ||| ||| {:decoder=>-4.21202, :psb=>1.0} +11 ||| could you tell me that the highest point in oregon state is ||| ||| {:decoder=>-4.21315, :psb=>0.4495431954509448} +12 ||| could you tell me that the highest point in the oregon state is ||| ||| {:decoder=>-4.2402, :psb=>0.5249261301080137} +13 ||| could you tell me what the highest point in oregon state is ||| ||| {:decoder=>-4.2474, :psb=>0.5333225381284186} +14 ||| could you tell me which the highest point in oregon state is ||| ||| {:decoder=>-4.24765, :psb=>0.4495431954509448} +15 ||| could you tell me that is the highest point in the state of oregon ||| ||| {:decoder=>-4.26389, :psb=>0.7992975447228272} +16 ||| could you tell me what the highest point in the oregon state is ||| ||| {:decoder=>-4.27445, :psb=>0.6060954862666158} +17 ||| could you tell me which the highest point in the oregon state is ||| ||| {:decoder=>-4.2747, :psb=>0.5249261301080137} +18 ||| could you tell me which of the highest point in the state of oregon is ||| ||| {:decoder=>-4.3113, :psb=>0.6848661007688605} +19 ||| could you tell me which of the highest point in the state oregon is ||| ||| {:decoder=>-4.48658, :psb=>0.6049483675122199} +20 ||| could you tell me which of the highest point in oregon state is ||| ||| {:decoder=>-4.53705, :psb=>0.4605329793777294} +21 ||| you could tell me what the highest point in the state oregon is ||| ||| {:decoder=>-4.55444, :psb=>0.535536059195526} +22 ||| could you tell me which of the highest point in the oregon state is ||| ||| {:decoder=>-4.5641, :psb=>0.5345224838248488} +>>> + + [TOP1] + nrl: could you tell me that the highest point in the state of oregon is + mrl: population_1(state(highest(place(loc_2(stateid('oregon')))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7367565054628357} + + [HOPE] + nrl: could you tell me what the highest point in the state of oregon is + mrl: answer(highest(place(loc_2(loc_2(state(stateid('oregon'))))))) + output: [] + correct?: false + SCORES: {:decoder=>0.9406103693428123, :psb=>0.8142441551106129}, #1 + + [FEAR] + nrl: could you tell me that the highest point in the state of oregon is + mrl: population_1(state(highest(place(loc_2(stateid('oregon')))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7367565054628357}, #0 + +================= + EXAMPLE: 5 + REFERENCE: give me all the states of usa + GOLD MRL: answer(state(loc_2(countryid('usa')))) +GOLD OUTPUT: [stateid(alabama),stateid(alaska),stateid(arizona),stateid(arkansas),stateid(california),stateid(colorado),stateid(connecticut),stateid(delaware),stateid('district of columbia'),stateid(florida),stateid(georgia),stateid(hawaii),stateid(idaho),stateid(illinois),stateid(indiana),stateid(iowa),stateid(kansas),stateid(kentucky),stateid(louisiana),stateid(maine),stateid(maryland),stateid(massachusetts),stateid(michigan),stateid(minnesota),stateid(mississippi),stateid(missouri),stateid(montana),stateid(nebraska),stateid(nevada),stateid('new hampshire'),stateid('new jersey'),stateid('new mexico'),stateid('new york'),stateid('north carolina'),stateid('north dakota'),stateid(ohio),stateid(oklahoma),stateid(oregon),stateid(pennsylvania),stateid('rhode island'),stateid('south carolina'),stateid('south dakota'),stateid(tennessee),stateid(texas),stateid(utah),stateid(vermont),stateid(virginia),stateid(washington),stateid('west virginia'),stateid(wisconsin),stateid(wyoming)] + +<<< KBEST +0 ||| call all states ||| ||| {:decoder=>-2.26563, :psb=>0.15218787864872976} +1 ||| call all states the u.s. ||| ||| {:decoder=>-2.30394, :psb=>0.21197381067415416} +2 ||| call all states the usa ||| ||| {:decoder=>-2.32666, :psb=>0.22778068328450227} +3 ||| call all the usa ||| ||| {:decoder=>-2.42772, :psb=>0.23618327637050732} +4 ||| call all states of usa ||| ||| {:decoder=>-2.44782, :psb=>0.3564960350471317} +5 ||| call all states of the usa ||| ||| {:decoder=>-2.44932, :psb=>0.29059254080791846} +6 ||| all states call ||| ||| {:decoder=>-2.4986, :psb=>0.15218787864872976} +7 ||| call all countries the usa ||| ||| {:decoder=>-2.5191, :psb=>0.21197381067415416} +8 ||| all call states ||| ||| {:decoder=>-2.57315, :psb=>0.15218787864872976} +9 ||| call all states the united states ||| ||| {:decoder=>-2.65098, :psb=>0.21506254256566312} +10 ||| call all states the us ||| ||| {:decoder=>-2.66279, :psb=>0.21197381067415416} +11 ||| call all united states of america ||| ||| {:decoder=>-2.76251, :psb=>0.2557539057896621} +>>> + + [TOP1] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976} + + [HOPE] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976}, #0 + + [FEAR] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976}, #0 + +================= + EXAMPLE: 6 + REFERENCE: give me the cities in texas + GOLD MRL: answer(city(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + +<<< KBEST +0 ||| tell me what cities in texas are ||| ||| {:decoder=>-2.19707, :psb=>0.35745672884978397} +1 ||| tell me what cities in texas , ||| ||| {:decoder=>-2.41406, :psb=>0.35745672884978397} +2 ||| tell me what are cities in texas ||| ||| {:decoder=>-2.48748, :psb=>0.35745672884978397} +3 ||| tell me what cities in texas is ||| ||| {:decoder=>-2.48963, :psb=>0.35745672884978397} +4 ||| tell me which cities in texas are ||| ||| {:decoder=>-2.49004, :psb=>0.35745672884978397} +5 ||| tell me what cities are in texas ||| ||| {:decoder=>-2.4976, :psb=>0.271608378656351} +6 ||| tell me that cities in texas are ||| ||| {:decoder=>-2.58874, :psb=>0.35745672884978397} +7 ||| tell me what towns in texas are ||| ||| {:decoder=>-2.60186, :psb=>0.2527600769504433} +8 ||| tell me the cities in texas are ||| ||| {:decoder=>-2.65822, :psb=>0.672126440078521} +9 ||| tell me which cities in texas , ||| ||| {:decoder=>-2.70704, :psb=>0.35745672884978397} +>>> + + [TOP1] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397} + + [HOPE] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397}, #0 + + [FEAR] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397}, #0 + +================= + EXAMPLE: 7 + REFERENCE: give me the cities in usa + GOLD MRL: answer(city(loc_2(countryid('usa')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + +<<< KBEST +0 ||| call the cities of usa ||| ||| {:decoder=>-1.9387, :psb=>0.3078921402430011} +1 ||| call the cities the usa ||| ||| {:decoder=>-2.01213, :psb=>0.3078921402430011} +2 ||| call the cities of the usa ||| ||| {:decoder=>-2.06626, :psb=>0.3021375397356768} +3 ||| call the cities of u.s. ||| ||| {:decoder=>-2.08364, :psb=>0.27821195481929917} +4 ||| call the cities the u.s. ||| ||| {:decoder=>-2.08532, :psb=>0.27821195481929917} +5 ||| call the cities in the usa ||| ||| {:decoder=>-2.27222, :psb=>0.4272870063962341} +6 ||| call the towns of usa ||| ||| {:decoder=>-2.30112, :psb=>0.23394743548827707} +7 ||| call the cities of us ||| ||| {:decoder=>-2.31409, :psb=>0.27821195481929917} +8 ||| call the u.s. cities ||| ||| {:decoder=>-2.32271, :psb=>0.2304318198457308} +9 ||| call the cities of the united states ||| ||| {:decoder=>-2.32819, :psb=>0.2283945119649991} +10 ||| call the cities the united states ||| ||| {:decoder=>-2.33646, :psb=>0.2730120862709067} +11 ||| call the cities in the united states ||| ||| {:decoder=>-2.33668, :psb=>0.33265096878635064} +>>> + + [TOP1] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011} + + [HOPE] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011}, #0 + + [FEAR] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011}, #0 + +================= + EXAMPLE: 8 + REFERENCE: give me the cities in virginia + GOLD MRL: answer(city(loc_2(stateid('virginia')))) +GOLD OUTPUT: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + +<<< KBEST +0 ||| call me the cities in virginia ||| ||| {:decoder=>-2.12435, :psb=>0.8034284189446518} +1 ||| call me the towns in virginia ||| ||| {:decoder=>-2.41531, :psb=>0.35930411196308426} +2 ||| the call me cities in virginia ||| ||| {:decoder=>-2.50236, :psb=>0.45180100180492244} +3 ||| me call the cities in virginia ||| ||| {:decoder=>-2.56986, :psb=>0.6389431042462724} +4 ||| call me in the cities virginia ||| ||| {:decoder=>-2.57372, :psb=>0.34329452398451965} +5 ||| call me the cities virginia ||| ||| {:decoder=>-2.61388, :psb=>0.43542524047973125} +6 ||| the cities in virginia call me ||| ||| {:decoder=>-2.63093, :psb=>0.6389431042462724} +7 ||| i call the cities in virginia ||| ||| {:decoder=>-2.64068, :psb=>0.6042750794713536} +8 ||| call me the cities to virginia ||| ||| {:decoder=>-2.66484, :psb=>0.4272870063962341} +9 ||| call the cities in virginia me ||| ||| {:decoder=>-2.67013, :psb=>0.6389431042462724} +10 ||| call i the cities in virginia ||| ||| {:decoder=>-2.67208, :psb=>0.6042750794713536} +11 ||| call me the cities in va ||| ||| {:decoder=>-2.67613, :psb=>0.6042750794713536} +12 ||| call me cities in the virginia ||| ||| {:decoder=>-2.69001, :psb=>0.34329452398451965} +>>> + + [TOP1] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 9 + REFERENCE: give me the cities which are in texas + GOLD MRL: answer(city(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + +<<< KBEST +0 ||| call me cities in texas ||| ||| {:decoder=>-1.85974, :psb=>0.221776483974985} +1 ||| call me towns in texas ||| ||| {:decoder=>-2.20454, :psb=>0.20638627362169998} +2 ||| call me in texas cities ||| ||| {:decoder=>-2.24615, :psb=>0.221776483974985} +3 ||| call i cities in texas ||| ||| {:decoder=>-2.29025, :psb=>0.20638627362169998} +4 ||| me call cities in texas ||| ||| {:decoder=>-2.36612, :psb=>0.221776483974985} +5 ||| cities in texas call me ||| ||| {:decoder=>-2.39526, :psb=>0.221776483974985} +6 ||| cities call me in texas ||| ||| {:decoder=>-2.40861, :psb=>0.221776483974985} +7 ||| call cities in texas me ||| ||| {:decoder=>-2.45898, :psb=>0.221776483974985} +8 ||| cities in call me texas ||| ||| {:decoder=>-2.51053, :psb=>0.18649105036213778} +9 ||| call cities me in texas ||| ||| {:decoder=>-2.51799, :psb=>0.221776483974985} +10 ||| me cities in texas call ||| ||| {:decoder=>-2.52114, :psb=>0.221776483974985} +>>> + + [TOP1] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985} + + [HOPE] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985}, #0 + + [FEAR] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985}, #0 + +================= + EXAMPLE: 10 + REFERENCE: give me the lakes in california + GOLD MRL: answer(lake(loc_2(stateid('california')))) +GOLD OUTPUT: [] + +<<< KBEST +0 ||| call me the lakes in california ||| ||| {:decoder=>-2.04111, :psb=>0.8034284189446518} +1 ||| the call me lakes in california ||| ||| {:decoder=>-2.42578, :psb=>0.45180100180492244} +2 ||| the lakes call me in california ||| ||| {:decoder=>-2.48008, :psb=>0.37991784282579627} +3 ||| me call the lakes in california ||| ||| {:decoder=>-2.48661, :psb=>0.6389431042462724} +4 ||| call me the lakes to california ||| ||| {:decoder=>-2.50608, :psb=>0.4272870063962341} +5 ||| call me the lakes of california ||| ||| {:decoder=>-2.52394, :psb=>0.4272870063962341} +6 ||| the lakes in california call me ||| ||| {:decoder=>-2.53061, :psb=>0.6389431042462724} +7 ||| call the me lakes in california ||| ||| {:decoder=>-2.54373, :psb=>0.45180100180492244} +8 ||| i call the lakes in california ||| ||| {:decoder=>-2.54412, :psb=>0.6042750794713536} +9 ||| call the lakes me in california ||| ||| {:decoder=>-2.54644, :psb=>0.37991784282579627} +10 ||| call me lakes in the california ||| ||| {:decoder=>-2.59348, :psb=>0.34329452398451965} +11 ||| call the lakes in california me ||| ||| {:decoder=>-2.60022, :psb=>0.6389431042462724} +12 ||| call me in the lakes california ||| ||| {:decoder=>-2.61399, :psb=>0.34329452398451965} +13 ||| call i the lakes in california ||| ||| {:decoder=>-2.62112, :psb=>0.6042750794713536} +14 ||| call me in california the lakes ||| ||| {:decoder=>-2.6326, :psb=>0.37991784282579627} +15 ||| called me the lakes in california ||| ||| {:decoder=>-2.72303, :psb=>0.8034284189446518} +16 ||| me the call lakes in california ||| ||| {:decoder=>-2.7312, :psb=>0.48549177170732344} +>>> + + [TOP1] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 11 + REFERENCE: give me the largest state + GOLD MRL: answer(largest(state(all))) +GOLD OUTPUT: [stateid(alaska)] + +<<< KBEST +0 ||| the city is the largest ||| ||| {:decoder=>-1.52264, :psb=>0.33980884896942454} +1 ||| which city is the largest ||| ||| {:decoder=>-1.54833, :psb=>0.33980884896942454} +2 ||| what city is the largest ||| ||| {:decoder=>-1.58285, :psb=>0.33980884896942454} +3 ||| which city is the biggest ||| ||| {:decoder=>-1.69687, :psb=>0.24028114141347542} +4 ||| what city is the biggest ||| ||| {:decoder=>-1.73139, :psb=>0.24028114141347542} +5 ||| what is the largest city ||| ||| {:decoder=>-1.74759, :psb=>0.33980884896942454} +6 ||| which town is the largest ||| ||| {:decoder=>-1.75937, :psb=>0.33980884896942454} +7 ||| which is the largest city ||| ||| {:decoder=>-1.8243, :psb=>0.33980884896942454} +8 ||| what town is the largest ||| ||| {:decoder=>-1.8533, :psb=>0.33980884896942454} +9 ||| that city is the largest ||| ||| {:decoder=>-1.86788, :psb=>0.33980884896942454} +10 ||| which town is the biggest ||| ||| {:decoder=>-1.90791, :psb=>0.24028114141347542} +>>> + + [TOP1] + nrl: the city is the largest + mrl: largest(city(all)) + output: [cityid('new york',ny)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454} + + [HOPE] + nrl: the city is the largest + mrl: largest(city(all)) + output: [cityid('new york',ny)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454}, #0 + + [FEAR] + nrl: the city is the largest + mrl: largest(city(all)) + output: [cityid('new york',ny)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454}, #0 + +================= + EXAMPLE: 12 + REFERENCE: give me the longest river that passes through the us + GOLD MRL: answer(longest(river(traverse_2(countryid('usa'))))) +GOLD OUTPUT: [] + +<<< KBEST +0 ||| call me the longest river , which flows through the usa ||| ||| {:decoder=>-3.42816, :psb=>0.3585594362259136} +1 ||| call me the longest river , which flows by the usa ||| ||| {:decoder=>-3.47408, :psb=>0.32399502498695193} +2 ||| call me the longest river , which by the usa flows ||| ||| {:decoder=>-3.50156, :psb=>0.32399502498695193} +3 ||| call me the longest river , which through the usa flows ||| ||| {:decoder=>-3.71014, :psb=>0.3585594362259136} +4 ||| call me the longest river flows , which by the usa ||| ||| {:decoder=>-3.72391, :psb=>0.32399502498695193} +5 ||| call me the longest river , which flows through the united states ||| ||| {:decoder=>-3.73799, :psb=>0.3264971028628052} +6 ||| call me the longest river which by the usa flows ||| ||| {:decoder=>-3.74014, :psb=>0.35930411196308426} +7 ||| call me the longest river , which by the usa is ||| ||| {:decoder=>-3.79029, :psb=>0.32399502498695193} +8 ||| call me the longest river which flows through the usa ||| ||| {:decoder=>-3.79473, :psb=>0.3976353643835253} +9 ||| call me the longest river which by the usa is ||| ||| {:decoder=>-4.02887, :psb=>0.35930411196308426} +>>> + + [TOP1] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136} + + [HOPE] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136}, #0 + + [FEAR] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136}, #0 + +================= + EXAMPLE: 13 + REFERENCE: how big is alaska + GOLD MRL: answer(size(stateid('alaska'))) +GOLD OUTPUT: [591000.0] + +<<< KBEST +0 ||| how big is alaska ||| ||| {:decoder=>-1.76349, :psb=>1.0} +1 ||| how large is alaska ||| ||| {:decoder=>-1.87235, :psb=>0.49999999999999994} +2 ||| how great is alaska ||| ||| {:decoder=>-1.93435, :psb=>0.49999999999999994} +3 ||| as large is alaska ||| ||| {:decoder=>-1.97498, :psb=>0.4518010018049224} +4 ||| as big is alaska ||| ||| {:decoder=>-2.06714, :psb=>0.6580370064762462} +5 ||| as great is alaska ||| ||| {:decoder=>-2.13864, :psb=>0.4518010018049224} +6 ||| like large is alaska ||| ||| {:decoder=>-2.14212, :psb=>0.4518010018049224} +7 ||| like big is alaska ||| ||| {:decoder=>-2.19425, :psb=>0.6580370064762462} +8 ||| as large alaska ||| ||| {:decoder=>-2.19548, :psb=>0.3478700554542393} +9 ||| how big alaska is ||| ||| {:decoder=>-2.19985, :psb=>0.537284965911771} +10 ||| such as large is alaska ||| ||| {:decoder=>-2.21714, :psb=>0.33980884896942454} +11 ||| as large alaska is ||| ||| {:decoder=>-2.21774, :psb=>0.37991784282579627} +12 ||| is as large alaska ||| ||| {:decoder=>-2.22111, :psb=>0.37991784282579627} +13 ||| how large alaska is ||| ||| {:decoder=>-2.25183, :psb=>0.42044820762685725} +14 ||| is as big alaska ||| ||| {:decoder=>-2.28654, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how large is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>0.7918745817799446, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 14 + REFERENCE: how big is massachusetts + GOLD MRL: answer(size(stateid('massachusetts'))) +GOLD OUTPUT: [8284.0] + +<<< KBEST +0 ||| how big is massachusetts ||| ||| {:decoder=>-2.32488, :psb=>1.0} +1 ||| how large is massachusetts ||| ||| {:decoder=>-2.52005, :psb=>0.49999999999999994} +2 ||| how great is massachusetts ||| ||| {:decoder=>-2.55259, :psb=>0.49999999999999994} +3 ||| what is massachusetts ||| ||| {:decoder=>-2.59019, :psb=>0.4919625503668659} +4 ||| how much is massachusetts ||| ||| {:decoder=>-2.65221, :psb=>0.49999999999999994} +5 ||| how great massachusetts ||| ||| {:decoder=>-2.67531, :psb=>0.41368954504257255} +6 ||| as large is massachusetts ||| ||| {:decoder=>-2.73596, :psb=>0.4518010018049224} +7 ||| how big , massachusetts ||| ||| {:decoder=>-2.7697, :psb=>0.49999999999999994} +8 ||| as large massachusetts ||| ||| {:decoder=>-2.77246, :psb=>0.3478700554542393} +9 ||| as big is massachusetts ||| ||| {:decoder=>-2.82198, :psb=>0.6580370064762462} +10 ||| as big massachusetts ||| ||| {:decoder=>-2.82936, :psb=>0.41368954504257255} +11 ||| is as large massachusetts ||| ||| {:decoder=>-2.84073, :psb=>0.37991784282579627} +12 ||| how large , massachusetts ||| ||| {:decoder=>-2.84939, :psb=>0.37991784282579627} +13 ||| is how big massachusetts ||| ||| {:decoder=>-2.88164, :psb=>0.537284965911771} +14 ||| is as big massachusetts ||| ||| {:decoder=>-2.90651, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how large is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>0.664443030792772, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 15 + REFERENCE: how big is new mexico + GOLD MRL: answer(size(stateid('new mexico'))) +GOLD OUTPUT: [121600.0] + +<<< KBEST +0 ||| how big is new mexico ||| ||| {:decoder=>-2.9497, :psb=>1.0} +1 ||| what is new mexico ||| ||| {:decoder=>-2.97296, :psb=>0.5124797359336637} +2 ||| how big is mexico ||| ||| {:decoder=>-3.10052, :psb=>0.5506953149031837} +3 ||| how great is new mexico ||| ||| {:decoder=>-3.2228, :psb=>0.5318295896944989} +4 ||| how much is new mexico ||| ||| {:decoder=>-3.25635, :psb=>0.5318295896944989} +5 ||| how great new mexico ||| ||| {:decoder=>-3.27266, :psb=>0.3894003915357024} +6 ||| how large is new mexico ||| ||| {:decoder=>-3.30084, :psb=>0.5318295896944989} +7 ||| what is mexico ||| ||| {:decoder=>-3.33284, :psb=>0.29642151188002913} +8 ||| how great is mexico ||| ||| {:decoder=>-3.37363, :psb=>0.32744539334076506} +9 ||| how much is mexico ||| ||| {:decoder=>-3.40718, :psb=>0.32744539334076506} +10 ||| how large is mexico ||| ||| {:decoder=>-3.47877, :psb=>0.32744539334076506} +11 ||| just how big is new mexico ||| ||| {:decoder=>-3.49964, :psb=>0.8034284189446518} +12 ||| how big , new mexico ||| ||| {:decoder=>-3.53163, :psb=>0.4472135954999579} +13 ||| how large , new mexico ||| ||| {:decoder=>-3.59678, :psb=>0.3760603093086393} +>>> + + [TOP1] + nrl: how big is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: what is new mexico + mrl: answer('new mexico') + output: []. + correct?: false + SCORES: {:decoder=>0.9640539036904245, :psb=>0.5124797359336637}, #1 + +================= + EXAMPLE: 16 + REFERENCE: how big is north dakota + GOLD MRL: answer(size(stateid('north dakota'))) +GOLD OUTPUT: [70700.0] + +<<< KBEST +0 ||| how big is north dakota ||| ||| {:decoder=>-2.28488, :psb=>1.0} +1 ||| how great is north dakota ||| ||| {:decoder=>-2.53473, :psb=>0.5318295896944989} +2 ||| how large is north dakota ||| ||| {:decoder=>-2.61467, :psb=>0.5318295896944989} +3 ||| how much is north dakota ||| ||| {:decoder=>-2.67928, :psb=>0.5318295896944989} +4 ||| what is north dakota ||| ||| {:decoder=>-2.7223, :psb=>0.5124797359336637} +5 ||| how great north dakota ||| ||| {:decoder=>-2.72936, :psb=>0.3894003915357024} +6 ||| just how big is north dakota ||| ||| {:decoder=>-2.77501, :psb=>0.8034284189446518} +7 ||| as big is north dakota ||| ||| {:decoder=>-2.92364, :psb=>0.7521206186172787} +8 ||| how big north dakota is ||| ||| {:decoder=>-2.92694, :psb=>0.47287080450158786} +9 ||| is how large north dakota ||| ||| {:decoder=>-2.94444, :psb=>0.40410310093532464} +10 ||| is how big north dakota ||| ||| {:decoder=>-2.95559, :psb=>0.47287080450158786} +11 ||| is as large north dakota ||| ||| {:decoder=>-2.96007, :psb=>0.3760603093086393} +12 ||| is as big north dakota ||| ||| {:decoder=>-3.02045, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how big is north dakota + mrl: answer(size(stateid('north dakota'))) + output: [70700.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is north dakota + mrl: answer(size(stateid('north dakota'))) + output: [70700.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is north dakota + mrl: answer(loc_1(stateid('north dakota'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.660331443642345, :psb=>0.5318295896944989}, #1 + +================= + EXAMPLE: 17 + REFERENCE: how big is texas + GOLD MRL: answer(size(stateid('texas'))) +GOLD OUTPUT: [266807.0] + +<<< KBEST +0 ||| how big is texas ||| ||| {:decoder=>-2.12049, :psb=>1.0} +1 ||| how great is texas ||| ||| {:decoder=>-2.3974, :psb=>0.49999999999999994} +2 ||| how large is texas ||| ||| {:decoder=>-2.46891, :psb=>0.49999999999999994} +3 ||| how much is texas ||| ||| {:decoder=>-2.51827, :psb=>0.49999999999999994} +4 ||| what is texas ||| ||| {:decoder=>-2.55362, :psb=>0.4919625503668659} +5 ||| how great texas ||| ||| {:decoder=>-2.55674, :psb=>0.41368954504257255} +6 ||| just how big is texas ||| ||| {:decoder=>-2.63412, :psb=>0.7521206186172787} +7 ||| is how large texas ||| ||| {:decoder=>-2.76055, :psb=>0.42044820762685725} +8 ||| how big texas is ||| ||| {:decoder=>-2.77502, :psb=>0.537284965911771} +9 ||| like big is texas ||| ||| {:decoder=>-2.77661, :psb=>0.6580370064762462} +10 ||| as big is texas ||| ||| {:decoder=>-2.78607, :psb=>0.6580370064762462} +11 ||| is how big texas ||| ||| {:decoder=>-2.78856, :psb=>0.537284965911771} +12 ||| is as large texas ||| ||| {:decoder=>-2.78943, :psb=>0.37991784282579627} +13 ||| is as big texas ||| ||| {:decoder=>-2.85645, :psb=>0.42044820762685725} +14 ||| how large texas is ||| ||| {:decoder=>-2.85938, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is texas + mrl: answer(size(stateid('texas'))) + output: [266807.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is texas + mrl: answer(size(stateid('texas'))) + output: [266807.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is texas + mrl: answer(loc_1(stateid('texas'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.6252351500223307, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 18 + REFERENCE: how big is the city of new york + GOLD MRL: answer(size(city(cityid('new york', _)))) +GOLD OUTPUT: [7071639] + +<<< KBEST +0 ||| how big is the city of new york ||| ||| {:decoder=>-2.71666, :psb=>1.0} +1 ||| how big is the new york city ||| ||| {:decoder=>-2.8007, :psb=>0.5329462628216856} +2 ||| how much is the new york city ||| ||| {:decoder=>-2.841, :psb=>0.2883677731713749} +3 ||| how great is the city of new york ||| ||| {:decoder=>-3.03119, :psb=>0.7476743906106103} +4 ||| how much is the city of new york ||| ||| {:decoder=>-3.0723, :psb=>0.7476743906106103} +5 ||| how large is the new york city ||| ||| {:decoder=>-3.07251, :psb=>0.2883677731713749} +6 ||| how big is the york city ||| ||| {:decoder=>-3.09541, :psb=>0.4791733671582712} +7 ||| how large is the city of new york ||| ||| {:decoder=>-3.1168, :psb=>0.7476743906106103} +8 ||| how great is the new york city ||| ||| {:decoder=>-3.15951, :psb=>0.2883677731713749} +>>> + + [TOP1] + nrl: how big is the city of new york + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is the city of new york + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how much is the new york city + mrl: answer(loc_1(cityid('new york',_))) + output: [countryid(usa),stateid('new york')] + correct?: false + SCORES: {:decoder=>0.7192277294795074, :psb=>0.2883677731713749}, #2 + +================= + EXAMPLE: 19 + REFERENCE: how high are the highest points of all the states + GOLD MRL: answer(elevation_1(highest(place(loc_2(state(all)))))) +GOLD OUTPUT: [6194] + +<<< KBEST +0 ||| what are the highest surveys all states ||| ||| {:decoder=>-4.43622, :psb=>0.257703362342899} +1 ||| how high are the highest surveys all states ||| ||| {:decoder=>-4.47177, :psb=>0.5062667121584363} +2 ||| how much are the highest surveys all states ||| ||| {:decoder=>-4.48915, :psb=>0.27534765745159184} +3 ||| what are the highest surveys of all states ||| ||| {:decoder=>-4.49133, :psb=>0.29588031349552907} +4 ||| how high is the highest surveys all states ||| ||| {:decoder=>-4.49862, :psb=>0.2315388580995513} +5 ||| how much are the highest surveys of all states ||| ||| {:decoder=>-4.50295, :psb=>0.3084052692656029} +6 ||| how high are the highest surveys of all states ||| ||| {:decoder=>-4.57617, :psb=>0.5501366107724776} +7 ||| what are the supreme surveys of all states ||| ||| {:decoder=>-4.58769, :psb=>0.22278563765922124} +8 ||| how high is the highest surveys of all states ||| ||| {:decoder=>-4.60303, :psb=>0.25933688537080213} +9 ||| how much are the supreme surveys of all states ||| ||| {:decoder=>-4.62665, :psb=>0.2334165360649142} +10 ||| how high are the supreme surveys of all states ||| ||| {:decoder=>-4.69201, :psb=>0.429170247427184} +11 ||| as highly are the highest surveys all states ||| ||| {:decoder=>-4.73159, :psb=>0.26493826542476406} +12 ||| what are the highest uplift all states ||| ||| {:decoder=>-4.92044, :psb=>0.257703362342899} +13 ||| what are the highest eminence all states ||| ||| {:decoder=>-4.92179, :psb=>0.257703362342899} +14 ||| what are the highest hills all states ||| ||| {:decoder=>-4.93793, :psb=>0.257703362342899} +15 ||| how high are the highest uplift all states ||| ||| {:decoder=>-4.94146, :psb=>0.5062667121584363} +16 ||| how high are the highest eminence all states ||| ||| {:decoder=>-4.94281, :psb=>0.5062667121584363} +17 ||| how high are the highest hills all states ||| ||| {:decoder=>-4.95895, :psb=>0.5062667121584363} +18 ||| how high is the highest uplift all states ||| ||| {:decoder=>-4.96832, :psb=>0.2315388580995513} +19 ||| how high is the highest eminence all states ||| ||| {:decoder=>-4.96967, :psb=>0.2315388580995513} +20 ||| how high is the highest hills all states ||| ||| {:decoder=>-4.9858, :psb=>0.2315388580995513} +>>> + + [TOP1] + nrl: what are the highest surveys all states + mrl: answer(highest(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.257703362342899} + + [HOPE] + nrl: how high are the highest surveys all states + mrl: answer(elevation_1(highest(state(all)))) + output: [] + correct?: false + SCORES: {:decoder=>0.9353142399650631, :psb=>0.5062667121584363}, #1 + + [FEAR] + nrl: what are the highest surveys all states + mrl: answer(highest(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.257703362342899}, #0 + +================= + EXAMPLE: 20 + REFERENCE: how high is guadalupe peak + GOLD MRL: answer(elevation_1(placeid('guadalupe peak'))) +GOLD OUTPUT: [2667] + +<<< KBEST +0 ||| how high is guadalupe peak ||| ||| {:decoder=>-1.73189, :psb=>1.0} +1 ||| how high is located guadalupe peak ||| ||| {:decoder=>-1.92751, :psb=>0.48549177170732344} +2 ||| how much is guadalupe peak ||| ||| {:decoder=>-1.95238, :psb=>0.5318295896944989} +3 ||| how high is situated guadalupe peak ||| ||| {:decoder=>-1.95515, :psb=>0.48549177170732344} +4 ||| as high is guadalupe peak ||| ||| {:decoder=>-1.95689, :psb=>0.7521206186172787} +5 ||| how highly is guadalupe peak ||| ||| {:decoder=>-1.96778, :psb=>0.5318295896944989} +6 ||| like high is guadalupe peak ||| ||| {:decoder=>-1.98541, :psb=>0.7521206186172787} +7 ||| as highly is guadalupe peak ||| ||| {:decoder=>-2.00361, :psb=>0.4949232003839765} +8 ||| such as high is guadalupe peak ||| ||| {:decoder=>-2.00426, :psb=>0.6042750794713536} +9 ||| like highly is guadalupe peak ||| ||| {:decoder=>-2.03269, :psb=>0.4949232003839765} +10 ||| such as highly is guadalupe peak ||| ||| {:decoder=>-2.05014, :psb=>0.39763536438352537} +11 ||| how highly is located guadalupe peak ||| ||| {:decoder=>-2.16904, :psb=>0.32466791547509893} +12 ||| as high is located guadalupe peak ||| ||| {:decoder=>-2.17339, :psb=>0.35930411196308426} +13 ||| how highly is situated guadalupe peak ||| ||| {:decoder=>-2.19668, :psb=>0.32466791547509893} +14 ||| like high is located guadalupe peak ||| ||| {:decoder=>-2.20191, :psb=>0.35930411196308426} +15 ||| as highly is located guadalupe peak ||| ||| {:decoder=>-2.20486, :psb=>0.3021375397356768} +16 ||| as high is situated guadalupe peak ||| ||| {:decoder=>-2.22041, :psb=>0.35930411196308426} +17 ||| such as high is located guadalupe peak ||| ||| {:decoder=>-2.22077, :psb=>0.3005840818981024} +18 ||| how high is situated of guadalupe peak ||| ||| {:decoder=>-2.22753, :psb=>0.40614925799324625} +19 ||| as highly is situated guadalupe peak ||| ||| {:decoder=>-2.23251, :psb=>0.3021375397356768} +20 ||| like highly is located guadalupe peak ||| ||| {:decoder=>-2.23394, :psb=>0.3021375397356768} +21 ||| like high is situated guadalupe peak ||| ||| {:decoder=>-2.24893, :psb=>0.35930411196308426} +22 ||| such as highly is located guadalupe peak ||| ||| {:decoder=>-2.2514, :psb=>0.2527600769504433} +23 ||| like highly is situated guadalupe peak ||| ||| {:decoder=>-2.26159, :psb=>0.3021375397356768} +24 ||| such as high is situated guadalupe peak ||| ||| {:decoder=>-2.26779, :psb=>0.3005840818981024} +25 ||| such as highly is situated guadalupe peak ||| ||| {:decoder=>-2.27904, :psb=>0.2527600769504433} +26 ||| how highly is situated of guadalupe peak ||| ||| {:decoder=>-2.46906, :psb=>0.271608378656351} +27 ||| as high is situated of guadalupe peak ||| ||| {:decoder=>-2.50104, :psb=>0.3005840818981024} +28 ||| as highly is situated of guadalupe peak ||| ||| {:decoder=>-2.50489, :psb=>0.2527600769504433} +29 ||| like high is situated of guadalupe peak ||| ||| {:decoder=>-2.52956, :psb=>0.3005840818981024} +30 ||| like highly is situated of guadalupe peak ||| ||| {:decoder=>-2.53397, :psb=>0.2527600769504433} +31 ||| such as high is situated of guadalupe peak ||| ||| {:decoder=>-2.54842, :psb=>0.25848657697858524} +32 ||| such as highly is situated of guadalupe peak ||| ||| {:decoder=>-2.55142, :psb=>0.21736043597249566} +>>> + + [TOP1] + nrl: how high is guadalupe peak + mrl: answer(elevation_1(placeid('guadalupe peak'))) + output: [2667] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is guadalupe peak + mrl: answer(elevation_1(placeid('guadalupe peak'))) + output: [2667] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is located guadalupe peak + mrl: answer(elevation_1(loc_1(placeid('guadalupe peak')))) + output: [] + correct?: false + SCORES: {:decoder=>0.7613022098031796, :psb=>0.48549177170732344}, #1 + +================= + EXAMPLE: 21 + REFERENCE: how high is the highest point in montana + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) +GOLD OUTPUT: [3901] + +<<< KBEST +0 ||| how high is the highest point in montana ||| ||| {:decoder=>-2.6542, :psb=>1.0} +1 ||| how much is the highest point in montana ||| ||| {:decoder=>-2.84267, :psb=>0.7476743906106103} +2 ||| as high is the highest point in montana ||| ||| {:decoder=>-2.8651, :psb=>0.8599476570625982} +3 ||| how high is the highest point montana ||| ||| {:decoder=>-2.87224, :psb=>0.7536998328984837} +4 ||| like high is the highest point in montana ||| ||| {:decoder=>-2.89628, :psb=>0.8599476570625982} +5 ||| as highly is the highest point in montana ||| ||| {:decoder=>-2.91773, :psb=>0.719408902854813} +6 ||| like highly is the highest point in montana ||| ||| {:decoder=>-2.95364, :psb=>0.719408902854813} +7 ||| what is the highest point in montana ||| ||| {:decoder=>-2.97724, :psb=>0.7252065560578255} +8 ||| as are highly is the highest point in montana ||| ||| {:decoder=>-3.0663, :psb=>0.631196907822589} +9 ||| montana how high is the highest point in ||| ||| {:decoder=>-3.16872, :psb=>0.8891397050194614} +10 ||| how high is the highest point in in montana ||| ||| {:decoder=>-3.76973, :psb=>0.8065978233496266} +11 ||| how much is the highest point in in montana ||| ||| {:decoder=>-3.95819, :psb=>0.5773502691896258} +12 ||| as high is the highest point in in montana ||| ||| {:decoder=>-4.01866, :psb=>0.6817705815391745} +13 ||| what is the highest point in in montana ||| ||| {:decoder=>-4.08474, :psb=>0.633160218956179} +14 ||| as highly is the highest point in in montana ||| ||| {:decoder=>-4.18169, :psb=>0.5555238068023581} +>>> + + [TOP1] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how much is the highest point in montana + mrl: answer(highest(place(loc_2(stateid('montana'))))) + output: [placeid('granite peak')] + correct?: false + SCORES: {:decoder=>0.8766145768548402, :psb=>0.7476743906106103}, #1 + +================= + EXAMPLE: 22 + REFERENCE: how high is the highest point in the largest state + GOLD MRL: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) +GOLD OUTPUT: [6194] + +<<< KBEST +0 ||| how high lies the highest point of the largest state ||| ||| {:decoder=>-3.80279, :psb=>0.3760603093086394} +1 ||| as high lies the highest point of the largest state ||| ||| {:decoder=>-3.88146, :psb=>0.347507514861063} +2 ||| how high lies the highest point of the biggest state ||| ||| {:decoder=>-3.88991, :psb=>0.29697089145035693} +3 ||| as highly lies the highest point of the largest state ||| ||| {:decoder=>-3.89829, :psb=>0.33437015248821106} +4 ||| as high is the highest point of the largest state ||| ||| {:decoder=>-3.96347, :psb=>0.5844356470407898} +5 ||| how high lies the highest point the largest state ||| ||| {:decoder=>-3.97275, :psb=>0.3777177718431893} +6 ||| as high is situated the highest point of the largest state ||| ||| {:decoder=>-4.01081, :psb=>0.33910457813956707} +7 ||| how high lies of the highest point the biggest state ||| ||| {:decoder=>-4.01371, :psb=>0.29697089145035693} +8 ||| as high located the highest point of the largest state ||| ||| {:decoder=>-4.02343, :psb=>0.347507514861063} +9 ||| how high lies the highest point the biggest state ||| ||| {:decoder=>-4.03546, :psb=>0.2982797722714565} +10 ||| how much lies the highest point of the largest state ||| ||| {:decoder=>-4.04784, :psb=>0.347507514861063} +11 ||| how high lies the highest point of the greatest state ||| ||| {:decoder=>-4.04875, :psb=>0.29697089145035693} +12 ||| as high is located the highest point of the largest state ||| ||| {:decoder=>-4.05371, :psb=>0.33910457813956707} +13 ||| as high is the highest point of the biggest state ||| ||| {:decoder=>-4.05397, :psb=>0.49144984054308527} +14 ||| as high is the highest point of largest state ||| ||| {:decoder=>-4.06535, :psb=>0.5341735956899846} +15 ||| as highly is situated the highest point of the largest state ||| ||| {:decoder=>-4.06745, :psb=>0.31335768840889733} +16 ||| how high is situated the highest point of the largest state ||| ||| {:decoder=>-4.07762, :psb=>0.3900256463674887} +17 ||| how high is the highest point of the largest state ||| ||| {:decoder=>-4.07829, :psb=>0.6999271023161167} +18 ||| as high is located the highest point of the biggest state ||| ||| {:decoder=>-4.07942, :psb=>0.2677873372229633} +19 ||| as highly is the highest point of the biggest state ||| ||| {:decoder=>-4.08176, :psb=>0.3760603093086393} +20 ||| how high lies of the highest point of the largest state ||| ||| {:decoder=>-4.08893, :psb=>0.33910457813956707} +21 ||| how high is the highest point of the biggest state ||| ||| {:decoder=>-4.08908, :psb=>0.6042750794713536} +22 ||| how high lies the the highest point of the largest state ||| ||| {:decoder=>-4.09269, :psb=>0.33910457813956707} +23 ||| how high located the highest point of the largest state ||| ||| {:decoder=>-4.14486, :psb=>0.3760603093086394} +24 ||| how much lies the highest point of the biggest state ||| ||| {:decoder=>-4.15496, :psb=>0.2659147948472494} +25 ||| how high lies the highest point of biggest state ||| ||| {:decoder=>-4.15815, :psb=>0.2982797722714565} +26 ||| how high lies the highest point the greatest state ||| ||| {:decoder=>-4.18232, :psb=>0.2982797722714565} +27 ||| how high , located the highest point of the largest state ||| ||| {:decoder=>-4.19526, :psb=>0.33910457813956707} +28 ||| how high is located the highest point of the largest state ||| ||| {:decoder=>-4.19731, :psb=>0.3900256463674887} +29 ||| how high is of the highest point of the largest state ||| ||| {:decoder=>-4.23343, :psb=>0.3900256463674887} +30 ||| how much lies the highest point the biggest state ||| ||| {:decoder=>-4.30052, :psb=>0.26708679784499234} +31 ||| how much lies the the highest point of the largest state ||| ||| {:decoder=>-4.34336, :psb=>0.31335768840889733} +32 ||| as highly lies the the highest point of the largest state ||| ||| {:decoder=>-4.38304, :psb=>0.30151134457776363} +>>> + + [TOP1] + nrl: how high lies the highest point of the largest state + mrl: answer(elevation_1(traverse_1(longest(highest(place(loc_2(largest(state(all))))))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3760603093086394} + + [HOPE] + nrl: how high lies the highest point of the largest state + mrl: answer(elevation_1(traverse_1(longest(highest(place(loc_2(largest(state(all))))))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3760603093086394}, #0 + + [FEAR] + nrl: how high lies the highest point of the largest state + mrl: answer(elevation_1(traverse_1(longest(highest(place(loc_2(largest(state(all))))))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3760603093086394}, #0 + +================= + EXAMPLE: 23 + REFERENCE: how high is the highest point of delaware + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) +GOLD OUTPUT: [135] + +<<< KBEST +0 ||| how high is the highest point of delaware ||| ||| {:decoder=>-3.12849, :psb=>1.0} +1 ||| how high is located the highest point of delaware ||| ||| {:decoder=>-3.18423, :psb=>0.6559965570884768} +2 ||| like high is the highest point of delaware ||| ||| {:decoder=>-3.22366, :psb=>0.8599476570625982} +3 ||| how high is situated the highest point of delaware ||| ||| {:decoder=>-3.23867, :psb=>0.6559965570884768} +4 ||| as high is the highest point of delaware ||| ||| {:decoder=>-3.24334, :psb=>0.8599476570625982} +5 ||| like high is located the highest point of delaware ||| ||| {:decoder=>-3.3139, :psb=>0.5773502691896258} +6 ||| like high is situated the highest point of delaware ||| ||| {:decoder=>-3.32506, :psb=>0.5773502691896258} +7 ||| as high is located the highest point of delaware ||| ||| {:decoder=>-3.33358, :psb=>0.5773502691896258} +8 ||| how high lies the highest point of delaware ||| ||| {:decoder=>-3.33709, :psb=>0.6580370064762462} +9 ||| as high is situated the highest point of delaware ||| ||| {:decoder=>-3.34474, :psb=>0.5773502691896258} +10 ||| how high situated the highest point of delaware ||| ||| {:decoder=>-3.35049, :psb=>0.6580370064762462} +11 ||| how high located the highest point of delaware ||| ||| {:decoder=>-3.35291, :psb=>0.6580370064762462} +12 ||| how high is of the highest point of delaware ||| ||| {:decoder=>-3.3759, :psb=>0.6559965570884768} +13 ||| like high lies the highest point of delaware ||| ||| {:decoder=>-3.38886, :psb=>0.6049483675122199} +14 ||| like high situated the highest point of delaware ||| ||| {:decoder=>-3.40226, :psb=>0.6049483675122199} +15 ||| like high located the highest point of delaware ||| ||| {:decoder=>-3.40468, :psb=>0.6049483675122199} +16 ||| as high lies the highest point of delaware ||| ||| {:decoder=>-3.40854, :psb=>0.6049483675122199} +17 ||| as high situated the highest point of delaware ||| ||| {:decoder=>-3.42194, :psb=>0.6049483675122199} +18 ||| as high located the highest point of delaware ||| ||| {:decoder=>-3.42436, :psb=>0.6049483675122199} +19 ||| how much is located the highest point of delaware ||| ||| {:decoder=>-3.45227, :psb=>0.5516251532744788} +20 ||| how high is located the the highest point of delaware ||| ||| {:decoder=>-3.81267, :psb=>0.5844356470407898} +21 ||| how high is the highest point of of delaware ||| ||| {:decoder=>-3.83505, :psb=>0.8065978233496266} +22 ||| how high lays the highest point of delaware ||| ||| {:decoder=>-3.87202, :psb=>0.6580370064762462} +23 ||| as high is the highest point of of delaware ||| ||| {:decoder=>-3.91688, :psb=>0.6817705815391745} +24 ||| like high lays the highest point of delaware ||| ||| {:decoder=>-3.92379, :psb=>0.6049483675122199} +25 ||| as high lays the highest point of delaware ||| ||| {:decoder=>-3.94347, :psb=>0.6049483675122199} +26 ||| how much is located the the highest point of delaware ||| ||| {:decoder=>-4.06633, :psb=>0.49144984054308527} +27 ||| how high is located the highest point of of delaware ||| ||| {:decoder=>-4.16884, :psb=>0.49944351611061033} +28 ||| how high is situated the highest point of of delaware ||| ||| {:decoder=>-4.18551, :psb=>0.49944351611061033} +29 ||| how high lies the highest point of of delaware ||| ||| {:decoder=>-4.2445, :psb=>0.48549177170732344} +30 ||| how high is of the highest point of of delaware ||| ||| {:decoder=>-4.27029, :psb=>0.49944351611061033} +31 ||| how much is located the highest point of of delaware ||| ||| {:decoder=>-4.43688, :psb=>0.41325840918969} +>>> + + [TOP1] + nrl: how high is the highest point of delaware + mrl: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) + output: [135] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point of delaware + mrl: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) + output: [135] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is located the highest point of delaware + mrl: answer(elevation_1(loc_1(highest(place(loc_2(stateid('delaware'))))))) + output: [] + correct?: false + SCORES: {:decoder=>0.957398023525096, :psb=>0.6559965570884768}, #1 + +================= + EXAMPLE: 24 + REFERENCE: how high is the highest point of florida + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('florida')))))) +GOLD OUTPUT: [105] + +<<< KBEST +0 ||| how high is the highest point florida ||| ||| {:decoder=>-4.55496, :psb=>0.7536998328984837} +1 ||| how high is the highest point floridas ||| ||| {:decoder=>-4.6746, :psb=>0.7252065560578255} +2 ||| as high is the highest point florida ||| ||| {:decoder=>-4.78174, :psb=>0.6098235933075173} +3 ||| florida how high is the highest point ||| ||| {:decoder=>-4.81695, :psb=>0.7536998328984837} +4 ||| how high the highest point is florida ||| ||| {:decoder=>-4.82526, :psb=>0.3829795673743804} +5 ||| like high is the highest point florida ||| ||| {:decoder=>-4.85749, :psb=>0.6098235933075173} +6 ||| how high is situated the highest point florida ||| ||| {:decoder=>-4.86497, :psb=>0.4445698525097307} +7 ||| how high is located the highest point florida ||| ||| {:decoder=>-4.90203, :psb=>0.4445698525097307} +8 ||| as high is the highest point floridas ||| ||| {:decoder=>-4.92138, :psb=>0.5826515567418346} +9 ||| how much is the highest point florida ||| ||| {:decoder=>-4.93144, :psb=>0.48497485346901076} +10 ||| florida as high is the highest point ||| ||| {:decoder=>-4.94433, :psb=>0.6098235933075173} +11 ||| florida like high is the highest point ||| ||| {:decoder=>-4.97678, :psb=>0.6098235933075173} +12 ||| like high is the highest point floridas ||| ||| {:decoder=>-4.97713, :psb=>0.5826515567418346} +13 ||| how high is of the highest point florida ||| ||| {:decoder=>-5.0234, :psb=>0.45966135761245924} +14 ||| how high florida is the highest point ||| ||| {:decoder=>-5.04683, :psb=>0.5329462628216856} +15 ||| how high lies the highest point florida ||| ||| {:decoder=>-5.04749, :psb=>0.3685012003094065} +16 ||| as high is situated the highest point florida ||| ||| {:decoder=>-5.05718, :psb=>0.3655552228545123} +17 ||| how high situated the highest point florida ||| ||| {:decoder=>-5.06152, :psb=>0.3685012003094065} +18 ||| as high is located the highest point florida ||| ||| {:decoder=>-5.11423, :psb=>0.3655552228545123} +19 ||| how high located the highest point florida ||| ||| {:decoder=>-5.11462, :psb=>0.3685012003094065} +20 ||| florida how high is situated the highest point ||| ||| {:decoder=>-5.1215, :psb=>0.4445698525097307} +21 ||| as high lies the highest point florida ||| ||| {:decoder=>-5.12349, :psb=>0.3276490485424231} +22 ||| like high is situated the highest point florida ||| ||| {:decoder=>-5.13293, :psb=>0.3655552228545123} +23 ||| as high situated the highest point florida ||| ||| {:decoder=>-5.13752, :psb=>0.3276490485424231} +24 ||| florida how high is located the highest point ||| ||| {:decoder=>-5.15856, :psb=>0.4445698525097307} +25 ||| like high is located the highest point florida ||| ||| {:decoder=>-5.16999, :psb=>0.3655552228545123} +26 ||| like high lies the highest point florida ||| ||| {:decoder=>-5.17924, :psb=>0.3276490485424231} +27 ||| as high located the highest point florida ||| ||| {:decoder=>-5.19062, :psb=>0.3276490485424231} +28 ||| like high located the highest point florida ||| ||| {:decoder=>-5.24637, :psb=>0.3276490485424231} +29 ||| florida how high is of the highest point ||| ||| {:decoder=>-5.27993, :psb=>0.45966135761245924} +>>> + + [TOP1] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837} + + [HOPE] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837}, #0 + + [FEAR] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837}, #0 + +================= + EXAMPLE: 25 + REFERENCE: how high is the highest point of louisiana + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) +GOLD OUTPUT: [163] + +<<< KBEST +0 ||| how high is the highest point of louisiana ||| ||| {:decoder=>-4.45688, :psb=>1.0} +1 ||| as high is the highest point of louisiana ||| ||| {:decoder=>-4.70366, :psb=>0.8599476570625982} +2 ||| like high is the highest point of louisiana ||| ||| {:decoder=>-4.75942, :psb=>0.8599476570625982} +3 ||| how high is located the highest point of louisiana ||| ||| {:decoder=>-4.78347, :psb=>0.6559965570884768} +4 ||| how high is situated the highest point of louisiana ||| ||| {:decoder=>-4.79725, :psb=>0.6559965570884768} +5 ||| how much is the highest point of louisiana ||| ||| {:decoder=>-4.83336, :psb=>0.7476743906106103} +6 ||| what is the highest point of louisiana ||| ||| {:decoder=>-4.88511, :psb=>0.7252065560578255} +7 ||| how high located the highest point of louisiana ||| ||| {:decoder=>-4.89227, :psb=>0.6580370064762462} +8 ||| how high is of the highest point of louisiana ||| ||| {:decoder=>-4.95567, :psb=>0.6559965570884768} +9 ||| how high lies the highest point of louisiana ||| ||| {:decoder=>-4.99977, :psb=>0.6580370064762462} +10 ||| as high is situated the highest point of louisiana ||| ||| {:decoder=>-5.00945, :psb=>0.5773502691896258} +11 ||| how high situated the highest point of louisiana ||| ||| {:decoder=>-5.0138, :psb=>0.6580370064762462} +12 ||| as high is located the highest point of louisiana ||| ||| {:decoder=>-5.04651, :psb=>0.5773502691896258} +13 ||| as high lies the highest point of louisiana ||| ||| {:decoder=>-5.05576, :psb=>0.6049483675122199} +14 ||| like high is situated the highest point of louisiana ||| ||| {:decoder=>-5.06521, :psb=>0.5773502691896258} +15 ||| as high situated the highest point of louisiana ||| ||| {:decoder=>-5.06979, :psb=>0.6049483675122199} +16 ||| like high is located the highest point of louisiana ||| ||| {:decoder=>-5.10226, :psb=>0.5773502691896258} +17 ||| like high lies the highest point of louisiana ||| ||| {:decoder=>-5.11152, :psb=>0.6049483675122199} +18 ||| as high located the highest point of louisiana ||| ||| {:decoder=>-5.12289, :psb=>0.6049483675122199} +19 ||| like high located the highest point of louisiana ||| ||| {:decoder=>-5.17864, :psb=>0.6049483675122199} +20 ||| how high is the highest point of of louisiana ||| ||| {:decoder=>-5.40085, :psb=>0.8065978233496266} +21 ||| as high is the highest point of of louisiana ||| ||| {:decoder=>-5.62548, :psb=>0.6817705815391745} +22 ||| how much is the highest point of of louisiana ||| ||| {:decoder=>-5.91813, :psb=>0.5773502691896258} +23 ||| how high is located the highest point of of louisiana ||| ||| {:decoder=>-6.068, :psb=>0.49944351611061033} +>>> + + [TOP1] + nrl: how high is the highest point of louisiana + mrl: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) + output: [163] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point of louisiana + mrl: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) + output: [163] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is located the highest point of louisiana + mrl: answer(elevation_1(loc_1(highest(place(loc_2(stateid('louisiana'))))))) + output: [] + correct?: false + SCORES: {:decoder=>0.7972900839167781, :psb=>0.6559965570884768}, #3 + +================= + EXAMPLE: 26 + REFERENCE: how large is the largest city in alaska + GOLD MRL: answer(size(largest(city(loc_2(stateid('alaska')))))) +GOLD OUTPUT: [174431] + +<<< KBEST +0 ||| how big is the largest city in alaska ||| ||| {:decoder=>-4.66569, :psb=>0.7476743906106103} +1 ||| how big is the biggest city in alaska ||| ||| {:decoder=>-4.83987, :psb=>0.3655552228545123} +2 ||| as big is the largest city in alaska ||| ||| {:decoder=>-4.92986, :psb=>0.719408902854813} +3 ||| how much is the largest city in alaska ||| ||| {:decoder=>-4.96272, :psb=>0.7476743906106103} +4 ||| how great is the largest city in alaska ||| ||| {:decoder=>-4.97905, :psb=>0.7476743906106103} +5 ||| how big is the largest town in alaska ||| ||| {:decoder=>-5.02657, :psb=>0.3655552228545123} +6 ||| how large is the largest city in alaska ||| ||| {:decoder=>-5.03371, :psb=>1.0} +7 ||| as great is the largest city in alaska ||| ||| {:decoder=>-5.06928, :psb=>0.719408902854813} +8 ||| as big is the biggest city in alaska ||| ||| {:decoder=>-5.07609, :psb=>0.3492671028290048} +9 ||| how great is the biggest city in alaska ||| ||| {:decoder=>-5.17803, :psb=>0.3655552228545123} +10 ||| how large is the biggest city in alaska ||| ||| {:decoder=>-5.21079, :psb=>0.5946035575013605} +11 ||| how big is the largest city of in alaska ||| ||| {:decoder=>-5.50709, :psb=>0.4638596139522901} +12 ||| how large is the largest city of in alaska ||| ||| {:decoder=>-5.70876, :psb=>0.7049141756270427} +13 ||| how great is the largest city of in alaska ||| ||| {:decoder=>-5.77997, :psb=>0.4638596139522901} +>>> + + [TOP1] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103} + + [HOPE] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103}, #0 + + [FEAR] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103}, #0 + +================= + EXAMPLE: 27 + REFERENCE: how long is rio grande + GOLD MRL: answer(len(riverid('rio grande'))) +GOLD OUTPUT: [3033] + +<<< KBEST +0 ||| how long rio grande ||| ||| {:decoder=>-3.44215, :psb=>0.4630777161991027} +1 ||| how long has rio grande ||| ||| {:decoder=>-3.63115, :psb=>0.4472135954999579} +2 ||| how long is rio grande ||| ||| {:decoder=>-3.68587, :psb=>1.0} +3 ||| how long will rio grande ||| ||| {:decoder=>-3.76379, :psb=>0.4472135954999579} +4 ||| like long is rio grande ||| ||| {:decoder=>-3.83572, :psb=>0.7521206186172787} +5 ||| like lang is rio grande ||| ||| {:decoder=>-3.86803, :psb=>0.4949232003839765} +6 ||| how long will it rio grande ||| ||| {:decoder=>-3.95253, :psb=>0.35930411196308426} +7 ||| how long has of rio grande ||| ||| {:decoder=>-4.06843, :psb=>0.35930411196308426} +8 ||| how long rÃo grande ||| ||| {:decoder=>-4.14496, :psb=>0.3894003915357024} +9 ||| how long rio grande is ||| ||| {:decoder=>-4.1618, :psb=>0.47287080450158786} +10 ||| how long , rio grande ||| ||| {:decoder=>-4.21092, :psb=>0.4472135954999579} +11 ||| rio grande is how long ||| ||| {:decoder=>-4.23318, :psb=>0.47287080450158786} +12 ||| how long is of rio grande ||| ||| {:decoder=>-4.23611, :psb=>0.48549177170732344} +13 ||| like long has rio grande ||| ||| {:decoder=>-4.29615, :psb=>0.3760603093086393} +14 ||| is how long rio grande ||| ||| {:decoder=>-4.30336, :psb=>0.47287080450158786} +15 ||| like long rio grande ||| ||| {:decoder=>-4.33319, :psb=>0.3894003915357024} +16 ||| rio grande is as long ||| ||| {:decoder=>-4.36009, :psb=>0.40410310093532464} +17 ||| as long is rio grande ||| ||| {:decoder=>-4.38834, :psb=>0.7521206186172787} +18 ||| as long rio grande ||| ||| {:decoder=>-4.50078, :psb=>0.3894003915357024} +19 ||| as long rio grande is ||| ||| {:decoder=>-4.62146, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how long rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.4630777161991027} + + [HOPE] + nrl: how long is rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>0.7933367816774214, :psb=>1.0}, #2 + + [FEAR] + nrl: how long rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.4630777161991027}, #0 + +================= + EXAMPLE: 28 + REFERENCE: how long is the mississippi + GOLD MRL: answer(len(riverid('mississippi'))) +GOLD OUTPUT: [3778] + +<<< KBEST +0 ||| how long is the mississippi river ||| ||| {:decoder=>-3.85459, :psb=>0.8034284189446518} +1 ||| how long the mississippi river ||| ||| {:decoder=>-3.94595, :psb=>0.4472135954999579} +2 ||| how long will the mississippi river ||| ||| {:decoder=>-4.18693, :psb=>0.35930411196308426} +3 ||| how long has the mississippi river ||| ||| {:decoder=>-4.19349, :psb=>0.35930411196308426} +4 ||| how long is the mississippi river and ||| ||| {:decoder=>-4.4355, :psb=>0.672126440078521} +5 ||| is how long the mississippi river ||| ||| {:decoder=>-4.48862, :psb=>0.37991784282579627} +6 ||| how long is mississippi river ||| ||| {:decoder=>-4.54241, :psb=>0.5318295896944989} +7 ||| how long has mississippi river ||| ||| {:decoder=>-4.55425, :psb=>0.3760603093086393} +8 ||| how long the mississippi river is ||| ||| {:decoder=>-4.57413, :psb=>0.37991784282579627} +9 ||| like long is the mississippi river ||| ||| {:decoder=>-4.6366, :psb=>0.6042750794713536} +10 ||| as long is the mississippi river ||| ||| {:decoder=>-4.74238, :psb=>0.6042750794713536} +11 ||| like lang is the mississippi river ||| ||| {:decoder=>-4.75457, :psb=>0.39763536438352537} +12 ||| how long is der mississippi river ||| ||| {:decoder=>-5.17746, :psb=>0.4272870063962341} +>>> + + [TOP1] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 29 + REFERENCE: how long is the mississippi river + GOLD MRL: answer(len(river(riverid('mississippi')))) +GOLD OUTPUT: [3778] + +<<< KBEST +0 ||| how long the mississippi ||| ||| {:decoder=>-3.38154, :psb=>0.36064528799877893} +1 ||| how long is the mississippi ||| ||| {:decoder=>-3.53382, :psb=>0.8187307530779819} +2 ||| how long will the mississippi ||| ||| {:decoder=>-3.62252, :psb=>0.36614752383039256} +3 ||| how long has the mississippi ||| ||| {:decoder=>-3.62908, :psb=>0.36614752383039256} +4 ||| how long has mississippi ||| ||| {:decoder=>-3.78592, :psb=>0.30326532985631666} +5 ||| how long is mississippi ||| ||| {:decoder=>-3.79303, :psb=>0.4288819424803534} +6 ||| how long will mississippi ||| ||| {:decoder=>-3.92198, :psb=>0.30326532985631666} +7 ||| how long the mississippi is ||| ||| {:decoder=>-4.09379, :psb=>0.3871538698781762} +8 ||| is how long the mississippi ||| ||| {:decoder=>-4.09503, :psb=>0.3871538698781762} +9 ||| the mississippi is how long ||| ||| {:decoder=>-4.13368, :psb=>0.3871538698781762} +10 ||| how long , mississippi ||| ||| {:decoder=>-4.20676, :psb=>0.30326532985631666} +11 ||| as long is the mississippi ||| ||| {:decoder=>-4.25544, :psb=>0.6157842804860023} +12 ||| the mississippi is as long ||| ||| {:decoder=>-4.26501, :psb=>0.3308516361499261} +13 ||| is how long mississippi ||| ||| {:decoder=>-4.28547, :psb=>0.3258798048281462} +14 ||| how long is der mississippi ||| ||| {:decoder=>-4.40912, :psb=>0.43542524047973125} +15 ||| like long is the mississippi ||| ||| {:decoder=>-4.44212, :psb=>0.6157842804860023} +16 ||| as long the mississippi ||| ||| {:decoder=>-4.46871, :psb=>0.30326532985631666} +>>> + + [TOP1] + nrl: how long the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.36064528799877893} + + [HOPE] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>0.8599299097657221, :psb=>0.8187307530779819}, #1 + + [FEAR] + nrl: how long the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.36064528799877893}, #0 + +--- + iteration #1/3: 30 examples + type1 updates: 11 + type2 updates: 18 + top1 hits: 11 + top1 variant: 9 + top1 true variant: 9 + hope hits: 12 + hope variant: 8 + hope true variant: 8 + kbest size: 16.0 + 3.33% without translations (abs: 1) + 6.67% no good gold output (abs: 2) + + top1 with parse 96.67% abs=29.0 + top1 with output 96.67% abs=29.0 +top1 with correct output 66.67% adj=68.97 abs=20.0 + + hope with parse 96.67% abs=29.0 + hope with output 96.67% abs=29.0 +hope with correct output 66.67% adj=68.97 abs=20.0 + + fear with parse 96.67% abs=29.0 + fear with output 96.67% abs=29.0 +fear with correct output 40.0% adj=41.38 abs=12.0 + + + +================= + EXAMPLE: 0 + REFERENCE: give me the cities in virginia + GOLD MRL: answer(city(loc_2(stateid('virginia')))) +GOLD OUTPUT: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + +<<< KBEST +0 ||| give me the cities in virginia ||| ||| {:decoder=>-4.39202, :psb=>1.0} +1 ||| give me cities in virginia ||| ||| {:decoder=>-4.62797, :psb=>0.49473859088183875} +2 ||| give me the towns in virginia ||| ||| {:decoder=>-4.74129, :psb=>0.48549177170732344} +3 ||| give me the cities virginia ||| ||| {:decoder=>-4.7643, :psb=>0.6511126026643229} +4 ||| the cities in virginia give me ||| ||| {:decoder=>-4.7757, :psb=>0.7071067811865476} +5 ||| give me the cities of virginia ||| ||| {:decoder=>-4.83635, :psb=>0.6389431042462724} +6 ||| unto me the cities in virginia ||| ||| {:decoder=>-4.87582, :psb=>0.8034284189446518} +7 ||| give unto me the cities in virginia ||| ||| {:decoder=>-4.91304, :psb=>0.7034711503007025} +8 ||| me give the cities in virginia ||| ||| {:decoder=>-4.94826, :psb=>0.668740304976422} +9 ||| unto me the towns in virginia ||| ||| {:decoder=>-5.3093, :psb=>0.35930411196308426} +10 ||| give unto me the towns in virginia ||| ||| {:decoder=>-5.34653, :psb=>0.3178289704467185} +>>> + + [TOP1] + nrl: give me the cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: give me the cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: give me cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>0.7528051041895834, :psb=>0.49473859088183875}, #1 + +================= + EXAMPLE: 1 + REFERENCE: what are the high points of states surrounding mississippi + GOLD MRL: answer(high_point_1(state(next_to_2(stateid('mississippi'))))) +GOLD OUTPUT: [placeid('cheaha mountain'),placeid('clingmans dome'),placeid('driskill mountain'),placeid('magazine mountain')] +NO MT OUTPUT, skipping example + +================= + EXAMPLE: 2 + REFERENCE: name the rivers in arkansas + GOLD MRL: answer(river(loc_2(stateid('arkansas')))) +GOLD OUTPUT: [riverid(arkansas),riverid(mississippi),riverid(ouachita),riverid(red),riverid('st. francis'),riverid(white)] + +<<< KBEST +0 ||| call the rivers in arkansas ||| ||| {:decoder=>-4.11703, :psb=>0.7521206186172787} +1 ||| call rivers in the arkansas ||| ||| {:decoder=>-4.23043, :psb=>0.40410310093532464} +2 ||| the rivers in arkansas call ||| ||| {:decoder=>-4.44413, :psb=>0.7521206186172787} +3 ||| call the rivers into arkansas ||| ||| {:decoder=>-4.47248, :psb=>0.3760603093086393} +4 ||| the rivers call in arkansas ||| ||| {:decoder=>-4.49541, :psb=>0.4472135954999579} +5 ||| the call rivers in arkansas ||| ||| {:decoder=>-4.53319, :psb=>0.5318295896944989} +6 ||| rivers in the call arkansas ||| ||| {:decoder=>-4.56979, :psb=>0.40410310093532464} +7 ||| rivers in the arkansas call ||| ||| {:decoder=>-4.57475, :psb=>0.40410310093532464} +8 ||| call the rivers in the arkansas ||| ||| {:decoder=>-4.60689, :psb=>0.4272870063962341} +9 ||| call the rivers arkansas ||| ||| {:decoder=>-4.66407, :psb=>0.3894003915357024} +10 ||| rivers in call the arkansas ||| ||| {:decoder=>-4.68148, :psb=>0.40410310093532464} +11 ||| call the rivers , in arkansas ||| ||| {:decoder=>-4.70798, :psb=>0.35930411196308426} +12 ||| in arkansas call the rivers ||| ||| {:decoder=>-4.73262, :psb=>0.4472135954999579} +13 ||| call rivers in arkansas ||| ||| {:decoder=>-4.76755, :psb=>0.5124797359336637} +14 ||| name rivers in the arkansas ||| ||| {:decoder=>-4.86883, :psb=>0.4272870063962341} +15 ||| call rivers in the the arkansas ||| ||| {:decoder=>-4.90226, :psb=>0.32466791547509893} +16 ||| rivers in the the arkansas call ||| ||| {:decoder=>-5.23461, :psb=>0.32466791547509893} +>>> + + [TOP1] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787} + + [HOPE] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787}, #0 + + [FEAR] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787}, #0 + +================= + EXAMPLE: 3 + REFERENCE: can you tell me the capital of texas + GOLD MRL: answer(capital(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(austin,tx)] + +<<< KBEST +0 ||| can you tell me , what is the capital of texas ||| ||| {:decoder=>-5.1274, :psb=>0.5270027137504644} +1 ||| you can tell me what the capital of texas is ||| ||| {:decoder=>-5.20615, :psb=>0.4272870063962341} +2 ||| can you tell me what is the capital of texas ||| ||| {:decoder=>-5.21848, :psb=>0.5844356470407898} +3 ||| can you tell me , was the capital of texas is ||| ||| {:decoder=>-5.22012, :psb=>0.5270027137504644} +4 ||| can you tell me what the capital of texas is ||| ||| {:decoder=>-5.27686, :psb=>0.5844356470407898} +5 ||| you can tell me , what is the capital of texas ||| ||| {:decoder=>-5.29333, :psb=>0.3852971889399676} +6 ||| can you tell me , what is the capital of the texas ||| ||| {:decoder=>-5.31683, :psb=>0.39458812555917666} +7 ||| you can tell me what is the capital of texas ||| ||| {:decoder=>-5.34753, :psb=>0.4272870063962341} +8 ||| can you tell me , which is the capital of texas ||| ||| {:decoder=>-5.36883, :psb=>0.5270027137504644} +9 ||| can you tell me , what is the capital city of texas ||| ||| {:decoder=>-5.39836, :psb=>0.3672056269893592} +10 ||| can you tell me , what the capital of texas is ||| ||| {:decoder=>-5.40518, :psb=>0.5270027137504644} +11 ||| can you tell me what is the capital of the texas ||| ||| {:decoder=>-5.40791, :psb=>0.43333706364123536} +12 ||| you can tell me , what the capital of texas is ||| ||| {:decoder=>-5.4104, :psb=>0.3852971889399676} +13 ||| you can tell me , was the capital of texas is ||| ||| {:decoder=>-5.42651, :psb=>0.3852971889399676} +14 ||| you can tell me was the capital of texas is ||| ||| {:decoder=>-5.55128, :psb=>0.4272870063962341} +15 ||| you can tell me that the capital of texas is ||| ||| {:decoder=>-5.57293, :psb=>0.4272870063962341} +16 ||| you can tell me , what is the capital city of texas ||| ||| {:decoder=>-5.60475, :psb=>0.21200626759025185} +17 ||| can you tell me , was the capital city of texas is ||| ||| {:decoder=>-5.71531, :psb=>0.3672056269893592} +18 ||| you can tell me , was the capital city of texas is ||| ||| {:decoder=>-5.9217, :psb=>0.21200626759025185} +>>> + + [TOP1] + nrl: can you tell me , what is the capital of texas + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.5270027137504644} + + [HOPE] + nrl: can you tell me , what is the capital of texas + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.5270027137504644}, #0 + + [FEAR] + nrl: can you tell me , what is the capital of texas + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.5270027137504644}, #0 + +================= + EXAMPLE: 4 + REFERENCE: could you tell me what is the highest point in the state of oregon + GOLD MRL: answer(highest(place(loc_2(state(stateid('oregon')))))) +GOLD OUTPUT: [placeid('mount hood')] + +<<< KBEST +0 ||| you could tell me which is the highest point in the state of oregon ||| ||| {:decoder=>-7.5084, :psb=>0.6985342056580097} +1 ||| could you tell me which is the highest point in the state of oregon ||| ||| {:decoder=>-7.69601, :psb=>0.7992975447228272} +2 ||| you could tell me which is the highest point in oregon state ||| ||| {:decoder=>-7.74873, :psb=>0.4090790395780627} +3 ||| you could say which is the highest point in the state of oregon me ||| ||| {:decoder=>-7.79918, :psb=>0.6668954865619205} +4 ||| you could say me which is the highest point in the state of oregon ||| ||| {:decoder=>-7.83782, :psb=>0.6668954865619205} +5 ||| you could tell me what is the highest point in the state of oregon ||| ||| {:decoder=>-7.85909, :psb=>0.8817192535122739} +6 ||| you could tell me what the highest point in the state of oregon is ||| ||| {:decoder=>-7.88394, :psb=>0.684694991173804} +7 ||| you could tell me that the highest point in the state of oregon is ||| ||| {:decoder=>-7.90874, :psb=>0.633160218956179} +8 ||| you could tell me which the highest point in the state of oregon is ||| ||| {:decoder=>-7.91722, :psb=>0.633160218956179} +9 ||| you could say what me is the highest point in the state of oregon ||| ||| {:decoder=>-7.93162, :psb=>0.6803749333171202} +10 ||| could you tell me which is the highest point in oregon state ||| ||| {:decoder=>-7.94634, :psb=>0.5227564628601822} +11 ||| you could tell me which is the highest point in the oregon state ||| ||| {:decoder=>-7.95397, :psb=>0.48507053375249726} +12 ||| you could tell me that is the highest point in the state of oregon ||| ||| {:decoder=>-7.98744, :psb=>0.6985342056580097} +13 ||| could you tell me that the highest point in the state of oregon is ||| ||| {:decoder=>-8.01419, :psb=>0.7367565054628357} +14 ||| you could tell me what the highest point is in the state of oregon ||| ||| {:decoder=>-8.03135, :psb=>0.521873921269267} +15 ||| could you tell me which the highest point in the state of oregon is ||| ||| {:decoder=>-8.11483, :psb=>0.7367565054628357} +16 ||| could you tell me what the highest point in the state of oregon is ||| ||| {:decoder=>-8.11766, :psb=>0.8142441551106129} +17 ||| you could tell me which of the highest point in the state of oregon is ||| ||| {:decoder=>-8.16515, :psb=>0.5885661912765424} +18 ||| you could tell me which highest point in the state of oregon is ||| ||| {:decoder=>-8.2233, :psb=>0.566260439529554} +19 ||| you could tell me what the highest point in the state oregon is ||| ||| {:decoder=>-8.40307, :psb=>0.535536059195526} +20 ||| you could tell me which the highest point in the state oregon is ||| ||| {:decoder=>-8.48039, :psb=>0.48507053375249726} +>>> + + [TOP1] + nrl: you could tell me which is the highest point in the state of oregon + mrl: answer(highest(place(loc_2(state(stateid('oregon')))))) + output: [placeid('mount hood')] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6985342056580097} + + [HOPE] + nrl: you could tell me which is the highest point in the state of oregon + mrl: answer(highest(place(loc_2(state(stateid('oregon')))))) + output: [placeid('mount hood')] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6985342056580097}, #0 + + [FEAR] + nrl: you could tell me which is the highest point in the state of oregon + mrl: answer(highest(place(loc_2(state(stateid('oregon')))))) + output: [placeid('mount hood')] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6985342056580097}, #0 + +================= + EXAMPLE: 5 + REFERENCE: give me all the states of usa + GOLD MRL: answer(state(loc_2(countryid('usa')))) +GOLD OUTPUT: [stateid(alabama),stateid(alaska),stateid(arizona),stateid(arkansas),stateid(california),stateid(colorado),stateid(connecticut),stateid(delaware),stateid('district of columbia'),stateid(florida),stateid(georgia),stateid(hawaii),stateid(idaho),stateid(illinois),stateid(indiana),stateid(iowa),stateid(kansas),stateid(kentucky),stateid(louisiana),stateid(maine),stateid(maryland),stateid(massachusetts),stateid(michigan),stateid(minnesota),stateid(mississippi),stateid(missouri),stateid(montana),stateid(nebraska),stateid(nevada),stateid('new hampshire'),stateid('new jersey'),stateid('new mexico'),stateid('new york'),stateid('north carolina'),stateid('north dakota'),stateid(ohio),stateid(oklahoma),stateid(oregon),stateid(pennsylvania),stateid('rhode island'),stateid('south carolina'),stateid('south dakota'),stateid(tennessee),stateid(texas),stateid(utah),stateid(vermont),stateid(virginia),stateid(washington),stateid('west virginia'),stateid(wisconsin),stateid(wyoming)] + +<<< KBEST +0 ||| call all states ||| ||| {:decoder=>-3.92581, :psb=>0.15218787864872976} +1 ||| all states call ||| ||| {:decoder=>-4.11349, :psb=>0.15218787864872976} +2 ||| all call states ||| ||| {:decoder=>-4.23272, :psb=>0.15218787864872976} +3 ||| call all states of the usa ||| ||| {:decoder=>-4.27556, :psb=>0.29059254080791846} +4 ||| call all the usa ||| ||| {:decoder=>-4.34422, :psb=>0.23618327637050732} +5 ||| call all states the usa ||| ||| {:decoder=>-4.45563, :psb=>0.22778068328450227} +6 ||| call all states of usa ||| ||| {:decoder=>-4.50267, :psb=>0.3564960350471317} +7 ||| call states all ||| ||| {:decoder=>-4.55982, :psb=>0.15218787864872976} +8 ||| call all united states of america ||| ||| {:decoder=>-4.56554, :psb=>0.2557539057896621} +9 ||| call all states in the usa ||| ||| {:decoder=>-4.574, :psb=>0.2310997417025822} +10 ||| name all states ||| ||| {:decoder=>-4.59699, :psb=>0.15218787864872976} +>>> + + [TOP1] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976} + + [HOPE] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976}, #0 + + [FEAR] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976}, #0 + +================= + EXAMPLE: 6 + REFERENCE: give me the cities in texas + GOLD MRL: answer(city(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + +<<< KBEST +0 ||| tell me what cities in texas are ||| ||| {:decoder=>-4.55611, :psb=>0.35745672884978397} +1 ||| tell me what are cities in texas ||| ||| {:decoder=>-4.69943, :psb=>0.35745672884978397} +2 ||| tell me which cities in texas are ||| ||| {:decoder=>-4.82504, :psb=>0.35745672884978397} +3 ||| tell me which are cities in texas ||| ||| {:decoder=>-4.85315, :psb=>0.35745672884978397} +4 ||| tell me what cities in texas , ||| ||| {:decoder=>-4.87254, :psb=>0.35745672884978397} +5 ||| tell me what cities are in texas ||| ||| {:decoder=>-4.87256, :psb=>0.271608378656351} +6 ||| tell me what cities in texas is ||| ||| {:decoder=>-4.9474, :psb=>0.35745672884978397} +7 ||| tell me which cities are in texas ||| ||| {:decoder=>-5.00184, :psb=>0.271608378656351} +8 ||| are tell me what cities in texas ||| ||| {:decoder=>-5.1019, :psb=>0.35745672884978397} +9 ||| tell me downloading cities in texas are ||| ||| {:decoder=>-5.26545, :psb=>0.35745672884978397} +>>> + + [TOP1] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397} + + [HOPE] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397}, #0 + + [FEAR] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397}, #0 + +================= + EXAMPLE: 7 + REFERENCE: give me the cities in usa + GOLD MRL: answer(city(loc_2(countryid('usa')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + +<<< KBEST +0 ||| call the cities of usa ||| ||| {:decoder=>-3.66011, :psb=>0.3078921402430011} +1 ||| call the cities of the usa ||| ||| {:decoder=>-3.72726, :psb=>0.3021375397356768} +2 ||| call the cities of the u.s. ||| ||| {:decoder=>-3.72881, :psb=>0.2730120862709067} +3 ||| call the cities of the united states ||| ||| {:decoder=>-3.85241, :psb=>0.2283945119649991} +4 ||| the cities of usa call ||| ||| {:decoder=>-3.91981, :psb=>0.3078921402430011} +5 ||| call the cities in the united states ||| ||| {:decoder=>-3.94392, :psb=>0.33265096878635064} +6 ||| call the usa cities ||| ||| {:decoder=>-3.98003, :psb=>0.255014728746912} +7 ||| call the cities the usa ||| ||| {:decoder=>-4.08224, :psb=>0.3078921402430011} +8 ||| call the cities in the usa ||| ||| {:decoder=>-4.09771, :psb=>0.4272870063962341} +>>> + + [TOP1] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011} + + [HOPE] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011}, #0 + + [FEAR] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011}, #0 + +================= + EXAMPLE: 8 + REFERENCE: give me the cities in virginia + GOLD MRL: answer(city(loc_2(stateid('virginia')))) +GOLD OUTPUT: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + +<<< KBEST +0 ||| call me the cities in virginia ||| ||| {:decoder=>-4.33256, :psb=>0.8034284189446518} +1 ||| call me cities in virginia ||| ||| {:decoder=>-4.6442, :psb=>0.43542524047973125} +2 ||| the cities in virginia call me ||| ||| {:decoder=>-4.72759, :psb=>0.6389431042462724} +3 ||| the call me cities in virginia ||| ||| {:decoder=>-4.73668, :psb=>0.45180100180492244} +4 ||| call me the cities virginia ||| ||| {:decoder=>-4.77874, :psb=>0.43542524047973125} +5 ||| call me the towns in virginia ||| ||| {:decoder=>-4.79512, :psb=>0.35930411196308426} +6 ||| call the cities in virginia me ||| ||| {:decoder=>-4.82678, :psb=>0.6389431042462724} +7 ||| me call the cities in virginia ||| ||| {:decoder=>-4.88323, :psb=>0.6389431042462724} +8 ||| call the me cities in virginia ||| ||| {:decoder=>-4.92687, :psb=>0.45180100180492244} +9 ||| call me the cities , virginia in ||| ||| {:decoder=>-4.93699, :psb=>0.3779644730092272} +10 ||| in virginia call me the cities ||| ||| {:decoder=>-5.03151, :psb=>0.48549177170732344} +11 ||| me the call cities in virginia ||| ||| {:decoder=>-5.07227, :psb=>0.48549177170732344} +12 ||| cities in virginia call me the ||| ||| {:decoder=>-5.09447, :psb=>0.48549177170732344} +>>> + + [TOP1] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 9 + REFERENCE: give me the cities which are in texas + GOLD MRL: answer(city(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + +<<< KBEST +0 ||| call me cities in texas ||| ||| {:decoder=>-3.75541, :psb=>0.221776483974985} +1 ||| cities in texas call me ||| ||| {:decoder=>-4.16914, :psb=>0.221776483974985} +2 ||| call me in texas cities ||| ||| {:decoder=>-4.18458, :psb=>0.221776483974985} +3 ||| call me towns in texas ||| ||| {:decoder=>-4.2724, :psb=>0.20638627362169998} +4 ||| call cities in texas me ||| ||| {:decoder=>-4.29096, :psb=>0.221776483974985} +5 ||| me call cities in texas ||| ||| {:decoder=>-4.3131, :psb=>0.221776483974985} +6 ||| i call cities in texas ||| ||| {:decoder=>-4.33349, :psb=>0.20638627362169998} +7 ||| me cities in texas call ||| ||| {:decoder=>-4.34565, :psb=>0.221776483974985} +8 ||| call i cities in texas ||| ||| {:decoder=>-4.34595, :psb=>0.20638627362169998} +9 ||| cities in call me texas ||| ||| {:decoder=>-4.38155, :psb=>0.18649105036213778} +10 ||| cities call me in texas ||| ||| {:decoder=>-4.40393, :psb=>0.221776483974985} +11 ||| call cities in me texas ||| ||| {:decoder=>-4.45751, :psb=>0.18649105036213778} +12 ||| call cities me in texas ||| ||| {:decoder=>-4.52574, :psb=>0.221776483974985} +>>> + + [TOP1] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985} + + [HOPE] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985}, #0 + + [FEAR] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985}, #0 + +================= + EXAMPLE: 10 + REFERENCE: give me the lakes in california + GOLD MRL: answer(lake(loc_2(stateid('california')))) +GOLD OUTPUT: [] + +<<< KBEST +0 ||| call me the lakes in california ||| ||| {:decoder=>-4.21994, :psb=>0.8034284189446518} +1 ||| the lakes in california call me ||| ||| {:decoder=>-4.52517, :psb=>0.6389431042462724} +2 ||| call the lakes in california me ||| ||| {:decoder=>-4.6544, :psb=>0.6389431042462724} +3 ||| the lakes call me in california ||| ||| {:decoder=>-4.68485, :psb=>0.37991784282579627} +4 ||| me call the lakes in california ||| ||| {:decoder=>-4.70086, :psb=>0.6389431042462724} +5 ||| the call me lakes in california ||| ||| {:decoder=>-4.71777, :psb=>0.45180100180492244} +6 ||| i call the lakes in california ||| ||| {:decoder=>-4.79032, :psb=>0.6042750794713536} +7 ||| call me the lakes , california in ||| ||| {:decoder=>-4.79205, :psb=>0.3779644730092272} +8 ||| call the lakes me in california ||| ||| {:decoder=>-4.80999, :psb=>0.37991784282579627} +9 ||| call me a lakes in california ||| ||| {:decoder=>-4.81742, :psb=>0.4272870063962341} +10 ||| call me lakes in california ||| ||| {:decoder=>-4.82882, :psb=>0.43542524047973125} +11 ||| me the lakes in california call ||| ||| {:decoder=>-4.8569, :psb=>0.8034284189446518} +12 ||| call i the lakes in california ||| ||| {:decoder=>-4.95829, :psb=>0.6042750794713536} +13 ||| in california call me the lakes ||| ||| {:decoder=>-4.96477, :psb=>0.48549177170732344} +14 ||| call me the the lakes in california ||| ||| {:decoder=>-5.01259, :psb=>0.5651887140592688} +15 ||| me the the lakes in california call ||| ||| {:decoder=>-5.65175, :psb=>0.5651887140592688} +>>> + + [TOP1] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 11 + REFERENCE: give me the largest state + GOLD MRL: answer(largest(state(all))) +GOLD OUTPUT: [stateid(alaska)] + +<<< KBEST +0 ||| which city is the largest ||| ||| {:decoder=>-3.12489, :psb=>0.33980884896942454} +1 ||| the town is the largest ||| ||| {:decoder=>-3.2327, :psb=>0.33980884896942454} +2 ||| what is the largest city ||| ||| {:decoder=>-3.24458, :psb=>0.33980884896942454} +3 ||| what town is the largest ||| ||| {:decoder=>-3.27745, :psb=>0.33980884896942454} +4 ||| what city is the largest ||| ||| {:decoder=>-3.31533, :psb=>0.33980884896942454} +5 ||| which city is the biggest ||| ||| {:decoder=>-3.36251, :psb=>0.24028114141347542} +6 ||| which is the largest city ||| ||| {:decoder=>-3.36404, :psb=>0.33980884896942454} +7 ||| what city is the biggest ||| ||| {:decoder=>-3.41863, :psb=>0.24028114141347542} +8 ||| which town is the largest ||| ||| {:decoder=>-3.51234, :psb=>0.33980884896942454} +9 ||| which city has the largest ||| ||| {:decoder=>-3.52084, :psb=>0.33980884896942454} +10 ||| what town is the biggest ||| ||| {:decoder=>-3.52506, :psb=>0.24028114141347542} +11 ||| downloading city is the largest ||| ||| {:decoder=>-3.57107, :psb=>0.33980884896942454} +12 ||| which town is the biggest ||| ||| {:decoder=>-3.64376, :psb=>0.24028114141347542} +>>> + + [TOP1] + nrl: which city is the largest + mrl: answer(city(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454} + + [HOPE] + nrl: which city is the largest + mrl: answer(city(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454}, #0 + + [FEAR] + nrl: which city is the largest + mrl: answer(city(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454}, #0 + +================= + EXAMPLE: 12 + REFERENCE: give me the longest river that passes through the us + GOLD MRL: answer(longest(river(traverse_2(countryid('usa'))))) +GOLD OUTPUT: [] + +<<< KBEST +0 ||| call me the longest river , which flows through the usa ||| ||| {:decoder=>-6.95509, :psb=>0.3585594362259136} +1 ||| call me the longest river , which flows through the u.s. ||| ||| {:decoder=>-6.97475, :psb=>0.3585594362259136} +2 ||| call me the longest river which flows through the usa ||| ||| {:decoder=>-7.22326, :psb=>0.3976353643835253} +3 ||| the longest river , which flows through the usa call me ||| ||| {:decoder=>-7.29612, :psb=>0.257663759638272} +4 ||| call me the longest river , which flows through the united states ||| ||| {:decoder=>-7.30196, :psb=>0.3264971028628052} +5 ||| call me , the longest river , which flows through the usa ||| ||| {:decoder=>-7.36955, :psb=>0.23462350320528} +6 ||| call the longest river , which flows through the usa me ||| ||| {:decoder=>-7.37955, :psb=>0.257663759638272} +7 ||| call me the longest river , which flows by the usa ||| ||| {:decoder=>-7.38281, :psb=>0.32399502498695193} +8 ||| call me the longest river that flows through the usa ||| ||| {:decoder=>-7.39227, :psb=>0.5143686723610401} +9 ||| me call the longest river , which flows through the usa ||| ||| {:decoder=>-7.436, :psb=>0.257663759638272} +10 ||| the longest river which flows through the usa call me ||| ||| {:decoder=>-7.5643, :psb=>0.28574404296988} +11 ||| call the longest river which flows through the usa me ||| ||| {:decoder=>-7.64772, :psb=>0.28574404296988} +12 ||| me call the longest river which flows through the usa ||| ||| {:decoder=>-7.70417, :psb=>0.28574404296988} +13 ||| the longest river that flows through the usa call me ||| ||| {:decoder=>-7.73331, :psb=>0.41325840918969} +14 ||| call the longest river that flows through the usa me ||| ||| {:decoder=>-7.81673, :psb=>0.41325840918969} +15 ||| me call the longest river that flows through the usa ||| ||| {:decoder=>-7.87318, :psb=>0.41325840918969} +>>> + + [TOP1] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136} + + [HOPE] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136}, #0 + + [FEAR] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136}, #0 + +================= + EXAMPLE: 13 + REFERENCE: how big is alaska + GOLD MRL: answer(size(stateid('alaska'))) +GOLD OUTPUT: [591000.0] + +<<< KBEST +0 ||| how big is alaska ||| ||| {:decoder=>-2.98892, :psb=>1.0} +1 ||| how large is alaska ||| ||| {:decoder=>-3.33252, :psb=>0.49999999999999994} +2 ||| how great is alaska ||| ||| {:decoder=>-3.33959, :psb=>0.49999999999999994} +3 ||| how great alaska ||| ||| {:decoder=>-3.51397, :psb=>0.41368954504257255} +4 ||| how much is alaska ||| ||| {:decoder=>-3.52043, :psb=>0.49999999999999994} +5 ||| as big is alaska ||| ||| {:decoder=>-3.54857, :psb=>0.6580370064762462} +6 ||| what is alaska ||| ||| {:decoder=>-3.58248, :psb=>0.4919625503668659} +7 ||| as large is alaska ||| ||| {:decoder=>-3.58385, :psb=>0.4518010018049224} +8 ||| how big alaska is ||| ||| {:decoder=>-3.59645, :psb=>0.537284965911771} +9 ||| is how large alaska ||| ||| {:decoder=>-3.644, :psb=>0.42044820762685725} +10 ||| is how big alaska ||| ||| {:decoder=>-3.67593, :psb=>0.537284965911771} +11 ||| how large alaska is ||| ||| {:decoder=>-3.69363, :psb=>0.42044820762685725} +12 ||| is as large alaska ||| ||| {:decoder=>-3.70024, :psb=>0.37991784282579627} +13 ||| how great alaska is ||| ||| {:decoder=>-3.76508, :psb=>0.42044820762685725} +14 ||| is as big alaska ||| ||| {:decoder=>-3.76676, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how large is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>0.5582639103157458, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 14 + REFERENCE: how big is massachusetts + GOLD MRL: answer(size(stateid('massachusetts'))) +GOLD OUTPUT: [8284.0] + +<<< KBEST +0 ||| how big is massachusetts ||| ||| {:decoder=>-3.48825, :psb=>1.0} +1 ||| how great is massachusetts ||| ||| {:decoder=>-3.845, :psb=>0.49999999999999994} +2 ||| how much is massachusetts ||| ||| {:decoder=>-3.98945, :psb=>0.49999999999999994} +3 ||| how great massachusetts ||| ||| {:decoder=>-3.99393, :psb=>0.41368954504257255} +4 ||| how large is massachusetts ||| ||| {:decoder=>-4.00153, :psb=>0.49999999999999994} +5 ||| what is massachusetts ||| ||| {:decoder=>-4.03909, :psb=>0.4919625503668659} +6 ||| how big is , massachusetts ||| ||| {:decoder=>-4.09322, :psb=>0.5318295896944989} +7 ||| just how big is massachusetts ||| ||| {:decoder=>-4.15455, :psb=>0.7521206186172787} +8 ||| how big is ma ||| ||| {:decoder=>-4.27101, :psb=>0.6580370064762462} +9 ||| is how large massachusetts ||| ||| {:decoder=>-4.31332, :psb=>0.42044820762685725} +10 ||| how big is ( massachusetts ||| ||| {:decoder=>-4.31763, :psb=>0.5318295896944989} +11 ||| how big , massachusetts ||| ||| {:decoder=>-4.3379, :psb=>0.49999999999999994} +12 ||| is how big massachusetts ||| ||| {:decoder=>-4.33837, :psb=>0.537284965911771} +13 ||| is as large massachusetts ||| ||| {:decoder=>-4.37489, :psb=>0.37991784282579627} +14 ||| how big massachusetts is ||| ||| {:decoder=>-4.38305, :psb=>0.537284965911771} +15 ||| how large , massachusetts ||| ||| {:decoder=>-4.43004, :psb=>0.37991784282579627} +16 ||| is as big massachusetts ||| ||| {:decoder=>-4.43439, :psb=>0.42044820762685725} +17 ||| how large massachusetts is ||| ||| {:decoder=>-4.48433, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is massachusetts + mrl: answer(loc_1(stateid('massachusetts'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.6418460364629344, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 15 + REFERENCE: how big is new mexico + GOLD MRL: answer(size(stateid('new mexico'))) +GOLD OUTPUT: [121600.0] + +<<< KBEST +0 ||| how big is new mexico ||| ||| {:decoder=>-4.05458, :psb=>1.0} +1 ||| how big is mexico ||| ||| {:decoder=>-4.33683, :psb=>0.5506953149031837} +2 ||| what is new mexico ||| ||| {:decoder=>-4.35235, :psb=>0.5124797359336637} +3 ||| how great is new mexico ||| ||| {:decoder=>-4.44119, :psb=>0.5318295896944989} +4 ||| how large is new mexico ||| ||| {:decoder=>-4.52288, :psb=>0.5318295896944989} +5 ||| how much is new mexico ||| ||| {:decoder=>-4.55958, :psb=>0.5318295896944989} +6 ||| how great new mexico ||| ||| {:decoder=>-4.56912, :psb=>0.3894003915357024} +7 ||| how great is mexico ||| ||| {:decoder=>-4.72344, :psb=>0.32744539334076506} +8 ||| just how big is new mexico ||| ||| {:decoder=>-4.76495, :psb=>0.8034284189446518} +9 ||| how large is mexico ||| ||| {:decoder=>-4.87078, :psb=>0.32744539334076506} +10 ||| how big new mexico is ||| ||| {:decoder=>-4.89277, :psb=>0.47287080450158786} +11 ||| how large new mexico is ||| ||| {:decoder=>-4.97659, :psb=>0.40410310093532464} +12 ||| how large , new mexico ||| ||| {:decoder=>-4.97943, :psb=>0.3760603093086393} +13 ||| is as large new mexico ||| ||| {:decoder=>-5.00405, :psb=>0.3760603093086393} +>>> + + [TOP1] + nrl: how big is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: what is new mexico + mrl: answer('new mexico') + output: []. + correct?: false + SCORES: {:decoder=>0.6863829294237832, :psb=>0.5124797359336637}, #2 + +================= + EXAMPLE: 16 + REFERENCE: how big is north dakota + GOLD MRL: answer(size(stateid('north dakota'))) +GOLD OUTPUT: [70700.0] + +<<< KBEST +0 ||| how big is north dakota ||| ||| {:decoder=>-3.35756, :psb=>1.0} +1 ||| how great is north dakota ||| ||| {:decoder=>-3.70091, :psb=>0.5318295896944989} +2 ||| how large is north dakota ||| ||| {:decoder=>-3.81679, :psb=>0.5318295896944989} +3 ||| how much is north dakota ||| ||| {:decoder=>-3.9503, :psb=>0.5318295896944989} +4 ||| just how big is north dakota ||| ||| {:decoder=>-3.98813, :psb=>0.8034284189446518} +5 ||| how great north dakota ||| ||| {:decoder=>-3.99944, :psb=>0.3894003915357024} +6 ||| what is north dakota ||| ||| {:decoder=>-4.12953, :psb=>0.5124797359336637} +7 ||| how big north dakota is ||| ||| {:decoder=>-4.15327, :psb=>0.47287080450158786} +8 ||| is how large north dakota ||| ||| {:decoder=>-4.18577, :psb=>0.40410310093532464} +9 ||| is how big north dakota ||| ||| {:decoder=>-4.19059, :psb=>0.47287080450158786} +10 ||| as big is north dakota ||| ||| {:decoder=>-4.21926, :psb=>0.7521206186172787} +11 ||| is as large north dakota ||| ||| {:decoder=>-4.25994, :psb=>0.3760603093086393} +12 ||| how large north dakota is ||| ||| {:decoder=>-4.27708, :psb=>0.40410310093532464} +13 ||| is as big north dakota ||| ||| {:decoder=>-4.32673, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how big is north dakota + mrl: answer(size(stateid('north dakota'))) + output: [70700.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is north dakota + mrl: answer(size(stateid('north dakota'))) + output: [70700.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is north dakota + mrl: answer(loc_1(stateid('north dakota'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.6457277876946255, :psb=>0.5318295896944989}, #1 + +================= + EXAMPLE: 17 + REFERENCE: how big is texas + GOLD MRL: answer(size(stateid('texas'))) +GOLD OUTPUT: [266807.0] + +<<< KBEST +0 ||| how big is texas ||| ||| {:decoder=>-2.99927, :psb=>1.0} +1 ||| how great is texas ||| ||| {:decoder=>-3.37968, :psb=>0.49999999999999994} +2 ||| how large is texas ||| ||| {:decoder=>-3.48712, :psb=>0.49999999999999994} +3 ||| how much is texas ||| ||| {:decoder=>-3.59539, :psb=>0.49999999999999994} +4 ||| how great texas ||| ||| {:decoder=>-3.62859, :psb=>0.41368954504257255} +5 ||| just how big is texas ||| ||| {:decoder=>-3.65333, :psb=>0.7521206186172787} +6 ||| how big is of texas ||| ||| {:decoder=>-3.72698, :psb=>0.5318295896944989} +7 ||| what is texas ||| ||| {:decoder=>-3.76694, :psb=>0.4919625503668659} +8 ||| like big is texas ||| ||| {:decoder=>-3.77439, :psb=>0.6580370064762462} +9 ||| how big texas is ||| ||| {:decoder=>-3.81161, :psb=>0.537284965911771} +10 ||| is how large texas ||| ||| {:decoder=>-3.81498, :psb=>0.42044820762685725} +11 ||| is how big texas ||| ||| {:decoder=>-3.84152, :psb=>0.537284965911771} +12 ||| as big is texas ||| ||| {:decoder=>-3.8878, :psb=>0.6580370064762462} +13 ||| is as large texas ||| ||| {:decoder=>-3.89239, :psb=>0.37991784282579627} +14 ||| how large texas is ||| ||| {:decoder=>-3.90906, :psb=>0.42044820762685725} +15 ||| is as big texas ||| ||| {:decoder=>-3.96071, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is texas + mrl: answer(size(stateid('texas'))) + output: [266807.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is texas + mrl: answer(size(stateid('texas'))) + output: [266807.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is texas + mrl: answer(loc_1(stateid('texas'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.6043330837077718, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 18 + REFERENCE: how big is the city of new york + GOLD MRL: answer(size(city(cityid('new york', _)))) +GOLD OUTPUT: [7071639] + +<<< KBEST +0 ||| how big is the city of new york ||| ||| {:decoder=>-3.86018, :psb=>1.0} +1 ||| how big is the new york city ||| ||| {:decoder=>-3.93768, :psb=>0.5329462628216856} +2 ||| how much is the new york city ||| ||| {:decoder=>-4.12003, :psb=>0.2883677731713749} +3 ||| how big is the york city ||| ||| {:decoder=>-4.2701, :psb=>0.4791733671582712} +4 ||| how great is the city of new york ||| ||| {:decoder=>-4.27963, :psb=>0.7476743906106103} +5 ||| how large is the new york city ||| ||| {:decoder=>-4.32869, :psb=>0.2883677731713749} +6 ||| how large is the city of new york ||| ||| {:decoder=>-4.40741, :psb=>0.7476743906106103} +7 ||| how great is the new york city ||| ||| {:decoder=>-4.486, :psb=>0.2883677731713749} +8 ||| how big is the city new york ||| ||| {:decoder=>-4.48641, :psb=>0.6633400689531749} +9 ||| just how big is the city of new york ||| ||| {:decoder=>-4.57721, :psb=>0.8773826753016616} +>>> + + [TOP1] + nrl: how big is the city of new york + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is the city of new york + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how big is the new york city + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>0.8919152615650674, :psb=>0.5329462628216856}, #1 + +================= + EXAMPLE: 19 + REFERENCE: how high are the highest points of all the states + GOLD MRL: answer(elevation_1(highest(place(loc_2(state(all)))))) +GOLD OUTPUT: [6194] + +<<< KBEST +0 ||| what are the highest surveys all states ||| ||| {:decoder=>-6.84432, :psb=>0.257703362342899} +1 ||| how much are the highest surveys all states ||| ||| {:decoder=>-6.89276, :psb=>0.27534765745159184} +2 ||| how high is the highest surveys all states ||| ||| {:decoder=>-6.91242, :psb=>0.2315388580995513} +3 ||| how high are the highest surveys all states ||| ||| {:decoder=>-6.92558, :psb=>0.5062667121584363} +4 ||| how high is the highest surveys of all states ||| ||| {:decoder=>-7.03658, :psb=>0.25933688537080213} +5 ||| how high are the highest surveys of all states ||| ||| {:decoder=>-7.04974, :psb=>0.5501366107724776} +6 ||| all states how high are the highest surveys ||| ||| {:decoder=>-7.09249, :psb=>0.5062667121584363} +7 ||| how high are the highest peaks all states ||| ||| {:decoder=>-7.15425, :psb=>0.5062667121584363} +8 ||| as highly are the highest surveys all states ||| ||| {:decoder=>-7.25011, :psb=>0.26493826542476406} +9 ||| how high are the supreme surveys of all states ||| ||| {:decoder=>-7.29429, :psb=>0.429170247427184} +10 ||| what are the highest uplift all states ||| ||| {:decoder=>-7.4484, :psb=>0.257703362342899} +11 ||| what are the highest eminence all states ||| ||| {:decoder=>-7.45127, :psb=>0.257703362342899} +12 ||| what are the highest hills all states ||| ||| {:decoder=>-7.50095, :psb=>0.257703362342899} +13 ||| how high is the highest uplift all states ||| ||| {:decoder=>-7.50111, :psb=>0.2315388580995513} +14 ||| how high is the highest eminence all states ||| ||| {:decoder=>-7.50399, :psb=>0.2315388580995513} +15 ||| how high are the highest uplift all states ||| ||| {:decoder=>-7.51427, :psb=>0.5062667121584363} +16 ||| how high are the highest eminence all states ||| ||| {:decoder=>-7.51715, :psb=>0.5062667121584363} +17 ||| how high is the highest hills all states ||| ||| {:decoder=>-7.55366, :psb=>0.2315388580995513} +18 ||| how high are the highest hills all states ||| ||| {:decoder=>-7.56682, :psb=>0.5062667121584363} +19 ||| how much are the highest uplift all states ||| ||| {:decoder=>-7.69884, :psb=>0.27534765745159184} +20 ||| how much are the highest eminence all states ||| ||| {:decoder=>-7.70171, :psb=>0.27534765745159184} +21 ||| how much are the highest hills all states ||| ||| {:decoder=>-7.75139, :psb=>0.27534765745159184} +>>> + + [TOP1] + nrl: what are the highest surveys all states + mrl: answer(highest(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.257703362342899} + + [HOPE] + nrl: how high are the highest surveys all states + mrl: answer(elevation_1(highest(state(all)))) + output: [] + correct?: false + SCORES: {:decoder=>0.9104148522164769, :psb=>0.5062667121584363}, #3 + + [FEAR] + nrl: what are the highest surveys all states + mrl: answer(highest(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.257703362342899}, #0 + +================= + EXAMPLE: 20 + REFERENCE: how high is guadalupe peak + GOLD MRL: answer(elevation_1(placeid('guadalupe peak'))) +GOLD OUTPUT: [2667] + +<<< KBEST +0 ||| how high is guadalupe peak ||| ||| {:decoder=>-3.4003, :psb=>1.0} +1 ||| how much is guadalupe peak ||| ||| {:decoder=>-3.70178, :psb=>0.5318295896944989} +2 ||| how high is located guadalupe peak ||| ||| {:decoder=>-3.79957, :psb=>0.48549177170732344} +3 ||| how high is situated guadalupe peak ||| ||| {:decoder=>-3.83114, :psb=>0.48549177170732344} +4 ||| as high is guadalupe peak ||| ||| {:decoder=>-3.85381, :psb=>0.7521206186172787} +5 ||| how highly is guadalupe peak ||| ||| {:decoder=>-3.88094, :psb=>0.5318295896944989} +6 ||| as highly is guadalupe peak ||| ||| {:decoder=>-3.91956, :psb=>0.4949232003839765} +7 ||| like high is guadalupe peak ||| ||| {:decoder=>-3.92346, :psb=>0.7521206186172787} +8 ||| how high lies guadalupe peak ||| ||| {:decoder=>-3.94484, :psb=>0.4472135954999579} +9 ||| such as high is guadalupe peak ||| ||| {:decoder=>-3.96174, :psb=>0.6042750794713536} +10 ||| how high located guadalupe peak ||| ||| {:decoder=>-3.97385, :psb=>0.4472135954999579} +11 ||| how high situated guadalupe peak ||| ||| {:decoder=>-3.98086, :psb=>0.4472135954999579} +12 ||| like highly is guadalupe peak ||| ||| {:decoder=>-4.00156, :psb=>0.4949232003839765} +13 ||| as high is located guadalupe peak ||| ||| {:decoder=>-4.27579, :psb=>0.35930411196308426} +14 ||| how highly is located guadalupe peak ||| ||| {:decoder=>-4.28634, :psb=>0.32466791547509893} +15 ||| how highly is situated guadalupe peak ||| ||| {:decoder=>-4.3179, :psb=>0.32466791547509893} +16 ||| as high is situated guadalupe peak ||| ||| {:decoder=>-4.32847, :psb=>0.35930411196308426} +17 ||| as highly is located guadalupe peak ||| ||| {:decoder=>-4.33047, :psb=>0.3021375397356768} +18 ||| like high is located guadalupe peak ||| ||| {:decoder=>-4.34544, :psb=>0.35930411196308426} +19 ||| as highly is situated guadalupe peak ||| ||| {:decoder=>-4.36203, :psb=>0.3021375397356768} +20 ||| such as high is located guadalupe peak ||| ||| {:decoder=>-4.38372, :psb=>0.3005840818981024} +21 ||| like high is situated guadalupe peak ||| ||| {:decoder=>-4.39812, :psb=>0.35930411196308426} +22 ||| like highly is located guadalupe peak ||| ||| {:decoder=>-4.40696, :psb=>0.3021375397356768} +23 ||| such as high is situated guadalupe peak ||| ||| {:decoder=>-4.4364, :psb=>0.3005840818981024} +24 ||| like highly is situated guadalupe peak ||| ||| {:decoder=>-4.43853, :psb=>0.3021375397356768} +25 ||| how high is situated of guadalupe peak ||| ||| {:decoder=>-4.55033, :psb=>0.40614925799324625} +26 ||| as high is situated of guadalupe peak ||| ||| {:decoder=>-4.7445, :psb=>0.3005840818981024} +27 ||| like high is situated of guadalupe peak ||| ||| {:decoder=>-4.81415, :psb=>0.3005840818981024} +28 ||| such as high is situated of guadalupe peak ||| ||| {:decoder=>-4.85243, :psb=>0.25848657697858524} +>>> + + [TOP1] + nrl: how high is guadalupe peak + mrl: answer(elevation_1(placeid('guadalupe peak'))) + output: [2667] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is guadalupe peak + mrl: answer(elevation_1(placeid('guadalupe peak'))) + output: [2667] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how much is guadalupe peak + mrl: answer(loc_1(placeid('guadalupe peak'))) + output: [countryid(usa),stateid(texas)] + correct?: false + SCORES: {:decoder=>0.7923877338805756, :psb=>0.5318295896944989}, #1 + +================= + EXAMPLE: 21 + REFERENCE: how high is the highest point in montana + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) +GOLD OUTPUT: [3901] + +<<< KBEST +0 ||| how high is the highest point in montana ||| ||| {:decoder=>-3.38519, :psb=>1.0} +1 ||| as high is the highest point in montana ||| ||| {:decoder=>-3.6603, :psb=>0.8599476570625982} +2 ||| like high is the highest point in montana ||| ||| {:decoder=>-3.67525, :psb=>0.8599476570625982} +3 ||| how much is the highest point in montana ||| ||| {:decoder=>-3.70863, :psb=>0.7476743906106103} +4 ||| how high is the highest point montana ||| ||| {:decoder=>-3.75306, :psb=>0.7536998328984837} +5 ||| as highly is the highest point in montana ||| ||| {:decoder=>-3.77606, :psb=>0.719408902854813} +6 ||| like highly is the highest point in montana ||| ||| {:decoder=>-3.79648, :psb=>0.719408902854813} +7 ||| how highly is the highest point in montana ||| ||| {:decoder=>-3.8789, :psb=>0.7476743906106103} +8 ||| such as high is the highest point in montana ||| ||| {:decoder=>-3.97329, :psb=>0.7545031759729783} +>>> + + [TOP1] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + +================= + EXAMPLE: 22 + REFERENCE: how high is the highest point in the largest state + GOLD MRL: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) +GOLD OUTPUT: [6194] + +<<< KBEST +0 ||| how high lies the highest point of the largest state ||| ||| {:decoder=>-5.13697, :psb=>0.3760603093086394} +1 ||| how high lies the highest point of the biggest state ||| ||| {:decoder=>-5.25588, :psb=>0.29697089145035693} +2 ||| as high is the highest point of the largest state ||| ||| {:decoder=>-5.29645, :psb=>0.5844356470407898} +3 ||| as high lies the highest point of the largest state ||| ||| {:decoder=>-5.29965, :psb=>0.347507514861063} +4 ||| as highly lies the highest point of the largest state ||| ||| {:decoder=>-5.31402, :psb=>0.33437015248821106} +5 ||| how high is the highest point of the largest state ||| ||| {:decoder=>-5.3277, :psb=>0.6999271023161167} +6 ||| how high lies the highest point the largest state ||| ||| {:decoder=>-5.39775, :psb=>0.3777177718431893} +7 ||| how high is the highest point of the biggest state ||| ||| {:decoder=>-5.41523, :psb=>0.6042750794713536} +8 ||| as high is situated the highest point of the largest state ||| ||| {:decoder=>-5.42835, :psb=>0.33910457813956707} +9 ||| as high is the highest point of the biggest state ||| ||| {:decoder=>-5.43372, :psb=>0.49144984054308527} +10 ||| how high is situated the highest point of the largest state ||| ||| {:decoder=>-5.43385, :psb=>0.3900256463674887} +11 ||| as high is located the highest point of the largest state ||| ||| {:decoder=>-5.43934, :psb=>0.33910457813956707} +12 ||| as high is the highest point of largest state ||| ||| {:decoder=>-5.45679, :psb=>0.5341735956899846} +13 ||| how much lies the highest point of the largest state ||| ||| {:decoder=>-5.471, :psb=>0.347507514861063} +14 ||| how high lies the highest point of the greatest state ||| ||| {:decoder=>-5.47439, :psb=>0.29697089145035693} +15 ||| how high lies the highest point the biggest state ||| ||| {:decoder=>-5.48456, :psb=>0.2982797722714565} +16 ||| how highly lies the highest point of the largest state ||| ||| {:decoder=>-5.50574, :psb=>0.347507514861063} +17 ||| how high is located the highest point of the largest state ||| ||| {:decoder=>-5.51361, :psb=>0.3900256463674887} +18 ||| as high situated the highest point of the largest state ||| ||| {:decoder=>-5.54247, :psb=>0.347507514861063} +19 ||| as high located the highest point of the largest state ||| ||| {:decoder=>-5.54826, :psb=>0.347507514861063} +20 ||| like high lies the highest point of the largest state ||| ||| {:decoder=>-5.56125, :psb=>0.347507514861063} +21 ||| how high located the highest point of the largest state ||| ||| {:decoder=>-5.5768, :psb=>0.3760603093086394} +22 ||| how high situated the highest point of the largest state ||| ||| {:decoder=>-5.83805, :psb=>0.3760603093086394} +23 ||| like high situated the highest point of the largest state ||| ||| {:decoder=>-5.99662, :psb=>0.347507514861063} +24 ||| like high located the highest point of the largest state ||| ||| {:decoder=>-6.00241, :psb=>0.347507514861063} +>>> + + [TOP1] + nrl: how high lies the highest point of the largest state + mrl: answer(elevation_1(traverse_1(longest(highest(place(loc_2(largest(state(all))))))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3760603093086394} + + [HOPE] + nrl: how high is the highest point of the largest state + mrl: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) + output: [6194] + correct?: true + SCORES: {:decoder=>0.7796149935293027, :psb=>0.6999271023161167}, #5 + + [FEAR] + nrl: how high lies the highest point of the largest state + mrl: answer(elevation_1(traverse_1(longest(highest(place(loc_2(largest(state(all))))))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.3760603093086394}, #0 + +================= + EXAMPLE: 23 + REFERENCE: how high is the highest point of delaware + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) +GOLD OUTPUT: [135] + +<<< KBEST +0 ||| how high is the highest point of delaware ||| ||| {:decoder=>-5.29326, :psb=>1.0} +1 ||| how much is the highest point of delaware ||| ||| {:decoder=>-5.54301, :psb=>0.7476743906106103} +2 ||| how high is located the highest point of delaware ||| ||| {:decoder=>-5.58546, :psb=>0.6559965570884768} +3 ||| as high is the highest point of delaware ||| ||| {:decoder=>-5.66772, :psb=>0.8599476570625982} +4 ||| how high is of the highest point of delaware ||| ||| {:decoder=>-5.68158, :psb=>0.6559965570884768} +5 ||| like high is the highest point of delaware ||| ||| {:decoder=>-5.72751, :psb=>0.8599476570625982} +6 ||| how high is situated the highest point of delaware ||| ||| {:decoder=>-5.84379, :psb=>0.6559965570884768} +7 ||| such as high is the highest point of delaware ||| ||| {:decoder=>-5.89577, :psb=>0.7545031759729783} +8 ||| how high is the highest point of of delaware ||| ||| {:decoder=>-6.09751, :psb=>0.8065978233496266} +9 ||| as high is the highest point of of delaware ||| ||| {:decoder=>-6.44204, :psb=>0.6817705815391745} +10 ||| how much is the highest point of of delaware ||| ||| {:decoder=>-6.46402, :psb=>0.5773502691896258} +11 ||| like high is the highest point of of delaware ||| ||| {:decoder=>-6.58852, :psb=>0.6817705815391745} +12 ||| such as high is the highest point of of delaware ||| ||| {:decoder=>-6.66799, :psb=>0.6073980520929507} +13 ||| how high is located the highest point of of delaware ||| ||| {:decoder=>-6.69356, :psb=>0.49944351611061033} +14 ||| how high is of the highest point of of delaware ||| ||| {:decoder=>-7.01692, :psb=>0.49944351611061033} +>>> + + [TOP1] + nrl: how high is the highest point of delaware + mrl: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) + output: [135] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point of delaware + mrl: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) + output: [135] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is located the highest point of delaware + mrl: answer(elevation_1(loc_1(highest(place(loc_2(stateid('delaware'))))))) + output: [] + correct?: false + SCORES: {:decoder=>0.8304770082266802, :psb=>0.6559965570884768}, #2 + +================= + EXAMPLE: 24 + REFERENCE: how high is the highest point of florida + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('florida')))))) +GOLD OUTPUT: [105] + +<<< KBEST +0 ||| how high is the highest point florida ||| ||| {:decoder=>-6.68086, :psb=>0.7536998328984837} +1 ||| how high is the highest point floridas ||| ||| {:decoder=>-6.88691, :psb=>0.7252065560578255} +2 ||| how much is the highest point florida ||| ||| {:decoder=>-6.88731, :psb=>0.48497485346901076} +3 ||| what is the highest point florida ||| ||| {:decoder=>-6.91488, :psb=>0.45782273986766686} +4 ||| how high the highest point is florida ||| ||| {:decoder=>-6.9883, :psb=>0.3829795673743804} +5 ||| florida how high is the highest point ||| ||| {:decoder=>-7.13067, :psb=>0.7536998328984837} +6 ||| as high is the highest point florida ||| ||| {:decoder=>-7.15724, :psb=>0.6098235933075173} +7 ||| how high florida is the highest point ||| ||| {:decoder=>-7.20824, :psb=>0.5329462628216856} +8 ||| like high is the highest point florida ||| ||| {:decoder=>-7.32247, :psb=>0.6098235933075173} +9 ||| florida as high is the highest point ||| ||| {:decoder=>-7.37726, :psb=>0.6098235933075173} +10 ||| how much florida is the highest point ||| ||| {:decoder=>-7.44916, :psb=>0.48497485346901076} +11 ||| florida like high is the highest point ||| ||| {:decoder=>-7.4618, :psb=>0.6098235933075173} +12 ||| florida how much is the highest point ||| ||| {:decoder=>-7.46192, :psb=>0.48497485346901076} +13 ||| florida what is the highest point ||| ||| {:decoder=>-7.51419, :psb=>0.45782273986766686} +14 ||| as high florida is the highest point ||| ||| {:decoder=>-7.58351, :psb=>0.48497485346901076} +15 ||| as florida high is the highest point ||| ||| {:decoder=>-7.5905, :psb=>0.6098235933075173} +16 ||| as florida highly is the highest point ||| ||| {:decoder=>-7.99345, :psb=>0.4633657281473354} +>>> + + [TOP1] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837} + + [HOPE] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837}, #0 + + [FEAR] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837}, #0 + +================= + EXAMPLE: 25 + REFERENCE: how high is the highest point of louisiana + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) +GOLD OUTPUT: [163] + +<<< KBEST +0 ||| how high is the highest point of louisiana ||| ||| {:decoder=>-6.65751, :psb=>1.0} +1 ||| how much is the highest point of louisiana ||| ||| {:decoder=>-6.86396, :psb=>0.7476743906106103} +2 ||| what is the highest point of louisiana ||| ||| {:decoder=>-6.88351, :psb=>0.7252065560578255} +3 ||| as high is the highest point of louisiana ||| ||| {:decoder=>-7.1639, :psb=>0.8599476570625982} +4 ||| like high is the highest point of louisiana ||| ||| {:decoder=>-7.29912, :psb=>0.8599476570625982} +5 ||| louisiana how high is the highest point of ||| ||| {:decoder=>-7.50406, :psb=>0.8891397050194614} +6 ||| how high is the highest point of of louisiana ||| ||| {:decoder=>-7.7048, :psb=>0.8065978233496266} +7 ||| how much is the highest point of of louisiana ||| ||| {:decoder=>-8.00776, :psb=>0.5773502691896258} +8 ||| what is the highest point of of louisiana ||| ||| {:decoder=>-8.04695, :psb=>0.633160218956179} +9 ||| as high is the highest point of of louisiana ||| ||| {:decoder=>-8.18649, :psb=>0.6817705815391745} +10 ||| like high is the highest point of of louisiana ||| ||| {:decoder=>-8.40118, :psb=>0.6817705815391745} +>>> + + [TOP1] + nrl: how high is the highest point of louisiana + mrl: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) + output: [163] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point of louisiana + mrl: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) + output: [163] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: what is the highest point of louisiana + mrl: answer(highest(place(loc_2(stateid('louisiana'))))) + output: [placeid('driskill mountain')] + correct?: false + SCORES: {:decoder=>0.8703883188906157, :psb=>0.7252065560578255}, #2 + +================= + EXAMPLE: 26 + REFERENCE: how large is the largest city in alaska + GOLD MRL: answer(size(largest(city(loc_2(stateid('alaska')))))) +GOLD OUTPUT: [174431] + +<<< KBEST +0 ||| how big is the largest city in alaska ||| ||| {:decoder=>-4.96206, :psb=>0.7476743906106103} +1 ||| how big is the biggest city in alaska ||| ||| {:decoder=>-5.19406, :psb=>0.3655552228545123} +2 ||| how large is the largest city in alaska ||| ||| {:decoder=>-5.30781, :psb=>1.0} +3 ||| how great is the largest city in alaska ||| ||| {:decoder=>-5.36404, :psb=>0.7476743906106103} +4 ||| as big is the largest city in alaska ||| ||| {:decoder=>-5.3762, :psb=>0.719408902854813} +5 ||| how big is the largest town in alaska ||| ||| {:decoder=>-5.40489, :psb=>0.3655552228545123} +6 ||| how large is the biggest city in alaska ||| ||| {:decoder=>-5.65073, :psb=>0.5946035575013605} +>>> + + [TOP1] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103} + + [HOPE] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103}, #0 + + [FEAR] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103}, #0 + +================= + EXAMPLE: 27 + REFERENCE: how long is rio grande + GOLD MRL: answer(len(riverid('rio grande'))) +GOLD OUTPUT: [3033] + +<<< KBEST +0 ||| how long is rio grande ||| ||| {:decoder=>-3.52588, :psb=>1.0} +1 ||| how long rio grande ||| ||| {:decoder=>-3.7083, :psb=>0.4630777161991027} +2 ||| how long has rio grande ||| ||| {:decoder=>-3.84633, :psb=>0.4472135954999579} +3 ||| like lang is rio grande ||| ||| {:decoder=>-4.00042, :psb=>0.4949232003839765} +4 ||| how long will rio grande ||| ||| {:decoder=>-4.05779, :psb=>0.4472135954999579} +5 ||| how long rio grande is ||| ||| {:decoder=>-4.0743, :psb=>0.47287080450158786} +6 ||| like long is rio grande ||| ||| {:decoder=>-4.0844, :psb=>0.7521206186172787} +7 ||| how long is a rio grande ||| ||| {:decoder=>-4.1369, :psb=>0.48549177170732344} +8 ||| how long is of rio grande ||| ||| {:decoder=>-4.15176, :psb=>0.48549177170732344} +9 ||| rio grande is how long ||| ||| {:decoder=>-4.17458, :psb=>0.47287080450158786} +10 ||| is how long rio grande ||| ||| {:decoder=>-4.20232, :psb=>0.47287080450158786} +11 ||| how long , rio grande ||| ||| {:decoder=>-4.26115, :psb=>0.4472135954999579} +12 ||| as long is rio grande ||| ||| {:decoder=>-4.34869, :psb=>0.7521206186172787} +13 ||| rio grande is as long ||| ||| {:decoder=>-4.35013, :psb=>0.40410310093532464} +14 ||| as lang is rio grande ||| ||| {:decoder=>-4.48982, :psb=>0.4949232003839765} +15 ||| is as long rio grande ||| ||| {:decoder=>-4.49959, :psb=>0.40410310093532464} +16 ||| is like long rio grande ||| ||| {:decoder=>-4.56531, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how long is rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how long is rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how long rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>0.8244999663276988, :psb=>0.4630777161991027}, #1 + +================= + EXAMPLE: 28 + REFERENCE: how long is the mississippi + GOLD MRL: answer(len(riverid('mississippi'))) +GOLD OUTPUT: [3778] + +<<< KBEST +0 ||| how long is the mississippi river ||| ||| {:decoder=>-4.02421, :psb=>0.8034284189446518} +1 ||| how long the mississippi river ||| ||| {:decoder=>-4.33982, :psb=>0.4472135954999579} +2 ||| how long has the mississippi river ||| ||| {:decoder=>-4.53518, :psb=>0.35930411196308426} +3 ||| how long is mississippi river ||| ||| {:decoder=>-4.59269, :psb=>0.5318295896944989} +4 ||| how long the mississippi river is ||| ||| {:decoder=>-4.61284, :psb=>0.37991784282579627} +5 ||| is how long the mississippi river ||| ||| {:decoder=>-4.69413, :psb=>0.37991784282579627} +6 ||| as long is the mississippi river ||| ||| {:decoder=>-4.846, :psb=>0.6042750794713536} +7 ||| like long is the mississippi river ||| ||| {:decoder=>-5.07372, :psb=>0.6042750794713536} +8 ||| how long is der mississippi river ||| ||| {:decoder=>-5.15498, :psb=>0.4272870063962341} +9 ||| is as long the mississippi river ||| ||| {:decoder=>-5.16513, :psb=>0.32466791547509893} +10 ||| is like long the mississippi river ||| ||| {:decoder=>-5.24122, :psb=>0.32466791547509893} +>>> + + [TOP1] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 29 + REFERENCE: how long is the mississippi river + GOLD MRL: answer(len(river(riverid('mississippi')))) +GOLD OUTPUT: [3778] + +<<< KBEST +0 ||| how long is the mississippi ||| ||| {:decoder=>-3.59409, :psb=>0.8187307530779819} +1 ||| how long the mississippi ||| ||| {:decoder=>-3.86179, :psb=>0.36064528799877893} +2 ||| how long is mississippi ||| ||| {:decoder=>-3.93407, :psb=>0.4288819424803534} +3 ||| how long has the mississippi ||| ||| {:decoder=>-4.05715, :psb=>0.36614752383039256} +4 ||| is how long the mississippi ||| ||| {:decoder=>-4.2161, :psb=>0.3871538698781762} +5 ||| the mississippi is how long ||| ||| {:decoder=>-4.21615, :psb=>0.3871538698781762} +6 ||| how long the mississippi is ||| ||| {:decoder=>-4.21634, :psb=>0.3871538698781762} +7 ||| how long is of mississippi ||| ||| {:decoder=>-4.26868, :psb=>0.43542524047973125} +8 ||| as long is the mississippi ||| ||| {:decoder=>-4.33797, :psb=>0.6157842804860023} +9 ||| the mississippi is as long ||| ||| {:decoder=>-4.39611, :psb=>0.3308516361499261} +10 ||| how long is der mississippi ||| ||| {:decoder=>-4.46637, :psb=>0.43542524047973125} +11 ||| is how long mississippi ||| ||| {:decoder=>-4.48856, :psb=>0.3258798048281462} +12 ||| like long is the mississippi ||| ||| {:decoder=>-4.55192, :psb=>0.6157842804860023} +13 ||| is as long the mississippi ||| ||| {:decoder=>-4.6871, :psb=>0.3308516361499261} +14 ||| mississippi how long is the ||| ||| {:decoder=>-4.68789, :psb=>0.6511126026643229} +>>> + + [TOP1] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8187307530779819} + + [HOPE] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8187307530779819}, #0 + + [FEAR] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8187307530779819}, #0 + +--- + iteration #2/3: 30 examples + type1 updates: 12 + type2 updates: 17 + top1 hits: 12 + top1 variant: 9 + top1 true variant: 9 + hope hits: 12 + hope variant: 10 + hope true variant: 10 + kbest size: 14.0 + 3.33% without translations (abs: 1) + 6.67% no good gold output (abs: 2) + + top1 with parse 96.67% abs=29.0 + top1 with output 96.67% abs=29.0 +top1 with correct output 70.0% adj=72.41 abs=21.0 + + hope with parse 96.67% abs=29.0 + hope with output 96.67% abs=29.0 +hope with correct output 73.33% adj=75.86 abs=22.0 + + fear with parse 96.67% abs=29.0 + fear with output 96.67% abs=29.0 +fear with correct output 46.67% adj=48.28 abs=14.0 + + + +================= + EXAMPLE: 0 + REFERENCE: give me the cities in virginia + GOLD MRL: answer(city(loc_2(stateid('virginia')))) +GOLD OUTPUT: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + +<<< KBEST +0 ||| give me the cities in virginia ||| ||| {:decoder=>-4.90046, :psb=>1.0} +1 ||| the cities in virginia give me ||| ||| {:decoder=>-5.29943, :psb=>0.7071067811865476} +2 ||| give me cities in virginia ||| ||| {:decoder=>-5.30919, :psb=>0.49473859088183875} +3 ||| give me the towns in virginia ||| ||| {:decoder=>-5.39031, :psb=>0.48549177170732344} +4 ||| unto me the cities in virginia ||| ||| {:decoder=>-5.41471, :psb=>0.8034284189446518} +5 ||| me give the cities in virginia ||| ||| {:decoder=>-5.43012, :psb=>0.668740304976422} +6 ||| give me the cities , virginia in ||| ||| {:decoder=>-5.4455, :psb=>0.559450014366235} +7 ||| give unto me the cities in virginia ||| ||| {:decoder=>-5.45788, :psb=>0.7034711503007025} +8 ||| give the cities in virginia me ||| ||| {:decoder=>-5.4907, :psb=>0.668740304976422} +9 ||| me the cities in virginia give ||| ||| {:decoder=>-5.50409, :psb=>0.8408964152537146} +10 ||| unto me the towns in virginia ||| ||| {:decoder=>-5.96261, :psb=>0.35930411196308426} +11 ||| give unto me the towns in virginia ||| ||| {:decoder=>-6.00578, :psb=>0.3178289704467185} +12 ||| me the towns in virginia give ||| ||| {:decoder=>-6.052, :psb=>0.37991784282579627} +>>> + + [TOP1] + nrl: give me the cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: give me the cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: give me cities in virginia + mrl: answer(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>0.645057922434305, :psb=>0.49473859088183875}, #2 + +================= + EXAMPLE: 1 + REFERENCE: what are the high points of states surrounding mississippi + GOLD MRL: answer(high_point_1(state(next_to_2(stateid('mississippi'))))) +GOLD OUTPUT: [placeid('cheaha mountain'),placeid('clingmans dome'),placeid('driskill mountain'),placeid('magazine mountain')] +NO MT OUTPUT, skipping example + +================= + EXAMPLE: 2 + REFERENCE: name the rivers in arkansas + GOLD MRL: answer(river(loc_2(stateid('arkansas')))) +GOLD OUTPUT: [riverid(arkansas),riverid(mississippi),riverid(ouachita),riverid(red),riverid('st. francis'),riverid(white)] + +<<< KBEST +0 ||| call the rivers in arkansas ||| ||| {:decoder=>-4.75288, :psb=>0.7521206186172787} +1 ||| call rivers in the arkansas ||| ||| {:decoder=>-4.91936, :psb=>0.40410310093532464} +2 ||| the rivers in arkansas call ||| ||| {:decoder=>-5.05979, :psb=>0.7521206186172787} +3 ||| the rivers call in arkansas ||| ||| {:decoder=>-5.12949, :psb=>0.4472135954999579} +4 ||| the call rivers in arkansas ||| ||| {:decoder=>-5.1567, :psb=>0.5318295896944989} +5 ||| call the rivers into arkansas ||| ||| {:decoder=>-5.16957, :psb=>0.3760603093086393} +6 ||| the rivers in call arkansas ||| ||| {:decoder=>-5.20856, :psb=>0.5318295896944989} +7 ||| rivers in the arkansas call ||| ||| {:decoder=>-5.26077, :psb=>0.40410310093532464} +8 ||| rivers in the call arkansas ||| ||| {:decoder=>-5.268, :psb=>0.40410310093532464} +9 ||| call the rivers in the arkansas ||| ||| {:decoder=>-5.27856, :psb=>0.4272870063962341} +10 ||| rivers in call the arkansas ||| ||| {:decoder=>-5.32639, :psb=>0.40410310093532464} +11 ||| call the rivers , in arkansas ||| ||| {:decoder=>-5.34179, :psb=>0.35930411196308426} +12 ||| in arkansas call the rivers ||| ||| {:decoder=>-5.37467, :psb=>0.4472135954999579} +13 ||| call the rivers arkansas ||| ||| {:decoder=>-5.40607, :psb=>0.3894003915357024} +14 ||| call rivers in arkansas the ||| ||| {:decoder=>-5.51696, :psb=>0.5318295896944989} +15 ||| call rivers in the the arkansas ||| ||| {:decoder=>-5.633, :psb=>0.32466791547509893} +16 ||| call the rivers in in arkansas ||| ||| {:decoder=>-5.86784, :psb=>0.4591497693322866} +17 ||| rivers in the the arkansas call ||| ||| {:decoder=>-5.97442, :psb=>0.32466791547509893} +18 ||| the rivers in in arkansas call ||| ||| {:decoder=>-6.18241, :psb=>0.4591497693322866} +19 ||| the call rivers in in arkansas ||| ||| {:decoder=>-6.25475, :psb=>0.35930411196308426} +20 ||| call rivers in in arkansas the ||| ||| {:decoder=>-6.81386, :psb=>0.35930411196308426} +>>> + + [TOP1] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787} + + [HOPE] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787}, #0 + + [FEAR] + nrl: call the rivers in arkansas + mrl: city(loc_2(stateid('arkansas'))) + output: [cityid('fort smith',ar),cityid('little rock',ar),cityid('north little rock',ar)] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7521206186172787}, #0 + +================= + EXAMPLE: 3 + REFERENCE: can you tell me the capital of texas + GOLD MRL: answer(capital(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(austin,tx)] + +<<< KBEST +0 ||| can you tell me , what is the capital of texas ||| ||| {:decoder=>-5.98132, :psb=>0.5270027137504644} +1 ||| can you tell me what is the capital of texas ||| ||| {:decoder=>-6.16151, :psb=>0.5844356470407898} +2 ||| you can tell me , what is the capital of texas ||| ||| {:decoder=>-6.16317, :psb=>0.3852971889399676} +3 ||| you can tell me what the capital of texas is ||| ||| {:decoder=>-6.18376, :psb=>0.4272870063962341} +4 ||| can you tell me , what is the capital of the texas ||| ||| {:decoder=>-6.21355, :psb=>0.39458812555917666} +5 ||| can you tell me what the capital of texas is ||| ||| {:decoder=>-6.22759, :psb=>0.5844356470407898} +6 ||| can you tell me , was the capital of texas is ||| ||| {:decoder=>-6.23065, :psb=>0.5270027137504644} +7 ||| you can tell me what is the capital of texas ||| ||| {:decoder=>-6.28271, :psb=>0.4272870063962341} +8 ||| can you tell me , what is the capital city of texas ||| ||| {:decoder=>-6.28756, :psb=>0.3672056269893592} +9 ||| can you tell me , what the capital of texas is ||| ||| {:decoder=>-6.29963, :psb=>0.5270027137504644} +10 ||| you can tell me , what the capital of texas is ||| ||| {:decoder=>-6.3575, :psb=>0.3852971889399676} +11 ||| you can tell me , was the capital of texas is ||| ||| {:decoder=>-6.4125, :psb=>0.3852971889399676} +12 ||| you can tell me , what is the capital city of texas ||| ||| {:decoder=>-6.47983, :psb=>0.21200626759025185} +13 ||| can you tell me , what the capital city of texas is ||| ||| {:decoder=>-6.70185, :psb=>0.3672056269893592} +14 ||| you can tell me what the capital city of texas is ||| ||| {:decoder=>-6.78796, :psb=>0.2328254894667881} +15 ||| you can tell me , what the capital city of texas is ||| ||| {:decoder=>-6.88369, :psb=>0.21200626759025185} +>>> + + [TOP1] + nrl: can you tell me , what is the capital of texas + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.5270027137504644} + + [HOPE] + nrl: can you tell me , what is the capital of texas + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.5270027137504644}, #0 + + [FEAR] + nrl: can you tell me , what is the capital of texas + mrl: population_1(answer(capital(loc_2(stateid('texas'))))) + output: [345496] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.5270027137504644}, #0 + +================= + EXAMPLE: 4 + REFERENCE: could you tell me what is the highest point in the state of oregon + GOLD MRL: answer(highest(place(loc_2(state(stateid('oregon')))))) +GOLD OUTPUT: [placeid('mount hood')] + +<<< KBEST +0 ||| you could tell me which is the highest point in the state of oregon ||| ||| {:decoder=>-9.21284, :psb=>0.6985342056580097} +1 ||| could you tell me which is the highest point in the state of oregon ||| ||| {:decoder=>-9.3765, :psb=>0.7992975447228272} +2 ||| you could say which is the highest point in the state of oregon me ||| ||| {:decoder=>-9.41639, :psb=>0.6668954865619205} +3 ||| you could say me which is the highest point in the state of oregon ||| ||| {:decoder=>-9.4563, :psb=>0.6668954865619205} +4 ||| you could tell me which is the highest point in oregon state ||| ||| {:decoder=>-9.47219, :psb=>0.4090790395780627} +5 ||| you could tell me which the highest point in the state of oregon is ||| ||| {:decoder=>-9.60631, :psb=>0.633160218956179} +6 ||| you could tell me what is the highest point in the state of oregon ||| ||| {:decoder=>-9.60802, :psb=>0.8817192535122739} +7 ||| you could tell me what the highest point in the state of oregon is ||| ||| {:decoder=>-9.62761, :psb=>0.684694991173804} +8 ||| could you tell me which is the highest point in oregon state ||| ||| {:decoder=>-9.63584, :psb=>0.5227564628601822} +9 ||| you could say what me is the highest point in the state of oregon ||| ||| {:decoder=>-9.64509, :psb=>0.6803749333171202} +10 ||| you could tell me that the highest point in the state of oregon is ||| ||| {:decoder=>-9.69288, :psb=>0.633160218956179} +11 ||| you could tell me which is the highest point in the oregon state ||| ||| {:decoder=>-9.69908, :psb=>0.48507053375249726} +12 ||| you could tell me which oregon is the highest point in the state ||| ||| {:decoder=>-9.84646, :psb=>0.5550418054052764} +13 ||| you could tell me which of the highest point in the state of oregon is ||| ||| {:decoder=>-9.88721, :psb=>0.5885661912765424} +14 ||| you could tell me which the highest point in the state oregon is ||| ||| {:decoder=>-10.156, :psb=>0.48507053375249726} +15 ||| you could tell me what the highest point in the state oregon is ||| ||| {:decoder=>-10.1657, :psb=>0.535536059195526} +>>> + + [TOP1] + nrl: you could tell me which is the highest point in the state of oregon + mrl: answer(highest(place(loc_2(state(stateid('oregon')))))) + output: [placeid('mount hood')] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6985342056580097} + + [HOPE] + nrl: you could tell me which is the highest point in the state of oregon + mrl: answer(highest(place(loc_2(state(stateid('oregon')))))) + output: [placeid('mount hood')] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6985342056580097}, #0 + + [FEAR] + nrl: you could tell me which is the highest point in the state of oregon + mrl: answer(highest(place(loc_2(state(stateid('oregon')))))) + output: [placeid('mount hood')] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6985342056580097}, #0 + +================= + EXAMPLE: 5 + REFERENCE: give me all the states of usa + GOLD MRL: answer(state(loc_2(countryid('usa')))) +GOLD OUTPUT: [stateid(alabama),stateid(alaska),stateid(arizona),stateid(arkansas),stateid(california),stateid(colorado),stateid(connecticut),stateid(delaware),stateid('district of columbia'),stateid(florida),stateid(georgia),stateid(hawaii),stateid(idaho),stateid(illinois),stateid(indiana),stateid(iowa),stateid(kansas),stateid(kentucky),stateid(louisiana),stateid(maine),stateid(maryland),stateid(massachusetts),stateid(michigan),stateid(minnesota),stateid(mississippi),stateid(missouri),stateid(montana),stateid(nebraska),stateid(nevada),stateid('new hampshire'),stateid('new jersey'),stateid('new mexico'),stateid('new york'),stateid('north carolina'),stateid('north dakota'),stateid(ohio),stateid(oklahoma),stateid(oregon),stateid(pennsylvania),stateid('rhode island'),stateid('south carolina'),stateid('south dakota'),stateid(tennessee),stateid(texas),stateid(utah),stateid(vermont),stateid(virginia),stateid(washington),stateid('west virginia'),stateid(wisconsin),stateid(wyoming)] + +<<< KBEST +0 ||| call all states ||| ||| {:decoder=>-4.78129, :psb=>0.15218787864872976} +1 ||| all states call ||| ||| {:decoder=>-4.97326, :psb=>0.15218787864872976} +2 ||| call all states of the usa ||| ||| {:decoder=>-5.01221, :psb=>0.29059254080791846} +3 ||| all call states ||| ||| {:decoder=>-5.09643, :psb=>0.15218787864872976} +4 ||| call all states the usa ||| ||| {:decoder=>-5.12451, :psb=>0.22778068328450227} +5 ||| call all the usa ||| ||| {:decoder=>-5.13119, :psb=>0.23618327637050732} +6 ||| call all states of usa ||| ||| {:decoder=>-5.15816, :psb=>0.3564960350471317} +7 ||| call all states in the usa ||| ||| {:decoder=>-5.3356, :psb=>0.2310997417025822} +8 ||| call all united states of america ||| ||| {:decoder=>-5.42748, :psb=>0.2557539057896621} +>>> + + [TOP1] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976} + + [HOPE] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976}, #0 + + [FEAR] + nrl: call all states + mrl: city(state(all)) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.15218787864872976}, #0 + +================= + EXAMPLE: 6 + REFERENCE: give me the cities in texas + GOLD MRL: answer(city(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + +<<< KBEST +0 ||| tell me what cities in texas are ||| ||| {:decoder=>-5.18623, :psb=>0.35745672884978397} +1 ||| tell me what are cities in texas ||| ||| {:decoder=>-5.36006, :psb=>0.35745672884978397} +2 ||| tell me which are cities in texas ||| ||| {:decoder=>-5.55054, :psb=>0.35745672884978397} +3 ||| tell me what cities are in texas ||| ||| {:decoder=>-5.57587, :psb=>0.271608378656351} +4 ||| tell me which cities in texas are ||| ||| {:decoder=>-5.57615, :psb=>0.35745672884978397} +5 ||| tell me what cities in texas , ||| ||| {:decoder=>-5.68419, :psb=>0.35745672884978397} +6 ||| tell me what cities in are texas ||| ||| {:decoder=>-5.68769, :psb=>0.271608378656351} +7 ||| tell me are what cities in texas ||| ||| {:decoder=>-5.71578, :psb=>0.35745672884978397} +8 ||| tell me what cities in texas is ||| ||| {:decoder=>-5.72165, :psb=>0.35745672884978397} +9 ||| are tell me what cities in texas ||| ||| {:decoder=>-5.7417, :psb=>0.35745672884978397} +10 ||| tell me which cities are in texas ||| ||| {:decoder=>-5.77551, :psb=>0.271608378656351} +11 ||| tell me downloading cities in texas are ||| ||| {:decoder=>-5.85841, :psb=>0.35745672884978397} +12 ||| sag me what cities in texas are ||| ||| {:decoder=>-6.00996, :psb=>0.35745672884978397} +13 ||| sag me which cities in texas are ||| ||| {:decoder=>-6.16294, :psb=>0.35745672884978397} +>>> + + [TOP1] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397} + + [HOPE] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397}, #0 + + [FEAR] + nrl: tell me what cities in texas are + mrl: answer(city(loc_2(loc_2(stateid('texas'))))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.35745672884978397}, #0 + +================= + EXAMPLE: 7 + REFERENCE: give me the cities in usa + GOLD MRL: answer(city(loc_2(countryid('usa')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + +<<< KBEST +0 ||| call the cities of usa ||| ||| {:decoder=>-4.25718, :psb=>0.3078921402430011} +1 ||| call the cities of the usa ||| ||| {:decoder=>-4.30348, :psb=>0.3021375397356768} +2 ||| call the cities of the u.s. ||| ||| {:decoder=>-4.43573, :psb=>0.2730120862709067} +3 ||| call the cities of the united states ||| ||| {:decoder=>-4.53806, :psb=>0.2283945119649991} +4 ||| call the usa cities ||| ||| {:decoder=>-4.55436, :psb=>0.255014728746912} +5 ||| call the cities the usa ||| ||| {:decoder=>-4.59985, :psb=>0.3078921402430011} +6 ||| call the cities in the usa ||| ||| {:decoder=>-4.64782, :psb=>0.4272870063962341} +7 ||| call the cities in the united states ||| ||| {:decoder=>-4.64897, :psb=>0.33265096878635064} +8 ||| call cities of the usa ||| ||| {:decoder=>-4.93486, :psb=>0.25890539701513354} +>>> + + [TOP1] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011} + + [HOPE] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011}, #0 + + [FEAR] + nrl: call the cities of usa + mrl: city(loc_2(countryid('usa'))) + output: [cityid(abilene,tx),cityid(abingdon,pa),cityid(akron,oh),cityid(alameda,ca),cityid(albany,ga),cityid(albany,ny),cityid(albuquerque,nm),cityid(alexandria,va),cityid(alhambra,ca),cityid(allentown,pa),cityid(altoona,pa),cityid(amarillo,tx),cityid(anaheim,ca),cityid(anchorage,ak),cityid(anderson,in),cityid('ann arbor',mi),cityid(appleton,wi),cityid(arlington,tx),cityid(arlington,va),cityid('arlington heights',il),cityid(arvada,co),cityid(atlanta,ga),cityid(aurora,co),cityid(aurora,il),cityid(austin,tx),cityid(bakersfield,ca),cityid(baltimore,md),cityid('baton rouge',la),cityid(bayonne,nj),cityid(beaumont,tx),cityid(bellevue,wa),cityid(berkeley,ca),cityid(bethesda,md),cityid(bethlehem,pa),cityid(billings,mt),cityid(birmingham,al),cityid(bloomington,mn),cityid(boise,id),cityid(boston,ma),cityid(boulder,co),cityid(bridgeport,ct),cityid(bristol,ct),cityid('bristol township',pa),cityid(brockton,ma),cityid(brownsville,tx),cityid('buena park',ca),cityid(buffalo,ny),cityid(burbank,ca),cityid(cambridge,ma),cityid(camden,nj),cityid(canton,oh),cityid(carson,ca),cityid(casper,wy),cityid('cedar rapids',ia),cityid(champaign,il),cityid(charleston,sc),cityid(charleston,wv),cityid(charlotte,nc),cityid(chattanooga,tn),cityid(cheektowaga,ny),cityid('cherry hill',nj),cityid(chesapeake,va),cityid(chicago,il),cityid('chula vista',ca),cityid(cicero,il),cityid(cincinnati,oh),cityid('citrus heights',ca),cityid(clearwater,fl),cityid(cleveland,oh),cityid(clifton,nj),cityid(clinton,mi),cityid('colorado springs',co),cityid(columbia,mo),cityid(columbia,sc),cityid(columbus,ga),cityid(columbus,oh),cityid(compton,ca),cityid(concord,ca),cityid('corpus christi',tx),cityid('costa mesa',ca),cityid(cranston,ri),cityid(dallas,tx),cityid('daly city',ca),cityid(danbury,ct),cityid(davenport,ia),cityid(dayton,oh),cityid(dearborn,mi),cityid('dearborn heights',mi),cityid(decatur,il),cityid(denver,co),cityid('des moines',ia),cityid(detroit,mi),cityid(downey,ca),cityid(dubuque,ia),cityid(duluth,mn),cityid(dundalk,md),cityid(durham,nc),cityid('east los angeles',ca),cityid('east orange',nj),cityid(edison,nj),cityid('el cajon',ca),cityid('el monte',ca),cityid('el paso',tx),cityid(elgin,il),cityid(elizabeth,nj),cityid(elyria,oh),cityid(erie,pa),cityid(escondido,ca),cityid(euclid,oh),cityid(eugene,or),cityid(evanston,il),cityid(evansville,in),cityid(ewa,hi),cityid(fairfield,ca),cityid('fall river',ma),cityid(fargo,nd),cityid('farmington hills',mi),cityid(fayetteville,nc),cityid(flint,mi),cityid('fort collins',co),cityid('fort lauderdale',fl),cityid('fort smith',ar),cityid('fort wayne',in),cityid('fort worth',tx),cityid(framingham,ma),cityid(fremont,ca),cityid(fresno,ca),cityid(fullerton,ca),cityid(gainesville,fl),cityid('garden grove',ca),cityid(garland,tx),cityid(gary,in),cityid(glendale,az),cityid(glendale,ca),cityid('grand prairie',tx),cityid('grand rapids',mi),cityid('great falls',mt),cityid('green bay',wi),cityid(greensboro,nc),cityid(greenville,sc),cityid(greenwich,ct),cityid(hamilton,oh),cityid(hammond,in),cityid(hampton,va),cityid(hartford,ct),cityid(hayward,ca),cityid('high point',nc),cityid(hollywood,fl),cityid(honolulu,hi),cityid(houston,tx),cityid(huntington,wv),cityid('huntington beach',ca),cityid(huntsville,al),cityid(independence,mo),cityid(indianapolis,in),cityid(inglewood,ca),cityid(irondequoit,ny),cityid(irvine,ca),cityid(irving,tx),cityid(irvington,nj),cityid(jackson,ms),cityid(jacksonville,fl),cityid('jersey city',nj),cityid(joliet,il),cityid(kalamazoo,mi),cityid('kansas city',ks),cityid('kansas city',mo),cityid(kendall,fl),cityid(kenner,la),cityid(kenosha,wi),cityid(kettering,oh),cityid(knoxville,tn),cityid(koolaupoko,hi),cityid(lafayette,la),cityid('lake charles',la),cityid(lakewood,ca),cityid(lakewood,co),cityid(lakewood,oh),cityid(lansing,mi),cityid(laredo,tx),cityid(largo,fl),cityid('las vegas',nv),cityid(lawrence,ma),cityid(lawton,ok),cityid(levittown,ny),cityid(lexington,ky),cityid(lincoln,ne),cityid('little rock',ar),cityid(livonia,mi),cityid('long beach',ca),cityid(longview,tx),cityid(lorain,oh),cityid('los angeles',ca),cityid(louisville,ky),cityid(lowell,ma),cityid('lower merion',pa),cityid(lubbock,tx),cityid(lynchburg,va),cityid(lynn,ma),cityid(macon,ga),cityid(madison,wi),cityid(manchester,nh),cityid(mcallen,tx),cityid(medford,ma),cityid(memphis,tn),cityid(meriden,ct),cityid(mesa,az),cityid(mesquite,tx),cityid(metairie,la),cityid(miami,fl),cityid('miami beach',fl),cityid(middletown,nj),cityid(midland,tx),cityid(milwaukee,wi),cityid(minneapolis,mn),cityid(mobile,al),cityid(modesto,ca),cityid(monroe,la),cityid(montgomery,al),cityid('mount vernon',ny),cityid('mountain view',ca),cityid(muncie,in),cityid(nashua,nh),cityid(nashville,tn),cityid('new bedford',ma),cityid('new britain',ct),cityid('new haven',ct),cityid('new orleans',la),cityid('new rochelle',ny),cityid('new york',ny),cityid(newark,nj),cityid('newport beach',ca),cityid('newport news',va),cityid(newton,ma),cityid('niagara falls',ny),cityid(norfolk,va),cityid(norman,ok),cityid('north charleston',sc),cityid('north little rock',ar),cityid(norwalk,ca),cityid(norwalk,ct),cityid('oak lawn',il),cityid(oakland,ca),cityid(oceanside,ca),cityid(odessa,tx),cityid(ogden,ut),cityid('oklahoma city',ok),cityid(omaha,ne),cityid(ontario,ca),cityid(orange,ca),cityid(orlando,fl),cityid('overland park',ks),cityid(oxnard,ca),cityid(parma,oh),cityid(pasadena,ca),cityid(pasadena,tx),cityid(paterson,nj),cityid(pawtucket,ri),cityid('penn hills',pa),cityid(pensacola,fl),cityid(peoria,il),cityid(philadelphia,pa),cityid(phoenix,az),cityid(pittsburgh,pa),cityid(plano,tx),cityid(pomona,ca),cityid(pontiac,mi),cityid('port arthur',tx),cityid(portland,me),cityid(portland,or),cityid(portsmouth,va),cityid(providence,ri),cityid(provo,ut),cityid(pueblo,co),cityid(quincy,ma),cityid(racine,wi),cityid(raleigh,nc),cityid(reading,pa),cityid(redford,mi),cityid('redondo beach',ca),cityid(reno,nv),cityid(richardson,tx),cityid(richmond,ca),cityid(richmond,va),cityid(riverside,ca),cityid(roanoke,va),cityid(rochester,mn),cityid(rochester,ny),cityid(rockford,il),cityid('royal oak',mi),cityid(sacramento,ca),cityid(saginaw,mi),cityid(salem,or),cityid(salinas,ca),cityid('salt lake city',ut),cityid('san angelo',tx),cityid('san antonio',tx),cityid('san bernardino',ca),cityid('san diego',ca),cityid('san francisco',ca),cityid('san jose',ca),cityid('san leandro',ca),cityid('san mateo',ca),cityid('santa ana',ca),cityid('santa barbara',ca),cityid('santa clara',ca),cityid('santa monica',ca),cityid('santa rosa',ca),cityid(savannah,ga),cityid(schenectady,ny),cityid('scotts valley',ca),cityid(scottsdale,az),cityid(scranton,pa),cityid(seattle,wa),cityid(shreveport,la),cityid('silver spring',md),cityid('simi valley',ca),cityid('sioux city',ia),cityid('sioux falls',sd),cityid(skokie,il),cityid(somerville,ma),cityid('south bend',in),cityid('south gate',ca),cityid(southfield,mi),cityid(spokane,wa),cityid(springfield,il),cityid(springfield,ma),cityid(springfield,mo),cityid(springfield,oh),cityid('st. clair shores',mi),cityid('st. joseph',mo),cityid('st. louis',mo),cityid('st. paul',mn),cityid('st. petersburg',fl),cityid(stamford,ct),cityid('sterling heights',mi),cityid(stockton,ca),cityid(sunnyvale,ca),cityid(syracuse,ny),cityid(tacoma,wa),cityid(tallahassee,fl),cityid(tampa,fl),cityid(taylor,mi),cityid(tempe,az),cityid('terre haute',in),cityid('thousand oaks',ca),cityid(toledo,oh),cityid(topeka,ks),cityid(torrance,ca),cityid(trenton,nj),cityid(troy,mi),cityid(tucson,az),cityid(tulsa,ok),cityid(tuscaloosa,al),cityid(tyler,tx),cityid('upper darby',pa),cityid(utica,ny),cityid(vallejo,ca),cityid(ventura,ca),cityid('virginia beach',va),cityid(waco,tx),cityid(waltham,ma),cityid(warren,mi),cityid(warwick,ri),cityid(washington,dc),cityid(waterbury,ct),cityid(waterford,mi),cityid(waterloo,ia),cityid(waukegan,il),cityid('west allis',wi),cityid('west covina',ca),cityid('west hartford',ct),cityid('west palm beach',fl),cityid('west valley',ut),cityid(westland,mi),cityid(westminster,ca),cityid(whittier,ca),cityid(wichita,ks),cityid('wichita falls',tx),cityid(wilmington,de),cityid('winston-salem',nc),cityid(woodbridge,nj),cityid(worcester,ma),cityid(wyoming,mi),cityid(yonkers,ny),cityid(youngstown,oh)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3078921402430011}, #0 + +================= + EXAMPLE: 8 + REFERENCE: give me the cities in virginia + GOLD MRL: answer(city(loc_2(stateid('virginia')))) +GOLD OUTPUT: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + +<<< KBEST +0 ||| call me the cities in virginia ||| ||| {:decoder=>-4.93827, :psb=>0.8034284189446518} +1 ||| the cities in virginia call me ||| ||| {:decoder=>-5.31284, :psb=>0.6389431042462724} +2 ||| the call me cities in virginia ||| ||| {:decoder=>-5.31367, :psb=>0.45180100180492244} +3 ||| call the cities in virginia me ||| ||| {:decoder=>-5.42424, :psb=>0.6389431042462724} +4 ||| call me cities in virginia ||| ||| {:decoder=>-5.43661, :psb=>0.43542524047973125} +5 ||| me call the cities in virginia ||| ||| {:decoder=>-5.47804, :psb=>0.6389431042462724} +6 ||| call me the towns in virginia ||| ||| {:decoder=>-5.49818, :psb=>0.35930411196308426} +7 ||| call the me cities in virginia ||| ||| {:decoder=>-5.52764, :psb=>0.45180100180492244} +8 ||| call me the cities virginia ||| ||| {:decoder=>-5.55828, :psb=>0.43542524047973125} +9 ||| call me in the cities virginia ||| ||| {:decoder=>-5.56823, :psb=>0.34329452398451965} +10 ||| me the cities in virginia call ||| ||| {:decoder=>-5.68903, :psb=>0.8034284189446518} +11 ||| in virginia call me the cities ||| ||| {:decoder=>-5.68952, :psb=>0.48549177170732344} +12 ||| me the call cities in virginia ||| ||| {:decoder=>-5.70501, :psb=>0.48549177170732344} +13 ||| cities in virginia call me the ||| ||| {:decoder=>-5.76462, :psb=>0.48549177170732344} +14 ||| me the towns in virginia call ||| ||| {:decoder=>-6.26143, :psb=>0.35930411196308426} +>>> + + [TOP1] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: call me the cities in virginia + mrl: city(city(loc_2(stateid('virginia')))) + output: [cityid(alexandria,va),cityid(arlington,va),cityid(chesapeake,va),cityid(hampton,va),cityid(lynchburg,va),cityid('newport news',va),cityid(norfolk,va),cityid(portsmouth,va),cityid(richmond,va),cityid(roanoke,va),cityid('virginia beach',va)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 9 + REFERENCE: give me the cities which are in texas + GOLD MRL: answer(city(loc_2(stateid('texas')))) +GOLD OUTPUT: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + +<<< KBEST +0 ||| call me cities in texas ||| ||| {:decoder=>-4.23114, :psb=>0.221776483974985} +1 ||| cities in texas call me ||| ||| {:decoder=>-4.66326, :psb=>0.221776483974985} +2 ||| call me in texas cities ||| ||| {:decoder=>-4.7176, :psb=>0.221776483974985} +3 ||| me call cities in texas ||| ||| {:decoder=>-4.80532, :psb=>0.221776483974985} +4 ||| call cities in texas me ||| ||| {:decoder=>-4.80659, :psb=>0.221776483974985} +5 ||| me cities in texas call ||| ||| {:decoder=>-4.83894, :psb=>0.221776483974985} +6 ||| call me towns in texas ||| ||| {:decoder=>-4.88967, :psb=>0.20638627362169998} +7 ||| cities in call me texas ||| ||| {:decoder=>-4.9023, :psb=>0.18649105036213778} +8 ||| cities call me in texas ||| ||| {:decoder=>-4.94992, :psb=>0.221776483974985} +9 ||| call i cities in texas ||| ||| {:decoder=>-4.97096, :psb=>0.20638627362169998} +10 ||| call cities in me texas ||| ||| {:decoder=>-4.99825, :psb=>0.18649105036213778} +11 ||| in texas call me cities ||| ||| {:decoder=>-5.00417, :psb=>0.221776483974985} +12 ||| cities in texas me call ||| ||| {:decoder=>-5.07199, :psb=>0.221776483974985} +>>> + + [TOP1] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985} + + [HOPE] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985}, #0 + + [FEAR] + nrl: call me cities in texas + mrl: city(city(loc_2(stateid('texas')))) + output: [cityid(abilene,tx),cityid(amarillo,tx),cityid(arlington,tx),cityid(austin,tx),cityid(beaumont,tx),cityid(brownsville,tx),cityid('corpus christi',tx),cityid(dallas,tx),cityid('el paso',tx),cityid('fort worth',tx),cityid(garland,tx),cityid('grand prairie',tx),cityid(houston,tx),cityid(irving,tx),cityid(laredo,tx),cityid(longview,tx),cityid(lubbock,tx),cityid(mcallen,tx),cityid(mesquite,tx),cityid(midland,tx),cityid(odessa,tx),cityid(pasadena,tx),cityid(plano,tx),cityid('port arthur',tx),cityid(richardson,tx),cityid('san angelo',tx),cityid('san antonio',tx),cityid(tyler,tx),cityid(waco,tx),cityid('wichita falls',tx)] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.221776483974985}, #0 + +================= + EXAMPLE: 10 + REFERENCE: give me the lakes in california + GOLD MRL: answer(lake(loc_2(stateid('california')))) +GOLD OUTPUT: [] + +<<< KBEST +0 ||| call me the lakes in california ||| ||| {:decoder=>-4.84491, :psb=>0.8034284189446518} +1 ||| the lakes in california call me ||| ||| {:decoder=>-5.16499, :psb=>0.6389431042462724} +2 ||| call the lakes in california me ||| ||| {:decoder=>-5.31599, :psb=>0.6389431042462724} +3 ||| the call me lakes in california ||| ||| {:decoder=>-5.33129, :psb=>0.45180100180492244} +4 ||| me call the lakes in california ||| ||| {:decoder=>-5.33979, :psb=>0.6389431042462724} +5 ||| the lakes call me in california ||| ||| {:decoder=>-5.34005, :psb=>0.37991784282579627} +6 ||| call me the lakes , california in ||| ||| {:decoder=>-5.44488, :psb=>0.3779644730092272} +7 ||| call the lakes me in california ||| ||| {:decoder=>-5.48682, :psb=>0.37991784282579627} +8 ||| me the lakes in california call ||| ||| {:decoder=>-5.50098, :psb=>0.8034284189446518} +9 ||| call me lakes in the california ||| ||| {:decoder=>-5.5134, :psb=>0.34329452398451965} +10 ||| call me of the lakes in california ||| ||| {:decoder=>-5.51503, :psb=>0.5345224838248488} +11 ||| in california call me the lakes ||| ||| {:decoder=>-5.56098, :psb=>0.48549177170732344} +12 ||| call the me lakes in california ||| ||| {:decoder=>-5.56826, :psb=>0.45180100180492244} +13 ||| call me in california the lakes ||| ||| {:decoder=>-5.57408, :psb=>0.37991784282579627} +14 ||| call me the in california lakes ||| ||| {:decoder=>-5.60393, :psb=>0.37991784282579627} +15 ||| call me the lakes , in california ||| ||| {:decoder=>-5.6148, :psb=>0.40614925799324625} +16 ||| call me the the lakes in california ||| ||| {:decoder=>-5.62073, :psb=>0.5651887140592688} +17 ||| me the call lakes in california ||| ||| {:decoder=>-5.74564, :psb=>0.48549177170732344} +18 ||| me the the lakes in california call ||| ||| {:decoder=>-6.27679, :psb=>0.5651887140592688} +>>> + + [TOP1] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: call me the lakes in california + mrl: city(lake(loc_2(stateid('california')))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 11 + REFERENCE: give me the largest state + GOLD MRL: answer(largest(state(all))) +GOLD OUTPUT: [stateid(alaska)] + +<<< KBEST +0 ||| which city is the largest ||| ||| {:decoder=>-3.60384, :psb=>0.33980884896942454} +1 ||| what is the largest city ||| ||| {:decoder=>-3.6763, :psb=>0.33980884896942454} +2 ||| what city is the largest ||| ||| {:decoder=>-3.72761, :psb=>0.33980884896942454} +3 ||| what town is the largest ||| ||| {:decoder=>-3.76011, :psb=>0.33980884896942454} +4 ||| which is the largest city ||| ||| {:decoder=>-3.82416, :psb=>0.33980884896942454} +5 ||| what city is the biggest ||| ||| {:decoder=>-3.85311, :psb=>0.24028114141347542} +6 ||| which city is the biggest ||| ||| {:decoder=>-3.85609, :psb=>0.24028114141347542} +7 ||| downloading city is the largest ||| ||| {:decoder=>-3.94522, :psb=>0.33980884896942454} +8 ||| which town is the largest ||| ||| {:decoder=>-3.97893, :psb=>0.33980884896942454} +9 ||| downloading is the largest city ||| ||| {:decoder=>-4.17845, :psb=>0.33980884896942454} +>>> + + [TOP1] + nrl: which city is the largest + mrl: answer(city(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454} + + [HOPE] + nrl: which city is the largest + mrl: answer(city(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454}, #0 + + [FEAR] + nrl: which city is the largest + mrl: answer(city(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.33980884896942454}, #0 + +================= + EXAMPLE: 12 + REFERENCE: give me the longest river that passes through the us + GOLD MRL: answer(longest(river(traverse_2(countryid('usa'))))) +GOLD OUTPUT: [] + +<<< KBEST +0 ||| call me the longest river , which flows through the usa ||| ||| {:decoder=>-8.23459, :psb=>0.3585594362259136} +1 ||| call me the longest river , which flows through the u.s. ||| ||| {:decoder=>-8.35556, :psb=>0.3585594362259136} +2 ||| call me the longest river which flows through the usa ||| ||| {:decoder=>-8.55136, :psb=>0.3976353643835253} +3 ||| the longest river , which flows through the usa call me ||| ||| {:decoder=>-8.572, :psb=>0.257663759638272} +4 ||| call me , the longest river , which flows through the usa ||| ||| {:decoder=>-8.67255, :psb=>0.23462350320528} +5 ||| call the longest river , which flows through the usa me ||| ||| {:decoder=>-8.67567, :psb=>0.257663759638272} +6 ||| call me the longest river , which flows by the usa ||| ||| {:decoder=>-8.68234, :psb=>0.32399502498695193} +7 ||| call me the longest river , which flows through the united states ||| ||| {:decoder=>-8.70065, :psb=>0.3264971028628052} +8 ||| me call the longest river , which flows through the usa ||| ||| {:decoder=>-8.72947, :psb=>0.257663759638272} +9 ||| the longest river which flows through the usa call me ||| ||| {:decoder=>-8.88877, :psb=>0.28574404296988} +10 ||| call me the longest river that flows through the usa ||| ||| {:decoder=>-8.97901, :psb=>0.5143686723610401} +11 ||| call the longest river which flows through the usa me ||| ||| {:decoder=>-8.99243, :psb=>0.28574404296988} +12 ||| me call the longest river which flows through the usa ||| ||| {:decoder=>-9.04623, :psb=>0.28574404296988} +13 ||| the longest river that flows through the usa call me ||| ||| {:decoder=>-9.35138, :psb=>0.41325840918969} +14 ||| call the longest river that flows through the usa me ||| ||| {:decoder=>-9.45505, :psb=>0.41325840918969} +15 ||| me call the longest river that flows through the usa ||| ||| {:decoder=>-9.50885, :psb=>0.41325840918969} +>>> + + [TOP1] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136} + + [HOPE] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136}, #0 + + [FEAR] + nrl: call me the longest river , which flows through the usa + mrl: city(answer(longest(river(traverse_2(countryid('usa')))))) + output: [] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.3585594362259136}, #0 + +================= + EXAMPLE: 13 + REFERENCE: how big is alaska + GOLD MRL: answer(size(stateid('alaska'))) +GOLD OUTPUT: [591000.0] + +<<< KBEST +0 ||| how big is alaska ||| ||| {:decoder=>-3.352, :psb=>1.0} +1 ||| how large is alaska ||| ||| {:decoder=>-3.66406, :psb=>0.49999999999999994} +2 ||| how great is alaska ||| ||| {:decoder=>-3.79165, :psb=>0.49999999999999994} +3 ||| how big alaska is ||| ||| {:decoder=>-3.9478, :psb=>0.537284965911771} +4 ||| as large is alaska ||| ||| {:decoder=>-3.98806, :psb=>0.4518010018049224} +5 ||| is how large alaska ||| ||| {:decoder=>-4.00369, :psb=>0.42044820762685725} +6 ||| as big is alaska ||| ||| {:decoder=>-4.0091, :psb=>0.6580370064762462} +7 ||| is how big alaska ||| ||| {:decoder=>-4.03758, :psb=>0.537284965911771} +8 ||| how large alaska is ||| ||| {:decoder=>-4.05471, :psb=>0.42044820762685725} +9 ||| is as large alaska ||| ||| {:decoder=>-4.10908, :psb=>0.37991784282579627} +10 ||| is as big alaska ||| ||| {:decoder=>-4.18126, :psb=>0.42044820762685725} +11 ||| how great alaska is ||| ||| {:decoder=>-4.20096, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how large is alaska + mrl: answer(size(stateid('alaska'))) + output: [591000.0] + correct?: true + SCORES: {:decoder=>0.6324208443271767, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 14 + REFERENCE: how big is massachusetts + GOLD MRL: answer(size(stateid('massachusetts'))) +GOLD OUTPUT: [8284.0] + +<<< KBEST +0 ||| how big is massachusetts ||| ||| {:decoder=>-3.87508, :psb=>1.0} +1 ||| how great is massachusetts ||| ||| {:decoder=>-4.31177, :psb=>0.49999999999999994} +2 ||| how large is massachusetts ||| ||| {:decoder=>-4.34604, :psb=>0.49999999999999994} +3 ||| how big is , massachusetts ||| ||| {:decoder=>-4.54065, :psb=>0.5318295896944989} +4 ||| how great massachusetts ||| ||| {:decoder=>-4.59867, :psb=>0.41368954504257255} +5 ||| how much is massachusetts ||| ||| {:decoder=>-4.61509, :psb=>0.49999999999999994} +6 ||| just how big is massachusetts ||| ||| {:decoder=>-4.61696, :psb=>0.7521206186172787} +7 ||| is how large massachusetts ||| ||| {:decoder=>-4.68735, :psb=>0.42044820762685725} +8 ||| is how big massachusetts ||| ||| {:decoder=>-4.71459, :psb=>0.537284965911771} +9 ||| how big massachusetts is ||| ||| {:decoder=>-4.74691, :psb=>0.537284965911771} +10 ||| is as large massachusetts ||| ||| {:decoder=>-4.79783, :psb=>0.37991784282579627} +11 ||| as big is massachusetts ||| ||| {:decoder=>-4.82551, :psb=>0.6580370064762462} +12 ||| how big , massachusetts ||| ||| {:decoder=>-4.84744, :psb=>0.49999999999999994} +13 ||| is as big massachusetts ||| ||| {:decoder=>-4.86349, :psb=>0.42044820762685725} +14 ||| as large is massachusetts ||| ||| {:decoder=>-4.86998, :psb=>0.4518010018049224} +15 ||| how large massachusetts is ||| ||| {:decoder=>-4.88264, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is massachusetts + mrl: answer(size(stateid('massachusetts'))) + output: [8284.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is massachusetts + mrl: answer(loc_1(stateid('massachusetts'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.5665866052642027, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 15 + REFERENCE: how big is new mexico + GOLD MRL: answer(size(stateid('new mexico'))) +GOLD OUTPUT: [121600.0] + +<<< KBEST +0 ||| how big is new mexico ||| ||| {:decoder=>-4.61619, :psb=>1.0} +1 ||| how large is new mexico ||| ||| {:decoder=>-5.05484, :psb=>0.5318295896944989} +2 ||| how great is new mexico ||| ||| {:decoder=>-5.09129, :psb=>0.5318295896944989} +3 ||| how big is mexico ||| ||| {:decoder=>-5.09245, :psb=>0.5506953149031837} +4 ||| how great new mexico ||| ||| {:decoder=>-5.34872, :psb=>0.3894003915357024} +5 ||| how much is new mexico ||| ||| {:decoder=>-5.36, :psb=>0.5318295896944989} +6 ||| just how big is new mexico ||| ||| {:decoder=>-5.41215, :psb=>0.8034284189446518} +7 ||| how big new mexico is ||| ||| {:decoder=>-5.45388, :psb=>0.47287080450158786} +8 ||| new mexico how big is ||| ||| {:decoder=>-5.4629, :psb=>0.6042750794713536} +9 ||| is how large new mexico ||| ||| {:decoder=>-5.49776, :psb=>0.40410310093532464} +10 ||| how large new mexico is ||| ||| {:decoder=>-5.54671, :psb=>0.40410310093532464} +11 ||| is how big new mexico ||| ||| {:decoder=>-5.55166, :psb=>0.47287080450158786} +12 ||| how great is mexico ||| ||| {:decoder=>-5.56797, :psb=>0.32744539334076506} +13 ||| how large is mexico ||| ||| {:decoder=>-5.59306, :psb=>0.32744539334076506} +14 ||| as big is new mexico ||| ||| {:decoder=>-5.59339, :psb=>0.7521206186172787} +15 ||| is as large new mexico ||| ||| {:decoder=>-5.61138, :psb=>0.3760603093086393} +16 ||| how great new mexico is ||| ||| {:decoder=>-5.64434, :psb=>0.40410310093532464} +17 ||| is as big new mexico ||| ||| {:decoder=>-5.70361, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how big is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how large is new mexico + mrl: answer(size(stateid('new mexico'))) + output: [121600.0] + correct?: true + SCORES: {:decoder=>0.5966140037887839, :psb=>0.5318295896944989}, #1 + +================= + EXAMPLE: 16 + REFERENCE: how big is north dakota + GOLD MRL: answer(size(stateid('north dakota'))) +GOLD OUTPUT: [70700.0] + +<<< KBEST +0 ||| how big is north dakota ||| ||| {:decoder=>-4.82486, :psb=>1.0} +1 ||| how great is north dakota ||| ||| {:decoder=>-5.2808, :psb=>0.5318295896944989} +2 ||| how large is north dakota ||| ||| {:decoder=>-5.43474, :psb=>0.5318295896944989} +3 ||| how much is north dakota ||| ||| {:decoder=>-5.52132, :psb=>0.5318295896944989} +4 ||| how great north dakota ||| ||| {:decoder=>-5.58252, :psb=>0.3894003915357024} +5 ||| just how big is north dakota ||| ||| {:decoder=>-5.59917, :psb=>0.8034284189446518} +6 ||| what is north dakota ||| ||| {:decoder=>-5.74702, :psb=>0.5124797359336637} +7 ||| how big north dakota is ||| ||| {:decoder=>-5.81288, :psb=>0.47287080450158786} +8 ||| is how big north dakota ||| ||| {:decoder=>-5.91808, :psb=>0.47287080450158786} +9 ||| is how large north dakota ||| ||| {:decoder=>-5.92961, :psb=>0.40410310093532464} +10 ||| as big is north dakota ||| ||| {:decoder=>-5.95059, :psb=>0.7521206186172787} +11 ||| how large north dakota is ||| ||| {:decoder=>-5.97472, :psb=>0.40410310093532464} +12 ||| is as large north dakota ||| ||| {:decoder=>-6.04774, :psb=>0.3760603093086393} +13 ||| how great north dakota is ||| ||| {:decoder=>-6.05242, :psb=>0.40410310093532464} +14 ||| is as big north dakota ||| ||| {:decoder=>-6.07447, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how big is north dakota + mrl: answer(size(stateid('north dakota'))) + output: [70700.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is north dakota + mrl: answer(size(stateid('north dakota'))) + output: [70700.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is north dakota + mrl: answer(loc_1(stateid('north dakota'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.6351341618584997, :psb=>0.5318295896944989}, #1 + +================= + EXAMPLE: 17 + REFERENCE: how big is texas + GOLD MRL: answer(size(stateid('texas'))) +GOLD OUTPUT: [266807.0] + +<<< KBEST +0 ||| how big is texas ||| ||| {:decoder=>-4.27286, :psb=>1.0} +1 ||| how great is texas ||| ||| {:decoder=>-4.77554, :psb=>0.49999999999999994} +2 ||| how large is texas ||| ||| {:decoder=>-4.92137, :psb=>0.49999999999999994} +3 ||| how much is texas ||| ||| {:decoder=>-4.97271, :psb=>0.49999999999999994} +4 ||| how great texas ||| ||| {:decoder=>-5.00691, :psb=>0.41368954504257255} +5 ||| just how big is texas ||| ||| {:decoder=>-5.07066, :psb=>0.7521206186172787} +6 ||| how big is of texas ||| ||| {:decoder=>-5.11615, :psb=>0.5318295896944989} +7 ||| what is texas ||| ||| {:decoder=>-5.19072, :psb=>0.4919625503668659} +8 ||| like big is texas ||| ||| {:decoder=>-5.25398, :psb=>0.6580370064762462} +9 ||| how big texas is ||| ||| {:decoder=>-5.27452, :psb=>0.537284965911771} +10 ||| is how large texas ||| ||| {:decoder=>-5.30358, :psb=>0.42044820762685725} +11 ||| is how big texas ||| ||| {:decoder=>-5.3345, :psb=>0.537284965911771} +12 ||| how large texas is ||| ||| {:decoder=>-5.41049, :psb=>0.42044820762685725} +13 ||| is as large texas ||| ||| {:decoder=>-5.42496, :psb=>0.37991784282579627} +14 ||| as big is texas ||| ||| {:decoder=>-5.42542, :psb=>0.6580370064762462} +15 ||| is as big texas ||| ||| {:decoder=>-5.49392, :psb=>0.42044820762685725} +16 ||| how great texas is ||| ||| {:decoder=>-5.54629, :psb=>0.42044820762685725} +>>> + + [TOP1] + nrl: how big is texas + mrl: answer(size(stateid('texas'))) + output: [266807.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is texas + mrl: answer(size(stateid('texas'))) + output: [266807.0] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how great is texas + mrl: answer(loc_1(stateid('texas'))) + output: [countryid(usa)] + correct?: false + SCORES: {:decoder=>0.6052550984349351, :psb=>0.49999999999999994}, #1 + +================= + EXAMPLE: 18 + REFERENCE: how big is the city of new york + GOLD MRL: answer(size(city(cityid('new york', _)))) +GOLD OUTPUT: [7071639] + +<<< KBEST +0 ||| how big is the city of new york ||| ||| {:decoder=>-5.35187, :psb=>1.0} +1 ||| how big is the new york city ||| ||| {:decoder=>-5.40803, :psb=>0.5329462628216856} +2 ||| how much is the new york city ||| ||| {:decoder=>-5.60488, :psb=>0.2883677731713749} +3 ||| how big is the york city ||| ||| {:decoder=>-5.83406, :psb=>0.4791733671582712} +4 ||| how great is the city of new york ||| ||| {:decoder=>-5.89202, :psb=>0.7476743906106103} +5 ||| how large is the new york city ||| ||| {:decoder=>-5.96135, :psb=>0.2883677731713749} +6 ||| how large is the city of new york ||| ||| {:decoder=>-6.0681, :psb=>0.7476743906106103} +7 ||| how great is the new york city ||| ||| {:decoder=>-6.13984, :psb=>0.2883677731713749} +8 ||| how big is the new york city 's ||| ||| {:decoder=>-6.14515, :psb=>0.528685631720282} +9 ||| just how big is the city of new york ||| ||| {:decoder=>-6.21917, :psb=>0.8773826753016616} +>>> + + [TOP1] + nrl: how big is the city of new york + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how big is the city of new york + mrl: answer(size(city(cityid('new york',_)))) + output: [7071639] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how much is the new york city + mrl: answer(loc_1(cityid('new york',_))) + output: [countryid(usa),stateid('new york')] + correct?: false + SCORES: {:decoder=>0.708278565663554, :psb=>0.2883677731713749}, #2 + +================= + EXAMPLE: 19 + REFERENCE: how high are the highest points of all the states + GOLD MRL: answer(elevation_1(highest(place(loc_2(state(all)))))) +GOLD OUTPUT: [6194] + +<<< KBEST +0 ||| how much are the highest surveys all states ||| ||| {:decoder=>-8.95102, :psb=>0.27534765745159184} +1 ||| what are the highest surveys all states ||| ||| {:decoder=>-8.95549, :psb=>0.257703362342899} +2 ||| how high is the highest surveys all states ||| ||| {:decoder=>-8.9672, :psb=>0.2315388580995513} +3 ||| how high are the highest surveys all states ||| ||| {:decoder=>-9.01094, :psb=>0.5062667121584363} +4 ||| how high is the highest surveys of all states ||| ||| {:decoder=>-9.11773, :psb=>0.25933688537080213} +5 ||| all states how high are the highest surveys ||| ||| {:decoder=>-9.14587, :psb=>0.5062667121584363} +6 ||| how high are the highest surveys of all states ||| ||| {:decoder=>-9.16148, :psb=>0.5501366107724776} +7 ||| how high all states are the highest surveys ||| ||| {:decoder=>-9.20504, :psb=>0.30592435772324006} +8 ||| all states how high is the highest surveys ||| ||| {:decoder=>-9.22113, :psb=>0.2315388580995513} +9 ||| how high are the highest peaks all states ||| ||| {:decoder=>-9.23844, :psb=>0.5062667121584363} +10 ||| how high are the highest surveys , all states ||| ||| {:decoder=>-9.2844, :psb=>0.5103723117878854} +11 ||| as highly are the highest surveys all states ||| ||| {:decoder=>-9.36742, :psb=>0.26493826542476406} +12 ||| how highly are the highest surveys all states ||| ||| {:decoder=>-9.39154, :psb=>0.27534765745159184} +13 ||| what are the highest uplift all states ||| ||| {:decoder=>-9.6381, :psb=>0.257703362342899} +14 ||| how high is the highest uplift all states ||| ||| {:decoder=>-9.64389, :psb=>0.2315388580995513} +15 ||| how high is the highest eminence all states ||| ||| {:decoder=>-9.64683, :psb=>0.2315388580995513} +16 ||| what are the highest eminence all states ||| ||| {:decoder=>-9.6507, :psb=>0.257703362342899} +17 ||| how high are the highest uplift all states ||| ||| {:decoder=>-9.68763, :psb=>0.5062667121584363} +18 ||| how high are the highest eminence all states ||| ||| {:decoder=>-9.69057, :psb=>0.5062667121584363} +19 ||| how high is the highest hills all states ||| ||| {:decoder=>-9.73746, :psb=>0.2315388580995513} +20 ||| what are the highest hills all states ||| ||| {:decoder=>-9.74134, :psb=>0.257703362342899} +21 ||| how high are the highest hills all states ||| ||| {:decoder=>-9.78121, :psb=>0.5062667121584363} +>>> + + [TOP1] + nrl: how much are the highest surveys all states + mrl: answer(highest(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.27534765745159184} + + [HOPE] + nrl: how high are the highest surveys all states + mrl: answer(elevation_1(highest(state(all)))) + output: [] + correct?: false + SCORES: {:decoder=>0.9278237511894868, :psb=>0.5062667121584363}, #3 + + [FEAR] + nrl: how much are the highest surveys all states + mrl: answer(highest(state(all))) + output: [] + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.27534765745159184}, #0 + +================= + EXAMPLE: 20 + REFERENCE: how high is guadalupe peak + GOLD MRL: answer(elevation_1(placeid('guadalupe peak'))) +GOLD OUTPUT: [2667] + +<<< KBEST +0 ||| how high is guadalupe peak ||| ||| {:decoder=>-5.25889, :psb=>1.0} +1 ||| how much is guadalupe peak ||| ||| {:decoder=>-5.63113, :psb=>0.5318295896944989} +2 ||| how high is located guadalupe peak ||| ||| {:decoder=>-5.87022, :psb=>0.48549177170732344} +3 ||| what is guadalupe peak ||| ||| {:decoder=>-5.89176, :psb=>0.5124797359336637} +4 ||| as high is guadalupe peak ||| ||| {:decoder=>-5.89329, :psb=>0.7521206186172787} +5 ||| how high is situated guadalupe peak ||| ||| {:decoder=>-5.92847, :psb=>0.48549177170732344} +6 ||| how tall is guadalupe peak ||| ||| {:decoder=>-5.95205, :psb=>0.5318295896944989} +7 ||| how highly is guadalupe peak ||| ||| {:decoder=>-5.99923, :psb=>0.5318295896944989} +8 ||| as highly is guadalupe peak ||| ||| {:decoder=>-6.00386, :psb=>0.4949232003839765} +9 ||| like high is guadalupe peak ||| ||| {:decoder=>-6.03796, :psb=>0.7521206186172787} +10 ||| how high lies guadalupe peak ||| ||| {:decoder=>-6.05851, :psb=>0.4472135954999579} +11 ||| how high located guadalupe peak ||| ||| {:decoder=>-6.07554, :psb=>0.4472135954999579} +12 ||| how high situated guadalupe peak ||| ||| {:decoder=>-6.10028, :psb=>0.4472135954999579} +13 ||| such as high is guadalupe peak ||| ||| {:decoder=>-6.12183, :psb=>0.6042750794713536} +14 ||| like highly is guadalupe peak ||| ||| {:decoder=>-6.17188, :psb=>0.4949232003839765} +15 ||| as high is located guadalupe peak ||| ||| {:decoder=>-6.52941, :psb=>0.35930411196308426} +16 ||| as high is situated guadalupe peak ||| ||| {:decoder=>-6.60485, :psb=>0.35930411196308426} +17 ||| like high is located guadalupe peak ||| ||| {:decoder=>-6.67408, :psb=>0.35930411196308426} +18 ||| like high is situated guadalupe peak ||| ||| {:decoder=>-6.74952, :psb=>0.35930411196308426} +19 ||| such as high is located guadalupe peak ||| ||| {:decoder=>-6.75795, :psb=>0.3005840818981024} +20 ||| such as high is situated guadalupe peak ||| ||| {:decoder=>-6.83338, :psb=>0.3005840818981024} +>>> + + [TOP1] + nrl: how high is guadalupe peak + mrl: answer(elevation_1(placeid('guadalupe peak'))) + output: [2667] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is guadalupe peak + mrl: answer(elevation_1(placeid('guadalupe peak'))) + output: [2667] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how much is guadalupe peak + mrl: answer(loc_1(placeid('guadalupe peak'))) + output: [countryid(usa),stateid(texas)] + correct?: false + SCORES: {:decoder=>0.763580587999924, :psb=>0.5318295896944989}, #1 + +================= + EXAMPLE: 21 + REFERENCE: how high is the highest point in montana + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) +GOLD OUTPUT: [3901] + +<<< KBEST +0 ||| how high is the highest point in montana ||| ||| {:decoder=>-5.28617, :psb=>1.0} +1 ||| how much is the highest point in montana ||| ||| {:decoder=>-5.64393, :psb=>0.7476743906106103} +2 ||| as high is the highest point in montana ||| ||| {:decoder=>-5.75403, :psb=>0.8599476570625982} +3 ||| how high is the highest point montana ||| ||| {:decoder=>-5.76096, :psb=>0.7536998328984837} +4 ||| like high is the highest point in montana ||| ||| {:decoder=>-5.83992, :psb=>0.8599476570625982} +5 ||| as highly is the highest point in montana ||| ||| {:decoder=>-5.95092, :psb=>0.719408902854813} +6 ||| like highly is the highest point in montana ||| ||| {:decoder=>-6.05081, :psb=>0.719408902854813} +7 ||| how highly is the highest point in montana ||| ||| {:decoder=>-6.11243, :psb=>0.7476743906106103} +8 ||| montana how high is the highest point in ||| ||| {:decoder=>-6.19152, :psb=>0.8891397050194614} +>>> + + [TOP1] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is the highest point in montana + mrl: answer(elevation_1(highest(place(loc_2(stateid('montana')))))) + output: [3901] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + +================= + EXAMPLE: 22 + REFERENCE: how high is the highest point in the largest state + GOLD MRL: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) +GOLD OUTPUT: [6194] + +<<< KBEST +0 ||| how high is the highest point of the largest state ||| ||| {:decoder=>-7.71391, :psb=>0.6999271023161167} +1 ||| how high lies the highest point of the largest state ||| ||| {:decoder=>-7.82842, :psb=>0.3760603093086394} +2 ||| as high is the highest point of the largest state ||| ||| {:decoder=>-7.85246, :psb=>0.5844356470407898} +3 ||| how high is the highest point of the biggest state ||| ||| {:decoder=>-7.87491, :psb=>0.6042750794713536} +4 ||| how high is the highest point of largest state ||| ||| {:decoder=>-7.98584, :psb=>0.6496350258549114} +5 ||| how high lies the highest point of the biggest state ||| ||| {:decoder=>-8.01327, :psb=>0.29697089145035693} +6 ||| as high lies the highest point of the largest state ||| ||| {:decoder=>-8.04457, :psb=>0.347507514861063} +7 ||| as high is the highest point of the biggest state ||| ||| {:decoder=>-8.05738, :psb=>0.49144984054308527} +8 ||| as highly lies the highest point of the largest state ||| ||| {:decoder=>-8.09543, :psb=>0.33437015248821106} +9 ||| as high is the highest point of largest state ||| ||| {:decoder=>-8.13951, :psb=>0.5341735956899846} +10 ||| like high is the highest point of the largest state ||| ||| {:decoder=>-8.20983, :psb=>0.5844356470407898} +11 ||| as high located the highest point of the largest state ||| ||| {:decoder=>-8.37807, :psb=>0.347507514861063} +>>> + + [TOP1] + nrl: how high is the highest point of the largest state + mrl: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) + output: [6194] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6999271023161167} + + [HOPE] + nrl: how high is the highest point of the largest state + mrl: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) + output: [6194] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6999271023161167}, #0 + + [FEAR] + nrl: how high is the highest point of the largest state + mrl: answer(elevation_1(highest(place(loc_2(largest(state(all))))))) + output: [6194] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.6999271023161167}, #0 + +================= + EXAMPLE: 23 + REFERENCE: how high is the highest point of delaware + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) +GOLD OUTPUT: [135] + +<<< KBEST +0 ||| how high is the highest point of delaware ||| ||| {:decoder=>-6.37084, :psb=>1.0} +1 ||| how high is located the highest point of delaware ||| ||| {:decoder=>-6.68632, :psb=>0.6559965570884768} +2 ||| how high is situated the highest point of delaware ||| ||| {:decoder=>-6.77427, :psb=>0.6559965570884768} +3 ||| as high is the highest point of delaware ||| ||| {:decoder=>-6.78307, :psb=>0.8599476570625982} +4 ||| like high is the highest point of delaware ||| ||| {:decoder=>-6.82648, :psb=>0.8599476570625982} +5 ||| how high is of the highest point of delaware ||| ||| {:decoder=>-6.82934, :psb=>0.6559965570884768} +6 ||| how high lies the highest point of delaware ||| ||| {:decoder=>-6.98391, :psb=>0.6580370064762462} +7 ||| how high located the highest point of delaware ||| ||| {:decoder=>-7.1696, :psb=>0.6580370064762462} +8 ||| how high is the highest point of of delaware ||| ||| {:decoder=>-7.34794, :psb=>0.8065978233496266} +9 ||| as high is the highest point of of delaware ||| ||| {:decoder=>-7.7287, :psb=>0.6817705815391745} +10 ||| like high is the highest point of of delaware ||| ||| {:decoder=>-7.86183, :psb=>0.6817705815391745} +11 ||| how high is situated the highest point of of delaware ||| ||| {:decoder=>-7.94157, :psb=>0.49944351611061033} +12 ||| how high is located the highest point of of delaware ||| ||| {:decoder=>-7.99414, :psb=>0.49944351611061033} +13 ||| how high lies the highest point of of delaware ||| ||| {:decoder=>-8.13092, :psb=>0.48549177170732344} +14 ||| how high is of the highest point of of delaware ||| ||| {:decoder=>-8.5008, :psb=>0.49944351611061033} +>>> + + [TOP1] + nrl: how high is the highest point of delaware + mrl: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) + output: [135] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point of delaware + mrl: answer(elevation_1(highest(place(loc_2(stateid('delaware')))))) + output: [135] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is located the highest point of delaware + mrl: answer(elevation_1(loc_1(highest(place(loc_2(stateid('delaware'))))))) + output: [] + correct?: false + SCORES: {:decoder=>0.8518845424327217, :psb=>0.6559965570884768}, #1 + +================= + EXAMPLE: 24 + REFERENCE: how high is the highest point of florida + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('florida')))))) +GOLD OUTPUT: [105] + +<<< KBEST +0 ||| how high is the highest point florida ||| ||| {:decoder=>-7.82412, :psb=>0.7536998328984837} +1 ||| how high is the highest point floridas ||| ||| {:decoder=>-7.93368, :psb=>0.7252065560578255} +2 ||| how high the highest point is florida ||| ||| {:decoder=>-8.18731, :psb=>0.3829795673743804} +3 ||| how much is the highest point florida ||| ||| {:decoder=>-8.24641, :psb=>0.48497485346901076} +4 ||| florida how high is the highest point ||| ||| {:decoder=>-8.29252, :psb=>0.7536998328984837} +5 ||| as high is the highest point florida ||| ||| {:decoder=>-8.33827, :psb=>0.6098235933075173} +6 ||| how high is situated the highest point florida ||| ||| {:decoder=>-8.39189, :psb=>0.4445698525097307} +7 ||| how high florida is the highest point ||| ||| {:decoder=>-8.45005, :psb=>0.5329462628216856} +8 ||| as high is the highest point floridas ||| ||| {:decoder=>-8.47784, :psb=>0.5826515567418346} +9 ||| like high is the highest point florida ||| ||| {:decoder=>-8.48712, :psb=>0.6098235933075173} +10 ||| how high is of the highest point florida ||| ||| {:decoder=>-8.51859, :psb=>0.45966135761245924} +11 ||| florida as high is the highest point ||| ||| {:decoder=>-8.57761, :psb=>0.6098235933075173} +12 ||| like high is the highest point floridas ||| ||| {:decoder=>-8.59668, :psb=>0.5826515567418346} +13 ||| how high is located the highest point florida ||| ||| {:decoder=>-8.63847, :psb=>0.4445698525097307} +14 ||| florida like high is the highest point ||| ||| {:decoder=>-8.64591, :psb=>0.6098235933075173} +15 ||| floridas how high is the highest point ||| ||| {:decoder=>-8.91756, :psb=>0.7252065560578255} +>>> + + [TOP1] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837} + + [HOPE] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837}, #0 + + [FEAR] + nrl: how high is the highest point florida + mrl: answer(elevation_1(highest(place('florida')))) + output: []. + correct?: false + SCORES: {:decoder=>1.0, :psb=>0.7536998328984837}, #0 + +================= + EXAMPLE: 25 + REFERENCE: how high is the highest point of louisiana + GOLD MRL: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) +GOLD OUTPUT: [163] + +<<< KBEST +0 ||| how high is the highest point of louisiana ||| ||| {:decoder=>-7.7586, :psb=>1.0} +1 ||| how much is the highest point of louisiana ||| ||| {:decoder=>-8.19942, :psb=>0.7476743906106103} +2 ||| as high is the highest point of louisiana ||| ||| {:decoder=>-8.30276, :psb=>0.8599476570625982} +3 ||| what is the highest point of louisiana ||| ||| {:decoder=>-8.33672, :psb=>0.7252065560578255} +4 ||| how high is located the highest point of louisiana ||| ||| {:decoder=>-8.3378, :psb=>0.6559965570884768} +5 ||| like high is the highest point of louisiana ||| ||| {:decoder=>-8.4216, :psb=>0.8599476570625982} +6 ||| how high is situated the highest point of louisiana ||| ||| {:decoder=>-8.4222, :psb=>0.6559965570884768} +7 ||| how high is the highest point from louisiana ||| ||| {:decoder=>-8.47028, :psb=>0.7476743906106103} +8 ||| how high is of the highest point of louisiana ||| ||| {:decoder=>-8.5489, :psb=>0.6559965570884768} +9 ||| louisiana how high is the highest point of ||| ||| {:decoder=>-8.78017, :psb=>0.8891397050194614} +10 ||| how high is the highest point of of louisiana ||| ||| {:decoder=>-8.97875, :psb=>0.8065978233496266} +11 ||| as louisiana high is the highest point of ||| ||| {:decoder=>-9.01218, :psb=>0.7476743906106103} +12 ||| as high is the highest point of of louisiana ||| ||| {:decoder=>-9.49666, :psb=>0.6817705815391745} +13 ||| how much is the highest point of of louisiana ||| ||| {:decoder=>-9.54731, :psb=>0.5773502691896258} +14 ||| like high is the highest point of of louisiana ||| ||| {:decoder=>-9.69801, :psb=>0.6817705815391745} +>>> + + [TOP1] + nrl: how high is the highest point of louisiana + mrl: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) + output: [163] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how high is the highest point of louisiana + mrl: answer(elevation_1(highest(place(loc_2(stateid('louisiana')))))) + output: [163] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how high is located the highest point of louisiana + mrl: answer(elevation_1(loc_1(highest(place(loc_2(stateid('louisiana'))))))) + output: [] + correct?: false + SCORES: {:decoder=>0.7013524731748318, :psb=>0.6559965570884768}, #4 + +================= + EXAMPLE: 26 + REFERENCE: how large is the largest city in alaska + GOLD MRL: answer(size(largest(city(loc_2(stateid('alaska')))))) +GOLD OUTPUT: [174431] + +<<< KBEST +0 ||| how big is the largest city in alaska ||| ||| {:decoder=>-7.30307, :psb=>0.7476743906106103} +1 ||| how big is the biggest city in alaska ||| ||| {:decoder=>-7.58876, :psb=>0.3655552228545123} +2 ||| how large is the largest city in alaska ||| ||| {:decoder=>-7.81022, :psb=>1.0} +3 ||| how great is the largest city in alaska ||| ||| {:decoder=>-7.84126, :psb=>0.7476743906106103} +4 ||| how big is the largest town in alaska ||| ||| {:decoder=>-7.87324, :psb=>0.3655552228545123} +5 ||| how great is the biggest city in alaska ||| ||| {:decoder=>-8.16696, :psb=>0.3655552228545123} +6 ||| how large is the biggest city in alaska ||| ||| {:decoder=>-8.22306, :psb=>0.5946035575013605} +>>> + + [TOP1] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103} + + [HOPE] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103}, #0 + + [FEAR] + nrl: how big is the largest city in alaska + mrl: answer(size(largest(city(loc_2(stateid('alaska')))))) + output: [174431] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.7476743906106103}, #0 + +================= + EXAMPLE: 27 + REFERENCE: how long is rio grande + GOLD MRL: answer(len(riverid('rio grande'))) +GOLD OUTPUT: [3033] + +<<< KBEST +0 ||| how long is rio grande ||| ||| {:decoder=>-5.3791, :psb=>1.0} +1 ||| how long rio grande ||| ||| {:decoder=>-5.51937, :psb=>0.4630777161991027} +2 ||| how long has rio grande ||| ||| {:decoder=>-5.7721, :psb=>0.4472135954999579} +3 ||| how long will rio grande ||| ||| {:decoder=>-6.00931, :psb=>0.4472135954999579} +4 ||| how long rio grande is ||| ||| {:decoder=>-6.11793, :psb=>0.47287080450158786} +5 ||| like lang is rio grande ||| ||| {:decoder=>-6.14756, :psb=>0.4949232003839765} +6 ||| rio grande is how long ||| ||| {:decoder=>-6.17474, :psb=>0.47287080450158786} +7 ||| like long is rio grande ||| ||| {:decoder=>-6.17789, :psb=>0.7521206186172787} +8 ||| how long is of rio grande ||| ||| {:decoder=>-6.222, :psb=>0.48549177170732344} +9 ||| how long , rio grande ||| ||| {:decoder=>-6.26844, :psb=>0.4472135954999579} +10 ||| is how long rio grande ||| ||| {:decoder=>-6.36224, :psb=>0.47287080450158786} +11 ||| rio grande is as long ||| ||| {:decoder=>-6.43718, :psb=>0.40410310093532464} +12 ||| as long is rio grande ||| ||| {:decoder=>-6.57339, :psb=>0.7521206186172787} +13 ||| like long rio grande ||| ||| {:decoder=>-6.60378, :psb=>0.3894003915357024} +14 ||| as lang is rio grande ||| ||| {:decoder=>-6.77637, :psb=>0.4949232003839765} +15 ||| is as long rio grande ||| ||| {:decoder=>-6.84056, :psb=>0.40410310093532464} +16 ||| as long rio grande is ||| ||| {:decoder=>-6.91337, :psb=>0.40410310093532464} +>>> + + [TOP1] + nrl: how long is rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0} + + [HOPE] + nrl: how long is rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>1.0, :psb=>1.0}, #0 + + [FEAR] + nrl: how long rio grande + mrl: answer(len(riverid('rio grande'))) + output: [3033] + correct?: true + SCORES: {:decoder=>0.9085754137146655, :psb=>0.4630777161991027}, #1 + +================= + EXAMPLE: 28 + REFERENCE: how long is the mississippi + GOLD MRL: answer(len(riverid('mississippi'))) +GOLD OUTPUT: [3778] + +<<< KBEST +0 ||| how long is the mississippi river ||| ||| {:decoder=>-5.99251, :psb=>0.8034284189446518} +1 ||| how long the mississippi river ||| ||| {:decoder=>-6.3028, :psb=>0.4472135954999579} +2 ||| how long has the mississippi river ||| ||| {:decoder=>-6.65798, :psb=>0.35930411196308426} +3 ||| how long will the mississippi river ||| ||| {:decoder=>-6.70278, :psb=>0.35930411196308426} +4 ||| how long the mississippi river is ||| ||| {:decoder=>-6.86439, :psb=>0.37991784282579627} +5 ||| how long is the mississippi river and ||| ||| {:decoder=>-6.88944, :psb=>0.672126440078521} +6 ||| how long is mississippi river ||| ||| {:decoder=>-6.93108, :psb=>0.5318295896944989} +7 ||| is how long the mississippi river ||| ||| {:decoder=>-6.98489, :psb=>0.37991784282579627} +8 ||| like long is the mississippi river ||| ||| {:decoder=>-7.3585, :psb=>0.6042750794713536} +9 ||| as long is the mississippi river ||| ||| {:decoder=>-7.39935, :psb=>0.6042750794713536} +10 ||| is as long the mississippi river ||| ||| {:decoder=>-7.76952, :psb=>0.32466791547509893} +>>> + + [TOP1] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518} + + [HOPE] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + + [FEAR] + nrl: how long is the mississippi river + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8034284189446518}, #0 + +================= + EXAMPLE: 29 + REFERENCE: how long is the mississippi river + GOLD MRL: answer(len(river(riverid('mississippi')))) +GOLD OUTPUT: [3778] + +<<< KBEST +0 ||| how long is the mississippi ||| ||| {:decoder=>-5.4476, :psb=>0.8187307530779819} +1 ||| how long the mississippi ||| ||| {:decoder=>-5.58584, :psb=>0.36064528799877893} +2 ||| how long is mississippi ||| ||| {:decoder=>-5.89517, :psb=>0.4288819424803534} +3 ||| how long has the mississippi ||| ||| {:decoder=>-5.94102, :psb=>0.36614752383039256} +4 ||| how long will the mississippi ||| ||| {:decoder=>-5.98582, :psb=>0.36614752383039256} +5 ||| how long the mississippi is ||| ||| {:decoder=>-6.29413, :psb=>0.3871538698781762} +6 ||| is how long the mississippi ||| ||| {:decoder=>-6.30808, :psb=>0.3871538698781762} +7 ||| how long , mississippi ||| ||| {:decoder=>-6.58296, :psb=>0.30326532985631666} +8 ||| as long is the mississippi ||| ||| {:decoder=>-6.63839, :psb=>0.6157842804860023} +9 ||| is how long mississippi ||| ||| {:decoder=>-6.64193, :psb=>0.3258798048281462} +10 ||| how long , the mississippi ||| ||| {:decoder=>-6.65378, :psb=>0.36614752383039256} +11 ||| how long mississippi is ||| ||| {:decoder=>-6.76988, :psb=>0.3258798048281462} +12 ||| like long is the mississippi ||| ||| {:decoder=>-6.88757, :psb=>0.6157842804860023} +13 ||| is as long the mississippi ||| ||| {:decoder=>-7.05256, :psb=>0.3308516361499261} +14 ||| mississippi how long is the ||| ||| {:decoder=>-7.09624, :psb=>0.6511126026643229} +>>> + + [TOP1] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8187307530779819} + + [HOPE] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8187307530779819}, #0 + + [FEAR] + nrl: how long is the mississippi + mrl: answer(len(riverid('mississippi'))) + output: [3778] + correct?: true + SCORES: {:decoder=>1.0, :psb=>0.8187307530779819}, #0 + +--- + iteration #3/3: 30 examples + type1 updates: 12 + type2 updates: 17 + top1 hits: 12 + top1 variant: 10 + top1 true variant: 10 + hope hits: 12 + hope variant: 10 + hope true variant: 10 + kbest size: 13.0 + 3.33% without translations (abs: 1) + 6.67% no good gold output (abs: 2) + + top1 with parse 96.67% abs=29.0 + top1 with output 96.67% abs=29.0 +top1 with correct output 73.33% adj=75.86 abs=22.0 + + hope with parse 96.67% abs=29.0 + hope with output 96.67% abs=29.0 +hope with correct output 73.33% adj=75.86 abs=22.0 + + fear with parse 96.67% abs=29.0 + fear with output 96.67% abs=29.0 +fear with correct output 50.0% adj=51.72 abs=15.0 + + |