summaryrefslogtreecommitdiff
path: root/gi/morf-segmentation/morfsegment.py
blob: 85b9d4fb6e9137befe9cf1363b4b4851254bdd76 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#!/usr/bin/python

import sys
import gzip

#usage: morfsegment.py inputvocab.gz segmentation.ready
#  stdin: the data to segment
#  stdout: the segmented data

if len(sys.argv) < 3:
  print "usage: morfsegment.py inputvocab.gz segmentation.ready [marker]"
  print "  stdin: the data to segment"
  print "  stdout: the segmented data"
  sys.exit()

#read index:
split_index={}

marker="##"

if len(sys.argv) > 3:
  marker=sys.argv[3]

word_vocab=gzip.open(sys.argv[1], 'rb') #inputvocab.gz
seg_vocab=open(sys.argv[2], 'r') #segm.ready..

for seg in seg_vocab:
  #seg = ver# #wonder\n
  #wordline = 1 verwonder\n
  word = word_vocab.readline().strip().split(' ')
  assert(len(word) == 2)
  word = word[1]
  seg=seg.strip()

  if seg != word:
    split_index[word] = seg

word_vocab.close()
seg_vocab.close()

for line in sys.stdin:
  words = line.strip().split()

  newsent = []
  for word in words:
    splitword = split_index.get(word, word)
    newsent.append(splitword)

  print ' '.join(newsent)