blob: 85b9d4fb6e9137befe9cf1363b4b4851254bdd76 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
#!/usr/bin/python
import sys
import gzip
#usage: morfsegment.py inputvocab.gz segmentation.ready
# stdin: the data to segment
# stdout: the segmented data
if len(sys.argv) < 3:
print "usage: morfsegment.py inputvocab.gz segmentation.ready [marker]"
print " stdin: the data to segment"
print " stdout: the segmented data"
sys.exit()
#read index:
split_index={}
marker="##"
if len(sys.argv) > 3:
marker=sys.argv[3]
word_vocab=gzip.open(sys.argv[1], 'rb') #inputvocab.gz
seg_vocab=open(sys.argv[2], 'r') #segm.ready..
for seg in seg_vocab:
#seg = ver# #wonder\n
#wordline = 1 verwonder\n
word = word_vocab.readline().strip().split(' ')
assert(len(word) == 2)
word = word[1]
seg=seg.strip()
if seg != word:
split_index[word] = seg
word_vocab.close()
seg_vocab.close()
for line in sys.stdin:
words = line.strip().split()
newsent = []
for word in words:
splitword = split_index.get(word, word)
newsent.append(splitword)
print ' '.join(newsent)
|
