From 733e1b1507d27d4f53055f740e8098f56215ab8f Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Fri, 27 Jul 2012 17:31:00 -0400 Subject: remove old suffix array extractor (use the one in python/ instead) --- sa-extract/cdat.pyx | 178 ---------------------------------------------------- 1 file changed, 178 deletions(-) delete mode 100644 sa-extract/cdat.pyx (limited to 'sa-extract/cdat.pyx') diff --git a/sa-extract/cdat.pyx b/sa-extract/cdat.pyx deleted file mode 100644 index 57d3ad63..00000000 --- a/sa-extract/cdat.pyx +++ /dev/null @@ -1,178 +0,0 @@ -# cdat.pyx -# Defines "data arrays" that can be directly written to/read from disk in binary format -# In particular, the array itself is written/read directly as a glob of binary data -# Adam Lopez - -import sys -import gzip -import log -import cintlist - -from libc.stdio cimport FILE, fopen, fread, fwrite, fclose -from libc.stdlib cimport malloc, realloc, free -from libc.string cimport memset, strcpy, strlen - -cdef class DataArray: - - def __init__(self, filename=None, from_binary=False, use_sent_id=False): - self.word2id = {"END_OF_FILE":0, "END_OF_LINE":1} - self.id2word = ["END_OF_FILE", "END_OF_LINE"] - self.data = cintlist.CIntList(1000,1000) - self.sent_id = cintlist.CIntList(1000,1000) - self.sent_index = cintlist.CIntList(1000,1000) - self.use_sent_id = use_sent_id - - if filename is not None: - if from_binary: - self.read_binary(filename) - else: - self.read_text(filename) - - - def __len__(self): - return len(self.data) - - - def getSentId(self, i): - return self.sent_id.arr[i] - - - def getSent(self, i): - cdef int j, start, stop - sent = [] - start = self.sent_index.arr[i] - stop = self.sent_index.arr[i+1] - for i from start <= i < stop: - sent.append(self.id2word[self.data.arr[i]]) - return sent - - - def getSentPos(self, loc): - return loc - self.sent_index.arr[self.sent_id.arr[loc]] - - - def get_id(self, word): - if not word in self.word2id: - self.word2id[word] = len(self.id2word) - self.id2word.append(word) - return self.word2id[word] - - - def get_word(self, id): - return self.id2word[id] - - - def write_text(self, filename): - f = open(filename, "w") - for w_id in self.data: - if w_id > 1: - f.write("%s " % self.get_word(w_id)) - if w_id == 1: - f.write("\n") - f.close() - - - def read_text(self, filename): - cdef int word_count - - if filename[-2:] == "gz": - file = gzip.GzipFile(filename) - else: - file = open(filename) - word_count = 0 - for line_num, line in enumerate(file): - self.sent_index.append(word_count) - for word in line.split(): - self.data.append(self.get_id(word)) - if self.use_sent_id: - self.sent_id.append(line_num) - word_count = word_count + 1 - self.data.append(1) - if self.use_sent_id: - self.sent_id.append(line_num) - word_count = word_count + 1 - self.data.append(0) - self.sent_index.append(word_count) - - def read_binary(self, filename): - cdef FILE* f - cdef bytes bfilename = filename - cdef char* cfilename = bfilename - f = fopen(cfilename, "r") - self.read_handle(f) - fclose(f) - - - cdef void read_handle(self, FILE* f): - cdef int num_words - cdef int word_len - cdef char* c_word - cdef bytes py_word - self.data.read_handle(f) - self.sent_index.read_handle(f) - self.sent_id.read_handle(f) - fread(&(num_words), sizeof(int), 1, f) - for i in xrange(num_words): - fread(&(word_len), sizeof(int), 1, f) - c_word = malloc (word_len * sizeof(char)) - fread(c_word, sizeof(char), word_len, f) - py_word = c_word - free(c_word) - self.word2id[py_word] = len(self.id2word) - self.id2word.append(py_word) - if len(self.sent_id) == 0: - self.use_sent_id = False - else: - self.use_sent_id = True - - - cdef void write_handle(self, FILE* f): - cdef int word_len - cdef int num_words - cdef char* c_word - - self.data.write_handle(f) - self.sent_index.write_handle(f) - self.sent_id.write_handle(f) - num_words = len(self.id2word) - 2 - fwrite(&(num_words), sizeof(int), 1, f) - for word in self.id2word[2:]: - c_word = word - word_len = strlen(c_word) + 1 - fwrite(&(word_len), sizeof(int), 1, f) - fwrite(c_word, sizeof(char), word_len, f) - - - def write_binary(self, filename): - cdef FILE* f - cdef bytes bfilename = filename - cdef char* cfilename = bfilename - f = fopen(cfilename, "w") - self.write_handle(f) - fclose(f) - - - def write_enhanced_handle(self, f): - for i in self.data: - f.write("%d " %i) - f.write("\n") - for i in self.sent_index: - f.write("%d " %i) - f.write("\n") - for i in self.sent_id: - f.write("%d " %i) - f.write("\n") - for word in self.id2word: - f.write("%s %d " % (word, self.word2id[word])) - f.write("\n") - - - - def write_enhanced(self, filename): - f = open(filename, "w") - self.write_enhanced_handle(self, f) - f.close() - - - - -- cgit v1.2.3