added recipe offset in conllu generator

This commit is contained in:
Jonas Weinz 2019-07-14 10:54:28 +02:00
parent ce0a3488a7
commit 1c704c8df2
2 changed files with 56 additions and 44 deletions

View File

@ -15,34 +15,17 @@ import settings # noqa
class ConlluSentenceIterator(object): class ConlluSentenceIterator(object):
def __init__(self, conllu_reader): def __init__(self, conllu_reader):
self.conllu_reader = conllu_reader self.conllu_reader = conllu_reader
def __next__(self):
next_sent = self.conllu_reader.next_sentence()
if next_sent is None:
raise StopIteration
return next_sent
class ConlluDocumentIterator(object):
def __init__(self, conllu_reader):
self.conllu_reader = conllu_reader
def __next__(self):
next_sent = self.conllu_reader.next_document()
if next_sent is None:
raise StopIteration
return next_sent
class ConlluReader(object):
def __init__(self, path, iter_documents=False):
self._path = path
self._fileobj = None self._fileobj = None
self._open() self._open()
self.iter_documents = iter_documents
def _open(self): def _open(self):
self._fileobj = open(self._path, 'r') self._fileobj = open(self.conllu_reader._path, 'r')
def __next__(self):
next_sent = self.next_sentence()
if next_sent is None:
raise StopIteration
return next_sent
def next_sentence(self): def next_sentence(self):
data = "" data = ""
@ -63,6 +46,16 @@ class ConlluReader(object):
conllu_obj = parse(data) conllu_obj = parse(data)
return conllu_obj return conllu_obj
class ConlluDocumentIterator(object):
def __init__(self, conllu_reader):
self.conllu_reader = conllu_reader
self._fileobj = None
self._open()
def _open(self):
self._fileobj = open(self.conllu_reader._path, 'r')
def next_document(self): def next_document(self):
data = "" data = ""
last_line_empty = False last_line_empty = False
@ -87,6 +80,18 @@ class ConlluReader(object):
conllu_obj = parse(data) conllu_obj = parse(data)
return conllu_obj return conllu_obj
def __next__(self):
next_sent = self.next_document()
if next_sent is None:
raise StopIteration
return next_sent
class ConlluReader(object):
def __init__(self, path, iter_documents=False):
self._path = path
self.iter_documents = iter_documents
def __iter__(self): def __iter__(self):
return ConlluDocumentIterator(self) if self.iter_documents else ConlluSentenceIterator(self) return ConlluDocumentIterator(self) if self.iter_documents else ConlluSentenceIterator(self)

View File

@ -22,8 +22,9 @@ spec = importlib.util.spec_from_file_location(
actions = importlib.util.module_from_spec(spec) actions = importlib.util.module_from_spec(spec)
spec.loader.exec_module(actions) spec.loader.exec_module(actions)
# load json reader # skipping recipes:
n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0
print("start reading at recipe " + str(n_skipped_recipes))
# settings: # settings:
recipe_buffer_size = 1000 recipe_buffer_size = 1000
@ -35,8 +36,6 @@ buffered_reader_1M = JSON_br("../" + settings.one_million_recipes_file)
# open savefile: # open savefile:
def process_instructions(instructions: list): def process_instructions(instructions: list):
if len(instructions) == 0: if len(instructions) == 0:
@ -54,19 +53,24 @@ def process_instructions(instructions: list):
savefile.write(str(cg)) savefile.write(str(cg))
i = 0 i = 0
buffer_count = 0 buffer_count = n_skipped_recipes % recipe_buffer_size
file_count = 0 file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file)
savefile = open(f"recipes{file_count}.conllu", 'w') savefile = open(f"recipes{file_count}.conllu", 'w')
instructions = [] instructions = []
for raw_recipe in buffered_reader_1M: for raw_recipe in buffered_reader_1M:
i += 1
if i > n_skipped_recipes:
instruction = "" instruction = ""
for item in raw_recipe['instructions']: for item in raw_recipe['instructions']:
instruction += item['text'] + '\n' instruction += item['text'] + '\n'
instructions.append(instruction) instructions.append(instruction)
i += 1
if i % recipe_buffer_size == 0: if i % recipe_buffer_size == 0:
process_instructions(instructions) process_instructions(instructions)
print(f"processed {i} recipes") print(f"processed {i} recipes")
@ -78,6 +82,9 @@ for raw_recipe in buffered_reader_1M:
savefile = open(f"recipes{file_count}.conllu", 'w') savefile = open(f"recipes{file_count}.conllu", 'w')
process_instructions(instructions) process_instructions(instructions)
print(f"processed {i} recipes") print(f"processed {i} recipes")