From 1c704c8df2c0ea895df1ce126af83e2647b1dac2 Mon Sep 17 00:00:00 2001 From: Jonas Weinz Date: Sun, 14 Jul 2019 10:54:28 +0200 Subject: [PATCH] added recipe offset in conllu generator --- Tagging/conllu_batch_generator.py | 55 ++++++++++++++++-------------- Tagging/recipe_conllu_generator.py | 45 +++++++++++++----------- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/Tagging/conllu_batch_generator.py b/Tagging/conllu_batch_generator.py index 737639d..d7cf338 100644 --- a/Tagging/conllu_batch_generator.py +++ b/Tagging/conllu_batch_generator.py @@ -15,34 +15,17 @@ import settings # noqa class ConlluSentenceIterator(object): def __init__(self, conllu_reader): self.conllu_reader = conllu_reader - - def __next__(self): - next_sent = self.conllu_reader.next_sentence() - if next_sent is None: - raise StopIteration - return next_sent - - -class ConlluDocumentIterator(object): - def __init__(self, conllu_reader): - self.conllu_reader = conllu_reader - - def __next__(self): - next_sent = self.conllu_reader.next_document() - if next_sent is None: - raise StopIteration - return next_sent - - -class ConlluReader(object): - def __init__(self, path, iter_documents=False): - self._path = path self._fileobj = None self._open() - self.iter_documents = iter_documents - + def _open(self): - self._fileobj = open(self._path, 'r') + self._fileobj = open(self.conllu_reader._path, 'r') + + def __next__(self): + next_sent = self.next_sentence() + if next_sent is None: + raise StopIteration + return next_sent def next_sentence(self): data = "" @@ -63,6 +46,16 @@ class ConlluReader(object): conllu_obj = parse(data) return conllu_obj + +class ConlluDocumentIterator(object): + def __init__(self, conllu_reader): + self.conllu_reader = conllu_reader + self._fileobj = None + self._open() + + def _open(self): + self._fileobj = open(self.conllu_reader._path, 'r') + def next_document(self): data = "" last_line_empty = False @@ -87,6 +80,18 @@ class ConlluReader(object): conllu_obj = parse(data) return conllu_obj + def __next__(self): + next_sent = self.next_document() + if next_sent is None: + raise StopIteration + return next_sent + + +class ConlluReader(object): + def __init__(self, path, iter_documents=False): + self._path = path + self.iter_documents = iter_documents + def __iter__(self): return ConlluDocumentIterator(self) if self.iter_documents else ConlluSentenceIterator(self) diff --git a/Tagging/recipe_conllu_generator.py b/Tagging/recipe_conllu_generator.py index bef9157..a160e3a 100755 --- a/Tagging/recipe_conllu_generator.py +++ b/Tagging/recipe_conllu_generator.py @@ -22,8 +22,9 @@ spec = importlib.util.spec_from_file_location( actions = importlib.util.module_from_spec(spec) spec.loader.exec_module(actions) -# load json reader - +# skipping recipes: +n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0 +print("start reading at recipe " + str(n_skipped_recipes)) # settings: recipe_buffer_size = 1000 @@ -35,8 +36,6 @@ buffered_reader_1M = JSON_br("../" + settings.one_million_recipes_file) # open savefile: - - def process_instructions(instructions: list): if len(instructions) == 0: @@ -54,28 +53,36 @@ def process_instructions(instructions: list): savefile.write(str(cg)) i = 0 -buffer_count = 0 -file_count = 0 +buffer_count = n_skipped_recipes % recipe_buffer_size +file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file) savefile = open(f"recipes{file_count}.conllu", 'w') instructions = [] for raw_recipe in buffered_reader_1M: - instruction = "" - for item in raw_recipe['instructions']: - instruction += item['text'] + '\n' - instructions.append(instruction) i += 1 - if i % recipe_buffer_size == 0: - process_instructions(instructions) - print(f"processed {i} recipes") - instructions = [] - buffer_count += 1 - if buffer_count % recipe_buffers_per_file == 0: - savefile.close() - file_count += 1 - savefile = open(f"recipes{file_count}.conllu", 'w') + + if i > n_skipped_recipes: + + instruction = "" + for item in raw_recipe['instructions']: + instruction += item['text'] + '\n' + + instructions.append(instruction) + + if i % recipe_buffer_size == 0: + process_instructions(instructions) + print(f"processed {i} recipes") + instructions = [] + buffer_count += 1 + if buffer_count % recipe_buffers_per_file == 0: + savefile.close() + file_count += 1 + savefile = open(f"recipes{file_count}.conllu", 'w') + + + process_instructions(instructions)