added recipe offset in conllu generator
This commit is contained in:
		| @ -15,34 +15,17 @@ import settings  # noqa | ||||
| class ConlluSentenceIterator(object): | ||||
|     def __init__(self, conllu_reader): | ||||
|         self.conllu_reader = conllu_reader | ||||
|  | ||||
|     def __next__(self): | ||||
|         next_sent = self.conllu_reader.next_sentence() | ||||
|         if next_sent is None: | ||||
|             raise StopIteration | ||||
|         return next_sent | ||||
|  | ||||
|  | ||||
| class ConlluDocumentIterator(object): | ||||
|     def __init__(self, conllu_reader): | ||||
|         self.conllu_reader = conllu_reader | ||||
|  | ||||
|     def __next__(self): | ||||
|         next_sent = self.conllu_reader.next_document() | ||||
|         if next_sent is None: | ||||
|             raise StopIteration | ||||
|         return next_sent | ||||
|  | ||||
|  | ||||
| class ConlluReader(object): | ||||
|     def __init__(self, path, iter_documents=False): | ||||
|         self._path = path | ||||
|         self._fileobj = None | ||||
|         self._open() | ||||
|         self.iter_documents = iter_documents | ||||
|  | ||||
|      | ||||
|     def _open(self): | ||||
|         self._fileobj = open(self._path, 'r') | ||||
|         self._fileobj = open(self.conllu_reader._path, 'r') | ||||
|  | ||||
|     def __next__(self): | ||||
|         next_sent = self.next_sentence() | ||||
|         if next_sent is None: | ||||
|             raise StopIteration | ||||
|         return next_sent | ||||
|  | ||||
|     def next_sentence(self): | ||||
|         data = "" | ||||
| @ -63,6 +46,16 @@ class ConlluReader(object): | ||||
|         conllu_obj = parse(data) | ||||
|         return conllu_obj | ||||
|  | ||||
|  | ||||
| class ConlluDocumentIterator(object): | ||||
|     def __init__(self, conllu_reader): | ||||
|         self.conllu_reader = conllu_reader | ||||
|         self._fileobj = None | ||||
|         self._open() | ||||
|      | ||||
|     def _open(self): | ||||
|         self._fileobj = open(self.conllu_reader._path, 'r') | ||||
|      | ||||
|     def next_document(self): | ||||
|         data = "" | ||||
|         last_line_empty = False | ||||
| @ -87,6 +80,18 @@ class ConlluReader(object): | ||||
|         conllu_obj = parse(data) | ||||
|         return conllu_obj | ||||
|  | ||||
|     def __next__(self): | ||||
|         next_sent = self.next_document() | ||||
|         if next_sent is None: | ||||
|             raise StopIteration | ||||
|         return next_sent | ||||
|  | ||||
|  | ||||
| class ConlluReader(object): | ||||
|     def __init__(self, path, iter_documents=False): | ||||
|         self._path = path | ||||
|         self.iter_documents = iter_documents | ||||
|  | ||||
|     def __iter__(self): | ||||
|         return ConlluDocumentIterator(self) if self.iter_documents else ConlluSentenceIterator(self) | ||||
|  | ||||
|  | ||||
| @ -22,8 +22,9 @@ spec = importlib.util.spec_from_file_location( | ||||
| actions = importlib.util.module_from_spec(spec) | ||||
| spec.loader.exec_module(actions) | ||||
|  | ||||
| # load json reader | ||||
|  | ||||
| # skipping recipes: | ||||
| n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0 | ||||
| print("start reading at recipe " + str(n_skipped_recipes)) | ||||
|  | ||||
| # settings: | ||||
| recipe_buffer_size = 1000 | ||||
| @ -35,8 +36,6 @@ buffered_reader_1M = JSON_br("../" + settings.one_million_recipes_file) | ||||
|  | ||||
| # open savefile: | ||||
|  | ||||
|  | ||||
|  | ||||
| def process_instructions(instructions: list): | ||||
|  | ||||
|     if len(instructions) == 0: | ||||
| @ -54,28 +53,36 @@ def process_instructions(instructions: list): | ||||
|     savefile.write(str(cg)) | ||||
|  | ||||
| i = 0 | ||||
| buffer_count = 0 | ||||
| file_count = 0 | ||||
| buffer_count = n_skipped_recipes % recipe_buffer_size | ||||
| file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file) | ||||
|  | ||||
| savefile = open(f"recipes{file_count}.conllu", 'w') | ||||
| instructions = [] | ||||
|  | ||||
| for raw_recipe in buffered_reader_1M: | ||||
|     instruction = "" | ||||
|     for item in raw_recipe['instructions']: | ||||
|         instruction += item['text'] + '\n' | ||||
|  | ||||
|     instructions.append(instruction) | ||||
|     i += 1 | ||||
|     if i % recipe_buffer_size == 0: | ||||
|         process_instructions(instructions) | ||||
|         print(f"processed {i} recipes") | ||||
|         instructions = [] | ||||
|         buffer_count += 1 | ||||
|         if buffer_count % recipe_buffers_per_file == 0: | ||||
|             savefile.close() | ||||
|             file_count += 1 | ||||
|             savefile = open(f"recipes{file_count}.conllu", 'w') | ||||
|  | ||||
|     if i > n_skipped_recipes: | ||||
|  | ||||
|         instruction = "" | ||||
|         for item in raw_recipe['instructions']: | ||||
|             instruction += item['text'] + '\n' | ||||
|  | ||||
|         instructions.append(instruction) | ||||
|  | ||||
|         if i % recipe_buffer_size == 0: | ||||
|             process_instructions(instructions) | ||||
|             print(f"processed {i} recipes") | ||||
|             instructions = [] | ||||
|             buffer_count += 1 | ||||
|             if buffer_count % recipe_buffers_per_file == 0: | ||||
|                 savefile.close() | ||||
|                 file_count += 1 | ||||
|                 savefile = open(f"recipes{file_count}.conllu", 'w') | ||||
|      | ||||
|      | ||||
|      | ||||
|              | ||||
|  | ||||
| process_instructions(instructions) | ||||
|  | ||||
		Reference in New Issue
	
	Block a user