added recipe offset in conllu generator

Tagging/conllu_batch_generator.py

@@ -15,34 +15,17 @@ import settings  # noqa
 class ConlluSentenceIterator(object):
     def __init__(self, conllu_reader):
         self.conllu_reader = conllu_reader
-
-    def __next__(self):
-        next_sent = self.conllu_reader.next_sentence()
-        if next_sent is None:
-            raise StopIteration
-        return next_sent
-
-
-class ConlluDocumentIterator(object):
-    def __init__(self, conllu_reader):
-        self.conllu_reader = conllu_reader
-
-    def __next__(self):
-        next_sent = self.conllu_reader.next_document()
-        if next_sent is None:
-            raise StopIteration
-        return next_sent
-
-
-class ConlluReader(object):
-    def __init__(self, path, iter_documents=False):
-        self._path = path
         self._fileobj = None
         self._open()
-        self.iter_documents = iter_documents
     
     def _open(self):
-        self._fileobj = open(self._path, 'r')
+        self._fileobj = open(self.conllu_reader._path, 'r')
+
+    def __next__(self):
+        next_sent = self.next_sentence()
+        if next_sent is None:
+            raise StopIteration
+        return next_sent
 
     def next_sentence(self):
         data = ""
@@ -63,6 +46,16 @@ class ConlluReader(object):
         conllu_obj = parse(data)
         return conllu_obj
 
+
+class ConlluDocumentIterator(object):
+    def __init__(self, conllu_reader):
+        self.conllu_reader = conllu_reader
+        self._fileobj = None
+        self._open()
+    
+    def _open(self):
+        self._fileobj = open(self.conllu_reader._path, 'r')
+    
     def next_document(self):
         data = ""
         last_line_empty = False
@@ -87,6 +80,18 @@ class ConlluReader(object):
         conllu_obj = parse(data)
         return conllu_obj
 
+    def __next__(self):
+        next_sent = self.next_document()
+        if next_sent is None:
+            raise StopIteration
+        return next_sent
+
+
+class ConlluReader(object):
+    def __init__(self, path, iter_documents=False):
+        self._path = path
+        self.iter_documents = iter_documents
+
     def __iter__(self):
         return ConlluDocumentIterator(self) if self.iter_documents else ConlluSentenceIterator(self)
 

Tagging/recipe_conllu_generator.py

@@ -22,8 +22,9 @@ spec = importlib.util.spec_from_file_location(
 actions = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(actions)
 
-# load json reader
+# skipping recipes:
-
+n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0
+print("start reading at recipe " + str(n_skipped_recipes))
 
 # settings:
 recipe_buffer_size = 1000
@@ -35,8 +36,6 @@ buffered_reader_1M = JSON_br("../" + settings.one_million_recipes_file)
 
 # open savefile:
 
-
-
 def process_instructions(instructions: list):
 
     if len(instructions) == 0:
@@ -54,28 +53,36 @@ def process_instructions(instructions: list):
     savefile.write(str(cg))
 
 i = 0
-buffer_count = 0
+buffer_count = n_skipped_recipes % recipe_buffer_size
-file_count = 0
+file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file)
 
 savefile = open(f"recipes{file_count}.conllu", 'w')
 instructions = []
 
 for raw_recipe in buffered_reader_1M:
-    instruction = ""
-    for item in raw_recipe['instructions']:
-        instruction += item['text'] + '\n'
 
-    instructions.append(instruction)
     i += 1
-    if i % recipe_buffer_size == 0:
+
-        process_instructions(instructions)
+    if i > n_skipped_recipes:
-        print(f"processed {i} recipes")
+
-        instructions = []
+        instruction = ""
-        buffer_count += 1
+        for item in raw_recipe['instructions']:
-        if buffer_count % recipe_buffers_per_file == 0:
+            instruction += item['text'] + '\n'
-            savefile.close()
+
-            file_count += 1
+        instructions.append(instruction)
-            savefile = open(f"recipes{file_count}.conllu", 'w')
+
+        if i % recipe_buffer_size == 0:
+            process_instructions(instructions)
+            print(f"processed {i} recipes")
+            instructions = []
+            buffer_count += 1
+            if buffer_count % recipe_buffers_per_file == 0:
+                savefile.close()
+                file_count += 1
+                savefile = open(f"recipes{file_count}.conllu", 'w')
+    
+    
+    
             
 
 process_instructions(instructions)