saving recipe id

This commit is contained in:
Jonas Weinz 2019-07-21 16:41:23 +02:00
parent eae9478e59
commit d3bcb10efa
3 changed files with 47 additions and 10 deletions

View File

@ -11,6 +11,8 @@ import numpy as np
sys.path.insert(0, '..') sys.path.insert(0, '..')
import settings # noqa import settings # noqa
import gzip
class ConlluSentenceIterator(object): class ConlluSentenceIterator(object):
def __init__(self, conllu_reader): def __init__(self, conllu_reader):
@ -19,18 +21,29 @@ class ConlluSentenceIterator(object):
self._open() self._open()
def _open(self): def _open(self):
self._fileobj = open(self.conllu_reader._path, 'r') if self.conllu_reader._path.endswith(".gz"):
self._fileobj = gzip.open(self.conllu_reader._path, 'r')
self._nextline = self.read_byte_line
else:
self._fileobj = open(self.conllu_reader._path, 'r')
self._nextline = self.read_str_line
def __next__(self): def __next__(self):
next_sent = self.next_sentence() next_sent = self.next_sentence()
if next_sent is None: if next_sent is None:
raise StopIteration raise StopIteration
return next_sent return next_sent
def read_str_line(self):
return self._fileobj.readline()
def read_byte_line(self):
return self._fileobj.readline().decode("utf-8")
def next_sentence(self): def next_sentence(self):
data = "" data = ""
while True: while True:
line = self._fileobj.readline() line = self._nextline()
if line == "": if line == "":
break break
data += line data += line
@ -54,13 +67,24 @@ class ConlluDocumentIterator(object):
self._open() self._open()
def _open(self): def _open(self):
self._fileobj = open(self.conllu_reader._path, 'r') if self.conllu_reader._path.endswith(".gz"):
self._fileobj = gzip.open(self.conllu_reader._path, 'r')
self._nextline = self.read_byte_line
else:
self._fileobj = open(self.conllu_reader._path, 'r')
self._nextline = self.read_str_line
def read_str_line(self):
return self._fileobj.readline()
def read_byte_line(self):
return self._fileobj.readline().decode("utf-8")
def next_document(self): def next_document(self):
data = "" data = ""
last_line_empty = False last_line_empty = False
while True: while True:
line = self._fileobj.readline() line = self._nextline()
if line == "": if line == "":
break break
data += line data += line

View File

@ -121,14 +121,17 @@ class ConlluSentence(object):
class ConlluDocument(object): class ConlluDocument(object):
def __init__(self): def __init__(self, id=None):
self.conllu_sentences = [] self.conllu_sentences = []
self.id = id
def add(self, conllu_sentence: ConlluSentence): def add(self, conllu_sentence: ConlluSentence):
self.conllu_sentences.append(conllu_sentence) self.conllu_sentences.append(conllu_sentence)
def __repr__(self): def __repr__(self):
result = "# newdoc\n" result = "# newdoc\n"
if self.id is not None:
result += "# id: " + self.id + "\n"
for elem in self.conllu_sentences: for elem in self.conllu_sentences:
result += elem.__repr__() + "\n" result += elem.__repr__() + "\n"
@ -139,7 +142,7 @@ class ConlluDocument(object):
class ConlluGenerator(object): class ConlluGenerator(object):
def __init__(self, documents: list, stemmed_multi_word_tokens, stemmer=PorterStemmer()): def __init__(self, documents: list, stemmed_multi_word_tokens, stemmer=PorterStemmer(), ids=None):
self.documents = documents self.documents = documents
self.stemmed_multi_word_tokens = stemmed_multi_word_tokens self.stemmed_multi_word_tokens = stemmed_multi_word_tokens
self.mwe_tokenizer = StemmedMWETokenizer( self.mwe_tokenizer = StemmedMWETokenizer(
@ -150,9 +153,12 @@ class ConlluGenerator(object):
self.conllu_documents = [] self.conllu_documents = []
self.ids = ids
def tokenize_and_stem(self): def tokenize_and_stem(self):
tokenized_documents = [] tokenized_documents = []
i = 0
for doc in self.documents: for doc in self.documents:
tokenized_sentences = [] tokenized_sentences = []
sentences = doc.split("\n") sentences = doc.split("\n")
@ -165,7 +171,10 @@ class ConlluGenerator(object):
# now create initial colln-u elemnts # now create initial colln-u elemnts
for doc in tokenized_documents: for doc in tokenized_documents:
conllu_doc = ConlluDocument() if self.ids:
conllu_doc = ConlluDocument(self.ids[i])
else:
conllu_doc = ConlluDocument()
self.id_counter = 0 self.id_counter = 0
for sent in doc: for sent in doc:
conllu_sent = ConlluSentence() conllu_sent = ConlluSentence()
@ -184,6 +193,7 @@ class ConlluGenerator(object):
self.id_counter += 1 self.id_counter += 1
conllu_doc.add(conllu_sent) conllu_doc.add(conllu_sent)
self.conllu_documents.append(conllu_doc) self.conllu_documents.append(conllu_doc)
i += 1
def pos_tagging(self): def pos_tagging(self):
for conllu_document in self.conllu_documents: for conllu_document in self.conllu_documents:

View File

@ -36,7 +36,7 @@ buffered_reader_1M = JSON_br("../" + settings.one_million_recipes_file)
# open savefile: # open savefile:
def process_instructions(instructions: list): def process_instructions(instructions: list, document_ids=None):
if len(instructions) == 0: if len(instructions) == 0:
return return
@ -44,7 +44,7 @@ def process_instructions(instructions: list):
conllu_input_docs = instructions conllu_input_docs = instructions
cg = ConlluGenerator( cg = ConlluGenerator(
conllu_input_docs, ingredients.multi_word_ingredients_stemmed) conllu_input_docs, ingredients.multi_word_ingredients_stemmed, ids=document_ids)
cg.tokenize_and_stem() cg.tokenize_and_stem()
cg.pos_tagging() cg.pos_tagging()
cg.add_misc_value_by_list("food_type", "ingredient", [w.replace(" ","_") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed) cg.add_misc_value_by_list("food_type", "ingredient", [w.replace(" ","_") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)
@ -58,6 +58,7 @@ file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file)
savefile = open(f"recipes{file_count}.conllu", 'w') savefile = open(f"recipes{file_count}.conllu", 'w')
instructions = [] instructions = []
ids = []
for raw_recipe in buffered_reader_1M: for raw_recipe in buffered_reader_1M:
@ -68,13 +69,15 @@ for raw_recipe in buffered_reader_1M:
instruction = "" instruction = ""
for item in raw_recipe['instructions']: for item in raw_recipe['instructions']:
instruction += item['text'] + '\n' instruction += item['text'] + '\n'
ids.append(raw_recipe['id'])
instructions.append(instruction) instructions.append(instruction)
if i % recipe_buffer_size == 0: if i % recipe_buffer_size == 0:
process_instructions(instructions) process_instructions(instructions, ids)
print(f"processed {i} recipes") print(f"processed {i} recipes")
instructions = [] instructions = []
ids = []
buffer_count += 1 buffer_count += 1
if buffer_count % recipe_buffers_per_file == 0: if buffer_count % recipe_buffers_per_file == 0:
savefile.close() savefile.close()