115 lines
3.2 KiB
Python
Executable File
115 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# coding: utf-8
|
|
|
|
# # Recipe Conllu Generator
|
|
|
|
import sys
|
|
sys.path.insert(0, '..')
|
|
|
|
from conllu_generator import ConlluDict, ConlluElement, ConlluDocument, ConlluGenerator
|
|
import settings
|
|
import importlib.util
|
|
from json_buffered_reader import JSON_buffered_reader as JSON_br
|
|
|
|
|
|
# loading ingredients:
|
|
spec = importlib.util.spec_from_file_location(
|
|
"ingredients", "../" + settings.ingredients_file)
|
|
ingredients = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(ingredients)
|
|
|
|
# loading actions:
|
|
spec = importlib.util.spec_from_file_location(
|
|
"actions", "../" + settings.actions_file)
|
|
actions = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(actions)
|
|
|
|
# loading containers
|
|
spec = importlib.util.spec_from_file_location(
|
|
"containers", "../" + settings.container_file)
|
|
containers = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(containers)
|
|
|
|
# loading placeholders
|
|
spec = importlib.util.spec_from_file_location(
|
|
"placeholders", "../" + settings.placeholder_file)
|
|
placeholders = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(placeholders)
|
|
|
|
# skipping recipes:
|
|
n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0
|
|
print("start reading at recipe " + str(n_skipped_recipes))
|
|
|
|
# settings:
|
|
recipe_buffer_size = 1000
|
|
recipe_buffers_per_file = 5
|
|
|
|
|
|
# create reader
|
|
buffered_reader_1M = JSON_br("../" + settings.one_million_recipes_file)
|
|
|
|
|
|
def process_instructions(instructions: list, document_ids=None):
|
|
|
|
if len(instructions) == 0:
|
|
return
|
|
|
|
conllu_input_docs = instructions
|
|
|
|
cg = ConlluGenerator(
|
|
conllu_input_docs, ingredients.multi_word_ingredients_stemmed, ids=document_ids)
|
|
cg.tokenize()
|
|
cg.pos_tagging_and_lemmatization()
|
|
|
|
|
|
cg.add_misc_value_by_list("food_type", "ingredient", [w.replace(" ","_") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)
|
|
cg.add_misc_value_by_list("food_type", "action", actions.stemmed_cooking_verbs)
|
|
cg.add_misc_value_by_list("food_type", "containers", containers.stemmed_containers)
|
|
cg.add_misc_value_by_list("food_type", "placeholders", placeholders.stemmed_placeholders)
|
|
|
|
savefile.write(str(cg))
|
|
|
|
|
|
i = 0
|
|
buffer_count = n_skipped_recipes % recipe_buffer_size
|
|
file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file)
|
|
|
|
savefile = open(f"recipes{file_count}.conllu", 'w')
|
|
instructions = []
|
|
ids = []
|
|
|
|
|
|
for raw_recipe in buffered_reader_1M:
|
|
|
|
i += 1
|
|
|
|
if i > n_skipped_recipes:
|
|
|
|
instruction = ""
|
|
for item in raw_recipe['instructions']:
|
|
instruction += item['text'] + '\n'
|
|
ids.append(raw_recipe['id'])
|
|
|
|
instructions.append(instruction)
|
|
|
|
if i % recipe_buffer_size == 0:
|
|
process_instructions(instructions, ids)
|
|
print(f"processed {i} recipes")
|
|
instructions = []
|
|
ids = []
|
|
buffer_count += 1
|
|
if buffer_count % recipe_buffers_per_file == 0:
|
|
savefile.close()
|
|
file_count += 1
|
|
savefile = open(f"recipes{file_count}.conllu", 'w')
|
|
|
|
|
|
|
|
|
|
|
|
process_instructions(instructions)
|
|
print(f"processed {i} recipes")
|
|
|
|
savefile.close()
|
|
|