master-thesis/Tagging/recipe_conllu_generator.ipynb

179 lines
5.2 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Recipe Conllu Generator"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.insert(0, '..')\n",
"\n",
"from conllu_generator import ConlluDict, ConlluElement, ConlluDocument, ConlluGenerator\n",
"import settings\n",
"import importlib.util\n",
"from json_buffered_reader import JSON_buffered_reader as JSON_br"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# loading ingredients:\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"ingredients\", \"../\" + settings.ingredients_file)\n",
"ingredients = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(ingredients)\n",
"\n",
"# loading actions:\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"actions\", \"../\" + settings.actions_file)\n",
"actions = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(actions)\n",
"\n",
"# loading containers\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"containers\", \"../\" + settings.container_file)\n",
"containers = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(containers)\n",
"\n",
"# loading placeholders\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"placeholders\", \"../\" + settings.placeholder_file)\n",
"placeholders = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(placeholders)\n",
"\n",
"# skipping recipes:\n",
"n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0\n",
"print(\"start reading at recipe \" + str(n_skipped_recipes))\n",
"\n",
"# settings:\n",
"recipe_buffer_size = 1000\n",
"recipe_buffers_per_file = 5\n",
"\n",
"\n",
"# create reader\n",
"buffered_reader_1M = JSON_br(\"../\" + settings.one_million_recipes_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def process_instructions(instructions: list, document_ids=None):\n",
"\n",
" if len(instructions) == 0:\n",
" return\n",
"\n",
" conllu_input_docs = instructions\n",
"\n",
" cg = ConlluGenerator(\n",
" conllu_input_docs, ingredients.multi_word_ingredients_stemmed, ids=document_ids)\n",
" cg.tokenize()\n",
" cg.pos_tagging_and_lemmatization()\n",
" \n",
" \n",
" cg.add_misc_value_by_list(\"food_type\", \"ingredient\", [w.replace(\" \",\"_\") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)\n",
" cg.add_misc_value_by_list(\"food_type\", \"action\", actions.stemmed_cooking_verbs)\n",
" cg.add_misc_value_by_list(\"food_type\", \"containers\", containers.stemmed_containers)\n",
" cg.add_misc_value_by_list(\"food_type\", \"placeholders\", placeholders.stemmed_placeholders)\n",
"\n",
" savefile.write(str(cg))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"i = 0\n",
"buffer_count = n_skipped_recipes % recipe_buffer_size\n",
"file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file)\n",
"\n",
"savefile = open(f\"recipes{file_count}.conllu\", 'w')\n",
"instructions = []\n",
"ids = []"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for raw_recipe in buffered_reader_1M:\n",
"\n",
" i += 1\n",
"\n",
" if i > n_skipped_recipes:\n",
"\n",
" instruction = \"\"\n",
" for item in raw_recipe['instructions']:\n",
" instruction += item['text'] + '\\n'\n",
" ids.append(raw_recipe['id'])\n",
"\n",
" instructions.append(instruction)\n",
"\n",
" if i % recipe_buffer_size == 0:\n",
" process_instructions(instructions, ids)\n",
" print(f\"processed {i} recipes\")\n",
" instructions = []\n",
" ids = []\n",
" buffer_count += 1\n",
" if buffer_count % recipe_buffers_per_file == 0:\n",
" savefile.close()\n",
" file_count += 1\n",
" savefile = open(f\"recipes{file_count}.conllu\", 'w')\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
" \n",
"\n",
"process_instructions(instructions)\n",
"print(f\"processed {i} recipes\")\n",
"\n",
"savefile.close()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}