{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Recipe Conllu Generator" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.insert(0, '..')\n", "\n", "from conllu_generator import ConlluDict, ConlluElement, ConlluDocument, ConlluGenerator\n", "import settings\n", "import importlib.util\n", "from json_buffered_reader import JSON_buffered_reader as JSON_br" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# loading ingredients:\n", "spec = importlib.util.spec_from_file_location(\n", " \"ingredients\", \"../\" + settings.ingredients_file)\n", "ingredients = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(ingredients)\n", "\n", "# loading actions:\n", "spec = importlib.util.spec_from_file_location(\n", " \"actions\", \"../\" + settings.actions_file)\n", "actions = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(actions)\n", "\n", "# loading containers\n", "spec = importlib.util.spec_from_file_location(\n", " \"containers\", \"../\" + settings.container_file)\n", "containers = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(containers)\n", "\n", "# loading placeholders\n", "spec = importlib.util.spec_from_file_location(\n", " \"placeholders\", \"../\" + settings.placeholder_file)\n", "placeholders = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(placeholders)\n", "\n", "# skipping recipes:\n", "n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0\n", "print(\"start reading at recipe \" + str(n_skipped_recipes))\n", "\n", "# settings:\n", "recipe_buffer_size = 1000\n", "recipe_buffers_per_file = 5\n", "\n", "\n", "# create reader\n", "buffered_reader_1M = JSON_br(\"../\" + settings.one_million_recipes_file)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def process_instructions(instructions: list, document_ids=None):\n", "\n", " if len(instructions) == 0:\n", " return\n", "\n", " conllu_input_docs = instructions\n", "\n", " cg = ConlluGenerator(\n", " conllu_input_docs, ingredients.multi_word_ingredients_stemmed, ids=document_ids)\n", " cg.tokenize()\n", " cg.pos_tagging_and_lemmatization()\n", " \n", " \n", " cg.add_misc_value_by_list(\"food_type\", \"ingredient\", [w.replace(\" \",\"_\") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)\n", " cg.add_misc_value_by_list(\"food_type\", \"action\", actions.stemmed_cooking_verbs)\n", " cg.add_misc_value_by_list(\"food_type\", \"containers\", containers.stemmed_containers)\n", " cg.add_misc_value_by_list(\"food_type\", \"placeholders\", placeholders.stemmed_placeholders)\n", "\n", " savefile.write(str(cg))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "i = 0\n", "buffer_count = n_skipped_recipes % recipe_buffer_size\n", "file_count = n_skipped_recipes // (recipe_buffer_size * recipe_buffers_per_file)\n", "\n", "savefile = open(f\"recipes{file_count}.conllu\", 'w')\n", "instructions = []\n", "ids = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for raw_recipe in buffered_reader_1M:\n", "\n", " i += 1\n", "\n", " if i > n_skipped_recipes:\n", "\n", " instruction = \"\"\n", " for item in raw_recipe['instructions']:\n", " instruction += item['text'] + '\\n'\n", " ids.append(raw_recipe['id'])\n", "\n", " instructions.append(instruction)\n", "\n", " if i % recipe_buffer_size == 0:\n", " process_instructions(instructions, ids)\n", " print(f\"processed {i} recipes\")\n", " instructions = []\n", " ids = []\n", " buffer_count += 1\n", " if buffer_count % recipe_buffers_per_file == 0:\n", " savefile.close()\n", " file_count += 1\n", " savefile = open(f\"recipes{file_count}.conllu\", 'w')\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ " \n", "\n", "process_instructions(instructions)\n", "print(f\"processed {i} recipes\")\n", "\n", "savefile.close()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }