{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Recipe class" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.append(\"../\")\n", "\n", "import settings\n", "\n", "import pycrfsuite\n", "\n", "import json\n", "\n", "import db.db_settings as db_settings\n", "from db.database_connection import DatabaseConnection\n", "\n", "from Tagging.conllu_generator import ConlluGenerator\n", "from Tagging.crf_data_generator import *\n", "\n", "from IPython.display import Markdown, HTML, display" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* get vocabulary" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import importlib.util\n", "# loading ingredients:\n", "spec = importlib.util.spec_from_file_location(\n", " \"ingredients\", \"../\" + settings.ingredients_file)\n", "ingredients = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(ingredients)\n", "\n", "# loading actions:\n", "spec = importlib.util.spec_from_file_location(\n", " \"actions\", \"../\" + settings.actions_file)\n", "actions = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(actions)\n", "\n", "# loading containers\n", "spec = importlib.util.spec_from_file_location(\n", " \"containers\", \"../\" + settings.container_file)\n", "containers = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(containers)\n", "\n", "# loading placeholders\n", "spec = importlib.util.spec_from_file_location(\n", " \"placeholders\", \"../\" + settings.placeholder_file)\n", "placeholders = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(placeholders)\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tagger = pycrfsuite.Tagger()\n", "tagger.open('../Tagging/test.crfsuite')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "id_query = \"select * from recipes where id like %s\"" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def escape_md_chars(s):\n", " s = s.replace(\"*\", \"\\*\")\n", " s = s.replace(\"(\", \"\\(\")\n", " s = s.replace(\")\", \"\\)\")\n", " s = s.replace(\"[\", \"\\[\")\n", " s = s.replace(\"]\", \"\\]\")\n", " s = s.replace(\"_\", \"\\_\")\n", " \n", " return s" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "class Recipe(object):\n", " def __init__(self, recipe_db_id = None):\n", " \n", " self._sentences = None\n", " self._title = None\n", " self._part = None\n", " self._ingredients = None\n", " self._recipe_id = recipe_db_id\n", " self._get_from_db()\n", " \n", " self._extracted_ingredients = None # TODO\n", " \n", " self.annotate_ingredients()\n", " self.annotate_sentences()\n", " \n", " def _get_from_db(self):\n", " result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))\n", " assert len(result) > 0\n", " result = result[0]\n", " self._title = result['title']\n", " self._part = result['part']\n", " \n", " raw_sentences = json.loads(result['instructions'])\n", " raw_ingredients = json.loads(result['ingredients'])\n", " \n", " # throwing the raw data through our connlu generator to annotate them right\n", " cg_sents = ConlluGenerator([\"\\n\".join(raw_sentences)])\n", " cg_ings = ConlluGenerator([\"\\n\".join(raw_ingredients)])\n", " \n", " cg_sents.tokenize()\n", " cg_sents.pos_tagging_and_lemmatization()\n", " \n", " cg_ings.tokenize()\n", " cg_ings.pos_tagging_and_lemmatization()\n", " \n", " # TODO\n", " self._sentences = cg_sents.get_conllu_elements()[0]\n", " self._ingredients = cg_ings.get_conllu_elements()[0]\n", " #self._sentences = json.loads(result['instructions'])\n", " #self._ingredients = json.loads(result['ingredients'])\n", " \n", " def avg_sentence_length(self):\n", " return sum([len(s) for s in self._sentences])/len(self._sentences)\n", " \n", " def n_instructions(self):\n", " return len(self._sentences)\n", " \n", " def max_sentence_length(self):\n", " return max([len(s) for s in self._sentences])\n", " \n", " def keyword_ratio(self):\n", " sentence_ratios = []\n", " for sent in self._sentences:\n", " # FIXME: only works if there are no other misc annotations!\n", " sentence_ratios.append(sum([token['misc'] is not None for token in sent]))\n", " return sum(sentence_ratios) / len(sentence_ratios)\n", " \n", " def predict_labels(self):\n", " features = [sent2features(sent) for sent in self._sentences]\n", " labels = [tagger.tag(feat) for feat in features]\n", " return labels\n", " \n", " def predict_ingredient_labels(self):\n", " features = [sent2features(sent) for sent in self._ingredients]\n", " labels = [tagger.tag(feat) for feat in features]\n", " return labels\n", " \n", " def _annotate_sentences(self, sent_token_list, predictions):\n", " # test whether we predicted an label or found it in our label list\n", " for i, ing in enumerate(sent_token_list):\n", " for j, token in enumerate(ing):\n", " lemma = token['lemma']\n", " \n", " # check for ingredient\n", " if lemma in ingredients.ingredients_stemmed:\n", " token.add_misc(\"food_type\", \"ingredient\")\n", " elif predictions[i][j] == 'ingredient':\n", " token.add_misc(\"food_type\", \"ingredient\")\n", " \n", " # check for action\n", " if lemma in actions.stemmed_cooking_verbs:\n", " token.add_misc(\"food_type\", \"action\")\n", " elif predictions[i][j] == 'action':\n", " token.add_misc(\"food_type\", \"action\")\n", " \n", " # check for container\n", " if lemma in containers.stemmed_containers:\n", " token.add_misc(\"food_type\", \"container\")\n", " elif predictions[i][j] == 'container':\n", " token.add_misc(\"food_type\", \"container\")\n", " \n", " # check for placeholder\n", " if lemma in placeholders.stemmed_placeholders:\n", " token.add_misc(\"food_type\", \"placeholder\")\n", " elif predictions[i][j] == 'placeholder':\n", " token.add_misc(\"food_type\", \"placeholder\")\n", " \n", " def annotate_ingredients(self):\n", " self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())\n", " \n", " def annotate_sentences(self):\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " \n", " def recipe_id(self):\n", " return self._recipe_id\n", " \n", " def serialize(self):\n", " result = \"# newdoc\\n\"\n", " if self._recipe_id is not None:\n", " result += f\"# id: {self._recipe_id}\\n\"\n", " \n", " for sent in self._sentences:\n", " result += f\"{sent.serialize()}\"\n", " return result + \"\\n\"\n", " \n", " def display_recipe(self):\n", " display(Markdown(f\"## {self._title}\\n({self._recipe_id})\"))\n", " display(Markdown(f\"### Ingredients\"))\n", " display(Markdown(\"\\n\".join([f\" * '{escape_md_chars(self.tokenlist2str(ing))}'\" for ing in self._ingredients])))\n", " display(Markdown(f\"### Instructions\"))\n", " display(Markdown(\"\\n\".join([f\" * {escape_md_chars(self.tokenlist2str(ins))}\" for ins in self._sentences])))\n", " \n", " def tokenlist2str(self, tokenlist):\n", " return \" \".join([token['form'] for token in tokenlist])\n", " \n", " def tokenarray2str(self, tokenarray):\n", " return \"\\n\".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])\n", " \n", " \n", " def __repr__(self):\n", " s = \"recipe: \" + (self._recipe_id if self._recipe_id else \"\") + \"\\n\"\n", " s += \"instructions: \\n\"\n", " for sent in self._sentences:\n", " s += \" \".join([token['form'] for token in sent]) + \"\\n\"\n", " \n", " s += \"\\nscores:\\n\"\n", " s += f\"avg_sent_length: {self.avg_sentence_length()}\\n\"\n", " s += f\"n_instructions: {self.n_instructions()}\\n\"\n", " s += f\"keyword_ratio: {self.keyword_ratio()}\\n\\n\\n\"\n", " \n", " return s" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }