{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Recipe class" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import sys\n", "sys.path.append(\"../\")\n", "\n", "import settings\n", "\n", "import pycrfsuite\n", "\n", "import json\n", "\n", "import db.db_settings as db_settings\n", "from db.database_connection import DatabaseConnection\n", "\n", "from Tagging.conllu_generator import ConlluGenerator\n", "from Tagging.crf_data_generator import *\n", "\n", "from difflib import SequenceMatcher\n", "\n", "import numpy as np\n", "\n", "import plotly.graph_objs as go\n", "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n", "from plotly.subplots import make_subplots\n", "init_notebook_mode(connected=True)\n", "\n", "from graphviz import Digraph\n", "\n", "import itertools\n", "\n", "\n", "import plotly.io as pio\n", "pio.renderers.default = \"jupyterlab\"\n", "\n", "from IPython.display import Markdown, HTML, display" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* sequence similarity matcher" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def similar(a, b):\n", " return SequenceMatcher(None, a, b).ratio()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def string_similarity(a,b):\n", " \"\"\"\n", " does the same like `similar` but also compares single words of multi word tokens\n", " and returns the max similar value\n", " \"\"\"\n", " \n", " tokens_a = a.split()\n", " tokens_b = b.split()\n", " \n", " max_similarity = -1\n", " max_a = None\n", " max_b = None\n", " \n", " for t_a in tokens_a:\n", " for t_b in tokens_b:\n", " s = similar(t_a, t_b)\n", " if s > max_similarity:\n", " max_similarity = s\n", " max_a = t_a,\n", " max_b = t_b,\n", " \n", " return max_similarity, max_a, max_b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* get vocabulary" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import importlib.util\n", "# loading ingredients:\n", "spec = importlib.util.spec_from_file_location(\n", " \"ingredients\", \"../\" + settings.ingredients_file)\n", "ingredients = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(ingredients)\n", "\n", "# loading actions:\n", "spec = importlib.util.spec_from_file_location(\n", " \"actions\", \"../\" + settings.actions_file)\n", "actions = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(actions)\n", "\n", "# loading containers\n", "spec = importlib.util.spec_from_file_location(\n", " \"containers\", \"../\" + settings.container_file)\n", "containers = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(containers)\n", "\n", "# loading placeholders\n", "spec = importlib.util.spec_from_file_location(\n", " \"placeholders\", \"../\" + settings.placeholder_file)\n", "placeholders = importlib.util.module_from_spec(spec)\n", "spec.loader.exec_module(placeholders)\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# helper function since the lemmatizer not always lemmatize in a meaningful way :shrug:\n", "def check_ingredient(ing_token):\n", " form = ing_token['form'].lower()\n", " lemma = ing_token['lemma'].lower()\n", " \n", " if form in ingredients.ingredients:\n", " return True\n", " \n", " if lemma in ingredients.ingredients_stemmed:\n", " return True\n", " \n", " if lemma.endswith('s'):\n", " if lemma[:-1] in ingredients.ingredients_stemmed:\n", " return True\n", " \n", " else:\n", " if lemma + 's' in ingredients.ingredients_stemmed:\n", " return True\n", " \n", " return False" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tagger = pycrfsuite.Tagger()\n", "tagger.open('../Tagging/test.crfsuite')" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "id_query = \"select * from recipes where id like %s\"" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def escape_md_chars(s):\n", " s = s.replace(\"*\", \"\\*\")\n", " s = s.replace(\"(\", \"\\(\")\n", " s = s.replace(\")\", \"\\)\")\n", " s = s.replace(\"[\", \"\\[\")\n", " s = s.replace(\"]\", \"\\]\")\n", " s = s.replace(\"_\", \"\\_\")\n", " \n", " return s" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import json\n", "class Ingredient(object):\n", " \n", " @staticmethod\n", " def from_json(j):\n", " d = json.loads(j)\n", " ing = Ingredient(d['base'])\n", " ing._action_set = set(d['actions'])\n", " return ing\n", " \n", " def __init__(self, base_ingredient, last_touched_instruction=0):\n", " self._base_ingredient = base_ingredient\n", " self._action_set = set()\n", " self._last_touched_instruction = last_touched_instruction\n", " self._is_mixed = False\n", " \n", " def apply_action(self, action, instruction_number=0, touch=True):\n", " if action in actions.mixing_cooking_verbs:\n", " self.mark_for_mixing()\n", " else:\n", " self._action_set.add(action)\n", " \n", " if touch:\n", " self._last_touched_instruction = instruction_number\n", " \n", " return self\n", " \n", " def similarity(self, ingredient, use_actions=False, action_factor = 0.5):\n", " sim,_,_ = string_similarity(self._base_ingredient, ingredient._base_ingredient)\n", " if not use_actions:\n", " return sim\n", " \n", " return (1 - action_factor) + action_factor * similar(list(self._action_set), list(ingredient._action_set))\n", " \n", " def mark_for_mixing(self):\n", " self._is_mixed = True\n", " \n", " def unmark_mixing(self):\n", " self._is_mixed = False\n", " \n", " def is_mixed(self):\n", " return self._is_mixed\n", " \n", " def most_similar_ingredient(self, ing_list, use_actions=False, action_factor=0.5):\n", " best_index = -1\n", " best_value = -1\n", " \n", " for i, ing in enumerate(ing_list):\n", " sim = self.similarity(ing, use_actions=use_actions, action_factor=action_factor)\n", " if sim > best_value:\n", " best_value = sim\n", " best_index = i\n", " return best_value, ing_list[best_index]\n", " \n", " def copy(self):\n", " result = Ingredient(self._base_ingredient, self._last_touched_instruction)\n", " result._action_set = self._action_set.copy()\n", " result._is_mixed = self._is_mixed\n", " \n", " return result\n", " \n", " def to_json(self):\n", " result = {}\n", " result['base'] = self._base_ingredient\n", " result['actions'] = list(self._action_set)\n", " return json.dumps(result)\n", " \n", " def __repr__(self):\n", " return f\"{'|'.join(list(self._action_set))} 🠊 {self._base_ingredient} (last touched @ {self._last_touched_instruction})\" \n", " \n", " " ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "class RecipeState(object):\n", " def __init__(self, initial_ingredients):\n", " self._ingredients = initial_ingredients\n", " self._seen_ingredients = set()\n", " self._seen_actions = set()\n", " \n", " self._mix_matrix = None\n", " self._mix_labels = None\n", " self._act_matrix = None\n", " self._act_labels = None\n", " self._ing_labels = None\n", " self._mat_need_update = True\n", " \n", " # set of (ing_a, ing_b) tuples\n", " self._seen_mixes = set()\n", " \n", " # set of (action, ing) tuples\n", " self._seen_applied_actions = set()\n", " \n", " for ing in self._ingredients:\n", " self._seen_ingredients.add(ing.to_json())\n", " \n", " def copy(self):\n", " return RecipeState([ing.copy() for ing in self._ingredients])\n", " \n", " def apply_action(self, action: str, ing: Ingredient, instruction_number=0, sim_threshold = 0.6, add_new_if_not_similar=True):\n", " # find most similar ingredient to the given one and apply action on it\n", " sim_val, best_ing = ing.most_similar_ingredient(self._ingredients)\n", " \n", " # if sim_val is good enough, we apply the action to the best ingredient, otherwise\n", " # we add a new ingredient to our set (and assume that it was not detected or listed in the\n", " # ingredient set before)\n", " \n", " self._mat_need_update = True\n", " \n", " if sim_val > sim_threshold:\n", " if action not in actions.stemmed_mixing_cooking_verbs:\n", " self._seen_actions.add(action)\n", " self._seen_applied_actions.add((action, best_ing.to_json()))\n", " best_ing.apply_action(action, instruction_number)\n", " self._seen_ingredients.add(best_ing.to_json())\n", " elif add_new_if_not_similar:\n", " self._ingredients.append(ing)\n", " if action not in actions.stemmed_mixing_cooking_verbs:\n", " self._seen_actions.add(action)\n", " self._seen_ingredients.add(ing.to_json())\n", " self._seen_applied_actions.add((action, ing.to_json()))\n", " ing.apply_action(action, instruction_number)\n", " self._seen_ingredients.add(ing.to_json())\n", " \n", " def apply_action_on_all(self, action, instruction_number=0, exclude_instruction_number=None):\n", " self._mat_need_update = True\n", " for ing in self._ingredients:\n", " if exclude_instruction_number is None or exclude_instruction_number != ing._last_touched_instruction:\n", " if action not in actions.stemmed_mixing_cooking_verbs:\n", " self._seen_actions.add(action)\n", " self._seen_applied_actions.add((action, ing.to_json()))\n", " ing.apply_action(action, instruction_number)\n", " self._seen_ingredients.add(ing.to_json())\n", " \n", " def apply_action_by_last_touched(action, last_touched_instruction, instruction_number=0):\n", " self._mat_need_update = True\n", " for ing in self.get_ingredients_touched_in_instruction(last_touched_instruction):\n", " if action not in actions.stemmed_mixing_cooking_verbs:\n", " self._seen_actions.add(action)\n", " self._seen_applied_actions.add((action, ing.to_json()))\n", " ing.apply_action(action, instruction_number)\n", " self._seen_ingredients.add(ing.to_json())\n", " \n", " def get_combined_ingredients(self):\n", " combined = []\n", " for ing in self._ingredients:\n", " if ing.is_mixed():\n", " combined.append(ing)\n", " ing.unmark_mixing()\n", " \n", " for x in combined:\n", " for y in combined:\n", " self._seen_mixes.add((x.to_json(), y.to_json()))\n", " \n", " self._mat_need_update = True\n", " return combined\n", " \n", " def _update_matrices(self):\n", " \n", " ing_list = list(self._seen_ingredients)\n", " idx = {}\n", " \n", " m = np.zeros((len(ing_list), len(ing_list)))\n", " \n", " for i,ing in enumerate(ing_list):\n", " idx[ing] = i\n", " \n", " for x,y in self._seen_mixes:\n", " m[idx[x], idx[y]] = 1\n", " \n", " self._mix_matrix = m\n", " self._mix_labels = [Ingredient.from_json(j) for j in ing_list]\n", " \n", " ing_list = list(self._seen_ingredients)\n", " idx_i = {}\n", " \n", " act_list = list(self._seen_actions)\n", " idx_a = {}\n", " \n", " for i,ing in enumerate(ing_list):\n", " idx_i[ing] = i\n", " \n", " for i,act in enumerate(act_list):\n", " idx_a[act] = i\n", " \n", " m = np.zeros((len(act_list), len(ing_list)))\n", " \n", " for act, ing in self._seen_applied_actions:\n", " m[idx_a[act], idx_i[ing]] = 1\n", " \n", " self._act_matrix = m\n", " self._act_labels = act_list\n", " self._ing_labels = [Ingredient.from_json(j) for j in ing_list]\n", " \n", " self._mat_need_update = False\n", " \n", " \n", " def get_mixing_matrix(self): \n", " if self._mat_need_update:\n", " self._update_matrices()\n", " return self._mix_matrix, self._mix_labels\n", "\n", " \n", " def get_action_matrix(self):\n", " if self._mat_need_update:\n", " self._update_matrices()\n", " return self._act_matrix, self._act_labels, self._ing_labels\n", " \n", " \n", " def get_ingredients_touched_in_instruction(self, instruction_number = 0):\n", " ings = []\n", " for ing in self._ingredients:\n", " if ing._last_touched_instruction == instruction_number:\n", " ings.append(ing)\n", " return ings \n", " \n", " \n", " def get_ingredients(self):\n", " return self._ingredients\n", " \n", " def __repr__(self):\n", " s = \"\"\n", " for ing in self._ingredients:\n", " s += f\"• {str(ing)}\\n\"\n", " return s\n", " " ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "class Node(object):\n", " def __init__(self, id, label, shape):\n", " self.id = id\n", " self.label = label\n", " self.shape = shape" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "class GraphWrapper(object):\n", " def __init__(self, comment=\"recipe graph\"):\n", " self._comment = comment\n", " self._nodes = set()\n", " self._nodes_by_id = {}\n", " self._nodes_by_label = {}\n", " self._edges = set()\n", " self._to_node = {}\n", " self._from_node = {}\n", " \n", " def node(self, id, label, shape = None):\n", " assert id not in self._nodes_by_id\n", " n = Node(id, label, shape)\n", " self._nodes.add(n)\n", " self._nodes_by_id[id] = n\n", " if label not in self._nodes_by_label:\n", " self._nodes_by_label[label] = set()\n", " self._nodes_by_label[label].add(n)\n", " self._to_node[id] = set()\n", " self._from_node[id] = set()\n", " \n", " def edge(self, a, b):\n", " assert a in self._nodes_by_id and b in self._nodes_by_id\n", " self._edges.add((a,b))\n", " self._from_node[a].add(b)\n", " self._to_node[b].add(a)\n", " \n", " def remove_edge(self, a, b):\n", " self._edges.discard((a,b))\n", " if a in self._from_node:\n", " self._from_node[a].discard(b)\n", " if b in self._to_node:\n", " self._to_node[b].discard(a)\n", " \n", " def remove_node(self, id, redirect_edges=False):\n", " assert id in self._nodes_by_id\n", " \n", " if redirect_edges:\n", " f_set = self._from_node[id].copy()\n", " t_set = self._to_node[id].copy()\n", " \n", " self.remove_node(id)\n", " \n", " for a in t_set:\n", " for b in f_set:\n", " self.edge(a,b)\n", " return\n", " \n", " # remove all edges\n", " b_set = self._from_node[id].copy()\n", " for b in b_set:\n", " self.remove_edge(id, b)\n", "\n", " a_set = self._to_node[id].copy()\n", " for a in a_set:\n", " self.remove_edge(a, id)\n", " \n", " # remove node itself\n", " n = self._nodes_by_id[id]\n", " self._nodes_by_label[n.label].remove(n)\n", " if len(self._nodes_by_label[n.label]) == 0:\n", " del(self._nodes_by_label[n.label])\n", " self._nodes.remove(n)\n", " del(self._nodes_by_id[id])\n", " del(self._from_node[id])\n", " del(self._to_node[id])\n", " \n", " def merge(self, a, b):\n", " \"\"\"\n", " merge a with b and return id of merged node\n", " \"\"\"\n", " assert a in self._nodes_by_id and b in self._nodes_by_id\n", " \n", " if (a,b) in self._edges:\n", " self.remove_edge(a,b)\n", " if (b,a) in self._edges:\n", " self.remove_edge(b,a)\n", " \n", " to_merged = set()\n", " from_merged = set()\n", " \n", " if a in self._from_node:\n", " from_merged = from_merged.union(self._from_node[a])\n", " if b in self._from_node:\n", " from_merged = from_merged.union(self._from_node[b])\n", " \n", " if a in self._to_node:\n", " to_merged = to_merged.union(self._to_node[a])\n", " if b in self._to_node:\n", " to_merged = to_merged.union(self._to_node[b])\n", " \n", " from_merged.discard(a)\n", " from_merged.discard(b)\n", " \n", " to_merged.discard(a)\n", " to_merged.discard(b)\n", " \n", " merged_node = self._nodes_by_id[a]\n", " \n", " self.remove_node(a)\n", " self.remove_node(b)\n", " \n", " self.node(merged_node.id, merged_node.label, merged_node.shape)\n", " \n", " for x in to_merged:\n", " self.edge(x, merged_node.id)\n", " \n", " for x in from_merged:\n", " self.edge(merged_node.id, x)\n", " \n", " def insert_before(self, node_id, insert_id, insert_label, insert_shape):\n", " assert insert_id not in self._nodes_by_id\n", " assert node_id in self._nodes_by_id\n", " to_node = self._to_node[node_id].copy()\n", " \n", " for a in to_node:\n", " self.remove_edge(a, node_id)\n", " \n", " self.node(insert_id, insert_label, insert_shape)\n", " \n", " for a in to_node:\n", " self.edge(a, insert_id)\n", " self.edge(insert_id, node_id)\n", " \n", " def merge_adjacent_with_label(self, label):\n", " \"\"\"\n", " merge all adjacent nodes with given label\n", " \"\"\"\n", " \n", " assert label in self._nodes_by_label\n", " \n", " node_set = self._nodes_by_label[label]\n", " mix_set = set()\n", " \n", " connected_clusters = {}\n", " \n", " for x in node_set:\n", " for y in node_set:\n", " if (x.id, y.id) in self._edges:\n", " # mark for merge\n", " mix_set.add(x.id)\n", " mix_set.add(y.id)\n", " \n", " if x.id not in connected_clusters:\n", " connected_clusters[x.id] = set()\n", " if y.id not in connected_clusters:\n", " connected_clusters[y.id] = set()\n", " \n", " u = connected_clusters[x.id].union(connected_clusters[y.id])\n", " u.add(x.id)\n", " u.add(y.id)\n", " \n", " for n in u:\n", " connected_clusters[n] = u\n", " \n", " clusters = []\n", " while len(mix_set) > 0:\n", " arbitrary_node = mix_set.pop()\n", " # get cluster for node:\n", " c = connected_clusters[arbitrary_node]\n", " c_list = list(c)\n", " \n", " # merge all nodes:\n", " for i in range(len(c_list) - 1):\n", " # note: order matters since 'merge' keeps the id of the first node!\n", " self.merge(c_list[i + 1], c_list[i])\n", " \n", " # subtract cluster set from mix_set\n", " mix_set = mix_set.difference(c)\n", " \n", " def merge_sisters(self):\n", " sister_nodes = set()\n", " sisters = {}\n", " for label, node_set in self._nodes_by_label.items():\n", " for x in node_set:\n", " for y in node_set:\n", " if x.id == y.id:\n", " continue\n", " if len(self._from_node[x.id].intersection(self._from_node[y.id])) > 0:\n", " sister_nodes.add(x.id)\n", " sister_nodes.add(y.id)\n", " if x.id not in sisters:\n", " sisters[x.id] = set()\n", " if y.id not in sisters:\n", " sisters[y.id] = set()\n", " \n", " u = sisters[x.id].union(sisters[y.id])\n", " u.add(x.id)\n", " u.add(y.id)\n", " \n", " for n in u:\n", " sisters[n] = u\n", " \n", " if len(sister_nodes) <= 1:\n", " return False\n", " while len(sister_nodes) > 0:\n", " arbitrary_node = sister_nodes.pop()\n", " # get cluster for node:\n", " c = sisters[arbitrary_node]\n", " c_list = list(c)\n", " \n", " # merge all nodes:\n", " for i in range(len(c_list) - 1):\n", " # note: order matters since 'merge' keeps the id of the first node!\n", " self.merge(c_list[i + 1], c_list[i])\n", " \n", " i = 0\n", " mix_id = \"mix0\"\n", " while mix_id in self._nodes_by_id:\n", " i += 1\n", " mix_id = f\"mix{i}\"\n", " self.insert_before(c_list[-1], mix_id, \"mix\", \"diamond\")\n", " \n", " # subtract cluster set from mix_set\n", " sister_nodes = sister_nodes.difference(c)\n", " \n", " return True\n", " \n", " def get_paths(self):\n", " cluster = {}\n", " nodes = set()\n", " for a,b in self._edges:\n", " if len(self._from_node[a]) == 1 and len(self._to_node[b]) == 1:\n", " if a not in cluster:\n", " cluster[a] = set()\n", " if b not in cluster:\n", " cluster[b] = set()\n", " \n", " nodes.add(a)\n", " nodes.add(b)\n", " \n", " u = cluster[a].union(cluster[b])\n", " u.add(a)\n", " u.add(b)\n", " \n", " for n in u:\n", " cluster[n] = u\n", " \n", " paths = []\n", " while len(nodes) > 0:\n", " \n", " arbitrary_node = nodes.pop()\n", " # get cluster for node:\n", " c = cluster[arbitrary_node]\n", " \n", " paths.append(c)\n", " \n", " nodes = nodes.difference(c)\n", " \n", " return paths\n", " \n", " def clean_paths(self):\n", " for path in self.get_paths():\n", " seen_labels = set()\n", " for n in path:\n", " l = self._nodes_by_id[n].label\n", " if l == \"mix\" and len(self._to_node[n]) == 1:\n", " self.remove_node(n, redirect_edges=True)\n", " elif l in seen_labels:\n", " self.remove_node(n, redirect_edges=True)\n", " else:\n", " seen_labels.add(l)\n", " \n", " \n", " \n", " def simplify(self):\n", " \n", " changed = True\n", " \n", " while changed:\n", " \n", " # merge all adjacent nodes with the same label\n", " for key in self._nodes_by_label:\n", " self.merge_adjacent_with_label(key)\n", "\n", " # and now merge all sister nodes with the same label\n", " # (just to make it more clean structured)\n", "\n", " changed = self.merge_sisters()\n", " \n", " self.clean_paths()\n", " \n", " \n", " \n", " def compile_graph(self, simplify = False):\n", " if simplify:\n", " self.simplify()\n", " dot = Digraph(self._comment)\n", " for n in self._nodes:\n", " dot.node(n.id, label=n.label, shape=n.shape)\n", " \n", " for e in self._edges:\n", " dot.edge(e[0], e[1])\n", " \n", " return dot\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class RecipeGraph(object):\n", " def __init__(self, initial_ingreds=None):\n", " self._base_ing_nodes = set()\n", " self._dot = GraphWrapper(comment=\"recipe graph\")\n", " self._ing_state_mapping = {} # key: ingredient, value: state_id\n", " self._seen_actions = set()\n", " self._ings_connected_with_state = {} # key: state_id, value: set of ingreds \n", " \n", " self._seen_actions_for_ingredient = {}\n", " \n", " \n", " if initial_ingreds is not None:\n", " for ing in initial_ingreds:\n", " self.add_base_ingredient(ing)\n", " \n", " def add_base_ingredient(self, ingredient):\n", " if type(ingredient) == Ingredient:\n", " self.add_base_ingredient(ingredient._base_ingredient)\n", " return\n", " self._base_ing_nodes.add(ingredient)\n", " self._dot.node(ingredient, label=ingredient,shape=\"box\")\n", " self._ing_state_mapping[ingredient] = ingredient\n", " self._ings_connected_with_state[ingredient] = set([ingredient])\n", " self._seen_actions_for_ingredient[ingredient] = set() \n", " \n", " def add_action(self, action, ingredient):\n", " if type(ingredient) == Ingredient:\n", " return self.add_action(ingredient._base_ingredient)\n", " \n", " if ingredient not in self._seen_actions_for_ingredient:\n", " self._seen_actions_for_ingredient[ingredient] = set()\n", " \n", " if action in self._seen_actions_for_ingredient[ingredient]:\n", " return False\n", " \n", " self._seen_actions_for_ingredient[ingredient].add(action)\n", " \n", " action_id = action + \"0\"\n", " \n", " i = 0\n", " \n", " while action_id in self._seen_actions:\n", " i += 1\n", " action_id = action + str(i)\n", " \n", " self._seen_actions.add(action_id)\n", " \n", " self._dot.node(action_id, action)\n", " \n", " # get to the bottom of our tree (last known thing that happened to our ingredient)\n", " last_node = self._ing_state_mapping[ingredient]\n", " \n", " # update the reference of the last known state for all connected ingredients\n", " # (and for ourselve)\n", " \n", " connected_ingredients = self._ings_connected_with_state[last_node]\n", " \n", " for ing_id in connected_ingredients:\n", " self._ing_state_mapping[ing_id] = action_id\n", " \n", " # set ingredient set for new node\n", " self._ings_connected_with_state[action_id] = connected_ingredients.copy()\n", " \n", " # connect nodes with an edge\n", " self._dot.edge(last_node, action_id)\n", " \n", " return True\n", " \n", " def add_action_if_possible(self, action, ingredient):\n", " # extract actions for ingredient\n", " action_set = ingredient._action_set\n", " \n", " if action_set.issubset(self._seen_actions_for_ingredient[ingredient._base_ingredient]):\n", " return self.add_action(action, ingredient._base_ingredient)\n", " return False\n", " \n", " def mix_ingredients(self, ingredient_list):\n", " assert len(ingredient_list) > 0\n", " \n", " if type(ingredient_list[0]) == Ingredient:\n", " self.mix_ingredients([ing._base_ingredient for ing in ingredient_list])\n", " return\n", " \n", " last_nodes = set([self._ing_state_mapping[ing] for ing in ingredient_list])\n", " \n", " # create mixed ingredient set\n", " ing_set = set()\n", " \n", " for state in last_nodes:\n", " ing_set = ing_set.union(self._ings_connected_with_state[state])\n", " \n", " mix_action_id = \"mix0\"\n", " i = 0\n", " while mix_action_id in self._seen_actions:\n", " i += 1\n", " mix_action_id = f\"mix{i}\"\n", " \n", " self._seen_actions.add(mix_action_id)\n", " \n", " self._dot.node(mix_action_id, \"mix\", shape=\"diamond\")\n", " \n", " self._ings_connected_with_state[mix_action_id] = ing_set.copy()\n", " \n", " for ing in ing_set:\n", " self._ing_state_mapping[ing] = mix_action_id\n", " \n", " for state in last_nodes:\n", " self._dot.edge(state, mix_action_id)\n", " \n", " def mix_if_possible(self, ingredient_list):\n", " assert len(ingredient_list) > 0\n", " assert type(ingredient_list[0]) == Ingredient\n", " \n", " # check whether ingredients are mixed already\n", " state_set = set(\n", " [self._ing_state_mapping[ing._base_ingredient] for ing in ingredient_list]\n", " )\n", " \n", " if len(state_set) <= 1:\n", " # all ingredients have the same last state → they're mixed already\n", " return False\n", " \n", " # check if action sets are matching the requirements\n", " for ing in ingredient_list:\n", " for act in ing._action_set:\n", " if act not in self._seen_actions_for_ingredient[ing._base_ingredient]:\n", " return False\n", " \n", " # now we can mix the stuff:\n", " self.mix_ingredients(ingredient_list)\n", " return True\n", " \n", " @staticmethod\n", " def fromRecipeState(rec_state: RecipeState):\n", " # get all ingredients\n", " base_ingredients = set([ing._base_ingredient for ing in rec_state._ingredients])\n", " \n", " mix_m, mix_label = rec_state.get_mixing_matrix()\n", " act_m, act_a, act_i = rec_state.get_action_matrix()\n", " \n", " graph = RecipeGraph(base_ingredients)\n", " \n", " # create list of tuples: [action, ingredient]\n", " seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n", " \n", " # create list of tuples [ingredient, ingredient]\n", " seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n", " \n", " seen_actions = seen_actions[act_m == 1]\n", " seen_mixes = seen_mixes[mix_m == 1]\n", " \n", " seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n", " seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n", " \n", " # for each ingredient get the list of unseen applied actions. (They were applied\n", " # before the first instruction)\n", " \n", " seen_actions_per_ingred = {}\n", " for act, json_ing in rec_state._seen_applied_actions:\n", " ing = Ingredient.from_json(json_ing)._base_ingredient\n", " if ing not in seen_actions_per_ingred:\n", " seen_actions_per_ingred[ing] = set()\n", " seen_actions_per_ingred[ing].add(act)\n", " \n", " unseen_actions_per_ingred = {}\n", " for ing in rec_state._ingredients:\n", " base = ing._base_ingredient\n", " if base not in seen_actions_per_ingred:\n", " unseen_actions_per_ingred[base] = ing._action_set.copy()\n", " else:\n", " unseen_actions_per_ingred[base] = ing._action_set.difference(seen_actions_per_ingred[base])\n", " \n", " # for each ingredient: apply unseen actions first\n", " for ing in rec_state._ingredients:\n", " base = ing._base_ingredient\n", " for act in unseen_actions_per_ingred[base]:\n", " graph.add_action(act, base)\n", " \n", " # iterate over all mixes and actions until the graph does not change anymore\n", " # TODO: there are more efficient ways to do that!\n", " changed = True\n", " while changed:\n", " changed = False\n", " changed_ingreds = True\n", " while changed_ingreds:\n", " changed_ingreds = False\n", " for mix in list(seen_mixes):\n", " if graph.mix_if_possible([mix[0], mix[1]]):\n", " changed = True\n", " changed_ingreds = True\n", " changed_acts = True\n", " while changed_acts:\n", " changed_acts = False\n", " for act in list(seen_actions):\n", " if graph.add_action_if_possible(act[0], act[1]):\n", " changed = True\n", " changed_acts = True\n", " \n", " return graph\n", " \n", " \n", " \n", " " ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "class Recipe(object):\n", " def __init__(self, recipe_db_id = None):\n", " \n", " self._sentences = None\n", " self._title = None\n", " self._part = None\n", " self._ingredients = None\n", " self._recipe_id = recipe_db_id\n", " self._get_from_db()\n", " \n", " self._extracted_ingredients = None # TODO\n", " \n", " self.annotate_ingredients()\n", " self.annotate_sentences()\n", " \n", " def _get_from_db(self):\n", " result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))\n", " assert len(result) > 0\n", " result = result[0]\n", " self._title = result['title']\n", " self._part = result['part']\n", " \n", " raw_sentences = json.loads(result['instructions'])\n", " raw_ingredients = json.loads(result['ingredients'])\n", " \n", " # throwing the raw data through our connlu generator to annotate them right\n", " cg_sents = ConlluGenerator([\"\\n\".join(raw_sentences)])\n", " cg_ings = ConlluGenerator([\"\\n\".join(raw_ingredients)])\n", " \n", " cg_sents.tokenize()\n", " cg_sents.pos_tagging_and_lemmatization()\n", " \n", " cg_ings.tokenize()\n", " cg_ings.pos_tagging_and_lemmatization()\n", " \n", " # TODO\n", " self._sentences = cg_sents.get_conllu_elements()[0]\n", " self._ingredients = cg_ings.get_conllu_elements()[0]\n", " #self._sentences = json.loads(result['instructions'])\n", " #self._ingredients = json.loads(result['ingredients'])\n", " \n", " def avg_sentence_length(self):\n", " return sum([len(s) for s in self._sentences])/len(self._sentences)\n", " \n", " def n_instructions(self):\n", " return len(self._sentences)\n", " \n", " def max_sentence_length(self):\n", " return max([len(s) for s in self._sentences])\n", " \n", " def keyword_ratio(self):\n", " sentence_ratios = []\n", " for sent in self._sentences:\n", " # FIXME: only works if there are no other misc annotations!\n", " sentence_ratios.append(sum([token['misc'] is not None for token in sent]))\n", " return sum(sentence_ratios) / len(sentence_ratios)\n", " \n", " def predict_labels(self):\n", " features = [sent2features(sent) for sent in self._sentences]\n", " labels = [tagger.tag(feat) for feat in features]\n", " return labels\n", " \n", " def predict_ingredient_labels(self):\n", " features = [sent2features(sent) for sent in self._ingredients]\n", " labels = [tagger.tag(feat) for feat in features]\n", " return labels\n", " \n", " def _annotate_sentences(self, sent_token_list, predictions):\n", " # test whether we predicted an label or found it in our label list\n", " for i, ing in enumerate(sent_token_list):\n", " for j, token in enumerate(ing):\n", " lemma = token['lemma']\n", " \n", " # check for labels\n", " if check_ingredient(token):\n", " token.add_misc(\"food_type\", \"ingredient\")\n", " continue\n", " \n", " if lemma in actions.stemmed_curated_cooking_verbs:\n", " token.add_misc(\"food_type\", \"action\")\n", " continue\n", " \n", " if predictions[i][j] == 'ingredient':\n", " token.add_misc(\"food_type\", \"ingredient\")\n", " continue\n", " \n", " #if predictions[i][j] == 'action':\n", " # token.add_misc(\"food_type\", \"action\")\n", " # continue\n", " \n", " if lemma in containers.stemmed_containers:\n", " token.add_misc(\"food_type\", \"container\")\n", " continue\n", " if predictions[i][j] == 'container':\n", " token.add_misc(\"food_type\", \"container\")\n", " continue\n", " \n", " if lemma in placeholders.stemmed_placeholders:\n", " token.add_misc(\"food_type\", \"placeholder\")\n", " if predictions[i][j] == 'placeholder':\n", " token.add_misc(\"food_type\", \"placeholder\")\n", " \n", " def annotate_ingredients(self):\n", " self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())\n", " \n", " def annotate_sentences(self):\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " \n", " def recipe_id(self):\n", " return self._recipe_id\n", " \n", " '''\n", " # TODO: only conllu module compatible, and not with our own conllu classes\n", " def serialize(self):\n", " result = \"# newdoc\\n\"\n", " if self._recipe_id is not None:\n", " result += f\"# id: {self._recipe_id}\\n\"\n", " \n", " for sent in self._sentences:\n", " result += f\"{sent.serialize()}\"\n", " return result + \"\\n\"\n", " '''\n", " \n", " def display_recipe(self):\n", " display(Markdown(f\"## {self._title}\\n({self._recipe_id})\"))\n", " display(Markdown(f\"### Ingredients\"))\n", " display(Markdown(\"\\n\".join([f\" * '{escape_md_chars(self.tokenlist2str(ing))}'\" for ing in self._ingredients])))\n", " display(Markdown(f\"### Instructions\"))\n", " display(Markdown(\"\\n\".join([f\" * {escape_md_chars(self.tokenlist2str(ins))}\" for ins in self._sentences])))\n", " \n", " def tokenlist2str(self, tokenlist):\n", " return \" \".join([token['form'] for token in tokenlist])\n", " \n", " def tokenarray2str(self, tokenarray):\n", " return \"\\n\".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])\n", " \n", " \n", " def __repr__(self):\n", " s = \"recipe: \" + (self._recipe_id if self._recipe_id else \"\") + \"\\n\"\n", " s += \"instructions: \\n\"\n", " for sent in self._sentences:\n", " s += \" \".join([token['form'] for token in sent]) + \"\\n\"\n", " \n", " s += \"\\nscores:\\n\"\n", " s += f\"avg_sent_length: {self.avg_sentence_length()}\\n\"\n", " s += f\"n_instructions: {self.n_instructions()}\\n\"\n", " s += f\"keyword_ratio: {self.keyword_ratio()}\\n\\n\\n\"\n", " \n", " return s\n", " \n", " # --------------------------------------------------------------------------\n", " # functions for extracting ingredients\n", " \n", " def extract_ingredients(self):\n", " self._extracted_ingredients = []\n", " for ing in self._ingredients:\n", " entry_ing_tokens = []\n", " entry_act_tokens = []\n", " for token in ing:\n", " t_misc = token['misc']\n", " if t_misc is not None and \"food_type\" in t_misc:\n", " ftype = t_misc['food_type']\n", " if ftype == \"ingredient\":\n", " entry_ing_tokens.append(token)\n", " elif ftype == \"action\":\n", " entry_act_tokens.append(token)\n", " \n", " # find max cluster of ingredients and merge them\n", " index_best = 0\n", " best_size = 0\n", " current_size = 0\n", " for i, ing_token in enumerate(entry_ing_tokens):\n", " if i == 0 or entry_ing_tokens[i - 1]['id'] + 1 == ing_token['id']:\n", " current_size += 1\n", " if current_size > best_size:\n", " best_size = current_size\n", " index_best = i - current_size + 1\n", " \n", " if best_size == 0:\n", " # unfortunately, no ingredient is found :(\n", " continue\n", " \n", " ingredient = Ingredient(\" \".join([entry['lemma'] for entry in entry_ing_tokens[index_best:index_best + best_size]]))\n", " \n", " # apply found actions:\n", " for action in entry_act_tokens:\n", " ingredient.apply_action(action['lemma'])\n", " \n", " self._extracted_ingredients.append(ingredient)\n", " \n", " return self._extracted_ingredients\n", " \n", " def apply_instructions(self, confidence_threshold = 0.4, max_dist_last_token = 4, debug=False):\n", " current_state = RecipeState(self._extracted_ingredients)\n", " self._recipe_state = current_state\n", " \n", " instruction_number = 0\n", " \n", " for sent in self._sentences:\n", " \n", " instruction_number += 1\n", " \n", " if debug:\n", " display(Markdown(f\"----\\n* **instruction {instruction_number}**:\\n`\" + escape_md_chars(self.tokenlist2str(sent)) + \"`\\n\"))\n", " \n", " instruction_ing_tokens = []\n", " instruction_act_tokens = []\n", " \n", " ing_dist_last_token = []\n", " act_dist_last_token = []\n", " \n", " \n", " last_token = -1\n", " \n", " for i, token in enumerate(sent):\n", " t_misc = token['misc']\n", " if t_misc is not None and \"food_type\" in t_misc:\n", " ftype = t_misc['food_type']\n", " if ftype == \"ingredient\":\n", " instruction_ing_tokens.append(token)\n", " ing_dist_last_token.append(1000 if last_token < 0 else i - last_token)\n", " last_token = i\n", " elif ftype == \"action\":\n", " instruction_act_tokens.append(token)\n", " act_dist_last_token.append(1000 if last_token < 0 else i - last_token)\n", " last_token = i\n", " \n", " # cluster ingredient tokens together and apply actions on it:\n", " clustered_ingredients = []\n", " clustered_conllu_ids = []\n", " clustered_last_tokens = []\n", " i = 0\n", " n = len(instruction_ing_tokens)\n", " \n", " current_token_start = 0\n", " while i < n:\n", " current_token_start = i\n", " clustered_conllu_ids.append(instruction_ing_tokens[i]['id'])\n", " clustered_last_tokens.append(ing_dist_last_token[i])\n", " ing_str = instruction_ing_tokens[i]['lemma']\n", " while i+1 < n and instruction_ing_tokens[i+1]['id'] - instruction_ing_tokens[i]['id'] == 1:\n", " ing_str += \" \" + instruction_ing_tokens[i+1]['lemma']\n", " i += 1\n", " clustered_ingredients.append(ing_str)\n", " i += 1\n", " \n", " def matching_action(ing_str, ing_id, action_token_list):\n", " \n", " action = None\n", " action_dists = [act['id'] - ing_id for act in action_token_list]\n", " \n", " # so far: simple heuristic by matching to next action to the left\n", " # (or first action to the right, if there is no one left to the ingredient)\n", " \n", " for i in range(len(action_token_list)):\n", " if action_dists[i] < 0:\n", " action = action_token_list[i]\n", " \n", " return action\n", " \n", " ingredients_used = set()\n", " actions_used = set()\n", " \n", " if debug:\n", " print(\"apply actions regular rule based:\")\n", " \n", " for i, ing_str in enumerate(clustered_ingredients):\n", " \n", " ing = Ingredient(ing_str)\n", " \n", " # get matching action:\n", " action = matching_action(ing_str, clustered_conllu_ids[i], instruction_act_tokens)\n", " \n", " if clustered_last_tokens[i] < max_dist_last_token:\n", " if action is not None:\n", " actions_used.add(action['lemma'])\n", " ingredients_used.add(ing_str)\n", " # apply action on state\n", " current_state.apply_action(action['lemma'], ing, instruction_number=instruction_number, add_new_if_not_similar=False)\n", " if debug:\n", " print(f\"\\tapply {action['lemma']} on {ing}\")\n", " \n", " if debug:\n", " print(\"try to match unused actions:\")\n", " # go throuh all actions. if we found an unused one, we assume it is applied either on the next right ingredient.\n", " \n", " \n", " for act_token in instruction_act_tokens:\n", " if act_token['lemma'] not in actions_used:\n", " # fing next ingredient right to it\n", " next_ing = None\n", " for i, ing_str in enumerate(clustered_ingredients):\n", " if clustered_conllu_ids[i] > act_token['id']:\n", " actions_used.add(act_token['lemma'])\n", " ingredients_used.add(ing_str)\n", " ing = Ingredient(ing_str)\n", " current_state.apply_action(act_token['lemma'], ing, instruction_number=instruction_number, add_new_if_not_similar=False)\n", " if debug:\n", " print(f\"\\tapply {act_token['lemma']} on {ing}\")\n", " break\n", " \n", " \n", " actions_unused = []\n", " ingredients_unused = []\n", " \n", " \n", " for act_token in instruction_act_tokens:\n", " if act_token['lemma'] in actions_used:\n", " continue\n", " actions_unused.append(act_token['lemma'])\n", " \n", " for ing_str in clustered_ingredients:\n", " if ing_str in ingredients_used:\n", " continue\n", " ingredients_unused.append(ing_str)\n", " \n", " if debug:\n", " print(f\"\\nunused actions: {actions_unused} \\nunused ings: {ingredients_unused}\\n\")\n", " \n", " if (instruction_number > 1):\n", " if debug:\n", " print(\"mixing ingredients based on mixing actions with last instruction:\")\n", " for ing in current_state.get_ingredients_touched_in_instruction(instruction_number -1):\n", " ing.mark_for_mixing()\n", "\n", " for ing in current_state.get_combined_ingredients():\n", " if debug:\n", " print(f\"\\t* {ing}\")\n", " \n", " if debug:\n", " print(\"mixing all ingredients in this instruction:\")\n", " \n", " for ing_str in clustered_ingredients:\n", " current_state.apply_action(\"mix\", Ingredient(ing_str), instruction_number=instruction_number, add_new_if_not_similar=False)\n", " \n", " for ing in current_state.get_combined_ingredients():\n", " if debug:\n", " print(f\"\\t* {ing}\")\n", " \n", " \n", " # if no ingredient is found, apply actions on all ingredients so far used\n", " \n", " if len(clustered_ingredients) == 0 and len(actions_unused) > 0:\n", " if debug:\n", " print(\"\\nno ingredients found. So apply actions on all ingredients that are touched so far:\")\n", " for action in actions_unused:\n", " current_state.apply_action_on_all(action, instruction_number, exclude_instruction_number=0)\n", " \n", " if debug:\n", " print(f\"\\nstate after instruction {instruction_number}:\")\n", " print(current_state)\n", " print(\"\\n\")\n", " \n", " def plot_matrices(self):\n", " if self._recipe_state is None:\n", " print(\"Error: no recipe state found\")\n", " return\n", " \n", " mixings, mix_labels = self._recipe_state.get_mixing_matrix()\n", " \n", " x_labels = [f\"{ing._base_ingredient} 🡸 ({' '.join([act for act in ing._action_set])})\" for ing in mix_labels]\n", " y_labels = [f\"({' '.join([act for act in ing._action_set])}) 🢂 {ing._base_ingredient}\" for ing in mix_labels]\n", " \n", "\n", " fig = go.Figure(data=go.Heatmap(\n", " z=mixings,\n", " x=x_labels,\n", " y=y_labels,\n", " xgap = 1,\n", " ygap = 1,))\n", "\n", " fig.update_layout(\n", " width=1024,\n", " height=1024,\n", " yaxis = dict(\n", " scaleanchor = \"x\",\n", " scaleratio = 1,\n", " )\n", " )\n", " fig.show()\n", "\n", " \n", " actions, act_labels, ing_labels = self._recipe_state.get_action_matrix()\n", " \n", "\n", " fig = go.Figure(data=go.Heatmap(\n", " z=actions,\n", " x=[f\"{ing._base_ingredient} 🡸 ({' '.join([act for act in ing._action_set])})\" for ing in ing_labels],\n", " y=[str(a) for a in act_labels],\n", " xgap = 1,\n", " ygap = 1,))\n", "\n", " fig.update_layout(\n", " width=1024,\n", " height=1024,\n", " yaxis = dict(\n", " scaleanchor = \"x\",\n", " scaleratio = 1,\n", " )\n", " )\n", " fig.show()\n", "\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.5" } }, "nbformat": 4, "nbformat_minor": 4 }