master-thesis/RecipeAnalysis/Recipe.ipynb

299 lines
10 KiB
Plaintext
Raw Normal View History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Recipe class"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append(\"../\")\n",
"\n",
"import settings\n",
"\n",
"import pycrfsuite\n",
"\n",
"import json\n",
"\n",
"import db.db_settings as db_settings\n",
"from db.database_connection import DatabaseConnection\n",
"\n",
"from Tagging.conllu_generator import ConlluGenerator\n",
"from Tagging.crf_data_generator import *\n",
"\n",
"from IPython.display import Markdown, HTML, display"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get vocabulary"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import importlib.util\n",
"# loading ingredients:\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"ingredients\", \"../\" + settings.ingredients_file)\n",
"ingredients = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(ingredients)\n",
"\n",
"# loading actions:\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"actions\", \"../\" + settings.actions_file)\n",
"actions = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(actions)\n",
"\n",
"# loading containers\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"containers\", \"../\" + settings.container_file)\n",
"containers = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(containers)\n",
"\n",
"# loading placeholders\n",
"spec = importlib.util.spec_from_file_location(\n",
" \"placeholders\", \"../\" + settings.placeholder_file)\n",
"placeholders = importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(placeholders)\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<contextlib.closing at 0x7f6743611278>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tagger = pycrfsuite.Tagger()\n",
"tagger.open('../Tagging/test.crfsuite')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"id_query = \"select * from recipes where id like %s\""
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def escape_md_chars(s):\n",
" s = s.replace(\"*\", \"\\*\")\n",
" s = s.replace(\"(\", \"\\(\")\n",
" s = s.replace(\")\", \"\\)\")\n",
" s = s.replace(\"[\", \"\\[\")\n",
" s = s.replace(\"]\", \"\\]\")\n",
" s = s.replace(\"_\", \"\\_\")\n",
" \n",
" return s"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"class Recipe(object):\n",
" def __init__(self, recipe_db_id = None):\n",
" \n",
" self._sentences = None\n",
" self._title = None\n",
" self._part = None\n",
" self._ingredients = None\n",
" self._recipe_id = recipe_db_id\n",
" self._get_from_db()\n",
" \n",
" self._extracted_ingredients = None # TODO\n",
" \n",
" self.annotate_ingredients()\n",
" self.annotate_sentences()\n",
" \n",
" def _get_from_db(self):\n",
" result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))\n",
" assert len(result) > 0\n",
" result = result[0]\n",
" self._title = result['title']\n",
" self._part = result['part']\n",
" \n",
" raw_sentences = json.loads(result['instructions'])\n",
" raw_ingredients = json.loads(result['ingredients'])\n",
" \n",
" # throwing the raw data through our connlu generator to annotate them right\n",
" cg_sents = ConlluGenerator([\"\\n\".join(raw_sentences)])\n",
" cg_ings = ConlluGenerator([\"\\n\".join(raw_ingredients)])\n",
" \n",
" cg_sents.tokenize()\n",
" cg_sents.pos_tagging_and_lemmatization()\n",
" \n",
" cg_ings.tokenize()\n",
" cg_ings.pos_tagging_and_lemmatization()\n",
" \n",
" # TODO\n",
" self._sentences = cg_sents.get_conllu_elements()[0]\n",
" self._ingredients = cg_ings.get_conllu_elements()[0]\n",
" #self._sentences = json.loads(result['instructions'])\n",
" #self._ingredients = json.loads(result['ingredients'])\n",
" \n",
" def avg_sentence_length(self):\n",
" return sum([len(s) for s in self._sentences])/len(self._sentences)\n",
" \n",
" def n_instructions(self):\n",
" return len(self._sentences)\n",
" \n",
" def max_sentence_length(self):\n",
" return max([len(s) for s in self._sentences])\n",
" \n",
" def keyword_ratio(self):\n",
" sentence_ratios = []\n",
" for sent in self._sentences:\n",
" # FIXME: only works if there are no other misc annotations!\n",
" sentence_ratios.append(sum([token['misc'] is not None for token in sent]))\n",
" return sum(sentence_ratios) / len(sentence_ratios)\n",
" \n",
" def predict_labels(self):\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" labels = [tagger.tag(feat) for feat in features]\n",
" return labels\n",
" \n",
" def predict_ingredient_labels(self):\n",
" features = [sent2features(sent) for sent in self._ingredients]\n",
" labels = [tagger.tag(feat) for feat in features]\n",
" return labels\n",
" \n",
" def _annotate_sentences(self, sent_token_list, predictions):\n",
" # test whether we predicted an label or found it in our label list\n",
" for i, ing in enumerate(sent_token_list):\n",
" for j, token in enumerate(ing):\n",
" lemma = token['lemma']\n",
" \n",
" # check for ingredient\n",
" if lemma in ingredients.ingredients_stemmed:\n",
" token.add_misc(\"food_type\", \"ingredient\")\n",
" elif predictions[i][j] == 'ingredient':\n",
" token.add_misc(\"food_type\", \"ingredient\")\n",
" \n",
" # check for action\n",
" if lemma in actions.stemmed_cooking_verbs:\n",
" token.add_misc(\"food_type\", \"action\")\n",
" elif predictions[i][j] == 'action':\n",
" token.add_misc(\"food_type\", \"action\")\n",
" \n",
" # check for container\n",
" if lemma in containers.stemmed_containers:\n",
" token.add_misc(\"food_type\", \"container\")\n",
" elif predictions[i][j] == 'container':\n",
" token.add_misc(\"food_type\", \"container\")\n",
" \n",
" # check for placeholder\n",
" if lemma in placeholders.stemmed_placeholders:\n",
" token.add_misc(\"food_type\", \"placeholder\")\n",
" elif predictions[i][j] == 'placeholder':\n",
" token.add_misc(\"food_type\", \"placeholder\")\n",
" \n",
" def annotate_ingredients(self):\n",
" self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())\n",
" \n",
" def annotate_sentences(self):\n",
" self._annotate_sentences(self._sentences, self.predict_labels())\n",
" \n",
" def recipe_id(self):\n",
" return self._recipe_id\n",
" \n",
" def serialize(self):\n",
" result = \"# newdoc\\n\"\n",
" if self._recipe_id is not None:\n",
" result += f\"# id: {self._recipe_id}\\n\"\n",
" \n",
" for sent in self._sentences:\n",
" result += f\"{sent.serialize()}\"\n",
" return result + \"\\n\"\n",
" \n",
" def display_recipe(self):\n",
" display(Markdown(f\"## {self._title}\\n({self._recipe_id})\"))\n",
" display(Markdown(f\"### Ingredients\"))\n",
" display(Markdown(\"\\n\".join([f\" * '{escape_md_chars(self.tokenlist2str(ing))}'\" for ing in self._ingredients])))\n",
" display(Markdown(f\"### Instructions\"))\n",
" display(Markdown(\"\\n\".join([f\" * {escape_md_chars(self.tokenlist2str(ins))}\" for ins in self._sentences])))\n",
" \n",
" def tokenlist2str(self, tokenlist):\n",
" return \" \".join([token['form'] for token in tokenlist])\n",
" \n",
" def tokenarray2str(self, tokenarray):\n",
" return \"\\n\".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])\n",
" \n",
" \n",
" def __repr__(self):\n",
" s = \"recipe: \" + (self._recipe_id if self._recipe_id else \"\") + \"\\n\"\n",
" s += \"instructions: \\n\"\n",
" for sent in self._sentences:\n",
" s += \" \".join([token['form'] for token in sent]) + \"\\n\"\n",
" \n",
" s += \"\\nscores:\\n\"\n",
" s += f\"avg_sent_length: {self.avg_sentence_length()}\\n\"\n",
" s += f\"n_instructions: {self.n_instructions()}\\n\"\n",
" s += f\"keyword_ratio: {self.keyword_ratio()}\\n\\n\\n\"\n",
" \n",
" return s"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}