master-thesis/RecipeAnalysis/MatrixGeneration.ipynb

573 lines
17 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Matrix Generation"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
" </script>\n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.7/site-packages/ipykernel_launcher.py:13: TqdmExperimentalWarning:\n",
"\n",
"Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
"\n"
]
}
],
"source": [
"import sys\n",
"sys.path.append(\"../\")\n",
"from Recipe import Recipe, Ingredient, RecipeGraph\n",
"\n",
"import settings\n",
"import db.db_settings as db_settings\n",
"from db.database_connection import DatabaseConnection\n",
"\n",
"import random\n",
"\n",
"import itertools\n",
"\n",
"from tqdm.autonotebook import tqdm\n",
"\n",
"import traceback\n",
"\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<db.database_connection.DatabaseConnection at 0x7f3635212850>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"DatabaseConnection(db_settings.db_host,\n",
" db_settings.db_port,\n",
" db_settings.db_user,\n",
" db_settings.db_pw,\n",
" db_settings.db_db,\n",
" db_settings.db_charset)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 8.52 s, sys: 719 ms, total: 9.24 s\n",
"Wall time: 9.4 s\n"
]
}
],
"source": [
"%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import AdjacencyMatrix"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create Adjacency Matrix"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n",
" mix_m, mix_label = rec_state.get_mixing_matrix()\n",
" act_m, act_a, act_i = rec_state.get_action_matrix()\n",
"\n",
" # create list of tuples: [action, ingredient]\n",
" seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n",
"\n",
" # create list of tuples [ingredient, ingredient]\n",
" seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n",
"\n",
" seen_actions = seen_actions[act_m == 1]\n",
" seen_mixes = seen_mixes[mix_m == 1]\n",
"\n",
" seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n",
" seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n",
" \n",
" seen_base_actions = set()\n",
" seen_base_mixes = set()\n",
" \n",
" for act, ing in seen_actions:\n",
" m_act.add_entry(act, ing.to_json(), 1)\n",
" if (act, ing._base_ingredient) not in seen_base_actions:\n",
" seen_base_actions.add((act, ing._base_ingredient))\n",
" m_base_act.add_entry(act, ing._base_ingredient, 1)\n",
" \n",
" for x,y in seen_mixes:\n",
" xj = x.to_json()\n",
" yj = y.to_json()\n",
" if xj < yj:\n",
" m_mix.add_entry(xj,yj,1)\n",
" if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n",
" seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n",
" m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"m_act = AdjacencyMatrix.adj_matrix()\n",
"m_mix = AdjacencyMatrix.adj_matrix(True)\n",
"m_base_act = AdjacencyMatrix.adj_matrix()\n",
"m_base_mix = AdjacencyMatrix.adj_matrix(True)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "fa6557a61875425495f90a61f88115af",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 8be34a1b83 has no ingredient! skipping it\n",
"warning: recipe 5f02077e4c has no ingredient! skipping it\n",
"warning: recipe e5c317b3f1 has no ingredient! skipping it\n",
"warning: recipe 56016a1985 has no ingredient! skipping it\n",
"warning: recipe eb77bd6edc has no ingredient! skipping it\n",
"warning: recipe 5ce8c2cedd has no ingredient! skipping it\n",
"warning: recipe d7a9aaba3a has no ingredient! skipping it\n",
"warning: recipe 5d99d73716 has no ingredient! skipping it\n",
"warning: recipe 5a8462b13f has no ingredient! skipping it\n",
"an error occured : 'NoneType' object has no attribute 'lower'\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"<timed exec>\", line 4, in <module>\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n",
" self.annotate_sentences()\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n",
" self._annotate_sentences(self._sentences, self.predict_labels())\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in <listcomp>\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in <listcomp>\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 54, in word2features\n",
" '+1:word.lower=' + word1.lower(),\n",
"AttributeError: 'NoneType' object has no attribute 'lower'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 99ab771181 has no ingredient! skipping it\n",
"warning: recipe de79b67e1a has no ingredient! skipping it\n",
"warning: recipe ebc64e182c has no ingredient! skipping it\n",
"warning: recipe 7358870155 has no ingredient! skipping it\n",
"warning: recipe 3e06b9bc57 has no ingredient! skipping it\n",
"warning: recipe 7c67cd9da7 has no ingredient! skipping it\n",
"warning: recipe 1e5911a066 has no ingredient! skipping it\n",
"an error occured : 'NoneType' object is not subscriptable\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"<timed exec>\", line 4, in <module>\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n",
" self.annotate_sentences()\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n",
" self._annotate_sentences(self._sentences, self.predict_labels())\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in <listcomp>\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in <listcomp>\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 19, in word2features\n",
" 'word[-3:]=' + word[-3:],\n",
"TypeError: 'NoneType' object is not subscriptable\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 56016a1985 has no ingredient! skipping it\n",
"warning: recipe 1d14e148c9 has no ingredient! skipping it\n",
"warning: recipe f96375e11c has no ingredient! skipping it\n",
"warning: recipe 026e9af79d has no ingredient! skipping it\n",
"warning: recipe c884f5d8a0 has no ingredient! skipping it\n",
"warning: recipe 76b02fee35 has no ingredient! skipping it\n",
"an error occured : 'NoneType' object has no attribute 'lower'\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"<timed exec>\", line 4, in <module>\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n",
" self.annotate_sentences()\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n",
" self._annotate_sentences(self._sentences, self.predict_labels())\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in <listcomp>\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in <listcomp>\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 54, in word2features\n",
" '+1:word.lower=' + word1.lower(),\n",
"AttributeError: 'NoneType' object has no attribute 'lower'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 2fa5b920f4 has no ingredient! skipping it\n",
"\n",
"CPU times: user 2h 2min 39s, sys: 43.4 s, total: 2h 3min 23s\n",
"Wall time: 2h 4min 37s\n"
]
}
],
"source": [
"%%time\n",
"for i in tqdm(range(100000)):\n",
" try:\n",
" id = random.choice(ids)['id']\n",
" rec = Recipe(id)\n",
" #rec.display_recipe()\n",
" ing = rec.extract_ingredients()\n",
" if len(ing) == 0:\n",
" print(f\"warning: recipe {id} has no ingredient! skipping it\")\n",
" continue\n",
" rec.apply_instructions(debug=False)\n",
" add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)\n",
" except Exception as e:\n",
" print(\"an error occured : \" + str(e))\n",
" traceback.print_exc()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"99999"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"import dill"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"dill.dump(m_act, file=open(\"m_act_raw.dill\", 'wb'))\n",
"dill.dump(m_mix, file=open(\"m_mix_raw.dill\", 'wb'))\n",
"dill.dump(m_base_act, file=open(\"m_base_act_raw.dill\", 'wb'))\n",
"dill.dump(m_base_mix, file=open(\"m_base_mix_raw.dill\", 'wb'))"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"m_act.apply_threshold(10)\n",
"m_mix.apply_threshold(10)\n",
"m_base_act.apply_threshold(20)\n",
"m_base_mix.apply_threshold(20)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"c_mix = m_mix.get_csr()\n",
"c_act = m_act.get_csr()\n",
"c_base_mix = m_base_mix.get_csr()\n",
"c_base_act = m_base_act.get_csr()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"m_mix.compile()\n",
"m_act.compile()\n",
"m_base_mix.compile()\n",
"m_base_act.compile()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"dill.dump(m_act, file=open(\"m_act.dill\", 'wb'))\n",
"dill.dump(m_mix, file=open(\"m_mix.dill\", 'wb'))\n",
"dill.dump(m_base_act, file=open(\"m_base_act.dill\", 'wb'))\n",
"dill.dump(m_base_mix, file=open(\"m_base_mix.dill\", 'wb'))"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(64, 237792) (266772, 266772)\n",
"13170 17041\n",
"(64, 11455) (13528, 13528)\n",
"6083 10775\n"
]
}
],
"source": [
"print(c_act.shape, c_mix.shape)\n",
"print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n",
"print(c_base_act.shape, c_base_mix.shape)\n",
"print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(64, 237792) (266772, 266772)\n",
"13170 17041\n"
]
}
],
"source": [
"print(c_act.shape, c_mix.shape)\n",
"print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"13170"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sum(c_act.toarray() > 1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* values after 100:\n",
"```\n",
"(53, 1498) (1620, 1620)\n",
"1982 6489\n",
"```\n",
"\n",
"* after 1000:\n",
"```\n",
"(60, 9855) (10946, 10946)\n",
"15446 59943\n",
"```\n",
"\n",
"* after 10000:\n",
"```\n",
"(65, 65235) (72448, 72448)\n",
"114808 546217\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"file_extension": ".py",
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
},
"mimetype": "text/x-python",
"name": "python",
"npconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": 3
},
"nbformat": 4,
"nbformat_minor": 4
}