master-thesis/RecipeAnalysis/MatrixGeneration.ipynb
2019-09-05 12:03:01 +02:00

384 lines
9.3 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Matrix Generation"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
" </script>\n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import sys\n",
"sys.path.append(\"../\")\n",
"from Recipe import Recipe, Ingredient, RecipeGraph\n",
"\n",
"import settings\n",
"import db.db_settings as db_settings\n",
"from db.database_connection import DatabaseConnection\n",
"\n",
"import random\n",
"\n",
"import itertools\n",
"\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<db.database_connection.DatabaseConnection at 0x7fc3a1bedac8>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"DatabaseConnection(db_settings.db_host,\n",
" db_settings.db_port,\n",
" db_settings.db_user,\n",
" db_settings.db_pw,\n",
" db_settings.db_db,\n",
" db_settings.db_charset)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 8.71 s, sys: 942 ms, total: 9.66 s\n",
"Wall time: 9.77 s\n"
]
}
],
"source": [
"%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import AdjacencyMatrix"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create Adjacency Matrix"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n",
" mix_m, mix_label = rec_state.get_mixing_matrix()\n",
" act_m, act_a, act_i = rec_state.get_action_matrix()\n",
"\n",
" # create list of tuples: [action, ingredient]\n",
" seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n",
"\n",
" # create list of tuples [ingredient, ingredient]\n",
" seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n",
"\n",
" seen_actions = seen_actions[act_m == 1]\n",
" seen_mixes = seen_mixes[mix_m == 1]\n",
"\n",
" seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n",
" seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n",
" \n",
" seen_base_actions = set()\n",
" seen_base_mixes = set()\n",
" \n",
" for act, ing in seen_actions:\n",
" m_act.add_entry(act, ing.to_json(), 1)\n",
" if (act, ing._base_ingredient) not in seen_base_actions:\n",
" seen_base_actions.add((act, ing._base_ingredient))\n",
" m_base_act.add_entry(act, ing._base_ingredient, 1)\n",
" \n",
" for x,y in seen_mixes:\n",
" xj = x.to_json()\n",
" yj = y.to_json()\n",
" if xj < yj:\n",
" m_mix.add_entry(xj,yj,1)\n",
" if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n",
" seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n",
" m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"m_act = AdjacencyMatrix.adj_matrix()\n",
"m_mix = AdjacencyMatrix.adj_matrix(True)\n",
"m_base_act = AdjacencyMatrix.adj_matrix()\n",
"m_base_mix = AdjacencyMatrix.adj_matrix(True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe a9dc137b48 has no ingredient! skipping it\n",
"CPU times: user 13min 35s, sys: 3.52 s, total: 13min 39s\n",
"Wall time: 13min 50s\n"
]
}
],
"source": [
"%%time\n",
"for i in range(10000):\n",
" id = random.choice(ids)['id']\n",
" rec = Recipe(id)\n",
" #rec.display_recipe()\n",
" ing = rec.extract_ingredients()\n",
" if len(ing) == 0:\n",
" print(f\"warning: recipe {id} has no ingredient! skipping it\")\n",
" continue\n",
" rec.apply_instructions(debug=False)\n",
" add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"import pickle"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"pickle.dump(m_act, file=open(\"m_act.pickle\", 'wb'))\n",
"pickle.dump(m_mix, file=open(\"m_mix.pickle\", 'wb'))\n",
"pickle.dump(m_base_act, file=open(\"m_base_act.pickle\", 'wb'))\n",
"pickle.dump(m_base_mix, file=open(\"m_base_mix.pickle\", 'wb'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"c_mix = m_mix.get_csr()\n",
"c_act = m_act.get_csr()\n",
"c_base_mix = m_base_mix.get_csr()\n",
"c_base_act = m_base_act.get_csr()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(65, 64699) (71548, 71548)\n",
"113994 537369\n",
"(65, 4738) (5850, 5850)\n",
"30820 122390\n"
]
}
],
"source": [
"print(c_act.shape, c_mix.shape)\n",
"print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n",
"print(c_base_act.shape, c_base_mix.shape)\n",
"print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(64, 63787) (70933, 70933)\n",
"112841 524285\n"
]
}
],
"source": [
"print(c_act.shape, c_mix.shape)\n",
"print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"17560"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sum(c_act.toarray() > 1)"
]
},
{
"cell_type": "code",
"execution_count": 99,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[1, 1, 0, ..., 0, 0, 0],\n",
" [0, 0, 1, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
]
},
"execution_count": 99,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* values after 100:\n",
"```\n",
"(53, 1498) (1620, 1620)\n",
"1982 6489\n",
"```\n",
"\n",
"* after 1000:\n",
"```\n",
"(60, 9855) (10946, 10946)\n",
"15446 59943\n",
"```\n",
"\n",
"* after 10000:\n",
"```\n",
"(65, 65235) (72448, 72448)\n",
"114808 546217\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}