diff --git a/RecipeAnalysis/AdjacencyMatrix.ipynb b/RecipeAnalysis/AdjacencyMatrix.ipynb index dbd15b7..f738e38 100644 --- a/RecipeAnalysis/AdjacencyMatrix.ipynb +++ b/RecipeAnalysis/AdjacencyMatrix.ipynb @@ -1,136 +1 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Adjacency Matrix" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "from scipy.sparse import csr_matrix, lil_matrix, coo_matrix" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "class adj_matrix(object):\n", - " def __init__(self, symmetric_indices=False):\n", - " \n", - " self._sym = symmetric_indices\n", - " if not symmetric_indices:\n", - " self._x_labels = []\n", - " self._y_labels = []\n", - "\n", - " self._x_label_index={}\n", - " self._y_label_index={}\n", - " \n", - " else:\n", - " self._labels = []\n", - " self._label_index={}\n", - " \n", - " self._x = []\n", - " self._y = []\n", - " self._data = []\n", - " \n", - " self._mat = None\n", - " \n", - " def _get_ix(self, label):\n", - " i = self._x_label_index.get(label)\n", - " if i is None:\n", - " i = len(self._x_labels)\n", - " self._x_labels.append(label)\n", - " self._x_label_index[label] = i\n", - " return i\n", - " \n", - " def _get_iy(self, label):\n", - " i = self._y_label_index.get(label)\n", - " if i is None:\n", - " i = len(self._y_labels)\n", - " self._y_labels.append(label)\n", - " self._y_label_index[label] = i\n", - " return i\n", - " \n", - " def _get_i(self, label):\n", - " i = self._label_index.get(label)\n", - " if i is None:\n", - " i = len(self._labels)\n", - " self._labels.append(label)\n", - " self._label_index[label] = i\n", - " return i\n", - " \n", - " def add_entry(self, x, y, data):\n", - " \n", - " if self._sym:\n", - " ix = self._get_i(x)\n", - " iy = self._get_i(y)\n", - " \n", - " else:\n", - " ix = self._get_ix(x)\n", - " iy = self._get_iy(y)\n", - " \n", - " self._x.append(ix)\n", - " self._y.append(iy)\n", - " self._data.append(data)\n", - " \n", - " \n", - " def compile_to_mat(self):\n", - " if self._sym:\n", - " sx = len(self._labels)\n", - " sy = len(self._labels)\n", - " else:\n", - " sx = len(self._x_labels)\n", - " sy = len(self._y_labels)\n", - " \n", - " self._mat = coo_matrix((self._data, (self._x, self._y)), shape=(sx,sy))\n", - " return self._mat\n", - " \n", - " def get_csr(self):\n", - " return self.compile_to_mat().tocsr()\n", - " \n", - " def get_labels(self):\n", - " if self._sym:\n", - " return self._labels\n", - " return self._x_labels, self._y_labels" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} +{"cells":[{"cell_type":"markdown","metadata":{},"source":["# Adjacency Matrix"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":"import numpy as np\n\nfrom scipy.sparse import csr_matrix, lil_matrix, coo_matrix"},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[{"ename":"SyntaxError","evalue":"invalid syntax (, line 63)","output_type":"error","traceback":["\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m63\u001b[0m\n\u001b[0;31m if label in self._label_document_count[label] += 1\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"]}],"source":"class adj_matrix(object):\n def __init__(self, symmetric_indices=False):\n \n self._sym = symmetric_indices\n if not symmetric_indices:\n self._x_labels = []\n self._y_labels = []\n\n self._x_label_index={}\n self._y_label_index={}\n \n else:\n self._labels = []\n self._label_index={}\n \n self._x = []\n self._y = []\n self._data = []\n \n self._mat = None\n self._csr = None\n\n # for a TF-IDF like approach we need also a counter how frequently ingredients\n # and actions appear in documents. \n\n self._current_document_labels = set()\n self._label_document_count = {}\n \n self._document_count = 0\n \n # building type dependend functions:\n self._build_funcs()\n \n def _get_ix(self, label):\n i = self._x_label_index.get(label)\n if i is None:\n i = len(self._x_labels)\n self._x_labels.append(label)\n self._x_label_index[label] = i\n return i\n \n def _get_iy(self, label):\n i = self._y_label_index.get(label)\n if i is None:\n i = len(self._y_labels)\n self._y_labels.append(label)\n self._y_label_index[label] = i\n return i\n \n def _get_i(self, label):\n i = self._label_index.get(label)\n if i is None:\n i = len(self._labels)\n self._labels.append(label)\n self._label_index[label] = i\n return i\n\n def _end_document(self):\n self._document_count += 1\n\n # adding all seen labels to our counter:\n for label in self._current_document_labels:\n self._label_document_count[label] += 1\n else:\n self._label_document_count[label] = 1\n \n self._current_document_labels = set()\n \n def apply_threshold(self, min_count=5):\n csr = self.get_csr()\n\n new_x = []\n new_y = []\n new_data = []\n\n for i in range(len(self._data)):\n if csr[self._x[i],self._y[i]] >= min_count:\n new_x.append(self._x[i])\n new_y.append(self._y[i])\n new_data.append(self._data[i])\n \n self._x = new_x\n self._y = new_y\n self._data = new_data\n\n \n def next_document(self):\n self._end_document()\n\n \n def add_entry(self, x, y, data):\n \n if self._sym:\n ix = self._get_i(x)\n iy = self._get_i(y)\n \n else:\n ix = self._get_ix(x)\n iy = self._get_iy(y)\n \n self._x.append(ix)\n self._y.append(iy)\n self._data.append(data)\n\n self._current_document_labels.add(x)\n self._current_document_labels.add(y)\n \n def compile(self):\n self._csr = None\n self._csr = self.get_csr()\n if self._sym:\n self._np_labels = np.array(self._labels)\n else:\n self._np_x_labels = np.array(self._x_labels)\n self._np_y_labels = np.array(self._y_labels)\n \n \n def compile_to_mat(self):\n if self._sym:\n sx = len(self._labels)\n sy = len(self._labels)\n else:\n sx = len(self._x_labels)\n sy = len(self._y_labels)\n \n self._mat = coo_matrix((self._data, (self._x, self._y)), shape=(sx,sy))\n return self._mat\n \n def get_csr(self):\n if self._csr is None:\n return self.compile_to_mat().tocsr()\n return self._csr\n \n def get_labels(self):\n if self._sym:\n return self._labels\n return self._x_labels, self._y_labels\n \n def _build_funcs(self):\n \n def get_sym_adjacent(key):\n assert self._csr is not None\n \n c = self._csr\n \n index = self._label_index[key]\n i1 = c[index,:].nonzero()[1]\n i2 = c[:,index].nonzero()[0]\n\n i = np.concatenate((i1,i2))\n\n names = self._np_labels[i]\n\n counts = np.concatenate((c[index, i1].toarray().flatten(), c[i2, index].toarray().flatten()))\n\n s = np.argsort(-counts)\n\n return names[s], counts[s]\n \n def get_forward_adjacent(key):\n assert self._csr is not None\n \n c = self._csr\n \n index = self._x_label_index[key]\n i = c[index,:].nonzero()[1]\n\n names = self._np_y_labels[i]\n\n counts = c[index, i].toarray().flatten()\n\n s = np.argsort(-counts)\n\n return names[s], counts[s]\n \n def get_backward_adjacent(key):\n assert self._csr is not None\n \n c = self._csr\n \n index = self._y_label_index[key]\n i = c[:,index].nonzero()[0]\n\n \n names = self._np_x_labels[i]\n\n counts = c[i, index].toarray().flatten()\n\n s = np.argsort(-counts)\n\n return names[s], counts[s]\n \n # sum functions:\n def sym_sum(key):\n return np.sum(self.get_adjacent(key)[1])\n\n def fw_sum(key):\n return np.sum(self.get_forward_adjacent(key)[1])\n\n def bw_sum(key):\n return np.sum(self.get_backward_adjacent(key)[1])\n \n # normalization stuff:\n def fw_normalization_factor(key, quotient_func):\n assert self._csr is not None\n c = self._csr\n \n ia = self._x_label_index[key]\n\n occurances = c[ia,:].nonzero()[1]\n\n return 1. / quotient_func(c[ia,occurances].toarray())\n\n def bw_normalization_factor(key, quotient_func):\n assert self._csr is not None\n \n c = self._csr\n \n ib = m._y_label_index[key]\n\n occurances = c[:,ib].nonzero()[0]\n\n return 1. / quotient_func(c[occurances,ib].toarray())\n\n def sym_normalization_factor(key, quotient_func):\n assert self._csr is not None\n \n c = self._csr\n \n ii = m._label_index[key]\n\n fw_occurances = c[ii,:].nonzero()[1]\n bw_occurances = c[:,ii].nonzero()[0]\n\n return 1. / quotient_func(np.concatenate(\n [c[ii,fw_occurances].toarray().flatten(),\n c[bw_occurances,ii].toarray().flatten()]\n ))\n \n def sym_p_a_given_b(key_a, key_b, quot_func = np.max):\n assert self._csr is not None\n \n c = self._csr\n \n ia = m._label_index[key_a]\n ib = m._label_index[key_b]\n\n v = c[ia,ib] + c[ib,ia]\n\n return v * self.sym_normalization_factor(key_b, quot_func)\n\n def fw_p_a_given_b(key_a, key_b, quot_func = np.max):\n assert self._csr is not None\n \n c = self._csr\n \n ia = m._x_label_index[key_a]\n ib = m._y_label_index[key_b]\n\n v = c[ia,ib]\n\n return v * self.bw_normalization_factor(key_b, quot_func)\n\n def bw_p_a_given_b(key_a, key_b, quot_func = np.max):\n assert self._csr is not None\n \n c = self._csr\n \n ia = m._y_label_index[key_a]\n ib = m._x_label_index[key_b]\n\n v = c[ib,ia]\n\n return v * self.fw_normalization_factor(key_b, quot_func)\n\n \n if self._sym:\n self.get_adjacent = get_sym_adjacent\n self.get_sum = sym_sum\n self.get_sym_normalization_factor = sym_normalization_factor\n self.p_a_given_b = sym_p_a_given_b\n \n else:\n self.get_forward_adjacent = get_forward_adjacent\n self.get_backward_adjacent = get_backward_adjacent\n \n self.get_fw_sum = fw_sum\n self.get_bw_sum = bw_sum\n \n self.get_fw_normalization_factor = fw_normalization_factor\n self.get_bw_normalization_factor = bw_normalization_factor\n\n self.fw_p_a_given_b = fw_p_a_given_b\n self.bw_p_a_given_b = bw_p_a_given_b\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"orig_nbformat":2,"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3}} \ No newline at end of file diff --git a/RecipeAnalysis/AdjacencyMatrix.py b/RecipeAnalysis/AdjacencyMatrix.py index 4e22cf0..acefea0 100644 --- a/RecipeAnalysis/AdjacencyMatrix.py +++ b/RecipeAnalysis/AdjacencyMatrix.py @@ -28,6 +28,18 @@ class adj_matrix(object): self._data = [] self._mat = None + self._csr = None + + # for a TF-IDF like approach we need also a counter how frequently ingredients + # and actions appear in documents. + + self._current_document_labels = set() + self._label_document_count = {} + + self._document_count = 0 + + # building type dependend functions: + self._build_funcs() def _get_ix(self, label): i = self._x_label_index.get(label) @@ -52,6 +64,39 @@ class adj_matrix(object): self._labels.append(label) self._label_index[label] = i return i + + def _end_document(self): + self._document_count += 1 + + # adding all seen labels to our counter: + for label in self._current_document_labels: + self._label_document_count[label] += 1 + else: + self._label_document_count[label] = 1 + + self._current_document_labels = set() + + def apply_threshold(self, min_count=5): + csr = self.get_csr() + + new_x = [] + new_y = [] + new_data = [] + + for i in range(len(self._data)): + if csr[self._x[i],self._y[i]] >= min_count: + new_x.append(self._x[i]) + new_y.append(self._y[i]) + new_data.append(self._data[i]) + + self._x = new_x + self._y = new_y + self._data = new_data + + + def next_document(self): + self._end_document() + def add_entry(self, x, y, data): @@ -66,6 +111,18 @@ class adj_matrix(object): self._x.append(ix) self._y.append(iy) self._data.append(data) + + self._current_document_labels.add(x) + self._current_document_labels.add(y) + + def compile(self): + self._csr = self.get_csr() + if self._sym: + self._np_labels = np.array(self._labels) + else: + self._np_x_labels = np.array(self._x_labels) + self._np_y_labels = np.array(self._y_labels) + def compile_to_mat(self): if self._sym: @@ -85,6 +142,163 @@ class adj_matrix(object): if self._sym: return self._labels return self._x_labels, self._y_labels + + def _build_funcs(self): + + def get_sym_adjacent(key): + assert self._csr is not None + + c = self._csr + + index = self._label_index[key] + i1 = c[index,:].nonzero()[1] + i2 = c[:,index].nonzero()[0] + + i = np.concatenate((i1,i2)) + + names = self._np_labels[i] + + counts = np.concatenate((c[index, i1].toarray().flatten(), c[i2, index].toarray().flatten())) + + s = np.argsort(-counts) + + return names[s], counts[s] + + def get_forward_adjacent(key): + assert self._csr is not None + + c = self._csr + + index = self._x_label_index[key] + i = c[index,:].nonzero()[1] + + names = self._np_y_labels[i] + + counts = c[index, i].toarray().flatten() + + s = np.argsort(-counts) + + return names[s], counts[s] + + def get_backward_adjacent(key): + assert self._csr is not None + + c = self._csr + + index = self._y_label_index[key] + i = c[:,index].nonzero()[0] + + + names = self._np_x_labels[i] + + counts = c[i, index].toarray().flatten() + + s = np.argsort(-counts) + + return names[s], counts[s] + + # sum functions: + def sym_sum(key): + return np.sum(self.get_adjacent(key)[1]) + + def fw_sum(key): + return np.sum(self.get_forward_adjacent(key)[1]) + + def bw_sum(key): + return np.sum(self.get_backward_adjacent(key)[1]) + + # normalization stuff: + def fw_normalization_factor(key, quotient_func): + assert self._csr is not None + c = self._csr + + ia = self._x_label_index[key] + + occurances = c[ia,:].nonzero()[1] + + return 1. / quotient_func(c[ia,occurances].toarray()) + + def bw_normalization_factor(key, quotient_func): + assert self._csr is not None + + c = self._csr + + ib = m._y_label_index[key] + + occurances = c[:,ib].nonzero()[0] + + return 1. / quotient_func(c[occurances,ib].toarray()) + + def sym_normalization_factor(key, quotient_func): + assert self._csr is not None + + c = self._csr + + ii = m._label_index[key] + + fw_occurances = c[ii,:].nonzero()[1] + bw_occurances = c[:,ii].nonzero()[0] + + return 1. / quotient_func(np.concatenate( + [c[ii,fw_occurances].toarray().flatten(), + c[bw_occurances,ii].toarray().flatten()] + )) + + def sym_p_a_given_b(key_a, key_b, quot_func = np.max): + assert self._csr is not None + + c = self._csr + + ia = m._label_index[key_a] + ib = m._label_index[key_b] + + v = c[ia,ib] + c[ib,ia] + + return v * self.sym_normalization_factor(key_b, quot_func) + + def fw_p_a_given_b(key_a, key_b, quot_func = np.max): + assert self._csr is not None + + c = self._csr + + ia = m._x_label_index[key_a] + ib = m._y_label_index[key_b] + + v = c[ia,ib] + + return v * self.bw_normalization_factor(key_b, quot_func) + + def bw_p_a_given_b(key_a, key_b, quot_func = np.max): + assert self._csr is not None + + c = self._csr + + ia = m._y_label_index[key_a] + ib = m._x_label_index[key_b] + + v = c[ib,ia] + + return v * self.fw_normalization_factor(key_b, quot_func) + + + if self._sym: + self.get_adjacent = get_sym_adjacent + self.get_sum = sym_sum + self.get_sym_normalization_factor = sym_normalization_factor + self.p_a_given_b = sym_p_a_given_b + + else: + self.get_forward_adjacent = get_forward_adjacent + self.get_backward_adjacent = get_backward_adjacent + + self.get_fw_sum = fw_sum + self.get_bw_sum = bw_sum + + self.get_fw_normalization_factor = fw_normalization_factor + self.get_bw_normalization_factor = bw_normalization_factor + + self.fw_p_a_given_b = fw_p_a_given_b + self.bw_p_a_given_b = bw_p_a_given_b diff --git a/RecipeAnalysis/MatrixGeneration.ipynb b/RecipeAnalysis/MatrixGeneration.ipynb index 9f8d588..67e9ffb 100644 --- a/RecipeAnalysis/MatrixGeneration.ipynb +++ b/RecipeAnalysis/MatrixGeneration.ipynb @@ -1,383 +1 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Matrix Generation" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.append(\"../\")\n", - "from Recipe import Recipe, Ingredient, RecipeGraph\n", - "\n", - "import settings\n", - "import db.db_settings as db_settings\n", - "from db.database_connection import DatabaseConnection\n", - "\n", - "import random\n", - "\n", - "import itertools\n", - "\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "DatabaseConnection(db_settings.db_host,\n", - " db_settings.db_port,\n", - " db_settings.db_user,\n", - " db_settings.db_pw,\n", - " db_settings.db_db,\n", - " db_settings.db_charset)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 8.71 s, sys: 942 ms, total: 9.66 s\n", - "Wall time: 9.77 s\n" - ] - } - ], - "source": [ - "%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import AdjacencyMatrix" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* create Adjacency Matrix" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n", - " mix_m, mix_label = rec_state.get_mixing_matrix()\n", - " act_m, act_a, act_i = rec_state.get_action_matrix()\n", - "\n", - " # create list of tuples: [action, ingredient]\n", - " seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n", - "\n", - " # create list of tuples [ingredient, ingredient]\n", - " seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n", - "\n", - " seen_actions = seen_actions[act_m == 1]\n", - " seen_mixes = seen_mixes[mix_m == 1]\n", - "\n", - " seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n", - " seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n", - " \n", - " seen_base_actions = set()\n", - " seen_base_mixes = set()\n", - " \n", - " for act, ing in seen_actions:\n", - " m_act.add_entry(act, ing.to_json(), 1)\n", - " if (act, ing._base_ingredient) not in seen_base_actions:\n", - " seen_base_actions.add((act, ing._base_ingredient))\n", - " m_base_act.add_entry(act, ing._base_ingredient, 1)\n", - " \n", - " for x,y in seen_mixes:\n", - " xj = x.to_json()\n", - " yj = y.to_json()\n", - " if xj < yj:\n", - " m_mix.add_entry(xj,yj,1)\n", - " if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n", - " seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n", - " m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "m_act = AdjacencyMatrix.adj_matrix()\n", - "m_mix = AdjacencyMatrix.adj_matrix(True)\n", - "m_base_act = AdjacencyMatrix.adj_matrix()\n", - "m_base_mix = AdjacencyMatrix.adj_matrix(True)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "warning: recipe a9dc137b48 has no ingredient! skipping it\n", - "CPU times: user 13min 35s, sys: 3.52 s, total: 13min 39s\n", - "Wall time: 13min 50s\n" - ] - } - ], - "source": [ - "%%time\n", - "for i in range(10000):\n", - " id = random.choice(ids)['id']\n", - " rec = Recipe(id)\n", - " #rec.display_recipe()\n", - " ing = rec.extract_ingredients()\n", - " if len(ing) == 0:\n", - " print(f\"warning: recipe {id} has no ingredient! skipping it\")\n", - " continue\n", - " rec.apply_instructions(debug=False)\n", - " add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "import pickle" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "pickle.dump(m_act, file=open(\"m_act.pickle\", 'wb'))\n", - "pickle.dump(m_mix, file=open(\"m_mix.pickle\", 'wb'))\n", - "pickle.dump(m_base_act, file=open(\"m_base_act.pickle\", 'wb'))\n", - "pickle.dump(m_base_mix, file=open(\"m_base_mix.pickle\", 'wb'))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "c_mix = m_mix.get_csr()\n", - "c_act = m_act.get_csr()\n", - "c_base_mix = m_base_mix.get_csr()\n", - "c_base_act = m_base_act.get_csr()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(65, 64699) (71548, 71548)\n", - "113994 537369\n", - "(65, 4738) (5850, 5850)\n", - "30820 122390\n" - ] - } - ], - "source": [ - "print(c_act.shape, c_mix.shape)\n", - "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n", - "print(c_base_act.shape, c_base_mix.shape)\n", - "print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(64, 63787) (70933, 70933)\n", - "112841 524285\n" - ] - } - ], - "source": [ - "print(c_act.shape, c_mix.shape)\n", - "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "17560" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.sum(c_act.toarray() > 1)" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[1, 1, 0, ..., 0, 0, 0],\n", - " [0, 0, 1, ..., 0, 0, 0],\n", - " [0, 0, 0, ..., 0, 0, 0],\n", - " ...,\n", - " [0, 0, 0, ..., 0, 0, 0],\n", - " [0, 0, 0, ..., 0, 0, 0],\n", - " [0, 0, 0, ..., 0, 0, 0]], dtype=int64)" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* values after 100:\n", - "```\n", - "(53, 1498) (1620, 1620)\n", - "1982 6489\n", - "```\n", - "\n", - "* after 1000:\n", - "```\n", - "(60, 9855) (10946, 10946)\n", - "15446 59943\n", - "```\n", - "\n", - "* after 10000:\n", - "```\n", - "(65, 65235) (72448, 72448)\n", - "114808 546217\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} +{"cells":[{"cell_type":"markdown","metadata":{},"source":"# Matrix Generation"},{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[{"data":{"text/html":" \n "},"metadata":{},"output_type":"display_data"}],"source":"import sys\nsys.path.append(\"../\")\nfrom Recipe import Recipe, Ingredient, RecipeGraph\n\nimport settings\nimport db.db_settings as db_settings\nfrom db.database_connection import DatabaseConnection\n\nimport random\n\nimport itertools\n\nimport numpy as np"},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[{"data":{"text/plain":""},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":"DatabaseConnection(db_settings.db_host,\n db_settings.db_port,\n db_settings.db_user,\n db_settings.db_pw,\n db_settings.db_db,\n db_settings.db_charset)"},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"CPU times: user 8.56 s, sys: 924 ms, total: 9.48 s\nWall time: 9.5 s\n"}],"source":"%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")"},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":"import AdjacencyMatrix"},{"cell_type":"markdown","metadata":{},"source":"* create Adjacency Matrix"},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[],"source":"def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n mix_m, mix_label = rec_state.get_mixing_matrix()\n act_m, act_a, act_i = rec_state.get_action_matrix()\n\n # create list of tuples: [action, ingredient]\n seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n\n # create list of tuples [ingredient, ingredient]\n seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n\n seen_actions = seen_actions[act_m == 1]\n seen_mixes = seen_mixes[mix_m == 1]\n\n seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n \n seen_base_actions = set()\n seen_base_mixes = set()\n \n for act, ing in seen_actions:\n m_act.add_entry(act, ing.to_json(), 1)\n if (act, ing._base_ingredient) not in seen_base_actions:\n seen_base_actions.add((act, ing._base_ingredient))\n m_base_act.add_entry(act, ing._base_ingredient, 1)\n \n for x,y in seen_mixes:\n xj = x.to_json()\n yj = y.to_json()\n if xj < yj:\n m_mix.add_entry(xj,yj,1)\n if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n"},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[],"source":"m_act = AdjacencyMatrix.adj_matrix()\nm_mix = AdjacencyMatrix.adj_matrix(True)\nm_base_act = AdjacencyMatrix.adj_matrix()\nm_base_mix = AdjacencyMatrix.adj_matrix(True)"},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"warning: recipe b4fc8f359d has no ingredient! skipping it\nwarning: recipe f288592241 has no ingredient! skipping it\nwarning: recipe 4dbdc1d0b5 has no ingredient! skipping it\nwarning: recipe 37af7ba84f has no ingredient! skipping it\nwarning: recipe bbbf057e4c has no ingredient! skipping it\nwarning: recipe ebc64e182c has no ingredient! skipping it\nwarning: recipe d271f2815d has no ingredient! skipping it\nwarning: recipe 52f19fe220 has no ingredient! skipping it\nwarning: recipe b4cbec22f0 has no ingredient! skipping it\nwarning: recipe d2b7a7e703 has no ingredient! skipping it\nwarning: recipe 2a28b886fe has no ingredient! skipping it\nwarning: recipe ebbaf84483 has no ingredient! skipping it\nwarning: recipe 90d70c93c3 has no ingredient! skipping it\nwarning: recipe 9e8689ec9e has no ingredient! skipping it\nwarning: recipe da1ebe217b has no ingredient! skipping it\nwarning: recipe 733ae6a2f0 has no ingredient! skipping it\nwarning: recipe cfacb2bf66 has no ingredient! skipping it\nwarning: recipe a995b595f5 has no ingredient! skipping it\nwarning: recipe 1aa2453f8d has no ingredient! skipping it\nwarning: recipe a252fc20a4 has no ingredient! skipping it\nwarning: recipe 69e9a8bedf has no ingredient! skipping it\nwarning: recipe 258ea03e87 has no ingredient! skipping it\nwarning: recipe 3534f4dcbb has no ingredient! skipping it\nwarning: recipe 91b5df4e7b has no ingredient! skipping it\nwarning: recipe 1bfe2f4329 has no ingredient! skipping it\nwarning: recipe 5325af57cc has no ingredient! skipping it\nCPU times: user 2h 17min 59s, sys: 41 s, total: 2h 18min 40s\nWall time: 2h 20min 22s\n"}],"source":"%%time\nfor i in range(100000):\n id = random.choice(ids)['id']\n rec = Recipe(id)\n #rec.display_recipe()\n ing = rec.extract_ingredients()\n if len(ing) == 0:\n print(f\"warning: recipe {id} has no ingredient! skipping it\")\n continue\n rec.apply_instructions(debug=False)\n add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)"},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[{"data":{"text/plain":"99999"},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":"i"},{"cell_type":"code","execution_count":31,"metadata":{},"outputs":[],"source":"import dill"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""},{"cell_type":"code","execution_count":32,"metadata":{},"outputs":[],"source":"dill.dump(m_act, file=open(\"m_act_raw.dill\", 'wb'))\ndill.dump(m_mix, file=open(\"m_mix_raw.dill\", 'wb'))\ndill.dump(m_base_act, file=open(\"m_base_act_raw.dill\", 'wb'))\ndill.dump(m_base_mix, file=open(\"m_base_mix_raw.dill\", 'wb'))"},{"cell_type":"code","execution_count":37,"metadata":{},"outputs":[],"source":"m_act.apply_threshold(10)\nm_mix.apply_threshold(10)\nm_base_act.apply_threshold(50)\nm_base_mix.apply_threshold(50)"},{"cell_type":"code","execution_count":33,"metadata":{},"outputs":[],"source":"c_mix = m_mix.get_csr()\nc_act = m_act.get_csr()\nc_base_mix = m_base_mix.get_csr()\nc_base_act = m_base_act.get_csr()"},{"cell_type":"code","execution_count":39,"metadata":{},"outputs":[],"source":"m_mix.compile()\nm_act.compile()\nm_base_mix.compile()\nm_base_act.compile()\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""},{"cell_type":"code","execution_count":43,"metadata":{},"outputs":[],"source":"dill.dump(m_act, file=open(\"m_act.dill\", 'wb'))\ndill.dump(m_mix, file=open(\"m_mix.dill\", 'wb'))\ndill.dump(m_base_act, file=open(\"m_base_act.dill\", 'wb'))\ndill.dump(m_base_mix, file=open(\"m_base_mix.dill\", 'wb'))"},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"(65, 384445) (433582, 433582)\n749954 4526723\n(65, 17483) (22059, 22059)\n114758 590425\n"}],"source":"print(c_act.shape, c_mix.shape)\nprint(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\nprint(c_base_act.shape, c_base_mix.shape)\nprint(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))"},{"cell_type":"code","execution_count":35,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"(65, 384445) (433582, 433582)\n749954 4526723\n"}],"source":"print(c_act.shape, c_mix.shape)\nprint(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))"},{"cell_type":"code","execution_count":36,"metadata":{},"outputs":[{"data":{"text/plain":"171424"},"execution_count":36,"metadata":{},"output_type":"execute_result"}],"source":"np.sum(c_act.toarray() > 1)"},{"cell_type":"code","execution_count":99,"metadata":{},"outputs":[{"data":{"text/plain":["array([[1, 1, 0, ..., 0, 0, 0],\n"," [0, 0, 1, ..., 0, 0, 0],\n"," [0, 0, 0, ..., 0, 0, 0],\n"," ...,\n"," [0, 0, 0, ..., 0, 0, 0],\n"," [0, 0, 0, ..., 0, 0, 0],\n"," [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"]},"execution_count":99,"metadata":{},"output_type":"execute_result"}],"source":""},{"cell_type":"markdown","metadata":{},"source":["* values after 100:\n","```\n","(53, 1498) (1620, 1620)\n","1982 6489\n","```\n","\n","* after 1000:\n","```\n","(60, 9855) (10946, 10946)\n","15446 59943\n","```\n","\n","* after 10000:\n","```\n","(65, 65235) (72448, 72448)\n","114808 546217\n","```"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"orig_nbformat":2,"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3}} \ No newline at end of file diff --git a/RecipeAnalysis/Recipe Analysis.ipynb b/RecipeAnalysis/Recipe Analysis.ipynb index e443adf..ba4faf4 100644 --- a/RecipeAnalysis/Recipe Analysis.ipynb +++ b/RecipeAnalysis/Recipe Analysis.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -45,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -65,16 +65,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -97,15 +97,15 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.22 s, sys: 1.25 s, total: 9.46 s\n", - "Wall time: 9.56 s\n" + "CPU times: user 9 s, sys: 850 ms, total: 9.85 s\n", + "Wall time: 9.92 s\n" ] } ], @@ -122,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -131,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -217,8 +217,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 9.53 ms, sys: 0 ns, total: 9.53 ms\n", - "Wall time: 8.74 ms\n" + "CPU times: user 10.8 ms, sys: 274 µs, total: 11 ms\n", + "Wall time: 9.98 ms\n" ] } ], @@ -3142,8 +3142,8 @@ 1 ], "range": [ - -0.7394662921348321, - 30.739466292134832 + -0.7173913043478262, + 30.717391304347828 ], "type": "category" }, @@ -4197,8 +4197,8 @@ 1 ], "range": [ - -11.044917257683217, - 15.044917257683217 + -11.06323877068558, + 15.06323877068558 ], "scaleanchor": "x", "scaleratio": 1, @@ -5336,8 +5336,8 @@ 1 ], "range": [ - -0.9618863049095596, - 12.961886304909559 + -0.9702842377260978, + 12.970284237726098 ], "scaleanchor": "x", "scaleratio": 1, @@ -9113,8 +9113,8 @@ 1 ], "range": [ - -0.8529850746268686, - 42.85298507462687 + -0.88507462686567, + 42.88507462686567 ], "type": "category" }, @@ -13399,8 +13399,8 @@ 1 ], "range": [ - -0.9393241167434745, - 43.93932411674348 + -0.9049079754601266, + 43.90490797546013 ], "type": "category" }, @@ -14673,8 +14673,8 @@ 1 ], "range": [ - -13.429078014184398, - 20.429078014184398 + -13.455082742316787, + 20.455082742316787 ], "scaleanchor": "x", "scaleratio": 1, @@ -16670,8 +16670,8 @@ 1 ], "range": [ - -0.7020860495436789, - 30.70208604954368 + -0.681640625, + 30.681640625 ], "type": "category" }, @@ -17759,8 +17759,8 @@ 1 ], "range": [ - -11.55260047281324, - 16.55260047281324 + -11.570921985815605, + 16.570921985815605 ], "scaleanchor": "x", "scaleratio": 1,