{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Matrix Generation" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.7/site-packages/ipykernel_launcher.py:13: TqdmExperimentalWarning:\n", "\n", "Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", "\n" ] } ], "source": [ "import sys\n", "sys.path.append(\"../\")\n", "from Recipe import Recipe, Ingredient, RecipeGraph\n", "\n", "import settings\n", "import db.db_settings as db_settings\n", "from db.database_connection import DatabaseConnection\n", "\n", "import random\n", "\n", "import itertools\n", "\n", "from tqdm.autonotebook import tqdm\n", "\n", "import traceback\n", "\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "DatabaseConnection(db_settings.db_host,\n", " db_settings.db_port,\n", " db_settings.db_user,\n", " db_settings.db_pw,\n", " db_settings.db_db,\n", " db_settings.db_charset)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 8.52 s, sys: 719 ms, total: 9.24 s\n", "Wall time: 9.4 s\n" ] } ], "source": [ "%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import AdjacencyMatrix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* create Adjacency Matrix" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n", " mix_m, mix_label = rec_state.get_mixing_matrix()\n", " act_m, act_a, act_i = rec_state.get_action_matrix()\n", "\n", " # create list of tuples: [action, ingredient]\n", " seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n", "\n", " # create list of tuples [ingredient, ingredient]\n", " seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n", "\n", " seen_actions = seen_actions[act_m == 1]\n", " seen_mixes = seen_mixes[mix_m == 1]\n", "\n", " seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n", " seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n", " \n", " seen_base_actions = set()\n", " seen_base_mixes = set()\n", " \n", " for act, ing in seen_actions:\n", " m_act.add_entry(act, ing.to_json(), 1)\n", " if (act, ing._base_ingredient) not in seen_base_actions:\n", " seen_base_actions.add((act, ing._base_ingredient))\n", " m_base_act.add_entry(act, ing._base_ingredient, 1)\n", " \n", " for x,y in seen_mixes:\n", " xj = x.to_json()\n", " yj = y.to_json()\n", " if xj < yj:\n", " m_mix.add_entry(xj,yj,1)\n", " if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n", " seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n", " m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "m_act = AdjacencyMatrix.adj_matrix()\n", "m_mix = AdjacencyMatrix.adj_matrix(True)\n", "m_base_act = AdjacencyMatrix.adj_matrix()\n", "m_base_mix = AdjacencyMatrix.adj_matrix(True)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "fa6557a61875425495f90a61f88115af", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 8be34a1b83 has no ingredient! skipping it\n", "warning: recipe 5f02077e4c has no ingredient! skipping it\n", "warning: recipe e5c317b3f1 has no ingredient! skipping it\n", "warning: recipe 56016a1985 has no ingredient! skipping it\n", "warning: recipe eb77bd6edc has no ingredient! skipping it\n", "warning: recipe 5ce8c2cedd has no ingredient! skipping it\n", "warning: recipe d7a9aaba3a has no ingredient! skipping it\n", "warning: recipe 5d99d73716 has no ingredient! skipping it\n", "warning: recipe 5a8462b13f has no ingredient! skipping it\n", "an error occured : 'NoneType' object has no attribute 'lower'\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"\", line 4, in \n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n", " self.annotate_sentences()\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in \n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in \n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 54, in word2features\n", " '+1:word.lower=' + word1.lower(),\n", "AttributeError: 'NoneType' object has no attribute 'lower'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 99ab771181 has no ingredient! skipping it\n", "warning: recipe de79b67e1a has no ingredient! skipping it\n", "warning: recipe ebc64e182c has no ingredient! skipping it\n", "warning: recipe 7358870155 has no ingredient! skipping it\n", "warning: recipe 3e06b9bc57 has no ingredient! skipping it\n", "warning: recipe 7c67cd9da7 has no ingredient! skipping it\n", "warning: recipe 1e5911a066 has no ingredient! skipping it\n", "an error occured : 'NoneType' object is not subscriptable\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"\", line 4, in \n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n", " self.annotate_sentences()\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in \n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in \n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 19, in word2features\n", " 'word[-3:]=' + word[-3:],\n", "TypeError: 'NoneType' object is not subscriptable\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 56016a1985 has no ingredient! skipping it\n", "warning: recipe 1d14e148c9 has no ingredient! skipping it\n", "warning: recipe f96375e11c has no ingredient! skipping it\n", "warning: recipe 026e9af79d has no ingredient! skipping it\n", "warning: recipe c884f5d8a0 has no ingredient! skipping it\n", "warning: recipe 76b02fee35 has no ingredient! skipping it\n", "an error occured : 'NoneType' object has no attribute 'lower'\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"\", line 4, in \n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n", " self.annotate_sentences()\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in \n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in \n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 54, in word2features\n", " '+1:word.lower=' + word1.lower(),\n", "AttributeError: 'NoneType' object has no attribute 'lower'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 2fa5b920f4 has no ingredient! skipping it\n", "\n", "CPU times: user 2h 2min 39s, sys: 43.4 s, total: 2h 3min 23s\n", "Wall time: 2h 4min 37s\n" ] } ], "source": [ "%%time\n", "for i in tqdm(range(100000)):\n", " try:\n", " id = random.choice(ids)['id']\n", " rec = Recipe(id)\n", " #rec.display_recipe()\n", " ing = rec.extract_ingredients()\n", " if len(ing) == 0:\n", " print(f\"warning: recipe {id} has no ingredient! skipping it\")\n", " continue\n", " rec.apply_instructions(debug=False)\n", " add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)\n", " except Exception as e:\n", " print(\"an error occured : \" + str(e))\n", " traceback.print_exc()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "99999" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "i" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import dill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "dill.dump(m_act, file=open(\"m_act_raw.dill\", 'wb'))\n", "dill.dump(m_mix, file=open(\"m_mix_raw.dill\", 'wb'))\n", "dill.dump(m_base_act, file=open(\"m_base_act_raw.dill\", 'wb'))\n", "dill.dump(m_base_mix, file=open(\"m_base_mix_raw.dill\", 'wb'))" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "m_act.apply_threshold(10)\n", "m_mix.apply_threshold(10)\n", "m_base_act.apply_threshold(20)\n", "m_base_mix.apply_threshold(20)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "c_mix = m_mix.get_csr()\n", "c_act = m_act.get_csr()\n", "c_base_mix = m_base_mix.get_csr()\n", "c_base_act = m_base_act.get_csr()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "m_mix.compile()\n", "m_act.compile()\n", "m_base_mix.compile()\n", "m_base_act.compile()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "dill.dump(m_act, file=open(\"m_act.dill\", 'wb'))\n", "dill.dump(m_mix, file=open(\"m_mix.dill\", 'wb'))\n", "dill.dump(m_base_act, file=open(\"m_base_act.dill\", 'wb'))\n", "dill.dump(m_base_mix, file=open(\"m_base_mix.dill\", 'wb'))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(64, 237792) (266772, 266772)\n", "13170 17041\n", "(64, 11455) (13528, 13528)\n", "6083 10775\n" ] } ], "source": [ "print(c_act.shape, c_mix.shape)\n", "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n", "print(c_base_act.shape, c_base_mix.shape)\n", "print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(64, 237792) (266772, 266772)\n", "13170 17041\n" ] } ], "source": [ "print(c_act.shape, c_mix.shape)\n", "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "13170" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.sum(c_act.toarray() > 1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* values after 100:\n", "```\n", "(53, 1498) (1620, 1620)\n", "1982 6489\n", "```\n", "\n", "* after 1000:\n", "```\n", "(60, 9855) (10946, 10946)\n", "15446 59943\n", "```\n", "\n", "* after 10000:\n", "```\n", "(65, 65235) (72448, 72448)\n", "114808 546217\n", "```" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "file_extension": ".py", "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.5" }, "mimetype": "text/x-python", "name": "python", "npconvert_exporter": "python", "pygments_lexer": "ipython3", "version": 3 }, "nbformat": 4, "nbformat_minor": 4 }