{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Matrix Generation" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.7/site-packages/ipykernel_launcher.py:13: TqdmExperimentalWarning:\n", "\n", "Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", "\n" ] } ], "source": [ "import sys\n", "sys.path.append(\"../\")\n", "from Recipe import Recipe, Ingredient, RecipeGraph\n", "\n", "import settings\n", "import db.db_settings as db_settings\n", "from db.database_connection import DatabaseConnection\n", "\n", "import random\n", "\n", "import itertools\n", "\n", "from tqdm.autonotebook import tqdm\n", "\n", "import traceback\n", "\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "DatabaseConnection(db_settings.db_host,\n", " db_settings.db_port,\n", " db_settings.db_user,\n", " db_settings.db_pw,\n", " db_settings.db_db,\n", " db_settings.db_charset)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 8.17 s, sys: 632 ms, total: 8.8 s\n", "Wall time: 8.84 s\n" ] } ], "source": [ "%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import AdjacencyMatrix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* create Adjacency Matrix" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n", " mix_m, mix_label = rec_state.get_mixing_matrix()\n", " act_m, act_a, act_i = rec_state.get_action_matrix()\n", "\n", " # create list of tuples: [action, ingredient]\n", " seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n", "\n", " # create list of tuples [ingredient, ingredient]\n", " seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n", "\n", " seen_actions = seen_actions[act_m == 1]\n", " seen_mixes = seen_mixes[mix_m == 1]\n", "\n", " seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n", " seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n", " \n", " seen_base_actions = set()\n", " seen_base_mixes = set()\n", " \n", " for act, ing in seen_actions:\n", " m_act.add_entry(act, ing.to_json(), 1)\n", " if (act, ing._base_ingredient) not in seen_base_actions:\n", " seen_base_actions.add((act, ing._base_ingredient))\n", " m_base_act.add_entry(act, ing._base_ingredient, 1)\n", " \n", " for x,y in seen_mixes:\n", " xj = x.to_json()\n", " yj = y.to_json()\n", " if xj < yj:\n", " m_mix.add_entry(xj,yj,1)\n", " if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n", " seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n", " m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "m_act = AdjacencyMatrix.adj_matrix()\n", "m_mix = AdjacencyMatrix.adj_matrix(True)\n", "m_base_act = AdjacencyMatrix.adj_matrix()\n", "m_base_mix = AdjacencyMatrix.adj_matrix(True)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "96abe66e3b4340ea8031c54333c28431", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(IntProgress(value=0, max=100000), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 7a6657f1be has no ingredient! skipping it\n", "warning: recipe 985bf99cb0 has no ingredient! skipping it\n", "warning: recipe 7b06b89a00 has no ingredient! skipping it\n", "warning: recipe 5d99d73716 has no ingredient! skipping it\n", "warning: recipe 5ea97ce121 has no ingredient! skipping it\n", "warning: recipe 219c14975a has no ingredient! skipping it\n", "warning: recipe 312dab2c6e has no ingredient! skipping it\n", "warning: recipe 9847df394d has no ingredient! skipping it\n", "warning: recipe c456eca29f has no ingredient! skipping it\n", "warning: recipe 63809adf7d has no ingredient! skipping it\n", "warning: recipe d8097248d7 has no ingredient! skipping it\n", "warning: recipe 4f6df45cbe has no ingredient! skipping it\n", "warning: recipe fa8ffd2e9f has no ingredient! skipping it\n", "warning: recipe 1274925e80 has no ingredient! skipping it\n", "warning: recipe 780a3bfa7f has no ingredient! skipping it\n", "warning: recipe 2f58557f49 has no ingredient! skipping it\n", "warning: recipe 6e0201877c has no ingredient! skipping it\n", "warning: recipe 4be587da96 has no ingredient! skipping it\n", "warning: recipe b4fc8f359d has no ingredient! skipping it\n", "warning: recipe 83dc617c2a has no ingredient! skipping it\n", "warning: recipe fe406ab97d has no ingredient! skipping it\n", "an error occured : 'NoneType' object is not subscriptable\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"\", line 4, in \n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n", " self.annotate_sentences()\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in \n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in \n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 19, in word2features\n", " 'word[-3:]=' + word[-3:],\n", "TypeError: 'NoneType' object is not subscriptable\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 83e51c3bc6 has no ingredient! skipping it\n", "warning: recipe 4178a59023 has no ingredient! skipping it\n", "warning: recipe 37711ac6d0 has no ingredient! skipping it\n", "warning: recipe f93a2bfa73 has no ingredient! skipping it\n", "warning: recipe 5f02077e4c has no ingredient! skipping it\n", "warning: recipe 3cb76a27d1 has no ingredient! skipping it\n", "warning: recipe 4c786ec3d6 has no ingredient! skipping it\n", "warning: recipe 90d70c93c3 has no ingredient! skipping it\n", "warning: recipe a5f6838027 has no ingredient! skipping it\n", "\n", "CPU times: user 2h 14min 8s, sys: 42.9 s, total: 2h 14min 51s\n", "Wall time: 2h 15min 40s\n" ] } ], "source": [ "%%time\n", "for i in tqdm(range(100000)):\n", " try:\n", " id = random.choice(ids)['id']\n", " rec = Recipe(id)\n", " #rec.display_recipe()\n", " ing = rec.extract_ingredients()\n", " if len(ing) == 0:\n", " print(f\"warning: recipe {id} has no ingredient! skipping it\")\n", " continue\n", " rec.apply_instructions(debug=False)\n", " add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)\n", " except Exception as e:\n", " print(\"an error occured : \" + str(e))\n", " traceback.print_exc()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "99999" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "i" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "import dill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "dill.dump(m_act, file=open(\"m_act_raw.dill\", 'wb'))\n", "dill.dump(m_mix, file=open(\"m_mix_raw.dill\", 'wb'))\n", "dill.dump(m_base_act, file=open(\"m_base_act_raw.dill\", 'wb'))\n", "dill.dump(m_base_mix, file=open(\"m_base_mix_raw.dill\", 'wb'))" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "m_act.apply_threshold(5)\n", "m_mix.apply_threshold(5)\n", "m_base_act.apply_threshold(10)\n", "m_base_mix.apply_threshold(10)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "c_mix = m_mix.get_csr()\n", "c_act = m_act.get_csr()\n", "c_base_mix = m_base_mix.get_csr()\n", "c_base_act = m_base_act.get_csr()" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "m_mix.compile()\n", "m_act.compile()\n", "m_base_mix.compile()\n", "m_base_act.compile()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "dill.dump(m_act, file=open(\"m_act.dill\", 'wb'))\n", "dill.dump(m_mix, file=open(\"m_mix.dill\", 'wb'))\n", "dill.dump(m_base_act, file=open(\"m_base_act.dill\", 'wb'))\n", "dill.dump(m_base_mix, file=open(\"m_base_mix.dill\", 'wb'))" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(65, 378992) (428558, 428558)\n", "43498 74724\n", "(65, 17533) (22178, 22178)\n", "15465 42881\n" ] } ], "source": [ "print(c_act.shape, c_mix.shape)\n", "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n", "print(c_base_act.shape, c_base_mix.shape)\n", "print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(65, 384445) (433582, 433582)\n", "749954 4526723\n" ] } ], "source": [ "print(c_act.shape, c_mix.shape)\n", "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "43498" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.sum(c_act.toarray() > 1)" ] }, { "cell_type": "code", "execution_count": 99, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[1, 1, 0, ..., 0, 0, 0],\n", " [0, 0, 1, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " ...,\n", " [0, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0]], dtype=int64)" ] }, "execution_count": 99, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* values after 100:\n", "```\n", "(53, 1498) (1620, 1620)\n", "1982 6489\n", "```\n", "\n", "* after 1000:\n", "```\n", "(60, 9855) (10946, 10946)\n", "15446 59943\n", "```\n", "\n", "* after 10000:\n", "```\n", "(65, 65235) (72448, 72448)\n", "114808 546217\n", "```" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "file_extension": ".py", "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.5" }, "mimetype": "text/x-python", "name": "python", "npconvert_exporter": "python", "pygments_lexer": "ipython3", "version": 3 }, "nbformat": 4, "nbformat_minor": 4 }