{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Matrix Generation" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.7/site-packages/ipykernel_launcher.py:13: TqdmExperimentalWarning:\n", "\n", "Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", "\n" ] } ], "source": [ "import sys\n", "sys.path.append(\"../\")\n", "from Recipe import Recipe, Ingredient, RecipeGraph\n", "\n", "import settings\n", "import db.db_settings as db_settings\n", "from db.database_connection import DatabaseConnection\n", "\n", "import random\n", "\n", "import itertools\n", "\n", "from tqdm.autonotebook import tqdm\n", "\n", "import traceback\n", "\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "DatabaseConnection(db_settings.db_host,\n", " db_settings.db_port,\n", " db_settings.db_user,\n", " db_settings.db_pw,\n", " db_settings.db_db,\n", " db_settings.db_charset)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 8.68 s, sys: 778 ms, total: 9.46 s\n", "Wall time: 12.5 s\n" ] } ], "source": [ "%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import AdjacencyMatrix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* create Adjacency Matrix" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n", " mix_m, mix_label = rec_state.get_mixing_matrix()\n", " act_m, act_a, act_i = rec_state.get_action_matrix()\n", "\n", " # create list of tuples: [action, ingredient]\n", " seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n", "\n", " # create list of tuples [ingredient, ingredient]\n", " seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n", "\n", " seen_actions = seen_actions[act_m == 1]\n", " seen_mixes = seen_mixes[mix_m == 1]\n", "\n", " seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n", " seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n", " \n", " seen_base_actions = set()\n", " seen_base_mixes = set()\n", " \n", " for act, ing in seen_actions:\n", " m_act.add_entry(act, ing.to_json(), 1)\n", " if (act, ing._base_ingredient) not in seen_base_actions:\n", " seen_base_actions.add((act, ing._base_ingredient))\n", " m_base_act.add_entry(act, ing._base_ingredient, 1)\n", " \n", " for x,y in seen_mixes:\n", " xj = x.to_json()\n", " yj = y.to_json()\n", " if xj < yj:\n", " m_mix.add_entry(xj,yj,1)\n", " if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n", " seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n", " m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "m_act = AdjacencyMatrix.adj_matrix()\n", "m_mix = AdjacencyMatrix.adj_matrix(True)\n", "m_base_act = AdjacencyMatrix.adj_matrix()\n", "m_base_mix = AdjacencyMatrix.adj_matrix(True)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "70396063ce4140b48105751d0ece88c9", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 0a14762efd has no ingredient! skipping it\n", "warning: recipe 361b00dc50 has no ingredient! skipping it\n", "warning: recipe e3b081a317 has no ingredient! skipping it\n", "an error occured : 'NoneType' object has no attribute 'lower'\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"\", line 4, in \n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n", " self.annotate_sentences()\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n", " self._annotate_sentences(self._sentences, self.predict_labels())\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in \n", " features = [sent2features(sent) for sent in self._sentences]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 87, in \n", " return [word2features(sent, i) for i in range(len(sent))]\n", " File \"../Tagging/crf_data_generator.py\", line 54, in word2features\n", " '+1:word.lower=' + word1.lower(),\n", "AttributeError: 'NoneType' object has no attribute 'lower'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "warning: recipe 80dd4d10ab has no ingredient! skipping it\n", "warning: recipe ac1ab87418 has no ingredient! skipping it\n", "warning: recipe fa5b33cadc has no ingredient! skipping it\n", "warning: recipe 80eb0385d0 has no ingredient! skipping it\n", "warning: recipe 2dfb05c466 has no ingredient! skipping it\n", "warning: recipe 7b372faa16 has no ingredient! skipping it\n", "warning: recipe f53fe916f2 has no ingredient! skipping it\n", "warning: recipe b4ff30e1de has no ingredient! skipping it\n", "warning: recipe cf2300c7ac has no ingredient! skipping it\n", "warning: recipe d09e7e82f9 has no ingredient! skipping it\n", "warning: recipe edbdc9c52f has no ingredient! skipping it\n", "warning: recipe 4db0dadfce has no ingredient! skipping it\n", "warning: recipe d9933872c6 has no ingredient! skipping it\n", "warning: recipe 37af7ba84f has no ingredient! skipping it\n", "warning: recipe cfc02f6bd7 has no ingredient! skipping it\n", "warning: recipe fac985c58f has no ingredient! skipping it\n", "warning: recipe 91308ac1b2 has no ingredient! skipping it\n", "warning: recipe d3ff2bf3f9 has no ingredient! skipping it\n", "warning: recipe ed9a0d0d51 has no ingredient! skipping it\n", "warning: recipe 52f19fe220 has no ingredient! skipping it\n", "warning: recipe 6fa7839c14 has no ingredient! skipping it\n", "\n", "CPU times: user 1h 57min 25s, sys: 36.8 s, total: 1h 58min 2s\n", "Wall time: 1h 58min 41s\n" ] } ], "source": [ "%%time\n", "for i in tqdm(range(100000)):\n", " try:\n", " id = random.choice(ids)['id']\n", " rec = Recipe(id)\n", " #rec.display_recipe()\n", " ing = rec.extract_ingredients()\n", " if len(ing) == 0:\n", " print(f\"warning: recipe {id} has no ingredient! skipping it\")\n", " continue\n", " rec.apply_instructions(debug=False)\n", " add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)\n", " except Exception as e:\n", " print(\"an error occured : \" + str(e))\n", " traceback.print_exc()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "99999" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "i" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import dill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "dill.dump(m_act, file=open(\"m_act_raw.dill\", 'wb'))\n", "dill.dump(m_mix, file=open(\"m_mix_raw.dill\", 'wb'))\n", "dill.dump(m_base_act, file=open(\"m_base_act_raw.dill\", 'wb'))\n", "dill.dump(m_base_mix, file=open(\"m_base_mix_raw.dill\", 'wb'))" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "m_act.apply_threshold(5)\n", "m_mix.apply_threshold(5)\n", "m_base_act.apply_threshold(10)\n", "m_base_mix.apply_threshold(10)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "c_mix = m_mix.get_csr()\n", "c_act = m_act.get_csr()\n", "c_base_mix = m_base_mix.get_csr()\n", "c_base_act = m_base_act.get_csr()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "m_mix.compile()\n", "m_act.compile()\n", "m_base_mix.compile()\n", "m_base_act.compile()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "dill.dump(m_act, file=open(\"m_act.dill\", 'wb'))\n", "dill.dump(m_mix, file=open(\"m_mix.dill\", 'wb'))\n", "dill.dump(m_base_act, file=open(\"m_base_act.dill\", 'wb'))\n", "dill.dump(m_base_mix, file=open(\"m_base_mix.dill\", 'wb'))" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(65, 272623) (293753, 293753)\n", "30381 42885\n", "(65, 11807) (13603, 13603)\n", "10725 19775\n" ] } ], "source": [ "print(c_act.shape, c_mix.shape)\n", "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n", "print(c_base_act.shape, c_base_mix.shape)\n", "print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(65, 272623) (293753, 293753)\n", "30381 42885\n" ] } ], "source": [ "print(c_act.shape, c_mix.shape)\n", "print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "30381" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.sum(c_act.toarray() > 1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* values after 100:\n", "```\n", "(53, 1498) (1620, 1620)\n", "1982 6489\n", "```\n", "\n", "* after 1000:\n", "```\n", "(60, 9855) (10946, 10946)\n", "15446 59943\n", "```\n", "\n", "* after 10000:\n", "```\n", "(65, 65235) (72448, 72448)\n", "114808 546217\n", "```" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "file_extension": ".py", "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.5" }, "mimetype": "text/x-python", "name": "python", "npconvert_exporter": "python", "pygments_lexer": "ipython3", "version": 3 }, "nbformat": 4, "nbformat_minor": 4 }