{"cells":[{"cell_type":"markdown","metadata":{},"source":"# Matrix Generation"},{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[{"data":{"text/html":" \n "},"metadata":{},"output_type":"display_data"}],"source":"import sys\nsys.path.append(\"../\")\nfrom Recipe import Recipe, Ingredient, RecipeGraph\n\nimport settings\nimport db.db_settings as db_settings\nfrom db.database_connection import DatabaseConnection\n\nimport random\n\nimport itertools\n\nimport numpy as np"},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[{"data":{"text/plain":""},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":"DatabaseConnection(db_settings.db_host,\n db_settings.db_port,\n db_settings.db_user,\n db_settings.db_pw,\n db_settings.db_db,\n db_settings.db_charset)"},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"CPU times: user 8.56 s, sys: 924 ms, total: 9.48 s\nWall time: 9.5 s\n"}],"source":"%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")"},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":"import AdjacencyMatrix"},{"cell_type":"markdown","metadata":{},"source":"* create Adjacency Matrix"},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[],"source":"def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n mix_m, mix_label = rec_state.get_mixing_matrix()\n act_m, act_a, act_i = rec_state.get_action_matrix()\n\n # create list of tuples: [action, ingredient]\n seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n\n # create list of tuples [ingredient, ingredient]\n seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n\n seen_actions = seen_actions[act_m == 1]\n seen_mixes = seen_mixes[mix_m == 1]\n\n seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n \n seen_base_actions = set()\n seen_base_mixes = set()\n \n for act, ing in seen_actions:\n m_act.add_entry(act, ing.to_json(), 1)\n if (act, ing._base_ingredient) not in seen_base_actions:\n seen_base_actions.add((act, ing._base_ingredient))\n m_base_act.add_entry(act, ing._base_ingredient, 1)\n \n for x,y in seen_mixes:\n xj = x.to_json()\n yj = y.to_json()\n if xj < yj:\n m_mix.add_entry(xj,yj,1)\n if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n"},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[],"source":"m_act = AdjacencyMatrix.adj_matrix()\nm_mix = AdjacencyMatrix.adj_matrix(True)\nm_base_act = AdjacencyMatrix.adj_matrix()\nm_base_mix = AdjacencyMatrix.adj_matrix(True)"},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"warning: recipe b4fc8f359d has no ingredient! skipping it\nwarning: recipe f288592241 has no ingredient! skipping it\nwarning: recipe 4dbdc1d0b5 has no ingredient! skipping it\nwarning: recipe 37af7ba84f has no ingredient! skipping it\nwarning: recipe bbbf057e4c has no ingredient! skipping it\nwarning: recipe ebc64e182c has no ingredient! skipping it\nwarning: recipe d271f2815d has no ingredient! skipping it\nwarning: recipe 52f19fe220 has no ingredient! skipping it\nwarning: recipe b4cbec22f0 has no ingredient! skipping it\nwarning: recipe d2b7a7e703 has no ingredient! skipping it\nwarning: recipe 2a28b886fe has no ingredient! skipping it\nwarning: recipe ebbaf84483 has no ingredient! skipping it\nwarning: recipe 90d70c93c3 has no ingredient! skipping it\nwarning: recipe 9e8689ec9e has no ingredient! skipping it\nwarning: recipe da1ebe217b has no ingredient! skipping it\nwarning: recipe 733ae6a2f0 has no ingredient! skipping it\nwarning: recipe cfacb2bf66 has no ingredient! skipping it\nwarning: recipe a995b595f5 has no ingredient! skipping it\nwarning: recipe 1aa2453f8d has no ingredient! skipping it\nwarning: recipe a252fc20a4 has no ingredient! skipping it\nwarning: recipe 69e9a8bedf has no ingredient! skipping it\nwarning: recipe 258ea03e87 has no ingredient! skipping it\nwarning: recipe 3534f4dcbb has no ingredient! skipping it\nwarning: recipe 91b5df4e7b has no ingredient! skipping it\nwarning: recipe 1bfe2f4329 has no ingredient! skipping it\nwarning: recipe 5325af57cc has no ingredient! skipping it\nCPU times: user 2h 17min 59s, sys: 41 s, total: 2h 18min 40s\nWall time: 2h 20min 22s\n"}],"source":"%%time\nfor i in range(100000):\n id = random.choice(ids)['id']\n rec = Recipe(id)\n #rec.display_recipe()\n ing = rec.extract_ingredients()\n if len(ing) == 0:\n print(f\"warning: recipe {id} has no ingredient! skipping it\")\n continue\n rec.apply_instructions(debug=False)\n add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)"},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[{"data":{"text/plain":"99999"},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":"i"},{"cell_type":"code","execution_count":31,"metadata":{},"outputs":[],"source":"import dill"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""},{"cell_type":"code","execution_count":32,"metadata":{},"outputs":[],"source":"dill.dump(m_act, file=open(\"m_act_raw.dill\", 'wb'))\ndill.dump(m_mix, file=open(\"m_mix_raw.dill\", 'wb'))\ndill.dump(m_base_act, file=open(\"m_base_act_raw.dill\", 'wb'))\ndill.dump(m_base_mix, file=open(\"m_base_mix_raw.dill\", 'wb'))"},{"cell_type":"code","execution_count":37,"metadata":{},"outputs":[],"source":"m_act.apply_threshold(10)\nm_mix.apply_threshold(10)\nm_base_act.apply_threshold(50)\nm_base_mix.apply_threshold(50)"},{"cell_type":"code","execution_count":33,"metadata":{},"outputs":[],"source":"c_mix = m_mix.get_csr()\nc_act = m_act.get_csr()\nc_base_mix = m_base_mix.get_csr()\nc_base_act = m_base_act.get_csr()"},{"cell_type":"code","execution_count":39,"metadata":{},"outputs":[],"source":"m_mix.compile()\nm_act.compile()\nm_base_mix.compile()\nm_base_act.compile()\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""},{"cell_type":"code","execution_count":43,"metadata":{},"outputs":[],"source":"dill.dump(m_act, file=open(\"m_act.dill\", 'wb'))\ndill.dump(m_mix, file=open(\"m_mix.dill\", 'wb'))\ndill.dump(m_base_act, file=open(\"m_base_act.dill\", 'wb'))\ndill.dump(m_base_mix, file=open(\"m_base_mix.dill\", 'wb'))"},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"(65, 384445) (433582, 433582)\n749954 4526723\n(65, 17483) (22059, 22059)\n114758 590425\n"}],"source":"print(c_act.shape, c_mix.shape)\nprint(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\nprint(c_base_act.shape, c_base_mix.shape)\nprint(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))"},{"cell_type":"code","execution_count":35,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"(65, 384445) (433582, 433582)\n749954 4526723\n"}],"source":"print(c_act.shape, c_mix.shape)\nprint(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))"},{"cell_type":"code","execution_count":36,"metadata":{},"outputs":[{"data":{"text/plain":"171424"},"execution_count":36,"metadata":{},"output_type":"execute_result"}],"source":"np.sum(c_act.toarray() > 1)"},{"cell_type":"code","execution_count":99,"metadata":{},"outputs":[{"data":{"text/plain":["array([[1, 1, 0, ..., 0, 0, 0],\n"," [0, 0, 1, ..., 0, 0, 0],\n"," [0, 0, 0, ..., 0, 0, 0],\n"," ...,\n"," [0, 0, 0, ..., 0, 0, 0],\n"," [0, 0, 0, ..., 0, 0, 0],\n"," [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"]},"execution_count":99,"metadata":{},"output_type":"execute_result"}],"source":""},{"cell_type":"markdown","metadata":{},"source":["* values after 100:\n","```\n","(53, 1498) (1620, 1620)\n","1982 6489\n","```\n","\n","* after 1000:\n","```\n","(60, 9855) (10946, 10946)\n","15446 59943\n","```\n","\n","* after 10000:\n","```\n","(65, 65235) (72448, 72448)\n","114808 546217\n","```"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"orig_nbformat":2,"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3}}