Better Initialization
This commit is contained in:
		
							
								
								
									
										1
									
								
								RecipeAnalysis/AdjMatrixTests.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1
									
								
								RecipeAnalysis/AdjMatrixTests.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1 @@ | ||||
| {"cells":[{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[{"data":{"text/html":"        <script type=\"text/javascript\">\n        window.PlotlyConfig = {MathJaxConfig: 'local'};\n        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n        if (typeof require !== 'undefined') {\n        require.undef(\"plotly\");\n        requirejs.config({\n            paths: {\n                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n            }\n        });\n        require(['plotly'], function(Plotly) {\n            window._Plotly = Plotly;\n        });\n        }\n        </script>\n        "},"metadata":{},"output_type":"display_data"},{"data":{"text/html":"        <script type=\"text/javascript\">\n        window.PlotlyConfig = {MathJaxConfig: 'local'};\n        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n        if (typeof require !== 'undefined') {\n        require.undef(\"plotly\");\n        requirejs.config({\n            paths: {\n                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n            }\n        });\n        require(['plotly'], function(Plotly) {\n            window._Plotly = Plotly;\n        });\n        }\n        </script>\n        "},"metadata":{},"output_type":"display_data"}],"source":"import sys\nsys.path.append(\"../\")\n\nimport settings\n\nimport pycrfsuite\n\nimport json\n\nimport db.db_settings as db_settings\nfrom db.database_connection import DatabaseConnection\n\nfrom Tagging.conllu_generator import ConlluGenerator\nfrom Tagging.crf_data_generator import *\n\nfrom RecipeAnalysis.Recipe import Ingredient\n\nfrom difflib import SequenceMatcher\n\nimport numpy as np\n\nimport plotly.graph_objs as go\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nfrom plotly.subplots import make_subplots\ninit_notebook_mode(connected=True)\n\nfrom graphviz import Digraph\n\nimport itertools\n\nimport random\n\nimport plotly.io as pio\npio.renderers.default = \"jupyterlab\"\n\nfrom IPython.display import Markdown, HTML, display\n\nfrom copy import deepcopy"},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[],"source":"import dill\nm_act = dill.load(open(\"m_act.dill\", \"rb\"))\nm_mix = dill.load(open(\"m_mix.dill\", \"rb\"))\nm_base_act = dill.load(open(\"m_base_act.dill\", \"rb\"))\nm_base_mix = dill.load(open(\"m_base_mix.dill\", \"rb\"))\n\n#m_act.apply_threshold(3)\n#m_mix.apply_threshold(3)\n#m_base_act.apply_threshold(5)\n#m_base_mix.apply_threshold(5)\n\n\n#c_act = m_act.get_csr()\n#c_mix = m_mix.get_csr()\n#c_base_act = m_base_act.get_csr()\n#c_base_mix = m_base_mix.get_csr()\n\nc_act = m_act._csr\nc_mix = m_mix._csr\nc_base_act = m_base_act._csr\nc_base_mix = m_base_mix._csr"},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":"base_ingredients = m_base_mix.get_labels()\nactions = m_act.get_labels()[0]"},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":"## get most used ingredients"},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[],"source":"ings = []\ning_sum = []\n\nfor ing in base_ingredients:\n    ings.append(ing)\n    ing_sum.append(m_base_mix.get_sum(ing))"},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[],"source":"ing_sort = np.argsort(-np.array(ing_sum))"},{"cell_type":"code","execution_count":23,"metadata":{},"outputs":[],"source":"w = np.array(ing_sum)[ing_sort[:20]]\ntop_ings = np.array(ings)[ing_sort[:20]]"},{"cell_type":"code","execution_count":24,"metadata":{},"outputs":[{"data":{"text/plain":"array(['salt', 'sugar', 'water', 'butter', 'onion', 'egg', 'flour',\n       'olive oil', 'milk', 'sprinkle', 'pepper', 'cream', 'garlic clove',\n       'cheese', 'tomato', 'dough', 'sauce', 'black pepper', 'roll',\n       'vanilla extract'], dtype='<U148')"},"execution_count":24,"metadata":{},"output_type":"execute_result"}],"source":"top_ings"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"orig_nbformat":2,"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3}} | ||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							
							
								
								
									
										355
									
								
								RecipeAnalysis/AdjacencyMatrixRefinement.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										355
									
								
								RecipeAnalysis/AdjacencyMatrixRefinement.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,355 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "# Further Refinement of raw Adjacency Matrices" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import sys\n", | ||||
|     "sys.path.append(\"../\")" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "/home/jonas/.local/lib/python3.7/site-packages/ipykernel_launcher.py:5: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", | ||||
|       "  \"\"\"\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/html": [ | ||||
|        "        <script type=\"text/javascript\">\n", | ||||
|        "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n", | ||||
|        "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n", | ||||
|        "        if (typeof require !== 'undefined') {\n", | ||||
|        "        require.undef(\"plotly\");\n", | ||||
|        "        requirejs.config({\n", | ||||
|        "            paths: {\n", | ||||
|        "                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n", | ||||
|        "            }\n", | ||||
|        "        });\n", | ||||
|        "        require(['plotly'], function(Plotly) {\n", | ||||
|        "            window._Plotly = Plotly;\n", | ||||
|        "        });\n", | ||||
|        "        }\n", | ||||
|        "        </script>\n", | ||||
|        "        " | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import dill\n", | ||||
|     "import numpy as np\n", | ||||
|     "import settings\n", | ||||
|     "import AdjacencyMatrix\n", | ||||
|     "from tqdm.autonotebook import tqdm\n", | ||||
|     "from Recipe import Ingredient" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "m_act = dill.load(open(\"m_act_raw.dill\", \"rb\"))\n", | ||||
|     "m_mix = dill.load(open(\"m_mix_raw.dill\", \"rb\"))\n", | ||||
|     "m_base_act = dill.load(open(\"m_base_act_raw.dill\", \"rb\"))\n", | ||||
|     "m_base_mix = dill.load(open(\"m_base_mix_raw.dill\", \"rb\"))" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "## Grouping Actions" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "groups = {\n", | ||||
|     "    'place':None,\n", | ||||
|     "     'heat':'heat',\n", | ||||
|     "     'cook':'heat',\n", | ||||
|     "     'bake':'heat',\n", | ||||
|     "     'grill':'heat',\n", | ||||
|     "     'melt':'heat',\n", | ||||
|     "     'blend':None,\n", | ||||
|     "     'beat':'prepare',\n", | ||||
|     "     'spread':None,\n", | ||||
|     "     'cool':'cool',\n", | ||||
|     "     'brown':'heat',\n", | ||||
|     "     'cut':'prepare',\n", | ||||
|     "     'chill':'cool',\n", | ||||
|     "     'drain':None,\n", | ||||
|     "     'boil':'heat',\n", | ||||
|     "     'simmer':'heat',\n", | ||||
|     "     'pour':None,\n", | ||||
|     "     'freeze':'cool',\n", | ||||
|     "     'saute':'heat',\n", | ||||
|     "     'rinse':'prepare',\n", | ||||
|     "     'warm':'heat',\n", | ||||
|     "     'wash':'prepare',\n", | ||||
|     "     'knead':'prepare',\n", | ||||
|     "     'peel':'prepare',\n", | ||||
|     "     'parboil':'heat',\n", | ||||
|     "     'break':'prepare',\n", | ||||
|     "     'broil':'heat',\n", | ||||
|     "     'scorch':'heat',\n", | ||||
|     "     'skim':None,\n", | ||||
|     "     'fry':'heat',\n", | ||||
|     "     'refrigerate':'cool',\n", | ||||
|     "     'burn':'heat',\n", | ||||
|     "     'thicken':None,\n", | ||||
|     "     'grate':'prepare',\n", | ||||
|     "     'brush':'prepare',\n", | ||||
|     "     'open':'prepare',\n", | ||||
|     "     'crack':'prepare',\n", | ||||
|     "     'poach':'heat',\n", | ||||
|     "     'slice':'prepare',\n", | ||||
|     "     'whisk':None,\n", | ||||
|     "     'dice':'prepare',\n", | ||||
|     "     'marinate':None,\n", | ||||
|     "     'whip':None,\n", | ||||
|     "     'sour':None,\n", | ||||
|     "     'soak':None,\n", | ||||
|     "     'steam':'heat',\n", | ||||
|     "     'chop':'prepare',\n", | ||||
|     "     'mince':None,\n", | ||||
|     "     'mash':'prepare',\n", | ||||
|     "     'squeeze':'prepare',\n", | ||||
|     "     'wipe':'prepare',\n", | ||||
|     "     'thaw':'prepare',\n", | ||||
|     "     'curdle':'heat',\n", | ||||
|     "     'sweeten':None,\n", | ||||
|     "     'baste':None,\n", | ||||
|     "     'carve':None,\n", | ||||
|     "     'grind':'prepare',\n", | ||||
|     "     'debone':'prepare',\n", | ||||
|     "     'steep':None,\n", | ||||
|     "     'clarify':None,\n", | ||||
|     "     'macerate':'prepare',\n", | ||||
|     "     'crumple':'prepare',\n", | ||||
|     "     'braise':'heat',\n", | ||||
|     "     'gut':None,\n", | ||||
|     "     'bury':None\n", | ||||
|     "}" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* now refactor the matrices to new versions that only contain those groups" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "# create new matrices:\n", | ||||
|     "m_grouped_act = AdjacencyMatrix.adj_matrix()\n", | ||||
|     "m_grouped_mix = AdjacencyMatrix.adj_matrix(True)\n", | ||||
|     "m_grouped_base_act = AdjacencyMatrix.adj_matrix()\n", | ||||
|     "#m_grouped_base_mix = AdjacencyMatrix.adj_matrix(True)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "2db5dd15c87740729f16f5c258707db2", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/plain": [ | ||||
|        "HBox(children=(IntProgress(value=0, max=741659), HTML(value='')))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "c = m_act.get_csr()\n", | ||||
|     "\n", | ||||
|     "label_acts, labels_ings = m_act.get_labels()\n", | ||||
|     "acts, ings = c.nonzero()\n", | ||||
|     "for i_act,j_ing in tqdm(zip(acts,ings), total=len(acts)):\n", | ||||
|     "    ing = Ingredient.from_json(labels_ings[j_ing])\n", | ||||
|     "    act = label_acts[i_act]\n", | ||||
|     "    \n", | ||||
|     "    grouped_ing = Ingredient(ing._base_ingredient)\n", | ||||
|     "    for a in ing._action_set:\n", | ||||
|     "        grouped_ing.apply_action(groups[a])\n", | ||||
|     "    \n", | ||||
|     "    grouped_act = groups[act]\n", | ||||
|     "    \n", | ||||
|     "    m_grouped_act.add_entry(grouped_act, grouped_ing.to_json(),1)\n", | ||||
|     "    " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "1d3f802944654954a6abbf8cb70f116e", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/plain": [ | ||||
|        "HBox(children=(IntProgress(value=0, max=4472620), HTML(value='')))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "c = m_mix.get_csr()\n", | ||||
|     "\n", | ||||
|     "labels_ings = m_mix.get_labels()\n", | ||||
|     "ings_a, ings_b = c.nonzero()\n", | ||||
|     "for i_ing,j_ing in tqdm(zip(ings_a,ings_b), total=len(ings_a)):\n", | ||||
|     "    ing_a = Ingredient.from_json(labels_ings[i_ing])\n", | ||||
|     "    ing_b = Ingredient.from_json(labels_ings[j_ing])\n", | ||||
|     "    \n", | ||||
|     "    grouped_ing_a = Ingredient(ing_a._base_ingredient)\n", | ||||
|     "    for a in ing_a._action_set:\n", | ||||
|     "        grouped_ing_a.apply_action(groups[a])\n", | ||||
|     "    \n", | ||||
|     "    grouped_ing_b = Ingredient(ing_b._base_ingredient)\n", | ||||
|     "    for a in ing_b._action_set:\n", | ||||
|     "        grouped_ing_b.apply_action(groups[a])\n", | ||||
|     "        \n", | ||||
|     "    m_grouped_mix.add_entry(grouped_ing_a.to_json(), grouped_ing_b.to_json(),1)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "92548999e3a6427bb33af20d3e53d9ef", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/plain": [ | ||||
|        "HBox(children=(IntProgress(value=0, max=114804), HTML(value='')))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "c = m_base_act.get_csr()\n", | ||||
|     "\n", | ||||
|     "label_acts, labels_ings = m_base_act.get_labels()\n", | ||||
|     "acts, ings = c.nonzero()\n", | ||||
|     "for i_act,j_ing in tqdm(zip(acts,ings), total=len(acts)):\n", | ||||
|     "    base_ing = labels_ings[j_ing]\n", | ||||
|     "    act = label_acts[i_act]\n", | ||||
|     "    \n", | ||||
|     "    grouped_act = groups[act]\n", | ||||
|     "    \n", | ||||
|     "    m_grouped_base_act.add_entry(grouped_act, base_ing,1)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "dill.dump(m_grouped_act, file=open(\"m_grouped_act_raw.dill\", 'wb'))\n", | ||||
|     "dill.dump(m_grouped_mix, file=open(\"m_grouped_mix_raw.dill\", 'wb'))\n", | ||||
|     "dill.dump(m_grouped_base_act, file=open(\"m_grouped_base_act_raw.dill\", 'wb'))" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.7.5" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 4 | ||||
| } | ||||
| @ -64,7 +64,7 @@ | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.7.3" | ||||
|    "version": "3.7.5rc1" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  | ||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							| @ -264,6 +264,8 @@ | ||||
|     "        \n", | ||||
|     "        if touch:\n", | ||||
|     "            self._last_touched_instruction = instruction_number\n", | ||||
|     "        \n", | ||||
|     "        return self\n", | ||||
|     "    \n", | ||||
|     "    def similarity(self, ingredient, use_actions=False, action_factor = 0.5):\n", | ||||
|     "        sim,_,_ = string_similarity(self._base_ingredient, ingredient._base_ingredient)\n", | ||||
| @ -1434,7 +1436,7 @@ | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.7.3" | ||||
|    "version": "3.7.5" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  | ||||
| @ -160,6 +160,8 @@ class Ingredient(object): | ||||
|          | ||||
|         if touch: | ||||
|             self._last_touched_instruction = instruction_number | ||||
|          | ||||
|         return self | ||||
|      | ||||
|     def similarity(self, ingredient, use_actions=False, action_factor = 0.5): | ||||
|         sim,_,_ = string_similarity(self._base_ingredient, ingredient._base_ingredient) | ||||
|  | ||||
		Reference in New Issue
	
	Block a user