Better Initialization

This commit is contained in:
Jonas Weinz
2019-12-01 14:04:07 +01:00
parent 020685c86a
commit 9c406c3e48
14 changed files with 15621 additions and 1647 deletions

View File

@ -0,0 +1 @@
{"cells":[{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[{"data":{"text/html":" <script type=\"text/javascript\">\n window.PlotlyConfig = {MathJaxConfig: 'local'};\n if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n if (typeof require !== 'undefined') {\n require.undef(\"plotly\");\n requirejs.config({\n paths: {\n 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n }\n });\n require(['plotly'], function(Plotly) {\n window._Plotly = Plotly;\n });\n }\n </script>\n "},"metadata":{},"output_type":"display_data"},{"data":{"text/html":" <script type=\"text/javascript\">\n window.PlotlyConfig = {MathJaxConfig: 'local'};\n if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n if (typeof require !== 'undefined') {\n require.undef(\"plotly\");\n requirejs.config({\n paths: {\n 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n }\n });\n require(['plotly'], function(Plotly) {\n window._Plotly = Plotly;\n });\n }\n </script>\n "},"metadata":{},"output_type":"display_data"}],"source":"import sys\nsys.path.append(\"../\")\n\nimport settings\n\nimport pycrfsuite\n\nimport json\n\nimport db.db_settings as db_settings\nfrom db.database_connection import DatabaseConnection\n\nfrom Tagging.conllu_generator import ConlluGenerator\nfrom Tagging.crf_data_generator import *\n\nfrom RecipeAnalysis.Recipe import Ingredient\n\nfrom difflib import SequenceMatcher\n\nimport numpy as np\n\nimport plotly.graph_objs as go\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nfrom plotly.subplots import make_subplots\ninit_notebook_mode(connected=True)\n\nfrom graphviz import Digraph\n\nimport itertools\n\nimport random\n\nimport plotly.io as pio\npio.renderers.default = \"jupyterlab\"\n\nfrom IPython.display import Markdown, HTML, display\n\nfrom copy import deepcopy"},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[],"source":"import dill\nm_act = dill.load(open(\"m_act.dill\", \"rb\"))\nm_mix = dill.load(open(\"m_mix.dill\", \"rb\"))\nm_base_act = dill.load(open(\"m_base_act.dill\", \"rb\"))\nm_base_mix = dill.load(open(\"m_base_mix.dill\", \"rb\"))\n\n#m_act.apply_threshold(3)\n#m_mix.apply_threshold(3)\n#m_base_act.apply_threshold(5)\n#m_base_mix.apply_threshold(5)\n\n\n#c_act = m_act.get_csr()\n#c_mix = m_mix.get_csr()\n#c_base_act = m_base_act.get_csr()\n#c_base_mix = m_base_mix.get_csr()\n\nc_act = m_act._csr\nc_mix = m_mix._csr\nc_base_act = m_base_act._csr\nc_base_mix = m_base_mix._csr"},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":"base_ingredients = m_base_mix.get_labels()\nactions = m_act.get_labels()[0]"},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":"## get most used ingredients"},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[],"source":"ings = []\ning_sum = []\n\nfor ing in base_ingredients:\n ings.append(ing)\n ing_sum.append(m_base_mix.get_sum(ing))"},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[],"source":"ing_sort = np.argsort(-np.array(ing_sum))"},{"cell_type":"code","execution_count":23,"metadata":{},"outputs":[],"source":"w = np.array(ing_sum)[ing_sort[:20]]\ntop_ings = np.array(ings)[ing_sort[:20]]"},{"cell_type":"code","execution_count":24,"metadata":{},"outputs":[{"data":{"text/plain":"array(['salt', 'sugar', 'water', 'butter', 'onion', 'egg', 'flour',\n 'olive oil', 'milk', 'sprinkle', 'pepper', 'cream', 'garlic clove',\n 'cheese', 'tomato', 'dough', 'sauce', 'black pepper', 'roll',\n 'vanilla extract'], dtype='<U148')"},"execution_count":24,"metadata":{},"output_type":"execute_result"}],"source":"top_ings"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"orig_nbformat":2,"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3}}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,355 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Further Refinement of raw Adjacency Matrices"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append(\"../\")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.7/site-packages/ipykernel_launcher.py:5: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
" \"\"\"\n"
]
},
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
" </script>\n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import dill\n",
"import numpy as np\n",
"import settings\n",
"import AdjacencyMatrix\n",
"from tqdm.autonotebook import tqdm\n",
"from Recipe import Ingredient"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"m_act = dill.load(open(\"m_act_raw.dill\", \"rb\"))\n",
"m_mix = dill.load(open(\"m_mix_raw.dill\", \"rb\"))\n",
"m_base_act = dill.load(open(\"m_base_act_raw.dill\", \"rb\"))\n",
"m_base_mix = dill.load(open(\"m_base_mix_raw.dill\", \"rb\"))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Grouping Actions"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"groups = {\n",
" 'place':None,\n",
" 'heat':'heat',\n",
" 'cook':'heat',\n",
" 'bake':'heat',\n",
" 'grill':'heat',\n",
" 'melt':'heat',\n",
" 'blend':None,\n",
" 'beat':'prepare',\n",
" 'spread':None,\n",
" 'cool':'cool',\n",
" 'brown':'heat',\n",
" 'cut':'prepare',\n",
" 'chill':'cool',\n",
" 'drain':None,\n",
" 'boil':'heat',\n",
" 'simmer':'heat',\n",
" 'pour':None,\n",
" 'freeze':'cool',\n",
" 'saute':'heat',\n",
" 'rinse':'prepare',\n",
" 'warm':'heat',\n",
" 'wash':'prepare',\n",
" 'knead':'prepare',\n",
" 'peel':'prepare',\n",
" 'parboil':'heat',\n",
" 'break':'prepare',\n",
" 'broil':'heat',\n",
" 'scorch':'heat',\n",
" 'skim':None,\n",
" 'fry':'heat',\n",
" 'refrigerate':'cool',\n",
" 'burn':'heat',\n",
" 'thicken':None,\n",
" 'grate':'prepare',\n",
" 'brush':'prepare',\n",
" 'open':'prepare',\n",
" 'crack':'prepare',\n",
" 'poach':'heat',\n",
" 'slice':'prepare',\n",
" 'whisk':None,\n",
" 'dice':'prepare',\n",
" 'marinate':None,\n",
" 'whip':None,\n",
" 'sour':None,\n",
" 'soak':None,\n",
" 'steam':'heat',\n",
" 'chop':'prepare',\n",
" 'mince':None,\n",
" 'mash':'prepare',\n",
" 'squeeze':'prepare',\n",
" 'wipe':'prepare',\n",
" 'thaw':'prepare',\n",
" 'curdle':'heat',\n",
" 'sweeten':None,\n",
" 'baste':None,\n",
" 'carve':None,\n",
" 'grind':'prepare',\n",
" 'debone':'prepare',\n",
" 'steep':None,\n",
" 'clarify':None,\n",
" 'macerate':'prepare',\n",
" 'crumple':'prepare',\n",
" 'braise':'heat',\n",
" 'gut':None,\n",
" 'bury':None\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* now refactor the matrices to new versions that only contain those groups"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"# create new matrices:\n",
"m_grouped_act = AdjacencyMatrix.adj_matrix()\n",
"m_grouped_mix = AdjacencyMatrix.adj_matrix(True)\n",
"m_grouped_base_act = AdjacencyMatrix.adj_matrix()\n",
"#m_grouped_base_mix = AdjacencyMatrix.adj_matrix(True)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2db5dd15c87740729f16f5c258707db2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=741659), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"c = m_act.get_csr()\n",
"\n",
"label_acts, labels_ings = m_act.get_labels()\n",
"acts, ings = c.nonzero()\n",
"for i_act,j_ing in tqdm(zip(acts,ings), total=len(acts)):\n",
" ing = Ingredient.from_json(labels_ings[j_ing])\n",
" act = label_acts[i_act]\n",
" \n",
" grouped_ing = Ingredient(ing._base_ingredient)\n",
" for a in ing._action_set:\n",
" grouped_ing.apply_action(groups[a])\n",
" \n",
" grouped_act = groups[act]\n",
" \n",
" m_grouped_act.add_entry(grouped_act, grouped_ing.to_json(),1)\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1d3f802944654954a6abbf8cb70f116e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=4472620), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"c = m_mix.get_csr()\n",
"\n",
"labels_ings = m_mix.get_labels()\n",
"ings_a, ings_b = c.nonzero()\n",
"for i_ing,j_ing in tqdm(zip(ings_a,ings_b), total=len(ings_a)):\n",
" ing_a = Ingredient.from_json(labels_ings[i_ing])\n",
" ing_b = Ingredient.from_json(labels_ings[j_ing])\n",
" \n",
" grouped_ing_a = Ingredient(ing_a._base_ingredient)\n",
" for a in ing_a._action_set:\n",
" grouped_ing_a.apply_action(groups[a])\n",
" \n",
" grouped_ing_b = Ingredient(ing_b._base_ingredient)\n",
" for a in ing_b._action_set:\n",
" grouped_ing_b.apply_action(groups[a])\n",
" \n",
" m_grouped_mix.add_entry(grouped_ing_a.to_json(), grouped_ing_b.to_json(),1)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "92548999e3a6427bb33af20d3e53d9ef",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=114804), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"c = m_base_act.get_csr()\n",
"\n",
"label_acts, labels_ings = m_base_act.get_labels()\n",
"acts, ings = c.nonzero()\n",
"for i_act,j_ing in tqdm(zip(acts,ings), total=len(acts)):\n",
" base_ing = labels_ings[j_ing]\n",
" act = label_acts[i_act]\n",
" \n",
" grouped_act = groups[act]\n",
" \n",
" m_grouped_base_act.add_entry(grouped_act, base_ing,1)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"dill.dump(m_grouped_act, file=open(\"m_grouped_act_raw.dill\", 'wb'))\n",
"dill.dump(m_grouped_mix, file=open(\"m_grouped_mix_raw.dill\", 'wb'))\n",
"dill.dump(m_grouped_base_act, file=open(\"m_grouped_base_act_raw.dill\", 'wb'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -64,7 +64,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
"version": "3.7.5rc1"
}
},
"nbformat": 4,

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -264,6 +264,8 @@
" \n",
" if touch:\n",
" self._last_touched_instruction = instruction_number\n",
" \n",
" return self\n",
" \n",
" def similarity(self, ingredient, use_actions=False, action_factor = 0.5):\n",
" sim,_,_ = string_similarity(self._base_ingredient, ingredient._base_ingredient)\n",
@ -1434,7 +1436,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
"version": "3.7.5"
}
},
"nbformat": 4,

View File

@ -160,6 +160,8 @@ class Ingredient(object):
if touch:
self._last_touched_instruction = instruction_number
return self
def similarity(self, ingredient, use_actions=False, action_factor = 0.5):
sim,_,_ = string_similarity(self._base_ingredient, ingredient._base_ingredient)