paper version of Algorithm

This commit is contained in:
Jonas Weinz
2020-02-16 13:20:04 +01:00
parent 6115df7945
commit 3f075ff941
14 changed files with 36128 additions and 41184 deletions

View File

@ -0,0 +1,77 @@
groups = {
'heat':'heat',
'cook':'heat',
'bake':'heat',
'grill':'heat',
'melt':'heat',
'blend':None,
'beat':'prepare',
'spread':None,
'cool':'cool',
'brown':'heat',
'cut':'prepare',
'chill':'cool',
'drain':None,
'boil':'heat',
'simmer':'heat',
'pour':None,
'freeze':'cool',
'saute':'heat',
'rinse':'prepare',
'warm':'heat',
'wash':'prepare',
'knead':'prepare',
'peel':'prepare',
'parboil':'heat',
'break':'prepare',
'broil':'heat',
'scorch':'heat',
'skim':None,
'fry':'heat',
'refrigerate':'cool',
'burn':'heat',
'thicken':None,
'grate':'prepare',
'brush':'prepare',
'open':'prepare',
'crack':'prepare',
'poach':'heat',
'slice':'prepare',
'whisk':None,
'dice':'prepare',
'marinate':None,
'whip':None,
'sour':None,
'soak':None,
'steam':'heat',
'chop':'prepare',
'mince':None,
'mash':'prepare',
'squeeze':'prepare',
'wipe':'prepare',
'thaw':'prepare',
'curdle':'heat',
'sweeten':None,
'baste':None,
'carve':None,
'grind':'prepare',
'debone':'prepare',
'steep':None,
'clarify':None,
'macerate':'prepare',
'crumple':'prepare',
'braise':'heat',
'gut':None,
'bury':None,
'spread':None,
'place':None
}
inverse_groups = {}
for g in set(groups.values()):
if g is not None:
inverse_groups[g] = []
for a,g in groups.items():
if g is not None:
inverse_groups[g].append(a)

View File

@ -9,7 +9,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@ -91,7 +91,7 @@
"outputs": [],
"source": [
"groups = {\n",
" 'place':None,\n",
" #'place':None,\n",
" 'heat':'heat',\n",
" 'cook':'heat',\n",
" 'bake':'heat',\n",
@ -99,7 +99,6 @@
" 'melt':'heat',\n",
" 'blend':None,\n",
" 'beat':'prepare',\n",
" 'spread':None,\n",
" 'cool':'cool',\n",
" 'brown':'heat',\n",
" 'cut':'prepare',\n",
@ -152,6 +151,7 @@
" 'steep':None,\n",
" 'clarify':None,\n",
" 'macerate':'prepare',\n",
" #'spread':None,\n",
" 'crumple':'prepare',\n",
" 'braise':'heat',\n",
" 'gut':None,\n",
@ -187,22 +187,26 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6b079a15f1b94d9e9ebce8ad6eb82058",
"model_id": "77d9643f1116425eb40c8664edca0bf9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=527544.0), HTML(value='')))"
"HBox(children=(FloatProgress(value=0.0, max=467050.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
"ename": "KeyError",
"evalue": "'spread'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-6-8156ba150c10>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mgrouped_ing\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mIngredient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ming\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_base_ingredient\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0ming\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_action_set\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mgrouped_ing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroups\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mgrouped_act\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroups\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: 'spread'"
]
}
],
@ -227,31 +231,9 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b5b6c8ff52d1407495ef7fcd3ff409a8",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=2123892.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"outputs": [],
"source": [
"c = m_mix.get_csr()\n",
"\n",

File diff suppressed because it is too large Load Diff

View File

@ -75,7 +75,7 @@
{
"data": {
"text/plain": [
"<db.database_connection.DatabaseConnection at 0x7fbc4c131510>"
"<db.database_connection.DatabaseConnection at 0x7f3635212850>"
]
},
"execution_count": 2,
@ -101,8 +101,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 8.68 s, sys: 778 ms, total: 9.46 s\n",
"Wall time: 12.5 s\n"
"CPU times: user 8.52 s, sys: 719 ms, total: 9.24 s\n",
"Wall time: 9.4 s\n"
]
}
],
@ -187,7 +187,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "70396063ce4140b48105751d0ece88c9",
"model_id": "fa6557a61875425495f90a61f88115af",
"version_major": 2,
"version_minor": 0
},
@ -202,9 +202,15 @@
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 0a14762efd has no ingredient! skipping it\n",
"warning: recipe 361b00dc50 has no ingredient! skipping it\n",
"warning: recipe e3b081a317 has no ingredient! skipping it\n",
"warning: recipe 8be34a1b83 has no ingredient! skipping it\n",
"warning: recipe 5f02077e4c has no ingredient! skipping it\n",
"warning: recipe e5c317b3f1 has no ingredient! skipping it\n",
"warning: recipe 56016a1985 has no ingredient! skipping it\n",
"warning: recipe eb77bd6edc has no ingredient! skipping it\n",
"warning: recipe 5ce8c2cedd has no ingredient! skipping it\n",
"warning: recipe d7a9aaba3a has no ingredient! skipping it\n",
"warning: recipe 5d99d73716 has no ingredient! skipping it\n",
"warning: recipe 5a8462b13f has no ingredient! skipping it\n",
"an error occured : 'NoneType' object has no attribute 'lower'\n"
]
},
@ -235,30 +241,83 @@
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 80dd4d10ab has no ingredient! skipping it\n",
"warning: recipe ac1ab87418 has no ingredient! skipping it\n",
"warning: recipe fa5b33cadc has no ingredient! skipping it\n",
"warning: recipe 80eb0385d0 has no ingredient! skipping it\n",
"warning: recipe 2dfb05c466 has no ingredient! skipping it\n",
"warning: recipe 7b372faa16 has no ingredient! skipping it\n",
"warning: recipe f53fe916f2 has no ingredient! skipping it\n",
"warning: recipe b4ff30e1de has no ingredient! skipping it\n",
"warning: recipe cf2300c7ac has no ingredient! skipping it\n",
"warning: recipe d09e7e82f9 has no ingredient! skipping it\n",
"warning: recipe edbdc9c52f has no ingredient! skipping it\n",
"warning: recipe 4db0dadfce has no ingredient! skipping it\n",
"warning: recipe d9933872c6 has no ingredient! skipping it\n",
"warning: recipe 37af7ba84f has no ingredient! skipping it\n",
"warning: recipe cfc02f6bd7 has no ingredient! skipping it\n",
"warning: recipe fac985c58f has no ingredient! skipping it\n",
"warning: recipe 91308ac1b2 has no ingredient! skipping it\n",
"warning: recipe d3ff2bf3f9 has no ingredient! skipping it\n",
"warning: recipe ed9a0d0d51 has no ingredient! skipping it\n",
"warning: recipe 52f19fe220 has no ingredient! skipping it\n",
"warning: recipe 6fa7839c14 has no ingredient! skipping it\n",
"warning: recipe 99ab771181 has no ingredient! skipping it\n",
"warning: recipe de79b67e1a has no ingredient! skipping it\n",
"warning: recipe ebc64e182c has no ingredient! skipping it\n",
"warning: recipe 7358870155 has no ingredient! skipping it\n",
"warning: recipe 3e06b9bc57 has no ingredient! skipping it\n",
"warning: recipe 7c67cd9da7 has no ingredient! skipping it\n",
"warning: recipe 1e5911a066 has no ingredient! skipping it\n",
"an error occured : 'NoneType' object is not subscriptable\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"<timed exec>\", line 4, in <module>\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n",
" self.annotate_sentences()\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n",
" self._annotate_sentences(self._sentences, self.predict_labels())\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in <listcomp>\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in <listcomp>\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 19, in word2features\n",
" 'word[-3:]=' + word[-3:],\n",
"TypeError: 'NoneType' object is not subscriptable\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 56016a1985 has no ingredient! skipping it\n",
"warning: recipe 1d14e148c9 has no ingredient! skipping it\n",
"warning: recipe f96375e11c has no ingredient! skipping it\n",
"warning: recipe 026e9af79d has no ingredient! skipping it\n",
"warning: recipe c884f5d8a0 has no ingredient! skipping it\n",
"warning: recipe 76b02fee35 has no ingredient! skipping it\n",
"an error occured : 'NoneType' object has no attribute 'lower'\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"<timed exec>\", line 4, in <module>\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 888, in __init__\n",
" self.annotate_sentences()\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 981, in annotate_sentences\n",
" self._annotate_sentences(self._sentences, self.predict_labels())\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in predict_labels\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"/home/jonas/Dokumente/gitRepos/master_thesis/RecipeAnalysis/Recipe.py\", line 933, in <listcomp>\n",
" features = [sent2features(sent) for sent in self._sentences]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in sent2features\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 87, in <listcomp>\n",
" return [word2features(sent, i) for i in range(len(sent))]\n",
" File \"../Tagging/crf_data_generator.py\", line 54, in word2features\n",
" '+1:word.lower=' + word1.lower(),\n",
"AttributeError: 'NoneType' object has no attribute 'lower'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe 2fa5b920f4 has no ingredient! skipping it\n",
"\n",
"CPU times: user 1h 57min 25s, sys: 36.8 s, total: 1h 58min 2s\n",
"Wall time: 1h 58min 41s\n"
"CPU times: user 2h 2min 39s, sys: 43.4 s, total: 2h 3min 23s\n",
"Wall time: 2h 4min 37s\n"
]
}
],
@ -282,7 +341,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 10,
"metadata": {},
"outputs": [
{
@ -291,7 +350,7 @@
"99999"
]
},
"execution_count": 8,
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@ -302,7 +361,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@ -318,7 +377,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@ -330,19 +389,19 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"m_act.apply_threshold(5)\n",
"m_mix.apply_threshold(5)\n",
"m_base_act.apply_threshold(10)\n",
"m_base_mix.apply_threshold(10)"
"m_act.apply_threshold(10)\n",
"m_mix.apply_threshold(10)\n",
"m_base_act.apply_threshold(20)\n",
"m_base_mix.apply_threshold(20)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
@ -354,7 +413,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
@ -373,7 +432,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
@ -385,17 +444,17 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(65, 272623) (293753, 293753)\n",
"30381 42885\n",
"(65, 11807) (13603, 13603)\n",
"10725 19775\n"
"(64, 237792) (266772, 266772)\n",
"13170 17041\n",
"(64, 11455) (13528, 13528)\n",
"6083 10775\n"
]
}
],
@ -408,15 +467,15 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(65, 272623) (293753, 293753)\n",
"30381 42885\n"
"(64, 237792) (266772, 266772)\n",
"13170 17041\n"
]
}
],
@ -427,16 +486,16 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"30381"
"13170"
]
},
"execution_count": 18,
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}