master-thesis/Tagging/Recipe_Tagging_Analysis.ipynb

1468 lines
49 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Recipe Tagging analysis\n",
"\n",
"analyse and visualize preconfigured conllu files"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"\n",
"from conllu import parse\n",
"\n",
"sys.path.insert(0,'..')\n",
"import settings\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"conllu_path = \"recipes0.conllu\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"with open(conllu_path, 'r') as f:\n",
" data = f.readlines()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# newdoc\n",
"1 Set set NOUN NNP _ _ _ _ food_type=action \n",
"2 oven oven ADV RB _ _ _ _ _ \n",
"3 to to PRT TO _ _ _ _ _ \n",
"4 350 350 NUM CD _ _ _ _ _ \n",
"5 degrees degre NOUN NNS _ _ _ _ _ \n",
"6 F F NOUN NNP _ _ _ _ _ \n",
"7 . . . . _ _ _ _ _ \n",
"\n",
"8 Grease greas VERB VB _ _ _ _ _ \n",
"9 a a DET DT _ _ _ _ _ \n",
"10 13 13 NUM CD _ _ _ _ _ \n",
"11 x x ADJ JJ _ _ _ _ _ \n",
"12 9-inch 9-inch ADJ JJ _ _ _ _ _ \n",
"13 baking bake VERB VBG _ _ _ _ food_type=action \n",
"14 dish dish NOUN NN _ _ _ _ _ \n",
"15 . . . . _ _ _ _ _ \n",
"\n",
"16 Cook cook VERB VB _ _ _ _ food_type=action \n",
"17 the the DET DT _ _ _ _ _ \n",
"\n"
]
}
],
"source": [
"print(\"\".join(data[0:20]))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"ts = parse(\"\".join(data))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"sentence = ts[0]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"list"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"type(ts)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"\n",
"* visualize tags:"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"from tagging_tools import print_visualized_tags"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Set</span> oven to 350 degrees F . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:yellow'>Grease</span> a 13 x 9-inch <span style='background-color:orange'>baking</span> dish . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Cook</span> the <span style='background-color:cyan'>macaroni</span> in <span style='background-color:cyan'>boiling_water</span> until al dente ( <span style='background-color:yellow'>do</span> not <span style='background-color:orange'>cook</span> until very soft ) <span style='background-color:orange'>drain</span> , then <span style='background-color:orange'>place</span> into a bowl . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Place</span> the <span style='background-color:cyan'>milk</span> , <span style='background-color:orange'>flour</span> , <span style='background-color:cyan'>onion_salt</span> and <span style='background-color:cyan'>garlic_powder</span> in a 1-quart jar ; <span style='background-color:orange'>cover</span> tightly and <span style='background-color:yellow'>shake</span> vigorously for 1 minute ; <span style='background-color:orange'>pour</span> the mixture over the <span style='background-color:cyan'>macaroni</span> and <span style='background-color:orange'>add</span> in 3-1/2 cups <span style='background-color:cyan'>shredded_cheddar_cheese</span> ; <span style='background-color:orange'>mix</span> thoroughly to <span style='background-color:yellow'>combine</span> , then <span style='background-color:orange'>spread</span> into the <span style='background-color:orange'>baking</span> dish . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Sprinkle</span> with breadcrumbs then <span style='background-color:yellow'>remaining</span> 1 cup of <span style='background-color:cyan'>cheddar_cheese</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Drizzle 1/4 cup <span style='background-color:cyan'>melted_butter</span> or <span style='background-color:cyan'>margarine</span> over the <span style='background-color:cyan'>top</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Bake</span> <span style='background-color:yellow'>uncovered</span> for 45 minutes or until golden <span style='background-color:orange'>brown</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Stir</span> together the <span style='background-color:orange'>sugar</span> , <span style='background-color:orange'>salt</span> , <span style='background-color:cyan'>pepper</span> , <span style='background-color:cyan'>cumin</span> and <span style='background-color:cyan'>mustard</span> in small bowl . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Place</span> <span style='background-color:cyan'>salmon</span> so that the side the skin <span style='background-color:yellow'>was</span> <span style='background-color:orange'>removed</span> from <span style='background-color:yellow'>is</span> down . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Press <span style='background-color:orange'>sugar</span> mixture evenly onto <span style='background-color:cyan'>fish</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Heat</span> <span style='background-color:cyan'>oil</span> over medium <span style='background-color:orange'>heat</span> in large nonstick or cast-iron skillet . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"When <span style='background-color:cyan'>hot</span> , <span style='background-color:orange'>place</span> <span style='background-color:cyan'>fish</span> rub-side down in pan . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Cook</span> until <span style='background-color:cyan'>brown_sugar</span> <span style='background-color:orange'>rub</span> dissolves and darkens slightly , <span style='background-color:yellow'>being</span> careful not to <span style='background-color:orange'>burn</span> , about 4 minutes . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Flip <span style='background-color:cyan'>fish</span> and <span style='background-color:orange'>cook</span> to <span style='background-color:yellow'>medium</span> doneness , about 1 minute longer . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Mix</span> all ingredients in bowl in order <span style='background-color:yellow'>listed</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Make small or medium size balls out of mixture and <span style='background-color:orange'>place</span> on a greased <span style='background-color:cyan'>cookie</span> sheet <span style='background-color:yellow'>sprayed</span> with Pam . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"For <span style='background-color:cyan'>extra</span> happy horses you <span style='background-color:yellow'>can</span> <span style='background-color:orange'>sprinkle</span> a little bit of <span style='background-color:cyan'>honey</span> on them before <span style='background-color:orange'>cooking</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Bake</span> 350F for 15-20 minutes and <span style='background-color:yellow'>make</span> sure they <span style='background-color:yellow'>do</span> n't <span style='background-color:orange'>burn</span> ! "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"The day before you <span style='background-color:yellow'>want</span> to <span style='background-color:yellow'>eat</span> this <span style='background-color:orange'>cut</span> your <span style='background-color:cyan'>rolls</span> into 8 medium to <span style='background-color:yellow'>thick</span> <span style='background-color:orange'>slices</span> , <span style='background-color:orange'>put</span> on a light <span style='background-color:orange'>setting</span> and just slightly <span style='background-color:orange'>toast</span> them or <span style='background-color:orange'>put</span> them into the oven and slightly pre <span style='background-color:orange'>toast</span> them . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"They <span style='background-color:yellow'>will</span> <span style='background-color:yellow'>be</span> <span style='background-color:orange'>grilled</span> the next day again so <span style='background-color:yellow'>do</span> not <span style='background-color:orange'>brown</span> them to much , alternatively you <span style='background-color:yellow'>can</span> <span style='background-color:yellow'>buy</span> larger ready <span style='background-color:orange'>cooked</span> <span style='background-color:cyan'>crostini</span> , next day <span style='background-color:orange'>cut</span> the <span style='background-color:cyan'>cheese</span> to size to <span style='background-color:yellow'>fit</span> the <span style='background-color:orange'>slices</span> of <span style='background-color:orange'>bread</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"The next day , <span style='background-color:orange'>slice</span> some <span style='background-color:cyan'>cheese</span> the same size as each <span style='background-color:orange'>slice</span> of bread.. "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Layer first a <span style='background-color:orange'>slice</span> of <span style='background-color:orange'>bread</span> , then a <span style='background-color:orange'>slice</span> of <span style='background-color:cyan'>cheese</span> then a <span style='background-color:orange'>slice</span> of <span style='background-color:orange'>bread</span> and another <span style='background-color:orange'>slice</span> of <span style='background-color:cyan'>cheese</span> into ovenproof <span style='background-color:cyan'>soup</span> <span style='background-color:yellow'>plates</span> or <span style='background-color:yellow'>bowls</span> "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>prepare</span> you <span style='background-color:cyan'>soup</span> by <span style='background-color:yellow'>making</span> a clear <span style='background-color:cyan'>stock</span> of preference , <span style='background-color:cyan'>vegetable</span> , <span style='background-color:cyan'>chicken</span> , <span style='background-color:cyan'>fish</span> or <span style='background-color:cyan'>beef</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"You <span style='background-color:yellow'>can</span> <span style='background-color:yellow'>do</span> this by <span style='background-color:orange'>skimming</span> the grey foam as soon a it <span style='background-color:yellow'>comes</span> up with a very small net strainer or a ladle . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"This <span style='background-color:yellow'>needs</span> to <span style='background-color:yellow'>be</span> <span style='background-color:yellow'>done</span> as soon as it <span style='background-color:yellow'>comes</span> up as it first <span style='background-color:yellow'>starts</span> <span style='background-color:orange'>boiling</span> or the <span style='background-color:cyan'>soup</span> <span style='background-color:yellow'>will</span> <span style='background-color:yellow'>go</span> cloudy . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Season</span> to your liking . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Add</span> a little <span style='background-color:cyan'>sherry</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Drain</span> the <span style='background-color:cyan'>stock</span> , carefully <span style='background-color:yellow'>ladle</span> the clear <span style='background-color:cyan'>stock</span> over the <span style='background-color:orange'>bread</span> and <span style='background-color:cyan'>cheese</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Put</span> the bowls into the oven and <span style='background-color:orange'>melt</span> the <span style='background-color:cyan'>cheese</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Hope you <span style='background-color:yellow'>enjoy</span> "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"The <span style='background-color:orange'>bread</span> full of <span style='background-color:cyan'>soup</span> <span style='background-color:yellow'>is</span> the best bit "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:yellow'>Using</span> a 1/2 c ice-cream <span style='background-color:orange'>scoop</span> , <span style='background-color:orange'>put</span> one <span style='background-color:orange'>scoop</span> of <span style='background-color:cyan'>sherbet</span> on each <span style='background-color:cyan'>cookie</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Freeze</span> until firm , at least 15 minutes . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Heat</span> the oven to 475 degrees F . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Whip</span> the <span style='background-color:cyan'>egg_whites</span> in a glass or metal bowl until soft peaks form . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Add</span> the <span style='background-color:orange'>sugar</span> and <span style='background-color:orange'>beat</span> until stiff peaks form . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Place</span> the frozen <span style='background-color:cyan'>sherbet</span> <span style='background-color:cyan'>topped</span> <span style='background-color:cyan'>cookies</span> about 2 '' apart on a wire rack on a <span style='background-color:orange'>baking</span> sheet . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Spread</span> the <span style='background-color:cyan'>meringue</span> over each of the <span style='background-color:cyan'>sherbets</span> and also the <span style='background-color:cyan'>cookie</span> edges . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Bake</span> until lightly <span style='background-color:orange'>browned</span> , about 2 minutes . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Serve</span> immediately . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Spread</span> 1/2 tablespoon <span style='background-color:cyan'>butter</span> on one side of each piece of <span style='background-color:orange'>toasted</span> <span style='background-color:cyan'>Italian_bread</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Set</span> aside . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Heat</span> <span style='background-color:cyan'>olive_oil</span> in a skillet over high <span style='background-color:orange'>heat</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"When <span style='background-color:cyan'>oil</span> <span style='background-color:yellow'>begins</span> to <span style='background-color:yellow'>smoke</span> , <span style='background-color:orange'>pour</span> <span style='background-color:cyan'>scallops</span> into pan . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Cook</span> for 30 seconds without <span style='background-color:orange'>stirring</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"Toss <span style='background-color:cyan'>scallops</span> in pan and <span style='background-color:orange'>stir</span> in <span style='background-color:cyan'>garlic</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Cook</span> and <span style='background-color:orange'>stir</span> until fragrant , about 30 seconds . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Stir</span> <span style='background-color:cyan'>wine</span> and <span style='background-color:cyan'>lemon_juice</span> into <span style='background-color:cyan'>scallops</span> , <span style='background-color:yellow'>bring</span> to a <span style='background-color:orange'>boil</span> , and <span style='background-color:orange'>cook</span> for about 30 seconds . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Stir</span> <span style='background-color:cyan'>parsley</span> and cold <span style='background-color:cyan'>butter</span> into <span style='background-color:cyan'>scallops</span> and <span style='background-color:orange'>remove</span> from <span style='background-color:orange'>heat</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"When <span style='background-color:cyan'>butter</span> <span style='background-color:orange'>melts</span> , <span style='background-color:orange'>stir</span> in <span style='background-color:orange'>salt</span> , <span style='background-color:cyan'>black_pepper</span> , and <span style='background-color:cyan'>cayenne_pepper</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Spoon</span> <span style='background-color:cyan'>scallops</span> over <span style='background-color:cyan'>buttered</span> <span style='background-color:orange'>toast</span> and <span style='background-color:orange'>serve</span> immediately . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"1 . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Whisk</span> together <span style='background-color:cyan'>brownie</span> <span style='background-color:orange'>mix</span> , <span style='background-color:orange'>flour</span> , and <span style='background-color:orange'>sugar</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:yellow'>2.Using</span> a heavy <span style='background-color:orange'>spoon</span> , <span style='background-color:orange'>stir</span> in the <span style='background-color:cyan'>eggs</span> one at a time , <span style='background-color:yellow'>followed</span> by <span style='background-color:cyan'>butter</span> and 2 tablespoons <span style='background-color:cyan'>water</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"If the <span style='background-color:orange'>batter</span> <span style='background-color:yellow'>appears</span> like there <span style='background-color:yellow'>is</span> too much <span style='background-color:orange'>flour</span> , <span style='background-color:orange'>add</span> 1 more tablespoon of <span style='background-color:cyan'>water</span> . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Batter</span> <span style='background-color:yellow'>will</span> <span style='background-color:yellow'>be</span> very thick . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"3 . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Fold</span> in <span style='background-color:cyan'>chocolate_chunks</span> 4 . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"<span style='background-color:orange'>Cover</span> and <span style='background-color:orange'>refrigerate</span> for 2 hours . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"5 . "
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for i in range(60):\n",
" print_visualized_tags(ts[i])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## train word2vec model based on stemmed words"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from gensim.test.utils import common_texts, get_tmpfile\n",
"from gensim.models import Word2Vec\n",
"from nltk import PorterStemmer\n",
"import numpy as np\n",
"from sklearn import preprocessing\n",
"porter = PorterStemmer()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"sentence_symbols = set(('.', ';', '!', '?', ',')) "
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"stemmed_sentences = []\n",
"for sentence in ts:\n",
" stemmed_sentence = []\n",
" for token in sentence:\n",
" stemmed = token['lemma']\n",
" if stemmed not in sentence_symbols:\n",
" stemmed_sentence.append(stemmed)\n",
" stemmed_sentences.append(stemmed_sentence)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"52563"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(stemmed_sentences)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"model = Word2Vec(stemmed_sentences, size=128, window=1, min_count=1, workers=4)\n",
"def word_similarity(word_a:str, word_b:str, model=model, stemmer=porter):\n",
" return model.wv.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n",
"\n",
"def word_exists(word:str, model=model, stemmer=porter):\n",
" return stemmer.stem(word) in model.wv\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## train stuff"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create Data Arrays"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"upos_tags = []\n",
"word_vecs = []\n",
"bool_ingr = []\n",
"bool_actn = []\n",
"\n",
"lb = preprocessing.LabelBinarizer()\n",
"lb.fit(['.', 'ADJ', 'ADP', 'ADV', 'CONJ', 'DET', 'NOUN', 'NUM', 'PRON', 'PRT', 'VERB', 'X'])\n",
"\n",
"number_tokens = 0\n",
"\n",
"for sentence in ts:\n",
" sent_upos = []\n",
" sent_word = []\n",
" sent_ingr = []\n",
" sent_actn = []\n",
" \n",
" for token in sentence:\n",
" number_tokens += 1\n",
" if token['form'] not in sentence_symbols:\n",
" sent_word.append(model.wv[token['lemma']])\n",
" upos = token['upostag']\n",
" misc = token['misc']\n",
" actn = False\n",
" ingr = False\n",
" sent_upos.append(upos)\n",
" if misc != None and 'food_type' in misc:\n",
" if misc['food_type'] == \"action\":\n",
" actn = True\n",
" elif misc['food_type'] == \"ingredient\":\n",
" ingr = True\n",
" sent_ingr.append(ingr)\n",
" sent_actn.append(actn)\n",
" if len(sent_upos) > 0:\n",
" upos_tags.append(lb.transform(sent_upos))\n",
" word_vecs.append(sent_word)\n",
" bool_ingr.append(sent_ingr)\n",
" bool_actn.append(sent_actn)\n",
" \n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create one-vs-all binary labels from upos values"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([-0.68675774, -0.7883484 , 0.20251137, -0.05719408, 0.09501456,\n",
" 0.05182203, -0.25285384, -0.41048968, -0.71525794, -0.67254657,\n",
" -0.17055118, -0.17210464, -0.82208157, 0.2450692 , 0.5252676 ,\n",
" -0.19269118, 0.341501 , 0.34656093, 0.9737164 , 0.35325623,\n",
" -0.89236367, -0.64312077, -0.7638635 , 0.39858425, -0.27193114,\n",
" 0.49859434, -0.81326216, 0.7487419 , 0.16307887, 0.25901216,\n",
" 0.66147804, -0.58643436, -0.09952813, 0.17525254, -0.00194792,\n",
" 0.07784765, 0.90528905, -0.15917295, -0.3427578 , 1.0070069 ,\n",
" -0.57078034, 0.33585113, -0.27914697, -0.24289384, 0.17581789,\n",
" 0.2808861 , -0.35724574, 0.2030598 , 0.7853533 , 0.5153942 ,\n",
" -0.02380644, -0.34730977, -0.2862777 , -0.44545448, 0.04394584,\n",
" -0.7926984 , -0.3066928 , -0.4378877 , 0.06224217, -0.48833787,\n",
" -0.48612142, 0.33839923, -0.02597165, 0.06829736, -1.3514524 ,\n",
" -0.72032964, 0.81755555, 0.73999155, 0.35204384, 0.32564116,\n",
" 0.37047276, 0.3869023 , 0.73725784, 0.49011388, -0.06073807,\n",
" 0.40204594, 0.25720817, 0.3023497 , -0.7338208 , -1.2525887 ,\n",
" 0.758383 , 0.1704396 , 0.26385054, -0.6765845 , -0.80152684,\n",
" -0.47452438, -0.25873485, 0.05395245, -0.5654043 , 0.5105308 ,\n",
" 0.26053032, 0.34596896, -0.62164515, -0.02258987, 0.06654755,\n",
" 0.8050189 , -0.02911493, -0.74690014, -0.5086117 , -0.06318699,\n",
" 1.2542377 , 0.5853978 , -0.92408645, -0.6169342 , -0.5941347 ,\n",
" 0.2128975 , 0.76446146, 0.78195745, -0.3471526 , 0.51036954,\n",
" 0.5108987 , -0.09576786, -0.7606738 , 0.8236566 , -0.28881475,\n",
" -0.7867539 , 0.31546646, 0.40256152, 0.0999696 , 0.20387918,\n",
" -0.6289423 , 0.63682 , 0.8060169 , -0.98891175, 0.14811945,\n",
" 0.29527354, 0.31912726, -0.22632086], dtype=float32)"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"word_vecs[0][0]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* the most annoying part: create sliding windows around words and create huge arrays containing values ad it's neighbors"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"window_size = 3\n",
"batch_size = 1000"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"12"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(lb.classes_)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"train_upos_tags = np.zeros((batch_size, window_size * 2 + 1, len(lb.classes_)))\n",
"train_word_vecs = np.zeros((batch_size, window_size * 2 + 1, model.vector_size))\n",
"train_bool_ingr = np.zeros((batch_size, 1))\n",
"train_bool_actn = np.zeros((batch_size, 1))\n",
"\n",
"X = np.zeros((batch_size, window_size * 2 + 1, train_upos_tags.shape[2] + train_word_vecs.shape[2]))"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"sentence_counter = 0\n",
"token_counter = 0\n",
"i = 0"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"def next_batch():\n",
" global sentence_counter\n",
" global i\n",
" global token_counter\n",
" global train_upos_tags\n",
" global train_word_vecs\n",
" global train_bool_ingr\n",
" global train_bool_actn\n",
" \n",
" \n",
" train_upos_tags = np.zeros((batch_size, window_size * 2 + 1, len(lb.classes_)))\n",
" train_word_vecs = np.zeros((batch_size, window_size * 2 + 1, model.vector_size))\n",
" train_bool_ingr = np.zeros((batch_size, 1))\n",
" train_bool_actn = np.zeros((batch_size, 1))\n",
" \n",
" i = 0\n",
" \n",
" current_sent_upos = upos_tags[sentence_counter]\n",
" current_sent_word = word_vecs[sentence_counter]\n",
" current_sent_ingr = bool_ingr[sentence_counter]\n",
" current_sent_actn = bool_actn[sentence_counter]\n",
"\n",
" \n",
" for i in range(batch_size):\n",
" \n",
" for j in range(- window_size, window_size + 1):\n",
" if j + token_counter < 0:\n",
" pass\n",
" elif j + token_counter >= len(current_sent_upos):\n",
" pass\n",
" else:\n",
" train_upos_tags[i,j + window_size,:] = current_sent_upos[token_counter + j]\n",
" train_word_vecs[i,j + window_size,:] = current_sent_word[token_counter + j]\n",
"\n",
" train_bool_ingr[i] = current_sent_ingr[token_counter]\n",
" train_bool_actn[i] = current_sent_actn[token_counter]\n",
"\n",
" token_counter += 1\n",
"\n",
" if token_counter >= len(current_sent_upos):\n",
" current_sent_upos = upos_tags[sentence_counter]\n",
" current_sent_word = word_vecs[sentence_counter]\n",
" current_sent_ingr = bool_ingr[sentence_counter]\n",
" current_sent_actn = bool_actn[sentence_counter]\n",
" \n",
" sentence_counter += 1\n",
" token_counter = 0\n",
" \n",
" # flatten dimensions:\n",
" train_upos_tags = train_upos_tags.reshape((batch_size, train_upos_tags.shape[1] * train_upos_tags.shape[2]))\n",
" train_word_vecs = train_word_vecs.reshape((batch_size, train_word_vecs.shape[1] * train_word_vecs.shape[2]))"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"next_batch()"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1000"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(train_bool_ingr)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"9\n",
"10\n",
"11\n",
"12\n",
"13\n",
"14\n",
"15\n"
]
}
],
"source": [
"for j in range(token_counter - window_size, token_counter + window_size + 1):\n",
" print(j + window_size)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"896"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(train_word_vecs[2])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## create a decision tree classifier"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.tree import DecisionTreeClassifier\n",
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* train ingredients"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
"X = train_upos_tags\n",
"Y = train_bool_ingr"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(X,Y, random_state=0)"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"250"
]
},
"execution_count": 48,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"tree = DecisionTreeClassifier(random_state=0)"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
" max_features=None, max_leaf_nodes=None,\n",
" min_impurity_decrease=0.0, min_impurity_split=None,\n",
" min_samples_leaf=1, min_samples_split=2,\n",
" min_weight_fraction_leaf=0.0, presort=False,\n",
" random_state=0, splitter='best')"
]
},
"execution_count": 50,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tree.fit(X_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [],
"source": [
"pred = tree.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
" 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
" 1., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 1., 0., 1., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1.,\n",
" 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
" 0., 0., 0., 0., 1., 1., 1., 0., 1., 1., 0., 0., 1., 0., 0., 0., 1.,\n",
" 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
]
},
"execution_count": 70,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(pred - y_test[:,0])**2"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1.,\n",
" 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 1., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
" 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
" 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
]
},
"execution_count": 67,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"y_test[:,0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}