{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Recipe Tagging analysis\n", "\n", "analyse and visualize preconfigured conllu files" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "\n", "from conllu import parse\n", "\n", "sys.path.insert(0,'..')\n", "import settings\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "conllu_path = \"recipes0.conllu\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "with open(conllu_path, 'r') as f:\n", " data = f.readlines()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "# newdoc\n", "1 Set set NOUN NNP _ _ _ _ food_type=action \n", "2 oven oven ADV RB _ _ _ _ _ \n", "3 to to PRT TO _ _ _ _ _ \n", "4 350 350 NUM CD _ _ _ _ _ \n", "5 degrees degre NOUN NNS _ _ _ _ _ \n", "6 F F NOUN NNP _ _ _ _ _ \n", "7 . . . . _ _ _ _ _ \n", "\n", "8 Grease greas VERB VB _ _ _ _ _ \n", "9 a a DET DT _ _ _ _ _ \n", "10 13 13 NUM CD _ _ _ _ _ \n", "11 x x ADJ JJ _ _ _ _ _ \n", "12 9-inch 9-inch ADJ JJ _ _ _ _ _ \n", "13 baking bake VERB VBG _ _ _ _ food_type=action \n", "14 dish dish NOUN NN _ _ _ _ _ \n", "15 . . . . _ _ _ _ _ \n", "\n", "16 Cook cook VERB VB _ _ _ _ food_type=action \n", "17 the the DET DT _ _ _ _ _ \n", "\n" ] } ], "source": [ "print(\"\".join(data[0:20]))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "ts = parse(\"\".join(data))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "sentence = ts[0]" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "list" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "type(ts)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "---\n", "\n", "* visualize tags:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "from tagging_tools import print_visualized_tags" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "Set oven to 350 degrees F . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Grease a 13 x 9-inch baking dish . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Cook the macaroni in boiling_water until al dente ( do not cook until very soft ) drain , then place into a bowl . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Place the milk , flour , onion_salt and garlic_powder in a 1-quart jar ; cover tightly and shake vigorously for 1 minute ; pour the mixture over the macaroni and add in 3-1/2 cups shredded_cheddar_cheese ; mix thoroughly to combine , then spread into the baking dish . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Sprinkle with breadcrumbs then remaining 1 cup of cheddar_cheese . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Drizzle 1/4 cup melted_butter or margarine over the top . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Bake uncovered for 45 minutes or until golden brown . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Stir together the sugar , salt , pepper , cumin and mustard in small bowl . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Place salmon so that the side the skin was removed from is down . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Press sugar mixture evenly onto fish . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Heat oil over medium heat in large nonstick or cast-iron skillet . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "When hot , place fish rub-side down in pan . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Cook until brown_sugar rub dissolves and darkens slightly , being careful not to burn , about 4 minutes . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Flip fish and cook to medium doneness , about 1 minute longer . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Mix all ingredients in bowl in order listed . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Make small or medium size balls out of mixture and place on a greased cookie sheet sprayed with Pam . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "For extra happy horses you can sprinkle a little bit of honey on them before cooking . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Bake 350F for 15-20 minutes and make sure they do n't burn ! " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "The day before you want to eat this cut your rolls into 8 medium to thick slices , put on a light setting and just slightly toast them or put them into the oven and slightly pre toast them . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "They will be grilled the next day again so do not brown them to much , alternatively you can buy larger ready cooked crostini , next day cut the cheese to size to fit the slices of bread . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "The next day , slice some cheese the same size as each slice of bread.. " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Layer first a slice of bread , then a slice of cheese then a slice of bread and another slice of cheese into ovenproof soup plates or bowls " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "prepare you soup by making a clear stock of preference , vegetable , chicken , fish or beef . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "You can do this by skimming the grey foam as soon a it comes up with a very small net strainer or a ladle . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "This needs to be done as soon as it comes up as it first starts boiling or the soup will go cloudy . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Season to your liking . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Add a little sherry . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Drain the stock , carefully ladle the clear stock over the bread and cheese . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Put the bowls into the oven and melt the cheese . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Hope you enjoy " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "The bread full of soup is the best bit " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Using a 1/2 c ice-cream scoop , put one scoop of sherbet on each cookie . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Freeze until firm , at least 15 minutes . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Heat the oven to 475 degrees F . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Whip the egg_whites in a glass or metal bowl until soft peaks form . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Add the sugar and beat until stiff peaks form . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Place the frozen sherbet topped cookies about 2 '' apart on a wire rack on a baking sheet . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Spread the meringue over each of the sherbets and also the cookie edges . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Bake until lightly browned , about 2 minutes . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Serve immediately . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Spread 1/2 tablespoon butter on one side of each piece of toasted Italian_bread . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Set aside . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Heat olive_oil in a skillet over high heat . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "When oil begins to smoke , pour scallops into pan . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Cook for 30 seconds without stirring . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Toss scallops in pan and stir in garlic . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Cook and stir until fragrant , about 30 seconds . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Stir wine and lemon_juice into scallops , bring to a boil , and cook for about 30 seconds . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Stir parsley and cold butter into scallops and remove from heat . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "When butter melts , stir in salt , black_pepper , and cayenne_pepper . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Spoon scallops over buttered toast and serve immediately . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "1 . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Whisk together brownie mix , flour , and sugar . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "2.Using a heavy spoon , stir in the eggs one at a time , followed by butter and 2 tablespoons water . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "If the batter appears like there is too much flour , add 1 more tablespoon of water . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Batter will be very thick . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "3 . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Fold in chocolate_chunks 4 . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "Cover and refrigerate for 2 hours . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "5 . " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "for i in range(60):\n", " print_visualized_tags(ts[i])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## train word2vec model based on stemmed words" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "from gensim.test.utils import common_texts, get_tmpfile\n", "from gensim.models import Word2Vec\n", "from nltk import PorterStemmer\n", "import numpy as np\n", "from sklearn import preprocessing\n", "porter = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "sentence_symbols = set(('.', ';', '!', '?', ',')) " ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "stemmed_sentences = []\n", "for sentence in ts:\n", " stemmed_sentence = []\n", " for token in sentence:\n", " stemmed = token['lemma']\n", " if stemmed not in sentence_symbols:\n", " stemmed_sentence.append(stemmed)\n", " stemmed_sentences.append(stemmed_sentence)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "52563" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(stemmed_sentences)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "model = Word2Vec(stemmed_sentences, size=128, window=1, min_count=1, workers=4)\n", "def word_similarity(word_a:str, word_b:str, model=model, stemmer=porter):\n", " return model.wv.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n", "\n", "def word_exists(word:str, model=model, stemmer=porter):\n", " return stemmer.stem(word) in model.wv\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## train stuff" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* create Data Arrays" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "upos_tags = []\n", "word_vecs = []\n", "bool_ingr = []\n", "bool_actn = []\n", "\n", "lb = preprocessing.LabelBinarizer()\n", "lb.fit(['.', 'ADJ', 'ADP', 'ADV', 'CONJ', 'DET', 'NOUN', 'NUM', 'PRON', 'PRT', 'VERB', 'X'])\n", "\n", "number_tokens = 0\n", "\n", "for sentence in ts:\n", " sent_upos = []\n", " sent_word = []\n", " sent_ingr = []\n", " sent_actn = []\n", " \n", " for token in sentence:\n", " number_tokens += 1\n", " if token['form'] not in sentence_symbols:\n", " sent_word.append(model.wv[token['lemma']])\n", " upos = token['upostag']\n", " misc = token['misc']\n", " actn = False\n", " ingr = False\n", " sent_upos.append(upos)\n", " if misc != None and 'food_type' in misc:\n", " if misc['food_type'] == \"action\":\n", " actn = True\n", " elif misc['food_type'] == \"ingredient\":\n", " ingr = True\n", " sent_ingr.append(ingr)\n", " sent_actn.append(actn)\n", " if len(sent_upos) > 0:\n", " upos_tags.append(lb.transform(sent_upos))\n", " word_vecs.append(sent_word)\n", " bool_ingr.append(sent_ingr)\n", " bool_actn.append(sent_actn)\n", " \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* create one-vs-all binary labels from upos values" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([-0.68675774, -0.7883484 , 0.20251137, -0.05719408, 0.09501456,\n", " 0.05182203, -0.25285384, -0.41048968, -0.71525794, -0.67254657,\n", " -0.17055118, -0.17210464, -0.82208157, 0.2450692 , 0.5252676 ,\n", " -0.19269118, 0.341501 , 0.34656093, 0.9737164 , 0.35325623,\n", " -0.89236367, -0.64312077, -0.7638635 , 0.39858425, -0.27193114,\n", " 0.49859434, -0.81326216, 0.7487419 , 0.16307887, 0.25901216,\n", " 0.66147804, -0.58643436, -0.09952813, 0.17525254, -0.00194792,\n", " 0.07784765, 0.90528905, -0.15917295, -0.3427578 , 1.0070069 ,\n", " -0.57078034, 0.33585113, -0.27914697, -0.24289384, 0.17581789,\n", " 0.2808861 , -0.35724574, 0.2030598 , 0.7853533 , 0.5153942 ,\n", " -0.02380644, -0.34730977, -0.2862777 , -0.44545448, 0.04394584,\n", " -0.7926984 , -0.3066928 , -0.4378877 , 0.06224217, -0.48833787,\n", " -0.48612142, 0.33839923, -0.02597165, 0.06829736, -1.3514524 ,\n", " -0.72032964, 0.81755555, 0.73999155, 0.35204384, 0.32564116,\n", " 0.37047276, 0.3869023 , 0.73725784, 0.49011388, -0.06073807,\n", " 0.40204594, 0.25720817, 0.3023497 , -0.7338208 , -1.2525887 ,\n", " 0.758383 , 0.1704396 , 0.26385054, -0.6765845 , -0.80152684,\n", " -0.47452438, -0.25873485, 0.05395245, -0.5654043 , 0.5105308 ,\n", " 0.26053032, 0.34596896, -0.62164515, -0.02258987, 0.06654755,\n", " 0.8050189 , -0.02911493, -0.74690014, -0.5086117 , -0.06318699,\n", " 1.2542377 , 0.5853978 , -0.92408645, -0.6169342 , -0.5941347 ,\n", " 0.2128975 , 0.76446146, 0.78195745, -0.3471526 , 0.51036954,\n", " 0.5108987 , -0.09576786, -0.7606738 , 0.8236566 , -0.28881475,\n", " -0.7867539 , 0.31546646, 0.40256152, 0.0999696 , 0.20387918,\n", " -0.6289423 , 0.63682 , 0.8060169 , -0.98891175, 0.14811945,\n", " 0.29527354, 0.31912726, -0.22632086], dtype=float32)" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "word_vecs[0][0]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* the most annoying part: create sliding windows around words and create huge arrays containing values ad it's neighbors" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [], "source": [ "window_size = 3\n", "batch_size = 1000" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "12" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(lb.classes_)" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "\n", "\n", "train_upos_tags = np.zeros((batch_size, window_size * 2 + 1, len(lb.classes_)))\n", "train_word_vecs = np.zeros((batch_size, window_size * 2 + 1, model.vector_size))\n", "train_bool_ingr = np.zeros((batch_size, 1))\n", "train_bool_actn = np.zeros((batch_size, 1))\n", "\n", "X = np.zeros((batch_size, window_size * 2 + 1, train_upos_tags.shape[2] + train_word_vecs.shape[2]))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [], "source": [ "sentence_counter = 0\n", "token_counter = 0\n", "i = 0" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [], "source": [ "def next_batch():\n", " global sentence_counter\n", " global i\n", " global token_counter\n", " global train_upos_tags\n", " global train_word_vecs\n", " global train_bool_ingr\n", " global train_bool_actn\n", " \n", " \n", " train_upos_tags = np.zeros((batch_size, window_size * 2 + 1, len(lb.classes_)))\n", " train_word_vecs = np.zeros((batch_size, window_size * 2 + 1, model.vector_size))\n", " train_bool_ingr = np.zeros((batch_size, 1))\n", " train_bool_actn = np.zeros((batch_size, 1))\n", " \n", " i = 0\n", " \n", " current_sent_upos = upos_tags[sentence_counter]\n", " current_sent_word = word_vecs[sentence_counter]\n", " current_sent_ingr = bool_ingr[sentence_counter]\n", " current_sent_actn = bool_actn[sentence_counter]\n", "\n", " \n", " for i in range(batch_size):\n", " \n", " for j in range(- window_size, window_size + 1):\n", " if j + token_counter < 0:\n", " pass\n", " elif j + token_counter >= len(current_sent_upos):\n", " pass\n", " else:\n", " train_upos_tags[i,j + window_size,:] = current_sent_upos[token_counter + j]\n", " train_word_vecs[i,j + window_size,:] = current_sent_word[token_counter + j]\n", "\n", " train_bool_ingr[i] = current_sent_ingr[token_counter]\n", " train_bool_actn[i] = current_sent_actn[token_counter]\n", "\n", " token_counter += 1\n", "\n", " if token_counter >= len(current_sent_upos):\n", " current_sent_upos = upos_tags[sentence_counter]\n", " current_sent_word = word_vecs[sentence_counter]\n", " current_sent_ingr = bool_ingr[sentence_counter]\n", " current_sent_actn = bool_actn[sentence_counter]\n", " \n", " sentence_counter += 1\n", " token_counter = 0\n", " \n", " # flatten dimensions:\n", " train_upos_tags = train_upos_tags.reshape((batch_size, train_upos_tags.shape[1] * train_upos_tags.shape[2]))\n", " train_word_vecs = train_word_vecs.reshape((batch_size, train_word_vecs.shape[1] * train_word_vecs.shape[2]))" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "next_batch()" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1000" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(train_bool_ingr)" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9\n", "10\n", "11\n", "12\n", "13\n", "14\n", "15\n" ] } ], "source": [ "for j in range(token_counter - window_size, token_counter + window_size + 1):\n", " print(j + window_size)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "896" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(train_word_vecs[2])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## create a decision tree classifier" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* train ingredients" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [], "source": [ "X = train_upos_tags\n", "Y = train_bool_ingr" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [], "source": [ "X_train, X_test, y_train, y_test = train_test_split(X,Y, random_state=0)" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "250" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(X_test)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [], "source": [ "tree = DecisionTreeClassifier(random_state=0)" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n", " max_features=None, max_leaf_nodes=None,\n", " min_impurity_decrease=0.0, min_impurity_split=None,\n", " min_samples_leaf=1, min_samples_split=2,\n", " min_weight_fraction_leaf=0.0, presort=False,\n", " random_state=0, splitter='best')" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tree.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [], "source": [ "pred = tree.predict(X_test)" ] }, { "cell_type": "code", "execution_count": 70, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n", " 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n", " 1., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 1., 0., 1., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1.,\n", " 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n", " 0., 0., 0., 0., 1., 1., 1., 0., 1., 1., 0., 0., 1., 0., 0., 0., 1.,\n", " 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])" ] }, "execution_count": 70, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(pred - y_test[:,0])**2" ] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1.,\n", " 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 1., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n", " 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n", " 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])" ] }, "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_test[:,0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }