{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 1M Recipes Interactive Visualization\n", "interactive plots from the naive word2vec approach" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%matplotlib widget\n", "\n", "import ipywidgets as widgets\n", "import numpy as np\n", "import json\n", "\n", "import nltk\n", "from nltk.stem import PorterStemmer\n", "from nltk.stem import LancasterStemmer\n", "from nltk.corpus import stopwords as nltk_stopwords\n", "\n", "from pprint import pprint\n", "\n", "from gensim.test.utils import common_texts, get_tmpfile\n", "from gensim.models import Word2Vec, KeyedVectors\n", "\n", "from sklearn.manifold import TSNE\n", "from sklearn.decomposition import PCA\n", "\n", "import matplotlib.pyplot as plt\n", "\n", "from json_buffered_reader import JSON_buffered_reader as JSON_br\n", "\n", "import pandas as pd\n", "\n", "import settings\n", "\n", "from IPython.display import HTML, Markdown, clear_output\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "'found 109 of 111 valid actions and 160 of 648 valid ingredients'\n" ] } ], "source": [ "wv = KeyedVectors.load(\"data/wordvectors.kv\")\n", "porter = PorterStemmer()\n", "def word_similarity(word_a:str, word_b:str, model=wv, stemmer=porter):\n", " return model.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n", "\n", "def word_exists(word:str, model=wv, stemmer=porter):\n", " return stemmer.stem(word) in model\n", "\n", "from cooking_vocab import cooking_verbs\n", "from cooking_ingredients import ingredients\n", "\n", "model_actions = []\n", "model_ingredients = []\n", "\n", "for action in cooking_verbs:\n", " if word_exists(action):\n", " model_actions.append(action)\n", "\n", "for ingredient in ingredients:\n", " if word_exists(ingredient):\n", " model_ingredients.append(ingredient)\n", "\n", "pprint(f\"found {len(model_actions)} of {len(cooking_verbs)} valid actions and {len(model_ingredients)} of {len(ingredients)} valid ingredients\")\n", "\n", "stemmed_ingredients = [porter.stem(ing) for ing in model_ingredients]\n", "stemmed_actions = [porter.stem(act) for act in model_actions]\n", "\n", "def low_dim_plot(train_tokens, display_tokens, use_tsne=True, model=wv):\n", " train_vecs = []\n", " train_labels = []\n", " \n", " display_indices = []\n", " \n", " i = 0\n", " \n", " for token in train_tokens:\n", " train_vecs.append(model[token])\n", " train_labels.append(token)\n", " if token in display_tokens:\n", " display_indices.append(i)\n", " i += 1\n", " \n", " plot_values = []\n", " plot_labels = []\n", " dim_reduced_values = None\n", " \n", " if use_tsne:\n", " tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n", " dim_reduced_values = tsne_model.fit_transform(train_vecs)\n", " else:\n", " # use pca then\n", " pca_model = PCA(n_components=2)\n", " dim_reduced_values = pca_model.fit_transform(train_vecs)\n", " \n", " for index in display_indices:\n", " plot_values.append(dim_reduced_values[index])\n", " plot_labels.append(train_labels[index])\n", " \n", " \n", " x = []\n", " y = []\n", " \n", " \n", " for value in plot_values:\n", " x.append(value[0])\n", " y.append(value[1])\n", "\n", " \n", " plt.figure(figsize=(16, 16)) \n", " for i in range(len(x)):\n", " plt.scatter(x[i],y[i])\n", " plt.annotate(plot_labels[i],\n", " xy=(x[i], y[i]),\n", " xytext=(5, 2),\n", " textcoords='offset points',\n", " ha='right',\n", " va='bottom')\n", " plt.show()\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4f548cf574e44a4eb2e2021ac54b4c17", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Output()" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "action_selector = widgets.Dropdown(options=stemmed_actions, value=stemmed_actions[0], description=\"choose cooking action\")\n", "ing_selector = widgets.Dropdown(options=stemmed_ingredients, value=stemmed_ingredients[0], description=\"choose ingredient\")\n", "\n", "out = widgets.Output()\n", "\n", "def action_click(b):\n", " with out:\n", " clear_output(True)\n", " low_dim_plot(list(wv.vocab.keys()) ,stemmed_ingredients + [action_selector.value], use_tsne=False)\n", "\n", "def ing_click(b):\n", " with out:\n", " clear_output(True)\n", " low_dim_plot(list(wv.vocab.keys()), stemmed_actions + [ing_selector.value], use_tsne=False)\n", "\n", "\n", "action_button = widgets.Button(description=\"show 2D-Projection\")\n", "action_button.on_click(action_click)\n", "\n", "\n", "ing_button = widgets.Button(description=\"show 2D-Projection\")\n", "ing_button.on_click(ing_click)\n", "\n", "display(out)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## View Ingredient Space for specific cooking Action" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1647c39a541d4e8287644b4013a72a47", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Dropdown(description='choose cooking action', options=('add', 'adjust', 'arrang', 'bake', 'bast', 'batter', 'b…" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "620c6138ecdb410b8b6312e8abb6e360", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Button(description='show 2D-Projection', style=ButtonStyle())" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(action_selector)\n", "display(action_button)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## View Cook Action Space for specifig Ingredient" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b99e2e5b79ed4b75b348b5ee16f9bdff", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Dropdown(description='choose ingredient', options=('salt', 'garlic', 'onion', 'water', 'sugar', 'butter', 'pep…" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "18379db972c04a05b7fb3d16419d5a77", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Button(description='show 2D-Projection', style=ButtonStyle())" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(ing_selector)\n", "display(ing_button)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mType:\u001b[0m Word2VecKeyedVectors\n", "\u001b[0;31mString form:\u001b[0m \n", "\u001b[0;31mFile:\u001b[0m ~/.local/lib/python3.7/site-packages/gensim/models/keyedvectors.py\n", "\u001b[0;31mDocstring:\u001b[0m \n", "Mapping between words and vectors for the :class:`~gensim.models.Word2Vec` model.\n", "Used to perform operations on the vectors such as vector lookup, distance, similarity etc.\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "?wv" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "50741" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(list(wv.vocab.keys()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }