just synced everything important
This commit is contained in:
Jonas Weinz
parent
5c25e04143
commit
321a1c3fa4
489
1M_clustering.ipynb
Normal file
489
1M_clustering.ipynb
Normal file
File diff suppressed because one or more lines are too long
8506
1M_evaluation.ipynb
8506
1M_evaluation.ipynb
File diff suppressed because it is too large
Load Diff
353
1M_interactive_visualization.ipynb
Normal file
353
1M_interactive_visualization.ipynb
Normal file
@ -0,0 +1,353 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 1M Recipes Interactive Visualization\n",
|
||||
"interactive plots from the naive word2vec approach"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib widget\n",
|
||||
"\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"import numpy as np\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import nltk\n",
|
||||
"from nltk.stem import PorterStemmer\n",
|
||||
"from nltk.stem import LancasterStemmer\n",
|
||||
"from nltk.corpus import stopwords as nltk_stopwords\n",
|
||||
"\n",
|
||||
"from pprint import pprint\n",
|
||||
"\n",
|
||||
"from gensim.test.utils import common_texts, get_tmpfile\n",
|
||||
"from gensim.models import Word2Vec, KeyedVectors\n",
|
||||
"\n",
|
||||
"from sklearn.manifold import TSNE\n",
|
||||
"from sklearn.decomposition import PCA\n",
|
||||
"\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
|
||||
"\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"import settings\n",
|
||||
"\n",
|
||||
"from IPython.display import HTML, Markdown, clear_output\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"'found 109 of 111 valid actions and 160 of 648 valid ingredients'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"wv = KeyedVectors.load(\"data/wordvectors.kv\")\n",
|
||||
"porter = PorterStemmer()\n",
|
||||
"def word_similarity(word_a:str, word_b:str, model=wv, stemmer=porter):\n",
|
||||
" return model.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n",
|
||||
"\n",
|
||||
"def word_exists(word:str, model=wv, stemmer=porter):\n",
|
||||
" return stemmer.stem(word) in model\n",
|
||||
"\n",
|
||||
"from cooking_vocab import cooking_verbs\n",
|
||||
"from cooking_ingredients import ingredients\n",
|
||||
"\n",
|
||||
"model_actions = []\n",
|
||||
"model_ingredients = []\n",
|
||||
"\n",
|
||||
"for action in cooking_verbs:\n",
|
||||
" if word_exists(action):\n",
|
||||
" model_actions.append(action)\n",
|
||||
"\n",
|
||||
"for ingredient in ingredients:\n",
|
||||
" if word_exists(ingredient):\n",
|
||||
" model_ingredients.append(ingredient)\n",
|
||||
"\n",
|
||||
"pprint(f\"found {len(model_actions)} of {len(cooking_verbs)} valid actions and {len(model_ingredients)} of {len(ingredients)} valid ingredients\")\n",
|
||||
"\n",
|
||||
"stemmed_ingredients = [porter.stem(ing) for ing in model_ingredients]\n",
|
||||
"stemmed_actions = [porter.stem(act) for act in model_actions]\n",
|
||||
"\n",
|
||||
"def low_dim_plot(train_tokens, display_tokens, use_tsne=True, model=wv):\n",
|
||||
" train_vecs = []\n",
|
||||
" train_labels = []\n",
|
||||
" \n",
|
||||
" display_indices = []\n",
|
||||
" \n",
|
||||
" i = 0\n",
|
||||
" \n",
|
||||
" for token in train_tokens:\n",
|
||||
" train_vecs.append(model[token])\n",
|
||||
" train_labels.append(token)\n",
|
||||
" if token in display_tokens:\n",
|
||||
" display_indices.append(i)\n",
|
||||
" i += 1\n",
|
||||
" \n",
|
||||
" plot_values = []\n",
|
||||
" plot_labels = []\n",
|
||||
" dim_reduced_values = None\n",
|
||||
" \n",
|
||||
" if use_tsne:\n",
|
||||
" tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n",
|
||||
" dim_reduced_values = tsne_model.fit_transform(train_vecs)\n",
|
||||
" else:\n",
|
||||
" # use pca then\n",
|
||||
" pca_model = PCA(n_components=2)\n",
|
||||
" dim_reduced_values = pca_model.fit_transform(train_vecs)\n",
|
||||
" \n",
|
||||
" for index in display_indices:\n",
|
||||
" plot_values.append(dim_reduced_values[index])\n",
|
||||
" plot_labels.append(train_labels[index])\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" x = []\n",
|
||||
" y = []\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" for value in plot_values:\n",
|
||||
" x.append(value[0])\n",
|
||||
" y.append(value[1])\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" plt.figure(figsize=(16, 16)) \n",
|
||||
" for i in range(len(x)):\n",
|
||||
" plt.scatter(x[i],y[i])\n",
|
||||
" plt.annotate(plot_labels[i],\n",
|
||||
" xy=(x[i], y[i]),\n",
|
||||
" xytext=(5, 2),\n",
|
||||
" textcoords='offset points',\n",
|
||||
" ha='right',\n",
|
||||
" va='bottom')\n",
|
||||
" plt.show()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "4f548cf574e44a4eb2e2021ac54b4c17",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Output()"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"action_selector = widgets.Dropdown(options=stemmed_actions, value=stemmed_actions[0], description=\"choose cooking action\")\n",
|
||||
"ing_selector = widgets.Dropdown(options=stemmed_ingredients, value=stemmed_ingredients[0], description=\"choose ingredient\")\n",
|
||||
"\n",
|
||||
"out = widgets.Output()\n",
|
||||
"\n",
|
||||
"def action_click(b):\n",
|
||||
" with out:\n",
|
||||
" clear_output(True)\n",
|
||||
" low_dim_plot(list(wv.vocab.keys()) ,stemmed_ingredients + [action_selector.value], use_tsne=False)\n",
|
||||
"\n",
|
||||
"def ing_click(b):\n",
|
||||
" with out:\n",
|
||||
" clear_output(True)\n",
|
||||
" low_dim_plot(list(wv.vocab.keys()), stemmed_actions + [ing_selector.value], use_tsne=False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"action_button = widgets.Button(description=\"show 2D-Projection\")\n",
|
||||
"action_button.on_click(action_click)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"ing_button = widgets.Button(description=\"show 2D-Projection\")\n",
|
||||
"ing_button.on_click(ing_click)\n",
|
||||
"\n",
|
||||
"display(out)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## View Ingredient Space for specific cooking Action"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1647c39a541d4e8287644b4013a72a47",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Dropdown(description='choose cooking action', options=('add', 'adjust', 'arrang', 'bake', 'bast', 'batter', 'b…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "620c6138ecdb410b8b6312e8abb6e360",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Button(description='show 2D-Projection', style=ButtonStyle())"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"display(action_selector)\n",
|
||||
"display(action_button)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## View Cook Action Space for specifig Ingredient"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b99e2e5b79ed4b75b348b5ee16f9bdff",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Dropdown(description='choose ingredient', options=('salt', 'garlic', 'onion', 'water', 'sugar', 'butter', 'pep…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "18379db972c04a05b7fb3d16419d5a77",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Button(description='show 2D-Projection', style=ButtonStyle())"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"display(ing_selector)\n",
|
||||
"display(ing_button)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"\u001b[0;31mType:\u001b[0m Word2VecKeyedVectors\n",
|
||||
"\u001b[0;31mString form:\u001b[0m <gensim.models.keyedvectors.Word2VecKeyedVectors object at 0x7fc704c80dd8>\n",
|
||||
"\u001b[0;31mFile:\u001b[0m ~/.local/lib/python3.7/site-packages/gensim/models/keyedvectors.py\n",
|
||||
"\u001b[0;31mDocstring:\u001b[0m \n",
|
||||
"Mapping between words and vectors for the :class:`~gensim.models.Word2Vec` model.\n",
|
||||
"Used to perform operations on the vectors such as vector lookup, distance, similarity etc.\n"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"?wv"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"50741"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len(list(wv.vocab.keys()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,120 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import json \n",
|
||||
"from pprint import pprint\n",
|
||||
"from gensim.test.utils import common_texts, get_tmpfile\n",
|
||||
"from gensim.models import Word2Vec\n",
|
||||
"\n",
|
||||
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
|
||||
"\n",
|
||||
"import settings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* reading in all ingredients with json stream:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"buffered_reader = JSON_br(settings.yummly_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ingredient_sets = []"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for recipe in buffered_reader:\n",
|
||||
" ingredient_sets.append(recipe['ingredients'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* training a word2vec approach on the ingredient set"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = Word2Vec(ingredient_sets, size=100, window=5, min_count=1, workers=4)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"0.46477914"
|
||||
]
|
||||
},
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.wv.similarity('eggs', 'pepper')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
0
__init__.py
Normal file
0
__init__.py
Normal file
Loading…
Reference in New Issue
Block a user