jonas/master-thesis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

just synced everything important

Browse Source
This commit is contained in:
Jonas Weinz 2019-09-19 10:19:35 +02:00
parent 5c25e04143
commit 321a1c3fa4
6 changed files with 6414 additions and 5597 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

489
1M_clustering.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

8506
1M_evaluation.ipynb
View File

File diff suppressed because it is too large Load Diff

353
1M_interactive_visualization.ipynb Normal file
View File

@ -0,0 +1,353 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 1M Recipes Interactive Visualization\n",
"interactive plots from the naive word2vec approach"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib widget\n",
"\n",
"import ipywidgets as widgets\n",
"import numpy as np\n",
"import json\n",
"\n",
"import nltk\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.stem import LancasterStemmer\n",
"from nltk.corpus import stopwords as nltk_stopwords\n",
"\n",
"from pprint import pprint\n",
"\n",
"from gensim.test.utils import common_texts, get_tmpfile\n",
"from gensim.models import Word2Vec, KeyedVectors\n",
"\n",
"from sklearn.manifold import TSNE\n",
"from sklearn.decomposition import PCA\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
"\n",
"import pandas as pd\n",
"\n",
"import settings\n",
"\n",
"from IPython.display import HTML, Markdown, clear_output\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"'found 109 of 111 valid actions and 160 of 648 valid ingredients'\n"
]
}
],
"source": [
"wv = KeyedVectors.load(\"data/wordvectors.kv\")\n",
"porter = PorterStemmer()\n",
"def word_similarity(word_a:str, word_b:str, model=wv, stemmer=porter):\n",
" return model.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n",
"\n",
"def word_exists(word:str, model=wv, stemmer=porter):\n",
" return stemmer.stem(word) in model\n",
"\n",
"from cooking_vocab import cooking_verbs\n",
"from cooking_ingredients import ingredients\n",
"\n",
"model_actions = []\n",
"model_ingredients = []\n",
"\n",
"for action in cooking_verbs:\n",
" if word_exists(action):\n",
" model_actions.append(action)\n",
"\n",
"for ingredient in ingredients:\n",
" if word_exists(ingredient):\n",
" model_ingredients.append(ingredient)\n",
"\n",
"pprint(f\"found {len(model_actions)} of {len(cooking_verbs)} valid actions and {len(model_ingredients)} of {len(ingredients)} valid ingredients\")\n",
"\n",
"stemmed_ingredients = [porter.stem(ing) for ing in model_ingredients]\n",
"stemmed_actions = [porter.stem(act) for act in model_actions]\n",
"\n",
"def low_dim_plot(train_tokens, display_tokens, use_tsne=True, model=wv):\n",
" train_vecs = []\n",
" train_labels = []\n",
" \n",
" display_indices = []\n",
" \n",
" i = 0\n",
" \n",
" for token in train_tokens:\n",
" train_vecs.append(model[token])\n",
" train_labels.append(token)\n",
" if token in display_tokens:\n",
" display_indices.append(i)\n",
" i += 1\n",
" \n",
" plot_values = []\n",
" plot_labels = []\n",
" dim_reduced_values = None\n",
" \n",
" if use_tsne:\n",
" tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n",
" dim_reduced_values = tsne_model.fit_transform(train_vecs)\n",
" else:\n",
" # use pca then\n",
" pca_model = PCA(n_components=2)\n",
" dim_reduced_values = pca_model.fit_transform(train_vecs)\n",
" \n",
" for index in display_indices:\n",
" plot_values.append(dim_reduced_values[index])\n",
" plot_labels.append(train_labels[index])\n",
" \n",
" \n",
" x = []\n",
" y = []\n",
" \n",
" \n",
" for value in plot_values:\n",
" x.append(value[0])\n",
" y.append(value[1])\n",
"\n",
" \n",
" plt.figure(figsize=(16, 16)) \n",
" for i in range(len(x)):\n",
" plt.scatter(x[i],y[i])\n",
" plt.annotate(plot_labels[i],\n",
" xy=(x[i], y[i]),\n",
" xytext=(5, 2),\n",
" textcoords='offset points',\n",
" ha='right',\n",
" va='bottom')\n",
" plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4f548cf574e44a4eb2e2021ac54b4c17",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"action_selector = widgets.Dropdown(options=stemmed_actions, value=stemmed_actions[0], description=\"choose cooking action\")\n",
"ing_selector = widgets.Dropdown(options=stemmed_ingredients, value=stemmed_ingredients[0], description=\"choose ingredient\")\n",
"\n",
"out = widgets.Output()\n",
"\n",
"def action_click(b):\n",
" with out:\n",
" clear_output(True)\n",
" low_dim_plot(list(wv.vocab.keys()) ,stemmed_ingredients + [action_selector.value], use_tsne=False)\n",
"\n",
"def ing_click(b):\n",
" with out:\n",
" clear_output(True)\n",
" low_dim_plot(list(wv.vocab.keys()), stemmed_actions + [ing_selector.value], use_tsne=False)\n",
"\n",
"\n",
"action_button = widgets.Button(description=\"show 2D-Projection\")\n",
"action_button.on_click(action_click)\n",
"\n",
"\n",
"ing_button = widgets.Button(description=\"show 2D-Projection\")\n",
"ing_button.on_click(ing_click)\n",
"\n",
"display(out)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## View Ingredient Space for specific cooking Action"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1647c39a541d4e8287644b4013a72a47",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Dropdown(description='choose cooking action', options=('add', 'adjust', 'arrang', 'bake', 'bast', 'batter', 'b…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "620c6138ecdb410b8b6312e8abb6e360",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Button(description='show 2D-Projection', style=ButtonStyle())"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display(action_selector)\n",
"display(action_button)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## View Cook Action Space for specifig Ingredient"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b99e2e5b79ed4b75b348b5ee16f9bdff",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Dropdown(description='choose ingredient', options=('salt', 'garlic', 'onion', 'water', 'sugar', 'butter', 'pep…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "18379db972c04a05b7fb3d16419d5a77",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Button(description='show 2D-Projection', style=ButtonStyle())"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display(ing_selector)\n",
"display(ing_button)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[0;31mType:\u001b[0m Word2VecKeyedVectors\n",
"\u001b[0;31mString form:\u001b[0m <gensim.models.keyedvectors.Word2VecKeyedVectors object at 0x7fc704c80dd8>\n",
"\u001b[0;31mFile:\u001b[0m ~/.local/lib/python3.7/site-packages/gensim/models/keyedvectors.py\n",
"\u001b[0;31mDocstring:\u001b[0m \n",
"Mapping between words and vectors for the :class:`~gensim.models.Word2Vec` model.\n",
"Used to perform operations on the vectors such as vector lookup, distance, similarity etc.\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"?wv"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"50741"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(list(wv.vocab.keys()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

2543
EvolutionaryAlgorithm/EvolutionaryAlgorithm.ipynb
View File

File diff suppressed because it is too large Load Diff

120
Yummly_word2vec.ipynb
View File

@ -1,120 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import json \n",
"from pprint import pprint\n",
"from gensim.test.utils import common_texts, get_tmpfile\n",
"from gensim.models import Word2Vec\n",
"\n",
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
"\n",
"import settings"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* reading in all ingredients with json stream:"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"buffered_reader = JSON_br(settings.yummly_train)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"ingredient_sets = []"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"for recipe in buffered_reader:\n",
" ingredient_sets.append(recipe['ingredients'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* training a word2vec approach on the ingredient set"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"model = Word2Vec(ingredient_sets, size=100, window=5, min_count=1, workers=4)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.46477914"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.wv.similarity('eggs', 'pepper')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

0
__init__.py Normal file
View File