master-thesis/1M_clustering.ipynb

490 lines
187 KiB
Plaintext
Raw Permalink Normal View History

2019-09-19 10:19:35 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 1M_recipe clustering"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import numpy as np\n",
"import json\n",
"\n",
"import nltk\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.stem import LancasterStemmer\n",
"from nltk.corpus import stopwords as nltk_stopwords\n",
"\n",
"from pprint import pprint\n",
"\n",
"from gensim.test.utils import common_texts, get_tmpfile\n",
"from gensim.models import Word2Vec, KeyedVectors\n",
"\n",
"from sklearn.manifold import TSNE\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
"\n",
"import pandas as pd\n",
"\n",
"import settings"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from ipypb import track\n",
"from IPython.display import HTML, Markdown"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"our goal is to cluster out word to vec approcah to get some information about ingredients"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## clustering ingredients\n",
"\n",
"first we will load our predefined list of ingredients and then cluster them"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# load model and define helper funktions for stemming:\n",
"wv = KeyedVectors.load(\"data/wordvectors.kv\")\n",
"\n",
"porter = PorterStemmer()\n",
"def word_similarity(word_a:str, word_b:str, model=wv, stemmer=porter):\n",
" return model.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n",
"\n",
"def word_exists(word:str, model=wv, stemmer=porter):\n",
" return stemmer.stem(word) in model\n",
"\n",
"# load predefined vocabulary\n",
"from cooking_vocab import cooking_verbs\n",
"from cooking_ingredients import ingredients\n",
"\n",
"model_actions = []\n",
"model_ingredients = []\n",
"\n",
"for action in cooking_verbs:\n",
" if word_exists(action):\n",
" model_actions.append(action)\n",
"\n",
"for ingredient in ingredients:\n",
" if word_exists(ingredient):\n",
" model_ingredients.append(ingredient)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"'found 109 of 111 valid actions and 160 of 648 valid ingredients'\n"
]
}
],
"source": [
"pprint(f\"found {len(model_actions)} of {len(cooking_verbs)} valid actions and {len(model_ingredients)} of {len(ingredients)} valid ingredients\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"stemmed_ingredients = [porter.stem(ing) for ing in model_ingredients]\n",
"stemmed_actions = [porter.stem(act) for act in model_actions]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* generate_datapoints"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(160, 512)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ingredient_vectors = np.array([wv[ing] for ing in stemmed_ingredients])\n",
"ingredient_vectors.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* try clustering in a little bit reduced space "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.cluster import DBSCAN\n",
"from sklearn.decomposition import PCA\n",
"\n",
"# filter randomly in 32 dimensions\n",
"tsne_model_a = TSNE(perplexity=40, n_components=3, init='pca', n_iter=2500, random_state=23)\n",
"low_dim_vectors_a = tsne_model_a.fit_transform(ingredient_vectors)\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"\n",
"dbscan = DBSCAN(eps=100, min_samples=1)\n",
"clusters_original = dbscan.fit(low_dim_vectors_a)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,\n",
" 13, 14, 15, 2, 16, 17, 18, 19, 20, 21, 10, 20, 22,\n",
" 23, 24, 25, 9, 26, 27, 28, 29, 30, 31, 20, 32, 9,\n",
" 33, 34, 27, 35, 36, 20, 37, 38, 39, 40, 41, 27, 27,\n",
" 18, 42, 43, 44, 27, 45, 12, 46, 38, 37, 27, 47, 48,\n",
" 49, 50, 51, 52, 53, 26, 54, 55, 31, 56, 49, 20, 57,\n",
" 58, 59, 60, 61, 62, 20, 63, 64, 65, 66, 67, 68, 27,\n",
" 69, 70, 28, 71, 72, 73, 74, 27, 65, 75, 12, 76, 35,\n",
" 77, 78, 27, 79, 80, 81, 31, 82, 27, 83, 74, 55, 84,\n",
" 85, 27, 86, 74, 67, 87, 20, 88, 89, 90, 82, 91, 92,\n",
" 93, 94, 80, 95, 96, 51, 97, 98, 95, 22, 27, 99, 100,\n",
" 55, 27, 74, 38, 101, 102, 103, 104, 27, 105, 106, 16, 107,\n",
" 67, 108, 74, 109])"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"clusters_original.labels_"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* try a dimensionality reduction by pca:"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"# filter randomly in 32 dimensions\n",
"tsne_model_b = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n",
"low_dim_vectors_b = tsne_model_b.fit_transform(ingredient_vectors)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* analyse distribution a little bit"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"-7.2183714 9.767563 3.887664 2.6013005\n"
]
}
],
"source": [
"print(np.min(low_dim_vectors_b), np.max(low_dim_vectors_b), np.sqrt(np.average(low_dim_vectors_b**2)), np.sqrt(np.median(low_dim_vectors_b**2)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* clustering with DBSCAN"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.cluster import DBSCAN"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"dbscan = DBSCAN(eps=1, min_samples=1)\n",
"clusters = dbscan.fit(low_dim_vectors_b)"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 12, 13, 14, 2,\n",
" 4, 8, 13, 0, 0, 15, 10, 0, 15, 16, 17, 2, 16, 16, 15, 15, 9,\n",
" 18, 19, 0, 20, 9, 15, 21, 15, 15, 18, 0, 15, 0, 22, 12, 23, 16,\n",
" 24, 13, 4, 25, 26, 16, 15, 0, 15, 0, 15, 15, 9, 4, 17, 27, 28,\n",
" 15, 29, 16, 30, 26, 19, 5, 17, 0, 31, 15, 32, 32, 26, 32, 21, 15,\n",
" 19, 24, 26, 22, 15, 15, 24, 5, 15, 32, 5, 33, 4, 24, 24, 28, 0,\n",
" 31, 15, 15, 19, 24, 14, 32, 24, 19, 34, 24, 15, 4, 15, 5, 15, 24,\n",
" 11, 4, 22, 16, 0, 15, 14, 21, 34, 17, 15, 15, 15, 32, 15, 15, 28,\n",
" 35, 21, 25, 15, 16, 0, 15, 26, 16, 4, 0, 36, 15, 12, 15, 24, 21,\n",
" 37, 4, 0, 22, 15, 4, 24])"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"clusters.labels_"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* plot clusters"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"def plot_clusters(points, labels, cluster_labels):\n",
" x = []\n",
" y = []\n",
" for value in points:\n",
" x.append(value[0])\n",
" y.append(value[1])\n",
" \n",
" plt.figure(figsize=(20, 20))\n",
" \n",
" plt.scatter(x,y, c=cluster_labels,s=80)\n",
" for i in range(len(x)):\n",
" plt.annotate(labels[i],\n",
" xy=(x[i], y[i]),\n",
" xytext=(5, 2),\n",
" textcoords='offset points',\n",
" ha='right',\n",
" va='bottom')\n",
" #plt.legend()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAABIMAAARiCAYAAAA3EzfQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xl0VdXd//H3SW4GCIQpiKCpgQqoJGEWFBBEwNZ5QEDBoj6CgGjV4s/6WC1OfZRSbbVSa0sVJ0oFa1Vs64OCgMIjCWEuqGiKCCIIBEISMp3fH5S0yCwhgdz3ay0Wuefss89335WslfvJ3vsEYRgiSZIkSZKk6BBT3QVIkiRJkiSp6hgGSZIkSZIkRRHDIEmSJEmSpChiGCRJkiRJkhRFDIMkSZIkSZKiiGGQJEmSJElSFDEMkiRJkiRJiiKGQZIkSZIkSVHEMEiSJEmSJCmKGAZJkiRJkiRFkUh13DQlJSVMS0urjltLkiRJkiTVSNnZ2ZvCMGx8sHbVEgalpaWRlZVVHbeWJEmSJEmqkYIg+OehtHOZmCRJkiRJUhQxDJIkSZIkSYoihkGSJEmSJElRxDBIkiRJkiQpihgGSZIkSZIkRRHDIEmSJEmSpChiGCRJkiRJkhRFDIMkSZIkSZKiiGGQJEmSJElSFDEMkiRJkiRJiiKGQZIkSZIkSVHEMEiSJEmSJCmKGAZJkiRJkiRFEcMgSZIkSZKkKGIYJEmSJEmSFEUMgyRJkiRJkqKIYZAkSZIkSVIUMQySJEmSJEmKIoZBkiRJkiRJUcQwSJIkSZIkKYoYBkmSJEmSJEURwyBJkiRJkqQoUilhUBAEtwdBsDwIgmVBEEwOgiCxMvqVJEmSJElS5TriMCgIgpOAW4FOYRimA7HAoCPtV5IkSZIkSZWvspaJRYBaQRBEgNrAukrqV5IkSZIkSZXoiMOgMAy/AMYDa4D1QF4Yhm8fab+SJEmSJEmqfJWxTKwBcCnQHGgGJAVBMGQf7YYHQZAVBEHWxo0bj/S2kiRJkiRJ+hYqY5lYH+CzMAw3hmFYArwKnP3NRmEYPhOGYacwDDs1bty4Em4rSZIkSZKkw1UZYdAaoGsQBLWDIAiA84B/VEK/kiRJkiRJqmSVsWfQ/wFTgYXA0n/1+cyR9itJkiRJkqTKF6mMTsIw/Cnw08roS5IkSZIkSUdPZT1aXpIkSZIkSccBwyBJkiRJkqQoYhgkSZIkSZIURQyDJEmSJEmSoohhkCRFidzcXNLT04/6fdLS0ti0adNRv48kSZKkb8cwSJJUoaysrLpLkCRJknSUGQZJUhQpLS1l6NChZGZm0r9/fwoKCkhLS+OBBx6ge/fuvPLKK6xevZrvfe97dOzYkR49erBy5UoA3njjDbp06UL79u3p06cPGzZsAODrr7+mX79+tG/fnptuuokwDKtziJIkSZIOwjBIkmqAQ10CtmrVKoYPH86SJUtITk5mwoQJACQmJjJ37lwGDRrE8OHDefLJJ8nOzmb8+PGMGjUKgO7duzN//nxycnIYNGgQ48aNA+Dyyy+nS5cu5OTkcMkll7BmzZqjN1BJkiRJRyxS3QVIkqpOamoq3bp1A2DIkCE88cQTAAwcOBCA/Px8PvjgA6666qqKa3bu3AnA2rVrGThwIOvXr6e4uJjmzZsDMH/+fB577DEALrzwwiobiyRJkqRvxzBIkmqI3UvAcnJyaNWqFc8//zzjx4/njTfeoLCwkIyMDIIgAGDBggXceOONbN26lbKyMs4//3xWrlzJCy+8QExMDIsWLQLgoosuYsyYMcCu8KioqIikpCQaNmzIjh07eOKJJygpKWHw4ME0a9aMLl26ANCrVy/atm3LSy+9VD1vhiRJkqT9cpmYJNUQ+1oCNnr0aBYsWMCyZcsoKipizZo1zJs3j+uvv5709HTuueeePfqoVasWycnJvPLKKwCEYcgnn3zCpk2bWL16NRMnTmThwoXs2LGDtWvXcuutt1K3bl2uuOIKZs6cSc+ePQGYNWuWQZAkSZJ0jDIMkqQa4ptLwObOncvMmTPp0qULGRkZfPDBB5xwwgk888wzfPTRR8TFxTFy5Ejq1KmzRz99+/Zl4sSJtG3bltmzZ/P+++8zf/58YmJiOO+886hTpw6fffYZRUVFzNu0nCApjpdmvMp32nyXP7/1esXsI0mSJEnHJpeJSVIN8c0QJggCRo0aRVZWFqmpqYwdOxaAH/7wh7z77rtMmzYNgL/+9a9cc801AEQiEerUqcPzzz8PQJ8+fRg6dCjbt2/nwgsvZPLkyQBs2pnHHTlP8fCKFykOSznlnt7Url+Hz4OA+D8kkJKSUkWjliRJknS4nBkkSTXE7iVgAJMnT6Z79+4ApKSkkJ+fz9SpUwFo0KABdevWZf78+QD88Y9/rOgjLS2NRYsWUV5ezueff86HH34IQNeuXXn//ff55JNPKAvLuXXe46z++BMKy3YSWzuOsoJiSsMyistLKYsJef2fc6ty6JIkSZIOgzODJKmGOP3005k0aRI33XQTLVu2ZOTIkWzZsoWMjAzS0tLo3LlzRduJEycybNgwkpKS6NWrF/Xq1QOgW7duNG/enIyMDNLT0+nQoQMAjRs35rnnnuPqq69mS0EeXxVtJe36s0g8uT7NLsxg8Y//THyjJDo8dhXNLspgSM/LuahbX15+6eVqeS8kSZIk7V8QhmGV37RTp05hVlZWld9XkrRLfn5+xV5BjzzyCOvXr+dXv/rVIV17/7LnmL1xyYEblccwru1IOqZ890hLlSRJknSIgiDIDsOw08HauUxMkqLQ9OnTadeuHenp6cyZM4ef/OQnh3xtfmnhQduUhSH3znmb6viDgyRJkqQDc5mYJEWhgQMHMnDgwG91bYs6zViy9VNKw7L9tgmCkH9uLmDxV1/SrknTb1umJEmSpKPAmUGSpMNy6Und4QATfsIQigrjKdgZ8vdPP666wiRJkiQdEsMgSdJhaVarEW3rpFNeHux1LgyhvDxg0/r6hEBBaUnVFyhJkiTpgAyDJEmH7bq077NtY31KS2MoLwsoKwsoL4fCgni++KwxJSURakfiaJNyQnWXKkmSJOkb3DNIknTYzmh8AimksurjROITS4iJCSkpjlBWGlvRJgQuPvW06itSkiRJ0j45M0iS9K38qu+FJMUlUFIUT1FBwh5BUGIkwvje36NWXFw1VihJkiRpXwyDJEnfymmNGvOX/oPp9Z3mxMfGUicunvjYWNIbN+EPF1zBhae2ru4SJUmSJO2Dy8QkSd/aqQ0a8exFV7K1qJCvCnZQLyGRJkl1qrssSZIkSQdgGCRJOmL1E2tRP7FWdZchSZIk6RC4TEySJEmSJCmKGAZJkiRJkiRFEcMgSZIkSZKkKGIYJEmSJEmSFEUMgyRJkiRJkqKIYZAkSZIkSVIUMQySJEmSJEmKIoZBkiRJkiRJUcQwSJIkSZIkKYoYBkmSJEmSJEURwyBJkiRJkqQoYhgkSZIkSZIURQyDJEmqQbZu3cqECROO6Jp169bRv39/AGbNmsVFF10EwHPPPcfo0aMrr1hJkiRVC8MgSZJqkMMNg8rKyva6plmzZkydOvVolCdJkqRjgGGQJEk1yI9//GNWr15Nu3btuPPOO7nzzjtJT08nIyODKVOmALtm+5x77rlcc801ZGRk7HVNbm4u6enpB7zPG2+8QZcuXWjfvj19+vRhw4YNVTE8SZIkVYJIdRcgSZIqzyOPPMKyZctYtGgR06ZN4+mnn2bx4sVs2rSJzp07c8455wDw4YcfsmzZMpo3b05ubm7FNQC5ubkHvU/37t2ZP38+QRDw+9//nnHjxvGLX/ziaA5NkiRJlcQwSJKkGmru3LlcffXVxMbG0qRJE3r27MmCBQtITk7mzDPPpHnz5t+677Vr1zJw4EDWr19PcXHxEfUlSZKkquUyMUmSaqgwDPd7Likp6Yj6vuWWWxg9ejRLly7lt7/9LUVFRUfUnyRJkqqOYZAkSTVI3bp12b59OwDnnHMOU6ZMoaysjI0bNzJ79mzOPPPMA15zqPLy8jjppJMAmDR
"text/plain": [
"<Figure size 1440x1440 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot_clusters(low_dim_vectors_b, stemmed_ingredients, clusters_original.labels_)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get sets"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0: mirin, salt, dry white wine, buttermilk, parmesan cheese, cream cheese, lime, dried thyme, fresh parsley, ground ginger, extra-virgin olive oil, ground cumin, cayenne pepper, yellow onion, carrots, \n",
"1: garlic, \n",
"2: eggs, soy sauce, jalapeno chilies, onions, \n",
"3: olive oil, \n",
"4: large eggs, cold water, shredded cheddar cheese, water, lemon, fresh ginger, egg yolks, oyster sauce, mayonaise, cumin seed, \n",
"5: black beans, flour tortillas, chopped cilantro, bay leaves, sugar, \n",
"6: ground black pepper, \n",
"7: garlic cloves, \n",
"8: butter, unsalted butter, \n",
"9: pepper, canola oil, diced tomatoes, chicken broth, \n",
"10: all-purpose flour, milk, \n",
"11: mushrooms, vegetable oil, \n",
"12: dried basil, minced garlic, green onions, \n",
"13: tomatoes, shallots, kosher salt, \n",
"14: whipping cream, ground nutmeg, black pepper, \n",
"15: ground coriander, lime wedges, ground turmeric, purple onion, corn starch, sesame seeds, red bell pepper, onion powder, hot sauce, cooking oil, vanilla extract, curry powder, freshly ground pepper, honey, salsa, baking soda, chopped cilantro fresh, crushed red pepper, cinnamon sticks, peanut oil, boneless skinless chicken breast halves, coarse salt, potatoes, dijon mustard, green bell pepper, coriander, flat leaf parsley, celery ribs, sour cream, avocado, garlic powder, chicken stock, flour, beansprouts, \n",
"16: oil, tomato paste, cinnamon, grated parmesan cheese, brown sugar, cooking spray, baking powder, chopped celery, ground pork, \n",
"17: warm water, paprika, chili powder, heavy cream, \n",
"18: fresh lemon juice, sesame oil, \n",
"19: bay leaf, dried oregano, worcestershire sauce, boneless skinless chicken breasts, corn tortillas, \n",
"20: scallions, \n",
"21: granulated sugar, lemon juice, balsamic vinegar, crushed red pepper flakes, bacon, \n",
"22: red wine vinegar, sea salt, ginger, clove, \n",
"23: ground cinnamon, \n",
"24: fresh basil, cucumber, zucchini, fish sauce, red pepper flakes, fresh cilantro, tumeric, capers, large egg whites, garam masala, celery, \n",
"25: large egg yolks, fresh lime juice, \n",
"26: chopped onion, chicken, fresh basil leaves, large garlic cloves, whole milk, \n",
"27: white sugar, \n",
"28: tomato sauce, cilantro, ground red pepper, \n",
"29: rice vinegar, \n",
"30: cilantro leaves, \n",
"31: cumin, chicken breasts, \n",
"32: lime juice, green chilies, plum tomatoes, coconut milk, peeled fresh ginger, shrimp, \n",
"33: ground beef, \n",
"34: white onion, red chili peppers, \n",
"35: white vinegar, \n",
"36: black peppercorns, \n",
"37: leeks, \n"
]
}
],
"source": [
"sets = {}\n",
"for i in range(len(model_ingredients)):\n",
" label = clusters.labels_[i] \n",
" if label not in sets:\n",
" sets[label] = set()\n",
" sets[label].add(ingredients[i])\n",
"for s in sets:\n",
" print(\"\" + str(s) + \": \" + \"\".join([ing + \", \" for ing in sets[s]]))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"648"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(ingredients)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}