{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 1M_recipes dataset word2vec experiments" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import json\n", "\n", "import nltk\n", "from nltk.stem import PorterStemmer\n", "from nltk.stem import LancasterStemmer\n", "from nltk.corpus import stopwords as nltk_stopwords\n", "\n", "from pprint import pprint\n", "\n", "from gensim.test.utils import common_texts, get_tmpfile\n", "from gensim.models import Word2Vec\n", "\n", "from json_buffered_reader import JSON_buffered_reader as JSON_br\n", "\n", "import pandas as pd\n", "\n", "import settings" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from ipypb import track" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from IPython.display import HTML, Markdown" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* reading firs n recipes from json stream" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "n = 1000000" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "buffered_reader = JSON_br(settings.one_million_recipes_file)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "instructions = []" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "stopwords = set(nltk_stopwords.words('english'))\n", "sentence_symbols = set(('.', ';', '!', '?', ',')) \n", "porter = PorterStemmer()\n", "model = None\n", "\n", "stemmed_stopwords = set([porter.stem(word) for word in stopwords])\n", "\n", "def recipe2instructions(stream_item):\n", " return [t['text'] for t in stream_item['instructions']]\n", "\n", "def stemmed_recipe_instruction(json_item, stemmer = porter):\n", " item_instructions = recipe2instructions(json_item)\n", " stemmed_list = []\n", " for instruction in item_instructions:\n", " stemmed_list.append([stemmer.stem(i).lower() for i in nltk.word_tokenize(instruction)])\n", " \n", " result = []\n", " for stemmed in stemmed_list:\n", " stemmed_without_stopwords = []\n", " for word in stemmed:\n", " if (word not in stopwords) and (word not in sentence_symbols):\n", " stemmed_without_stopwords.append(word)\n", " result.append(stemmed_without_stopwords)\n", " \n", " return result\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* example:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "#item = buffered_reader.__next__()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "#stemmed_recipe_instruction(item)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* reading up to n recipes:" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "100%\n", "1000000/1000000\n", "[01:01:03<00:00, 0.00s/it]
" ], "text/plain": [ "\u001b[A\u001b[2K\r", " [████████████████████████████████████████████████████████████] 1000000/1000000 [01:01:03<00:00, 0.00s/it]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "\n", "for i in track(range(n)):\n", " try:\n", " json_recipe = buffered_reader.__next__()\n", " except StopIteration:\n", " print(\"reached end of stream after \" + i + \"iterations\")\n", " break\n", " instructions += stemmed_recipe_instruction(json_recipe)\n", " #print(i)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* train word2vec on that instructions" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "model = Word2Vec(instructions, size=512, window=1, min_count=3, workers=4)\n", "def word_similarity(word_a:str, word_b:str, model=model, stemmer=porter):\n", " return model.wv.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n", "\n", "def word_exists(word:str, model=model, stemmer=porter):\n", " return stemmer.stem(word) in model.wv\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.46360567" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "word_similarity(\"dice\", \"onions\")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "path = get_tmpfile(\"wordvectors.kv\")" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "model.wv.save(path)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'potato'" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model.wv.most_similar_to_given(\"mash\", [\"salad\",\"potato\", \"oil\", \"tomato\"])" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }