297 lines
6.9 KiB
Plaintext
297 lines
6.9 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 1M_recipes dataset word2vec experiments"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import json\n",
|
|
"\n",
|
|
"import nltk\n",
|
|
"from nltk.stem import PorterStemmer\n",
|
|
"from nltk.stem import LancasterStemmer\n",
|
|
"from nltk.corpus import stopwords as nltk_stopwords\n",
|
|
"\n",
|
|
"from pprint import pprint\n",
|
|
"\n",
|
|
"from gensim.test.utils import common_texts, get_tmpfile\n",
|
|
"from gensim.models import Word2Vec\n",
|
|
"\n",
|
|
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
|
|
"\n",
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"import settings"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from ipypb import track"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from IPython.display import HTML, Markdown"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* reading firs n recipes from json stream"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"n = 1000000"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"buffered_reader = JSON_br(settings.one_million_recipes_file)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"instructions = []"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"stopwords = set(nltk_stopwords.words('english'))\n",
|
|
"sentence_symbols = set(('.', ';', '!', '?', ',')) \n",
|
|
"porter = PorterStemmer()\n",
|
|
"model = None\n",
|
|
"\n",
|
|
"stemmed_stopwords = set([porter.stem(word) for word in stopwords])\n",
|
|
"\n",
|
|
"def recipe2instructions(stream_item):\n",
|
|
" return [t['text'] for t in stream_item['instructions']]\n",
|
|
"\n",
|
|
"def stemmed_recipe_instruction(json_item, stemmer = porter):\n",
|
|
" item_instructions = recipe2instructions(json_item)\n",
|
|
" stemmed_list = []\n",
|
|
" for instruction in item_instructions:\n",
|
|
" stemmed_list.append([stemmer.stem(i).lower() for i in nltk.word_tokenize(instruction)])\n",
|
|
" \n",
|
|
" result = []\n",
|
|
" for stemmed in stemmed_list:\n",
|
|
" stemmed_without_stopwords = []\n",
|
|
" for word in stemmed:\n",
|
|
" if (word not in stopwords) and (word not in sentence_symbols):\n",
|
|
" stemmed_without_stopwords.append(word)\n",
|
|
" result.append(stemmed_without_stopwords)\n",
|
|
" \n",
|
|
" return result\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* example:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#item = buffered_reader.__next__()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#stemmed_recipe_instruction(item)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* reading up to n recipes:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
|
|
"<progress style=\"width:60ex\" max=\"1000000\" value=\"1000000\" class=\"Progress-main\"/></progress>\n",
|
|
"<span class=\"Progress-label\"><strong>100%</strong></span>\n",
|
|
"<span class=\"Iteration-label\">1000000/1000000</span>\n",
|
|
"<span class=\"Time-label\">[01:01:03<00:00, 0.00s/it]</span></div>"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[A\u001b[2K\r",
|
|
" [████████████████████████████████████████████████████████████] 1000000/1000000 [01:01:03<00:00, 0.00s/it]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"\n",
|
|
"for i in track(range(n)):\n",
|
|
" try:\n",
|
|
" json_recipe = buffered_reader.__next__()\n",
|
|
" except StopIteration:\n",
|
|
" print(\"reached end of stream after \" + i + \"iterations\")\n",
|
|
" break\n",
|
|
" instructions += stemmed_recipe_instruction(json_recipe)\n",
|
|
" #print(i)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* train word2vec on that instructions"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = Word2Vec(instructions, size=512, window=1, min_count=3, workers=4)\n",
|
|
"def word_similarity(word_a:str, word_b:str, model=model, stemmer=porter):\n",
|
|
" return model.wv.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n",
|
|
"\n",
|
|
"def word_exists(word:str, model=model, stemmer=porter):\n",
|
|
" return stemmer.stem(word) in model.wv\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"0.46360567"
|
|
]
|
|
},
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"word_similarity(\"dice\", \"onions\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"path = get_tmpfile(\"wordvectors.kv\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.wv.save(path)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'potato'"
|
|
]
|
|
},
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"model.wv.most_similar_to_given(\"mash\", [\"salad\",\"potato\", \"oil\", \"tomato\"])"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|