master-thesis/1M_word2vec.ipynb

297 lines
6.9 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 1M_recipes dataset word2vec experiments"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import json\n",
"\n",
"import nltk\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.stem import LancasterStemmer\n",
"from nltk.corpus import stopwords as nltk_stopwords\n",
"\n",
"from pprint import pprint\n",
"\n",
"from gensim.test.utils import common_texts, get_tmpfile\n",
"from gensim.models import Word2Vec\n",
"\n",
"from json_buffered_reader import JSON_buffered_reader as JSON_br\n",
"\n",
"import pandas as pd\n",
"\n",
"import settings"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from ipypb import track"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from IPython.display import HTML, Markdown"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* reading firs n recipes from json stream"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"n = 1000000"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"buffered_reader = JSON_br(settings.one_million_recipes_file)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"instructions = []"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"stopwords = set(nltk_stopwords.words('english'))\n",
"sentence_symbols = set(('.', ';', '!', '?', ',')) \n",
"porter = PorterStemmer()\n",
"model = None\n",
"\n",
"stemmed_stopwords = set([porter.stem(word) for word in stopwords])\n",
"\n",
"def recipe2instructions(stream_item):\n",
" return [t['text'] for t in stream_item['instructions']]\n",
"\n",
"def stemmed_recipe_instruction(json_item, stemmer = porter):\n",
" item_instructions = recipe2instructions(json_item)\n",
" stemmed_list = []\n",
" for instruction in item_instructions:\n",
" stemmed_list.append([stemmer.stem(i).lower() for i in nltk.word_tokenize(instruction)])\n",
" \n",
" result = []\n",
" for stemmed in stemmed_list:\n",
" stemmed_without_stopwords = []\n",
" for word in stemmed:\n",
" if (word not in stopwords) and (word not in sentence_symbols):\n",
" stemmed_without_stopwords.append(word)\n",
" result.append(stemmed_without_stopwords)\n",
" \n",
" return result\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* example:"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"#item = buffered_reader.__next__()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"#stemmed_recipe_instruction(item)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* reading up to n recipes:"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
"<progress style=\"width:60ex\" max=\"1000000\" value=\"1000000\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>100%</strong></span>\n",
"<span class=\"Iteration-label\">1000000/1000000</span>\n",
"<span class=\"Time-label\">[01:01:03<00:00, 0.00s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
" [████████████████████████████████████████████████████████████] 1000000/1000000 [01:01:03<00:00, 0.00s/it]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"for i in track(range(n)):\n",
" try:\n",
" json_recipe = buffered_reader.__next__()\n",
" except StopIteration:\n",
" print(\"reached end of stream after \" + i + \"iterations\")\n",
" break\n",
" instructions += stemmed_recipe_instruction(json_recipe)\n",
" #print(i)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* train word2vec on that instructions"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"model = Word2Vec(instructions, size=512, window=1, min_count=3, workers=4)\n",
"def word_similarity(word_a:str, word_b:str, model=model, stemmer=porter):\n",
" return model.wv.similarity(stemmer.stem(word_a), stemmer.stem(word_b))\n",
"\n",
"def word_exists(word:str, model=model, stemmer=porter):\n",
" return stemmer.stem(word) in model.wv\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.46360567"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"word_similarity(\"dice\", \"onions\")"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"path = get_tmpfile(\"wordvectors.kv\")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"model.wv.save(path)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'potato'"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.wv.most_similar_to_given(\"mash\", [\"salad\",\"potato\", \"oil\", \"tomato\"])"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}