{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 1M_recipes dataset word2vec experiments" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import json\n", "\n", "import nltk\n", "from nltk.stem import PorterStemmer\n", "from nltk.stem import LancasterStemmer\n", "from nltk.corpus import stopwords as nltk_stopwords\n", "\n", "from pprint import pprint\n", "\n", "from gensim.test.utils import common_texts, get_tmpfile\n", "from gensim.models import Word2Vec\n", "\n", "from json_buffered_reader import JSON_buffered_reader as JSON_br\n", "\n", "import pandas as pd\n", "\n", "import settings" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from ipypb import track" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from IPython.display import HTML, Markdown" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* reading firs n recipes from json stream" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "n = 1000000" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "buffered_reader = JSON_br(settings.one_million_recipes_file)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "instructions = []" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "stopwords = set(nltk_stopwords.words('english'))\n", "sentence_symbols = set(('.', ';', '!', '?', ',')) \n", "porter = PorterStemmer()\n", "model = None\n", "\n", "stemmed_stopwords = set([porter.stem(word) for word in stopwords])\n", "\n", "def recipe2instructions(stream_item):\n", " return [t['text'] for t in stream_item['instructions']]\n", "\n", "def stemmed_recipe_instruction(json_item, stemmer = porter):\n", " item_instructions = recipe2instructions(json_item)\n", " stemmed_list = []\n", " for instruction in item_instructions:\n", " stemmed_list.append([stemmer.stem(i).lower() for i in nltk.word_tokenize(instruction)])\n", " \n", " result = []\n", " for stemmed in stemmed_list:\n", " stemmed_without_stopwords = []\n", " for word in stemmed:\n", " if (word not in stopwords) and (word not in sentence_symbols):\n", " stemmed_without_stopwords.append(word)\n", " result.append(stemmed_without_stopwords)\n", " \n", " return result\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* example:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "#item = buffered_reader.__next__()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "#stemmed_recipe_instruction(item)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* reading up to n recipes:" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "