{ "cells": [ { "cell_type": "code", "execution_count": 149, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import unicodedata as uni\n", "import numpy as np\n", "from nltk.stem import PorterStemmer\n", "from nltk.tokenize import sent_tokenize, word_tokenize\n", "from nltk.corpus import wordnet\n", "import math\n", "import pprint\n", "\n", "pp=pprint.PrettyPrinter(indent=4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Naive Approach" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read in table" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptionUnnamed: 4
00126980๐Ÿ€„MAHJONG TILE RED DRAGONNaN
11129525๐ŸงตSPOOL OF THREADNaN
22129526๐ŸงถBALL OF YARNNaN
33127183๐ŸƒPLAYING CARD BLACK JOKERNaN
44129296๐ŸคZIPPER-MOUTH FACENaN
\n", "
" ], "text/plain": [ " Unnamed: 0 code character description Unnamed: 4\n", "0 0 126980 ๐Ÿ€„ MAHJONG TILE RED DRAGON NaN\n", "1 1 129525 ๐Ÿงต SPOOL OF THREAD NaN\n", "2 2 129526 ๐Ÿงถ BALL OF YARN NaN\n", "3 3 127183 ๐Ÿƒ PLAYING CARD BLACK JOKER NaN\n", "4 4 129296 ๐Ÿค ZIPPER-MOUTH FACE NaN" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n", "table.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* todo: read in a lot of messages" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "ps = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "stemmed_messages = []\n", "for m in messages:\n", " words = word_tokenize(m)\n", " sm = []\n", " for w in words:\n", " sm.append(ps.stem(w))\n", " stemmed_messages.append(sm)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n", " ['thi',\n", " 'is',\n", " 'a',\n", " 'veri',\n", " 'sunni',\n", " 'day',\n", " 'today',\n", " ',',\n", " 'i',\n", " 'am',\n", " 'veri',\n", " 'happi']]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "stemmed_messages" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1027, 5)" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table.shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* compare words to emoji descriptions" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "def evaluate_sentence(sentence):\n", " tokenized_sentence = word_tokenize(sentence)\n", " n = len(tokenized_sentence)\n", " l = table.shape[0]\n", " matrix_list = []\n", " \n", " for index, row in table.iterrows():\n", " emoji_tokens = word_tokenize(row['description'])\n", " m = len(emoji_tokens)\n", "\n", " mat = np.zeros(shape=(m,n))\n", " for i in range(len(emoji_tokens)):\n", " for j in range(len(tokenized_sentence)):\n", " syn1 = wordnet.synsets(emoji_tokens[i])\n", " if len(syn1) == 0:\n", " continue\n", " w1 = syn1[0]\n", " #print(j, tokenized_sentence)\n", " syn2 = wordnet.synsets(tokenized_sentence[j])\n", " if len(syn2) == 0:\n", " continue\n", " w2 = syn2[0]\n", " val = w1.wup_similarity(w2)\n", " if val is None:\n", " continue\n", " mat[i,j] = val\n", " #print(row['character'], mat)\n", " matrix_list.append(mat)\n", " \n", " return matrix_list\n", " \n", " " ] }, { "cell_type": "code", "execution_count": 177, "metadata": {}, "outputs": [], "source": [ "result = evaluate_sentence(\"horse back riding\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* building a lookup table:" ] }, { "cell_type": "code", "execution_count": 178, "metadata": {}, "outputs": [], "source": [ "lookup = {}\n", "emoji_set = []\n", "for index, row in table.iterrows():\n", " lookup[index] = row['character']\n", " emoji_set.append(row['character'])\n", "\n", "emoji_set = set(emoji_set)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* sorting" ] }, { "cell_type": "code", "execution_count": 179, "metadata": {}, "outputs": [], "source": [ "summed = np.argsort([-np.sum(x) for x in result])\n", "max_val = np.argsort([-np.max(x) for x in result])\n", "avg = np.argsort([-np.mean(x) for x in result])\n", "\n", "t = 0.9\n", "threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n" ] }, { "cell_type": "code", "execution_count": 180, "metadata": {}, "outputs": [], "source": [ "def print_best_results(sorted_indices, n=10):\n", " pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n", " pp.pprint([result[x] for x in sorted_indices[:10]])" ] }, { "cell_type": "code", "execution_count": 181, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[ '๐ŸŽ -- HORSE',\n", " '๐Ÿ‡ -- HORSE RACING',\n", " '๐Ÿด -- HORSE FACE',\n", " '\\U0001f993 -- ZEBRA FACE',\n", " '๐Ÿ… -- SPORTS MEDAL',\n", " '๐Ÿ—ป -- MOUNT FUJI',\n", " '๐ŸŽ  -- CAROUSEL HORSE',\n", " '๐Ÿคš -- RAISED BACK OF HAND',\n", " '๐Ÿ”™ -- BACK WITH LEFTWARDS ARROW ABOVE',\n", " '๐Ÿ“ฎ -- POSTBOX']\n", "[ array([[1. , 0.19047619, 0.08333333]]),\n", " array([[1. , 0.19047619, 0.08333333],\n", " [0.08333333, 0.13333333, 0.88888889]]),\n", " array([[1. , 0.19047619, 0.08333333],\n", " [0.18181818, 0.76923077, 0.125 ]]),\n", " array([[0.93333333, 0.19047619, 0.08333333],\n", " [0.18181818, 0.76923077, 0.125 ]]),\n", " array([[0.08695652, 0.14285714, 0.94117647],\n", " [0.09090909, 0.15384615, 0.25 ]]),\n", " array([[0.96774194, 0.18181818, 0.08 ],\n", " [0.4 , 0.19047619, 0.08333333]]),\n", " array([[0.07407407, 0.11111111, 0.19047619],\n", " [1. , 0.19047619, 0.08333333]]),\n", " array([[0.1 , 0.18181818, 0.14285714],\n", " [0.19047619, 1. , 0.13333333],\n", " [0. , 0. , 0. ],\n", " [0.17391304, 0.71428571, 0.11764706]]),\n", " array([[0.19047619, 1. , 0.13333333],\n", " [0. , 0. , 0. ],\n", " [0. , 0. , 0. ],\n", " [0.08695652, 0.14285714, 0.23529412],\n", " [0.09090909, 0.15384615, 0.25 ]]),\n", " array([[0.32 , 0.25 , 0.10526316]])]\n" ] } ], "source": [ "print_best_results(threshold)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }