{ "cells": [ { "cell_type": "code", "execution_count": 149, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import unicodedata as uni\n", "import numpy as np\n", "from nltk.stem import PorterStemmer\n", "from nltk.tokenize import sent_tokenize, word_tokenize\n", "from nltk.corpus import wordnet\n", "import math\n", "import pprint\n", "\n", "pp=pprint.PrettyPrinter(indent=4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Naive Approach" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read in table" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptionUnnamed: 4
00126980๐Ÿ€„MAHJONG TILE RED DRAGONNaN
11129525๐ŸงตSPOOL OF THREADNaN
22129526๐ŸงถBALL OF YARNNaN
33127183๐ŸƒPLAYING CARD BLACK JOKERNaN
44129296๐ŸคZIPPER-MOUTH FACENaN
\n", "
" ], "text/plain": [ " Unnamed: 0 code character description Unnamed: 4\n", "0 0 126980 ๐Ÿ€„ MAHJONG TILE RED DRAGON NaN\n", "1 1 129525 ๐Ÿงต SPOOL OF THREAD NaN\n", "2 2 129526 ๐Ÿงถ BALL OF YARN NaN\n", "3 3 127183 ๐Ÿƒ PLAYING CARD BLACK JOKER NaN\n", "4 4 129296 ๐Ÿค ZIPPER-MOUTH FACE NaN" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n", "table.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* todo: read in a lot of messages" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "ps = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "stemmed_messages = []\n", "for m in messages:\n", " words = word_tokenize(m)\n", " sm = []\n", " for w in words:\n", " sm.append(ps.stem(w))\n", " stemmed_messages.append(sm)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n", " ['thi',\n", " 'is',\n", " 'a',\n", " 'veri',\n", " 'sunni',\n", " 'day',\n", " 'today',\n", " ',',\n", " 'i',\n", " 'am',\n", " 'veri',\n", " 'happi']]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "stemmed_messages" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1027, 5)" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table.shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* compare words to emoji descriptions" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "def evaluate_sentence(sentence):\n", " tokenized_sentence = word_tokenize(sentence)\n", " n = len(tokenized_sentence)\n", " l = table.shape[0]\n", " matrix_list = []\n", " \n", " for index, row in table.iterrows():\n", " emoji_tokens = word_tokenize(row['description'])\n", " m = len(emoji_tokens)\n", "\n", " mat = np.zeros(shape=(m,n))\n", " for i in range(len(emoji_tokens)):\n", " for j in range(len(tokenized_sentence)):\n", " syn1 = wordnet.synsets(emoji_tokens[i])\n", " if len(syn1) == 0:\n", " continue\n", " w1 = syn1[0]\n", " #print(j, tokenized_sentence)\n", " syn2 = wordnet.synsets(tokenized_sentence[j])\n", " if len(syn2) == 0:\n", " continue\n", " w2 = syn2[0]\n", " val = w1.wup_similarity(w2)\n", " if val is None:\n", " continue\n", " mat[i,j] = val\n", " #print(row['character'], mat)\n", " matrix_list.append(mat)\n", " \n", " return matrix_list\n", " \n", " " ] }, { "cell_type": "code", "execution_count": 130, "metadata": {}, "outputs": [], "source": [ "result = evaluate_sentence(\"I like playing soccer\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* building a lookup table:" ] }, { "cell_type": "code", "execution_count": 131, "metadata": {}, "outputs": [], "source": [ "lookup = {}\n", "for index, row in table.iterrows():\n", " lookup[index] = row['character']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* sorting" ] }, { "cell_type": "code", "execution_count": 139, "metadata": {}, "outputs": [], "source": [ "summed = np.argsort([-np.sum(x) for x in result])\n", "max_val = np.argsort([-np.max(x) for x in result])\n", "avg = np.argsort([-np.mean(x) for x in result])\n", "\n", "t = 0.9\n", "threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n" ] }, { "cell_type": "code", "execution_count": 156, "metadata": {}, "outputs": [], "source": [ "def print_best_results(sorted_indices, n=10):\n", " pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n", " pp.pprint([result[x] for x in sorted_indices[:10]])" ] }, { "cell_type": "code", "execution_count": 157, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[ 'โšฝ -- SOCCER BALL',\n", " '๐Ÿ‰ -- RUGBY FOOTBALL',\n", " '๐Ÿˆ -- AMERICAN FOOTBALL',\n", " '๐ŸŽด -- FLOWER PLAYING CARDS',\n", " '๐Ÿƒ -- PLAYING CARD BLACK JOKER',\n", " '๐Ÿ‡ฎ -- REGIONAL INDICATOR SYMBOL LETTER I',\n", " '\\U0001f91f -- I LOVE YOU HAND SIGN',\n", " '๐Ÿ“ง -- E-MAIL SYMBOL',\n", " '๐Ÿ“ญ -- OPEN MAILBOX WITH LOWERED FLAG',\n", " '๐Ÿ“ฅ -- INBOX TRAY']\n", "[ array([[0.25 , 0.28571429, 0.58333333, 1. ],\n", " [0.26666667, 0.10526316, 0.1 , 0.1 ]]),\n", " array([[0.25 , 0.28571429, 0.58333333, 0.84615385],\n", " [0.26666667, 0.3 , 0.60869565, 0.96 ]]),\n", " array([[0.33333333, 0.125 , 0.11764706, 0.11764706],\n", " [0.26666667, 0.3 , 0.60869565, 0.96 ]]),\n", " array([[0.23529412, 0.0952381 , 0.09090909, 0.09090909],\n", " [0.25 , 0.47619048, 1. , 0.58333333],\n", " [0.30769231, 0.33333333, 0.57142857, 0.7 ]]),\n", " array([[0.25 , 0.47619048, 1. , 0.58333333],\n", " [0.53333333, 0.22222222, 0.21052632, 0.21052632],\n", " [0.30769231, 0.22222222, 0.21052632, 0.21052632],\n", " [0.28571429, 0.11111111, 0.10526316, 0.10526316]]),\n", " array([[0. , 0. , 0. , 0. ],\n", " [0.33333333, 0.23529412, 0.22222222, 0.22222222],\n", " [0.4 , 0.26666667, 0.25 , 0.25 ],\n", " [0.30769231, 0.22222222, 0.21052632, 0.21052632],\n", " [1. , 0.26666667, 0.25 , 0.25 ]]),\n", " array([[1. , 0.26666667, 0.25 , 0.25 ],\n", " [0.33333333, 0.23529412, 0.22222222, 0.22222222],\n", " [0. , 0. , 0. , 0. ],\n", " [0.28571429, 0.11111111, 0.10526316, 0.10526316],\n", " [0.33333333, 0.23529412, 0.22222222, 0.22222222]]),\n", " array([[0.28571429, 0.31578947, 0.45454545, 0.5 ],\n", " [0.4 , 0.26666667, 0.25 , 0.25 ]]),\n", " array([[0.30769231, 0.11764706, 0.11111111, 0.11111111],\n", " [0.26666667, 0.10526316, 0.1 , 0.1 ],\n", " [0. , 0. , 0. , 0. ],\n", " [0.22222222, 0.14285714, 0.13333333, 0.13333333],\n", " [0.26666667, 0.10526316, 0.1 , 0.1 ]]),\n", " array([[0. , 0. , 0. , 0. ],\n", " [0.26666667, 0.10526316, 0.1 , 0.1 ]])]\n" ] } ], "source": [ "print_best_results(threshold)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }