{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import unicodedata as uni\n", "import numpy as np\n", "from nltk.stem import PorterStemmer\n", "from nltk.tokenize import sent_tokenize, word_tokenize\n", "from nltk.corpus import wordnet\n", "import math\n", "import pprint\n", "\n", "pp=pprint.PrettyPrinter(indent=4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Naive Approach" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read in table" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptiondescription_de
00126980๐Ÿ€„MAHJONG TILE RED DRAGONMAHJONG FLIESE ROT DRACHE
11129525๐ŸงตSPOOL OF THREADSpool Gewinde
22129526๐ŸงถBALL OF YARNBALL OF YARN
33127183๐ŸƒPLAYING CARD BLACK JOKERSPIELKARTE BLACK JOKER
44129296๐ŸคZIPPER-MOUTH FACEZIPPER-MUND Gesicht
\n", "
" ], "text/plain": [ " Unnamed: 0 code character description \\\n", "0 0 126980 ๐Ÿ€„ MAHJONG TILE RED DRAGON \n", "1 1 129525 ๐Ÿงต SPOOL OF THREAD \n", "2 2 129526 ๐Ÿงถ BALL OF YARN \n", "3 3 127183 ๐Ÿƒ PLAYING CARD BLACK JOKER \n", "4 4 129296 ๐Ÿค ZIPPER-MOUTH FACE \n", "\n", " description_de \n", "0 MAHJONG FLIESE ROT DRACHE \n", "1 Spool Gewinde \n", "2 BALL OF YARN \n", "3 SPIELKARTE BLACK JOKER \n", "4 ZIPPER-MUND Gesicht " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n", "table.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* todo: read in a lot of messages" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* using a Stemmer to get the main 'Part' of each word" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "ps = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "stemmed_messages = []\n", "for m in messages:\n", " words = word_tokenize(m)\n", " sm = []\n", " for w in words:\n", " sm.append(ps.stem(w))\n", " stemmed_messages.append(sm)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n", " ['thi',\n", " 'is',\n", " 'a',\n", " 'veri',\n", " 'sunni',\n", " 'day',\n", " 'today',\n", " ',',\n", " 'i',\n", " 'am',\n", " 'veri',\n", " 'happi']]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "stemmed_messages" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1027, 5)" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table.shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* compare words to emoji descriptions" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def evaluate_sentence(sentence, description_key = 'description'):\n", " tokenized_sentence = word_tokenize(sentence)\n", " n = len(tokenized_sentence)\n", " l = table.shape[0]\n", " matrix_list = []\n", " \n", " for index, row in table.iterrows():\n", " emoji_tokens = word_tokenize(row[description_key])\n", " m = len(emoji_tokens)\n", "\n", " mat = np.zeros(shape=(m,n))\n", " for i in range(len(emoji_tokens)):\n", " for j in range(len(tokenized_sentence)):\n", " syn1 = wordnet.synsets(emoji_tokens[i])\n", " if len(syn1) == 0:\n", " continue\n", " w1 = syn1[0]\n", " #print(j, tokenized_sentence)\n", " syn2 = wordnet.synsets(tokenized_sentence[j])\n", " if len(syn2) == 0:\n", " continue\n", " w2 = syn2[0]\n", " val = w1.wup_similarity(w2)\n", " if val is None:\n", " continue\n", " mat[i,j] = val\n", " #print(row['character'], mat)\n", " matrix_list.append(mat)\n", " \n", " return matrix_list\n", " \n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* building a lookup table:" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "lookup = {}\n", "emoji_set = []\n", "for index, row in table.iterrows():\n", " lookup[index] = row['character']\n", " emoji_set.append(row['character'])\n", "\n", "emoji_set = set(emoji_set)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* sorting" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def predict(sentence, description_key='description', n=10, t=0.9):\n", "\n", " result = evaluate_sentence(sentence, description_key)\n", " \n", " summed = np.argsort([-np.sum(x) for x in result])[0:n]\n", " max_val = np.argsort([-np.max(x) for x in result])[0:n]\n", " avg = np.argsort([-np.mean(x) for x in result])[0:n]\n", " threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n", " \n", " # build a result table\n", " table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n", " lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n", " lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n", " lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n", " \n", " \n", " table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n", " \n", " display(table_frame)\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
summedsummed_descriptionmax_valmax_val_descriptionavgavg_descriptionthresholdthreshold_description
0๐ŸคชGRINNING FACE WITH ONE LARGE AND ONE SMALL EYE๐ŸคŸI LOVE YOU HAND SIGN๐Ÿ’ขANGER SYMBOL๐Ÿ’ŒLOVE LETTER
1๐ŸšฎPUT LITTER IN ITS PLACE SYMBOL๐Ÿ’ŒLOVE LETTER๐Ÿ“ฆPACKAGE๐ŸฉLOVE HOTEL
2๐Ÿ––RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I๐Ÿ’ŒLOVE LETTER๐ŸคŸI LOVE YOU HAND SIGN
3๐ŸฅฐSMILING FACE WITH SMILING EYES AND THREE HEARTS๐ŸฉLOVE HOTEL๐Ÿ†‘SQUARED CL๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I
4๐ŸคญSMILING FACE WITH SMILING EYES AND HAND COVERI...๐Ÿ˜ฑFACE SCREAMING IN FEAR๐ŸŒ‰BRIDGE AT NIGHT๐Ÿ“จINCOMING ENVELOPE
5๐Ÿ”‚CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...๐Ÿ‡ญREGIONAL INDICATOR SYMBOL LETTER H๐Ÿ’BOUQUET๐Ÿ“ฉENVELOPE WITH DOWNWARDS ARROW ABOVE
6๐Ÿ”›ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...๐Ÿ‡ณREGIONAL INDICATOR SYMBOL LETTER N๐Ÿ“œSCROLL๐Ÿ“ชCLOSED MAILBOX WITH LOWERED FLAG
7๐Ÿ” INPUT SYMBOL FOR LATIN CAPITAL LETTERS๐Ÿ‡ดREGIONAL INDICATOR SYMBOL LETTER O๐ŸŽWRAPPED PRESENT๐Ÿ“ซCLOSED MAILBOX WITH RAISED FLAG
8๐ŸคŸI LOVE YOU HAND SIGN๐Ÿ“ฒMOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT๐Ÿ’จDASH SYMBOL๐Ÿ“ฐNEWSPAPER
9๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I๐ŸŒ†CITYSCAPE AT DUSK๐Ÿ’คSLEEPING SYMBOL๐Ÿ“ฌOPEN MAILBOX WITH RAISED FLAG
\n", "
" ], "text/plain": [ " summed summed_description max_val \\\n", "0 ๐Ÿคช GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE ๐ŸคŸ \n", "1 ๐Ÿšฎ PUT LITTER IN ITS PLACE SYMBOL ๐Ÿ’Œ \n", "2 ๐Ÿ–– RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... ๐Ÿ‡ฎ \n", "3 ๐Ÿฅฐ SMILING FACE WITH SMILING EYES AND THREE HEARTS ๐Ÿฉ \n", "4 ๐Ÿคญ SMILING FACE WITH SMILING EYES AND HAND COVERI... ๐Ÿ˜ฑ \n", "5 ๐Ÿ”‚ CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... ๐Ÿ‡ญ \n", "6 ๐Ÿ”› ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... ๐Ÿ‡ณ \n", "7 ๐Ÿ”  INPUT SYMBOL FOR LATIN CAPITAL LETTERS ๐Ÿ‡ด \n", "8 ๐ŸคŸ I LOVE YOU HAND SIGN ๐Ÿ“ฒ \n", "9 ๐Ÿ‡ฎ REGIONAL INDICATOR SYMBOL LETTER I ๐ŸŒ† \n", "\n", " max_val_description avg avg_description threshold \\\n", "0 I LOVE YOU HAND SIGN ๐Ÿ’ข ANGER SYMBOL ๐Ÿ’Œ \n", "1 LOVE LETTER ๐Ÿ“ฆ PACKAGE ๐Ÿฉ \n", "2 REGIONAL INDICATOR SYMBOL LETTER I ๐Ÿ’Œ LOVE LETTER ๐ŸคŸ \n", "3 LOVE HOTEL ๐Ÿ†‘ SQUARED CL ๐Ÿ‡ฎ \n", "4 FACE SCREAMING IN FEAR ๐ŸŒ‰ BRIDGE AT NIGHT ๐Ÿ“จ \n", "5 REGIONAL INDICATOR SYMBOL LETTER H ๐Ÿ’ BOUQUET ๐Ÿ“ฉ \n", "6 REGIONAL INDICATOR SYMBOL LETTER N ๐Ÿ“œ SCROLL ๐Ÿ“ช \n", "7 REGIONAL INDICATOR SYMBOL LETTER O ๐ŸŽ WRAPPED PRESENT ๐Ÿ“ซ \n", "8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT ๐Ÿ’จ DASH SYMBOL ๐Ÿ“ฐ \n", "9 CITYSCAPE AT DUSK ๐Ÿ’ค SLEEPING SYMBOL ๐Ÿ“ฌ \n", "\n", " threshold_description \n", "0 LOVE LETTER \n", "1 LOVE HOTEL \n", "2 I LOVE YOU HAND SIGN \n", "3 REGIONAL INDICATOR SYMBOL LETTER I \n", "4 INCOMING ENVELOPE \n", "5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n", "6 CLOSED MAILBOX WITH LOWERED FLAG \n", "7 CLOSED MAILBOX WITH RAISED FLAG \n", "8 NEWSPAPER \n", "9 OPEN MAILBOX WITH RAISED FLAG " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "predict(\"I love sunny days!\", description_key='description' )" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }