{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os\n",
"import unicodedata as uni\n",
"import numpy as np\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.tokenize import sent_tokenize, word_tokenize\n",
"from nltk.corpus import wordnet\n",
"import math\n",
"import pprint\n",
"\n",
"pp=pprint.PrettyPrinter(indent=4)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Naive Approach"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read in table"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Unnamed: 0 | \n",
" code | \n",
" character | \n",
" description | \n",
" description_de | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 0 | \n",
" 126980 | \n",
" ๐ | \n",
" MAHJONG TILE RED DRAGON | \n",
" MAHJONG FLIESE ROT DRACHE | \n",
"
\n",
" \n",
" 1 | \n",
" 1 | \n",
" 129525 | \n",
" ๐งต | \n",
" SPOOL OF THREAD | \n",
" Spool Gewinde | \n",
"
\n",
" \n",
" 2 | \n",
" 2 | \n",
" 129526 | \n",
" ๐งถ | \n",
" BALL OF YARN | \n",
" BALL OF YARN | \n",
"
\n",
" \n",
" 3 | \n",
" 3 | \n",
" 127183 | \n",
" ๐ | \n",
" PLAYING CARD BLACK JOKER | \n",
" SPIELKARTE BLACK JOKER | \n",
"
\n",
" \n",
" 4 | \n",
" 4 | \n",
" 129296 | \n",
" ๐ค | \n",
" ZIPPER-MOUTH FACE | \n",
" ZIPPER-MUND Gesicht | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Unnamed: 0 code character description \\\n",
"0 0 126980 ๐ MAHJONG TILE RED DRAGON \n",
"1 1 129525 ๐งต SPOOL OF THREAD \n",
"2 2 129526 ๐งถ BALL OF YARN \n",
"3 3 127183 ๐ PLAYING CARD BLACK JOKER \n",
"4 4 129296 ๐ค ZIPPER-MOUTH FACE \n",
"\n",
" description_de \n",
"0 MAHJONG FLIESE ROT DRACHE \n",
"1 Spool Gewinde \n",
"2 BALL OF YARN \n",
"3 SPIELKARTE BLACK JOKER \n",
"4 ZIPPER-MUND Gesicht "
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"table = pd.read_csv('../Tools/emoji_descriptions.csv')\n",
"table.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* todo: read in a lot of messages"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* using a Stemmer to get the main 'Part' of each word"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"ps = PorterStemmer()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"stemmed_messages = []\n",
"for m in messages:\n",
" words = word_tokenize(m)\n",
" sm = []\n",
" for w in words:\n",
" sm.append(ps.stem(w))\n",
" stemmed_messages.append(sm)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n",
" ['thi',\n",
" 'is',\n",
" 'a',\n",
" 'veri',\n",
" 'sunni',\n",
" 'day',\n",
" 'today',\n",
" ',',\n",
" 'i',\n",
" 'am',\n",
" 'veri',\n",
" 'happi']]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"stemmed_messages"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1027, 5)"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"table.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* compare words to emoji descriptions"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def evaluate_sentence(sentence, description_key = 'description'):\n",
" tokenized_sentence = word_tokenize(sentence)\n",
" n = len(tokenized_sentence)\n",
" l = table.shape[0]\n",
" matrix_list = []\n",
" \n",
" for index, row in table.iterrows():\n",
" emoji_tokens = word_tokenize(row[description_key])\n",
" m = len(emoji_tokens)\n",
"\n",
" mat = np.zeros(shape=(m,n))\n",
" for i in range(len(emoji_tokens)):\n",
" for j in range(len(tokenized_sentence)):\n",
" syn1 = wordnet.synsets(emoji_tokens[i])\n",
" if len(syn1) == 0:\n",
" continue\n",
" w1 = syn1[0]\n",
" #print(j, tokenized_sentence)\n",
" syn2 = wordnet.synsets(tokenized_sentence[j])\n",
" if len(syn2) == 0:\n",
" continue\n",
" w2 = syn2[0]\n",
" val = w1.wup_similarity(w2)\n",
" if val is None:\n",
" continue\n",
" mat[i,j] = val\n",
" #print(row['character'], mat)\n",
" matrix_list.append(mat)\n",
" \n",
" return matrix_list\n",
" \n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* building a lookup table:"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"lookup = {}\n",
"emoji_set = []\n",
"for index, row in table.iterrows():\n",
" lookup[index] = row['character']\n",
" emoji_set.append(row['character'])\n",
"\n",
"emoji_set = set(emoji_set)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* sorting"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def predict(sentence, description_key='description', n=10, t=0.9):\n",
"\n",
" result = evaluate_sentence(sentence, description_key)\n",
" \n",
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
" avg = np.argsort([-np.mean(x) for x in result])[0:n]\n",
" threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n",
" \n",
" # build a result table\n",
" table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n",
" lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n",
" lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n",
" lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n",
" \n",
" \n",
" table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n",
" \n",
" display(table_frame)\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" summed | \n",
" summed_description | \n",
" max_val | \n",
" max_val_description | \n",
" avg | \n",
" avg_description | \n",
" threshold | \n",
" threshold_description | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" ๐คช | \n",
" GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE | \n",
" ๐ค | \n",
" I LOVE YOU HAND SIGN | \n",
" ๐ข | \n",
" ANGER SYMBOL | \n",
" ๐ | \n",
" LOVE LETTER | \n",
"
\n",
" \n",
" 1 | \n",
" ๐ฎ | \n",
" PUT LITTER IN ITS PLACE SYMBOL | \n",
" ๐ | \n",
" LOVE LETTER | \n",
" ๐ฆ | \n",
" PACKAGE | \n",
" ๐ฉ | \n",
" LOVE HOTEL | \n",
"
\n",
" \n",
" 2 | \n",
" ๐ | \n",
" RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... | \n",
" ๐ฎ | \n",
" REGIONAL INDICATOR SYMBOL LETTER I | \n",
" ๐ | \n",
" LOVE LETTER | \n",
" ๐ค | \n",
" I LOVE YOU HAND SIGN | \n",
"
\n",
" \n",
" 3 | \n",
" ๐ฅฐ | \n",
" SMILING FACE WITH SMILING EYES AND THREE HEARTS | \n",
" ๐ฉ | \n",
" LOVE HOTEL | \n",
" ๐ | \n",
" SQUARED CL | \n",
" ๐ฎ | \n",
" REGIONAL INDICATOR SYMBOL LETTER I | \n",
"
\n",
" \n",
" 4 | \n",
" ๐คญ | \n",
" SMILING FACE WITH SMILING EYES AND HAND COVERI... | \n",
" ๐ฑ | \n",
" FACE SCREAMING IN FEAR | \n",
" ๐ | \n",
" BRIDGE AT NIGHT | \n",
" ๐จ | \n",
" INCOMING ENVELOPE | \n",
"
\n",
" \n",
" 5 | \n",
" ๐ | \n",
" CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... | \n",
" ๐ญ | \n",
" REGIONAL INDICATOR SYMBOL LETTER H | \n",
" ๐ | \n",
" BOUQUET | \n",
" ๐ฉ | \n",
" ENVELOPE WITH DOWNWARDS ARROW ABOVE | \n",
"
\n",
" \n",
" 6 | \n",
" ๐ | \n",
" ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... | \n",
" ๐ณ | \n",
" REGIONAL INDICATOR SYMBOL LETTER N | \n",
" ๐ | \n",
" SCROLL | \n",
" ๐ช | \n",
" CLOSED MAILBOX WITH LOWERED FLAG | \n",
"
\n",
" \n",
" 7 | \n",
" ๐ | \n",
" INPUT SYMBOL FOR LATIN CAPITAL LETTERS | \n",
" ๐ด | \n",
" REGIONAL INDICATOR SYMBOL LETTER O | \n",
" ๐ | \n",
" WRAPPED PRESENT | \n",
" ๐ซ | \n",
" CLOSED MAILBOX WITH RAISED FLAG | \n",
"
\n",
" \n",
" 8 | \n",
" ๐ค | \n",
" I LOVE YOU HAND SIGN | \n",
" ๐ฒ | \n",
" MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT | \n",
" ๐จ | \n",
" DASH SYMBOL | \n",
" ๐ฐ | \n",
" NEWSPAPER | \n",
"
\n",
" \n",
" 9 | \n",
" ๐ฎ | \n",
" REGIONAL INDICATOR SYMBOL LETTER I | \n",
" ๐ | \n",
" CITYSCAPE AT DUSK | \n",
" ๐ค | \n",
" SLEEPING SYMBOL | \n",
" ๐ฌ | \n",
" OPEN MAILBOX WITH RAISED FLAG | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" summed summed_description max_val \\\n",
"0 ๐คช GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE ๐ค \n",
"1 ๐ฎ PUT LITTER IN ITS PLACE SYMBOL ๐ \n",
"2 ๐ RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... ๐ฎ \n",
"3 ๐ฅฐ SMILING FACE WITH SMILING EYES AND THREE HEARTS ๐ฉ \n",
"4 ๐คญ SMILING FACE WITH SMILING EYES AND HAND COVERI... ๐ฑ \n",
"5 ๐ CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... ๐ญ \n",
"6 ๐ ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... ๐ณ \n",
"7 ๐ INPUT SYMBOL FOR LATIN CAPITAL LETTERS ๐ด \n",
"8 ๐ค I LOVE YOU HAND SIGN ๐ฒ \n",
"9 ๐ฎ REGIONAL INDICATOR SYMBOL LETTER I ๐ \n",
"\n",
" max_val_description avg avg_description threshold \\\n",
"0 I LOVE YOU HAND SIGN ๐ข ANGER SYMBOL ๐ \n",
"1 LOVE LETTER ๐ฆ PACKAGE ๐ฉ \n",
"2 REGIONAL INDICATOR SYMBOL LETTER I ๐ LOVE LETTER ๐ค \n",
"3 LOVE HOTEL ๐ SQUARED CL ๐ฎ \n",
"4 FACE SCREAMING IN FEAR ๐ BRIDGE AT NIGHT ๐จ \n",
"5 REGIONAL INDICATOR SYMBOL LETTER H ๐ BOUQUET ๐ฉ \n",
"6 REGIONAL INDICATOR SYMBOL LETTER N ๐ SCROLL ๐ช \n",
"7 REGIONAL INDICATOR SYMBOL LETTER O ๐ WRAPPED PRESENT ๐ซ \n",
"8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT ๐จ DASH SYMBOL ๐ฐ \n",
"9 CITYSCAPE AT DUSK ๐ค SLEEPING SYMBOL ๐ฌ \n",
"\n",
" threshold_description \n",
"0 LOVE LETTER \n",
"1 LOVE HOTEL \n",
"2 I LOVE YOU HAND SIGN \n",
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
"4 INCOMING ENVELOPE \n",
"5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n",
"6 CLOSED MAILBOX WITH LOWERED FLAG \n",
"7 CLOSED MAILBOX WITH RAISED FLAG \n",
"8 NEWSPAPER \n",
"9 OPEN MAILBOX WITH RAISED FLAG "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"predict(\"I love sunny days!\", description_key='description' )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}