better naive approach

This commit is contained in:
Jonas Weinz 2018-06-05 10:02:35 +02:00
parent 9a922d88fa
commit 12f99af9a9

View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 149, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -65,7 +65,7 @@
" <th>code</th>\n", " <th>code</th>\n",
" <th>character</th>\n", " <th>character</th>\n",
" <th>description</th>\n", " <th>description</th>\n",
" <th>Unnamed: 4</th>\n", " <th>description_de</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
@ -75,7 +75,7 @@
" <td>126980</td>\n", " <td>126980</td>\n",
" <td>🀄</td>\n", " <td>🀄</td>\n",
" <td>MAHJONG TILE RED DRAGON</td>\n", " <td>MAHJONG TILE RED DRAGON</td>\n",
" <td>NaN</td>\n", " <td>MAHJONG FLIESE ROT DRACHE</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>1</th>\n", " <th>1</th>\n",
@ -83,7 +83,7 @@
" <td>129525</td>\n", " <td>129525</td>\n",
" <td>🧵</td>\n", " <td>🧵</td>\n",
" <td>SPOOL OF THREAD</td>\n", " <td>SPOOL OF THREAD</td>\n",
" <td>NaN</td>\n", " <td>Spool Gewinde</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>2</th>\n", " <th>2</th>\n",
@ -91,7 +91,7 @@
" <td>129526</td>\n", " <td>129526</td>\n",
" <td>🧶</td>\n", " <td>🧶</td>\n",
" <td>BALL OF YARN</td>\n", " <td>BALL OF YARN</td>\n",
" <td>NaN</td>\n", " <td>BALL OF YARN</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>3</th>\n", " <th>3</th>\n",
@ -99,7 +99,7 @@
" <td>127183</td>\n", " <td>127183</td>\n",
" <td>🃏</td>\n", " <td>🃏</td>\n",
" <td>PLAYING CARD BLACK JOKER</td>\n", " <td>PLAYING CARD BLACK JOKER</td>\n",
" <td>NaN</td>\n", " <td>SPIELKARTE BLACK JOKER</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>4</th>\n", " <th>4</th>\n",
@ -107,19 +107,26 @@
" <td>129296</td>\n", " <td>129296</td>\n",
" <td>🤐</td>\n", " <td>🤐</td>\n",
" <td>ZIPPER-MOUTH FACE</td>\n", " <td>ZIPPER-MOUTH FACE</td>\n",
" <td>NaN</td>\n", " <td>ZIPPER-MUND Gesicht</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" Unnamed: 0 code character description Unnamed: 4\n", " Unnamed: 0 code character description \\\n",
"0 0 126980 🀄 MAHJONG TILE RED DRAGON NaN\n", "0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
"1 1 129525 🧵 SPOOL OF THREAD NaN\n", "1 1 129525 🧵 SPOOL OF THREAD \n",
"2 2 129526 🧶 BALL OF YARN NaN\n", "2 2 129526 🧶 BALL OF YARN \n",
"3 3 127183 🃏 PLAYING CARD BLACK JOKER NaN\n", "3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
"4 4 129296 🤐 ZIPPER-MOUTH FACE NaN" "4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
"\n",
" description_de \n",
"0 MAHJONG FLIESE ROT DRACHE \n",
"1 Spool Gewinde \n",
"2 BALL OF YARN \n",
"3 SPIELKARTE BLACK JOKER \n",
"4 ZIPPER-MUND Gesicht "
] ]
}, },
"execution_count": 2, "execution_count": 2,
@ -148,6 +155,13 @@
"messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
] ]
}, },
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* using a Stemmer to get the main 'Part' of each word"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 4,
@ -233,18 +247,18 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 59, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def evaluate_sentence(sentence):\n", "def evaluate_sentence(sentence, description_key = 'description'):\n",
" tokenized_sentence = word_tokenize(sentence)\n", " tokenized_sentence = word_tokenize(sentence)\n",
" n = len(tokenized_sentence)\n", " n = len(tokenized_sentence)\n",
" l = table.shape[0]\n", " l = table.shape[0]\n",
" matrix_list = []\n", " matrix_list = []\n",
" \n", " \n",
" for index, row in table.iterrows():\n", " for index, row in table.iterrows():\n",
" emoji_tokens = word_tokenize(row['description'])\n", " emoji_tokens = word_tokenize(row[description_key])\n",
" m = len(emoji_tokens)\n", " m = len(emoji_tokens)\n",
"\n", "\n",
" mat = np.zeros(shape=(m,n))\n", " mat = np.zeros(shape=(m,n))\n",
@ -271,15 +285,6 @@
" " " "
] ]
}, },
{
"cell_type": "code",
"execution_count": 177,
"metadata": {},
"outputs": [],
"source": [
"result = evaluate_sentence(\"horse back riding\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
@ -289,7 +294,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 178, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -311,76 +316,226 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 179, "execution_count": 10,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"summed = np.argsort([-np.sum(x) for x in result])\n", "def predict(sentence, description_key='description', n=10, t=0.9):\n",
"max_val = np.argsort([-np.max(x) for x in result])\n",
"avg = np.argsort([-np.mean(x) for x in result])\n",
"\n", "\n",
"t = 0.9\n", " result = evaluate_sentence(sentence, description_key)\n",
"threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n" " \n",
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
" avg = np.argsort([-np.mean(x) for x in result])[0:n]\n",
" threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n",
" \n",
" # build a result table\n",
" table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n",
" lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n",
" lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n",
" lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n",
" \n",
" \n",
" table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n",
" \n",
" display(table_frame)\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 180, "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def print_best_results(sorted_indices, n=10):\n",
" pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n",
" pp.pprint([result[x] for x in sorted_indices[:10]])"
]
},
{
"cell_type": "code",
"execution_count": 181,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "data": {
"output_type": "stream", "text/html": [
"text": [ "<div>\n",
"[ '🐎 -- HORSE',\n", "<style scoped>\n",
" '🏇 -- HORSE RACING',\n", " .dataframe tbody tr th:only-of-type {\n",
" '🐴 -- HORSE FACE',\n", " vertical-align: middle;\n",
" '\\U0001f993 -- ZEBRA FACE',\n", " }\n",
" '🏅 -- SPORTS MEDAL',\n", "\n",
" '🗻 -- MOUNT FUJI',\n", " .dataframe tbody tr th {\n",
" '🎠 -- CAROUSEL HORSE',\n", " vertical-align: top;\n",
" '🤚 -- RAISED BACK OF HAND',\n", " }\n",
" '🔙 -- BACK WITH LEFTWARDS ARROW ABOVE',\n", "\n",
" '📮 -- POSTBOX']\n", " .dataframe thead th {\n",
"[ array([[1. , 0.19047619, 0.08333333]]),\n", " text-align: right;\n",
" array([[1. , 0.19047619, 0.08333333],\n", " }\n",
" [0.08333333, 0.13333333, 0.88888889]]),\n", "</style>\n",
" array([[1. , 0.19047619, 0.08333333],\n", "<table border=\"1\" class=\"dataframe\">\n",
" [0.18181818, 0.76923077, 0.125 ]]),\n", " <thead>\n",
" array([[0.93333333, 0.19047619, 0.08333333],\n", " <tr style=\"text-align: right;\">\n",
" [0.18181818, 0.76923077, 0.125 ]]),\n", " <th></th>\n",
" array([[0.08695652, 0.14285714, 0.94117647],\n", " <th>summed</th>\n",
" [0.09090909, 0.15384615, 0.25 ]]),\n", " <th>summed_description</th>\n",
" array([[0.96774194, 0.18181818, 0.08 ],\n", " <th>max_val</th>\n",
" [0.4 , 0.19047619, 0.08333333]]),\n", " <th>max_val_description</th>\n",
" array([[0.07407407, 0.11111111, 0.19047619],\n", " <th>avg</th>\n",
" [1. , 0.19047619, 0.08333333]]),\n", " <th>avg_description</th>\n",
" array([[0.1 , 0.18181818, 0.14285714],\n", " <th>threshold</th>\n",
" [0.19047619, 1. , 0.13333333],\n", " <th>threshold_description</th>\n",
" [0. , 0. , 0. ],\n", " </tr>\n",
" [0.17391304, 0.71428571, 0.11764706]]),\n", " </thead>\n",
" array([[0.19047619, 1. , 0.13333333],\n", " <tbody>\n",
" [0. , 0. , 0. ],\n", " <tr>\n",
" [0. , 0. , 0. ],\n", " <th>0</th>\n",
" [0.08695652, 0.14285714, 0.23529412],\n", " <td>🤪</td>\n",
" [0.09090909, 0.15384615, 0.25 ]]),\n", " <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
" array([[0.32 , 0.25 , 0.10526316]])]\n" " <td>🤟</td>\n",
] " <td>I LOVE YOU HAND SIGN</td>\n",
" <td>💢</td>\n",
" <td>ANGER SYMBOL</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>🚮</td>\n",
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>📦</td>\n",
" <td>PACKAGE</td>\n",
" <td>🏩</td>\n",
" <td>LOVE HOTEL</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>🖖</td>\n",
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>🥰</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
" <td>🏩</td>\n",
" <td>LOVE HOTEL</td>\n",
" <td>🆑</td>\n",
" <td>SQUARED CL</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>🤭</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
" <td>😱</td>\n",
" <td>FACE SCREAMING IN FEAR</td>\n",
" <td>🌉</td>\n",
" <td>BRIDGE AT NIGHT</td>\n",
" <td>📨</td>\n",
" <td>INCOMING ENVELOPE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>🔂</td>\n",
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
" <td>🇭</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER H</td>\n",
" <td>💐</td>\n",
" <td>BOUQUET</td>\n",
" <td>📩</td>\n",
" <td>ENVELOPE WITH DOWNWARDS ARROW ABOVE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>🔛</td>\n",
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
" <td>🇳</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER N</td>\n",
" <td>📜</td>\n",
" <td>SCROLL</td>\n",
" <td>📪</td>\n",
" <td>CLOSED MAILBOX WITH LOWERED FLAG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>🔠</td>\n",
" <td>INPUT SYMBOL FOR LATIN CAPITAL LETTERS</td>\n",
" <td>🇴</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER O</td>\n",
" <td>🎁</td>\n",
" <td>WRAPPED PRESENT</td>\n",
" <td>📫</td>\n",
" <td>CLOSED MAILBOX WITH RAISED FLAG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" <td>📲</td>\n",
" <td>MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT</td>\n",
" <td>💨</td>\n",
" <td>DASH SYMBOL</td>\n",
" <td>📰</td>\n",
" <td>NEWSPAPER</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>🌆</td>\n",
" <td>CITYSCAPE AT DUSK</td>\n",
" <td>💤</td>\n",
" <td>SLEEPING SYMBOL</td>\n",
" <td>📬</td>\n",
" <td>OPEN MAILBOX WITH RAISED FLAG</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" summed summed_description max_val \\\n",
"0 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🤟 \n",
"1 🚮 PUT LITTER IN ITS PLACE SYMBOL 💌 \n",
"2 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🇮 \n",
"3 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🏩 \n",
"4 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 😱 \n",
"5 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🇭 \n",
"6 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇳 \n",
"7 🔠 INPUT SYMBOL FOR LATIN CAPITAL LETTERS 🇴 \n",
"8 🤟 I LOVE YOU HAND SIGN 📲 \n",
"9 🇮 REGIONAL INDICATOR SYMBOL LETTER I 🌆 \n",
"\n",
" max_val_description avg avg_description threshold \\\n",
"0 I LOVE YOU HAND SIGN 💢 ANGER SYMBOL 💌 \n",
"1 LOVE LETTER 📦 PACKAGE 🏩 \n",
"2 REGIONAL INDICATOR SYMBOL LETTER I 💌 LOVE LETTER 🤟 \n",
"3 LOVE HOTEL 🆑 SQUARED CL 🇮 \n",
"4 FACE SCREAMING IN FEAR 🌉 BRIDGE AT NIGHT 📨 \n",
"5 REGIONAL INDICATOR SYMBOL LETTER H 💐 BOUQUET 📩 \n",
"6 REGIONAL INDICATOR SYMBOL LETTER N 📜 SCROLL 📪 \n",
"7 REGIONAL INDICATOR SYMBOL LETTER O 🎁 WRAPPED PRESENT 📫 \n",
"8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT 💨 DASH SYMBOL 📰 \n",
"9 CITYSCAPE AT DUSK 💤 SLEEPING SYMBOL 📬 \n",
"\n",
" threshold_description \n",
"0 LOVE LETTER \n",
"1 LOVE HOTEL \n",
"2 I LOVE YOU HAND SIGN \n",
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
"4 INCOMING ENVELOPE \n",
"5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n",
"6 CLOSED MAILBOX WITH LOWERED FLAG \n",
"7 CLOSED MAILBOX WITH RAISED FLAG \n",
"8 NEWSPAPER \n",
"9 OPEN MAILBOX WITH RAISED FLAG "
]
},
"metadata": {},
"output_type": "display_data"
} }
], ],
"source": [ "source": [
"print_best_results(threshold)" "predict(\"I love sunny days!\", description_key='description' )"
] ]
}, },
{ {