better naive approach
This commit is contained in:
Jonas Weinz
parent
9a922d88fa
commit
12f99af9a9
@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 149,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -65,7 +65,7 @@
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" <th>description_de</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
@ -75,7 +75,7 @@
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>MAHJONG TILE RED DRAGON</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>MAHJONG FLIESE ROT DRACHE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
@ -83,7 +83,7 @@
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>SPOOL OF THREAD</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>Spool Gewinde</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
@ -91,7 +91,7 @@
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
@ -99,7 +99,7 @@
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>PLAYING CARD BLACK JOKER</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>SPIELKARTE BLACK JOKER</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
@ -107,19 +107,26 @@
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>ZIPPER-MOUTH FACE</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>ZIPPER-MUND Gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON NaN\n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD NaN\n",
|
||||
"2 2 129526 🧶 BALL OF YARN NaN\n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER NaN\n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE NaN"
|
||||
" Unnamed: 0 code character description \\\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD \n",
|
||||
"2 2 129526 🧶 BALL OF YARN \n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
|
||||
"\n",
|
||||
" description_de \n",
|
||||
"0 MAHJONG FLIESE ROT DRACHE \n",
|
||||
"1 Spool Gewinde \n",
|
||||
"2 BALL OF YARN \n",
|
||||
"3 SPIELKARTE BLACK JOKER \n",
|
||||
"4 ZIPPER-MUND Gesicht "
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
@ -148,6 +155,13 @@
|
||||
"messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* using a Stemmer to get the main 'Part' of each word"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
@ -233,18 +247,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 59,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def evaluate_sentence(sentence):\n",
|
||||
"def evaluate_sentence(sentence, description_key = 'description'):\n",
|
||||
" tokenized_sentence = word_tokenize(sentence)\n",
|
||||
" n = len(tokenized_sentence)\n",
|
||||
" l = table.shape[0]\n",
|
||||
" matrix_list = []\n",
|
||||
" \n",
|
||||
" for index, row in table.iterrows():\n",
|
||||
" emoji_tokens = word_tokenize(row['description'])\n",
|
||||
" emoji_tokens = word_tokenize(row[description_key])\n",
|
||||
" m = len(emoji_tokens)\n",
|
||||
"\n",
|
||||
" mat = np.zeros(shape=(m,n))\n",
|
||||
@ -271,15 +285,6 @@
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 177,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result = evaluate_sentence(\"horse back riding\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@ -289,7 +294,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 178,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -311,76 +316,226 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 179,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summed = np.argsort([-np.sum(x) for x in result])\n",
|
||||
"max_val = np.argsort([-np.max(x) for x in result])\n",
|
||||
"avg = np.argsort([-np.mean(x) for x in result])\n",
|
||||
"def predict(sentence, description_key='description', n=10, t=0.9):\n",
|
||||
"\n",
|
||||
"t = 0.9\n",
|
||||
"threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n"
|
||||
" result = evaluate_sentence(sentence, description_key)\n",
|
||||
" \n",
|
||||
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
|
||||
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
|
||||
" avg = np.argsort([-np.mean(x) for x in result])[0:n]\n",
|
||||
" threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n",
|
||||
" \n",
|
||||
" # build a result table\n",
|
||||
" table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n",
|
||||
" lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n",
|
||||
" lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n",
|
||||
" lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n",
|
||||
" \n",
|
||||
" display(table_frame)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 180,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def print_best_results(sorted_indices, n=10):\n",
|
||||
" pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n",
|
||||
" pp.pprint([result[x] for x in sorted_indices[:10]])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 181,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[ '🐎 -- HORSE',\n",
|
||||
" '🏇 -- HORSE RACING',\n",
|
||||
" '🐴 -- HORSE FACE',\n",
|
||||
" '\\U0001f993 -- ZEBRA FACE',\n",
|
||||
" '🏅 -- SPORTS MEDAL',\n",
|
||||
" '🗻 -- MOUNT FUJI',\n",
|
||||
" '🎠 -- CAROUSEL HORSE',\n",
|
||||
" '🤚 -- RAISED BACK OF HAND',\n",
|
||||
" '🔙 -- BACK WITH LEFTWARDS ARROW ABOVE',\n",
|
||||
" '📮 -- POSTBOX']\n",
|
||||
"[ array([[1. , 0.19047619, 0.08333333]]),\n",
|
||||
" array([[1. , 0.19047619, 0.08333333],\n",
|
||||
" [0.08333333, 0.13333333, 0.88888889]]),\n",
|
||||
" array([[1. , 0.19047619, 0.08333333],\n",
|
||||
" [0.18181818, 0.76923077, 0.125 ]]),\n",
|
||||
" array([[0.93333333, 0.19047619, 0.08333333],\n",
|
||||
" [0.18181818, 0.76923077, 0.125 ]]),\n",
|
||||
" array([[0.08695652, 0.14285714, 0.94117647],\n",
|
||||
" [0.09090909, 0.15384615, 0.25 ]]),\n",
|
||||
" array([[0.96774194, 0.18181818, 0.08 ],\n",
|
||||
" [0.4 , 0.19047619, 0.08333333]]),\n",
|
||||
" array([[0.07407407, 0.11111111, 0.19047619],\n",
|
||||
" [1. , 0.19047619, 0.08333333]]),\n",
|
||||
" array([[0.1 , 0.18181818, 0.14285714],\n",
|
||||
" [0.19047619, 1. , 0.13333333],\n",
|
||||
" [0. , 0. , 0. ],\n",
|
||||
" [0.17391304, 0.71428571, 0.11764706]]),\n",
|
||||
" array([[0.19047619, 1. , 0.13333333],\n",
|
||||
" [0. , 0. , 0. ],\n",
|
||||
" [0. , 0. , 0. ],\n",
|
||||
" [0.08695652, 0.14285714, 0.23529412],\n",
|
||||
" [0.09090909, 0.15384615, 0.25 ]]),\n",
|
||||
" array([[0.32 , 0.25 , 0.10526316]])]\n"
|
||||
]
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>summed</th>\n",
|
||||
" <th>summed_description</th>\n",
|
||||
" <th>max_val</th>\n",
|
||||
" <th>max_val_description</th>\n",
|
||||
" <th>avg</th>\n",
|
||||
" <th>avg_description</th>\n",
|
||||
" <th>threshold</th>\n",
|
||||
" <th>threshold_description</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>🤪</td>\n",
|
||||
" <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>I LOVE YOU HAND SIGN</td>\n",
|
||||
" <td>💢</td>\n",
|
||||
" <td>ANGER SYMBOL</td>\n",
|
||||
" <td>💌</td>\n",
|
||||
" <td>LOVE LETTER</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>🚮</td>\n",
|
||||
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
|
||||
" <td>💌</td>\n",
|
||||
" <td>LOVE LETTER</td>\n",
|
||||
" <td>📦</td>\n",
|
||||
" <td>PACKAGE</td>\n",
|
||||
" <td>🏩</td>\n",
|
||||
" <td>LOVE HOTEL</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>🖖</td>\n",
|
||||
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
|
||||
" <td>🇮</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
|
||||
" <td>💌</td>\n",
|
||||
" <td>LOVE LETTER</td>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>I LOVE YOU HAND SIGN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>🥰</td>\n",
|
||||
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
|
||||
" <td>🏩</td>\n",
|
||||
" <td>LOVE HOTEL</td>\n",
|
||||
" <td>🆑</td>\n",
|
||||
" <td>SQUARED CL</td>\n",
|
||||
" <td>🇮</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>🤭</td>\n",
|
||||
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
|
||||
" <td>😱</td>\n",
|
||||
" <td>FACE SCREAMING IN FEAR</td>\n",
|
||||
" <td>🌉</td>\n",
|
||||
" <td>BRIDGE AT NIGHT</td>\n",
|
||||
" <td>📨</td>\n",
|
||||
" <td>INCOMING ENVELOPE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>🔂</td>\n",
|
||||
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
|
||||
" <td>🇭</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER H</td>\n",
|
||||
" <td>💐</td>\n",
|
||||
" <td>BOUQUET</td>\n",
|
||||
" <td>📩</td>\n",
|
||||
" <td>ENVELOPE WITH DOWNWARDS ARROW ABOVE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>🔛</td>\n",
|
||||
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
|
||||
" <td>🇳</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER N</td>\n",
|
||||
" <td>📜</td>\n",
|
||||
" <td>SCROLL</td>\n",
|
||||
" <td>📪</td>\n",
|
||||
" <td>CLOSED MAILBOX WITH LOWERED FLAG</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>🔠</td>\n",
|
||||
" <td>INPUT SYMBOL FOR LATIN CAPITAL LETTERS</td>\n",
|
||||
" <td>🇴</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER O</td>\n",
|
||||
" <td>🎁</td>\n",
|
||||
" <td>WRAPPED PRESENT</td>\n",
|
||||
" <td>📫</td>\n",
|
||||
" <td>CLOSED MAILBOX WITH RAISED FLAG</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>I LOVE YOU HAND SIGN</td>\n",
|
||||
" <td>📲</td>\n",
|
||||
" <td>MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT</td>\n",
|
||||
" <td>💨</td>\n",
|
||||
" <td>DASH SYMBOL</td>\n",
|
||||
" <td>📰</td>\n",
|
||||
" <td>NEWSPAPER</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>🇮</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
|
||||
" <td>🌆</td>\n",
|
||||
" <td>CITYSCAPE AT DUSK</td>\n",
|
||||
" <td>💤</td>\n",
|
||||
" <td>SLEEPING SYMBOL</td>\n",
|
||||
" <td>📬</td>\n",
|
||||
" <td>OPEN MAILBOX WITH RAISED FLAG</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" summed summed_description max_val \\\n",
|
||||
"0 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🤟 \n",
|
||||
"1 🚮 PUT LITTER IN ITS PLACE SYMBOL 💌 \n",
|
||||
"2 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🇮 \n",
|
||||
"3 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🏩 \n",
|
||||
"4 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 😱 \n",
|
||||
"5 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🇭 \n",
|
||||
"6 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇳 \n",
|
||||
"7 🔠 INPUT SYMBOL FOR LATIN CAPITAL LETTERS 🇴 \n",
|
||||
"8 🤟 I LOVE YOU HAND SIGN 📲 \n",
|
||||
"9 🇮 REGIONAL INDICATOR SYMBOL LETTER I 🌆 \n",
|
||||
"\n",
|
||||
" max_val_description avg avg_description threshold \\\n",
|
||||
"0 I LOVE YOU HAND SIGN 💢 ANGER SYMBOL 💌 \n",
|
||||
"1 LOVE LETTER 📦 PACKAGE 🏩 \n",
|
||||
"2 REGIONAL INDICATOR SYMBOL LETTER I 💌 LOVE LETTER 🤟 \n",
|
||||
"3 LOVE HOTEL 🆑 SQUARED CL 🇮 \n",
|
||||
"4 FACE SCREAMING IN FEAR 🌉 BRIDGE AT NIGHT 📨 \n",
|
||||
"5 REGIONAL INDICATOR SYMBOL LETTER H 💐 BOUQUET 📩 \n",
|
||||
"6 REGIONAL INDICATOR SYMBOL LETTER N 📜 SCROLL 📪 \n",
|
||||
"7 REGIONAL INDICATOR SYMBOL LETTER O 🎁 WRAPPED PRESENT 📫 \n",
|
||||
"8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT 💨 DASH SYMBOL 📰 \n",
|
||||
"9 CITYSCAPE AT DUSK 💤 SLEEPING SYMBOL 📬 \n",
|
||||
"\n",
|
||||
" threshold_description \n",
|
||||
"0 LOVE LETTER \n",
|
||||
"1 LOVE HOTEL \n",
|
||||
"2 I LOVE YOU HAND SIGN \n",
|
||||
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
|
||||
"4 INCOMING ENVELOPE \n",
|
||||
"5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n",
|
||||
"6 CLOSED MAILBOX WITH LOWERED FLAG \n",
|
||||
"7 CLOSED MAILBOX WITH RAISED FLAG \n",
|
||||
"8 NEWSPAPER \n",
|
||||
"9 OPEN MAILBOX WITH RAISED FLAG "
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print_best_results(threshold)"
|
||||
"predict(\"I love sunny days!\", description_key='description' )"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user