updated naive approach

This commit is contained in:
Jonas Weinz 2018-06-05 15:03:33 +02:00
parent f5e72e8d21
commit b6ed452fc6

View File

@ -247,11 +247,12 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"def evaluate_sentence(sentence, description_key = 'description'):\n",
"def evaluate_sentence(sentence, description_key = 'description', lang = 'eng'):\n",
" \n",
" tokenized_sentence = word_tokenize(sentence)\n",
" n = len(tokenized_sentence)\n",
" l = table.shape[0]\n",
@ -264,12 +265,12 @@
" mat = np.zeros(shape=(m,n))\n",
" for i in range(len(emoji_tokens)):\n",
" for j in range(len(tokenized_sentence)):\n",
" syn1 = wordnet.synsets(emoji_tokens[i])\n",
" syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)\n",
" if len(syn1) == 0:\n",
" continue\n",
" w1 = syn1[0]\n",
" #print(j, tokenized_sentence)\n",
" syn2 = wordnet.synsets(tokenized_sentence[j])\n",
" syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)\n",
" if len(syn2) == 0:\n",
" continue\n",
" w2 = syn2[0]\n",
@ -294,7 +295,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
@ -316,13 +317,13 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"def predict(sentence, description_key='description', n=10, t=0.9):\n",
"def predict(sentence, description_key='description', lang = 'eng', n=10, t=0.9):\n",
"\n",
" result = evaluate_sentence(sentence, description_key)\n",
" result = evaluate_sentence(sentence, description_key, lang)\n",
" \n",
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
@ -343,7 +344,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 29,
"metadata": {},
"outputs": [
{
@ -380,113 +381,113 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>🤪</td>\n",
" <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
" <td>🤭</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" <td>💢</td>\n",
" <td>ANGER SYMBOL</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>💐</td>\n",
" <td>BOUQUET</td>\n",
" <td>🚆</td>\n",
" <td>TRAIN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>🚮</td>\n",
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>📦</td>\n",
" <td>PACKAGE</td>\n",
" <td>🏩</td>\n",
" <td>LOVE HOTEL</td>\n",
" <td>🔛</td>\n",
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>🍳</td>\n",
" <td>COOKING</td>\n",
" <td>🚸</td>\n",
" <td>CHILDREN CROSSING</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>🖖</td>\n",
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" <td>🚮</td>\n",
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
" <td>🚆</td>\n",
" <td>TRAIN</td>\n",
" <td>🚆</td>\n",
" <td>TRAIN</td>\n",
" <td>🚄</td>\n",
" <td>HIGH-SPEED TRAIN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>🥰</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
" <td>🏩</td>\n",
" <td>LOVE HOTEL</td>\n",
" <td>🆑</td>\n",
" <td>SQUARED CL</td>\n",
" <td>🤪</td>\n",
" <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
" <td>🚅</td>\n",
" <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
" <td>🎥</td>\n",
" <td>MOVIE CAMERA</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>🤭</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
" <td>😱</td>\n",
" <td>FACE SCREAMING IN FEAR</td>\n",
" <td>🌉</td>\n",
" <td>BRIDGE AT NIGHT</td>\n",
" <td>📨</td>\n",
" <td>INCOMING ENVELOPE</td>\n",
" <td>🥰</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
" <td>🚄</td>\n",
" <td>HIGH-SPEED TRAIN</td>\n",
" <td>🎭</td>\n",
" <td>PERFORMING ARTS</td>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>🔂</td>\n",
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
" <td>🇭</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER H</td>\n",
" <td>💐</td>\n",
" <td>BOUQUET</td>\n",
" <td>📩</td>\n",
" <td>ENVELOPE WITH DOWNWARDS ARROW ABOVE</td>\n",
" <td>🙌</td>\n",
" <td>PERSON RAISING BOTH HANDS IN CELEBRATION</td>\n",
" <td>🚸</td>\n",
" <td>CHILDREN CROSSING</td>\n",
" <td>🎹</td>\n",
" <td>MUSICAL KEYBOARD</td>\n",
" <td>🚅</td>\n",
" <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>🔛</td>\n",
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
" <td>🇳</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER N</td>\n",
" <td>📜</td>\n",
" <td>SCROLL</td>\n",
" <td>📪</td>\n",
" <td>CLOSED MAILBOX WITH LOWERED FLAG</td>\n",
" <td>🖖</td>\n",
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
" <td>🛸</td>\n",
" <td>FLYING SAUCER</td>\n",
" <td>🧾</td>\n",
" <td>RECEIPT</td>\n",
" <td>🔹</td>\n",
" <td>SMALL BLUE DIAMOND</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>🔠</td>\n",
" <td>INPUT SYMBOL FOR LATIN CAPITAL LETTERS</td>\n",
" <td>🇴</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER O</td>\n",
" <td>🎁</td>\n",
" <td>WRAPPED PRESENT</td>\n",
" <td>📫</td>\n",
" <td>CLOSED MAILBOX WITH RAISED FLAG</td>\n",
" <td>🔂</td>\n",
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
" <td>🥾</td>\n",
" <td>HIKING BOOT</td>\n",
" <td>💏</td>\n",
" <td>KISS</td>\n",
" <td>📠</td>\n",
" <td>FAX MACHINE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" <td>📲</td>\n",
" <td>MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT</td>\n",
" <td>💨</td>\n",
" <td>DASH SYMBOL</td>\n",
" <td>📰</td>\n",
" <td>NEWSPAPER</td>\n",
" <td>😄</td>\n",
" <td>SMILING FACE WITH OPEN MOUTH AND SMILING EYES</td>\n",
" <td>🥏</td>\n",
" <td>FLYING DISC</td>\n",
" <td>🔥</td>\n",
" <td>FIRE</td>\n",
" <td>📡</td>\n",
" <td>SATELLITE ANTENNA</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>🌆</td>\n",
" <td>CITYSCAPE AT DUSK</td>\n",
" <td>💤</td>\n",
" <td>SLEEPING SYMBOL</td>\n",
" <td>📬</td>\n",
" <td>OPEN MAILBOX WITH RAISED FLAG</td>\n",
" <td>👉</td>\n",
" <td>WHITE RIGHT POINTING BACKHAND INDEX</td>\n",
" <td>🚏</td>\n",
" <td>BUS STOP</td>\n",
" <td>🎼</td>\n",
" <td>MUSICAL SCORE</td>\n",
" <td>📢</td>\n",
" <td>PUBLIC ADDRESS LOUDSPEAKER</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -494,40 +495,40 @@
],
"text/plain": [
" summed summed_description max_val \\\n",
"0 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🤟 \n",
"1 🚮 PUT LITTER IN ITS PLACE SYMBOL 💌 \n",
"2 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🇮 \n",
"3 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🏩 \n",
"4 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 😱 \n",
"5 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🇭 \n",
"6 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇳 \n",
"7 🔠 INPUT SYMBOL FOR LATIN CAPITAL LETTERS 🇴 \n",
"8 🤟 I LOVE YOU HAND SIGN 📲 \n",
"9 🇮 REGIONAL INDICATOR SYMBOL LETTER I 🌆 \n",
"0 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 🤟 \n",
"1 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇮 \n",
"2 🚮 PUT LITTER IN ITS PLACE SYMBOL 🚆 \n",
"3 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🚅 \n",
"4 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🚄 \n",
"5 🙌 PERSON RAISING BOTH HANDS IN CELEBRATION 🚸 \n",
"6 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🛸 \n",
"7 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🥾 \n",
"8 😄 SMILING FACE WITH OPEN MOUTH AND SMILING EYES 🥏 \n",
"9 👉 WHITE RIGHT POINTING BACKHAND INDEX 🚏 \n",
"\n",
" max_val_description avg avg_description threshold \\\n",
"0 I LOVE YOU HAND SIGN 💢 ANGER SYMBOL 💌 \n",
"1 LOVE LETTER 📦 PACKAGE 🏩 \n",
"2 REGIONAL INDICATOR SYMBOL LETTER I 💌 LOVE LETTER 🤟 \n",
"3 LOVE HOTEL 🆑 SQUARED CL 🇮 \n",
"4 FACE SCREAMING IN FEAR 🌉 BRIDGE AT NIGHT 📨 \n",
"5 REGIONAL INDICATOR SYMBOL LETTER H 💐 BOUQUET 📩 \n",
"6 REGIONAL INDICATOR SYMBOL LETTER N 📜 SCROLL 📪 \n",
"7 REGIONAL INDICATOR SYMBOL LETTER O 🎁 WRAPPED PRESENT 📫 \n",
"8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT 💨 DASH SYMBOL 📰 \n",
"9 CITYSCAPE AT DUSK 💤 SLEEPING SYMBOL 📬 \n",
" max_val_description avg avg_description threshold \\\n",
"0 I LOVE YOU HAND SIGN 💐 BOUQUET 🚆 \n",
"1 REGIONAL INDICATOR SYMBOL LETTER I 🍳 COOKING 🚸 \n",
"2 TRAIN 🚆 TRAIN 🚄 \n",
"3 HIGH-SPEED TRAIN WITH BULLET NOSE 🎥 MOVIE CAMERA 🇮 \n",
"4 HIGH-SPEED TRAIN 🎭 PERFORMING ARTS 🤟 \n",
"5 CHILDREN CROSSING 🎹 MUSICAL KEYBOARD 🚅 \n",
"6 FLYING SAUCER 🧾 RECEIPT 🔹 \n",
"7 HIKING BOOT 💏 KISS 📠 \n",
"8 FLYING DISC 🔥 FIRE 📡 \n",
"9 BUS STOP 🎼 MUSICAL SCORE 📢 \n",
"\n",
" threshold_description \n",
"0 LOVE LETTER \n",
"1 LOVE HOTEL \n",
"2 I LOVE YOU HAND SIGN \n",
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
"4 INCOMING ENVELOPE \n",
"5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n",
"6 CLOSED MAILBOX WITH LOWERED FLAG \n",
"7 CLOSED MAILBOX WITH RAISED FLAG \n",
"8 NEWSPAPER \n",
"9 OPEN MAILBOX WITH RAISED FLAG "
" threshold_description \n",
"0 TRAIN \n",
"1 CHILDREN CROSSING \n",
"2 HIGH-SPEED TRAIN \n",
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
"4 I LOVE YOU HAND SIGN \n",
"5 HIGH-SPEED TRAIN WITH BULLET NOSE \n",
"6 SMALL BLUE DIAMOND \n",
"7 FAX MACHINE \n",
"8 SATELLITE ANTENNA \n",
"9 PUBLIC ADDRESS LOUDSPEAKER "
]
},
"metadata": {},
@ -535,7 +536,7 @@
}
],
"source": [
"predict(\"I love sunny days!\", description_key='description' )"
"predict(\"I like to travel by train\", description_key='description' , lang='eng')"
]
},
{