updated naive approach

This commit is contained in:
Jonas Weinz 2018-06-05 15:03:33 +02:00
parent f5e72e8d21
commit b6ed452fc6

View File

@ -247,11 +247,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 21,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def evaluate_sentence(sentence, description_key = 'description'):\n", "def evaluate_sentence(sentence, description_key = 'description', lang = 'eng'):\n",
" \n",
" tokenized_sentence = word_tokenize(sentence)\n", " tokenized_sentence = word_tokenize(sentence)\n",
" n = len(tokenized_sentence)\n", " n = len(tokenized_sentence)\n",
" l = table.shape[0]\n", " l = table.shape[0]\n",
@ -264,12 +265,12 @@
" mat = np.zeros(shape=(m,n))\n", " mat = np.zeros(shape=(m,n))\n",
" for i in range(len(emoji_tokens)):\n", " for i in range(len(emoji_tokens)):\n",
" for j in range(len(tokenized_sentence)):\n", " for j in range(len(tokenized_sentence)):\n",
" syn1 = wordnet.synsets(emoji_tokens[i])\n", " syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)\n",
" if len(syn1) == 0:\n", " if len(syn1) == 0:\n",
" continue\n", " continue\n",
" w1 = syn1[0]\n", " w1 = syn1[0]\n",
" #print(j, tokenized_sentence)\n", " #print(j, tokenized_sentence)\n",
" syn2 = wordnet.synsets(tokenized_sentence[j])\n", " syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)\n",
" if len(syn2) == 0:\n", " if len(syn2) == 0:\n",
" continue\n", " continue\n",
" w2 = syn2[0]\n", " w2 = syn2[0]\n",
@ -294,7 +295,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 22,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -316,13 +317,13 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 23,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def predict(sentence, description_key='description', n=10, t=0.9):\n", "def predict(sentence, description_key='description', lang = 'eng', n=10, t=0.9):\n",
"\n", "\n",
" result = evaluate_sentence(sentence, description_key)\n", " result = evaluate_sentence(sentence, description_key, lang)\n",
" \n", " \n",
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n", " summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n", " max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
@ -343,7 +344,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 29,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -380,113 +381,113 @@
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <td>🤪</td>\n", " <td>🤭</td>\n",
" <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n", " <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
" <td>🤟</td>\n", " <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n", " <td>I LOVE YOU HAND SIGN</td>\n",
" <td>💢</td>\n", " <td>💐</td>\n",
" <td>ANGER SYMBOL</td>\n", " <td>BOUQUET</td>\n",
" <td>💌</td>\n", " <td>🚆</td>\n",
" <td>LOVE LETTER</td>\n", " <td>TRAIN</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>1</th>\n", " <th>1</th>\n",
" <td>🚮</td>\n", " <td>🔛</td>\n",
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n", " <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
" <td>💌</td>\n", " <td>🇮</td>\n",
" <td>LOVE LETTER</td>\n", " <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>📦</td>\n", " <td>🍳</td>\n",
" <td>PACKAGE</td>\n", " <td>COOKING</td>\n",
" <td>🏩</td>\n", " <td>🚸</td>\n",
" <td>LOVE HOTEL</td>\n", " <td>CHILDREN CROSSING</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>2</th>\n", " <th>2</th>\n",
" <td>🖖</td>\n", " <td>🚮</td>\n",
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n", " <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
" <td>🇮</td>\n", " <td>🚆</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n", " <td>TRAIN</td>\n",
" <td>💌</td>\n", " <td>🚆</td>\n",
" <td>LOVE LETTER</td>\n", " <td>TRAIN</td>\n",
" <td>🤟</td>\n", " <td>🚄</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n", " <td>HIGH-SPEED TRAIN</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>3</th>\n", " <th>3</th>\n",
" <td>🥰</td>\n", " <td>🤪</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n", " <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
" <td>🏩</td>\n", " <td>🚅</td>\n",
" <td>LOVE HOTEL</td>\n", " <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
" <td>🆑</td>\n", " <td>🎥</td>\n",
" <td>SQUARED CL</td>\n", " <td>MOVIE CAMERA</td>\n",
" <td>🇮</td>\n", " <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n", " <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>4</th>\n", " <th>4</th>\n",
" <td>🤭</td>\n", " <td>🥰</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n", " <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
" <td>😱</td>\n", " <td>🚄</td>\n",
" <td>FACE SCREAMING IN FEAR</td>\n", " <td>HIGH-SPEED TRAIN</td>\n",
" <td>🌉</td>\n", " <td>🎭</td>\n",
" <td>BRIDGE AT NIGHT</td>\n", " <td>PERFORMING ARTS</td>\n",
" <td>📨</td>\n", " <td>🤟</td>\n",
" <td>INCOMING ENVELOPE</td>\n", " <td>I LOVE YOU HAND SIGN</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>5</th>\n", " <th>5</th>\n",
" <td>🔂</td>\n", " <td>🙌</td>\n",
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n", " <td>PERSON RAISING BOTH HANDS IN CELEBRATION</td>\n",
" <td>🇭</td>\n", " <td>🚸</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER H</td>\n", " <td>CHILDREN CROSSING</td>\n",
" <td>💐</td>\n", " <td>🎹</td>\n",
" <td>BOUQUET</td>\n", " <td>MUSICAL KEYBOARD</td>\n",
" <td>📩</td>\n", " <td>🚅</td>\n",
" <td>ENVELOPE WITH DOWNWARDS ARROW ABOVE</td>\n", " <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>6</th>\n", " <th>6</th>\n",
" <td>🔛</td>\n", " <td>🖖</td>\n",
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n", " <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
" <td>🇳</td>\n", " <td>🛸</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER N</td>\n", " <td>FLYING SAUCER</td>\n",
" <td>📜</td>\n", " <td>🧾</td>\n",
" <td>SCROLL</td>\n", " <td>RECEIPT</td>\n",
" <td>📪</td>\n", " <td>🔹</td>\n",
" <td>CLOSED MAILBOX WITH LOWERED FLAG</td>\n", " <td>SMALL BLUE DIAMOND</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>7</th>\n", " <th>7</th>\n",
" <td>🔠</td>\n", " <td>🔂</td>\n",
" <td>INPUT SYMBOL FOR LATIN CAPITAL LETTERS</td>\n", " <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
" <td>🇴</td>\n", " <td>🥾</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER O</td>\n", " <td>HIKING BOOT</td>\n",
" <td>🎁</td>\n", " <td>💏</td>\n",
" <td>WRAPPED PRESENT</td>\n", " <td>KISS</td>\n",
" <td>📫</td>\n", " <td>📠</td>\n",
" <td>CLOSED MAILBOX WITH RAISED FLAG</td>\n", " <td>FAX MACHINE</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>8</th>\n", " <th>8</th>\n",
" <td>🤟</td>\n", " <td>😄</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n", " <td>SMILING FACE WITH OPEN MOUTH AND SMILING EYES</td>\n",
" <td>📲</td>\n", " <td>🥏</td>\n",
" <td>MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT</td>\n", " <td>FLYING DISC</td>\n",
" <td>💨</td>\n", " <td>🔥</td>\n",
" <td>DASH SYMBOL</td>\n", " <td>FIRE</td>\n",
" <td>📰</td>\n", " <td>📡</td>\n",
" <td>NEWSPAPER</td>\n", " <td>SATELLITE ANTENNA</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>9</th>\n", " <th>9</th>\n",
" <td>🇮</td>\n", " <td>👉</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n", " <td>WHITE RIGHT POINTING BACKHAND INDEX</td>\n",
" <td>🌆</td>\n", " <td>🚏</td>\n",
" <td>CITYSCAPE AT DUSK</td>\n", " <td>BUS STOP</td>\n",
" <td>💤</td>\n", " <td>🎼</td>\n",
" <td>SLEEPING SYMBOL</td>\n", " <td>MUSICAL SCORE</td>\n",
" <td>📬</td>\n", " <td>📢</td>\n",
" <td>OPEN MAILBOX WITH RAISED FLAG</td>\n", " <td>PUBLIC ADDRESS LOUDSPEAKER</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
@ -494,40 +495,40 @@
], ],
"text/plain": [ "text/plain": [
" summed summed_description max_val \\\n", " summed summed_description max_val \\\n",
"0 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🤟 \n", "0 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 🤟 \n",
"1 🚮 PUT LITTER IN ITS PLACE SYMBOL 💌 \n", "1 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇮 \n",
"2 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🇮 \n", "2 🚮 PUT LITTER IN ITS PLACE SYMBOL 🚆 \n",
"3 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🏩 \n", "3 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🚅 \n",
"4 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 😱 \n", "4 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🚄 \n",
"5 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🇭 \n", "5 🙌 PERSON RAISING BOTH HANDS IN CELEBRATION 🚸 \n",
"6 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇳 \n", "6 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🛸 \n",
"7 🔠 INPUT SYMBOL FOR LATIN CAPITAL LETTERS 🇴 \n", "7 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🥾 \n",
"8 🤟 I LOVE YOU HAND SIGN 📲 \n", "8 😄 SMILING FACE WITH OPEN MOUTH AND SMILING EYES 🥏 \n",
"9 🇮 REGIONAL INDICATOR SYMBOL LETTER I 🌆 \n", "9 👉 WHITE RIGHT POINTING BACKHAND INDEX 🚏 \n",
"\n", "\n",
" max_val_description avg avg_description threshold \\\n", " max_val_description avg avg_description threshold \\\n",
"0 I LOVE YOU HAND SIGN 💢 ANGER SYMBOL 💌 \n", "0 I LOVE YOU HAND SIGN 💐 BOUQUET 🚆 \n",
"1 LOVE LETTER 📦 PACKAGE 🏩 \n", "1 REGIONAL INDICATOR SYMBOL LETTER I 🍳 COOKING 🚸 \n",
"2 REGIONAL INDICATOR SYMBOL LETTER I 💌 LOVE LETTER 🤟 \n", "2 TRAIN 🚆 TRAIN 🚄 \n",
"3 LOVE HOTEL 🆑 SQUARED CL 🇮 \n", "3 HIGH-SPEED TRAIN WITH BULLET NOSE 🎥 MOVIE CAMERA 🇮 \n",
"4 FACE SCREAMING IN FEAR 🌉 BRIDGE AT NIGHT 📨 \n", "4 HIGH-SPEED TRAIN 🎭 PERFORMING ARTS 🤟 \n",
"5 REGIONAL INDICATOR SYMBOL LETTER H 💐 BOUQUET 📩 \n", "5 CHILDREN CROSSING 🎹 MUSICAL KEYBOARD 🚅 \n",
"6 REGIONAL INDICATOR SYMBOL LETTER N 📜 SCROLL 📪 \n", "6 FLYING SAUCER 🧾 RECEIPT 🔹 \n",
"7 REGIONAL INDICATOR SYMBOL LETTER O 🎁 WRAPPED PRESENT 📫 \n", "7 HIKING BOOT 💏 KISS 📠 \n",
"8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT 💨 DASH SYMBOL 📰 \n", "8 FLYING DISC 🔥 FIRE 📡 \n",
"9 CITYSCAPE AT DUSK 💤 SLEEPING SYMBOL 📬 \n", "9 BUS STOP 🎼 MUSICAL SCORE 📢 \n",
"\n", "\n",
" threshold_description \n", " threshold_description \n",
"0 LOVE LETTER \n", "0 TRAIN \n",
"1 LOVE HOTEL \n", "1 CHILDREN CROSSING \n",
"2 I LOVE YOU HAND SIGN \n", "2 HIGH-SPEED TRAIN \n",
"3 REGIONAL INDICATOR SYMBOL LETTER I \n", "3 REGIONAL INDICATOR SYMBOL LETTER I \n",
"4 INCOMING ENVELOPE \n", "4 I LOVE YOU HAND SIGN \n",
"5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n", "5 HIGH-SPEED TRAIN WITH BULLET NOSE \n",
"6 CLOSED MAILBOX WITH LOWERED FLAG \n", "6 SMALL BLUE DIAMOND \n",
"7 CLOSED MAILBOX WITH RAISED FLAG \n", "7 FAX MACHINE \n",
"8 NEWSPAPER \n", "8 SATELLITE ANTENNA \n",
"9 OPEN MAILBOX WITH RAISED FLAG " "9 PUBLIC ADDRESS LOUDSPEAKER "
] ]
}, },
"metadata": {}, "metadata": {},
@ -535,7 +536,7 @@
} }
], ],
"source": [ "source": [
"predict(\"I love sunny days!\", description_key='description' )" "predict(\"I like to travel by train\", description_key='description' , lang='eng')"
] ]
}, },
{ {