Merge branch 'master' of ssh://the-cake-is-a-lie.net:20022/jonas/NLP-LAB
This commit is contained in:
Carsten
commit
f59f0bc131
4
.gitignore
vendored
4
.gitignore
vendored
@ -60,3 +60,7 @@ target/
|
||||
|
||||
# jupyter:
|
||||
.ipynb_checkpoints/
|
||||
|
||||
# hardcoded stuff
|
||||
data_en
|
||||
data_de
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -2,38 +2,28 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 36,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[nltk_data] Downloading package stopwords to\n",
|
||||
"[nltk_data] C:\\Users\\Maren\\AppData\\Roaming\\nltk_data...\n",
|
||||
"[nltk_data] Unzipping corpora\\stopwords.zip.\n"
|
||||
"[nltk_data] Downloading package stopwords to /home/jonas/nltk_data...\n",
|
||||
"[nltk_data] Package stopwords is already up-to-date!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"True"
|
||||
]
|
||||
},
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import nltk\n",
|
||||
"nltk.download('stopwords')"
|
||||
"nltk.download('stopwords')\n",
|
||||
"from nltk.corpus import stopwords"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -42,7 +32,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -70,7 +60,7 @@
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" <th>description_de</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
@ -80,7 +70,7 @@
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>MAHJONG TILE RED DRAGON</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>MAHJONG FLIESE ROT DRACHE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
@ -88,7 +78,7 @@
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>SPOOL OF THREAD</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>Spool Gewinde</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
@ -96,7 +86,7 @@
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
@ -104,7 +94,7 @@
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>PLAYING CARD BLACK JOKER</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>SPIELKARTE BLACK JOKER</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
@ -112,22 +102,29 @@
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>ZIPPER-MOUTH FACE</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>ZIPPER-MUND Gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON NaN\n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD NaN\n",
|
||||
"2 2 129526 🧶 BALL OF YARN NaN\n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER NaN\n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE NaN"
|
||||
" Unnamed: 0 code character description \\\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD \n",
|
||||
"2 2 129526 🧶 BALL OF YARN \n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
|
||||
"\n",
|
||||
" description_de \n",
|
||||
"0 MAHJONG FLIESE ROT DRACHE \n",
|
||||
"1 Spool Gewinde \n",
|
||||
"2 BALL OF YARN \n",
|
||||
"3 SPIELKARTE BLACK JOKER \n",
|
||||
"4 ZIPPER-MUND Gesicht "
|
||||
]
|
||||
},
|
||||
"execution_count": 15,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -138,23 +135,31 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 46,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||||
" \"\"\"\n",
|
||||
"C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
|
||||
" \n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||||
" # Remove the CWD from sys.path while we load stuff.\n"
|
||||
" del sys.path[0]\n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:22: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -182,7 +187,7 @@
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" <th>description_de</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
@ -192,7 +197,7 @@
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>mahjong tile red dragon</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>mahjong fliese rot drache</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
@ -200,7 +205,7 @@
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>spool thread</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>spool gewinde</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
@ -208,7 +213,7 @@
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>ball yarn</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>ball of yarn</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
@ -216,7 +221,7 @@
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>playing card black joker</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>spielkarte black joker</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
@ -224,22 +229,592 @@
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>zipper-mouth face</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>zipper-mund gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>129297</td>\n",
|
||||
" <td>🤑</td>\n",
|
||||
" <td>money-mouth face</td>\n",
|
||||
" <td>geld-mund-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>129298</td>\n",
|
||||
" <td>🤒</td>\n",
|
||||
" <td>face thermometer</td>\n",
|
||||
" <td>gesicht thermometer</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>7</td>\n",
|
||||
" <td>129299</td>\n",
|
||||
" <td>🤓</td>\n",
|
||||
" <td>nerd face</td>\n",
|
||||
" <td>nerd gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>8</td>\n",
|
||||
" <td>129300</td>\n",
|
||||
" <td>🤔</td>\n",
|
||||
" <td>thinking face</td>\n",
|
||||
" <td>denken gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>9</td>\n",
|
||||
" <td>129301</td>\n",
|
||||
" <td>🤕</td>\n",
|
||||
" <td>face head-bandage</td>\n",
|
||||
" <td>gesicht head-bandage</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10</th>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>129302</td>\n",
|
||||
" <td>🤖</td>\n",
|
||||
" <td>robot face</td>\n",
|
||||
" <td>roboter-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>11</th>\n",
|
||||
" <td>11</td>\n",
|
||||
" <td>129303</td>\n",
|
||||
" <td>🤗</td>\n",
|
||||
" <td>hugging face</td>\n",
|
||||
" <td>umarmen gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>12</th>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>129304</td>\n",
|
||||
" <td>🤘</td>\n",
|
||||
" <td>sign horns</td>\n",
|
||||
" <td>zeichen hörner</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>13</th>\n",
|
||||
" <td>13</td>\n",
|
||||
" <td>129305</td>\n",
|
||||
" <td>🤙</td>\n",
|
||||
" <td>call hand</td>\n",
|
||||
" <td>ruf</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>14</th>\n",
|
||||
" <td>14</td>\n",
|
||||
" <td>129306</td>\n",
|
||||
" <td>🤚</td>\n",
|
||||
" <td>raised back hand</td>\n",
|
||||
" <td>rückseite hand</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>15</th>\n",
|
||||
" <td>15</td>\n",
|
||||
" <td>129307</td>\n",
|
||||
" <td>🤛</td>\n",
|
||||
" <td>left-facing fist</td>\n",
|
||||
" <td>links-facing fist</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>16</th>\n",
|
||||
" <td>16</td>\n",
|
||||
" <td>129308</td>\n",
|
||||
" <td>🤜</td>\n",
|
||||
" <td>right-facing fist</td>\n",
|
||||
" <td>rechts-facing fist</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>17</th>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>129309</td>\n",
|
||||
" <td>🤝</td>\n",
|
||||
" <td>handshake</td>\n",
|
||||
" <td>handshake</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>18</th>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>129310</td>\n",
|
||||
" <td>🤞</td>\n",
|
||||
" <td>hand index middle fingers crossed</td>\n",
|
||||
" <td>hand index mittleren fingern gekreuzt</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>19</th>\n",
|
||||
" <td>19</td>\n",
|
||||
" <td>129311</td>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>love hand sign</td>\n",
|
||||
" <td>liebe handzeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>20</th>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>129312</td>\n",
|
||||
" <td>🤠</td>\n",
|
||||
" <td>face cowboy hat</td>\n",
|
||||
" <td>gesicht cowboy-hut</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>21</th>\n",
|
||||
" <td>21</td>\n",
|
||||
" <td>129313</td>\n",
|
||||
" <td>🤡</td>\n",
|
||||
" <td>clown face</td>\n",
|
||||
" <td>clown-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>22</th>\n",
|
||||
" <td>22</td>\n",
|
||||
" <td>129314</td>\n",
|
||||
" <td>🤢</td>\n",
|
||||
" <td>nauseated face</td>\n",
|
||||
" <td>ungeätztes gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>23</th>\n",
|
||||
" <td>23</td>\n",
|
||||
" <td>129315</td>\n",
|
||||
" <td>🤣</td>\n",
|
||||
" <td>rolling floor laughing</td>\n",
|
||||
" <td>rollen boden lachen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>24</th>\n",
|
||||
" <td>24</td>\n",
|
||||
" <td>129316</td>\n",
|
||||
" <td>🤤</td>\n",
|
||||
" <td>drooling face</td>\n",
|
||||
" <td>gesicht abtastend</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>25</th>\n",
|
||||
" <td>25</td>\n",
|
||||
" <td>129317</td>\n",
|
||||
" <td>🤥</td>\n",
|
||||
" <td>lying face</td>\n",
|
||||
" <td>liegendes gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>26</th>\n",
|
||||
" <td>26</td>\n",
|
||||
" <td>129318</td>\n",
|
||||
" <td>🤦</td>\n",
|
||||
" <td>face palm</td>\n",
|
||||
" <td>gesicht palme</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>27</th>\n",
|
||||
" <td>27</td>\n",
|
||||
" <td>129319</td>\n",
|
||||
" <td>🤧</td>\n",
|
||||
" <td>sneezing face</td>\n",
|
||||
" <td>snezing-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>28</th>\n",
|
||||
" <td>28</td>\n",
|
||||
" <td>129320</td>\n",
|
||||
" <td>🤨</td>\n",
|
||||
" <td>face one eyebrow raised</td>\n",
|
||||
" <td>gesicht augenbrauen erhöht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>29</th>\n",
|
||||
" <td>29</td>\n",
|
||||
" <td>129321</td>\n",
|
||||
" <td>🤩</td>\n",
|
||||
" <td>grinning face star eyes</td>\n",
|
||||
" <td>gesicht sternenaugen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>...</th>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>997</th>\n",
|
||||
" <td>997</td>\n",
|
||||
" <td>9995</td>\n",
|
||||
" <td>✋</td>\n",
|
||||
" <td>raised hand</td>\n",
|
||||
" <td>raised hand</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>998</th>\n",
|
||||
" <td>998</td>\n",
|
||||
" <td>128581</td>\n",
|
||||
" <td>🙅</td>\n",
|
||||
" <td>face good gesture</td>\n",
|
||||
" <td>gesicht gute geste</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999</th>\n",
|
||||
" <td>999</td>\n",
|
||||
" <td>128582</td>\n",
|
||||
" <td>🙆</td>\n",
|
||||
" <td>face ok gesture</td>\n",
|
||||
" <td>gesicht ok gestern</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1000</th>\n",
|
||||
" <td>1000</td>\n",
|
||||
" <td>128583</td>\n",
|
||||
" <td>🙇</td>\n",
|
||||
" <td>person bowing deeply</td>\n",
|
||||
" <td>person, tief biegt</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1001</th>\n",
|
||||
" <td>1001</td>\n",
|
||||
" <td>128584</td>\n",
|
||||
" <td>🙈</td>\n",
|
||||
" <td>see-no-evil monkey</td>\n",
|
||||
" <td>see-no-evil-affe</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1002</th>\n",
|
||||
" <td>1002</td>\n",
|
||||
" <td>129511</td>\n",
|
||||
" <td>🧧</td>\n",
|
||||
" <td>red gift envelope</td>\n",
|
||||
" <td>roter geschenkumschlag</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1003</th>\n",
|
||||
" <td>1003</td>\n",
|
||||
" <td>128585</td>\n",
|
||||
" <td>🙉</td>\n",
|
||||
" <td>hear-no-evil monkey</td>\n",
|
||||
" <td>hear-no-evil affe</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1004</th>\n",
|
||||
" <td>1004</td>\n",
|
||||
" <td>127476</td>\n",
|
||||
" <td>🇴</td>\n",
|
||||
" <td>regional indicator symbol letter</td>\n",
|
||||
" <td>regionaler indikator symbol letter o</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1005</th>\n",
|
||||
" <td>1005</td>\n",
|
||||
" <td>128586</td>\n",
|
||||
" <td>🙊</td>\n",
|
||||
" <td>speak-no-evil monkey</td>\n",
|
||||
" <td>speak-no-evil-affe</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1006</th>\n",
|
||||
" <td>1006</td>\n",
|
||||
" <td>10024</td>\n",
|
||||
" <td>✨</td>\n",
|
||||
" <td>sparkles</td>\n",
|
||||
" <td>funkeln</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1007</th>\n",
|
||||
" <td>1007</td>\n",
|
||||
" <td>129512</td>\n",
|
||||
" <td>🧨</td>\n",
|
||||
" <td>firecracker</td>\n",
|
||||
" <td>firecracker</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1008</th>\n",
|
||||
" <td>1008</td>\n",
|
||||
" <td>127477</td>\n",
|
||||
" <td>🇵</td>\n",
|
||||
" <td>regional indicator symbol letter p</td>\n",
|
||||
" <td>regionaler indikator symbol letter p</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1009</th>\n",
|
||||
" <td>1009</td>\n",
|
||||
" <td>10060</td>\n",
|
||||
" <td>❌</td>\n",
|
||||
" <td>cross mark</td>\n",
|
||||
" <td>kreuzmarke</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1010</th>\n",
|
||||
" <td>1010</td>\n",
|
||||
" <td>10062</td>\n",
|
||||
" <td>❎</td>\n",
|
||||
" <td>negative squared cross mark</td>\n",
|
||||
" <td>negative quadrierte kreuzmarke</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1011</th>\n",
|
||||
" <td>1011</td>\n",
|
||||
" <td>129513</td>\n",
|
||||
" <td>🧩</td>\n",
|
||||
" <td>jigsaw puzzle piece</td>\n",
|
||||
" <td>jigsaw puzzle stück</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1012</th>\n",
|
||||
" <td>1012</td>\n",
|
||||
" <td>127478</td>\n",
|
||||
" <td>🇶</td>\n",
|
||||
" <td>regional indicator symbol letter q</td>\n",
|
||||
" <td>regionaler indikator symbol letter q</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1013</th>\n",
|
||||
" <td>1013</td>\n",
|
||||
" <td>10067</td>\n",
|
||||
" <td>❓</td>\n",
|
||||
" <td>black question mark ornament</td>\n",
|
||||
" <td>schwarzes fragenzeichen-verzierung</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1014</th>\n",
|
||||
" <td>1014</td>\n",
|
||||
" <td>10068</td>\n",
|
||||
" <td>❔</td>\n",
|
||||
" <td>white question mark ornament</td>\n",
|
||||
" <td>weisses frage-zeichen-verzierung</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1015</th>\n",
|
||||
" <td>1015</td>\n",
|
||||
" <td>10069</td>\n",
|
||||
" <td>❕</td>\n",
|
||||
" <td>white exclamation mark ornament</td>\n",
|
||||
" <td>weisses ausrufs-markierungs-verzierung</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1016</th>\n",
|
||||
" <td>1016</td>\n",
|
||||
" <td>10071</td>\n",
|
||||
" <td>❗</td>\n",
|
||||
" <td>heavy exclamation mark symbol</td>\n",
|
||||
" <td>schweres ausrufzeichen symbol</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1017</th>\n",
|
||||
" <td>1017</td>\n",
|
||||
" <td>129514</td>\n",
|
||||
" <td>🧪</td>\n",
|
||||
" <td>test tube</td>\n",
|
||||
" <td>testrohr</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1018</th>\n",
|
||||
" <td>1018</td>\n",
|
||||
" <td>129515</td>\n",
|
||||
" <td>🧫</td>\n",
|
||||
" <td>petri dish</td>\n",
|
||||
" <td>petrischale</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1019</th>\n",
|
||||
" <td>1019</td>\n",
|
||||
" <td>10133</td>\n",
|
||||
" <td>➕</td>\n",
|
||||
" <td>heavy plus sign</td>\n",
|
||||
" <td>schwere plus zeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1020</th>\n",
|
||||
" <td>1020</td>\n",
|
||||
" <td>10134</td>\n",
|
||||
" <td>➖</td>\n",
|
||||
" <td>heavy minus sign</td>\n",
|
||||
" <td>schwere minuszeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1021</th>\n",
|
||||
" <td>1021</td>\n",
|
||||
" <td>10135</td>\n",
|
||||
" <td>➗</td>\n",
|
||||
" <td>heavy division sign</td>\n",
|
||||
" <td>schweres abteilungszeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1022</th>\n",
|
||||
" <td>1022</td>\n",
|
||||
" <td>129424</td>\n",
|
||||
" <td>🦐</td>\n",
|
||||
" <td>shrimp</td>\n",
|
||||
" <td>garnele</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1023</th>\n",
|
||||
" <td>1023</td>\n",
|
||||
" <td>129425</td>\n",
|
||||
" <td>🦑</td>\n",
|
||||
" <td>squid</td>\n",
|
||||
" <td>tintenfisch</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1024</th>\n",
|
||||
" <td>1024</td>\n",
|
||||
" <td>10160</td>\n",
|
||||
" <td>➰</td>\n",
|
||||
" <td>curly loop</td>\n",
|
||||
" <td>lockige schleife</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1025</th>\n",
|
||||
" <td>1025</td>\n",
|
||||
" <td>10175</td>\n",
|
||||
" <td>➿</td>\n",
|
||||
" <td>double curly loop</td>\n",
|
||||
" <td>doppelt lockige schleife</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1026</th>\n",
|
||||
" <td>1026</td>\n",
|
||||
" <td>129520</td>\n",
|
||||
" <td>🧰</td>\n",
|
||||
" <td>TOOLBOX</td>\n",
|
||||
" <td>WERKZEUGKASTEN</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"<p>1027 rows × 5 columns</p>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 mahjong tile red dragon NaN\n",
|
||||
"1 1 129525 🧵 spool thread NaN\n",
|
||||
"2 2 129526 🧶 ball yarn NaN\n",
|
||||
"3 3 127183 🃏 playing card black joker NaN\n",
|
||||
"4 4 129296 🤐 zipper-mouth face NaN"
|
||||
" Unnamed: 0 code character description \\\n",
|
||||
"0 0 126980 🀄 mahjong tile red dragon \n",
|
||||
"1 1 129525 🧵 spool thread \n",
|
||||
"2 2 129526 🧶 ball yarn \n",
|
||||
"3 3 127183 🃏 playing card black joker \n",
|
||||
"4 4 129296 🤐 zipper-mouth face \n",
|
||||
"5 5 129297 🤑 money-mouth face \n",
|
||||
"6 6 129298 🤒 face thermometer \n",
|
||||
"7 7 129299 🤓 nerd face \n",
|
||||
"8 8 129300 🤔 thinking face \n",
|
||||
"9 9 129301 🤕 face head-bandage \n",
|
||||
"10 10 129302 🤖 robot face \n",
|
||||
"11 11 129303 🤗 hugging face \n",
|
||||
"12 12 129304 🤘 sign horns \n",
|
||||
"13 13 129305 🤙 call hand \n",
|
||||
"14 14 129306 🤚 raised back hand \n",
|
||||
"15 15 129307 🤛 left-facing fist \n",
|
||||
"16 16 129308 🤜 right-facing fist \n",
|
||||
"17 17 129309 🤝 handshake \n",
|
||||
"18 18 129310 🤞 hand index middle fingers crossed \n",
|
||||
"19 19 129311 🤟 love hand sign \n",
|
||||
"20 20 129312 🤠 face cowboy hat \n",
|
||||
"21 21 129313 🤡 clown face \n",
|
||||
"22 22 129314 🤢 nauseated face \n",
|
||||
"23 23 129315 🤣 rolling floor laughing \n",
|
||||
"24 24 129316 🤤 drooling face \n",
|
||||
"25 25 129317 🤥 lying face \n",
|
||||
"26 26 129318 🤦 face palm \n",
|
||||
"27 27 129319 🤧 sneezing face \n",
|
||||
"28 28 129320 🤨 face one eyebrow raised \n",
|
||||
"29 29 129321 🤩 grinning face star eyes \n",
|
||||
"... ... ... ... ... \n",
|
||||
"997 997 9995 ✋ raised hand \n",
|
||||
"998 998 128581 🙅 face good gesture \n",
|
||||
"999 999 128582 🙆 face ok gesture \n",
|
||||
"1000 1000 128583 🙇 person bowing deeply \n",
|
||||
"1001 1001 128584 🙈 see-no-evil monkey \n",
|
||||
"1002 1002 129511 🧧 red gift envelope \n",
|
||||
"1003 1003 128585 🙉 hear-no-evil monkey \n",
|
||||
"1004 1004 127476 🇴 regional indicator symbol letter \n",
|
||||
"1005 1005 128586 🙊 speak-no-evil monkey \n",
|
||||
"1006 1006 10024 ✨ sparkles \n",
|
||||
"1007 1007 129512 🧨 firecracker \n",
|
||||
"1008 1008 127477 🇵 regional indicator symbol letter p \n",
|
||||
"1009 1009 10060 ❌ cross mark \n",
|
||||
"1010 1010 10062 ❎ negative squared cross mark \n",
|
||||
"1011 1011 129513 🧩 jigsaw puzzle piece \n",
|
||||
"1012 1012 127478 🇶 regional indicator symbol letter q \n",
|
||||
"1013 1013 10067 ❓ black question mark ornament \n",
|
||||
"1014 1014 10068 ❔ white question mark ornament \n",
|
||||
"1015 1015 10069 ❕ white exclamation mark ornament \n",
|
||||
"1016 1016 10071 ❗ heavy exclamation mark symbol \n",
|
||||
"1017 1017 129514 🧪 test tube \n",
|
||||
"1018 1018 129515 🧫 petri dish \n",
|
||||
"1019 1019 10133 ➕ heavy plus sign \n",
|
||||
"1020 1020 10134 ➖ heavy minus sign \n",
|
||||
"1021 1021 10135 ➗ heavy division sign \n",
|
||||
"1022 1022 129424 🦐 shrimp \n",
|
||||
"1023 1023 129425 🦑 squid \n",
|
||||
"1024 1024 10160 ➰ curly loop \n",
|
||||
"1025 1025 10175 ➿ double curly loop \n",
|
||||
"1026 1026 129520 🧰 TOOLBOX \n",
|
||||
"\n",
|
||||
" description_de \n",
|
||||
"0 mahjong fliese rot drache \n",
|
||||
"1 spool gewinde \n",
|
||||
"2 ball of yarn \n",
|
||||
"3 spielkarte black joker \n",
|
||||
"4 zipper-mund gesicht \n",
|
||||
"5 geld-mund-gesicht \n",
|
||||
"6 gesicht thermometer \n",
|
||||
"7 nerd gesicht \n",
|
||||
"8 denken gesicht \n",
|
||||
"9 gesicht head-bandage \n",
|
||||
"10 roboter-gesicht \n",
|
||||
"11 umarmen gesicht \n",
|
||||
"12 zeichen hörner \n",
|
||||
"13 ruf \n",
|
||||
"14 rückseite hand \n",
|
||||
"15 links-facing fist \n",
|
||||
"16 rechts-facing fist \n",
|
||||
"17 handshake \n",
|
||||
"18 hand index mittleren fingern gekreuzt \n",
|
||||
"19 liebe handzeichen \n",
|
||||
"20 gesicht cowboy-hut \n",
|
||||
"21 clown-gesicht \n",
|
||||
"22 ungeätztes gesicht \n",
|
||||
"23 rollen boden lachen \n",
|
||||
"24 gesicht abtastend \n",
|
||||
"25 liegendes gesicht \n",
|
||||
"26 gesicht palme \n",
|
||||
"27 snezing-gesicht \n",
|
||||
"28 gesicht augenbrauen erhöht \n",
|
||||
"29 gesicht sternenaugen \n",
|
||||
"... ... \n",
|
||||
"997 raised hand \n",
|
||||
"998 gesicht gute geste \n",
|
||||
"999 gesicht ok gestern \n",
|
||||
"1000 person, tief biegt \n",
|
||||
"1001 see-no-evil-affe \n",
|
||||
"1002 roter geschenkumschlag \n",
|
||||
"1003 hear-no-evil affe \n",
|
||||
"1004 regionaler indikator symbol letter o \n",
|
||||
"1005 speak-no-evil-affe \n",
|
||||
"1006 funkeln \n",
|
||||
"1007 firecracker \n",
|
||||
"1008 regionaler indikator symbol letter p \n",
|
||||
"1009 kreuzmarke \n",
|
||||
"1010 negative quadrierte kreuzmarke \n",
|
||||
"1011 jigsaw puzzle stück \n",
|
||||
"1012 regionaler indikator symbol letter q \n",
|
||||
"1013 schwarzes fragenzeichen-verzierung \n",
|
||||
"1014 weisses frage-zeichen-verzierung \n",
|
||||
"1015 weisses ausrufs-markierungs-verzierung \n",
|
||||
"1016 schweres ausrufzeichen symbol \n",
|
||||
"1017 testrohr \n",
|
||||
"1018 petrischale \n",
|
||||
"1019 schwere plus zeichen \n",
|
||||
"1020 schwere minuszeichen \n",
|
||||
"1021 schweres abteilungszeichen \n",
|
||||
"1022 garnele \n",
|
||||
"1023 tintenfisch \n",
|
||||
"1024 lockige schleife \n",
|
||||
"1025 doppelt lockige schleife \n",
|
||||
"1026 WERKZEUGKASTEN \n",
|
||||
"\n",
|
||||
"[1027 rows x 5 columns]"
|
||||
]
|
||||
},
|
||||
"execution_count": 46,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -247,16 +822,28 @@
|
||||
"source": [
|
||||
"prepStep1Table = table\n",
|
||||
"\n",
|
||||
"stopwords_en = set(stopwords.words('english'))\n",
|
||||
"stopwords_de = set(stopwords.words('german'))\n",
|
||||
"\n",
|
||||
"# lowercasing and stopword removal\n",
|
||||
"for entry in range(len(table[\"description\"][1:])):\n",
|
||||
" table[\"description\"][entry] = table[\"description\"][entry].lower()\n",
|
||||
" newEntry = []\n",
|
||||
" for word in table[\"description\"][entry].split(\" \"):\n",
|
||||
" if word not in set(stopwords.words('english')):\n",
|
||||
" if word not in stopwords_en:\n",
|
||||
" newEntry.append(word)\n",
|
||||
" table[\"description\"][entry] = (\" \").join(newEntry).lower()\n",
|
||||
" \n",
|
||||
"# just copied block from above and adjusted for german column :)\n",
|
||||
"for entry in range(len(table[\"description_de\"][1:])):\n",
|
||||
" table[\"description_de\"][entry] = table[\"description_de\"][entry].lower()\n",
|
||||
" newEntry = []\n",
|
||||
" for word in table[\"description_de\"][entry].split(\" \"):\n",
|
||||
" if word not in stopwords_de:\n",
|
||||
" newEntry.append(word)\n",
|
||||
" table[\"description_de\"][entry] = (\" \").join(newEntry).lower()\n",
|
||||
" \n",
|
||||
"table.head()"
|
||||
"table"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -283,7 +870,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
"version": "3.6.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 149,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -65,7 +65,7 @@
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" <th>description_de</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
@ -75,7 +75,7 @@
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>MAHJONG TILE RED DRAGON</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>MAHJONG FLIESE ROT DRACHE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
@ -83,7 +83,7 @@
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>SPOOL OF THREAD</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>Spool Gewinde</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
@ -91,7 +91,7 @@
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
@ -99,7 +99,7 @@
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>PLAYING CARD BLACK JOKER</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>SPIELKARTE BLACK JOKER</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
@ -107,19 +107,26 @@
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>ZIPPER-MOUTH FACE</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>ZIPPER-MUND Gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON NaN\n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD NaN\n",
|
||||
"2 2 129526 🧶 BALL OF YARN NaN\n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER NaN\n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE NaN"
|
||||
" Unnamed: 0 code character description \\\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD \n",
|
||||
"2 2 129526 🧶 BALL OF YARN \n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
|
||||
"\n",
|
||||
" description_de \n",
|
||||
"0 MAHJONG FLIESE ROT DRACHE \n",
|
||||
"1 Spool Gewinde \n",
|
||||
"2 BALL OF YARN \n",
|
||||
"3 SPIELKARTE BLACK JOKER \n",
|
||||
"4 ZIPPER-MUND Gesicht "
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
@ -148,6 +155,13 @@
|
||||
"messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* using a Stemmer to get the main 'Part' of each word"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
@ -233,29 +247,30 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 59,
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def evaluate_sentence(sentence):\n",
|
||||
"def evaluate_sentence(sentence, description_key = 'description', lang = 'eng'):\n",
|
||||
" \n",
|
||||
" tokenized_sentence = word_tokenize(sentence)\n",
|
||||
" n = len(tokenized_sentence)\n",
|
||||
" l = table.shape[0]\n",
|
||||
" matrix_list = []\n",
|
||||
" \n",
|
||||
" for index, row in table.iterrows():\n",
|
||||
" emoji_tokens = word_tokenize(row['description'])\n",
|
||||
" emoji_tokens = word_tokenize(row[description_key])\n",
|
||||
" m = len(emoji_tokens)\n",
|
||||
"\n",
|
||||
" mat = np.zeros(shape=(m,n))\n",
|
||||
" for i in range(len(emoji_tokens)):\n",
|
||||
" for j in range(len(tokenized_sentence)):\n",
|
||||
" syn1 = wordnet.synsets(emoji_tokens[i])\n",
|
||||
" syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)\n",
|
||||
" if len(syn1) == 0:\n",
|
||||
" continue\n",
|
||||
" w1 = syn1[0]\n",
|
||||
" #print(j, tokenized_sentence)\n",
|
||||
" syn2 = wordnet.synsets(tokenized_sentence[j])\n",
|
||||
" syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)\n",
|
||||
" if len(syn2) == 0:\n",
|
||||
" continue\n",
|
||||
" w2 = syn2[0]\n",
|
||||
@ -271,15 +286,6 @@
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 130,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result = evaluate_sentence(\"I like playing soccer\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@ -289,13 +295,17 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 131,
|
||||
"execution_count": 22,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lookup = {}\n",
|
||||
"emoji_set = []\n",
|
||||
"for index, row in table.iterrows():\n",
|
||||
" lookup[index] = row['character']"
|
||||
" lookup[index] = row['character']\n",
|
||||
" emoji_set.append(row['character'])\n",
|
||||
"\n",
|
||||
"emoji_set = set(emoji_set)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -307,94 +317,228 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 139,
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summed = np.argsort([-np.sum(x) for x in result])\n",
|
||||
"max_val = np.argsort([-np.max(x) for x in result])\n",
|
||||
"avg = np.argsort([-np.mean(x) for x in result])\n",
|
||||
"def predict(sentence, description_key='description', lang = 'eng', n=10, t=0.9):\n",
|
||||
"\n",
|
||||
"t = 0.9\n",
|
||||
"threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n"
|
||||
" result = evaluate_sentence(sentence, description_key, lang)\n",
|
||||
" \n",
|
||||
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
|
||||
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
|
||||
" avg = np.argsort([-np.mean(x) for x in result])[0:n]\n",
|
||||
" threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n",
|
||||
" \n",
|
||||
" # build a result table\n",
|
||||
" table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n",
|
||||
" lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n",
|
||||
" lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n",
|
||||
" lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n",
|
||||
" \n",
|
||||
" display(table_frame)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 156,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def print_best_results(sorted_indices, n=10):\n",
|
||||
" pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n",
|
||||
" pp.pprint([result[x] for x in sorted_indices[:10]])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 157,
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[ '⚽ -- SOCCER BALL',\n",
|
||||
" '🏉 -- RUGBY FOOTBALL',\n",
|
||||
" '🏈 -- AMERICAN FOOTBALL',\n",
|
||||
" '🎴 -- FLOWER PLAYING CARDS',\n",
|
||||
" '🃏 -- PLAYING CARD BLACK JOKER',\n",
|
||||
" '🇮 -- REGIONAL INDICATOR SYMBOL LETTER I',\n",
|
||||
" '\\U0001f91f -- I LOVE YOU HAND SIGN',\n",
|
||||
" '📧 -- E-MAIL SYMBOL',\n",
|
||||
" '📭 -- OPEN MAILBOX WITH LOWERED FLAG',\n",
|
||||
" '📥 -- INBOX TRAY']\n",
|
||||
"[ array([[0.25 , 0.28571429, 0.58333333, 1. ],\n",
|
||||
" [0.26666667, 0.10526316, 0.1 , 0.1 ]]),\n",
|
||||
" array([[0.25 , 0.28571429, 0.58333333, 0.84615385],\n",
|
||||
" [0.26666667, 0.3 , 0.60869565, 0.96 ]]),\n",
|
||||
" array([[0.33333333, 0.125 , 0.11764706, 0.11764706],\n",
|
||||
" [0.26666667, 0.3 , 0.60869565, 0.96 ]]),\n",
|
||||
" array([[0.23529412, 0.0952381 , 0.09090909, 0.09090909],\n",
|
||||
" [0.25 , 0.47619048, 1. , 0.58333333],\n",
|
||||
" [0.30769231, 0.33333333, 0.57142857, 0.7 ]]),\n",
|
||||
" array([[0.25 , 0.47619048, 1. , 0.58333333],\n",
|
||||
" [0.53333333, 0.22222222, 0.21052632, 0.21052632],\n",
|
||||
" [0.30769231, 0.22222222, 0.21052632, 0.21052632],\n",
|
||||
" [0.28571429, 0.11111111, 0.10526316, 0.10526316]]),\n",
|
||||
" array([[0. , 0. , 0. , 0. ],\n",
|
||||
" [0.33333333, 0.23529412, 0.22222222, 0.22222222],\n",
|
||||
" [0.4 , 0.26666667, 0.25 , 0.25 ],\n",
|
||||
" [0.30769231, 0.22222222, 0.21052632, 0.21052632],\n",
|
||||
" [1. , 0.26666667, 0.25 , 0.25 ]]),\n",
|
||||
" array([[1. , 0.26666667, 0.25 , 0.25 ],\n",
|
||||
" [0.33333333, 0.23529412, 0.22222222, 0.22222222],\n",
|
||||
" [0. , 0. , 0. , 0. ],\n",
|
||||
" [0.28571429, 0.11111111, 0.10526316, 0.10526316],\n",
|
||||
" [0.33333333, 0.23529412, 0.22222222, 0.22222222]]),\n",
|
||||
" array([[0.28571429, 0.31578947, 0.45454545, 0.5 ],\n",
|
||||
" [0.4 , 0.26666667, 0.25 , 0.25 ]]),\n",
|
||||
" array([[0.30769231, 0.11764706, 0.11111111, 0.11111111],\n",
|
||||
" [0.26666667, 0.10526316, 0.1 , 0.1 ],\n",
|
||||
" [0. , 0. , 0. , 0. ],\n",
|
||||
" [0.22222222, 0.14285714, 0.13333333, 0.13333333],\n",
|
||||
" [0.26666667, 0.10526316, 0.1 , 0.1 ]]),\n",
|
||||
" array([[0. , 0. , 0. , 0. ],\n",
|
||||
" [0.26666667, 0.10526316, 0.1 , 0.1 ]])]\n"
|
||||
]
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>summed</th>\n",
|
||||
" <th>summed_description</th>\n",
|
||||
" <th>max_val</th>\n",
|
||||
" <th>max_val_description</th>\n",
|
||||
" <th>avg</th>\n",
|
||||
" <th>avg_description</th>\n",
|
||||
" <th>threshold</th>\n",
|
||||
" <th>threshold_description</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>🤭</td>\n",
|
||||
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>I LOVE YOU HAND SIGN</td>\n",
|
||||
" <td>💐</td>\n",
|
||||
" <td>BOUQUET</td>\n",
|
||||
" <td>🚆</td>\n",
|
||||
" <td>TRAIN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>🔛</td>\n",
|
||||
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
|
||||
" <td>🇮</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
|
||||
" <td>🍳</td>\n",
|
||||
" <td>COOKING</td>\n",
|
||||
" <td>🚸</td>\n",
|
||||
" <td>CHILDREN CROSSING</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>🚮</td>\n",
|
||||
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
|
||||
" <td>🚆</td>\n",
|
||||
" <td>TRAIN</td>\n",
|
||||
" <td>🚆</td>\n",
|
||||
" <td>TRAIN</td>\n",
|
||||
" <td>🚄</td>\n",
|
||||
" <td>HIGH-SPEED TRAIN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>🤪</td>\n",
|
||||
" <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
|
||||
" <td>🚅</td>\n",
|
||||
" <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
|
||||
" <td>🎥</td>\n",
|
||||
" <td>MOVIE CAMERA</td>\n",
|
||||
" <td>🇮</td>\n",
|
||||
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>🥰</td>\n",
|
||||
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
|
||||
" <td>🚄</td>\n",
|
||||
" <td>HIGH-SPEED TRAIN</td>\n",
|
||||
" <td>🎭</td>\n",
|
||||
" <td>PERFORMING ARTS</td>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>I LOVE YOU HAND SIGN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>🙌</td>\n",
|
||||
" <td>PERSON RAISING BOTH HANDS IN CELEBRATION</td>\n",
|
||||
" <td>🚸</td>\n",
|
||||
" <td>CHILDREN CROSSING</td>\n",
|
||||
" <td>🎹</td>\n",
|
||||
" <td>MUSICAL KEYBOARD</td>\n",
|
||||
" <td>🚅</td>\n",
|
||||
" <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>🖖</td>\n",
|
||||
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
|
||||
" <td>🛸</td>\n",
|
||||
" <td>FLYING SAUCER</td>\n",
|
||||
" <td>🧾</td>\n",
|
||||
" <td>RECEIPT</td>\n",
|
||||
" <td>🔹</td>\n",
|
||||
" <td>SMALL BLUE DIAMOND</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>🔂</td>\n",
|
||||
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
|
||||
" <td>🥾</td>\n",
|
||||
" <td>HIKING BOOT</td>\n",
|
||||
" <td>💏</td>\n",
|
||||
" <td>KISS</td>\n",
|
||||
" <td>📠</td>\n",
|
||||
" <td>FAX MACHINE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>😄</td>\n",
|
||||
" <td>SMILING FACE WITH OPEN MOUTH AND SMILING EYES</td>\n",
|
||||
" <td>🥏</td>\n",
|
||||
" <td>FLYING DISC</td>\n",
|
||||
" <td>🔥</td>\n",
|
||||
" <td>FIRE</td>\n",
|
||||
" <td>📡</td>\n",
|
||||
" <td>SATELLITE ANTENNA</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>👉</td>\n",
|
||||
" <td>WHITE RIGHT POINTING BACKHAND INDEX</td>\n",
|
||||
" <td>🚏</td>\n",
|
||||
" <td>BUS STOP</td>\n",
|
||||
" <td>🎼</td>\n",
|
||||
" <td>MUSICAL SCORE</td>\n",
|
||||
" <td>📢</td>\n",
|
||||
" <td>PUBLIC ADDRESS LOUDSPEAKER</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" summed summed_description max_val \\\n",
|
||||
"0 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 🤟 \n",
|
||||
"1 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇮 \n",
|
||||
"2 🚮 PUT LITTER IN ITS PLACE SYMBOL 🚆 \n",
|
||||
"3 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🚅 \n",
|
||||
"4 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🚄 \n",
|
||||
"5 🙌 PERSON RAISING BOTH HANDS IN CELEBRATION 🚸 \n",
|
||||
"6 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🛸 \n",
|
||||
"7 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🥾 \n",
|
||||
"8 😄 SMILING FACE WITH OPEN MOUTH AND SMILING EYES 🥏 \n",
|
||||
"9 👉 WHITE RIGHT POINTING BACKHAND INDEX 🚏 \n",
|
||||
"\n",
|
||||
" max_val_description avg avg_description threshold \\\n",
|
||||
"0 I LOVE YOU HAND SIGN 💐 BOUQUET 🚆 \n",
|
||||
"1 REGIONAL INDICATOR SYMBOL LETTER I 🍳 COOKING 🚸 \n",
|
||||
"2 TRAIN 🚆 TRAIN 🚄 \n",
|
||||
"3 HIGH-SPEED TRAIN WITH BULLET NOSE 🎥 MOVIE CAMERA 🇮 \n",
|
||||
"4 HIGH-SPEED TRAIN 🎭 PERFORMING ARTS 🤟 \n",
|
||||
"5 CHILDREN CROSSING 🎹 MUSICAL KEYBOARD 🚅 \n",
|
||||
"6 FLYING SAUCER 🧾 RECEIPT 🔹 \n",
|
||||
"7 HIKING BOOT 💏 KISS 📠 \n",
|
||||
"8 FLYING DISC 🔥 FIRE 📡 \n",
|
||||
"9 BUS STOP 🎼 MUSICAL SCORE 📢 \n",
|
||||
"\n",
|
||||
" threshold_description \n",
|
||||
"0 TRAIN \n",
|
||||
"1 CHILDREN CROSSING \n",
|
||||
"2 HIGH-SPEED TRAIN \n",
|
||||
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
|
||||
"4 I LOVE YOU HAND SIGN \n",
|
||||
"5 HIGH-SPEED TRAIN WITH BULLET NOSE \n",
|
||||
"6 SMALL BLUE DIAMOND \n",
|
||||
"7 FAX MACHINE \n",
|
||||
"8 SATELLITE ANTENNA \n",
|
||||
"9 PUBLIC ADDRESS LOUDSPEAKER "
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print_best_results(threshold)"
|
||||
"predict(\"I like to travel by train\", description_key='description' , lang='eng')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
||||
6
Project/simple_approach/README.md
Normal file
6
Project/simple_approach/README.md
Normal file
@ -0,0 +1,6 @@
|
||||
# Simple Approach
|
||||
|
||||
----
|
||||
|
||||
just a slightly better approach than the naive one 💁
|
||||
|
||||
2010
Project/simple_approach/simple_twitter_learning.ipynb
Normal file
2010
Project/simple_approach/simple_twitter_learning.ipynb
Normal file
File diff suppressed because it is too large
Load Diff
17664
Project/simple_approach/test.csv
Normal file
17664
Project/simple_approach/test.csv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user