Let Preprocessing.ipynb also process german translation of emoji descriptions
This commit is contained in:
Jonas Weinz
parent
4baaaa6341
commit
9a922d88fa
@ -2,38 +2,28 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 36,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[nltk_data] Downloading package stopwords to\n",
|
||||
"[nltk_data] C:\\Users\\Maren\\AppData\\Roaming\\nltk_data...\n",
|
||||
"[nltk_data] Unzipping corpora\\stopwords.zip.\n"
|
||||
"[nltk_data] Downloading package stopwords to /home/jonas/nltk_data...\n",
|
||||
"[nltk_data] Package stopwords is already up-to-date!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"True"
|
||||
]
|
||||
},
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import nltk\n",
|
||||
"nltk.download('stopwords')"
|
||||
"nltk.download('stopwords')\n",
|
||||
"from nltk.corpus import stopwords"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -42,7 +32,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -70,7 +60,7 @@
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" <th>description_de</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
@ -80,7 +70,7 @@
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>MAHJONG TILE RED DRAGON</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>MAHJONG FLIESE ROT DRACHE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
@ -88,7 +78,7 @@
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>SPOOL OF THREAD</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>Spool Gewinde</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
@ -96,7 +86,7 @@
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
@ -104,7 +94,7 @@
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>PLAYING CARD BLACK JOKER</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>SPIELKARTE BLACK JOKER</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
@ -112,22 +102,29 @@
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>ZIPPER-MOUTH FACE</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>ZIPPER-MUND Gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON NaN\n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD NaN\n",
|
||||
"2 2 129526 🧶 BALL OF YARN NaN\n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER NaN\n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE NaN"
|
||||
" Unnamed: 0 code character description \\\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD \n",
|
||||
"2 2 129526 🧶 BALL OF YARN \n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
|
||||
"\n",
|
||||
" description_de \n",
|
||||
"0 MAHJONG FLIESE ROT DRACHE \n",
|
||||
"1 Spool Gewinde \n",
|
||||
"2 BALL OF YARN \n",
|
||||
"3 SPIELKARTE BLACK JOKER \n",
|
||||
"4 ZIPPER-MUND Gesicht "
|
||||
]
|
||||
},
|
||||
"execution_count": 15,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -138,23 +135,31 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 46,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||||
" \"\"\"\n",
|
||||
"C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
|
||||
" \n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||||
" # Remove the CWD from sys.path while we load stuff.\n"
|
||||
" del sys.path[0]\n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||||
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:22: SettingWithCopyWarning: \n",
|
||||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||||
"\n",
|
||||
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -182,7 +187,7 @@
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" <th>description_de</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
@ -192,7 +197,7 @@
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>mahjong tile red dragon</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>mahjong fliese rot drache</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
@ -200,7 +205,7 @@
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>spool thread</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>spool gewinde</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
@ -208,7 +213,7 @@
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>ball yarn</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>ball of yarn</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
@ -216,7 +221,7 @@
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>playing card black joker</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>spielkarte black joker</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
@ -224,22 +229,592 @@
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>zipper-mouth face</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>zipper-mund gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>129297</td>\n",
|
||||
" <td>🤑</td>\n",
|
||||
" <td>money-mouth face</td>\n",
|
||||
" <td>geld-mund-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>129298</td>\n",
|
||||
" <td>🤒</td>\n",
|
||||
" <td>face thermometer</td>\n",
|
||||
" <td>gesicht thermometer</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>7</td>\n",
|
||||
" <td>129299</td>\n",
|
||||
" <td>🤓</td>\n",
|
||||
" <td>nerd face</td>\n",
|
||||
" <td>nerd gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>8</td>\n",
|
||||
" <td>129300</td>\n",
|
||||
" <td>🤔</td>\n",
|
||||
" <td>thinking face</td>\n",
|
||||
" <td>denken gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>9</td>\n",
|
||||
" <td>129301</td>\n",
|
||||
" <td>🤕</td>\n",
|
||||
" <td>face head-bandage</td>\n",
|
||||
" <td>gesicht head-bandage</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10</th>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>129302</td>\n",
|
||||
" <td>🤖</td>\n",
|
||||
" <td>robot face</td>\n",
|
||||
" <td>roboter-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>11</th>\n",
|
||||
" <td>11</td>\n",
|
||||
" <td>129303</td>\n",
|
||||
" <td>🤗</td>\n",
|
||||
" <td>hugging face</td>\n",
|
||||
" <td>umarmen gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>12</th>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>129304</td>\n",
|
||||
" <td>🤘</td>\n",
|
||||
" <td>sign horns</td>\n",
|
||||
" <td>zeichen hörner</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>13</th>\n",
|
||||
" <td>13</td>\n",
|
||||
" <td>129305</td>\n",
|
||||
" <td>🤙</td>\n",
|
||||
" <td>call hand</td>\n",
|
||||
" <td>ruf</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>14</th>\n",
|
||||
" <td>14</td>\n",
|
||||
" <td>129306</td>\n",
|
||||
" <td>🤚</td>\n",
|
||||
" <td>raised back hand</td>\n",
|
||||
" <td>rückseite hand</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>15</th>\n",
|
||||
" <td>15</td>\n",
|
||||
" <td>129307</td>\n",
|
||||
" <td>🤛</td>\n",
|
||||
" <td>left-facing fist</td>\n",
|
||||
" <td>links-facing fist</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>16</th>\n",
|
||||
" <td>16</td>\n",
|
||||
" <td>129308</td>\n",
|
||||
" <td>🤜</td>\n",
|
||||
" <td>right-facing fist</td>\n",
|
||||
" <td>rechts-facing fist</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>17</th>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>129309</td>\n",
|
||||
" <td>🤝</td>\n",
|
||||
" <td>handshake</td>\n",
|
||||
" <td>handshake</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>18</th>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>129310</td>\n",
|
||||
" <td>🤞</td>\n",
|
||||
" <td>hand index middle fingers crossed</td>\n",
|
||||
" <td>hand index mittleren fingern gekreuzt</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>19</th>\n",
|
||||
" <td>19</td>\n",
|
||||
" <td>129311</td>\n",
|
||||
" <td>🤟</td>\n",
|
||||
" <td>love hand sign</td>\n",
|
||||
" <td>liebe handzeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>20</th>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>129312</td>\n",
|
||||
" <td>🤠</td>\n",
|
||||
" <td>face cowboy hat</td>\n",
|
||||
" <td>gesicht cowboy-hut</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>21</th>\n",
|
||||
" <td>21</td>\n",
|
||||
" <td>129313</td>\n",
|
||||
" <td>🤡</td>\n",
|
||||
" <td>clown face</td>\n",
|
||||
" <td>clown-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>22</th>\n",
|
||||
" <td>22</td>\n",
|
||||
" <td>129314</td>\n",
|
||||
" <td>🤢</td>\n",
|
||||
" <td>nauseated face</td>\n",
|
||||
" <td>ungeätztes gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>23</th>\n",
|
||||
" <td>23</td>\n",
|
||||
" <td>129315</td>\n",
|
||||
" <td>🤣</td>\n",
|
||||
" <td>rolling floor laughing</td>\n",
|
||||
" <td>rollen boden lachen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>24</th>\n",
|
||||
" <td>24</td>\n",
|
||||
" <td>129316</td>\n",
|
||||
" <td>🤤</td>\n",
|
||||
" <td>drooling face</td>\n",
|
||||
" <td>gesicht abtastend</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>25</th>\n",
|
||||
" <td>25</td>\n",
|
||||
" <td>129317</td>\n",
|
||||
" <td>🤥</td>\n",
|
||||
" <td>lying face</td>\n",
|
||||
" <td>liegendes gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>26</th>\n",
|
||||
" <td>26</td>\n",
|
||||
" <td>129318</td>\n",
|
||||
" <td>🤦</td>\n",
|
||||
" <td>face palm</td>\n",
|
||||
" <td>gesicht palme</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>27</th>\n",
|
||||
" <td>27</td>\n",
|
||||
" <td>129319</td>\n",
|
||||
" <td>🤧</td>\n",
|
||||
" <td>sneezing face</td>\n",
|
||||
" <td>snezing-gesicht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>28</th>\n",
|
||||
" <td>28</td>\n",
|
||||
" <td>129320</td>\n",
|
||||
" <td>🤨</td>\n",
|
||||
" <td>face one eyebrow raised</td>\n",
|
||||
" <td>gesicht augenbrauen erhöht</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>29</th>\n",
|
||||
" <td>29</td>\n",
|
||||
" <td>129321</td>\n",
|
||||
" <td>🤩</td>\n",
|
||||
" <td>grinning face star eyes</td>\n",
|
||||
" <td>gesicht sternenaugen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>...</th>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>997</th>\n",
|
||||
" <td>997</td>\n",
|
||||
" <td>9995</td>\n",
|
||||
" <td>✋</td>\n",
|
||||
" <td>raised hand</td>\n",
|
||||
" <td>raised hand</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>998</th>\n",
|
||||
" <td>998</td>\n",
|
||||
" <td>128581</td>\n",
|
||||
" <td>🙅</td>\n",
|
||||
" <td>face good gesture</td>\n",
|
||||
" <td>gesicht gute geste</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999</th>\n",
|
||||
" <td>999</td>\n",
|
||||
" <td>128582</td>\n",
|
||||
" <td>🙆</td>\n",
|
||||
" <td>face ok gesture</td>\n",
|
||||
" <td>gesicht ok gestern</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1000</th>\n",
|
||||
" <td>1000</td>\n",
|
||||
" <td>128583</td>\n",
|
||||
" <td>🙇</td>\n",
|
||||
" <td>person bowing deeply</td>\n",
|
||||
" <td>person, tief biegt</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1001</th>\n",
|
||||
" <td>1001</td>\n",
|
||||
" <td>128584</td>\n",
|
||||
" <td>🙈</td>\n",
|
||||
" <td>see-no-evil monkey</td>\n",
|
||||
" <td>see-no-evil-affe</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1002</th>\n",
|
||||
" <td>1002</td>\n",
|
||||
" <td>129511</td>\n",
|
||||
" <td>🧧</td>\n",
|
||||
" <td>red gift envelope</td>\n",
|
||||
" <td>roter geschenkumschlag</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1003</th>\n",
|
||||
" <td>1003</td>\n",
|
||||
" <td>128585</td>\n",
|
||||
" <td>🙉</td>\n",
|
||||
" <td>hear-no-evil monkey</td>\n",
|
||||
" <td>hear-no-evil affe</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1004</th>\n",
|
||||
" <td>1004</td>\n",
|
||||
" <td>127476</td>\n",
|
||||
" <td>🇴</td>\n",
|
||||
" <td>regional indicator symbol letter</td>\n",
|
||||
" <td>regionaler indikator symbol letter o</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1005</th>\n",
|
||||
" <td>1005</td>\n",
|
||||
" <td>128586</td>\n",
|
||||
" <td>🙊</td>\n",
|
||||
" <td>speak-no-evil monkey</td>\n",
|
||||
" <td>speak-no-evil-affe</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1006</th>\n",
|
||||
" <td>1006</td>\n",
|
||||
" <td>10024</td>\n",
|
||||
" <td>✨</td>\n",
|
||||
" <td>sparkles</td>\n",
|
||||
" <td>funkeln</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1007</th>\n",
|
||||
" <td>1007</td>\n",
|
||||
" <td>129512</td>\n",
|
||||
" <td>🧨</td>\n",
|
||||
" <td>firecracker</td>\n",
|
||||
" <td>firecracker</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1008</th>\n",
|
||||
" <td>1008</td>\n",
|
||||
" <td>127477</td>\n",
|
||||
" <td>🇵</td>\n",
|
||||
" <td>regional indicator symbol letter p</td>\n",
|
||||
" <td>regionaler indikator symbol letter p</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1009</th>\n",
|
||||
" <td>1009</td>\n",
|
||||
" <td>10060</td>\n",
|
||||
" <td>❌</td>\n",
|
||||
" <td>cross mark</td>\n",
|
||||
" <td>kreuzmarke</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1010</th>\n",
|
||||
" <td>1010</td>\n",
|
||||
" <td>10062</td>\n",
|
||||
" <td>❎</td>\n",
|
||||
" <td>negative squared cross mark</td>\n",
|
||||
" <td>negative quadrierte kreuzmarke</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1011</th>\n",
|
||||
" <td>1011</td>\n",
|
||||
" <td>129513</td>\n",
|
||||
" <td>🧩</td>\n",
|
||||
" <td>jigsaw puzzle piece</td>\n",
|
||||
" <td>jigsaw puzzle stück</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1012</th>\n",
|
||||
" <td>1012</td>\n",
|
||||
" <td>127478</td>\n",
|
||||
" <td>🇶</td>\n",
|
||||
" <td>regional indicator symbol letter q</td>\n",
|
||||
" <td>regionaler indikator symbol letter q</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1013</th>\n",
|
||||
" <td>1013</td>\n",
|
||||
" <td>10067</td>\n",
|
||||
" <td>❓</td>\n",
|
||||
" <td>black question mark ornament</td>\n",
|
||||
" <td>schwarzes fragenzeichen-verzierung</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1014</th>\n",
|
||||
" <td>1014</td>\n",
|
||||
" <td>10068</td>\n",
|
||||
" <td>❔</td>\n",
|
||||
" <td>white question mark ornament</td>\n",
|
||||
" <td>weisses frage-zeichen-verzierung</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1015</th>\n",
|
||||
" <td>1015</td>\n",
|
||||
" <td>10069</td>\n",
|
||||
" <td>❕</td>\n",
|
||||
" <td>white exclamation mark ornament</td>\n",
|
||||
" <td>weisses ausrufs-markierungs-verzierung</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1016</th>\n",
|
||||
" <td>1016</td>\n",
|
||||
" <td>10071</td>\n",
|
||||
" <td>❗</td>\n",
|
||||
" <td>heavy exclamation mark symbol</td>\n",
|
||||
" <td>schweres ausrufzeichen symbol</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1017</th>\n",
|
||||
" <td>1017</td>\n",
|
||||
" <td>129514</td>\n",
|
||||
" <td>🧪</td>\n",
|
||||
" <td>test tube</td>\n",
|
||||
" <td>testrohr</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1018</th>\n",
|
||||
" <td>1018</td>\n",
|
||||
" <td>129515</td>\n",
|
||||
" <td>🧫</td>\n",
|
||||
" <td>petri dish</td>\n",
|
||||
" <td>petrischale</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1019</th>\n",
|
||||
" <td>1019</td>\n",
|
||||
" <td>10133</td>\n",
|
||||
" <td>➕</td>\n",
|
||||
" <td>heavy plus sign</td>\n",
|
||||
" <td>schwere plus zeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1020</th>\n",
|
||||
" <td>1020</td>\n",
|
||||
" <td>10134</td>\n",
|
||||
" <td>➖</td>\n",
|
||||
" <td>heavy minus sign</td>\n",
|
||||
" <td>schwere minuszeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1021</th>\n",
|
||||
" <td>1021</td>\n",
|
||||
" <td>10135</td>\n",
|
||||
" <td>➗</td>\n",
|
||||
" <td>heavy division sign</td>\n",
|
||||
" <td>schweres abteilungszeichen</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1022</th>\n",
|
||||
" <td>1022</td>\n",
|
||||
" <td>129424</td>\n",
|
||||
" <td>🦐</td>\n",
|
||||
" <td>shrimp</td>\n",
|
||||
" <td>garnele</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1023</th>\n",
|
||||
" <td>1023</td>\n",
|
||||
" <td>129425</td>\n",
|
||||
" <td>🦑</td>\n",
|
||||
" <td>squid</td>\n",
|
||||
" <td>tintenfisch</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1024</th>\n",
|
||||
" <td>1024</td>\n",
|
||||
" <td>10160</td>\n",
|
||||
" <td>➰</td>\n",
|
||||
" <td>curly loop</td>\n",
|
||||
" <td>lockige schleife</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1025</th>\n",
|
||||
" <td>1025</td>\n",
|
||||
" <td>10175</td>\n",
|
||||
" <td>➿</td>\n",
|
||||
" <td>double curly loop</td>\n",
|
||||
" <td>doppelt lockige schleife</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1026</th>\n",
|
||||
" <td>1026</td>\n",
|
||||
" <td>129520</td>\n",
|
||||
" <td>🧰</td>\n",
|
||||
" <td>TOOLBOX</td>\n",
|
||||
" <td>WERKZEUGKASTEN</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"<p>1027 rows × 5 columns</p>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 mahjong tile red dragon NaN\n",
|
||||
"1 1 129525 🧵 spool thread NaN\n",
|
||||
"2 2 129526 🧶 ball yarn NaN\n",
|
||||
"3 3 127183 🃏 playing card black joker NaN\n",
|
||||
"4 4 129296 🤐 zipper-mouth face NaN"
|
||||
" Unnamed: 0 code character description \\\n",
|
||||
"0 0 126980 🀄 mahjong tile red dragon \n",
|
||||
"1 1 129525 🧵 spool thread \n",
|
||||
"2 2 129526 🧶 ball yarn \n",
|
||||
"3 3 127183 🃏 playing card black joker \n",
|
||||
"4 4 129296 🤐 zipper-mouth face \n",
|
||||
"5 5 129297 🤑 money-mouth face \n",
|
||||
"6 6 129298 🤒 face thermometer \n",
|
||||
"7 7 129299 🤓 nerd face \n",
|
||||
"8 8 129300 🤔 thinking face \n",
|
||||
"9 9 129301 🤕 face head-bandage \n",
|
||||
"10 10 129302 🤖 robot face \n",
|
||||
"11 11 129303 🤗 hugging face \n",
|
||||
"12 12 129304 🤘 sign horns \n",
|
||||
"13 13 129305 🤙 call hand \n",
|
||||
"14 14 129306 🤚 raised back hand \n",
|
||||
"15 15 129307 🤛 left-facing fist \n",
|
||||
"16 16 129308 🤜 right-facing fist \n",
|
||||
"17 17 129309 🤝 handshake \n",
|
||||
"18 18 129310 🤞 hand index middle fingers crossed \n",
|
||||
"19 19 129311 🤟 love hand sign \n",
|
||||
"20 20 129312 🤠 face cowboy hat \n",
|
||||
"21 21 129313 🤡 clown face \n",
|
||||
"22 22 129314 🤢 nauseated face \n",
|
||||
"23 23 129315 🤣 rolling floor laughing \n",
|
||||
"24 24 129316 🤤 drooling face \n",
|
||||
"25 25 129317 🤥 lying face \n",
|
||||
"26 26 129318 🤦 face palm \n",
|
||||
"27 27 129319 🤧 sneezing face \n",
|
||||
"28 28 129320 🤨 face one eyebrow raised \n",
|
||||
"29 29 129321 🤩 grinning face star eyes \n",
|
||||
"... ... ... ... ... \n",
|
||||
"997 997 9995 ✋ raised hand \n",
|
||||
"998 998 128581 🙅 face good gesture \n",
|
||||
"999 999 128582 🙆 face ok gesture \n",
|
||||
"1000 1000 128583 🙇 person bowing deeply \n",
|
||||
"1001 1001 128584 🙈 see-no-evil monkey \n",
|
||||
"1002 1002 129511 🧧 red gift envelope \n",
|
||||
"1003 1003 128585 🙉 hear-no-evil monkey \n",
|
||||
"1004 1004 127476 🇴 regional indicator symbol letter \n",
|
||||
"1005 1005 128586 🙊 speak-no-evil monkey \n",
|
||||
"1006 1006 10024 ✨ sparkles \n",
|
||||
"1007 1007 129512 🧨 firecracker \n",
|
||||
"1008 1008 127477 🇵 regional indicator symbol letter p \n",
|
||||
"1009 1009 10060 ❌ cross mark \n",
|
||||
"1010 1010 10062 ❎ negative squared cross mark \n",
|
||||
"1011 1011 129513 🧩 jigsaw puzzle piece \n",
|
||||
"1012 1012 127478 🇶 regional indicator symbol letter q \n",
|
||||
"1013 1013 10067 ❓ black question mark ornament \n",
|
||||
"1014 1014 10068 ❔ white question mark ornament \n",
|
||||
"1015 1015 10069 ❕ white exclamation mark ornament \n",
|
||||
"1016 1016 10071 ❗ heavy exclamation mark symbol \n",
|
||||
"1017 1017 129514 🧪 test tube \n",
|
||||
"1018 1018 129515 🧫 petri dish \n",
|
||||
"1019 1019 10133 ➕ heavy plus sign \n",
|
||||
"1020 1020 10134 ➖ heavy minus sign \n",
|
||||
"1021 1021 10135 ➗ heavy division sign \n",
|
||||
"1022 1022 129424 🦐 shrimp \n",
|
||||
"1023 1023 129425 🦑 squid \n",
|
||||
"1024 1024 10160 ➰ curly loop \n",
|
||||
"1025 1025 10175 ➿ double curly loop \n",
|
||||
"1026 1026 129520 🧰 TOOLBOX \n",
|
||||
"\n",
|
||||
" description_de \n",
|
||||
"0 mahjong fliese rot drache \n",
|
||||
"1 spool gewinde \n",
|
||||
"2 ball of yarn \n",
|
||||
"3 spielkarte black joker \n",
|
||||
"4 zipper-mund gesicht \n",
|
||||
"5 geld-mund-gesicht \n",
|
||||
"6 gesicht thermometer \n",
|
||||
"7 nerd gesicht \n",
|
||||
"8 denken gesicht \n",
|
||||
"9 gesicht head-bandage \n",
|
||||
"10 roboter-gesicht \n",
|
||||
"11 umarmen gesicht \n",
|
||||
"12 zeichen hörner \n",
|
||||
"13 ruf \n",
|
||||
"14 rückseite hand \n",
|
||||
"15 links-facing fist \n",
|
||||
"16 rechts-facing fist \n",
|
||||
"17 handshake \n",
|
||||
"18 hand index mittleren fingern gekreuzt \n",
|
||||
"19 liebe handzeichen \n",
|
||||
"20 gesicht cowboy-hut \n",
|
||||
"21 clown-gesicht \n",
|
||||
"22 ungeätztes gesicht \n",
|
||||
"23 rollen boden lachen \n",
|
||||
"24 gesicht abtastend \n",
|
||||
"25 liegendes gesicht \n",
|
||||
"26 gesicht palme \n",
|
||||
"27 snezing-gesicht \n",
|
||||
"28 gesicht augenbrauen erhöht \n",
|
||||
"29 gesicht sternenaugen \n",
|
||||
"... ... \n",
|
||||
"997 raised hand \n",
|
||||
"998 gesicht gute geste \n",
|
||||
"999 gesicht ok gestern \n",
|
||||
"1000 person, tief biegt \n",
|
||||
"1001 see-no-evil-affe \n",
|
||||
"1002 roter geschenkumschlag \n",
|
||||
"1003 hear-no-evil affe \n",
|
||||
"1004 regionaler indikator symbol letter o \n",
|
||||
"1005 speak-no-evil-affe \n",
|
||||
"1006 funkeln \n",
|
||||
"1007 firecracker \n",
|
||||
"1008 regionaler indikator symbol letter p \n",
|
||||
"1009 kreuzmarke \n",
|
||||
"1010 negative quadrierte kreuzmarke \n",
|
||||
"1011 jigsaw puzzle stück \n",
|
||||
"1012 regionaler indikator symbol letter q \n",
|
||||
"1013 schwarzes fragenzeichen-verzierung \n",
|
||||
"1014 weisses frage-zeichen-verzierung \n",
|
||||
"1015 weisses ausrufs-markierungs-verzierung \n",
|
||||
"1016 schweres ausrufzeichen symbol \n",
|
||||
"1017 testrohr \n",
|
||||
"1018 petrischale \n",
|
||||
"1019 schwere plus zeichen \n",
|
||||
"1020 schwere minuszeichen \n",
|
||||
"1021 schweres abteilungszeichen \n",
|
||||
"1022 garnele \n",
|
||||
"1023 tintenfisch \n",
|
||||
"1024 lockige schleife \n",
|
||||
"1025 doppelt lockige schleife \n",
|
||||
"1026 WERKZEUGKASTEN \n",
|
||||
"\n",
|
||||
"[1027 rows x 5 columns]"
|
||||
]
|
||||
},
|
||||
"execution_count": 46,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -247,16 +822,28 @@
|
||||
"source": [
|
||||
"prepStep1Table = table\n",
|
||||
"\n",
|
||||
"stopwords_en = set(stopwords.words('english'))\n",
|
||||
"stopwords_de = set(stopwords.words('german'))\n",
|
||||
"\n",
|
||||
"# lowercasing and stopword removal\n",
|
||||
"for entry in range(len(table[\"description\"][1:])):\n",
|
||||
" table[\"description\"][entry] = table[\"description\"][entry].lower()\n",
|
||||
" newEntry = []\n",
|
||||
" for word in table[\"description\"][entry].split(\" \"):\n",
|
||||
" if word not in set(stopwords.words('english')):\n",
|
||||
" if word not in stopwords_en:\n",
|
||||
" newEntry.append(word)\n",
|
||||
" table[\"description\"][entry] = (\" \").join(newEntry).lower()\n",
|
||||
" \n",
|
||||
"# just copied block from above and adjusted for german column :)\n",
|
||||
"for entry in range(len(table[\"description_de\"][1:])):\n",
|
||||
" table[\"description_de\"][entry] = table[\"description_de\"][entry].lower()\n",
|
||||
" newEntry = []\n",
|
||||
" for word in table[\"description_de\"][entry].split(\" \"):\n",
|
||||
" if word not in stopwords_de:\n",
|
||||
" newEntry.append(word)\n",
|
||||
" table[\"description_de\"][entry] = (\" \").join(newEntry).lower()\n",
|
||||
" \n",
|
||||
"table.head()"
|
||||
"table"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -283,7 +870,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
"version": "3.6.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user