Let Preprocessing.ipynb also process german translation of emoji descriptions
This commit is contained in:
		| @ -2,38 +2,28 @@ | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 36, | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "[nltk_data] Downloading package stopwords to\n", | ||||
|       "[nltk_data]     C:\\Users\\Maren\\AppData\\Roaming\\nltk_data...\n", | ||||
|       "[nltk_data]   Unzipping corpora\\stopwords.zip.\n" | ||||
|       "[nltk_data] Downloading package stopwords to /home/jonas/nltk_data...\n", | ||||
|       "[nltk_data]   Package stopwords is already up-to-date!\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "True" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 36, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import pandas as pd\n", | ||||
|     "import nltk\n", | ||||
|     "nltk.download('stopwords')" | ||||
|     "nltk.download('stopwords')\n", | ||||
|     "from nltk.corpus import stopwords" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 33, | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -42,7 +32,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 15, | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -70,7 +60,7 @@ | ||||
|        "      <th>code</th>\n", | ||||
|        "      <th>character</th>\n", | ||||
|        "      <th>description</th>\n", | ||||
|        "      <th>Unnamed: 4</th>\n", | ||||
|        "      <th>description_de</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
| @ -80,7 +70,7 @@ | ||||
|        "      <td>126980</td>\n", | ||||
|        "      <td>🀄</td>\n", | ||||
|        "      <td>MAHJONG TILE RED DRAGON</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>MAHJONG FLIESE ROT DRACHE</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1</th>\n", | ||||
| @ -88,7 +78,7 @@ | ||||
|        "      <td>129525</td>\n", | ||||
|        "      <td>🧵</td>\n", | ||||
|        "      <td>SPOOL OF THREAD</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>Spool Gewinde</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>2</th>\n", | ||||
| @ -96,7 +86,7 @@ | ||||
|        "      <td>129526</td>\n", | ||||
|        "      <td>🧶</td>\n", | ||||
|        "      <td>BALL OF YARN</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>BALL OF YARN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>3</th>\n", | ||||
| @ -104,7 +94,7 @@ | ||||
|        "      <td>127183</td>\n", | ||||
|        "      <td>🃏</td>\n", | ||||
|        "      <td>PLAYING CARD BLACK JOKER</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>SPIELKARTE BLACK JOKER</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>4</th>\n", | ||||
| @ -112,22 +102,29 @@ | ||||
|        "      <td>129296</td>\n", | ||||
|        "      <td>🤐</td>\n", | ||||
|        "      <td>ZIPPER-MOUTH FACE</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>ZIPPER-MUND Gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "   Unnamed: 0    code character               description  Unnamed: 4\n", | ||||
|        "0           0  126980         🀄   MAHJONG TILE RED DRAGON         NaN\n", | ||||
|        "1           1  129525         🧵           SPOOL OF THREAD         NaN\n", | ||||
|        "2           2  129526         🧶              BALL OF YARN         NaN\n", | ||||
|        "3           3  127183         🃏  PLAYING CARD BLACK JOKER         NaN\n", | ||||
|        "4           4  129296         🤐         ZIPPER-MOUTH FACE         NaN" | ||||
|        "   Unnamed: 0    code character               description  \\\n", | ||||
|        "0           0  126980         🀄   MAHJONG TILE RED DRAGON   \n", | ||||
|        "1           1  129525         🧵           SPOOL OF THREAD   \n", | ||||
|        "2           2  129526         🧶              BALL OF YARN   \n", | ||||
|        "3           3  127183         🃏  PLAYING CARD BLACK JOKER   \n", | ||||
|        "4           4  129296         🤐         ZIPPER-MOUTH FACE   \n", | ||||
|        "\n", | ||||
|        "              description_de  \n", | ||||
|        "0  MAHJONG FLIESE ROT DRACHE  \n", | ||||
|        "1              Spool Gewinde  \n", | ||||
|        "2               BALL OF YARN  \n", | ||||
|        "3     SPIELKARTE BLACK JOKER  \n", | ||||
|        "4        ZIPPER-MUND Gesicht  " | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 15, | ||||
|      "execution_count": 3, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -138,23 +135,31 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 46, | ||||
|    "execution_count": 4, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n", | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "  \"\"\"\n", | ||||
|       "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n", | ||||
|       "  \n", | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "  # Remove the CWD from sys.path while we load stuff.\n" | ||||
|       "  del sys.path[0]\n", | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:22: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
| @ -182,7 +187,7 @@ | ||||
|        "      <th>code</th>\n", | ||||
|        "      <th>character</th>\n", | ||||
|        "      <th>description</th>\n", | ||||
|        "      <th>Unnamed: 4</th>\n", | ||||
|        "      <th>description_de</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
| @ -192,7 +197,7 @@ | ||||
|        "      <td>126980</td>\n", | ||||
|        "      <td>🀄</td>\n", | ||||
|        "      <td>mahjong tile red dragon</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>mahjong fliese rot drache</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1</th>\n", | ||||
| @ -200,7 +205,7 @@ | ||||
|        "      <td>129525</td>\n", | ||||
|        "      <td>🧵</td>\n", | ||||
|        "      <td>spool thread</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>spool gewinde</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>2</th>\n", | ||||
| @ -208,7 +213,7 @@ | ||||
|        "      <td>129526</td>\n", | ||||
|        "      <td>🧶</td>\n", | ||||
|        "      <td>ball yarn</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>ball of yarn</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>3</th>\n", | ||||
| @ -216,7 +221,7 @@ | ||||
|        "      <td>127183</td>\n", | ||||
|        "      <td>🃏</td>\n", | ||||
|        "      <td>playing card black joker</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>spielkarte black joker</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>4</th>\n", | ||||
| @ -224,22 +229,592 @@ | ||||
|        "      <td>129296</td>\n", | ||||
|        "      <td>🤐</td>\n", | ||||
|        "      <td>zipper-mouth face</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "      <td>zipper-mund gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>5</th>\n", | ||||
|        "      <td>5</td>\n", | ||||
|        "      <td>129297</td>\n", | ||||
|        "      <td>🤑</td>\n", | ||||
|        "      <td>money-mouth face</td>\n", | ||||
|        "      <td>geld-mund-gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>6</th>\n", | ||||
|        "      <td>6</td>\n", | ||||
|        "      <td>129298</td>\n", | ||||
|        "      <td>🤒</td>\n", | ||||
|        "      <td>face thermometer</td>\n", | ||||
|        "      <td>gesicht thermometer</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>7</th>\n", | ||||
|        "      <td>7</td>\n", | ||||
|        "      <td>129299</td>\n", | ||||
|        "      <td>🤓</td>\n", | ||||
|        "      <td>nerd face</td>\n", | ||||
|        "      <td>nerd gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>8</th>\n", | ||||
|        "      <td>8</td>\n", | ||||
|        "      <td>129300</td>\n", | ||||
|        "      <td>🤔</td>\n", | ||||
|        "      <td>thinking face</td>\n", | ||||
|        "      <td>denken gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>9</th>\n", | ||||
|        "      <td>9</td>\n", | ||||
|        "      <td>129301</td>\n", | ||||
|        "      <td>🤕</td>\n", | ||||
|        "      <td>face head-bandage</td>\n", | ||||
|        "      <td>gesicht head-bandage</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>10</th>\n", | ||||
|        "      <td>10</td>\n", | ||||
|        "      <td>129302</td>\n", | ||||
|        "      <td>🤖</td>\n", | ||||
|        "      <td>robot face</td>\n", | ||||
|        "      <td>roboter-gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>11</th>\n", | ||||
|        "      <td>11</td>\n", | ||||
|        "      <td>129303</td>\n", | ||||
|        "      <td>🤗</td>\n", | ||||
|        "      <td>hugging face</td>\n", | ||||
|        "      <td>umarmen gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>12</th>\n", | ||||
|        "      <td>12</td>\n", | ||||
|        "      <td>129304</td>\n", | ||||
|        "      <td>🤘</td>\n", | ||||
|        "      <td>sign horns</td>\n", | ||||
|        "      <td>zeichen hörner</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>13</th>\n", | ||||
|        "      <td>13</td>\n", | ||||
|        "      <td>129305</td>\n", | ||||
|        "      <td>🤙</td>\n", | ||||
|        "      <td>call hand</td>\n", | ||||
|        "      <td>ruf</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>14</th>\n", | ||||
|        "      <td>14</td>\n", | ||||
|        "      <td>129306</td>\n", | ||||
|        "      <td>🤚</td>\n", | ||||
|        "      <td>raised back hand</td>\n", | ||||
|        "      <td>rückseite hand</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>15</th>\n", | ||||
|        "      <td>15</td>\n", | ||||
|        "      <td>129307</td>\n", | ||||
|        "      <td>🤛</td>\n", | ||||
|        "      <td>left-facing fist</td>\n", | ||||
|        "      <td>links-facing fist</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>16</th>\n", | ||||
|        "      <td>16</td>\n", | ||||
|        "      <td>129308</td>\n", | ||||
|        "      <td>🤜</td>\n", | ||||
|        "      <td>right-facing fist</td>\n", | ||||
|        "      <td>rechts-facing fist</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>17</th>\n", | ||||
|        "      <td>17</td>\n", | ||||
|        "      <td>129309</td>\n", | ||||
|        "      <td>🤝</td>\n", | ||||
|        "      <td>handshake</td>\n", | ||||
|        "      <td>handshake</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>18</th>\n", | ||||
|        "      <td>18</td>\n", | ||||
|        "      <td>129310</td>\n", | ||||
|        "      <td>🤞</td>\n", | ||||
|        "      <td>hand index middle fingers crossed</td>\n", | ||||
|        "      <td>hand index mittleren fingern gekreuzt</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>19</th>\n", | ||||
|        "      <td>19</td>\n", | ||||
|        "      <td>129311</td>\n", | ||||
|        "      <td>🤟</td>\n", | ||||
|        "      <td>love hand sign</td>\n", | ||||
|        "      <td>liebe handzeichen</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>20</th>\n", | ||||
|        "      <td>20</td>\n", | ||||
|        "      <td>129312</td>\n", | ||||
|        "      <td>🤠</td>\n", | ||||
|        "      <td>face cowboy hat</td>\n", | ||||
|        "      <td>gesicht cowboy-hut</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>21</th>\n", | ||||
|        "      <td>21</td>\n", | ||||
|        "      <td>129313</td>\n", | ||||
|        "      <td>🤡</td>\n", | ||||
|        "      <td>clown face</td>\n", | ||||
|        "      <td>clown-gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>22</th>\n", | ||||
|        "      <td>22</td>\n", | ||||
|        "      <td>129314</td>\n", | ||||
|        "      <td>🤢</td>\n", | ||||
|        "      <td>nauseated face</td>\n", | ||||
|        "      <td>ungeätztes gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>23</th>\n", | ||||
|        "      <td>23</td>\n", | ||||
|        "      <td>129315</td>\n", | ||||
|        "      <td>🤣</td>\n", | ||||
|        "      <td>rolling floor laughing</td>\n", | ||||
|        "      <td>rollen boden lachen</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>24</th>\n", | ||||
|        "      <td>24</td>\n", | ||||
|        "      <td>129316</td>\n", | ||||
|        "      <td>🤤</td>\n", | ||||
|        "      <td>drooling face</td>\n", | ||||
|        "      <td>gesicht abtastend</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>25</th>\n", | ||||
|        "      <td>25</td>\n", | ||||
|        "      <td>129317</td>\n", | ||||
|        "      <td>🤥</td>\n", | ||||
|        "      <td>lying face</td>\n", | ||||
|        "      <td>liegendes gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>26</th>\n", | ||||
|        "      <td>26</td>\n", | ||||
|        "      <td>129318</td>\n", | ||||
|        "      <td>🤦</td>\n", | ||||
|        "      <td>face palm</td>\n", | ||||
|        "      <td>gesicht palme</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>27</th>\n", | ||||
|        "      <td>27</td>\n", | ||||
|        "      <td>129319</td>\n", | ||||
|        "      <td>🤧</td>\n", | ||||
|        "      <td>sneezing face</td>\n", | ||||
|        "      <td>snezing-gesicht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>28</th>\n", | ||||
|        "      <td>28</td>\n", | ||||
|        "      <td>129320</td>\n", | ||||
|        "      <td>🤨</td>\n", | ||||
|        "      <td>face one eyebrow raised</td>\n", | ||||
|        "      <td>gesicht augenbrauen erhöht</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>29</th>\n", | ||||
|        "      <td>29</td>\n", | ||||
|        "      <td>129321</td>\n", | ||||
|        "      <td>🤩</td>\n", | ||||
|        "      <td>grinning face star eyes</td>\n", | ||||
|        "      <td>gesicht sternenaugen</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>...</th>\n", | ||||
|        "      <td>...</td>\n", | ||||
|        "      <td>...</td>\n", | ||||
|        "      <td>...</td>\n", | ||||
|        "      <td>...</td>\n", | ||||
|        "      <td>...</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>997</th>\n", | ||||
|        "      <td>997</td>\n", | ||||
|        "      <td>9995</td>\n", | ||||
|        "      <td>✋</td>\n", | ||||
|        "      <td>raised hand</td>\n", | ||||
|        "      <td>raised hand</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>998</th>\n", | ||||
|        "      <td>998</td>\n", | ||||
|        "      <td>128581</td>\n", | ||||
|        "      <td>🙅</td>\n", | ||||
|        "      <td>face good gesture</td>\n", | ||||
|        "      <td>gesicht gute geste</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>999</th>\n", | ||||
|        "      <td>999</td>\n", | ||||
|        "      <td>128582</td>\n", | ||||
|        "      <td>🙆</td>\n", | ||||
|        "      <td>face ok gesture</td>\n", | ||||
|        "      <td>gesicht ok gestern</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1000</th>\n", | ||||
|        "      <td>1000</td>\n", | ||||
|        "      <td>128583</td>\n", | ||||
|        "      <td>🙇</td>\n", | ||||
|        "      <td>person bowing deeply</td>\n", | ||||
|        "      <td>person, tief biegt</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1001</th>\n", | ||||
|        "      <td>1001</td>\n", | ||||
|        "      <td>128584</td>\n", | ||||
|        "      <td>🙈</td>\n", | ||||
|        "      <td>see-no-evil monkey</td>\n", | ||||
|        "      <td>see-no-evil-affe</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1002</th>\n", | ||||
|        "      <td>1002</td>\n", | ||||
|        "      <td>129511</td>\n", | ||||
|        "      <td>🧧</td>\n", | ||||
|        "      <td>red gift envelope</td>\n", | ||||
|        "      <td>roter geschenkumschlag</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1003</th>\n", | ||||
|        "      <td>1003</td>\n", | ||||
|        "      <td>128585</td>\n", | ||||
|        "      <td>🙉</td>\n", | ||||
|        "      <td>hear-no-evil monkey</td>\n", | ||||
|        "      <td>hear-no-evil affe</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1004</th>\n", | ||||
|        "      <td>1004</td>\n", | ||||
|        "      <td>127476</td>\n", | ||||
|        "      <td>🇴</td>\n", | ||||
|        "      <td>regional indicator symbol letter</td>\n", | ||||
|        "      <td>regionaler indikator symbol letter o</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1005</th>\n", | ||||
|        "      <td>1005</td>\n", | ||||
|        "      <td>128586</td>\n", | ||||
|        "      <td>🙊</td>\n", | ||||
|        "      <td>speak-no-evil monkey</td>\n", | ||||
|        "      <td>speak-no-evil-affe</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1006</th>\n", | ||||
|        "      <td>1006</td>\n", | ||||
|        "      <td>10024</td>\n", | ||||
|        "      <td>✨</td>\n", | ||||
|        "      <td>sparkles</td>\n", | ||||
|        "      <td>funkeln</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1007</th>\n", | ||||
|        "      <td>1007</td>\n", | ||||
|        "      <td>129512</td>\n", | ||||
|        "      <td>🧨</td>\n", | ||||
|        "      <td>firecracker</td>\n", | ||||
|        "      <td>firecracker</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1008</th>\n", | ||||
|        "      <td>1008</td>\n", | ||||
|        "      <td>127477</td>\n", | ||||
|        "      <td>🇵</td>\n", | ||||
|        "      <td>regional indicator symbol letter p</td>\n", | ||||
|        "      <td>regionaler indikator symbol letter p</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1009</th>\n", | ||||
|        "      <td>1009</td>\n", | ||||
|        "      <td>10060</td>\n", | ||||
|        "      <td>❌</td>\n", | ||||
|        "      <td>cross mark</td>\n", | ||||
|        "      <td>kreuzmarke</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1010</th>\n", | ||||
|        "      <td>1010</td>\n", | ||||
|        "      <td>10062</td>\n", | ||||
|        "      <td>❎</td>\n", | ||||
|        "      <td>negative squared cross mark</td>\n", | ||||
|        "      <td>negative quadrierte kreuzmarke</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1011</th>\n", | ||||
|        "      <td>1011</td>\n", | ||||
|        "      <td>129513</td>\n", | ||||
|        "      <td>🧩</td>\n", | ||||
|        "      <td>jigsaw puzzle piece</td>\n", | ||||
|        "      <td>jigsaw puzzle stück</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1012</th>\n", | ||||
|        "      <td>1012</td>\n", | ||||
|        "      <td>127478</td>\n", | ||||
|        "      <td>🇶</td>\n", | ||||
|        "      <td>regional indicator symbol letter q</td>\n", | ||||
|        "      <td>regionaler indikator symbol letter q</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1013</th>\n", | ||||
|        "      <td>1013</td>\n", | ||||
|        "      <td>10067</td>\n", | ||||
|        "      <td>❓</td>\n", | ||||
|        "      <td>black question mark ornament</td>\n", | ||||
|        "      <td>schwarzes fragenzeichen-verzierung</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1014</th>\n", | ||||
|        "      <td>1014</td>\n", | ||||
|        "      <td>10068</td>\n", | ||||
|        "      <td>❔</td>\n", | ||||
|        "      <td>white question mark ornament</td>\n", | ||||
|        "      <td>weisses frage-zeichen-verzierung</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1015</th>\n", | ||||
|        "      <td>1015</td>\n", | ||||
|        "      <td>10069</td>\n", | ||||
|        "      <td>❕</td>\n", | ||||
|        "      <td>white exclamation mark ornament</td>\n", | ||||
|        "      <td>weisses ausrufs-markierungs-verzierung</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1016</th>\n", | ||||
|        "      <td>1016</td>\n", | ||||
|        "      <td>10071</td>\n", | ||||
|        "      <td>❗</td>\n", | ||||
|        "      <td>heavy exclamation mark symbol</td>\n", | ||||
|        "      <td>schweres ausrufzeichen symbol</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1017</th>\n", | ||||
|        "      <td>1017</td>\n", | ||||
|        "      <td>129514</td>\n", | ||||
|        "      <td>🧪</td>\n", | ||||
|        "      <td>test tube</td>\n", | ||||
|        "      <td>testrohr</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1018</th>\n", | ||||
|        "      <td>1018</td>\n", | ||||
|        "      <td>129515</td>\n", | ||||
|        "      <td>🧫</td>\n", | ||||
|        "      <td>petri dish</td>\n", | ||||
|        "      <td>petrischale</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1019</th>\n", | ||||
|        "      <td>1019</td>\n", | ||||
|        "      <td>10133</td>\n", | ||||
|        "      <td>➕</td>\n", | ||||
|        "      <td>heavy plus sign</td>\n", | ||||
|        "      <td>schwere plus zeichen</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1020</th>\n", | ||||
|        "      <td>1020</td>\n", | ||||
|        "      <td>10134</td>\n", | ||||
|        "      <td>➖</td>\n", | ||||
|        "      <td>heavy minus sign</td>\n", | ||||
|        "      <td>schwere minuszeichen</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1021</th>\n", | ||||
|        "      <td>1021</td>\n", | ||||
|        "      <td>10135</td>\n", | ||||
|        "      <td>➗</td>\n", | ||||
|        "      <td>heavy division sign</td>\n", | ||||
|        "      <td>schweres abteilungszeichen</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1022</th>\n", | ||||
|        "      <td>1022</td>\n", | ||||
|        "      <td>129424</td>\n", | ||||
|        "      <td>🦐</td>\n", | ||||
|        "      <td>shrimp</td>\n", | ||||
|        "      <td>garnele</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1023</th>\n", | ||||
|        "      <td>1023</td>\n", | ||||
|        "      <td>129425</td>\n", | ||||
|        "      <td>🦑</td>\n", | ||||
|        "      <td>squid</td>\n", | ||||
|        "      <td>tintenfisch</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1024</th>\n", | ||||
|        "      <td>1024</td>\n", | ||||
|        "      <td>10160</td>\n", | ||||
|        "      <td>➰</td>\n", | ||||
|        "      <td>curly loop</td>\n", | ||||
|        "      <td>lockige schleife</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1025</th>\n", | ||||
|        "      <td>1025</td>\n", | ||||
|        "      <td>10175</td>\n", | ||||
|        "      <td>➿</td>\n", | ||||
|        "      <td>double curly loop</td>\n", | ||||
|        "      <td>doppelt lockige schleife</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1026</th>\n", | ||||
|        "      <td>1026</td>\n", | ||||
|        "      <td>129520</td>\n", | ||||
|        "      <td>🧰</td>\n", | ||||
|        "      <td>TOOLBOX</td>\n", | ||||
|        "      <td>WERKZEUGKASTEN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "<p>1027 rows × 5 columns</p>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "   Unnamed: 0    code character               description  Unnamed: 4\n", | ||||
|        "0           0  126980         🀄   mahjong tile red dragon         NaN\n", | ||||
|        "1           1  129525         🧵              spool thread         NaN\n", | ||||
|        "2           2  129526         🧶                 ball yarn         NaN\n", | ||||
|        "3           3  127183         🃏  playing card black joker         NaN\n", | ||||
|        "4           4  129296         🤐         zipper-mouth face         NaN" | ||||
|        "      Unnamed: 0    code character                         description  \\\n", | ||||
|        "0              0  126980         🀄             mahjong tile red dragon   \n", | ||||
|        "1              1  129525         🧵                        spool thread   \n", | ||||
|        "2              2  129526         🧶                           ball yarn   \n", | ||||
|        "3              3  127183         🃏            playing card black joker   \n", | ||||
|        "4              4  129296         🤐                   zipper-mouth face   \n", | ||||
|        "5              5  129297         🤑                    money-mouth face   \n", | ||||
|        "6              6  129298         🤒                    face thermometer   \n", | ||||
|        "7              7  129299         🤓                           nerd face   \n", | ||||
|        "8              8  129300         🤔                       thinking face   \n", | ||||
|        "9              9  129301         🤕                   face head-bandage   \n", | ||||
|        "10            10  129302         🤖                          robot face   \n", | ||||
|        "11            11  129303         🤗                        hugging face   \n", | ||||
|        "12            12  129304         🤘                          sign horns   \n", | ||||
|        "13            13  129305         🤙                           call hand   \n", | ||||
|        "14            14  129306         🤚                    raised back hand   \n", | ||||
|        "15            15  129307         🤛                    left-facing fist   \n", | ||||
|        "16            16  129308         🤜                   right-facing fist   \n", | ||||
|        "17            17  129309         🤝                           handshake   \n", | ||||
|        "18            18  129310         🤞   hand index middle fingers crossed   \n", | ||||
|        "19            19  129311         🤟                      love hand sign   \n", | ||||
|        "20            20  129312         🤠                     face cowboy hat   \n", | ||||
|        "21            21  129313         🤡                          clown face   \n", | ||||
|        "22            22  129314         🤢                      nauseated face   \n", | ||||
|        "23            23  129315         🤣              rolling floor laughing   \n", | ||||
|        "24            24  129316         🤤                       drooling face   \n", | ||||
|        "25            25  129317         🤥                          lying face   \n", | ||||
|        "26            26  129318         🤦                           face palm   \n", | ||||
|        "27            27  129319         🤧                       sneezing face   \n", | ||||
|        "28            28  129320         🤨             face one eyebrow raised   \n", | ||||
|        "29            29  129321         🤩             grinning face star eyes   \n", | ||||
|        "...          ...     ...       ...                                 ...   \n", | ||||
|        "997          997    9995         ✋                         raised hand   \n", | ||||
|        "998          998  128581         🙅                   face good gesture   \n", | ||||
|        "999          999  128582         🙆                     face ok gesture   \n", | ||||
|        "1000        1000  128583         🙇                person bowing deeply   \n", | ||||
|        "1001        1001  128584         🙈                  see-no-evil monkey   \n", | ||||
|        "1002        1002  129511         🧧                   red gift envelope   \n", | ||||
|        "1003        1003  128585         🙉                 hear-no-evil monkey   \n", | ||||
|        "1004        1004  127476         🇴    regional indicator symbol letter   \n", | ||||
|        "1005        1005  128586         🙊                speak-no-evil monkey   \n", | ||||
|        "1006        1006   10024         ✨                            sparkles   \n", | ||||
|        "1007        1007  129512         🧨                         firecracker   \n", | ||||
|        "1008        1008  127477         🇵  regional indicator symbol letter p   \n", | ||||
|        "1009        1009   10060         ❌                          cross mark   \n", | ||||
|        "1010        1010   10062         ❎         negative squared cross mark   \n", | ||||
|        "1011        1011  129513         🧩                 jigsaw puzzle piece   \n", | ||||
|        "1012        1012  127478         🇶  regional indicator symbol letter q   \n", | ||||
|        "1013        1013   10067         ❓        black question mark ornament   \n", | ||||
|        "1014        1014   10068         ❔        white question mark ornament   \n", | ||||
|        "1015        1015   10069         ❕     white exclamation mark ornament   \n", | ||||
|        "1016        1016   10071         ❗       heavy exclamation mark symbol   \n", | ||||
|        "1017        1017  129514         🧪                           test tube   \n", | ||||
|        "1018        1018  129515         🧫                          petri dish   \n", | ||||
|        "1019        1019   10133         ➕                     heavy plus sign   \n", | ||||
|        "1020        1020   10134         ➖                    heavy minus sign   \n", | ||||
|        "1021        1021   10135         ➗                 heavy division sign   \n", | ||||
|        "1022        1022  129424         🦐                              shrimp   \n", | ||||
|        "1023        1023  129425         🦑                               squid   \n", | ||||
|        "1024        1024   10160         ➰                          curly loop   \n", | ||||
|        "1025        1025   10175         ➿                   double curly loop   \n", | ||||
|        "1026        1026  129520         🧰                             TOOLBOX   \n", | ||||
|        "\n", | ||||
|        "                              description_de  \n", | ||||
|        "0                  mahjong fliese rot drache  \n", | ||||
|        "1                              spool gewinde  \n", | ||||
|        "2                               ball of yarn  \n", | ||||
|        "3                     spielkarte black joker  \n", | ||||
|        "4                        zipper-mund gesicht  \n", | ||||
|        "5                          geld-mund-gesicht  \n", | ||||
|        "6                        gesicht thermometer  \n", | ||||
|        "7                               nerd gesicht  \n", | ||||
|        "8                             denken gesicht  \n", | ||||
|        "9                       gesicht head-bandage  \n", | ||||
|        "10                           roboter-gesicht  \n", | ||||
|        "11                           umarmen gesicht  \n", | ||||
|        "12                            zeichen hörner  \n", | ||||
|        "13                                       ruf  \n", | ||||
|        "14                            rückseite hand  \n", | ||||
|        "15                         links-facing fist  \n", | ||||
|        "16                        rechts-facing fist  \n", | ||||
|        "17                                 handshake  \n", | ||||
|        "18     hand index mittleren fingern gekreuzt  \n", | ||||
|        "19                         liebe handzeichen  \n", | ||||
|        "20                        gesicht cowboy-hut  \n", | ||||
|        "21                             clown-gesicht  \n", | ||||
|        "22                        ungeätztes gesicht  \n", | ||||
|        "23                       rollen boden lachen  \n", | ||||
|        "24                         gesicht abtastend  \n", | ||||
|        "25                         liegendes gesicht  \n", | ||||
|        "26                             gesicht palme  \n", | ||||
|        "27                           snezing-gesicht  \n", | ||||
|        "28                gesicht augenbrauen erhöht  \n", | ||||
|        "29                      gesicht sternenaugen  \n", | ||||
|        "...                                      ...  \n", | ||||
|        "997                              raised hand  \n", | ||||
|        "998                       gesicht gute geste  \n", | ||||
|        "999                       gesicht ok gestern  \n", | ||||
|        "1000                      person, tief biegt  \n", | ||||
|        "1001                        see-no-evil-affe  \n", | ||||
|        "1002                  roter geschenkumschlag  \n", | ||||
|        "1003                       hear-no-evil affe  \n", | ||||
|        "1004    regionaler indikator symbol letter o  \n", | ||||
|        "1005                      speak-no-evil-affe  \n", | ||||
|        "1006                                 funkeln  \n", | ||||
|        "1007                             firecracker  \n", | ||||
|        "1008    regionaler indikator symbol letter p  \n", | ||||
|        "1009                              kreuzmarke  \n", | ||||
|        "1010          negative quadrierte kreuzmarke  \n", | ||||
|        "1011                     jigsaw puzzle stück  \n", | ||||
|        "1012    regionaler indikator symbol letter q  \n", | ||||
|        "1013      schwarzes fragenzeichen-verzierung  \n", | ||||
|        "1014        weisses frage-zeichen-verzierung  \n", | ||||
|        "1015  weisses ausrufs-markierungs-verzierung  \n", | ||||
|        "1016           schweres ausrufzeichen symbol  \n", | ||||
|        "1017                                testrohr  \n", | ||||
|        "1018                             petrischale  \n", | ||||
|        "1019                    schwere plus zeichen  \n", | ||||
|        "1020                    schwere minuszeichen  \n", | ||||
|        "1021              schweres abteilungszeichen  \n", | ||||
|        "1022                                 garnele  \n", | ||||
|        "1023                             tintenfisch  \n", | ||||
|        "1024                        lockige schleife  \n", | ||||
|        "1025                doppelt lockige schleife  \n", | ||||
|        "1026                          WERKZEUGKASTEN  \n", | ||||
|        "\n", | ||||
|        "[1027 rows x 5 columns]" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 46, | ||||
|      "execution_count": 4, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -247,16 +822,28 @@ | ||||
|    "source": [ | ||||
|     "prepStep1Table = table\n", | ||||
|     "\n", | ||||
|     "stopwords_en = set(stopwords.words('english'))\n", | ||||
|     "stopwords_de = set(stopwords.words('german'))\n", | ||||
|     "\n", | ||||
|     "# lowercasing and stopword removal\n", | ||||
|     "for entry in range(len(table[\"description\"][1:])):\n", | ||||
|     "    table[\"description\"][entry] = table[\"description\"][entry].lower()\n", | ||||
|     "    newEntry = []\n", | ||||
|     "    for word in table[\"description\"][entry].split(\" \"):\n", | ||||
|     "        if word not in set(stopwords.words('english')):\n", | ||||
|     "        if word not in stopwords_en:\n", | ||||
|     "            newEntry.append(word)\n", | ||||
|     "    table[\"description\"][entry] = (\" \").join(newEntry).lower()\n", | ||||
|     "    \n", | ||||
|     "# just copied block from above and adjusted for german column :)\n", | ||||
|     "for entry in range(len(table[\"description_de\"][1:])):\n", | ||||
|     "    table[\"description_de\"][entry] = table[\"description_de\"][entry].lower()\n", | ||||
|     "    newEntry = []\n", | ||||
|     "    for word in table[\"description_de\"][entry].split(\" \"):\n", | ||||
|     "        if word not in stopwords_de:\n", | ||||
|     "            newEntry.append(word)\n", | ||||
|     "    table[\"description_de\"][entry] = (\" \").join(newEntry).lower()\n", | ||||
|     "        \n", | ||||
|     "table.head()" | ||||
|     "table" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
| @ -283,7 +870,7 @@ | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.6.4" | ||||
|    "version": "3.6.5" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  | ||||
		Reference in New Issue
	
	Block a user