nlp-lab/Project/Tools/Preprocessing.ipynb

879 lines
33 KiB
Plaintext
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[nltk_data] Downloading package stopwords to /home/jonas/nltk_data...\n",
"[nltk_data] Package stopwords is already up-to-date!\n"
]
}
],
"source": [
"import pandas as pd\n",
"import nltk\n",
"nltk.download('stopwords')\n",
"from nltk.corpus import stopwords"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"table = pd.read_csv(\"emoji_descriptions.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Unnamed: 0</th>\n",
" <th>code</th>\n",
" <th>character</th>\n",
" <th>description</th>\n",
" <th>description_de</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>126980</td>\n",
" <td>🀄</td>\n",
" <td>MAHJONG TILE RED DRAGON</td>\n",
" <td>MAHJONG FLIESE ROT DRACHE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>129525</td>\n",
" <td>🧵</td>\n",
" <td>SPOOL OF THREAD</td>\n",
" <td>Spool Gewinde</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>129526</td>\n",
" <td>🧶</td>\n",
" <td>BALL OF YARN</td>\n",
" <td>BALL OF YARN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>127183</td>\n",
" <td>🃏</td>\n",
" <td>PLAYING CARD BLACK JOKER</td>\n",
" <td>SPIELKARTE BLACK JOKER</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>129296</td>\n",
" <td>🤐</td>\n",
" <td>ZIPPER-MOUTH FACE</td>\n",
" <td>ZIPPER-MUND Gesicht</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Unnamed: 0 code character description \\\n",
"0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
"1 1 129525 🧵 SPOOL OF THREAD \n",
"2 2 129526 🧶 BALL OF YARN \n",
"3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
"4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
"\n",
" description_de \n",
"0 MAHJONG FLIESE ROT DRACHE \n",
"1 Spool Gewinde \n",
"2 BALL OF YARN \n",
"3 SPIELKARTE BLACK JOKER \n",
"4 ZIPPER-MUND Gesicht "
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"table.head()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" \n",
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" del sys.path[0]\n",
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
"/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:22: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Unnamed: 0</th>\n",
" <th>code</th>\n",
" <th>character</th>\n",
" <th>description</th>\n",
" <th>description_de</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>126980</td>\n",
" <td>🀄</td>\n",
" <td>mahjong tile red dragon</td>\n",
" <td>mahjong fliese rot drache</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>129525</td>\n",
" <td>🧵</td>\n",
" <td>spool thread</td>\n",
" <td>spool gewinde</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>129526</td>\n",
" <td>🧶</td>\n",
" <td>ball yarn</td>\n",
" <td>ball of yarn</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>127183</td>\n",
" <td>🃏</td>\n",
" <td>playing card black joker</td>\n",
" <td>spielkarte black joker</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>129296</td>\n",
" <td>🤐</td>\n",
" <td>zipper-mouth face</td>\n",
" <td>zipper-mund gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>5</td>\n",
" <td>129297</td>\n",
" <td>🤑</td>\n",
" <td>money-mouth face</td>\n",
" <td>geld-mund-gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>6</td>\n",
" <td>129298</td>\n",
" <td>🤒</td>\n",
" <td>face thermometer</td>\n",
" <td>gesicht thermometer</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>7</td>\n",
" <td>129299</td>\n",
" <td>🤓</td>\n",
" <td>nerd face</td>\n",
" <td>nerd gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>8</td>\n",
" <td>129300</td>\n",
" <td>🤔</td>\n",
" <td>thinking face</td>\n",
" <td>denken gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>9</td>\n",
" <td>129301</td>\n",
" <td>🤕</td>\n",
" <td>face head-bandage</td>\n",
" <td>gesicht head-bandage</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>10</td>\n",
" <td>129302</td>\n",
" <td>🤖</td>\n",
" <td>robot face</td>\n",
" <td>roboter-gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>11</td>\n",
" <td>129303</td>\n",
" <td>🤗</td>\n",
" <td>hugging face</td>\n",
" <td>umarmen gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>12</td>\n",
" <td>129304</td>\n",
" <td>🤘</td>\n",
" <td>sign horns</td>\n",
" <td>zeichen hörner</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>13</td>\n",
" <td>129305</td>\n",
" <td>🤙</td>\n",
" <td>call hand</td>\n",
" <td>ruf</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>14</td>\n",
" <td>129306</td>\n",
" <td>🤚</td>\n",
" <td>raised back hand</td>\n",
" <td>rückseite hand</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>15</td>\n",
" <td>129307</td>\n",
" <td>🤛</td>\n",
" <td>left-facing fist</td>\n",
" <td>links-facing fist</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>16</td>\n",
" <td>129308</td>\n",
" <td>🤜</td>\n",
" <td>right-facing fist</td>\n",
" <td>rechts-facing fist</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>17</td>\n",
" <td>129309</td>\n",
" <td>🤝</td>\n",
" <td>handshake</td>\n",
" <td>handshake</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>18</td>\n",
" <td>129310</td>\n",
" <td>🤞</td>\n",
" <td>hand index middle fingers crossed</td>\n",
" <td>hand index mittleren fingern gekreuzt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>19</td>\n",
" <td>129311</td>\n",
" <td>🤟</td>\n",
" <td>love hand sign</td>\n",
" <td>liebe handzeichen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>20</td>\n",
" <td>129312</td>\n",
" <td>🤠</td>\n",
" <td>face cowboy hat</td>\n",
" <td>gesicht cowboy-hut</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>21</td>\n",
" <td>129313</td>\n",
" <td>🤡</td>\n",
" <td>clown face</td>\n",
" <td>clown-gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>22</td>\n",
" <td>129314</td>\n",
" <td>🤢</td>\n",
" <td>nauseated face</td>\n",
" <td>ungeätztes gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>23</td>\n",
" <td>129315</td>\n",
" <td>🤣</td>\n",
" <td>rolling floor laughing</td>\n",
" <td>rollen boden lachen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>24</td>\n",
" <td>129316</td>\n",
" <td>🤤</td>\n",
" <td>drooling face</td>\n",
" <td>gesicht abtastend</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>25</td>\n",
" <td>129317</td>\n",
" <td>🤥</td>\n",
" <td>lying face</td>\n",
" <td>liegendes gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>26</td>\n",
" <td>129318</td>\n",
" <td>🤦</td>\n",
" <td>face palm</td>\n",
" <td>gesicht palme</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>27</td>\n",
" <td>129319</td>\n",
" <td>🤧</td>\n",
" <td>sneezing face</td>\n",
" <td>snezing-gesicht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>28</td>\n",
" <td>129320</td>\n",
" <td>🤨</td>\n",
" <td>face one eyebrow raised</td>\n",
" <td>gesicht augenbrauen erhöht</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>29</td>\n",
" <td>129321</td>\n",
" <td>🤩</td>\n",
" <td>grinning face star eyes</td>\n",
" <td>gesicht sternenaugen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>997</th>\n",
" <td>997</td>\n",
" <td>9995</td>\n",
" <td>✋</td>\n",
" <td>raised hand</td>\n",
" <td>raised hand</td>\n",
" </tr>\n",
" <tr>\n",
" <th>998</th>\n",
" <td>998</td>\n",
" <td>128581</td>\n",
" <td>🙅</td>\n",
" <td>face good gesture</td>\n",
" <td>gesicht gute geste</td>\n",
" </tr>\n",
" <tr>\n",
" <th>999</th>\n",
" <td>999</td>\n",
" <td>128582</td>\n",
" <td>🙆</td>\n",
" <td>face ok gesture</td>\n",
" <td>gesicht ok gestern</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1000</th>\n",
" <td>1000</td>\n",
" <td>128583</td>\n",
" <td>🙇</td>\n",
" <td>person bowing deeply</td>\n",
" <td>person, tief biegt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1001</th>\n",
" <td>1001</td>\n",
" <td>128584</td>\n",
" <td>🙈</td>\n",
" <td>see-no-evil monkey</td>\n",
" <td>see-no-evil-affe</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1002</th>\n",
" <td>1002</td>\n",
" <td>129511</td>\n",
" <td>🧧</td>\n",
" <td>red gift envelope</td>\n",
" <td>roter geschenkumschlag</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1003</th>\n",
" <td>1003</td>\n",
" <td>128585</td>\n",
" <td>🙉</td>\n",
" <td>hear-no-evil monkey</td>\n",
" <td>hear-no-evil affe</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1004</th>\n",
" <td>1004</td>\n",
" <td>127476</td>\n",
" <td>🇴</td>\n",
" <td>regional indicator symbol letter</td>\n",
" <td>regionaler indikator symbol letter o</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1005</th>\n",
" <td>1005</td>\n",
" <td>128586</td>\n",
" <td>🙊</td>\n",
" <td>speak-no-evil monkey</td>\n",
" <td>speak-no-evil-affe</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1006</th>\n",
" <td>1006</td>\n",
" <td>10024</td>\n",
" <td>✨</td>\n",
" <td>sparkles</td>\n",
" <td>funkeln</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1007</th>\n",
" <td>1007</td>\n",
" <td>129512</td>\n",
" <td>🧨</td>\n",
" <td>firecracker</td>\n",
" <td>firecracker</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1008</th>\n",
" <td>1008</td>\n",
" <td>127477</td>\n",
" <td>🇵</td>\n",
" <td>regional indicator symbol letter p</td>\n",
" <td>regionaler indikator symbol letter p</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1009</th>\n",
" <td>1009</td>\n",
" <td>10060</td>\n",
" <td>❌</td>\n",
" <td>cross mark</td>\n",
" <td>kreuzmarke</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1010</th>\n",
" <td>1010</td>\n",
" <td>10062</td>\n",
" <td>❎</td>\n",
" <td>negative squared cross mark</td>\n",
" <td>negative quadrierte kreuzmarke</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1011</th>\n",
" <td>1011</td>\n",
" <td>129513</td>\n",
" <td>🧩</td>\n",
" <td>jigsaw puzzle piece</td>\n",
" <td>jigsaw puzzle stück</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1012</th>\n",
" <td>1012</td>\n",
" <td>127478</td>\n",
" <td>🇶</td>\n",
" <td>regional indicator symbol letter q</td>\n",
" <td>regionaler indikator symbol letter q</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1013</th>\n",
" <td>1013</td>\n",
" <td>10067</td>\n",
" <td>❓</td>\n",
" <td>black question mark ornament</td>\n",
" <td>schwarzes fragenzeichen-verzierung</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1014</th>\n",
" <td>1014</td>\n",
" <td>10068</td>\n",
" <td>❔</td>\n",
" <td>white question mark ornament</td>\n",
" <td>weisses frage-zeichen-verzierung</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1015</th>\n",
" <td>1015</td>\n",
" <td>10069</td>\n",
" <td>❕</td>\n",
" <td>white exclamation mark ornament</td>\n",
" <td>weisses ausrufs-markierungs-verzierung</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1016</th>\n",
" <td>1016</td>\n",
" <td>10071</td>\n",
" <td>❗</td>\n",
" <td>heavy exclamation mark symbol</td>\n",
" <td>schweres ausrufzeichen symbol</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1017</th>\n",
" <td>1017</td>\n",
" <td>129514</td>\n",
" <td>🧪</td>\n",
" <td>test tube</td>\n",
" <td>testrohr</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1018</th>\n",
" <td>1018</td>\n",
" <td>129515</td>\n",
" <td>🧫</td>\n",
" <td>petri dish</td>\n",
" <td>petrischale</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1019</th>\n",
" <td>1019</td>\n",
" <td>10133</td>\n",
" <td></td>\n",
" <td>heavy plus sign</td>\n",
" <td>schwere plus zeichen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1020</th>\n",
" <td>1020</td>\n",
" <td>10134</td>\n",
" <td></td>\n",
" <td>heavy minus sign</td>\n",
" <td>schwere minuszeichen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1021</th>\n",
" <td>1021</td>\n",
" <td>10135</td>\n",
" <td>➗</td>\n",
" <td>heavy division sign</td>\n",
" <td>schweres abteilungszeichen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1022</th>\n",
" <td>1022</td>\n",
" <td>129424</td>\n",
" <td>🦐</td>\n",
" <td>shrimp</td>\n",
" <td>garnele</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1023</th>\n",
" <td>1023</td>\n",
" <td>129425</td>\n",
" <td>🦑</td>\n",
" <td>squid</td>\n",
" <td>tintenfisch</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1024</th>\n",
" <td>1024</td>\n",
" <td>10160</td>\n",
" <td>➰</td>\n",
" <td>curly loop</td>\n",
" <td>lockige schleife</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1025</th>\n",
" <td>1025</td>\n",
" <td>10175</td>\n",
" <td>➿</td>\n",
" <td>double curly loop</td>\n",
" <td>doppelt lockige schleife</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1026</th>\n",
" <td>1026</td>\n",
" <td>129520</td>\n",
" <td>🧰</td>\n",
" <td>TOOLBOX</td>\n",
" <td>WERKZEUGKASTEN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1027 rows × 5 columns</p>\n",
"</div>"
],
"text/plain": [
" Unnamed: 0 code character description \\\n",
"0 0 126980 🀄 mahjong tile red dragon \n",
"1 1 129525 🧵 spool thread \n",
"2 2 129526 🧶 ball yarn \n",
"3 3 127183 🃏 playing card black joker \n",
"4 4 129296 🤐 zipper-mouth face \n",
"5 5 129297 🤑 money-mouth face \n",
"6 6 129298 🤒 face thermometer \n",
"7 7 129299 🤓 nerd face \n",
"8 8 129300 🤔 thinking face \n",
"9 9 129301 🤕 face head-bandage \n",
"10 10 129302 🤖 robot face \n",
"11 11 129303 🤗 hugging face \n",
"12 12 129304 🤘 sign horns \n",
"13 13 129305 🤙 call hand \n",
"14 14 129306 🤚 raised back hand \n",
"15 15 129307 🤛 left-facing fist \n",
"16 16 129308 🤜 right-facing fist \n",
"17 17 129309 🤝 handshake \n",
"18 18 129310 🤞 hand index middle fingers crossed \n",
"19 19 129311 🤟 love hand sign \n",
"20 20 129312 🤠 face cowboy hat \n",
"21 21 129313 🤡 clown face \n",
"22 22 129314 🤢 nauseated face \n",
"23 23 129315 🤣 rolling floor laughing \n",
"24 24 129316 🤤 drooling face \n",
"25 25 129317 🤥 lying face \n",
"26 26 129318 🤦 face palm \n",
"27 27 129319 🤧 sneezing face \n",
"28 28 129320 🤨 face one eyebrow raised \n",
"29 29 129321 🤩 grinning face star eyes \n",
"... ... ... ... ... \n",
"997 997 9995 ✋ raised hand \n",
"998 998 128581 🙅 face good gesture \n",
"999 999 128582 🙆 face ok gesture \n",
"1000 1000 128583 🙇 person bowing deeply \n",
"1001 1001 128584 🙈 see-no-evil monkey \n",
"1002 1002 129511 🧧 red gift envelope \n",
"1003 1003 128585 🙉 hear-no-evil monkey \n",
"1004 1004 127476 🇴 regional indicator symbol letter \n",
"1005 1005 128586 🙊 speak-no-evil monkey \n",
"1006 1006 10024 ✨ sparkles \n",
"1007 1007 129512 🧨 firecracker \n",
"1008 1008 127477 🇵 regional indicator symbol letter p \n",
"1009 1009 10060 ❌ cross mark \n",
"1010 1010 10062 ❎ negative squared cross mark \n",
"1011 1011 129513 🧩 jigsaw puzzle piece \n",
"1012 1012 127478 🇶 regional indicator symbol letter q \n",
"1013 1013 10067 ❓ black question mark ornament \n",
"1014 1014 10068 ❔ white question mark ornament \n",
"1015 1015 10069 ❕ white exclamation mark ornament \n",
"1016 1016 10071 ❗ heavy exclamation mark symbol \n",
"1017 1017 129514 🧪 test tube \n",
"1018 1018 129515 🧫 petri dish \n",
"1019 1019 10133 heavy plus sign \n",
"1020 1020 10134 heavy minus sign \n",
"1021 1021 10135 ➗ heavy division sign \n",
"1022 1022 129424 🦐 shrimp \n",
"1023 1023 129425 🦑 squid \n",
"1024 1024 10160 ➰ curly loop \n",
"1025 1025 10175 ➿ double curly loop \n",
"1026 1026 129520 🧰 TOOLBOX \n",
"\n",
" description_de \n",
"0 mahjong fliese rot drache \n",
"1 spool gewinde \n",
"2 ball of yarn \n",
"3 spielkarte black joker \n",
"4 zipper-mund gesicht \n",
"5 geld-mund-gesicht \n",
"6 gesicht thermometer \n",
"7 nerd gesicht \n",
"8 denken gesicht \n",
"9 gesicht head-bandage \n",
"10 roboter-gesicht \n",
"11 umarmen gesicht \n",
"12 zeichen hörner \n",
"13 ruf \n",
"14 rückseite hand \n",
"15 links-facing fist \n",
"16 rechts-facing fist \n",
"17 handshake \n",
"18 hand index mittleren fingern gekreuzt \n",
"19 liebe handzeichen \n",
"20 gesicht cowboy-hut \n",
"21 clown-gesicht \n",
"22 ungeätztes gesicht \n",
"23 rollen boden lachen \n",
"24 gesicht abtastend \n",
"25 liegendes gesicht \n",
"26 gesicht palme \n",
"27 snezing-gesicht \n",
"28 gesicht augenbrauen erhöht \n",
"29 gesicht sternenaugen \n",
"... ... \n",
"997 raised hand \n",
"998 gesicht gute geste \n",
"999 gesicht ok gestern \n",
"1000 person, tief biegt \n",
"1001 see-no-evil-affe \n",
"1002 roter geschenkumschlag \n",
"1003 hear-no-evil affe \n",
"1004 regionaler indikator symbol letter o \n",
"1005 speak-no-evil-affe \n",
"1006 funkeln \n",
"1007 firecracker \n",
"1008 regionaler indikator symbol letter p \n",
"1009 kreuzmarke \n",
"1010 negative quadrierte kreuzmarke \n",
"1011 jigsaw puzzle stück \n",
"1012 regionaler indikator symbol letter q \n",
"1013 schwarzes fragenzeichen-verzierung \n",
"1014 weisses frage-zeichen-verzierung \n",
"1015 weisses ausrufs-markierungs-verzierung \n",
"1016 schweres ausrufzeichen symbol \n",
"1017 testrohr \n",
"1018 petrischale \n",
"1019 schwere plus zeichen \n",
"1020 schwere minuszeichen \n",
"1021 schweres abteilungszeichen \n",
"1022 garnele \n",
"1023 tintenfisch \n",
"1024 lockige schleife \n",
"1025 doppelt lockige schleife \n",
"1026 WERKZEUGKASTEN \n",
"\n",
"[1027 rows x 5 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prepStep1Table = table\n",
"\n",
"stopwords_en = set(stopwords.words('english'))\n",
"stopwords_de = set(stopwords.words('german'))\n",
"\n",
"# lowercasing and stopword removal\n",
"for entry in range(len(table[\"description\"][1:])):\n",
" table[\"description\"][entry] = table[\"description\"][entry].lower()\n",
" newEntry = []\n",
" for word in table[\"description\"][entry].split(\" \"):\n",
" if word not in stopwords_en:\n",
" newEntry.append(word)\n",
" table[\"description\"][entry] = (\" \").join(newEntry).lower()\n",
" \n",
"# just copied block from above and adjusted for german column :)\n",
"for entry in range(len(table[\"description_de\"][1:])):\n",
" table[\"description_de\"][entry] = table[\"description_de\"][entry].lower()\n",
" newEntry = []\n",
" for word in table[\"description_de\"][entry].split(\" \"):\n",
" if word not in stopwords_de:\n",
" newEntry.append(word)\n",
" table[\"description_de\"][entry] = (\" \").join(newEntry).lower()\n",
" \n",
"table"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}