{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[nltk_data] Downloading package stopwords to /home/jonas/nltk_data...\n", "[nltk_data] Package stopwords is already up-to-date!\n" ] } ], "source": [ "import pandas as pd\n", "import nltk\n", "nltk.download('stopwords')\n", "from nltk.corpus import stopwords" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "table = pd.read_csv(\"emoji_descriptions.csv\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptiondescription_de
00126980πŸ€„MAHJONG TILE RED DRAGONMAHJONG FLIESE ROT DRACHE
11129525🧡SPOOL OF THREADSpool Gewinde
22129526🧢BALL OF YARNBALL OF YARN
33127183πŸƒPLAYING CARD BLACK JOKERSPIELKARTE BLACK JOKER
44129296🀐ZIPPER-MOUTH FACEZIPPER-MUND Gesicht
\n", "
" ], "text/plain": [ " Unnamed: 0 code character description \\\n", "0 0 126980 πŸ€„ MAHJONG TILE RED DRAGON \n", "1 1 129525 🧡 SPOOL OF THREAD \n", "2 2 129526 🧢 BALL OF YARN \n", "3 3 127183 πŸƒ PLAYING CARD BLACK JOKER \n", "4 4 129296 🀐 ZIPPER-MOUTH FACE \n", "\n", " description_de \n", "0 MAHJONG FLIESE ROT DRACHE \n", "1 Spool Gewinde \n", "2 BALL OF YARN \n", "3 SPIELKARTE BLACK JOKER \n", "4 ZIPPER-MUND Gesicht " ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table.head()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " \n", "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " del sys.path[0]\n", "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", "/home/jonas/.local/lib/python3.6/site-packages/ipykernel_launcher.py:22: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptiondescription_de
00126980πŸ€„mahjong tile red dragonmahjong fliese rot drache
11129525🧡spool threadspool gewinde
22129526🧢ball yarnball of yarn
33127183πŸƒplaying card black jokerspielkarte black joker
44129296🀐zipper-mouth facezipper-mund gesicht
55129297πŸ€‘money-mouth facegeld-mund-gesicht
66129298πŸ€’face thermometergesicht thermometer
77129299πŸ€“nerd facenerd gesicht
88129300πŸ€”thinking facedenken gesicht
99129301πŸ€•face head-bandagegesicht head-bandage
1010129302πŸ€–robot faceroboter-gesicht
1111129303πŸ€—hugging faceumarmen gesicht
1212129304🀘sign hornszeichen hârner
1313129305πŸ€™call handruf
1414129306🀚raised back handrückseite hand
1515129307πŸ€›left-facing fistlinks-facing fist
1616129308🀜right-facing fistrechts-facing fist
1717129309🀝handshakehandshake
1818129310🀞hand index middle fingers crossedhand index mittleren fingern gekreuzt
1919129311🀟love hand signliebe handzeichen
2020129312🀠face cowboy hatgesicht cowboy-hut
2121129313🀑clown faceclown-gesicht
2222129314🀒nauseated faceungeÀtztes gesicht
2323129315🀣rolling floor laughingrollen boden lachen
2424129316🀀drooling facegesicht abtastend
2525129317πŸ€₯lying faceliegendes gesicht
2626129318🀦face palmgesicht palme
2727129319🀧sneezing facesnezing-gesicht
2828129320🀨face one eyebrow raisedgesicht augenbrauen erhâht
2929129321🀩grinning face star eyesgesicht sternenaugen
..................
9979979995βœ‹raised handraised hand
998998128581πŸ™…face good gesturegesicht gute geste
999999128582πŸ™†face ok gesturegesicht ok gestern
10001000128583πŸ™‡person bowing deeplyperson, tief biegt
10011001128584πŸ™ˆsee-no-evil monkeysee-no-evil-affe
10021002129511🧧red gift enveloperoter geschenkumschlag
10031003128585πŸ™‰hear-no-evil monkeyhear-no-evil affe
10041004127476πŸ‡΄regional indicator symbol letterregionaler indikator symbol letter o
10051005128586πŸ™Šspeak-no-evil monkeyspeak-no-evil-affe
1006100610024✨sparklesfunkeln
10071007129512🧨firecrackerfirecracker
10081008127477πŸ‡΅regional indicator symbol letter pregionaler indikator symbol letter p
1009100910060❌cross markkreuzmarke
1010101010062❎negative squared cross marknegative quadrierte kreuzmarke
10111011129513🧩jigsaw puzzle piecejigsaw puzzle stück
10121012127478πŸ‡Άregional indicator symbol letter qregionaler indikator symbol letter q
1013101310067❓black question mark ornamentschwarzes fragenzeichen-verzierung
1014101410068❔white question mark ornamentweisses frage-zeichen-verzierung
1015101510069❕white exclamation mark ornamentweisses ausrufs-markierungs-verzierung
1016101610071❗heavy exclamation mark symbolschweres ausrufzeichen symbol
10171017129514πŸ§ͺtest tubetestrohr
10181018129515🧫petri dishpetrischale
1019101910133βž•heavy plus signschwere plus zeichen
1020102010134βž–heavy minus signschwere minuszeichen
1021102110135βž—heavy division signschweres abteilungszeichen
10221022129424🦐shrimpgarnele
10231023129425πŸ¦‘squidtintenfisch
1024102410160➰curly looplockige schleife
1025102510175➿double curly loopdoppelt lockige schleife
10261026129520🧰TOOLBOXWERKZEUGKASTEN
\n", "

1027 rows Γ— 5 columns

\n", "
" ], "text/plain": [ " Unnamed: 0 code character description \\\n", "0 0 126980 πŸ€„ mahjong tile red dragon \n", "1 1 129525 🧡 spool thread \n", "2 2 129526 🧢 ball yarn \n", "3 3 127183 πŸƒ playing card black joker \n", "4 4 129296 🀐 zipper-mouth face \n", "5 5 129297 πŸ€‘ money-mouth face \n", "6 6 129298 πŸ€’ face thermometer \n", "7 7 129299 πŸ€“ nerd face \n", "8 8 129300 πŸ€” thinking face \n", "9 9 129301 πŸ€• face head-bandage \n", "10 10 129302 πŸ€– robot face \n", "11 11 129303 πŸ€— hugging face \n", "12 12 129304 🀘 sign horns \n", "13 13 129305 πŸ€™ call hand \n", "14 14 129306 🀚 raised back hand \n", "15 15 129307 πŸ€› left-facing fist \n", "16 16 129308 🀜 right-facing fist \n", "17 17 129309 🀝 handshake \n", "18 18 129310 🀞 hand index middle fingers crossed \n", "19 19 129311 🀟 love hand sign \n", "20 20 129312 🀠 face cowboy hat \n", "21 21 129313 🀑 clown face \n", "22 22 129314 🀒 nauseated face \n", "23 23 129315 🀣 rolling floor laughing \n", "24 24 129316 🀀 drooling face \n", "25 25 129317 πŸ€₯ lying face \n", "26 26 129318 🀦 face palm \n", "27 27 129319 🀧 sneezing face \n", "28 28 129320 🀨 face one eyebrow raised \n", "29 29 129321 🀩 grinning face star eyes \n", "... ... ... ... ... \n", "997 997 9995 βœ‹ raised hand \n", "998 998 128581 πŸ™… face good gesture \n", "999 999 128582 πŸ™† face ok gesture \n", "1000 1000 128583 πŸ™‡ person bowing deeply \n", "1001 1001 128584 πŸ™ˆ see-no-evil monkey \n", "1002 1002 129511 🧧 red gift envelope \n", "1003 1003 128585 πŸ™‰ hear-no-evil monkey \n", "1004 1004 127476 πŸ‡΄ regional indicator symbol letter \n", "1005 1005 128586 πŸ™Š speak-no-evil monkey \n", "1006 1006 10024 ✨ sparkles \n", "1007 1007 129512 🧨 firecracker \n", "1008 1008 127477 πŸ‡΅ regional indicator symbol letter p \n", "1009 1009 10060 ❌ cross mark \n", "1010 1010 10062 ❎ negative squared cross mark \n", "1011 1011 129513 🧩 jigsaw puzzle piece \n", "1012 1012 127478 πŸ‡Ά regional indicator symbol letter q \n", "1013 1013 10067 ❓ black question mark ornament \n", "1014 1014 10068 ❔ white question mark ornament \n", "1015 1015 10069 ❕ white exclamation mark ornament \n", "1016 1016 10071 ❗ heavy exclamation mark symbol \n", "1017 1017 129514 πŸ§ͺ test tube \n", "1018 1018 129515 🧫 petri dish \n", "1019 1019 10133 βž• heavy plus sign \n", "1020 1020 10134 βž– heavy minus sign \n", "1021 1021 10135 βž— heavy division sign \n", "1022 1022 129424 🦐 shrimp \n", "1023 1023 129425 πŸ¦‘ squid \n", "1024 1024 10160 ➰ curly loop \n", "1025 1025 10175 ➿ double curly loop \n", "1026 1026 129520 🧰 TOOLBOX \n", "\n", " description_de \n", "0 mahjong fliese rot drache \n", "1 spool gewinde \n", "2 ball of yarn \n", "3 spielkarte black joker \n", "4 zipper-mund gesicht \n", "5 geld-mund-gesicht \n", "6 gesicht thermometer \n", "7 nerd gesicht \n", "8 denken gesicht \n", "9 gesicht head-bandage \n", "10 roboter-gesicht \n", "11 umarmen gesicht \n", "12 zeichen hΓΆrner \n", "13 ruf \n", "14 rΓΌckseite hand \n", "15 links-facing fist \n", "16 rechts-facing fist \n", "17 handshake \n", "18 hand index mittleren fingern gekreuzt \n", "19 liebe handzeichen \n", "20 gesicht cowboy-hut \n", "21 clown-gesicht \n", "22 ungeΓ€tztes gesicht \n", "23 rollen boden lachen \n", "24 gesicht abtastend \n", "25 liegendes gesicht \n", "26 gesicht palme \n", "27 snezing-gesicht \n", "28 gesicht augenbrauen erhΓΆht \n", "29 gesicht sternenaugen \n", "... ... \n", "997 raised hand \n", "998 gesicht gute geste \n", "999 gesicht ok gestern \n", "1000 person, tief biegt \n", "1001 see-no-evil-affe \n", "1002 roter geschenkumschlag \n", "1003 hear-no-evil affe \n", "1004 regionaler indikator symbol letter o \n", "1005 speak-no-evil-affe \n", "1006 funkeln \n", "1007 firecracker \n", "1008 regionaler indikator symbol letter p \n", "1009 kreuzmarke \n", "1010 negative quadrierte kreuzmarke \n", "1011 jigsaw puzzle stΓΌck \n", "1012 regionaler indikator symbol letter q \n", "1013 schwarzes fragenzeichen-verzierung \n", "1014 weisses frage-zeichen-verzierung \n", "1015 weisses ausrufs-markierungs-verzierung \n", "1016 schweres ausrufzeichen symbol \n", "1017 testrohr \n", "1018 petrischale \n", "1019 schwere plus zeichen \n", "1020 schwere minuszeichen \n", "1021 schweres abteilungszeichen \n", "1022 garnele \n", "1023 tintenfisch \n", "1024 lockige schleife \n", "1025 doppelt lockige schleife \n", "1026 WERKZEUGKASTEN \n", "\n", "[1027 rows x 5 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "prepStep1Table = table\n", "\n", "stopwords_en = set(stopwords.words('english'))\n", "stopwords_de = set(stopwords.words('german'))\n", "\n", "# lowercasing and stopword removal\n", "for entry in range(len(table[\"description\"][1:])):\n", " table[\"description\"][entry] = table[\"description\"][entry].lower()\n", " newEntry = []\n", " for word in table[\"description\"][entry].split(\" \"):\n", " if word not in stopwords_en:\n", " newEntry.append(word)\n", " table[\"description\"][entry] = (\" \").join(newEntry).lower()\n", " \n", "# just copied block from above and adjusted for german column :)\n", "for entry in range(len(table[\"description_de\"][1:])):\n", " table[\"description_de\"][entry] = table[\"description_de\"][entry].lower()\n", " newEntry = []\n", " for word in table[\"description_de\"][entry].split(\" \"):\n", " if word not in stopwords_de:\n", " newEntry.append(word)\n", " table[\"description_de\"][entry] = (\" \").join(newEntry).lower()\n", " \n", "table" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }