{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read table `test.csv` exported by `whatsapp2csv.sh`" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetimepersonmessage
010/18/1611:18Lena CartaHallo Jonas πŸ˜‰
110/18/1611:21Jonas WeinzMoin moin ;) Vllt. bin ich aber gar nicht Jona...
210/18/1611:21Lena CartaπŸ˜±πŸ€•
310/18/1611:21Sonja VorwerkπŸ™ˆπŸ˜‚
410/18/1611:22Caner UcarπŸ˜‚πŸ˜‚πŸ˜‚
510/18/1611:22Patrick (Arithmeum)hallo jeff
610/18/1611:22Caner UcarPatrick kein Handy am Empfang
710/18/1611:26Patrick (Arithmeum)halts maul und geh ma duschen
810/18/1611:34Simon (Arithmeum)Hab dich als Carsten abgespeichert, selber Schuld
910/18/1611:36Caner UcarPatrick ich hab geduscht mit Kristina und Sonja v
1010/18/1611:36Lena CartaΓ€hm....
1110/18/1611:37Jonathan (Arithmeum)Caner benutzt das Damen-Bad
1210/18/1611:37Jonas WeinzLΓΌg doch nicht, du tanzt die ganze Zeit aufm T...
1310/18/1611:37Jonathan (Arithmeum)bÀÀh😱
1410/18/1611:40Patrick (Arithmeum)ja caner so ist das unter frauen..da geht man ...
1510/18/1612:02Thomas KoppelbergEeemmm was?
1610/18/1612:12Kristina StellwagThomas, du wolltest gestern ja nicht 😜
1710/18/1612:14Thomas KoppelbergIch wollte einiges, aber ich kann mich zurΓΌckh...
1810/18/1612:19Simon (Arithmeum)Genau πŸ™Š
1910/18/1613:36Caner UcarSex
\n", "
" ], "text/plain": [ " date time person \\\n", "0 10/18/16 11:18 Lena Carta \n", "1 10/18/16 11:21 Jonas Weinz \n", "2 10/18/16 11:21 Lena Carta \n", "3 10/18/16 11:21 Sonja Vorwerk \n", "4 10/18/16 11:22 Caner Ucar \n", "5 10/18/16 11:22 Patrick (Arithmeum) \n", "6 10/18/16 11:22 Caner Ucar \n", "7 10/18/16 11:26 Patrick (Arithmeum) \n", "8 10/18/16 11:34 Simon (Arithmeum) \n", "9 10/18/16 11:36 Caner Ucar \n", "10 10/18/16 11:36 Lena Carta \n", "11 10/18/16 11:37 Jonathan (Arithmeum) \n", "12 10/18/16 11:37 Jonas Weinz \n", "13 10/18/16 11:37 Jonathan (Arithmeum) \n", "14 10/18/16 11:40 Patrick (Arithmeum) \n", "15 10/18/16 12:02 Thomas Koppelberg \n", "16 10/18/16 12:12 Kristina Stellwag \n", "17 10/18/16 12:14 Thomas Koppelberg \n", "18 10/18/16 12:19 Simon (Arithmeum) \n", "19 10/18/16 13:36 Caner Ucar \n", "\n", " message \n", "0 Hallo Jonas πŸ˜‰ \n", "1 Moin moin ;) Vllt. bin ich aber gar nicht Jona... \n", "2 πŸ˜±πŸ€• \n", "3 πŸ™ˆπŸ˜‚ \n", "4 πŸ˜‚πŸ˜‚πŸ˜‚ \n", "5 hallo jeff \n", "6 Patrick kein Handy am Empfang \n", "7 halts maul und geh ma duschen \n", "8 Hab dich als Carsten abgespeichert, selber Schuld \n", "9 Patrick ich hab geduscht mit Kristina und Sonja v \n", "10 Γ€hm.... \n", "11 Caner benutzt das Damen-Bad \n", "12 LΓΌg doch nicht, du tanzt die ganze Zeit aufm T... \n", "13 bÀÀh😱 \n", "14 ja caner so ist das unter frauen..da geht man ... \n", "15 Eeemmm was? \n", "16 Thomas, du wolltest gestern ja nicht 😜 \n", "17 Ich wollte einiges, aber ich kann mich zurΓΌckh... \n", "18 Genau πŸ™Š \n", "19 Sex " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "messages = pd.read_csv('test.csv', delimiter='\\t')\n", "messages[:20]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "emoji_blacklist = set([\n", " 0x1F3FB,\n", " 0x1F3FC,\n", " 0x1F3FD,\n", " 0x1F3FE,\n", " 0x1F3FF\n", "])" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
unicodetype
228231A..231BEmoji_Presentation
22923E9..23ECEmoji_Presentation
23023F0Emoji_Presentation
23123F3Emoji_Presentation
23225FD..25FEEmoji_Presentation
2332614..2615Emoji_Presentation
2342648..2653Emoji_Presentation
235267FEmoji_Presentation
2362693Emoji_Presentation
23726A1Emoji_Presentation
23826AA..26ABEmoji_Presentation
23926BD..26BEEmoji_Presentation
24026C4..26C5Emoji_Presentation
24126CEEmoji_Presentation
24226D4Emoji_Presentation
24326EAEmoji_Presentation
24426F2..26F3Emoji_Presentation
24526F5Emoji_Presentation
24626FAEmoji_Presentation
24726FDEmoji_Presentation
2482705Emoji_Presentation
249270A..270BEmoji_Presentation
2502728Emoji_Presentation
251274CEmoji_Presentation
252274EEmoji_Presentation
2532753..2755Emoji_Presentation
2542757Emoji_Presentation
2552795..2797Emoji_Presentation
25627B0Emoji_Presentation
25727BFEmoji_Presentation
.........
3291F6F7..1F6F8Emoji_Presentation
3301F6F9Emoji_Presentation
3311F910..1F918Emoji_Presentation
3321F919..1F91EEmoji_Presentation
3331F91FEmoji_Presentation
3341F920..1F927Emoji_Presentation
3351F928..1F92FEmoji_Presentation
3361F930Emoji_Presentation
3371F931..1F932Emoji_Presentation
3381F933..1F93AEmoji_Presentation
3391F93C..1F93EEmoji_Presentation
3401F940..1F945Emoji_Presentation
3411F947..1F94BEmoji_Presentation
3421F94CEmoji_Presentation
3431F94D..1F94FEmoji_Presentation
3441F950..1F95EEmoji_Presentation
3451F95F..1F96BEmoji_Presentation
3461F96C..1F970Emoji_Presentation
3471F973..1F976Emoji_Presentation
3481F97AEmoji_Presentation
3491F97C..1F97FEmoji_Presentation
3501F980..1F984Emoji_Presentation
3511F985..1F991Emoji_Presentation
3521F992..1F997Emoji_Presentation
3531F998..1F9A2Emoji_Presentation
3541F9B0..1F9B9Emoji_Presentation
3551F9C0Emoji_Presentation
3561F9C1..1F9C2Emoji_Presentation
3571F9D0..1F9E6Emoji_Presentation
3581F9E7..1F9FFEmoji_Presentation
\n", "

131 rows Γ— 2 columns

\n", "
" ], "text/plain": [ " unicode type\n", "228 231A..231B Emoji_Presentation\n", "229 23E9..23EC Emoji_Presentation\n", "230 23F0 Emoji_Presentation\n", "231 23F3 Emoji_Presentation\n", "232 25FD..25FE Emoji_Presentation\n", "233 2614..2615 Emoji_Presentation\n", "234 2648..2653 Emoji_Presentation\n", "235 267F Emoji_Presentation\n", "236 2693 Emoji_Presentation\n", "237 26A1 Emoji_Presentation\n", "238 26AA..26AB Emoji_Presentation\n", "239 26BD..26BE Emoji_Presentation\n", "240 26C4..26C5 Emoji_Presentation\n", "241 26CE Emoji_Presentation\n", "242 26D4 Emoji_Presentation\n", "243 26EA Emoji_Presentation\n", "244 26F2..26F3 Emoji_Presentation\n", "245 26F5 Emoji_Presentation\n", "246 26FA Emoji_Presentation\n", "247 26FD Emoji_Presentation\n", "248 2705 Emoji_Presentation\n", "249 270A..270B Emoji_Presentation\n", "250 2728 Emoji_Presentation\n", "251 274C Emoji_Presentation\n", "252 274E Emoji_Presentation\n", "253 2753..2755 Emoji_Presentation\n", "254 2757 Emoji_Presentation\n", "255 2795..2797 Emoji_Presentation\n", "256 27B0 Emoji_Presentation\n", "257 27BF Emoji_Presentation\n", ".. ... ...\n", "329 1F6F7..1F6F8 Emoji_Presentation\n", "330 1F6F9 Emoji_Presentation\n", "331 1F910..1F918 Emoji_Presentation\n", "332 1F919..1F91E Emoji_Presentation\n", "333 1F91F Emoji_Presentation\n", "334 1F920..1F927 Emoji_Presentation\n", "335 1F928..1F92F Emoji_Presentation\n", "336 1F930 Emoji_Presentation\n", "337 1F931..1F932 Emoji_Presentation\n", "338 1F933..1F93A Emoji_Presentation\n", "339 1F93C..1F93E Emoji_Presentation\n", "340 1F940..1F945 Emoji_Presentation\n", "341 1F947..1F94B Emoji_Presentation\n", "342 1F94C Emoji_Presentation\n", "343 1F94D..1F94F Emoji_Presentation\n", "344 1F950..1F95E Emoji_Presentation\n", "345 1F95F..1F96B Emoji_Presentation\n", "346 1F96C..1F970 Emoji_Presentation\n", "347 1F973..1F976 Emoji_Presentation\n", "348 1F97A Emoji_Presentation\n", "349 1F97C..1F97F Emoji_Presentation\n", "350 1F980..1F984 Emoji_Presentation\n", "351 1F985..1F991 Emoji_Presentation\n", "352 1F992..1F997 Emoji_Presentation\n", "353 1F998..1F9A2 Emoji_Presentation\n", "354 1F9B0..1F9B9 Emoji_Presentation\n", "355 1F9C0 Emoji_Presentation\n", "356 1F9C1..1F9C2 Emoji_Presentation\n", "357 1F9D0..1F9E6 Emoji_Presentation\n", "358 1F9E7..1F9FF Emoji_Presentation\n", "\n", "[131 rows x 2 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n", "emoji_data['type'] = emoji_data['type'].str.strip()\n", "emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n", "emoji_data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* now build a set out of the unicode types" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ord(\"πŸ˜€\") == int('0x1f600',16)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "228 231A..231B \n", "229 23E9..23EC \n", "230 23F0 \n", "231 23F3 \n", "232 25FD..25FE \n", "Name: unicode, dtype: object" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "emoji_codes = emoji_data['unicode']\n", "emoji_codes.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* we have to iterate over the whole list and extract all given ranges:" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "emoji_code_list = []\n", "for entry in emoji_codes:\n", " # testing whether we have an entry or a range:\n", " if '.' in entry:\n", " # range\n", " a,b = entry.split(\"..\")\n", " for i in range(int(a,16),int(b,16) +1):\n", " if i not in emoji_blacklist:\n", " emoji_code_list.append(i)\n", " else:\n", " # single entry\n", " if i not in emoji_blacklist:\n", " emoji_code_list.append(int(entry,16))\n", "emoji_code_set = set(emoji_code_list)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "True False\n" ] } ], "source": [ "# simple test:\n", "print(ord(\"πŸ˜€\") in emoji_code_set, ord(\"a\") in emoji_code_set)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* expanding column and fill new emojis" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "messages[\"emojis\"] = None" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "for i in messages.index:\n", " emoji_list = []\n", " to_remove = []\n", " m = messages.iloc[i]['message']\n", " for c in str(m):\n", " if ord(c) in emoji_code_set:\n", " emoji_list.append(c)\n", " elif ord(c) in emoji_blacklist:\n", " to_remove.append(c)\n", " \n", " messages.loc[i,'emojis'] = emoji_list\n", " #remove emiΓ³jis from message\n", " for e in (emoji_list + to_remove):\n", " m = m.replace(e,\"\")\n", " messages.loc[i,'message'] = m\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetimepersonmessageemojis
010/18/1611:18Lena CartaHallo Jonas[πŸ˜‰]
110/18/1611:21Jonas WeinzMoin moin ;) Vllt. bin ich aber gar nicht Jona...[πŸ€”]
210/18/1611:21Lena Carta[😱, πŸ€•]
310/18/1611:21Sonja Vorwerk[πŸ™ˆ, πŸ˜‚]
410/18/1611:22Caner Ucar[πŸ˜‚, πŸ˜‚, πŸ˜‚]
510/18/1611:22Patrick (Arithmeum)hallo jeff[]
610/18/1611:22Caner UcarPatrick kein Handy am Empfang[]
710/18/1611:26Patrick (Arithmeum)halts maul und geh ma duschen[]
810/18/1611:34Simon (Arithmeum)Hab dich als Carsten abgespeichert, selber Schuld[]
910/18/1611:36Caner UcarPatrick ich hab geduscht mit Kristina und Sonja v[]
1010/18/1611:36Lena CartaΓ€hm....[]
1110/18/1611:37Jonathan (Arithmeum)Caner benutzt das Damen-Bad[]
1210/18/1611:37Jonas WeinzLΓΌg doch nicht, du tanzt die ganze Zeit aufm T...[]
1310/18/1611:37Jonathan (Arithmeum)bÀÀh[😱]
1410/18/1611:40Patrick (Arithmeum)ja caner so ist das unter frauen..da geht man ...[]
1510/18/1612:02Thomas KoppelbergEeemmm was?[]
1610/18/1612:12Kristina StellwagThomas, du wolltest gestern ja nicht[😜]
1710/18/1612:14Thomas KoppelbergIch wollte einiges, aber ich kann mich zurΓΌckh...[]
1810/18/1612:19Simon (Arithmeum)Genau[πŸ™Š]
1910/18/1613:36Caner UcarSex[]
\n", "
" ], "text/plain": [ " date time person \\\n", "0 10/18/16 11:18 Lena Carta \n", "1 10/18/16 11:21 Jonas Weinz \n", "2 10/18/16 11:21 Lena Carta \n", "3 10/18/16 11:21 Sonja Vorwerk \n", "4 10/18/16 11:22 Caner Ucar \n", "5 10/18/16 11:22 Patrick (Arithmeum) \n", "6 10/18/16 11:22 Caner Ucar \n", "7 10/18/16 11:26 Patrick (Arithmeum) \n", "8 10/18/16 11:34 Simon (Arithmeum) \n", "9 10/18/16 11:36 Caner Ucar \n", "10 10/18/16 11:36 Lena Carta \n", "11 10/18/16 11:37 Jonathan (Arithmeum) \n", "12 10/18/16 11:37 Jonas Weinz \n", "13 10/18/16 11:37 Jonathan (Arithmeum) \n", "14 10/18/16 11:40 Patrick (Arithmeum) \n", "15 10/18/16 12:02 Thomas Koppelberg \n", "16 10/18/16 12:12 Kristina Stellwag \n", "17 10/18/16 12:14 Thomas Koppelberg \n", "18 10/18/16 12:19 Simon (Arithmeum) \n", "19 10/18/16 13:36 Caner Ucar \n", "\n", " message emojis \n", "0 Hallo Jonas [πŸ˜‰] \n", "1 Moin moin ;) Vllt. bin ich aber gar nicht Jona... [πŸ€”] \n", "2 [😱, πŸ€•] \n", "3 [πŸ™ˆ, πŸ˜‚] \n", "4 [πŸ˜‚, πŸ˜‚, πŸ˜‚] \n", "5 hallo jeff [] \n", "6 Patrick kein Handy am Empfang [] \n", "7 halts maul und geh ma duschen [] \n", "8 Hab dich als Carsten abgespeichert, selber Schuld [] \n", "9 Patrick ich hab geduscht mit Kristina und Sonja v [] \n", "10 Γ€hm.... [] \n", "11 Caner benutzt das Damen-Bad [] \n", "12 LΓΌg doch nicht, du tanzt die ganze Zeit aufm T... [] \n", "13 bÀÀh [😱] \n", "14 ja caner so ist das unter frauen..da geht man ... [] \n", "15 Eeemmm was? [] \n", "16 Thomas, du wolltest gestern ja nicht [😜] \n", "17 Ich wollte einiges, aber ich kann mich zurΓΌckh... [] \n", "18 Genau [πŸ™Š] \n", "19 Sex [] " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "messages[:20]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* get a list only containing messaged with emojis" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n", "emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetimepersonmessageemojis
010/18/1611:18Lena CartaHallo Jonas[πŸ˜‰]
110/18/1611:21Jonas WeinzMoin moin ;) Vllt. bin ich aber gar nicht Jona...[πŸ€”]
1310/18/1611:37Jonathan (Arithmeum)bÀÀh[😱]
1610/18/1612:12Kristina StellwagThomas, du wolltest gestern ja nicht[😜]
1810/18/1612:19Simon (Arithmeum)Genau[πŸ™Š]
2910/18/1618:38Lena Cartawas du bist morgen nicht da ???[😱, 😯, πŸ˜”]
3110/18/1618:39Lena Cartaich dachte diese Woche komplett noch. Aber gut...[πŸ˜‡]
3210/18/1618:40Kristina StellwagNee das Semester hat doch angefangen[😜]
3310/18/1618:42Lena Cartaja hatte ich gerade vergessen . Dabei hatte ic...[πŸ˜‚, 😢]
4110/20/1612:50Steffi (Arithmeum)Klar[πŸ‘Ή]
4410/20/1617:54Steffi (Arithmeum)Mein Zug lÀsst sich auch heute Zeit[😞]
4510/20/1618:08Steffi (Arithmeum)Wir fangen doch wahrscheinlich nicht vor halb ...[😳]
5110/20/1620:58Anna BorutzkyIst das dein Ernst?[πŸ™„]
5210/20/1620:58Patrick (Arithmeum)Habs voll vercheckt sry ..gerade erst gemerkt[πŸ™„]
7010/20/1622:59Jonathan (Arithmeum)Ich bitte darum Caner....[😍, 😍, 😍]
7810/25/1601:03Caner UcarHaaaaaalloooo[πŸ€—, πŸ€—, πŸ€—, πŸ€—, πŸ€—, πŸ€—, πŸ€—, 😏, πŸ€—, πŸ€—, 😏]
7910/25/1601:04Michael (Arithmeum)Keine sorge, das hat bei dir nichts mit der St...[πŸ˜‰]
8810/25/1609:02Lena Cartaich wollte eigentlich schon da sein, aber ich ...[πŸ˜‡]
8910/25/1609:04Jonathan (Arithmeum)SIIIIEG...βš”[πŸŽ‰]
9010/25/1610:00Lena Cartabesser spÀt als nie. ich bin immerhin auf dem ...[😁]
9310/25/1621:49Jonathan (Arithmeum)Ich nicht[πŸ˜„]
9410/25/1621:50Lena Cartaich auch nicht[😳]
9510/25/1622:13Konstantin (Arithmeum)Wer hat den zuletzt was laminiert? lena und i...[πŸ€”]
9610/25/1623:28Caner UcarIch auch nicht[πŸ€”]
10210/27/1614:43Sonja Vorwerkhm mΓΌsste das am besten heute schon wissen...w...[πŸ™ˆ, πŸ˜„]
11510/28/1608:32Lena CartaGuten Morgen liebe Kollegen, kΓΆnnte einer von ...[πŸ™]
11610/28/1609:48Lena CartaIst noch niemand im BΓΌro?[πŸ˜‡]
11710/28/1609:50Caner Ucar10:10 ca.[πŸ™Œ]
11810/28/1609:51Caner UcarWann hat das woki geΓΆffnet[πŸ˜„]
12110/28/1610:54Lena CartaDa hÀtte ich jetzt auch nicht gesucht[😝]
..................
86635/9/1817:42Jonas Weinzmehrere BΓΌrostΓΌhle hintereinander?[πŸ€”]
86655/9/1817:43Lena CartaWollte Kristina das nicht immer schonmal machen[😁]
86755/10/1814:06Lena KlΓΆcknerErledigt[πŸ‘]
86765/10/1814:07Caner UcarDanke[😊]
86775/10/1814:23Anna ZensIch hatte es auch schon erledigt[πŸ˜„]
86785/11/1808:16Lena CartaGute morgen, ich habe gesehen neben dem Buch l...[πŸ™‚]
86795/11/1808:25Anna ZensDie handschriftlichen sind die, von denen kein...[πŸ˜„]
86805/11/1808:26Lena Cartaachso ich denke mal dann wird sich Ingo melden[πŸ˜€, πŸ˜€]
86815/11/1813:18Freya (Arithmeum)Mein Kalender funktioniert nicht - kΓΆnnte mir ...[πŸ€”]
86825/12/1814:53Freya (Arithmeum)KΓΆnnte jemand fΓΌr Montag Abend zwei Karten fΓΌr...[πŸ˜‡]
86835/12/1815:05Anna BorutzkyUnd eine Freikarte fΓΌr Sonja bitte![πŸ˜€]
86895/12/1815:16Lena CartaDanke Wieso kommen solche Fragen immer am Woc...[πŸ˜€]
86925/12/1815:17Anna BorutzkyMurphyβ€˜s Gesetz?[πŸ˜‰]
86935/12/1815:17Caner UcarCool[😍]
86955/12/1815:23Jonas Weinz...und die kommt auf Annas Tisch[πŸ˜‰]
86965/12/1815:23Freya (Arithmeum)Vielen, vielen Dank ☺ Sowas passiert immer am ...[πŸ™ˆ]
86975/12/1815:23Lena CartaDas war nicht auf dich bezogen .[πŸ˜€]
86985/12/1815:24Freya (Arithmeum)Na gut[πŸ˜„]
86995/12/1815:24Lena CartaEigentlich ist es halbwegs entspannt heute Ic...[πŸ˜€, πŸ˜›]
87005/12/1815:25Freya (Arithmeum)Es gibt keine doofen Fragen[πŸ˜‹]
87025/12/1815:25Lena Cartahttps://www.youtube.com/watch?v=uPHi5xn_q5c[πŸ˜›]
87085/12/1817:56Freya (Arithmeum)Ich mΓΆchte arbeiten und habe mega Bock drauf! ...[πŸ˜‡]
87195/12/1820:09Lena Cartabin leider verabredet .... mit Thilo[😬, πŸ€—]
87255/13/1817:24Thomas Koppelbergβ™₯[😍]
87265/14/1817:00Sonja SchÀferKann mir jemand noch eine zweite Freikarte res...[😊]
87395/14/1822:40Jonas WeinzHab Ingo beim vorbeilaufenden nicht gesehen, h...[πŸ’]
87445/15/1817:16Caner Ucar‍♀[🀷]
87555/17/1810:43Lena Cartasuper perfekt dann ist beiden geholfen[πŸ™‚, 😎]
87595/17/1810:43Lena Cartaob das so sinnvoll ist?[πŸ˜€]
87635/17/1810:45Lena Cartaich dachte es ging konkret um den Termin[πŸ˜€]
\n", "

2455 rows Γ— 5 columns

\n", "
" ], "text/plain": [ " date time person \\\n", "0 10/18/16 11:18 Lena Carta \n", "1 10/18/16 11:21 Jonas Weinz \n", "13 10/18/16 11:37 Jonathan (Arithmeum) \n", "16 10/18/16 12:12 Kristina Stellwag \n", "18 10/18/16 12:19 Simon (Arithmeum) \n", "29 10/18/16 18:38 Lena Carta \n", "31 10/18/16 18:39 Lena Carta \n", "32 10/18/16 18:40 Kristina Stellwag \n", "33 10/18/16 18:42 Lena Carta \n", "41 10/20/16 12:50 Steffi (Arithmeum) \n", "44 10/20/16 17:54 Steffi (Arithmeum) \n", "45 10/20/16 18:08 Steffi (Arithmeum) \n", "51 10/20/16 20:58 Anna Borutzky \n", "52 10/20/16 20:58 Patrick (Arithmeum) \n", "70 10/20/16 22:59 Jonathan (Arithmeum) \n", "78 10/25/16 01:03 Caner Ucar \n", "79 10/25/16 01:04 Michael (Arithmeum) \n", "88 10/25/16 09:02 Lena Carta \n", "89 10/25/16 09:04 Jonathan (Arithmeum) \n", "90 10/25/16 10:00 Lena Carta \n", "93 10/25/16 21:49 Jonathan (Arithmeum) \n", "94 10/25/16 21:50 Lena Carta \n", "95 10/25/16 22:13 Konstantin (Arithmeum) \n", "96 10/25/16 23:28 Caner Ucar \n", "102 10/27/16 14:43 Sonja Vorwerk \n", "115 10/28/16 08:32 Lena Carta \n", "116 10/28/16 09:48 Lena Carta \n", "117 10/28/16 09:50 Caner Ucar \n", "118 10/28/16 09:51 Caner Ucar \n", "121 10/28/16 10:54 Lena Carta \n", "... ... ... ... \n", "8663 5/9/18 17:42 Jonas Weinz \n", "8665 5/9/18 17:43 Lena Carta \n", "8675 5/10/18 14:06 Lena KlΓΆckner \n", "8676 5/10/18 14:07 Caner Ucar \n", "8677 5/10/18 14:23 Anna Zens \n", "8678 5/11/18 08:16 Lena Carta \n", "8679 5/11/18 08:25 Anna Zens \n", "8680 5/11/18 08:26 Lena Carta \n", "8681 5/11/18 13:18 Freya (Arithmeum) \n", "8682 5/12/18 14:53 Freya (Arithmeum) \n", "8683 5/12/18 15:05 Anna Borutzky \n", "8689 5/12/18 15:16 Lena Carta \n", "8692 5/12/18 15:17 Anna Borutzky \n", "8693 5/12/18 15:17 Caner Ucar \n", "8695 5/12/18 15:23 Jonas Weinz \n", "8696 5/12/18 15:23 Freya (Arithmeum) \n", "8697 5/12/18 15:23 Lena Carta \n", "8698 5/12/18 15:24 Freya (Arithmeum) \n", "8699 5/12/18 15:24 Lena Carta \n", "8700 5/12/18 15:25 Freya (Arithmeum) \n", "8702 5/12/18 15:25 Lena Carta \n", "8708 5/12/18 17:56 Freya (Arithmeum) \n", "8719 5/12/18 20:09 Lena Carta \n", "8725 5/13/18 17:24 Thomas Koppelberg \n", "8726 5/14/18 17:00 Sonja SchΓ€fer \n", "8739 5/14/18 22:40 Jonas Weinz \n", "8744 5/15/18 17:16 Caner Ucar \n", "8755 5/17/18 10:43 Lena Carta \n", "8759 5/17/18 10:43 Lena Carta \n", "8763 5/17/18 10:45 Lena Carta \n", "\n", " message \\\n", "0 Hallo Jonas \n", "1 Moin moin ;) Vllt. bin ich aber gar nicht Jona... \n", "13 bÀÀh \n", "16 Thomas, du wolltest gestern ja nicht \n", "18 Genau \n", "29 was du bist morgen nicht da ??? \n", "31 ich dachte diese Woche komplett noch. Aber gut... \n", "32 Nee das Semester hat doch angefangen \n", "33 ja hatte ich gerade vergessen . Dabei hatte ic... \n", "41 Klar \n", "44 Mein Zug lΓ€sst sich auch heute Zeit \n", "45 Wir fangen doch wahrscheinlich nicht vor halb ... \n", "51 Ist das dein Ernst? \n", "52 Habs voll vercheckt sry ..gerade erst gemerkt \n", "70 Ich bitte darum Caner.... \n", "78 Haaaaaalloooo \n", "79 Keine sorge, das hat bei dir nichts mit der St... \n", "88 ich wollte eigentlich schon da sein, aber ich ... \n", "89 SIIIIEG...βš” \n", "90 besser spΓ€t als nie. ich bin immerhin auf dem ... \n", "93 Ich nicht \n", "94 ich auch nicht \n", "95 Wer hat den zuletzt was laminiert? lena und i... \n", "96 Ich auch nicht \n", "102 hm mΓΌsste das am besten heute schon wissen...w... \n", "115 Guten Morgen liebe Kollegen, kΓΆnnte einer von ... \n", "116 Ist noch niemand im BΓΌro? \n", "117 10:10 ca. \n", "118 Wann hat das woki geΓΆffnet \n", "121 Da hΓ€tte ich jetzt auch nicht gesucht \n", "... ... \n", "8663 mehrere BΓΌrostΓΌhle hintereinander? \n", "8665 Wollte Kristina das nicht immer schonmal machen \n", "8675 Erledigt \n", "8676 Danke \n", "8677 Ich hatte es auch schon erledigt \n", "8678 Gute morgen, ich habe gesehen neben dem Buch l... \n", "8679 Die handschriftlichen sind die, von denen kein... \n", "8680 achso ich denke mal dann wird sich Ingo melden \n", "8681 Mein Kalender funktioniert nicht - kΓΆnnte mir ... \n", "8682 KΓΆnnte jemand fΓΌr Montag Abend zwei Karten fΓΌr... \n", "8683 Und eine Freikarte fΓΌr Sonja bitte! \n", "8689 Danke Wieso kommen solche Fragen immer am Woc... \n", "8692 Murphyβ€˜s Gesetz? \n", "8693 Cool \n", "8695 ...und die kommt auf Annas Tisch \n", "8696 Vielen, vielen Dank ☺ Sowas passiert immer am ... \n", "8697 Das war nicht auf dich bezogen . \n", "8698 Na gut \n", "8699 Eigentlich ist es halbwegs entspannt heute Ic... \n", "8700 Es gibt keine doofen Fragen \n", "8702 https://www.youtube.com/watch?v=uPHi5xn_q5c \n", "8708 Ich mΓΆchte arbeiten und habe mega Bock drauf! ... \n", "8719 bin leider verabredet .... mit Thilo \n", "8725 β™₯ \n", "8726 Kann mir jemand noch eine zweite Freikarte res... \n", "8739 Hab Ingo beim vorbeilaufenden nicht gesehen, h... \n", "8744 ‍♀ \n", "8755 super perfekt dann ist beiden geholfen \n", "8759 ob das so sinnvoll ist? \n", "8763 ich dachte es ging konkret um den Termin \n", "\n", " emojis \n", "0 [πŸ˜‰] \n", "1 [πŸ€”] \n", "13 [😱] \n", "16 [😜] \n", "18 [πŸ™Š] \n", "29 [😱, 😯, πŸ˜”] \n", "31 [πŸ˜‡] \n", "32 [😜] \n", "33 [πŸ˜‚, 😢] \n", "41 [πŸ‘Ή] \n", "44 [😞] \n", "45 [😳] \n", "51 [πŸ™„] \n", "52 [πŸ™„] \n", "70 [😍, 😍, 😍] \n", "78 [πŸ€—, πŸ€—, πŸ€—, πŸ€—, πŸ€—, πŸ€—, πŸ€—, 😏, πŸ€—, πŸ€—, 😏] \n", "79 [πŸ˜‰] \n", "88 [πŸ˜‡] \n", "89 [πŸŽ‰] \n", "90 [😁] \n", "93 [πŸ˜„] \n", "94 [😳] \n", "95 [πŸ€”] \n", "96 [πŸ€”] \n", "102 [πŸ™ˆ, πŸ˜„] \n", "115 [πŸ™] \n", "116 [πŸ˜‡] \n", "117 [πŸ™Œ] \n", "118 [πŸ˜„] \n", "121 [😝] \n", "... ... \n", "8663 [πŸ€”] \n", "8665 [😁] \n", "8675 [πŸ‘] \n", "8676 [😊] \n", "8677 [πŸ˜„] \n", "8678 [πŸ™‚] \n", "8679 [πŸ˜„] \n", "8680 [πŸ˜€, πŸ˜€] \n", "8681 [πŸ€”] \n", "8682 [πŸ˜‡] \n", "8683 [πŸ˜€] \n", "8689 [πŸ˜€] \n", "8692 [πŸ˜‰] \n", "8693 [😍] \n", "8695 [πŸ˜‰] \n", "8696 [πŸ™ˆ] \n", "8697 [πŸ˜€] \n", "8698 [πŸ˜„] \n", "8699 [πŸ˜€, πŸ˜›] \n", "8700 [πŸ˜‹] \n", "8702 [πŸ˜›] \n", "8708 [πŸ˜‡] \n", "8719 [😬, πŸ€—] \n", "8725 [😍] \n", "8726 [😊] \n", "8739 [πŸ’] \n", "8744 [🀷] \n", "8755 [πŸ™‚, 😎] \n", "8759 [πŸ˜€] \n", "8763 [πŸ˜€] \n", "\n", "[2455 rows x 5 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(emoji_messages)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## learning part" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import itertools\n", "import sklearn.utils as sku\n", "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "labels=[e[-1] for e in emoji_messages['emojis']]" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['πŸ˜‰', 'πŸ€”', '😱', '😜', 'πŸ™Š', 'πŸ˜”', 'πŸ˜‡', '😜', '😢', 'πŸ‘Ή']" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "labels[:10]" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "vectorizer = TfidfVectorizer(stop_words='english')\n", "vec_train = vectorizer.fit_transform(X1)\n", "vec_test = vectorizer.transform(Xt1)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.6/site-packages/sklearn/neural_network/multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", " % self.max_iter, ConvergenceWarning)\n" ] }, { "data": { "text/plain": [ "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n", " beta_2=0.999, early_stopping=False, epsilon=1e-08,\n", " hidden_layer_sizes=(100,), learning_rate='constant',\n", " learning_rate_init=0.001, max_iter=200, momentum=0.9,\n", " nesterovs_momentum=True, power_t=0.5, random_state=None,\n", " shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n", " verbose=False, warm_start=False)" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.ensemble import RandomForestClassifier as RFC\n", "from sklearn.neural_network import MLPClassifier as MLP\n", "from sklearn.naive_bayes import MultinomialNB as MNB\n", "#clf_a = RFC(criterion='entropy', random_state=4222)\n", "clf_a = MLP()\n", "clf_a.fit(vec_train, y1)" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "pred = clf_a.predict(vectorizer.transform(Xt1))\n" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
messagepredtrainedemojis
8663mehrere BΓΌrostΓΌhle hintereinander?πŸ™ˆπŸ€”[πŸ€”]
1862Ich will auch EssenπŸ˜‰πŸ˜ž[😞]
39212x orangeπŸ™‚πŸ™‚[πŸ™‚]
2222Man soll auch keine Lernsachen da lassenπŸ˜€πŸ˜€[πŸ˜€]
761Wir kΓΆnnen auch gerne wieder bei mir zusammen ...πŸ˜‡πŸ˜‰[πŸ˜‰]
3600Hab den Bus verpasst, den ich eigentlich nehme...πŸ˜ŸπŸ‘Έ[πŸ‘Έ]
1794Klaro, gerneπŸ˜ŽπŸ™‚[πŸ™‚]
861Sehr gut, Jan! Wenn sich noch jemand melden wΓΌ...πŸ˜˜πŸ˜‡[πŸ˜‡]
1469Jaaa aber mein Mitbewohner ist messyπŸ˜¬πŸ™Š[😱, πŸ˜”, 🌚, πŸ™Š]
6254HallΓΆchen Es hat ein Herr angerufen, der gern...πŸ’πŸ™‚[πŸ™‚]
7968IwoπŸ€¦πŸ˜‚[πŸ˜‚]
8102Keine Ahnung wer da haariger istπŸ˜…πŸ§[🧐]
6835Danke😊😊[😊]
7135Wurde bestimmt von Big Bang geklautπŸ€·πŸ˜‚[πŸ˜‚, πŸ˜‚, πŸ˜‚]
4556Ich kann leider nicht, aber arbeiten da nicht ...πŸ˜‚πŸ˜€[πŸ˜€]
6705Ist von heute, aber halt 3x in englischπŸ˜‡πŸ™„[πŸ™„]
2058..πŸ€¦πŸ˜…[πŸ˜…]
2072Oder fang endlich mal wieder an, was fΓΌrs Stud...πŸ˜„πŸ˜ˆ[😈]
756Ich hab vom letzten Jahr noch ein TraumaπŸ˜‚πŸ˜†[πŸ˜†]
5966Ist Freitag schon Wochenende?πŸ˜πŸ€”[πŸ€”]
7007Wenn wir die Klimaanlage im Museum ausschalten...πŸ€”πŸ€ͺ[πŸ€ͺ]
6968Ein Vorbild fΓΌr uns alleπŸ€”πŸ€©[🀩]
5665Kein Problem. Du kannst mich eintragen.πŸ™‚πŸ™‚[πŸ™‚]
2641Jonas, du bist gelb markiert! Du kannst keine ...πŸ˜‰πŸ™ƒ[πŸ™ƒ]
1981Da bin ich mir sehr sicher!πŸ˜˜πŸ˜„[πŸ˜„]
3334Das wΓ€re auf jeden Fall hilfreich! DankeπŸ˜πŸ™‚[πŸ™‚]
4447Ist das Schild denn auch laminiert?!πŸ€”πŸ˜[😏]
288da bin ich mal gespannt wie das aussieht, wen...πŸ˜‚πŸ‘[πŸ‘]
4265Aber es ist doch gebrochen...πŸ€”πŸ˜’[😒]
7405Partey🀦πŸ€ͺ[πŸ€ͺ]
...............
4910Vllt gilt das Schild auch nur fΓΌr schwarze Hun...πŸ˜‚πŸ˜±[😱]
2047Ich erprobe mich morgen mit Freya an Addieren ...πŸ™‚πŸ˜„[πŸ˜„]
2650Ich kann leider nichtπŸ˜•πŸ˜•[πŸ˜•]
299Das schafft meine Handykamera aber nichtπŸ˜‚πŸ˜‰[πŸ˜‰]
2499also ihr lieben ich arbeite gerade im arithmeu...πŸ˜πŸ˜…[πŸ˜…]
4783dankeπŸ˜ŠπŸ’Έ[πŸ’Έ, πŸ’Έ, πŸ’Έ]
4040HerrlichπŸ€¦πŸ˜…[πŸ€—, πŸ˜…]
7553babbage, meine Autokerrektur β€β™‚πŸ€”πŸ€¦[🀦]
6232Bin erst um 9.15Uhr da.....πŸ‘πŸ”ͺ[πŸ’£, πŸ”ͺ]
3514Dankeschân! Dann trage ich dich ein😘😊[😊]
3422SchnitzelπŸ˜…πŸ˜[😍]
2294Club discreto?πŸ˜‚πŸ˜³[😳, 😳, 😳]
8218auch da bin ich etwas neidischπŸ™„πŸ˜…[πŸ˜…]
4463MΓΆchte jemand was vom CafΓ© Roller?πŸ€§πŸ˜‚[πŸ˜‚]
8032Frohe Ostern an Euch alle...😊πŸ₯[🐰, 🐣, πŸ₯]
4505okay, ihr geht eigentlich ganz verantwortungsv...πŸ˜ŠπŸ’[πŸ’]
4810Meine Meinung: Wenn das Personal da ist, dann ...πŸ˜‚πŸ€·[🀷]
5762Ein shk ist aber billigerπŸ‘΅πŸ™„[πŸ™„]
4247Alle die morgen auf der Arbeit sind: nachher k...πŸ™ˆπŸ˜‚[πŸ˜‚]
2501ich glaube das wÀre eine stilvolle und romanti...😍😊[😊]
5278Ich kann das machenπŸ’βœ‹[βœ‹]
5747In der KΓΌche im KΓΌhlschrank steht eine grΓΌne T...πŸ˜‰πŸ™‚[πŸ˜‰, πŸ™‚]
7022Dann halt nicht β€β™‚πŸ€·πŸ€·[🀷]
1736Ich hab absolut keine Ahnung davonπŸ˜‚πŸ˜ž[😞]
5909Wir bekommen es bestimmt zu zweit hin, aber we...πŸ™ˆπŸ˜‰[πŸ˜‰]
5890Pass bloß auf, wenn du die isst mutierst du be...πŸ˜„πŸ˜±[😱]
566Das sind 5 Maß pro AbendπŸ™„πŸ˜‹[πŸ˜‹]
6044Sind sehr leckerπŸ˜€πŸ‘Œ[πŸ‘Œ]
1472Ich werde die alle drillen das sage ich dir, a...πŸ€”πŸ˜‡[πŸ˜‡]
5702Oh man, das wird ein TagπŸ˜€πŸ˜„[πŸ˜„]
\n", "

246 rows Γ— 4 columns

\n", "
" ], "text/plain": [ " message pred trained \\\n", "8663 mehrere BΓΌrostΓΌhle hintereinander? πŸ™ˆ πŸ€” \n", "1862 Ich will auch Essen πŸ˜‰ 😞 \n", "3921 2x orange πŸ™‚ πŸ™‚ \n", "2222 Man soll auch keine Lernsachen da lassen πŸ˜€ πŸ˜€ \n", "761 Wir kΓΆnnen auch gerne wieder bei mir zusammen ... πŸ˜‡ πŸ˜‰ \n", "3600 Hab den Bus verpasst, den ich eigentlich nehme... 😟 πŸ‘Έ \n", "1794 Klaro, gerne 😎 πŸ™‚ \n", "861 Sehr gut, Jan! Wenn sich noch jemand melden wΓΌ... 😘 πŸ˜‡ \n", "1469 Jaaa aber mein Mitbewohner ist messy 😬 πŸ™Š \n", "6254 HallΓΆchen Es hat ein Herr angerufen, der gern... πŸ’ πŸ™‚ \n", "7968 Iwo 🀦 πŸ˜‚ \n", "8102 Keine Ahnung wer da haariger ist πŸ˜… 🧐 \n", "6835 Danke 😊 😊 \n", "7135 Wurde bestimmt von Big Bang geklaut 🀷 πŸ˜‚ \n", "4556 Ich kann leider nicht, aber arbeiten da nicht ... πŸ˜‚ πŸ˜€ \n", "6705 Ist von heute, aber halt 3x in englisch πŸ˜‡ πŸ™„ \n", "2058 .. 🀦 πŸ˜… \n", "2072 Oder fang endlich mal wieder an, was fΓΌrs Stud... πŸ˜„ 😈 \n", "756 Ich hab vom letzten Jahr noch ein Trauma πŸ˜‚ πŸ˜† \n", "5966 Ist Freitag schon Wochenende? 😍 πŸ€” \n", "7007 Wenn wir die Klimaanlage im Museum ausschalten... πŸ€” πŸ€ͺ \n", "6968 Ein Vorbild fΓΌr uns alle πŸ€” 🀩 \n", "5665 Kein Problem. Du kannst mich eintragen. πŸ™‚ πŸ™‚ \n", "2641 Jonas, du bist gelb markiert! Du kannst keine ... πŸ˜‰ πŸ™ƒ \n", "1981 Da bin ich mir sehr sicher! 😘 πŸ˜„ \n", "3334 Das wΓ€re auf jeden Fall hilfreich! Danke 😍 πŸ™‚ \n", "4447 Ist das Schild denn auch laminiert?! πŸ€” 😏 \n", "288 da bin ich mal gespannt wie das aussieht, wen... πŸ˜‚ πŸ‘ \n", "4265 Aber es ist doch gebrochen... πŸ€” 😒 \n", "7405 Partey 🀦 πŸ€ͺ \n", "... ... ... ... \n", "4910 Vllt gilt das Schild auch nur fΓΌr schwarze Hun... πŸ˜‚ 😱 \n", "2047 Ich erprobe mich morgen mit Freya an Addieren ... πŸ™‚ πŸ˜„ \n", "2650 Ich kann leider nicht πŸ˜• πŸ˜• \n", "299 Das schafft meine Handykamera aber nicht πŸ˜‚ πŸ˜‰ \n", "2499 also ihr lieben ich arbeite gerade im arithmeu... 😁 πŸ˜… \n", "4783 danke 😊 πŸ’Έ \n", "4040 Herrlich 🀦 πŸ˜… \n", "7553 babbage, meine Autokerrektur ‍♂ πŸ€” 🀦 \n", "6232 Bin erst um 9.15Uhr da..... πŸ‘ πŸ”ͺ \n", "3514 DankeschΓΆn! Dann trage ich dich ein 😘 😊 \n", "3422 Schnitzel πŸ˜… 😍 \n", "2294 Club discreto? πŸ˜‚ 😳 \n", "8218 auch da bin ich etwas neidisch πŸ™„ πŸ˜… \n", "4463 MΓΆchte jemand was vom CafΓ© Roller? 🀧 πŸ˜‚ \n", "8032 Frohe Ostern an Euch alle... 😊 πŸ₯ \n", "4505 okay, ihr geht eigentlich ganz verantwortungsv... 😊 πŸ’ \n", "4810 Meine Meinung: Wenn das Personal da ist, dann ... πŸ˜‚ 🀷 \n", "5762 Ein shk ist aber billiger πŸ‘΅ πŸ™„ \n", "4247 Alle die morgen auf der Arbeit sind: nachher k... πŸ™ˆ πŸ˜‚ \n", "2501 ich glaube das wΓ€re eine stilvolle und romanti... 😍 😊 \n", "5278 Ich kann das machen πŸ’ βœ‹ \n", "5747 In der KΓΌche im KΓΌhlschrank steht eine grΓΌne T... πŸ˜‰ πŸ™‚ \n", "7022 Dann halt nicht ‍♂ 🀷 🀷 \n", "1736 Ich hab absolut keine Ahnung davon πŸ˜‚ 😞 \n", "5909 Wir bekommen es bestimmt zu zweit hin, aber we... πŸ™ˆ πŸ˜‰ \n", "5890 Pass bloß auf, wenn du die isst mutierst du be... πŸ˜„ 😱 \n", "566 Das sind 5 Maß pro Abend πŸ™„ πŸ˜‹ \n", "6044 Sind sehr lecker πŸ˜€ πŸ‘Œ \n", "1472 Ich werde die alle drillen das sage ich dir, a... πŸ€” πŸ˜‡ \n", "5702 Oh man, das wird ein Tag πŸ˜€ πŸ˜„ \n", "\n", " emojis \n", "8663 [πŸ€”] \n", "1862 [😞] \n", "3921 [πŸ™‚] \n", "2222 [πŸ˜€] \n", "761 [πŸ˜‰] \n", "3600 [πŸ‘Έ] \n", "1794 [πŸ™‚] \n", "861 [πŸ˜‡] \n", "1469 [😱, πŸ˜”, 🌚, πŸ™Š] \n", "6254 [πŸ™‚] \n", "7968 [πŸ˜‚] \n", "8102 [🧐] \n", "6835 [😊] \n", "7135 [πŸ˜‚, πŸ˜‚, πŸ˜‚] \n", "4556 [πŸ˜€] \n", "6705 [πŸ™„] \n", "2058 [πŸ˜…] \n", "2072 [😈] \n", "756 [πŸ˜†] \n", "5966 [πŸ€”] \n", "7007 [πŸ€ͺ] \n", "6968 [🀩] \n", "5665 [πŸ™‚] \n", "2641 [πŸ™ƒ] \n", "1981 [πŸ˜„] \n", "3334 [πŸ™‚] \n", "4447 [😏] \n", "288 [πŸ‘] \n", "4265 [😒] \n", "7405 [πŸ€ͺ] \n", "... ... \n", "4910 [😱] \n", "2047 [πŸ˜„] \n", "2650 [πŸ˜•] \n", "299 [πŸ˜‰] \n", "2499 [πŸ˜…] \n", "4783 [πŸ’Έ, πŸ’Έ, πŸ’Έ] \n", "4040 [πŸ€—, πŸ˜…] \n", "7553 [🀦] \n", "6232 [πŸ’£, πŸ”ͺ] \n", "3514 [😊] \n", "3422 [😍] \n", "2294 [😳, 😳, 😳] \n", "8218 [πŸ˜…] \n", "4463 [πŸ˜‚] \n", "8032 [🐰, 🐣, πŸ₯] \n", "4505 [πŸ’] \n", "4810 [🀷] \n", "5762 [πŸ™„] \n", "4247 [πŸ˜‚] \n", "2501 [😊] \n", "5278 [βœ‹] \n", "5747 [πŸ˜‰, πŸ™‚] \n", "7022 [🀷] \n", "1736 [😞] \n", "5909 [πŸ˜‰] \n", "5890 [😱] \n", "566 [πŸ˜‹] \n", "6044 [πŸ‘Œ] \n", "1472 [πŸ˜‡] \n", "5702 [πŸ˜„] \n", "\n", "[246 rows x 4 columns]" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n", "testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n", "testlist.to_csv('export.csv')\n", "testlist" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'πŸ™„'" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "αΌΏ\n" ] } ], "source": [ "print(chr(0x1F3F))" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "<1x3931 sparse matrix of type ''\n", "\twith 2 stored elements in Compressed Sparse Row format>" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vec_train[0]" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0b212fa64d5e457d89ad798983002a1c", "version_major": 2, "version_minor": 0 }, "text/html": [ "

Failed to display Jupyter Widget of type Text.

\n", "

\n", " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", " that the widgets JavaScript is still loading. If this message persists, it\n", " likely means that the widgets JavaScript library is either not installed or\n", " not enabled. See the Jupyter\n", " Widgets Documentation for setup instructions.\n", "

\n", "

\n", " If you're reading this message in another frontend (for example, a static\n", " rendering on GitHub or NBViewer),\n", " it may mean that your frontend doesn't currently support widgets.\n", "

\n" ], "text/plain": [ "Text(value='')" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f1d5e13f70a24e80acc95372e3f3db0d", "version_major": 2, "version_minor": 0 }, "text/html": [ "

Failed to display Jupyter Widget of type VBox.

\n", "

\n", " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", " that the widgets JavaScript is still loading. If this message persists, it\n", " likely means that the widgets JavaScript library is either not installed or\n", " not enabled. See the Jupyter\n", " Widgets Documentation for setup instructions.\n", "

\n", "

\n", " If you're reading this message in another frontend (for example, a static\n", " rendering on GitHub or NBViewer),\n", " it may mean that your frontend doesn't currently support widgets.\n", "

\n" ], "text/plain": [ "VBox(children=(Button(description='get smiley', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import ipywidgets as widgets \n", "\n", "out = widgets.Output()\n", "\n", "t = widgets.Text()\n", "b = widgets.Button(\n", " description='get smiley',\n", " disabled=False,\n", " button_style='', # 'success', 'info', 'warning', 'danger' or ''\n", " tooltip='Click me',\n", " icon='check'\n", ")\n", "\n", "\n", "\n", "def handle_submit(sender):\n", " with out:\n", " clear_output()\n", " with out:\n", " display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n", "\n", "b.on_click(handle_submit)\n", " \n", "display(t)\n", "display(widgets.VBox([b, out])) " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }