nlp-lab/Project/Tools/emoji tester.ipynb
2018-05-18 15:38:36 +02:00

2736 lines
96 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read table `test.csv` exported by `whatsapp2csv.sh`"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>date</th>\n",
" <th>time</th>\n",
" <th>person</th>\n",
" <th>message</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>10/18/16</td>\n",
" <td>11:18</td>\n",
" <td>Lena Carta</td>\n",
" <td>Hallo Jonas 😉</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>Moin moin ;) Vllt. bin ich aber gar nicht Jona...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Lena Carta</td>\n",
" <td>😱🤕</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Sonja Vorwerk</td>\n",
" <td>🙈😂</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>10/18/16</td>\n",
" <td>11:22</td>\n",
" <td>Caner Ucar</td>\n",
" <td>😂😂😂</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>10/18/16</td>\n",
" <td>11:22</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>hallo jeff</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>10/18/16</td>\n",
" <td>11:22</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Patrick kein Handy am Empfang</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>10/18/16</td>\n",
" <td>11:26</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>halts maul und geh ma duschen</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>10/18/16</td>\n",
" <td>11:34</td>\n",
" <td>Simon (Arithmeum)</td>\n",
" <td>Hab dich als Carsten abgespeichert, selber Schuld</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>10/18/16</td>\n",
" <td>11:36</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Patrick ich hab geduscht mit Kristina und Sonja v</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>10/18/16</td>\n",
" <td>11:36</td>\n",
" <td>Lena Carta</td>\n",
" <td>ähm....</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>Caner benutzt das Damen-Bad</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>Lüg doch nicht, du tanzt die ganze Zeit aufm T...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>bääh😱</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>10/18/16</td>\n",
" <td>11:40</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>ja caner so ist das unter frauen..da geht man ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>10/18/16</td>\n",
" <td>12:02</td>\n",
" <td>Thomas Koppelberg</td>\n",
" <td>Eeemmm was?</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>10/18/16</td>\n",
" <td>12:12</td>\n",
" <td>Kristina Stellwag</td>\n",
" <td>Thomas, du wolltest gestern ja nicht 😜</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>10/18/16</td>\n",
" <td>12:14</td>\n",
" <td>Thomas Koppelberg</td>\n",
" <td>Ich wollte einiges, aber ich kann mich zurückh...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>10/18/16</td>\n",
" <td>12:19</td>\n",
" <td>Simon (Arithmeum)</td>\n",
" <td>Genau 🙊</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>10/18/16</td>\n",
" <td>13:36</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Sex</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" date time person \\\n",
"0 10/18/16 11:18 Lena Carta \n",
"1 10/18/16 11:21 Jonas Weinz \n",
"2 10/18/16 11:21 Lena Carta \n",
"3 10/18/16 11:21 Sonja Vorwerk \n",
"4 10/18/16 11:22 Caner Ucar \n",
"5 10/18/16 11:22 Patrick (Arithmeum) \n",
"6 10/18/16 11:22 Caner Ucar \n",
"7 10/18/16 11:26 Patrick (Arithmeum) \n",
"8 10/18/16 11:34 Simon (Arithmeum) \n",
"9 10/18/16 11:36 Caner Ucar \n",
"10 10/18/16 11:36 Lena Carta \n",
"11 10/18/16 11:37 Jonathan (Arithmeum) \n",
"12 10/18/16 11:37 Jonas Weinz \n",
"13 10/18/16 11:37 Jonathan (Arithmeum) \n",
"14 10/18/16 11:40 Patrick (Arithmeum) \n",
"15 10/18/16 12:02 Thomas Koppelberg \n",
"16 10/18/16 12:12 Kristina Stellwag \n",
"17 10/18/16 12:14 Thomas Koppelberg \n",
"18 10/18/16 12:19 Simon (Arithmeum) \n",
"19 10/18/16 13:36 Caner Ucar \n",
"\n",
" message \n",
"0 Hallo Jonas 😉 \n",
"1 Moin moin ;) Vllt. bin ich aber gar nicht Jona... \n",
"2 😱🤕 \n",
"3 🙈😂 \n",
"4 😂😂😂 \n",
"5 hallo jeff \n",
"6 Patrick kein Handy am Empfang \n",
"7 halts maul und geh ma duschen \n",
"8 Hab dich als Carsten abgespeichert, selber Schuld \n",
"9 Patrick ich hab geduscht mit Kristina und Sonja v \n",
"10 ähm.... \n",
"11 Caner benutzt das Damen-Bad \n",
"12 Lüg doch nicht, du tanzt die ganze Zeit aufm T... \n",
"13 bääh😱 \n",
"14 ja caner so ist das unter frauen..da geht man ... \n",
"15 Eeemmm was? \n",
"16 Thomas, du wolltest gestern ja nicht 😜 \n",
"17 Ich wollte einiges, aber ich kann mich zurückh... \n",
"18 Genau 🙊 \n",
"19 Sex "
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"messages = pd.read_csv('test.csv', delimiter='\\t')\n",
"messages[:20]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"emoji_blacklist = set([\n",
" 0x1F3FB,\n",
" 0x1F3FC,\n",
" 0x1F3FD,\n",
" 0x1F3FE,\n",
" 0x1F3FF\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>unicode</th>\n",
" <th>type</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>228</th>\n",
" <td>231A..231B</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>229</th>\n",
" <td>23E9..23EC</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>230</th>\n",
" <td>23F0</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>231</th>\n",
" <td>23F3</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>232</th>\n",
" <td>25FD..25FE</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>233</th>\n",
" <td>2614..2615</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>234</th>\n",
" <td>2648..2653</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>235</th>\n",
" <td>267F</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>236</th>\n",
" <td>2693</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>237</th>\n",
" <td>26A1</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>238</th>\n",
" <td>26AA..26AB</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>239</th>\n",
" <td>26BD..26BE</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>240</th>\n",
" <td>26C4..26C5</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>241</th>\n",
" <td>26CE</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>242</th>\n",
" <td>26D4</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>243</th>\n",
" <td>26EA</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>244</th>\n",
" <td>26F2..26F3</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>245</th>\n",
" <td>26F5</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>246</th>\n",
" <td>26FA</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>247</th>\n",
" <td>26FD</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>248</th>\n",
" <td>2705</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>249</th>\n",
" <td>270A..270B</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>250</th>\n",
" <td>2728</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>251</th>\n",
" <td>274C</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>252</th>\n",
" <td>274E</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>253</th>\n",
" <td>2753..2755</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>254</th>\n",
" <td>2757</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>255</th>\n",
" <td>2795..2797</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>256</th>\n",
" <td>27B0</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>257</th>\n",
" <td>27BF</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>329</th>\n",
" <td>1F6F7..1F6F8</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>330</th>\n",
" <td>1F6F9</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>331</th>\n",
" <td>1F910..1F918</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>332</th>\n",
" <td>1F919..1F91E</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>333</th>\n",
" <td>1F91F</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>334</th>\n",
" <td>1F920..1F927</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>335</th>\n",
" <td>1F928..1F92F</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>336</th>\n",
" <td>1F930</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>337</th>\n",
" <td>1F931..1F932</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>338</th>\n",
" <td>1F933..1F93A</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>339</th>\n",
" <td>1F93C..1F93E</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>340</th>\n",
" <td>1F940..1F945</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>341</th>\n",
" <td>1F947..1F94B</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>342</th>\n",
" <td>1F94C</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>343</th>\n",
" <td>1F94D..1F94F</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>344</th>\n",
" <td>1F950..1F95E</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>345</th>\n",
" <td>1F95F..1F96B</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>346</th>\n",
" <td>1F96C..1F970</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>347</th>\n",
" <td>1F973..1F976</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>348</th>\n",
" <td>1F97A</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>349</th>\n",
" <td>1F97C..1F97F</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>350</th>\n",
" <td>1F980..1F984</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>351</th>\n",
" <td>1F985..1F991</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>352</th>\n",
" <td>1F992..1F997</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>353</th>\n",
" <td>1F998..1F9A2</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>354</th>\n",
" <td>1F9B0..1F9B9</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>355</th>\n",
" <td>1F9C0</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>356</th>\n",
" <td>1F9C1..1F9C2</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>357</th>\n",
" <td>1F9D0..1F9E6</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" <tr>\n",
" <th>358</th>\n",
" <td>1F9E7..1F9FF</td>\n",
" <td>Emoji_Presentation</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>131 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" unicode type\n",
"228 231A..231B Emoji_Presentation\n",
"229 23E9..23EC Emoji_Presentation\n",
"230 23F0 Emoji_Presentation\n",
"231 23F3 Emoji_Presentation\n",
"232 25FD..25FE Emoji_Presentation\n",
"233 2614..2615 Emoji_Presentation\n",
"234 2648..2653 Emoji_Presentation\n",
"235 267F Emoji_Presentation\n",
"236 2693 Emoji_Presentation\n",
"237 26A1 Emoji_Presentation\n",
"238 26AA..26AB Emoji_Presentation\n",
"239 26BD..26BE Emoji_Presentation\n",
"240 26C4..26C5 Emoji_Presentation\n",
"241 26CE Emoji_Presentation\n",
"242 26D4 Emoji_Presentation\n",
"243 26EA Emoji_Presentation\n",
"244 26F2..26F3 Emoji_Presentation\n",
"245 26F5 Emoji_Presentation\n",
"246 26FA Emoji_Presentation\n",
"247 26FD Emoji_Presentation\n",
"248 2705 Emoji_Presentation\n",
"249 270A..270B Emoji_Presentation\n",
"250 2728 Emoji_Presentation\n",
"251 274C Emoji_Presentation\n",
"252 274E Emoji_Presentation\n",
"253 2753..2755 Emoji_Presentation\n",
"254 2757 Emoji_Presentation\n",
"255 2795..2797 Emoji_Presentation\n",
"256 27B0 Emoji_Presentation\n",
"257 27BF Emoji_Presentation\n",
".. ... ...\n",
"329 1F6F7..1F6F8 Emoji_Presentation\n",
"330 1F6F9 Emoji_Presentation\n",
"331 1F910..1F918 Emoji_Presentation\n",
"332 1F919..1F91E Emoji_Presentation\n",
"333 1F91F Emoji_Presentation\n",
"334 1F920..1F927 Emoji_Presentation\n",
"335 1F928..1F92F Emoji_Presentation\n",
"336 1F930 Emoji_Presentation\n",
"337 1F931..1F932 Emoji_Presentation\n",
"338 1F933..1F93A Emoji_Presentation\n",
"339 1F93C..1F93E Emoji_Presentation\n",
"340 1F940..1F945 Emoji_Presentation\n",
"341 1F947..1F94B Emoji_Presentation\n",
"342 1F94C Emoji_Presentation\n",
"343 1F94D..1F94F Emoji_Presentation\n",
"344 1F950..1F95E Emoji_Presentation\n",
"345 1F95F..1F96B Emoji_Presentation\n",
"346 1F96C..1F970 Emoji_Presentation\n",
"347 1F973..1F976 Emoji_Presentation\n",
"348 1F97A Emoji_Presentation\n",
"349 1F97C..1F97F Emoji_Presentation\n",
"350 1F980..1F984 Emoji_Presentation\n",
"351 1F985..1F991 Emoji_Presentation\n",
"352 1F992..1F997 Emoji_Presentation\n",
"353 1F998..1F9A2 Emoji_Presentation\n",
"354 1F9B0..1F9B9 Emoji_Presentation\n",
"355 1F9C0 Emoji_Presentation\n",
"356 1F9C1..1F9C2 Emoji_Presentation\n",
"357 1F9D0..1F9E6 Emoji_Presentation\n",
"358 1F9E7..1F9FF Emoji_Presentation\n",
"\n",
"[131 rows x 2 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n",
"emoji_data['type'] = emoji_data['type'].str.strip()\n",
"emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n",
"emoji_data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* now build a set out of the unicode types"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ord(\"😀\") == int('0x1f600',16)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"228 231A..231B \n",
"229 23E9..23EC \n",
"230 23F0 \n",
"231 23F3 \n",
"232 25FD..25FE \n",
"Name: unicode, dtype: object"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"emoji_codes = emoji_data['unicode']\n",
"emoji_codes.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* we have to iterate over the whole list and extract all given ranges:"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"emoji_code_list = []\n",
"for entry in emoji_codes:\n",
" # testing whether we have an entry or a range:\n",
" if '.' in entry:\n",
" # range\n",
" a,b = entry.split(\"..\")\n",
" for i in range(int(a,16),int(b,16) +1):\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(i)\n",
" else:\n",
" # single entry\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(int(entry,16))\n",
"emoji_code_set = set(emoji_code_list)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"True False\n"
]
}
],
"source": [
"# simple test:\n",
"print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* expanding column and fill new emojis"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"messages[\"emojis\"] = None"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"for i in messages.index:\n",
" emoji_list = []\n",
" to_remove = []\n",
" m = messages.iloc[i]['message']\n",
" for c in str(m):\n",
" if ord(c) in emoji_code_set:\n",
" emoji_list.append(c)\n",
" elif ord(c) in emoji_blacklist:\n",
" to_remove.append(c)\n",
" \n",
" messages.loc[i,'emojis'] = emoji_list\n",
" #remove emiójis from message\n",
" for e in (emoji_list + to_remove):\n",
" m = m.replace(e,\"\")\n",
" messages.loc[i,'message'] = m\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>date</th>\n",
" <th>time</th>\n",
" <th>person</th>\n",
" <th>message</th>\n",
" <th>emojis</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>10/18/16</td>\n",
" <td>11:18</td>\n",
" <td>Lena Carta</td>\n",
" <td>Hallo Jonas</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>Moin moin ;) Vllt. bin ich aber gar nicht Jona...</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Lena Carta</td>\n",
" <td></td>\n",
" <td>[😱, 🤕]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Sonja Vorwerk</td>\n",
" <td></td>\n",
" <td>[🙈, 😂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>10/18/16</td>\n",
" <td>11:22</td>\n",
" <td>Caner Ucar</td>\n",
" <td></td>\n",
" <td>[😂, 😂, 😂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>10/18/16</td>\n",
" <td>11:22</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>hallo jeff</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>10/18/16</td>\n",
" <td>11:22</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Patrick kein Handy am Empfang</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>10/18/16</td>\n",
" <td>11:26</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>halts maul und geh ma duschen</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>10/18/16</td>\n",
" <td>11:34</td>\n",
" <td>Simon (Arithmeum)</td>\n",
" <td>Hab dich als Carsten abgespeichert, selber Schuld</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>10/18/16</td>\n",
" <td>11:36</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Patrick ich hab geduscht mit Kristina und Sonja v</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>10/18/16</td>\n",
" <td>11:36</td>\n",
" <td>Lena Carta</td>\n",
" <td>ähm....</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>Caner benutzt das Damen-Bad</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>Lüg doch nicht, du tanzt die ganze Zeit aufm T...</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>bääh</td>\n",
" <td>[😱]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>10/18/16</td>\n",
" <td>11:40</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>ja caner so ist das unter frauen..da geht man ...</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>10/18/16</td>\n",
" <td>12:02</td>\n",
" <td>Thomas Koppelberg</td>\n",
" <td>Eeemmm was?</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>10/18/16</td>\n",
" <td>12:12</td>\n",
" <td>Kristina Stellwag</td>\n",
" <td>Thomas, du wolltest gestern ja nicht</td>\n",
" <td>[😜]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>10/18/16</td>\n",
" <td>12:14</td>\n",
" <td>Thomas Koppelberg</td>\n",
" <td>Ich wollte einiges, aber ich kann mich zurückh...</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>10/18/16</td>\n",
" <td>12:19</td>\n",
" <td>Simon (Arithmeum)</td>\n",
" <td>Genau</td>\n",
" <td>[🙊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>10/18/16</td>\n",
" <td>13:36</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Sex</td>\n",
" <td>[]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" date time person \\\n",
"0 10/18/16 11:18 Lena Carta \n",
"1 10/18/16 11:21 Jonas Weinz \n",
"2 10/18/16 11:21 Lena Carta \n",
"3 10/18/16 11:21 Sonja Vorwerk \n",
"4 10/18/16 11:22 Caner Ucar \n",
"5 10/18/16 11:22 Patrick (Arithmeum) \n",
"6 10/18/16 11:22 Caner Ucar \n",
"7 10/18/16 11:26 Patrick (Arithmeum) \n",
"8 10/18/16 11:34 Simon (Arithmeum) \n",
"9 10/18/16 11:36 Caner Ucar \n",
"10 10/18/16 11:36 Lena Carta \n",
"11 10/18/16 11:37 Jonathan (Arithmeum) \n",
"12 10/18/16 11:37 Jonas Weinz \n",
"13 10/18/16 11:37 Jonathan (Arithmeum) \n",
"14 10/18/16 11:40 Patrick (Arithmeum) \n",
"15 10/18/16 12:02 Thomas Koppelberg \n",
"16 10/18/16 12:12 Kristina Stellwag \n",
"17 10/18/16 12:14 Thomas Koppelberg \n",
"18 10/18/16 12:19 Simon (Arithmeum) \n",
"19 10/18/16 13:36 Caner Ucar \n",
"\n",
" message emojis \n",
"0 Hallo Jonas [😉] \n",
"1 Moin moin ;) Vllt. bin ich aber gar nicht Jona... [🤔] \n",
"2 [😱, 🤕] \n",
"3 [🙈, 😂] \n",
"4 [😂, 😂, 😂] \n",
"5 hallo jeff [] \n",
"6 Patrick kein Handy am Empfang [] \n",
"7 halts maul und geh ma duschen [] \n",
"8 Hab dich als Carsten abgespeichert, selber Schuld [] \n",
"9 Patrick ich hab geduscht mit Kristina und Sonja v [] \n",
"10 ähm.... [] \n",
"11 Caner benutzt das Damen-Bad [] \n",
"12 Lüg doch nicht, du tanzt die ganze Zeit aufm T... [] \n",
"13 bääh [😱] \n",
"14 ja caner so ist das unter frauen..da geht man ... [] \n",
"15 Eeemmm was? [] \n",
"16 Thomas, du wolltest gestern ja nicht [😜] \n",
"17 Ich wollte einiges, aber ich kann mich zurückh... [] \n",
"18 Genau [🙊] \n",
"19 Sex [] "
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"messages[:20]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get a list only containing messaged with emojis"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n",
"emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>date</th>\n",
" <th>time</th>\n",
" <th>person</th>\n",
" <th>message</th>\n",
" <th>emojis</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>10/18/16</td>\n",
" <td>11:18</td>\n",
" <td>Lena Carta</td>\n",
" <td>Hallo Jonas</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10/18/16</td>\n",
" <td>11:21</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>Moin moin ;) Vllt. bin ich aber gar nicht Jona...</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>10/18/16</td>\n",
" <td>11:37</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>bääh</td>\n",
" <td>[😱]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>10/18/16</td>\n",
" <td>12:12</td>\n",
" <td>Kristina Stellwag</td>\n",
" <td>Thomas, du wolltest gestern ja nicht</td>\n",
" <td>[😜]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>10/18/16</td>\n",
" <td>12:19</td>\n",
" <td>Simon (Arithmeum)</td>\n",
" <td>Genau</td>\n",
" <td>[🙊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>10/18/16</td>\n",
" <td>18:38</td>\n",
" <td>Lena Carta</td>\n",
" <td>was du bist morgen nicht da ???</td>\n",
" <td>[😱, 😯, 😔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>31</th>\n",
" <td>10/18/16</td>\n",
" <td>18:39</td>\n",
" <td>Lena Carta</td>\n",
" <td>ich dachte diese Woche komplett noch. Aber gut...</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32</th>\n",
" <td>10/18/16</td>\n",
" <td>18:40</td>\n",
" <td>Kristina Stellwag</td>\n",
" <td>Nee das Semester hat doch angefangen</td>\n",
" <td>[😜]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>10/18/16</td>\n",
" <td>18:42</td>\n",
" <td>Lena Carta</td>\n",
" <td>ja hatte ich gerade vergessen . Dabei hatte ic...</td>\n",
" <td>[😂, 😶]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>41</th>\n",
" <td>10/20/16</td>\n",
" <td>12:50</td>\n",
" <td>Steffi (Arithmeum)</td>\n",
" <td>Klar</td>\n",
" <td>[👹]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44</th>\n",
" <td>10/20/16</td>\n",
" <td>17:54</td>\n",
" <td>Steffi (Arithmeum)</td>\n",
" <td>Mein Zug lässt sich auch heute Zeit</td>\n",
" <td>[😞]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>10/20/16</td>\n",
" <td>18:08</td>\n",
" <td>Steffi (Arithmeum)</td>\n",
" <td>Wir fangen doch wahrscheinlich nicht vor halb ...</td>\n",
" <td>[😳]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>51</th>\n",
" <td>10/20/16</td>\n",
" <td>20:58</td>\n",
" <td>Anna Borutzky</td>\n",
" <td>Ist das dein Ernst?</td>\n",
" <td>[🙄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>52</th>\n",
" <td>10/20/16</td>\n",
" <td>20:58</td>\n",
" <td>Patrick (Arithmeum)</td>\n",
" <td>Habs voll vercheckt sry ..gerade erst gemerkt</td>\n",
" <td>[🙄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>70</th>\n",
" <td>10/20/16</td>\n",
" <td>22:59</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>Ich bitte darum Caner....</td>\n",
" <td>[😍, 😍, 😍]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>78</th>\n",
" <td>10/25/16</td>\n",
" <td>01:03</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Haaaaaalloooo</td>\n",
" <td>[🤗, 🤗, 🤗, 🤗, 🤗, 🤗, 🤗, 😏, 🤗, 🤗, 😏]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>79</th>\n",
" <td>10/25/16</td>\n",
" <td>01:04</td>\n",
" <td>Michael (Arithmeum)</td>\n",
" <td>Keine sorge, das hat bei dir nichts mit der St...</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>88</th>\n",
" <td>10/25/16</td>\n",
" <td>09:02</td>\n",
" <td>Lena Carta</td>\n",
" <td>ich wollte eigentlich schon da sein, aber ich ...</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>89</th>\n",
" <td>10/25/16</td>\n",
" <td>09:04</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>SIIIIEG...⚔</td>\n",
" <td>[🎉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>90</th>\n",
" <td>10/25/16</td>\n",
" <td>10:00</td>\n",
" <td>Lena Carta</td>\n",
" <td>besser spät als nie. ich bin immerhin auf dem ...</td>\n",
" <td>[😁]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>93</th>\n",
" <td>10/25/16</td>\n",
" <td>21:49</td>\n",
" <td>Jonathan (Arithmeum)</td>\n",
" <td>Ich nicht</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>94</th>\n",
" <td>10/25/16</td>\n",
" <td>21:50</td>\n",
" <td>Lena Carta</td>\n",
" <td>ich auch nicht</td>\n",
" <td>[😳]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>10/25/16</td>\n",
" <td>22:13</td>\n",
" <td>Konstantin (Arithmeum)</td>\n",
" <td>Wer hat den zuletzt was laminiert? lena und i...</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>10/25/16</td>\n",
" <td>23:28</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Ich auch nicht</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>102</th>\n",
" <td>10/27/16</td>\n",
" <td>14:43</td>\n",
" <td>Sonja Vorwerk</td>\n",
" <td>hm müsste das am besten heute schon wissen...w...</td>\n",
" <td>[🙈, 😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>115</th>\n",
" <td>10/28/16</td>\n",
" <td>08:32</td>\n",
" <td>Lena Carta</td>\n",
" <td>Guten Morgen liebe Kollegen, könnte einer von ...</td>\n",
" <td>[🙁]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>116</th>\n",
" <td>10/28/16</td>\n",
" <td>09:48</td>\n",
" <td>Lena Carta</td>\n",
" <td>Ist noch niemand im Büro?</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>117</th>\n",
" <td>10/28/16</td>\n",
" <td>09:50</td>\n",
" <td>Caner Ucar</td>\n",
" <td>10:10 ca.</td>\n",
" <td>[🙌]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>118</th>\n",
" <td>10/28/16</td>\n",
" <td>09:51</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Wann hat das woki geöffnet</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>121</th>\n",
" <td>10/28/16</td>\n",
" <td>10:54</td>\n",
" <td>Lena Carta</td>\n",
" <td>Da hätte ich jetzt auch nicht gesucht</td>\n",
" <td>[😝]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8663</th>\n",
" <td>5/9/18</td>\n",
" <td>17:42</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>mehrere Bürostühle hintereinander?</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8665</th>\n",
" <td>5/9/18</td>\n",
" <td>17:43</td>\n",
" <td>Lena Carta</td>\n",
" <td>Wollte Kristina das nicht immer schonmal machen</td>\n",
" <td>[😁]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8675</th>\n",
" <td>5/10/18</td>\n",
" <td>14:06</td>\n",
" <td>Lena Klöckner</td>\n",
" <td>Erledigt</td>\n",
" <td>[👍]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8676</th>\n",
" <td>5/10/18</td>\n",
" <td>14:07</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Danke</td>\n",
" <td>[😊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8677</th>\n",
" <td>5/10/18</td>\n",
" <td>14:23</td>\n",
" <td>Anna Zens</td>\n",
" <td>Ich hatte es auch schon erledigt</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8678</th>\n",
" <td>5/11/18</td>\n",
" <td>08:16</td>\n",
" <td>Lena Carta</td>\n",
" <td>Gute morgen, ich habe gesehen neben dem Buch l...</td>\n",
" <td>[🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8679</th>\n",
" <td>5/11/18</td>\n",
" <td>08:25</td>\n",
" <td>Anna Zens</td>\n",
" <td>Die handschriftlichen sind die, von denen kein...</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8680</th>\n",
" <td>5/11/18</td>\n",
" <td>08:26</td>\n",
" <td>Lena Carta</td>\n",
" <td>achso ich denke mal dann wird sich Ingo melden</td>\n",
" <td>[😀, 😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8681</th>\n",
" <td>5/11/18</td>\n",
" <td>13:18</td>\n",
" <td>Freya (Arithmeum)</td>\n",
" <td>Mein Kalender funktioniert nicht - könnte mir ...</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8682</th>\n",
" <td>5/12/18</td>\n",
" <td>14:53</td>\n",
" <td>Freya (Arithmeum)</td>\n",
" <td>Könnte jemand für Montag Abend zwei Karten für...</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8683</th>\n",
" <td>5/12/18</td>\n",
" <td>15:05</td>\n",
" <td>Anna Borutzky</td>\n",
" <td>Und eine Freikarte für Sonja bitte!</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8689</th>\n",
" <td>5/12/18</td>\n",
" <td>15:16</td>\n",
" <td>Lena Carta</td>\n",
" <td>Danke Wieso kommen solche Fragen immer am Woc...</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8692</th>\n",
" <td>5/12/18</td>\n",
" <td>15:17</td>\n",
" <td>Anna Borutzky</td>\n",
" <td>Murphys Gesetz?</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8693</th>\n",
" <td>5/12/18</td>\n",
" <td>15:17</td>\n",
" <td>Caner Ucar</td>\n",
" <td>Cool</td>\n",
" <td>[😍]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8695</th>\n",
" <td>5/12/18</td>\n",
" <td>15:23</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>...und die kommt auf Annas Tisch</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8696</th>\n",
" <td>5/12/18</td>\n",
" <td>15:23</td>\n",
" <td>Freya (Arithmeum)</td>\n",
" <td>Vielen, vielen Dank ☺ Sowas passiert immer am ...</td>\n",
" <td>[🙈]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8697</th>\n",
" <td>5/12/18</td>\n",
" <td>15:23</td>\n",
" <td>Lena Carta</td>\n",
" <td>Das war nicht auf dich bezogen .</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8698</th>\n",
" <td>5/12/18</td>\n",
" <td>15:24</td>\n",
" <td>Freya (Arithmeum)</td>\n",
" <td>Na gut</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8699</th>\n",
" <td>5/12/18</td>\n",
" <td>15:24</td>\n",
" <td>Lena Carta</td>\n",
" <td>Eigentlich ist es halbwegs entspannt heute Ic...</td>\n",
" <td>[😀, 😛]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8700</th>\n",
" <td>5/12/18</td>\n",
" <td>15:25</td>\n",
" <td>Freya (Arithmeum)</td>\n",
" <td>Es gibt keine doofen Fragen</td>\n",
" <td>[😋]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8702</th>\n",
" <td>5/12/18</td>\n",
" <td>15:25</td>\n",
" <td>Lena Carta</td>\n",
" <td>https://www.youtube.com/watch?v=uPHi5xn_q5c</td>\n",
" <td>[😛]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8708</th>\n",
" <td>5/12/18</td>\n",
" <td>17:56</td>\n",
" <td>Freya (Arithmeum)</td>\n",
" <td>Ich möchte arbeiten und habe mega Bock drauf! ...</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8719</th>\n",
" <td>5/12/18</td>\n",
" <td>20:09</td>\n",
" <td>Lena Carta</td>\n",
" <td>bin leider verabredet .... mit Thilo</td>\n",
" <td>[😬, 🤗]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8725</th>\n",
" <td>5/13/18</td>\n",
" <td>17:24</td>\n",
" <td>Thomas Koppelberg</td>\n",
" <td>♥</td>\n",
" <td>[😍]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8726</th>\n",
" <td>5/14/18</td>\n",
" <td>17:00</td>\n",
" <td>Sonja Schäfer</td>\n",
" <td>Kann mir jemand noch eine zweite Freikarte res...</td>\n",
" <td>[😊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8739</th>\n",
" <td>5/14/18</td>\n",
" <td>22:40</td>\n",
" <td>Jonas Weinz</td>\n",
" <td>Hab Ingo beim vorbeilaufenden nicht gesehen, h...</td>\n",
" <td>[💁]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8744</th>\n",
" <td>5/15/18</td>\n",
" <td>17:16</td>\n",
" <td>Caner Ucar</td>\n",
" <td>‍♀</td>\n",
" <td>[🤷]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8755</th>\n",
" <td>5/17/18</td>\n",
" <td>10:43</td>\n",
" <td>Lena Carta</td>\n",
" <td>super perfekt dann ist beiden geholfen</td>\n",
" <td>[🙂, 😎]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8759</th>\n",
" <td>5/17/18</td>\n",
" <td>10:43</td>\n",
" <td>Lena Carta</td>\n",
" <td>ob das so sinnvoll ist?</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8763</th>\n",
" <td>5/17/18</td>\n",
" <td>10:45</td>\n",
" <td>Lena Carta</td>\n",
" <td>ich dachte es ging konkret um den Termin</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>2455 rows × 5 columns</p>\n",
"</div>"
],
"text/plain": [
" date time person \\\n",
"0 10/18/16 11:18 Lena Carta \n",
"1 10/18/16 11:21 Jonas Weinz \n",
"13 10/18/16 11:37 Jonathan (Arithmeum) \n",
"16 10/18/16 12:12 Kristina Stellwag \n",
"18 10/18/16 12:19 Simon (Arithmeum) \n",
"29 10/18/16 18:38 Lena Carta \n",
"31 10/18/16 18:39 Lena Carta \n",
"32 10/18/16 18:40 Kristina Stellwag \n",
"33 10/18/16 18:42 Lena Carta \n",
"41 10/20/16 12:50 Steffi (Arithmeum) \n",
"44 10/20/16 17:54 Steffi (Arithmeum) \n",
"45 10/20/16 18:08 Steffi (Arithmeum) \n",
"51 10/20/16 20:58 Anna Borutzky \n",
"52 10/20/16 20:58 Patrick (Arithmeum) \n",
"70 10/20/16 22:59 Jonathan (Arithmeum) \n",
"78 10/25/16 01:03 Caner Ucar \n",
"79 10/25/16 01:04 Michael (Arithmeum) \n",
"88 10/25/16 09:02 Lena Carta \n",
"89 10/25/16 09:04 Jonathan (Arithmeum) \n",
"90 10/25/16 10:00 Lena Carta \n",
"93 10/25/16 21:49 Jonathan (Arithmeum) \n",
"94 10/25/16 21:50 Lena Carta \n",
"95 10/25/16 22:13 Konstantin (Arithmeum) \n",
"96 10/25/16 23:28 Caner Ucar \n",
"102 10/27/16 14:43 Sonja Vorwerk \n",
"115 10/28/16 08:32 Lena Carta \n",
"116 10/28/16 09:48 Lena Carta \n",
"117 10/28/16 09:50 Caner Ucar \n",
"118 10/28/16 09:51 Caner Ucar \n",
"121 10/28/16 10:54 Lena Carta \n",
"... ... ... ... \n",
"8663 5/9/18 17:42 Jonas Weinz \n",
"8665 5/9/18 17:43 Lena Carta \n",
"8675 5/10/18 14:06 Lena Klöckner \n",
"8676 5/10/18 14:07 Caner Ucar \n",
"8677 5/10/18 14:23 Anna Zens \n",
"8678 5/11/18 08:16 Lena Carta \n",
"8679 5/11/18 08:25 Anna Zens \n",
"8680 5/11/18 08:26 Lena Carta \n",
"8681 5/11/18 13:18 Freya (Arithmeum) \n",
"8682 5/12/18 14:53 Freya (Arithmeum) \n",
"8683 5/12/18 15:05 Anna Borutzky \n",
"8689 5/12/18 15:16 Lena Carta \n",
"8692 5/12/18 15:17 Anna Borutzky \n",
"8693 5/12/18 15:17 Caner Ucar \n",
"8695 5/12/18 15:23 Jonas Weinz \n",
"8696 5/12/18 15:23 Freya (Arithmeum) \n",
"8697 5/12/18 15:23 Lena Carta \n",
"8698 5/12/18 15:24 Freya (Arithmeum) \n",
"8699 5/12/18 15:24 Lena Carta \n",
"8700 5/12/18 15:25 Freya (Arithmeum) \n",
"8702 5/12/18 15:25 Lena Carta \n",
"8708 5/12/18 17:56 Freya (Arithmeum) \n",
"8719 5/12/18 20:09 Lena Carta \n",
"8725 5/13/18 17:24 Thomas Koppelberg \n",
"8726 5/14/18 17:00 Sonja Schäfer \n",
"8739 5/14/18 22:40 Jonas Weinz \n",
"8744 5/15/18 17:16 Caner Ucar \n",
"8755 5/17/18 10:43 Lena Carta \n",
"8759 5/17/18 10:43 Lena Carta \n",
"8763 5/17/18 10:45 Lena Carta \n",
"\n",
" message \\\n",
"0 Hallo Jonas \n",
"1 Moin moin ;) Vllt. bin ich aber gar nicht Jona... \n",
"13 bääh \n",
"16 Thomas, du wolltest gestern ja nicht \n",
"18 Genau \n",
"29 was du bist morgen nicht da ??? \n",
"31 ich dachte diese Woche komplett noch. Aber gut... \n",
"32 Nee das Semester hat doch angefangen \n",
"33 ja hatte ich gerade vergessen . Dabei hatte ic... \n",
"41 Klar \n",
"44 Mein Zug lässt sich auch heute Zeit \n",
"45 Wir fangen doch wahrscheinlich nicht vor halb ... \n",
"51 Ist das dein Ernst? \n",
"52 Habs voll vercheckt sry ..gerade erst gemerkt \n",
"70 Ich bitte darum Caner.... \n",
"78 Haaaaaalloooo \n",
"79 Keine sorge, das hat bei dir nichts mit der St... \n",
"88 ich wollte eigentlich schon da sein, aber ich ... \n",
"89 SIIIIEG...⚔ \n",
"90 besser spät als nie. ich bin immerhin auf dem ... \n",
"93 Ich nicht \n",
"94 ich auch nicht \n",
"95 Wer hat den zuletzt was laminiert? lena und i... \n",
"96 Ich auch nicht \n",
"102 hm müsste das am besten heute schon wissen...w... \n",
"115 Guten Morgen liebe Kollegen, könnte einer von ... \n",
"116 Ist noch niemand im Büro? \n",
"117 10:10 ca. \n",
"118 Wann hat das woki geöffnet \n",
"121 Da hätte ich jetzt auch nicht gesucht \n",
"... ... \n",
"8663 mehrere Bürostühle hintereinander? \n",
"8665 Wollte Kristina das nicht immer schonmal machen \n",
"8675 Erledigt \n",
"8676 Danke \n",
"8677 Ich hatte es auch schon erledigt \n",
"8678 Gute morgen, ich habe gesehen neben dem Buch l... \n",
"8679 Die handschriftlichen sind die, von denen kein... \n",
"8680 achso ich denke mal dann wird sich Ingo melden \n",
"8681 Mein Kalender funktioniert nicht - könnte mir ... \n",
"8682 Könnte jemand für Montag Abend zwei Karten für... \n",
"8683 Und eine Freikarte für Sonja bitte! \n",
"8689 Danke Wieso kommen solche Fragen immer am Woc... \n",
"8692 Murphys Gesetz? \n",
"8693 Cool \n",
"8695 ...und die kommt auf Annas Tisch \n",
"8696 Vielen, vielen Dank ☺ Sowas passiert immer am ... \n",
"8697 Das war nicht auf dich bezogen . \n",
"8698 Na gut \n",
"8699 Eigentlich ist es halbwegs entspannt heute Ic... \n",
"8700 Es gibt keine doofen Fragen \n",
"8702 https://www.youtube.com/watch?v=uPHi5xn_q5c \n",
"8708 Ich möchte arbeiten und habe mega Bock drauf! ... \n",
"8719 bin leider verabredet .... mit Thilo \n",
"8725 ♥ \n",
"8726 Kann mir jemand noch eine zweite Freikarte res... \n",
"8739 Hab Ingo beim vorbeilaufenden nicht gesehen, h... \n",
"8744 ‍♀ \n",
"8755 super perfekt dann ist beiden geholfen \n",
"8759 ob das so sinnvoll ist? \n",
"8763 ich dachte es ging konkret um den Termin \n",
"\n",
" emojis \n",
"0 [😉] \n",
"1 [🤔] \n",
"13 [😱] \n",
"16 [😜] \n",
"18 [🙊] \n",
"29 [😱, 😯, 😔] \n",
"31 [😇] \n",
"32 [😜] \n",
"33 [😂, 😶] \n",
"41 [👹] \n",
"44 [😞] \n",
"45 [😳] \n",
"51 [🙄] \n",
"52 [🙄] \n",
"70 [😍, 😍, 😍] \n",
"78 [🤗, 🤗, 🤗, 🤗, 🤗, 🤗, 🤗, 😏, 🤗, 🤗, 😏] \n",
"79 [😉] \n",
"88 [😇] \n",
"89 [🎉] \n",
"90 [😁] \n",
"93 [😄] \n",
"94 [😳] \n",
"95 [🤔] \n",
"96 [🤔] \n",
"102 [🙈, 😄] \n",
"115 [🙁] \n",
"116 [😇] \n",
"117 [🙌] \n",
"118 [😄] \n",
"121 [😝] \n",
"... ... \n",
"8663 [🤔] \n",
"8665 [😁] \n",
"8675 [👍] \n",
"8676 [😊] \n",
"8677 [😄] \n",
"8678 [🙂] \n",
"8679 [😄] \n",
"8680 [😀, 😀] \n",
"8681 [🤔] \n",
"8682 [😇] \n",
"8683 [😀] \n",
"8689 [😀] \n",
"8692 [😉] \n",
"8693 [😍] \n",
"8695 [😉] \n",
"8696 [🙈] \n",
"8697 [😀] \n",
"8698 [😄] \n",
"8699 [😀, 😛] \n",
"8700 [😋] \n",
"8702 [😛] \n",
"8708 [😇] \n",
"8719 [😬, 🤗] \n",
"8725 [😍] \n",
"8726 [😊] \n",
"8739 [💁] \n",
"8744 [🤷] \n",
"8755 [🙂, 😎] \n",
"8759 [😀] \n",
"8763 [😀] \n",
"\n",
"[2455 rows x 5 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display(emoji_messages)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## learning part"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import itertools\n",
"import sklearn.utils as sku\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"labels=[e[-1] for e in emoji_messages['emojis']]"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['😉', '🤔', '😱', '😜', '🙊', '😔', '😇', '😜', '😶', '👹']"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"labels[:10]"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"vectorizer = TfidfVectorizer(stop_words='english')\n",
"vec_train = vectorizer.fit_transform(X1)\n",
"vec_test = vectorizer.transform(Xt1)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.6/site-packages/sklearn/neural_network/multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
" % self.max_iter, ConvergenceWarning)\n"
]
},
{
"data": {
"text/plain": [
"MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
" beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
" hidden_layer_sizes=(100,), learning_rate='constant',\n",
" learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
" nesterovs_momentum=True, power_t=0.5, random_state=None,\n",
" shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
" verbose=False, warm_start=False)"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from sklearn.ensemble import RandomForestClassifier as RFC\n",
"from sklearn.neural_network import MLPClassifier as MLP\n",
"from sklearn.naive_bayes import MultinomialNB as MNB\n",
"#clf_a = RFC(criterion='entropy', random_state=4222)\n",
"clf_a = MLP()\n",
"clf_a.fit(vec_train, y1)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"pred = clf_a.predict(vectorizer.transform(Xt1))\n"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>message</th>\n",
" <th>pred</th>\n",
" <th>trained</th>\n",
" <th>emojis</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>8663</th>\n",
" <td>mehrere Bürostühle hintereinander?</td>\n",
" <td>🙈</td>\n",
" <td>🤔</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1862</th>\n",
" <td>Ich will auch Essen</td>\n",
" <td>😉</td>\n",
" <td>😞</td>\n",
" <td>[😞]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3921</th>\n",
" <td>2x orange</td>\n",
" <td>🙂</td>\n",
" <td>🙂</td>\n",
" <td>[🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2222</th>\n",
" <td>Man soll auch keine Lernsachen da lassen</td>\n",
" <td>😀</td>\n",
" <td>😀</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>761</th>\n",
" <td>Wir können auch gerne wieder bei mir zusammen ...</td>\n",
" <td>😇</td>\n",
" <td>😉</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3600</th>\n",
" <td>Hab den Bus verpasst, den ich eigentlich nehme...</td>\n",
" <td>😟</td>\n",
" <td>👸</td>\n",
" <td>[👸]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1794</th>\n",
" <td>Klaro, gerne</td>\n",
" <td>😎</td>\n",
" <td>🙂</td>\n",
" <td>[🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>861</th>\n",
" <td>Sehr gut, Jan! Wenn sich noch jemand melden wü...</td>\n",
" <td>😘</td>\n",
" <td>😇</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1469</th>\n",
" <td>Jaaa aber mein Mitbewohner ist messy</td>\n",
" <td>😬</td>\n",
" <td>🙊</td>\n",
" <td>[😱, 😔, 🌚, 🙊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6254</th>\n",
" <td>Hallöchen Es hat ein Herr angerufen, der gern...</td>\n",
" <td>💁</td>\n",
" <td>🙂</td>\n",
" <td>[🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7968</th>\n",
" <td>Iwo</td>\n",
" <td>🤦</td>\n",
" <td>😂</td>\n",
" <td>[😂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8102</th>\n",
" <td>Keine Ahnung wer da haariger ist</td>\n",
" <td>😅</td>\n",
" <td>🧐</td>\n",
" <td>[🧐]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6835</th>\n",
" <td>Danke</td>\n",
" <td>😊</td>\n",
" <td>😊</td>\n",
" <td>[😊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7135</th>\n",
" <td>Wurde bestimmt von Big Bang geklaut</td>\n",
" <td>🤷</td>\n",
" <td>😂</td>\n",
" <td>[😂, 😂, 😂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4556</th>\n",
" <td>Ich kann leider nicht, aber arbeiten da nicht ...</td>\n",
" <td>😂</td>\n",
" <td>😀</td>\n",
" <td>[😀]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6705</th>\n",
" <td>Ist von heute, aber halt 3x in englisch</td>\n",
" <td>😇</td>\n",
" <td>🙄</td>\n",
" <td>[🙄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2058</th>\n",
" <td>..</td>\n",
" <td>🤦</td>\n",
" <td>😅</td>\n",
" <td>[😅]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2072</th>\n",
" <td>Oder fang endlich mal wieder an, was fürs Stud...</td>\n",
" <td>😄</td>\n",
" <td>😈</td>\n",
" <td>[😈]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>756</th>\n",
" <td>Ich hab vom letzten Jahr noch ein Trauma</td>\n",
" <td>😂</td>\n",
" <td>😆</td>\n",
" <td>[😆]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5966</th>\n",
" <td>Ist Freitag schon Wochenende?</td>\n",
" <td>😍</td>\n",
" <td>🤔</td>\n",
" <td>[🤔]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7007</th>\n",
" <td>Wenn wir die Klimaanlage im Museum ausschalten...</td>\n",
" <td>🤔</td>\n",
" <td>🤪</td>\n",
" <td>[🤪]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6968</th>\n",
" <td>Ein Vorbild für uns alle</td>\n",
" <td>🤔</td>\n",
" <td>🤩</td>\n",
" <td>[🤩]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5665</th>\n",
" <td>Kein Problem. Du kannst mich eintragen.</td>\n",
" <td>🙂</td>\n",
" <td>🙂</td>\n",
" <td>[🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2641</th>\n",
" <td>Jonas, du bist gelb markiert! Du kannst keine ...</td>\n",
" <td>😉</td>\n",
" <td>🙃</td>\n",
" <td>[🙃]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1981</th>\n",
" <td>Da bin ich mir sehr sicher!</td>\n",
" <td>😘</td>\n",
" <td>😄</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3334</th>\n",
" <td>Das wäre auf jeden Fall hilfreich! Danke</td>\n",
" <td>😍</td>\n",
" <td>🙂</td>\n",
" <td>[🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4447</th>\n",
" <td>Ist das Schild denn auch laminiert?!</td>\n",
" <td>🤔</td>\n",
" <td>😏</td>\n",
" <td>[😏]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>288</th>\n",
" <td>da bin ich mal gespannt wie das aussieht, wen...</td>\n",
" <td>😂</td>\n",
" <td>👍</td>\n",
" <td>[👍]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4265</th>\n",
" <td>Aber es ist doch gebrochen...</td>\n",
" <td>🤔</td>\n",
" <td>😢</td>\n",
" <td>[😢]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7405</th>\n",
" <td>Partey</td>\n",
" <td>🤦</td>\n",
" <td>🤪</td>\n",
" <td>[🤪]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4910</th>\n",
" <td>Vllt gilt das Schild auch nur für schwarze Hun...</td>\n",
" <td>😂</td>\n",
" <td>😱</td>\n",
" <td>[😱]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2047</th>\n",
" <td>Ich erprobe mich morgen mit Freya an Addieren ...</td>\n",
" <td>🙂</td>\n",
" <td>😄</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2650</th>\n",
" <td>Ich kann leider nicht</td>\n",
" <td>😕</td>\n",
" <td>😕</td>\n",
" <td>[😕]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>299</th>\n",
" <td>Das schafft meine Handykamera aber nicht</td>\n",
" <td>😂</td>\n",
" <td>😉</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2499</th>\n",
" <td>also ihr lieben ich arbeite gerade im arithmeu...</td>\n",
" <td>😁</td>\n",
" <td>😅</td>\n",
" <td>[😅]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4783</th>\n",
" <td>danke</td>\n",
" <td>😊</td>\n",
" <td>💸</td>\n",
" <td>[💸, 💸, 💸]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4040</th>\n",
" <td>Herrlich</td>\n",
" <td>🤦</td>\n",
" <td>😅</td>\n",
" <td>[🤗, 😅]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7553</th>\n",
" <td>babbage, meine Autokerrektur ‍♂</td>\n",
" <td>🤔</td>\n",
" <td>🤦</td>\n",
" <td>[🤦]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6232</th>\n",
" <td>Bin erst um 9.15Uhr da.....</td>\n",
" <td>👍</td>\n",
" <td>🔪</td>\n",
" <td>[💣, 🔪]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3514</th>\n",
" <td>Dankeschön! Dann trage ich dich ein</td>\n",
" <td>😘</td>\n",
" <td>😊</td>\n",
" <td>[😊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3422</th>\n",
" <td>Schnitzel</td>\n",
" <td>😅</td>\n",
" <td>😍</td>\n",
" <td>[😍]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2294</th>\n",
" <td>Club discreto?</td>\n",
" <td>😂</td>\n",
" <td>😳</td>\n",
" <td>[😳, 😳, 😳]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8218</th>\n",
" <td>auch da bin ich etwas neidisch</td>\n",
" <td>🙄</td>\n",
" <td>😅</td>\n",
" <td>[😅]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4463</th>\n",
" <td>Möchte jemand was vom Café Roller?</td>\n",
" <td>🤧</td>\n",
" <td>😂</td>\n",
" <td>[😂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8032</th>\n",
" <td>Frohe Ostern an Euch alle...</td>\n",
" <td>😊</td>\n",
" <td>🐥</td>\n",
" <td>[🐰, 🐣, 🐥]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4505</th>\n",
" <td>okay, ihr geht eigentlich ganz verantwortungsv...</td>\n",
" <td>😊</td>\n",
" <td>💁</td>\n",
" <td>[💁]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4810</th>\n",
" <td>Meine Meinung: Wenn das Personal da ist, dann ...</td>\n",
" <td>😂</td>\n",
" <td>🤷</td>\n",
" <td>[🤷]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5762</th>\n",
" <td>Ein shk ist aber billiger</td>\n",
" <td>👵</td>\n",
" <td>🙄</td>\n",
" <td>[🙄]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4247</th>\n",
" <td>Alle die morgen auf der Arbeit sind: nachher k...</td>\n",
" <td>🙈</td>\n",
" <td>😂</td>\n",
" <td>[😂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2501</th>\n",
" <td>ich glaube das wäre eine stilvolle und romanti...</td>\n",
" <td>😍</td>\n",
" <td>😊</td>\n",
" <td>[😊]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5278</th>\n",
" <td>Ich kann das machen</td>\n",
" <td>💁</td>\n",
" <td>✋</td>\n",
" <td>[✋]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5747</th>\n",
" <td>In der Küche im Kühlschrank steht eine grüne T...</td>\n",
" <td>😉</td>\n",
" <td>🙂</td>\n",
" <td>[😉, 🙂]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7022</th>\n",
" <td>Dann halt nicht ‍♂</td>\n",
" <td>🤷</td>\n",
" <td>🤷</td>\n",
" <td>[🤷]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1736</th>\n",
" <td>Ich hab absolut keine Ahnung davon</td>\n",
" <td>😂</td>\n",
" <td>😞</td>\n",
" <td>[😞]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5909</th>\n",
" <td>Wir bekommen es bestimmt zu zweit hin, aber we...</td>\n",
" <td>🙈</td>\n",
" <td>😉</td>\n",
" <td>[😉]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5890</th>\n",
" <td>Pass bloß auf, wenn du die isst mutierst du be...</td>\n",
" <td>😄</td>\n",
" <td>😱</td>\n",
" <td>[😱]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>566</th>\n",
" <td>Das sind 5 Maß pro Abend</td>\n",
" <td>🙄</td>\n",
" <td>😋</td>\n",
" <td>[😋]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6044</th>\n",
" <td>Sind sehr lecker</td>\n",
" <td>😀</td>\n",
" <td>👌</td>\n",
" <td>[👌]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1472</th>\n",
" <td>Ich werde die alle drillen das sage ich dir, a...</td>\n",
" <td>🤔</td>\n",
" <td>😇</td>\n",
" <td>[😇]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5702</th>\n",
" <td>Oh man, das wird ein Tag</td>\n",
" <td>😀</td>\n",
" <td>😄</td>\n",
" <td>[😄]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>246 rows × 4 columns</p>\n",
"</div>"
],
"text/plain": [
" message pred trained \\\n",
"8663 mehrere Bürostühle hintereinander? 🙈 🤔 \n",
"1862 Ich will auch Essen 😉 😞 \n",
"3921 2x orange 🙂 🙂 \n",
"2222 Man soll auch keine Lernsachen da lassen 😀 😀 \n",
"761 Wir können auch gerne wieder bei mir zusammen ... 😇 😉 \n",
"3600 Hab den Bus verpasst, den ich eigentlich nehme... 😟 👸 \n",
"1794 Klaro, gerne 😎 🙂 \n",
"861 Sehr gut, Jan! Wenn sich noch jemand melden wü... 😘 😇 \n",
"1469 Jaaa aber mein Mitbewohner ist messy 😬 🙊 \n",
"6254 Hallöchen Es hat ein Herr angerufen, der gern... 💁 🙂 \n",
"7968 Iwo 🤦 😂 \n",
"8102 Keine Ahnung wer da haariger ist 😅 🧐 \n",
"6835 Danke 😊 😊 \n",
"7135 Wurde bestimmt von Big Bang geklaut 🤷 😂 \n",
"4556 Ich kann leider nicht, aber arbeiten da nicht ... 😂 😀 \n",
"6705 Ist von heute, aber halt 3x in englisch 😇 🙄 \n",
"2058 .. 🤦 😅 \n",
"2072 Oder fang endlich mal wieder an, was fürs Stud... 😄 😈 \n",
"756 Ich hab vom letzten Jahr noch ein Trauma 😂 😆 \n",
"5966 Ist Freitag schon Wochenende? 😍 🤔 \n",
"7007 Wenn wir die Klimaanlage im Museum ausschalten... 🤔 🤪 \n",
"6968 Ein Vorbild für uns alle 🤔 🤩 \n",
"5665 Kein Problem. Du kannst mich eintragen. 🙂 🙂 \n",
"2641 Jonas, du bist gelb markiert! Du kannst keine ... 😉 🙃 \n",
"1981 Da bin ich mir sehr sicher! 😘 😄 \n",
"3334 Das wäre auf jeden Fall hilfreich! Danke 😍 🙂 \n",
"4447 Ist das Schild denn auch laminiert?! 🤔 😏 \n",
"288 da bin ich mal gespannt wie das aussieht, wen... 😂 👍 \n",
"4265 Aber es ist doch gebrochen... 🤔 😢 \n",
"7405 Partey 🤦 🤪 \n",
"... ... ... ... \n",
"4910 Vllt gilt das Schild auch nur für schwarze Hun... 😂 😱 \n",
"2047 Ich erprobe mich morgen mit Freya an Addieren ... 🙂 😄 \n",
"2650 Ich kann leider nicht 😕 😕 \n",
"299 Das schafft meine Handykamera aber nicht 😂 😉 \n",
"2499 also ihr lieben ich arbeite gerade im arithmeu... 😁 😅 \n",
"4783 danke 😊 💸 \n",
"4040 Herrlich 🤦 😅 \n",
"7553 babbage, meine Autokerrektur ‍♂ 🤔 🤦 \n",
"6232 Bin erst um 9.15Uhr da..... 👍 🔪 \n",
"3514 Dankeschön! Dann trage ich dich ein 😘 😊 \n",
"3422 Schnitzel 😅 😍 \n",
"2294 Club discreto? 😂 😳 \n",
"8218 auch da bin ich etwas neidisch 🙄 😅 \n",
"4463 Möchte jemand was vom Café Roller? 🤧 😂 \n",
"8032 Frohe Ostern an Euch alle... 😊 🐥 \n",
"4505 okay, ihr geht eigentlich ganz verantwortungsv... 😊 💁 \n",
"4810 Meine Meinung: Wenn das Personal da ist, dann ... 😂 🤷 \n",
"5762 Ein shk ist aber billiger 👵 🙄 \n",
"4247 Alle die morgen auf der Arbeit sind: nachher k... 🙈 😂 \n",
"2501 ich glaube das wäre eine stilvolle und romanti... 😍 😊 \n",
"5278 Ich kann das machen 💁 ✋ \n",
"5747 In der Küche im Kühlschrank steht eine grüne T... 😉 🙂 \n",
"7022 Dann halt nicht ‍♂ 🤷 🤷 \n",
"1736 Ich hab absolut keine Ahnung davon 😂 😞 \n",
"5909 Wir bekommen es bestimmt zu zweit hin, aber we... 🙈 😉 \n",
"5890 Pass bloß auf, wenn du die isst mutierst du be... 😄 😱 \n",
"566 Das sind 5 Maß pro Abend 🙄 😋 \n",
"6044 Sind sehr lecker 😀 👌 \n",
"1472 Ich werde die alle drillen das sage ich dir, a... 🤔 😇 \n",
"5702 Oh man, das wird ein Tag 😀 😄 \n",
"\n",
" emojis \n",
"8663 [🤔] \n",
"1862 [😞] \n",
"3921 [🙂] \n",
"2222 [😀] \n",
"761 [😉] \n",
"3600 [👸] \n",
"1794 [🙂] \n",
"861 [😇] \n",
"1469 [😱, 😔, 🌚, 🙊] \n",
"6254 [🙂] \n",
"7968 [😂] \n",
"8102 [🧐] \n",
"6835 [😊] \n",
"7135 [😂, 😂, 😂] \n",
"4556 [😀] \n",
"6705 [🙄] \n",
"2058 [😅] \n",
"2072 [😈] \n",
"756 [😆] \n",
"5966 [🤔] \n",
"7007 [🤪] \n",
"6968 [🤩] \n",
"5665 [🙂] \n",
"2641 [🙃] \n",
"1981 [😄] \n",
"3334 [🙂] \n",
"4447 [😏] \n",
"288 [👍] \n",
"4265 [😢] \n",
"7405 [🤪] \n",
"... ... \n",
"4910 [😱] \n",
"2047 [😄] \n",
"2650 [😕] \n",
"299 [😉] \n",
"2499 [😅] \n",
"4783 [💸, 💸, 💸] \n",
"4040 [🤗, 😅] \n",
"7553 [🤦] \n",
"6232 [💣, 🔪] \n",
"3514 [😊] \n",
"3422 [😍] \n",
"2294 [😳, 😳, 😳] \n",
"8218 [😅] \n",
"4463 [😂] \n",
"8032 [🐰, 🐣, 🐥] \n",
"4505 [💁] \n",
"4810 [🤷] \n",
"5762 [🙄] \n",
"4247 [😂] \n",
"2501 [😊] \n",
"5278 [✋] \n",
"5747 [😉, 🙂] \n",
"7022 [🤷] \n",
"1736 [😞] \n",
"5909 [😉] \n",
"5890 [😱] \n",
"566 [😋] \n",
"6044 [👌] \n",
"1472 [😇] \n",
"5702 [😄] \n",
"\n",
"[246 rows x 4 columns]"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n",
"testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n",
"testlist.to_csv('export.csv')\n",
"testlist"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'🙄'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Ἷ\n"
]
}
],
"source": [
"print(chr(0x1F3F))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<1x3931 sparse matrix of type '<class 'numpy.float64'>'\n",
"\twith 2 stored elements in Compressed Sparse Row format>"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vec_train[0]"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0b212fa64d5e457d89ad798983002a1c",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>Text</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"Text(value='')"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f1d5e13f70a24e80acc95372e3f3db0d",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(Button(description='get smiley', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import ipywidgets as widgets \n",
"\n",
"out = widgets.Output()\n",
"\n",
"t = widgets.Text()\n",
"b = widgets.Button(\n",
" description='get smiley',\n",
" disabled=False,\n",
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
" tooltip='Click me',\n",
" icon='check'\n",
")\n",
"\n",
"\n",
"\n",
"def handle_submit(sender):\n",
" with out:\n",
" clear_output()\n",
" with out:\n",
" display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n",
"\n",
"b.on_click(handle_submit)\n",
" \n",
"display(t)\n",
"display(widgets.VBox([b, out])) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}