nlp-lab/Project/simple_approach/simple_twitter_learning.ipynb
2018-06-05 15:57:26 +02:00

1770 lines
69 KiB
Plaintext
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os\n",
"import glob\n",
"import json\n",
"import numpy as np\n",
"import itertools\n",
"import sklearn.utils as sku\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.preprocessing import MultiLabelBinarizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# simple twitter approach\n",
"*for learning emoji usage by single (in the meaning of unconnected) twitter messages*"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## loading train data"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"data_root_folder = \"./data_en/\" # i created a symlink here"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get all json files in `data_root_folder`"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"json_files = sorted(glob.glob(data_root_folder + \"/*.json\"))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* so far, only load the first file"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>EMOJI</th>\n",
" <th>datetime</th>\n",
" <th>id</th>\n",
" <th>lang</th>\n",
" <th>person</th>\n",
" <th>reply_to</th>\n",
" <th>text</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>[🔥, 👏]</td>\n",
" <td>2017-11-01 13:29:00</td>\n",
" <td>925716304635547600</td>\n",
" <td>en</td>\n",
" <td>31507978</td>\n",
" <td>NaN</td>\n",
" <td>fashionbombdaily's photo &lt;EMOJI&gt;&lt;EMOJI&gt;🏼</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>[🤦]</td>\n",
" <td>2017-11-01 13:29:00</td>\n",
" <td>925716304664911900</td>\n",
" <td>en</td>\n",
" <td>231994649</td>\n",
" <td>NaN</td>\n",
" <td>Its scary how on point my horoscope be &lt;EMOJI...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>[🤧]</td>\n",
" <td>2017-11-01 13:29:00</td>\n",
" <td>925716304652308500</td>\n",
" <td>en</td>\n",
" <td>78123421</td>\n",
" <td>NaN</td>\n",
" <td>@mvyn_: Them boring ass licks, you supposed t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>[🐯, 🐥, 💕]</td>\n",
" <td>2017-11-01 13:29:01</td>\n",
" <td>925716308846661600</td>\n",
" <td>en</td>\n",
" <td>860396470540705800</td>\n",
" <td>NaN</td>\n",
" <td>@taehyungpic: #vmin s unit shoot &lt;EMOJI&gt;&lt;EM...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>[👍, 👍, 👍]</td>\n",
" <td>2017-11-01 13:29:02</td>\n",
" <td>925716313015586800</td>\n",
" <td>en</td>\n",
" <td>775381892321333200</td>\n",
" <td>NaN</td>\n",
" <td>@risa_del: Can I just say how much Meng respe...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>[😒]</td>\n",
" <td>2017-11-01 13:29:02</td>\n",
" <td>925716313032364000</td>\n",
" <td>en</td>\n",
" <td>3244140547</td>\n",
" <td>NaN</td>\n",
" <td>@Melodia_Muse: omg did y'all just cut off Red...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>[✅, 🍟]</td>\n",
" <td>2017-11-01 13:29:02</td>\n",
" <td>925716313036722200</td>\n",
" <td>en</td>\n",
" <td>4891498023</td>\n",
" <td>NaN</td>\n",
" <td>@Harrys1DEmpire: Follow everyone who LIKES th...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>[👶]</td>\n",
" <td>2017-11-01 13:29:03</td>\n",
" <td>925716317243555800</td>\n",
" <td>en</td>\n",
" <td>2660072274</td>\n",
" <td>NaN</td>\n",
" <td>@taehyungpic: he is so smollll &lt;EMOJI&gt;©rocket...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>[😄]</td>\n",
" <td>2017-11-01 13:29:03</td>\n",
" <td>925716317214089200</td>\n",
" <td>en</td>\n",
" <td>2592765104</td>\n",
" <td>NaN</td>\n",
" <td>Woooaaaahhh &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>[🌟, 🌟]</td>\n",
" <td>2017-11-01 13:29:03</td>\n",
" <td>925716317218508800</td>\n",
" <td>en</td>\n",
" <td>3011085514</td>\n",
" <td>NaN</td>\n",
" <td>@mymermaidIagoon: &lt;EMOJI&gt; HURRY! For your cha...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>[📷]</td>\n",
" <td>2017-11-01 13:29:04</td>\n",
" <td>925716321416949800</td>\n",
" <td>en</td>\n",
" <td>278737933</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt; vivalcli: Portraits by Zhao Guojing an...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>[🚨, 🚨, 🚨, 🚨, 🚨, 🚨]</td>\n",
" <td>2017-11-01 13:29:05</td>\n",
" <td>925716325615439900</td>\n",
" <td>en</td>\n",
" <td>876587129077477400</td>\n",
" <td>NaN</td>\n",
" <td>@BillRatchet: &lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;NO NUT NOV...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>[💕]</td>\n",
" <td>2017-11-01 13:29:05</td>\n",
" <td>925716325615505400</td>\n",
" <td>en</td>\n",
" <td>2402588155</td>\n",
" <td>NaN</td>\n",
" <td>@mysweetapplexxx: ❤Happy Birthday, @GoodGalC...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>[🙏]</td>\n",
" <td>2017-11-01 13:29:06</td>\n",
" <td>925716329797054500</td>\n",
" <td>en</td>\n",
" <td>1546678166</td>\n",
" <td>NaN</td>\n",
" <td>@pourlui: @realDonaldTrump @SSNjl @foxandfrie...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>[😩, 😩]</td>\n",
" <td>2017-11-01 13:29:06</td>\n",
" <td>925716329801310200</td>\n",
" <td>en</td>\n",
" <td>824586253634981900</td>\n",
" <td>9.257162e+17</td>\n",
" <td>@hiphopphiIes i wanna know too&lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>[😭, 💓]</td>\n",
" <td>2017-11-01 13:29:02</td>\n",
" <td>925716313019965400</td>\n",
" <td>en</td>\n",
" <td>1581953814</td>\n",
" <td>9.257088e+17</td>\n",
" <td>@WizMommma veda was yoda too &lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>[🙏, 🤕, 😂]</td>\n",
" <td>2017-11-01 13:29:06</td>\n",
" <td>925716329801371600</td>\n",
" <td>en</td>\n",
" <td>1737878390</td>\n",
" <td>NaN</td>\n",
" <td>@TiagoIlori4: Me and my family are humbled by...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>[😂]</td>\n",
" <td>2017-11-01 13:29:05</td>\n",
" <td>925716325607133200</td>\n",
" <td>en</td>\n",
" <td>1001999683</td>\n",
" <td>NaN</td>\n",
" <td>Im less stressed about turning 30 now &lt;EMOJI&gt;...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>[💯]</td>\n",
" <td>2017-11-01 13:29:07</td>\n",
" <td>925716334008082400</td>\n",
" <td>en</td>\n",
" <td>745222369183043600</td>\n",
" <td>NaN</td>\n",
" <td>Full charged. &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>[🙄]</td>\n",
" <td>2017-11-01 13:29:09</td>\n",
" <td>925716342401052700</td>\n",
" <td>en</td>\n",
" <td>798557155217539100</td>\n",
" <td>9.257147e+17</td>\n",
" <td>@SeaDimon Thats part of the problem, (they) @...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>[😟, 😥, 😢]</td>\n",
" <td>2017-11-01 13:29:10</td>\n",
" <td>925716346570240000</td>\n",
" <td>en</td>\n",
" <td>744396039126421500</td>\n",
" <td>9.250629e+17</td>\n",
" <td>@Ian_khetye got me emotional there&lt;EMOJI&gt;&lt;EMOJ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>[😂, 😂, 😂]</td>\n",
" <td>2017-11-01 13:29:10</td>\n",
" <td>925716346603753500</td>\n",
" <td>en</td>\n",
" <td>2428265479</td>\n",
" <td>NaN</td>\n",
" <td>@30SECONDFlGHTS: BRUHHH &lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>[👀]</td>\n",
" <td>2017-11-01 13:29:10</td>\n",
" <td>925716346595434500</td>\n",
" <td>en</td>\n",
" <td>2523596372</td>\n",
" <td>NaN</td>\n",
" <td>@BasebaIINation: Game 7 who you got?&lt;EMOJI&gt; f...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>[🔎]</td>\n",
" <td>2017-11-01 13:29:11</td>\n",
" <td>925716350764331000</td>\n",
" <td>en</td>\n",
" <td>2315896393</td>\n",
" <td>NaN</td>\n",
" <td>@JudicialWatch: JW says DOJ &amp;amp; Congress ar...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>[📣, 💖]</td>\n",
" <td>2017-11-01 13:29:12</td>\n",
" <td>925716354988236800</td>\n",
" <td>en</td>\n",
" <td>2316464875</td>\n",
" <td>NaN</td>\n",
" <td>@cssconfau: ✍🏻 &lt;EMOJI&gt; Ever thought about spe...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>[✨, 💖]</td>\n",
" <td>2017-11-01 13:29:12</td>\n",
" <td>925716354975465500</td>\n",
" <td>en</td>\n",
" <td>2614916743</td>\n",
" <td>NaN</td>\n",
" <td>@btspat_: Deal for Young Forever: Night Vers...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>[💪]</td>\n",
" <td>2017-11-01 13:29:13</td>\n",
" <td>925716359153041400</td>\n",
" <td>en</td>\n",
" <td>2887539044</td>\n",
" <td>NaN</td>\n",
" <td>@DFC_SARAVANA: @dhanushkraja Reached 6 Millio...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>[🌻]</td>\n",
" <td>2017-11-01 13:29:13</td>\n",
" <td>925716359182520300</td>\n",
" <td>en</td>\n",
" <td>721490010118205400</td>\n",
" <td>8.965900e+17</td>\n",
" <td>back to the yellow &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>[🍁, 🌺, 🍂]</td>\n",
" <td>2017-11-01 13:29:15</td>\n",
" <td>925716367558545400</td>\n",
" <td>en</td>\n",
" <td>520536723</td>\n",
" <td>9.257159e+17</td>\n",
" <td>@Dimafadma Happy month to you and your loved o...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>[🍃]</td>\n",
" <td>2017-11-01 13:29:16</td>\n",
" <td>925716371735900200</td>\n",
" <td>en</td>\n",
" <td>850852815941517300</td>\n",
" <td>NaN</td>\n",
" <td>Maxx and Patricia. Family hangs at For Good He...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176302</th>\n",
" <td>[😭, 😭, 💘]</td>\n",
" <td>2017-11-01 07:23:31</td>\n",
" <td>925624327755591700</td>\n",
" <td>en</td>\n",
" <td>845085544589672400</td>\n",
" <td>9.256009e+17</td>\n",
" <td>@peachshua1230 Awww &lt;EMOJI&gt; Ajsksjdjd im smili...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176303</th>\n",
" <td>[🤣, 🤣, 🤣, 🤣]</td>\n",
" <td>2017-11-01 07:23:33</td>\n",
" <td>925624336156938200</td>\n",
" <td>en</td>\n",
" <td>1154786048</td>\n",
" <td>NaN</td>\n",
" <td>@playboinathann: MY LIL BRO ATE AN EDIBLE THI...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176304</th>\n",
" <td>[😫, ✋]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340342812700</td>\n",
" <td>en</td>\n",
" <td>924752524871131100</td>\n",
" <td>NaN</td>\n",
" <td>I hate when I send a text or snap n I'm so anx...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176305</th>\n",
" <td>[👻, 🎃]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340334501900</td>\n",
" <td>en</td>\n",
" <td>868681065308631000</td>\n",
" <td>NaN</td>\n",
" <td>@LDNNOISE: Happy Halloween &lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176306</th>\n",
" <td>[😂, 🙁]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340355280900</td>\n",
" <td>en</td>\n",
" <td>419493819</td>\n",
" <td>NaN</td>\n",
" <td>The answer is no I have no plans and I never l...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176307</th>\n",
" <td>[😭]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340346937300</td>\n",
" <td>en</td>\n",
" <td>2457745952</td>\n",
" <td>9.256219e+17</td>\n",
" <td>@BeachBoy_Gab LMAOOO I'm so proud &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176308</th>\n",
" <td>[😢, 💔]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340338507800</td>\n",
" <td>en</td>\n",
" <td>1955767531</td>\n",
" <td>NaN</td>\n",
" <td>My cousin/little sister is leaving to San Fran...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176309</th>\n",
" <td>[🙃]</td>\n",
" <td>2017-11-01 07:23:35</td>\n",
" <td>925624344524361700</td>\n",
" <td>en</td>\n",
" <td>796490344581898200</td>\n",
" <td>NaN</td>\n",
" <td>Can't be alone w my thoughts tonight so just g...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176310</th>\n",
" <td>[✨]</td>\n",
" <td>2017-11-01 07:23:35</td>\n",
" <td>925624344511897600</td>\n",
" <td>en</td>\n",
" <td>825346617620328400</td>\n",
" <td>NaN</td>\n",
" <td>@ohmygirlpcs: Jine &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176311</th>\n",
" <td>[😂, 😂, 😂, 😂, 😂]</td>\n",
" <td>2017-11-01 07:23:36</td>\n",
" <td>925624348710285300</td>\n",
" <td>en</td>\n",
" <td>907808317124177900</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt; ambot!!!</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176312</th>\n",
" <td>[🚀]</td>\n",
" <td>2017-11-01 07:23:37</td>\n",
" <td>925624352921608200</td>\n",
" <td>en</td>\n",
" <td>1605371160</td>\n",
" <td>NaN</td>\n",
" <td>@WestHamUtd: It wasn't the result we wanted, ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176313</th>\n",
" <td>[👌, 👊, 🙌]</td>\n",
" <td>2017-11-01 07:23:37</td>\n",
" <td>925624352929910800</td>\n",
" <td>en</td>\n",
" <td>262162415</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt;🏽&lt;EMOJI&gt;🏽 1st of the month!!Happy 1st o...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176314</th>\n",
" <td>[😴]</td>\n",
" <td>2017-11-01 07:23:41</td>\n",
" <td>925624369715515400</td>\n",
" <td>en</td>\n",
" <td>574882525</td>\n",
" <td>NaN</td>\n",
" <td>#WednesdayWisdom ... stay in bed &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176315</th>\n",
" <td>[💀, 🖤]</td>\n",
" <td>2017-11-01 07:23:42</td>\n",
" <td>925624373905707000</td>\n",
" <td>en</td>\n",
" <td>2694136560</td>\n",
" <td>NaN</td>\n",
" <td>@Brandonwoelfel: Happy Halloween friends!&lt;EMO...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176316</th>\n",
" <td>[🔴]</td>\n",
" <td>2017-11-01 07:23:43</td>\n",
" <td>925624378078851100</td>\n",
" <td>en</td>\n",
" <td>799982767886872600</td>\n",
" <td>NaN</td>\n",
" <td>@carras16: Good win tonight and great to see ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176317</th>\n",
" <td>[😂, 😂, 😂, 😂]</td>\n",
" <td>2017-11-01 07:23:44</td>\n",
" <td>925624382269124600</td>\n",
" <td>en</td>\n",
" <td>556175173</td>\n",
" <td>9.254038e+17</td>\n",
" <td>@Louis_Tomlinson @NiallOfficial THIS IS GOLD. ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176318</th>\n",
" <td>[💖, 💗, 💞, 💕, 💘]</td>\n",
" <td>2017-11-01 07:23:44</td>\n",
" <td>925624382281547800</td>\n",
" <td>en</td>\n",
" <td>837324291637051400</td>\n",
" <td>NaN</td>\n",
" <td>@95duet: i love them sm☹&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176319</th>\n",
" <td>[😀]</td>\n",
" <td>2017-11-01 07:23:45</td>\n",
" <td>925624386455031800</td>\n",
" <td>en</td>\n",
" <td>1610265588</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt; thank you for the kind compliment</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176320</th>\n",
" <td>[😎]</td>\n",
" <td>2017-11-01 07:23:45</td>\n",
" <td>925624386454937600</td>\n",
" <td>en</td>\n",
" <td>4760724450</td>\n",
" <td>NaN</td>\n",
" <td>Enjoyed the silence &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176321</th>\n",
" <td>[✨]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390657572900</td>\n",
" <td>en</td>\n",
" <td>882858115636514800</td>\n",
" <td>NaN</td>\n",
" <td>OS: Spiderman Homecoming &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176322</th>\n",
" <td>[🤷]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390682849300</td>\n",
" <td>en</td>\n",
" <td>188129628</td>\n",
" <td>NaN</td>\n",
" <td>Go to hell &lt;EMOJI&gt;🏽‍♀️</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176323</th>\n",
" <td>[😘]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390666129400</td>\n",
" <td>en</td>\n",
" <td>2473135939</td>\n",
" <td>NaN</td>\n",
" <td>Thank you Yomi! &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176324</th>\n",
" <td>[😂]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390670106600</td>\n",
" <td>en</td>\n",
" <td>2373584209</td>\n",
" <td>9.256241e+17</td>\n",
" <td>@discopiggu Lol. Just enjoy the stars. Music K...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176325</th>\n",
" <td>[🙏]</td>\n",
" <td>2017-11-01 07:23:47</td>\n",
" <td>925624394860453900</td>\n",
" <td>en</td>\n",
" <td>40709255</td>\n",
" <td>NaN</td>\n",
" <td>@INS_Ecosystem: We are honored to have @eyal ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176326</th>\n",
" <td>[🔥]</td>\n",
" <td>2017-11-01 07:23:48</td>\n",
" <td>925624399063117800</td>\n",
" <td>en</td>\n",
" <td>889503742474768400</td>\n",
" <td>NaN</td>\n",
" <td>@EmiNiPr: I just love twitterMan's not hot &lt;E...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176327</th>\n",
" <td>[🙏]</td>\n",
" <td>2017-11-01 07:23:50</td>\n",
" <td>925624407459971100</td>\n",
" <td>en</td>\n",
" <td>181689756</td>\n",
" <td>NaN</td>\n",
" <td>Thoughts and prayers for NY&lt;EMOJI&gt;🏻 #NYCStrong</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176328</th>\n",
" <td>[💁]</td>\n",
" <td>2017-11-01 07:23:50</td>\n",
" <td>925624407460057100</td>\n",
" <td>en</td>\n",
" <td>60160788</td>\n",
" <td>9.254610e+17</td>\n",
" <td>@BrianyH I searched COCK, PENIS, SHLONG, WINKY...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176329</th>\n",
" <td>[🌠, 🎇, 🌠, 🌙]</td>\n",
" <td>2017-11-01 07:23:51</td>\n",
" <td>925624411620646900</td>\n",
" <td>en</td>\n",
" <td>4915042321</td>\n",
" <td>NaN</td>\n",
" <td>@eyesnoselips99: 170930 KMF &lt;EMOJI&gt;☄&lt;EMOJI&gt; ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176330</th>\n",
" <td>[🍃, 🌻, 🌻, 🍃, 🍃, 💐, 💐, 🍃, 🙋]</td>\n",
" <td>2017-11-01 07:23:53</td>\n",
" <td>925624420022063100</td>\n",
" <td>en</td>\n",
" <td>3792290725</td>\n",
" <td>9.256215e+17</td>\n",
" <td>@amitbarman520 Thank you so much&lt;EMOJI&gt;&lt;EMOJI&gt;...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>176331</th>\n",
" <td>[😻, 😫, 🤔, 💦, 💦, 💦]</td>\n",
" <td>2017-11-01 07:23:57</td>\n",
" <td>925624436799049700</td>\n",
" <td>en</td>\n",
" <td>612733752</td>\n",
" <td>NaN</td>\n",
" <td>@DarrionDaDon_: Vocals ? &lt;EMOJI&gt;High Notes ? ...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>176332 rows × 7 columns</p>\n",
"</div>"
],
"text/plain": [
" EMOJI datetime id \\\n",
"0 [🔥, 👏] 2017-11-01 13:29:00 925716304635547600 \n",
"1 [🤦] 2017-11-01 13:29:00 925716304664911900 \n",
"2 [🤧] 2017-11-01 13:29:00 925716304652308500 \n",
"3 [🐯, 🐥, 💕] 2017-11-01 13:29:01 925716308846661600 \n",
"4 [👍, 👍, 👍] 2017-11-01 13:29:02 925716313015586800 \n",
"5 [😒] 2017-11-01 13:29:02 925716313032364000 \n",
"6 [✅, 🍟] 2017-11-01 13:29:02 925716313036722200 \n",
"7 [👶] 2017-11-01 13:29:03 925716317243555800 \n",
"8 [😄] 2017-11-01 13:29:03 925716317214089200 \n",
"9 [🌟, 🌟] 2017-11-01 13:29:03 925716317218508800 \n",
"10 [📷] 2017-11-01 13:29:04 925716321416949800 \n",
"11 [🚨, 🚨, 🚨, 🚨, 🚨, 🚨] 2017-11-01 13:29:05 925716325615439900 \n",
"12 [💕] 2017-11-01 13:29:05 925716325615505400 \n",
"13 [🙏] 2017-11-01 13:29:06 925716329797054500 \n",
"14 [😩, 😩] 2017-11-01 13:29:06 925716329801310200 \n",
"15 [😭, 💓] 2017-11-01 13:29:02 925716313019965400 \n",
"16 [🙏, 🤕, 😂] 2017-11-01 13:29:06 925716329801371600 \n",
"17 [😂] 2017-11-01 13:29:05 925716325607133200 \n",
"18 [💯] 2017-11-01 13:29:07 925716334008082400 \n",
"19 [🙄] 2017-11-01 13:29:09 925716342401052700 \n",
"20 [😟, 😥, 😢] 2017-11-01 13:29:10 925716346570240000 \n",
"21 [😂, 😂, 😂] 2017-11-01 13:29:10 925716346603753500 \n",
"22 [👀] 2017-11-01 13:29:10 925716346595434500 \n",
"23 [🔎] 2017-11-01 13:29:11 925716350764331000 \n",
"24 [📣, 💖] 2017-11-01 13:29:12 925716354988236800 \n",
"25 [✨, 💖] 2017-11-01 13:29:12 925716354975465500 \n",
"26 [💪] 2017-11-01 13:29:13 925716359153041400 \n",
"27 [🌻] 2017-11-01 13:29:13 925716359182520300 \n",
"28 [🍁, 🌺, 🍂] 2017-11-01 13:29:15 925716367558545400 \n",
"29 [🍃] 2017-11-01 13:29:16 925716371735900200 \n",
"... ... ... ... \n",
"176302 [😭, 😭, 💘] 2017-11-01 07:23:31 925624327755591700 \n",
"176303 [🤣, 🤣, 🤣, 🤣] 2017-11-01 07:23:33 925624336156938200 \n",
"176304 [😫, ✋] 2017-11-01 07:23:34 925624340342812700 \n",
"176305 [👻, 🎃] 2017-11-01 07:23:34 925624340334501900 \n",
"176306 [😂, 🙁] 2017-11-01 07:23:34 925624340355280900 \n",
"176307 [😭] 2017-11-01 07:23:34 925624340346937300 \n",
"176308 [😢, 💔] 2017-11-01 07:23:34 925624340338507800 \n",
"176309 [🙃] 2017-11-01 07:23:35 925624344524361700 \n",
"176310 [✨] 2017-11-01 07:23:35 925624344511897600 \n",
"176311 [😂, 😂, 😂, 😂, 😂] 2017-11-01 07:23:36 925624348710285300 \n",
"176312 [🚀] 2017-11-01 07:23:37 925624352921608200 \n",
"176313 [👌, 👊, 🙌] 2017-11-01 07:23:37 925624352929910800 \n",
"176314 [😴] 2017-11-01 07:23:41 925624369715515400 \n",
"176315 [💀, 🖤] 2017-11-01 07:23:42 925624373905707000 \n",
"176316 [🔴] 2017-11-01 07:23:43 925624378078851100 \n",
"176317 [😂, 😂, 😂, 😂] 2017-11-01 07:23:44 925624382269124600 \n",
"176318 [💖, 💗, 💞, 💕, 💘] 2017-11-01 07:23:44 925624382281547800 \n",
"176319 [😀] 2017-11-01 07:23:45 925624386455031800 \n",
"176320 [😎] 2017-11-01 07:23:45 925624386454937600 \n",
"176321 [✨] 2017-11-01 07:23:46 925624390657572900 \n",
"176322 [🤷] 2017-11-01 07:23:46 925624390682849300 \n",
"176323 [😘] 2017-11-01 07:23:46 925624390666129400 \n",
"176324 [😂] 2017-11-01 07:23:46 925624390670106600 \n",
"176325 [🙏] 2017-11-01 07:23:47 925624394860453900 \n",
"176326 [🔥] 2017-11-01 07:23:48 925624399063117800 \n",
"176327 [🙏] 2017-11-01 07:23:50 925624407459971100 \n",
"176328 [💁] 2017-11-01 07:23:50 925624407460057100 \n",
"176329 [🌠, 🎇, 🌠, 🌙] 2017-11-01 07:23:51 925624411620646900 \n",
"176330 [🍃, 🌻, 🌻, 🍃, 🍃, 💐, 💐, 🍃, 🙋] 2017-11-01 07:23:53 925624420022063100 \n",
"176331 [😻, 😫, 🤔, 💦, 💦, 💦] 2017-11-01 07:23:57 925624436799049700 \n",
"\n",
" lang person reply_to \\\n",
"0 en 31507978 NaN \n",
"1 en 231994649 NaN \n",
"2 en 78123421 NaN \n",
"3 en 860396470540705800 NaN \n",
"4 en 775381892321333200 NaN \n",
"5 en 3244140547 NaN \n",
"6 en 4891498023 NaN \n",
"7 en 2660072274 NaN \n",
"8 en 2592765104 NaN \n",
"9 en 3011085514 NaN \n",
"10 en 278737933 NaN \n",
"11 en 876587129077477400 NaN \n",
"12 en 2402588155 NaN \n",
"13 en 1546678166 NaN \n",
"14 en 824586253634981900 9.257162e+17 \n",
"15 en 1581953814 9.257088e+17 \n",
"16 en 1737878390 NaN \n",
"17 en 1001999683 NaN \n",
"18 en 745222369183043600 NaN \n",
"19 en 798557155217539100 9.257147e+17 \n",
"20 en 744396039126421500 9.250629e+17 \n",
"21 en 2428265479 NaN \n",
"22 en 2523596372 NaN \n",
"23 en 2315896393 NaN \n",
"24 en 2316464875 NaN \n",
"25 en 2614916743 NaN \n",
"26 en 2887539044 NaN \n",
"27 en 721490010118205400 8.965900e+17 \n",
"28 en 520536723 9.257159e+17 \n",
"29 en 850852815941517300 NaN \n",
"... ... ... ... \n",
"176302 en 845085544589672400 9.256009e+17 \n",
"176303 en 1154786048 NaN \n",
"176304 en 924752524871131100 NaN \n",
"176305 en 868681065308631000 NaN \n",
"176306 en 419493819 NaN \n",
"176307 en 2457745952 9.256219e+17 \n",
"176308 en 1955767531 NaN \n",
"176309 en 796490344581898200 NaN \n",
"176310 en 825346617620328400 NaN \n",
"176311 en 907808317124177900 NaN \n",
"176312 en 1605371160 NaN \n",
"176313 en 262162415 NaN \n",
"176314 en 574882525 NaN \n",
"176315 en 2694136560 NaN \n",
"176316 en 799982767886872600 NaN \n",
"176317 en 556175173 9.254038e+17 \n",
"176318 en 837324291637051400 NaN \n",
"176319 en 1610265588 NaN \n",
"176320 en 4760724450 NaN \n",
"176321 en 882858115636514800 NaN \n",
"176322 en 188129628 NaN \n",
"176323 en 2473135939 NaN \n",
"176324 en 2373584209 9.256241e+17 \n",
"176325 en 40709255 NaN \n",
"176326 en 889503742474768400 NaN \n",
"176327 en 181689756 NaN \n",
"176328 en 60160788 9.254610e+17 \n",
"176329 en 4915042321 NaN \n",
"176330 en 3792290725 9.256215e+17 \n",
"176331 en 612733752 NaN \n",
"\n",
" text \n",
"0 fashionbombdaily's photo <EMOJI><EMOJI>🏼 \n",
"1 Its scary how on point my horoscope be <EMOJI... \n",
"2 @mvyn_: Them boring ass licks, you supposed t... \n",
"3 @taehyungpic: #vmin s unit shoot <EMOJI><EM... \n",
"4 @risa_del: Can I just say how much Meng respe... \n",
"5 @Melodia_Muse: omg did y'all just cut off Red... \n",
"6 @Harrys1DEmpire: Follow everyone who LIKES th... \n",
"7 @taehyungpic: he is so smollll <EMOJI>©rocket... \n",
"8 Woooaaaahhh <EMOJI> \n",
"9 @mymermaidIagoon: <EMOJI> HURRY! For your cha... \n",
"10 <EMOJI> vivalcli: Portraits by Zhao Guojing an... \n",
"11 @BillRatchet: <EMOJI><EMOJI><EMOJI>NO NUT NOV... \n",
"12 @mysweetapplexxx: ❤Happy Birthday, @GoodGalC... \n",
"13 @pourlui: @realDonaldTrump @SSNjl @foxandfrie... \n",
"14 @hiphopphiIes i wanna know too<EMOJI><EMOJI> \n",
"15 @WizMommma veda was yoda too <EMOJI><EMOJI> \n",
"16 @TiagoIlori4: Me and my family are humbled by... \n",
"17 Im less stressed about turning 30 now <EMOJI>... \n",
"18 Full charged. <EMOJI> \n",
"19 @SeaDimon Thats part of the problem, (they) @... \n",
"20 @Ian_khetye got me emotional there<EMOJI><EMOJ... \n",
"21 @30SECONDFlGHTS: BRUHHH <EMOJI><EMOJI><EMOJI> \n",
"22 @BasebaIINation: Game 7 who you got?<EMOJI> f... \n",
"23 @JudicialWatch: JW says DOJ &amp; Congress ar... \n",
"24 @cssconfau: ✍🏻 <EMOJI> Ever thought about spe... \n",
"25 @btspat_: Deal for Young Forever: Night Vers... \n",
"26 @DFC_SARAVANA: @dhanushkraja Reached 6 Millio... \n",
"27 back to the yellow <EMOJI> \n",
"28 @Dimafadma Happy month to you and your loved o... \n",
"29 Maxx and Patricia. Family hangs at For Good He... \n",
"... ... \n",
"176302 @peachshua1230 Awww <EMOJI> Ajsksjdjd im smili... \n",
"176303 @playboinathann: MY LIL BRO ATE AN EDIBLE THI... \n",
"176304 I hate when I send a text or snap n I'm so anx... \n",
"176305 @LDNNOISE: Happy Halloween <EMOJI><EMOJI> \n",
"176306 The answer is no I have no plans and I never l... \n",
"176307 @BeachBoy_Gab LMAOOO I'm so proud <EMOJI> \n",
"176308 My cousin/little sister is leaving to San Fran... \n",
"176309 Can't be alone w my thoughts tonight so just g... \n",
"176310 @ohmygirlpcs: Jine <EMOJI> \n",
"176311 <EMOJI><EMOJI><EMOJI><EMOJI><EMOJI> ambot!!! \n",
"176312 @WestHamUtd: It wasn't the result we wanted, ... \n",
"176313 <EMOJI>🏽<EMOJI>🏽 1st of the month!!Happy 1st o... \n",
"176314 #WednesdayWisdom ... stay in bed <EMOJI> \n",
"176315 @Brandonwoelfel: Happy Halloween friends!<EMO... \n",
"176316 @carras16: Good win tonight and great to see ... \n",
"176317 @Louis_Tomlinson @NiallOfficial THIS IS GOLD. ... \n",
"176318 @95duet: i love them sm☹<EMOJI><EMOJI><EMOJI... \n",
"176319 <EMOJI> thank you for the kind compliment \n",
"176320 Enjoyed the silence <EMOJI> \n",
"176321 OS: Spiderman Homecoming <EMOJI> \n",
"176322 Go to hell <EMOJI>🏽‍♀️ \n",
"176323 Thank you Yomi! <EMOJI> \n",
"176324 @discopiggu Lol. Just enjoy the stars. Music K... \n",
"176325 @INS_Ecosystem: We are honored to have @eyal ... \n",
"176326 @EmiNiPr: I just love twitterMan's not hot <E... \n",
"176327 Thoughts and prayers for NY<EMOJI>🏻 #NYCStrong \n",
"176328 @BrianyH I searched COCK, PENIS, SHLONG, WINKY... \n",
"176329 @eyesnoselips99: 170930 KMF <EMOJI>☄<EMOJI> ... \n",
"176330 @amitbarman520 Thank you so much<EMOJI><EMOJI>... \n",
"176331 @DarrionDaDon_: Vocals ? <EMOJI>High Notes ? ... \n",
"\n",
"[176332 rows x 7 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"twitter_data = pd.read_json(json_files[0])\n",
"twitter_data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* extracting emojis and text"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"emojis = twitter_data['EMOJI']\n",
"plain_text = twitter_data['text']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* make our plain text more \"plain\":\n",
" * removing the keyword `<EMOJI>` (just for the beginning)\n",
" * removing remaining useless emojis, like skin modifier etc."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"# defining blacklist for modifier emojis:\n",
"emoji_blacklist = set([\n",
" chr(0x1F3FB),\n",
" chr(0x1F3FC),\n",
" chr(0x1F3FD),\n",
" chr(0x1F3FE),\n",
" chr(0x1F3FF),\n",
" chr(0x2642),\n",
" chr(0x2640)\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"# filtering them and the EMOJI keyword out:\n",
"plain_text = plain_text.str.replace(\"<EMOJI>\",\"\").str.replace(\"[\" + \"\".join(list(emoji_blacklist)) + \"]\",\"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# making a set of the emoji list:\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* trying a multi label vectorizer"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"mlb = MultiLabelBinarizer()\n",
"\n",
"labels = mlb.fit_transform(emojis)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* generating train and test set:"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"X1, Xt1, y1, yt1 = train_test_split(plain_text, labels, test_size=0.1, random_state=4222)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"vectorizer = TfidfVectorizer(stop_words='english')\n",
"vec_train = vectorizer.fit_transform(X1)\n",
"vec_test = vectorizer.transform(Xt1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* train. this can take a very long time..."
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.neural_network import MLPClassifier as MLP\n",
"from sklearn.multiclass import OneVsRestClassifier as OVRC\n",
"from sklearn.tree import DecisionTreeClassifier as DTC"
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [],
"source": [
"def train(max_size = 10000):\n",
" clf = OVRC(DTC())\n",
"\n",
" clf.fit(vec_train[:max_size], y1[:max_size])\n",
" \n",
" return clf"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"clf = train()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* make a prediction and store it in a csv file:"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>predict</th>\n",
" <th>teacher</th>\n",
" <th>text</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>57624</th>\n",
" <td>(💥, 😡)</td>\n",
" <td>(💥, 😡)</td>\n",
" <td>@RealRoseTaylor1: IMO A terrorist doesnt hav...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>58180</th>\n",
" <td>()</td>\n",
" <td>(😒,)</td>\n",
" <td>@mog7546: NOT BREAKING NEWS - #KELLY's STUPID...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>122719</th>\n",
" <td>(😂, 🤦)</td>\n",
" <td>(😂, 🤦)</td>\n",
" <td>@SpecialMonroe: Im more of like a “ yes nigg...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>132425</th>\n",
" <td>()</td>\n",
" <td>(🤷,)</td>\n",
" <td>@xo_sweetttnesss Basically </td>\n",
" </tr>\n",
" <tr>\n",
" <th>95629</th>\n",
" <td>(😂,)</td>\n",
" <td>(😂,)</td>\n",
" <td>@JVino_: Never should of gave niggas technolo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>144805</th>\n",
" <td>(😂,)</td>\n",
" <td>(🤷,)</td>\n",
" <td>@_Erickaaaaaaa: People mad iffy Im not aroun...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50997</th>\n",
" <td>(💕, 🛫)</td>\n",
" <td>(💕, 🛫)</td>\n",
" <td>@MURASAKI_9394: &amp;lt;preview&amp;gt; 171101 ICN@mt...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>52294</th>\n",
" <td>()</td>\n",
" <td>(😍, 😣)</td>\n",
" <td>@JaDinePhils: Rogue x Gambit x Deadpool No cu...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>57600</th>\n",
" <td>(⚾, 😬)</td>\n",
" <td>(💙,)</td>\n",
" <td>@CailenHanzlik: So hyped watching my former t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12709</th>\n",
" <td>(😂,)</td>\n",
" <td>(💀,)</td>\n",
" <td>@C_eazyy @RuthlessLuke_ LMAO DED let a nigga s...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>159921</th>\n",
" <td>(😍,)</td>\n",
" <td>(👎,)</td>\n",
" <td>@ShawnHayes_Real @Mr_Ghostly @CR_Shelton @Rojo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1297</th>\n",
" <td>(💀, 😂)</td>\n",
" <td>(💀, 😂)</td>\n",
" <td>@Ieansquad: he really solved the equation I c...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>124445</th>\n",
" <td>()</td>\n",
" <td>(😩,)</td>\n",
" <td>It is really breaking my heart. Imma stick by ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>150159</th>\n",
" <td>()</td>\n",
" <td>(🐰,)</td>\n",
" <td>@jikookarchive: : aw sexy~: (jimin your cleav...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>170959</th>\n",
" <td>(✅, 🔥)</td>\n",
" <td>(✅, 🍋, 🔥)</td>\n",
" <td>@Harrys1DEmpire: 1: Like this2: Follow all th...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>46992</th>\n",
" <td>(🎬, 💋, 🔗)</td>\n",
" <td>(🎬, 💋, 🔗)</td>\n",
" <td>@ONGSEONGWU_TH: [] #WANNAONE X Innisfree | On...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>130772</th>\n",
" <td>(🎃,)</td>\n",
" <td>(🐺,)</td>\n",
" <td>@lbtreiman: Well i certainly didn't get 'em a...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63733</th>\n",
" <td>()</td>\n",
" <td>(🎃,)</td>\n",
" <td>@gamngbizzle: Follow everyone who retweets t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22026</th>\n",
" <td>()</td>\n",
" <td>(💕, 📷)</td>\n",
" <td>@fshion_me: Love it【@perlinek】Coupon code: zo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>109588</th>\n",
" <td>(😍, 😳)</td>\n",
" <td>(😍,)</td>\n",
" <td>@SoCuteBabies: Those eyes</td>\n",
" </tr>\n",
" <tr>\n",
" <th>167358</th>\n",
" <td>(🎃,)</td>\n",
" <td>(👻, 😈)</td>\n",
" <td>@Lepetit626: Happy Halloween from the cutest ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>132276</th>\n",
" <td>()</td>\n",
" <td>(💀,)</td>\n",
" <td>@marceoooo shuttup</td>\n",
" </tr>\n",
" <tr>\n",
" <th>150897</th>\n",
" <td>(😱, 🙈, 🙉)</td>\n",
" <td>(🔥, 😈)</td>\n",
" <td>@instaWANKtube: Drilling babe@mistersex17 @hq...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15211</th>\n",
" <td>(🎃,)</td>\n",
" <td>(🎃,)</td>\n",
" <td>@G_Eazy: Hopped out of the millennium falcon ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>58157</th>\n",
" <td>(⌛, 💊)</td>\n",
" <td>(⌛, 💊)</td>\n",
" <td>@NBCNewsNight: Stranger Things Neuro Upside D...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>87036</th>\n",
" <td>(🔥,)</td>\n",
" <td>(😂,)</td>\n",
" <td>I ran a lot harder back then, as well.I join v...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68909</th>\n",
" <td>()</td>\n",
" <td>(🚗,)</td>\n",
" <td>@Holy_Boost: SUMMER BOOST case GA! -Like+ ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>70063</th>\n",
" <td>()</td>\n",
" <td>(😭,)</td>\n",
" <td>I love you so much meliflua ❤️</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53508</th>\n",
" <td>()</td>\n",
" <td>(💥,)</td>\n",
" <td>@SMMmarkeing786: Get FREE CBD OIL Trial Bott...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>91590</th>\n",
" <td>(👀,)</td>\n",
" <td>(👀,)</td>\n",
" <td>@dezidoesit: Shea Moisture did not come to pl...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>82637</th>\n",
" <td>()</td>\n",
" <td>()</td>\n",
" <td>@KissaSins: Follow me on Instagram!!</td>\n",
" </tr>\n",
" <tr>\n",
" <th>151805</th>\n",
" <td>(👀,)</td>\n",
" <td>(👀,)</td>\n",
" <td>@dezidoesit: Shea Moisture did not come to pl...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25914</th>\n",
" <td>()</td>\n",
" <td>(🔘,)</td>\n",
" <td>@Heartless600904: He knows how badly he hurt ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>91602</th>\n",
" <td>()</td>\n",
" <td>(😡,)</td>\n",
" <td>@1776Stonewall @Frederick99m That's when I kne...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16911</th>\n",
" <td>()</td>\n",
" <td>(😩,)</td>\n",
" <td>Not even out of bed yet and Im ready to get o...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>94798</th>\n",
" <td>()</td>\n",
" <td>(🙃,)</td>\n",
" <td>nobody gets how lonely and hungry i get</td>\n",
" </tr>\n",
" <tr>\n",
" <th>112317</th>\n",
" <td>(💪,)</td>\n",
" <td>(🤗,)</td>\n",
" <td>@BIGAC_: new month, new money, new opportunit...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>72241</th>\n",
" <td>()</td>\n",
" <td>(😎,)</td>\n",
" <td>@ZeeTamil: Happy birthday @iamsrk #ZeeTamil #...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>160551</th>\n",
" <td>(✨, 💪)</td>\n",
" <td>(😭,)</td>\n",
" <td>@WORLDSTARNOW: Drake looks like he entering h...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>151889</th>\n",
" <td>(👅, 😻)</td>\n",
" <td>(😍,)</td>\n",
" <td>@GenderReveaIs: The moment of truth, hoop sty...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64998</th>\n",
" <td>(🐶, 💓)</td>\n",
" <td>(🐶, 💓)</td>\n",
" <td>@LA_Loving: Babys first Halloween</td>\n",
" </tr>\n",
" <tr>\n",
" <th>138015</th>\n",
" <td>()</td>\n",
" <td>(😂,)</td>\n",
" <td>@MikeChosen1: When you got a girl who doesn't...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>84447</th>\n",
" <td>()</td>\n",
" <td>(💕,)</td>\n",
" <td>@LiamPayne @pinkskiesljp @BestSimy @LiamPayne ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125899</th>\n",
" <td>()</td>\n",
" <td>(🙏,)</td>\n",
" <td>@Oh_Ashlaay We just have to pray to the baseba...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>47898</th>\n",
" <td>(😂,)</td>\n",
" <td>(😂,)</td>\n",
" <td>@JVino_: Never should of gave niggas technolo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7597</th>\n",
" <td>()</td>\n",
" <td>(😂,)</td>\n",
" <td>@junkiejunkrat Were doing goooooood. But fuck...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>173305</th>\n",
" <td>()</td>\n",
" <td>(🌈, 💛)</td>\n",
" <td>Ohh nice.</td>\n",
" </tr>\n",
" <tr>\n",
" <th>133827</th>\n",
" <td>(💙,)</td>\n",
" <td>(💙,)</td>\n",
" <td>@JenBluesnake: This is lovely. Unsworth's app...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>157487</th>\n",
" <td>()</td>\n",
" <td>(💻,)</td>\n",
" <td>@yesyours: : Vehicle Magnet #VehicleMagnet 🖨</td>\n",
" </tr>\n",
" <tr>\n",
" <th>142977</th>\n",
" <td>(🌸, 💓)</td>\n",
" <td>(🌸, 💓)</td>\n",
" <td>@namkook_daily: they are so cosy and soft ☁</td>\n",
" </tr>\n",
" <tr>\n",
" <th>47506</th>\n",
" <td>(😭,)</td>\n",
" <td>(💙, 🔥)</td>\n",
" <td>@Juli_SJ13: ••••••2017 SAVED ••••••#BlackSuit</td>\n",
" </tr>\n",
" <tr>\n",
" <th>164518</th>\n",
" <td>()</td>\n",
" <td>(😜,)</td>\n",
" <td>@marcogumabao: I hope you like your eggs SUNN...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96549</th>\n",
" <td>(🐶, 💓)</td>\n",
" <td>(🐶, 💓)</td>\n",
" <td>@LA_Loving: Babys first Halloween</td>\n",
" </tr>\n",
" <tr>\n",
" <th>69134</th>\n",
" <td>()</td>\n",
" <td>(😞, 😡)</td>\n",
" <td>Reading this @AshcroftBen with many mixed emot...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>116753</th>\n",
" <td>(💪,)</td>\n",
" <td>(🍋,)</td>\n",
" <td>@Caradelevingne: ❤️ NO_ONE EVER REALLY DIES ❤...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125133</th>\n",
" <td>(😑,)</td>\n",
" <td>(🖤, 😉)</td>\n",
" <td>@DannyTu3250187: @Kriszti7504 @filipina_krueg...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5081</th>\n",
" <td>(😔,)</td>\n",
" <td>(😘,)</td>\n",
" <td>@brysoxox: rt or gay</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62088</th>\n",
" <td>()</td>\n",
" <td>(💔,)</td>\n",
" <td>Why do say that hondas are for fboys??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32769</th>\n",
" <td>(😂, 😘, 🙏)</td>\n",
" <td>(😁,)</td>\n",
" <td>@menggalurks: Live na ba ang EB today? #ALDU...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>92385</th>\n",
" <td>(😍,)</td>\n",
" <td>(💔,)</td>\n",
" <td>OML, WHAT HAPPENED?!!</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>17634 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" predict teacher \\\n",
"57624 (💥, 😡) (💥, 😡) \n",
"58180 () (😒,) \n",
"122719 (😂, 🤦) (😂, 🤦) \n",
"132425 () (🤷,) \n",
"95629 (😂,) (😂,) \n",
"144805 (😂,) (🤷,) \n",
"50997 (💕, 🛫) (💕, 🛫) \n",
"52294 () (😍, 😣) \n",
"57600 (⚾, 😬) (💙,) \n",
"12709 (😂,) (💀,) \n",
"159921 (😍,) (👎,) \n",
"1297 (💀, 😂) (💀, 😂) \n",
"124445 () (😩,) \n",
"150159 () (🐰,) \n",
"170959 (✅, 🔥) (✅, 🍋, 🔥) \n",
"46992 (🎬, 💋, 🔗) (🎬, 💋, 🔗) \n",
"130772 (🎃,) (🐺,) \n",
"63733 () (🎃,) \n",
"22026 () (💕, 📷) \n",
"109588 (😍, 😳) (😍,) \n",
"167358 (🎃,) (👻, 😈) \n",
"132276 () (💀,) \n",
"150897 (😱, 🙈, 🙉) (🔥, 😈) \n",
"15211 (🎃,) (🎃,) \n",
"58157 (⌛, 💊) (⌛, 💊) \n",
"87036 (🔥,) (😂,) \n",
"68909 () (🚗,) \n",
"70063 () (😭,) \n",
"53508 () (💥,) \n",
"91590 (👀,) (👀,) \n",
"... ... ... \n",
"82637 () () \n",
"151805 (👀,) (👀,) \n",
"25914 () (🔘,) \n",
"91602 () (😡,) \n",
"16911 () (😩,) \n",
"94798 () (🙃,) \n",
"112317 (💪,) (🤗,) \n",
"72241 () (😎,) \n",
"160551 (✨, 💪) (😭,) \n",
"151889 (👅, 😻) (😍,) \n",
"64998 (🐶, 💓) (🐶, 💓) \n",
"138015 () (😂,) \n",
"84447 () (💕,) \n",
"125899 () (🙏,) \n",
"47898 (😂,) (😂,) \n",
"7597 () (😂,) \n",
"173305 () (🌈, 💛) \n",
"133827 (💙,) (💙,) \n",
"157487 () (💻,) \n",
"142977 (🌸, 💓) (🌸, 💓) \n",
"47506 (😭,) (💙, 🔥) \n",
"164518 () (😜,) \n",
"96549 (🐶, 💓) (🐶, 💓) \n",
"69134 () (😞, 😡) \n",
"116753 (💪,) (🍋,) \n",
"125133 (😑,) (🖤, 😉) \n",
"5081 (😔,) (😘,) \n",
"62088 () (💔,) \n",
"32769 (😂, 😘, 🙏) (😁,) \n",
"92385 (😍,) (💔,) \n",
"\n",
" text \n",
"57624 @RealRoseTaylor1: IMO A terrorist doesnt hav... \n",
"58180 @mog7546: NOT BREAKING NEWS - #KELLY's STUPID... \n",
"122719 @SpecialMonroe: Im more of like a “ yes nigg... \n",
"132425 @xo_sweetttnesss Basically \n",
"95629 @JVino_: Never should of gave niggas technolo... \n",
"144805 @_Erickaaaaaaa: People mad iffy Im not aroun... \n",
"50997 @MURASAKI_9394: &lt;preview&gt; 171101 ICN@mt... \n",
"52294 @JaDinePhils: Rogue x Gambit x Deadpool No cu... \n",
"57600 @CailenHanzlik: So hyped watching my former t... \n",
"12709 @C_eazyy @RuthlessLuke_ LMAO DED let a nigga s... \n",
"159921 @ShawnHayes_Real @Mr_Ghostly @CR_Shelton @Rojo... \n",
"1297 @Ieansquad: he really solved the equation I c... \n",
"124445 It is really breaking my heart. Imma stick by ... \n",
"150159 @jikookarchive: : aw sexy~: (jimin your cleav... \n",
"170959 @Harrys1DEmpire: 1: Like this2: Follow all th... \n",
"46992 @ONGSEONGWU_TH: [] #WANNAONE X Innisfree | On... \n",
"130772 @lbtreiman: Well i certainly didn't get 'em a... \n",
"63733 @gamngbizzle: Follow everyone who retweets t... \n",
"22026 @fshion_me: Love it【@perlinek】Coupon code: zo... \n",
"109588 @SoCuteBabies: Those eyes \n",
"167358 @Lepetit626: Happy Halloween from the cutest ... \n",
"132276 @marceoooo shuttup \n",
"150897 @instaWANKtube: Drilling babe@mistersex17 @hq... \n",
"15211 @G_Eazy: Hopped out of the millennium falcon ... \n",
"58157 @NBCNewsNight: Stranger Things Neuro Upside D... \n",
"87036 I ran a lot harder back then, as well.I join v... \n",
"68909 @Holy_Boost: SUMMER BOOST case GA! -Like+ ... \n",
"70063 I love you so much meliflua ❤️ \n",
"53508 @SMMmarkeing786: Get FREE CBD OIL Trial Bott... \n",
"91590 @dezidoesit: Shea Moisture did not come to pl... \n",
"... ... \n",
"82637 @KissaSins: Follow me on Instagram!! \n",
"151805 @dezidoesit: Shea Moisture did not come to pl... \n",
"25914 @Heartless600904: He knows how badly he hurt ... \n",
"91602 @1776Stonewall @Frederick99m That's when I kne... \n",
"16911 Not even out of bed yet and Im ready to get o... \n",
"94798 nobody gets how lonely and hungry i get \n",
"112317 @BIGAC_: new month, new money, new opportunit... \n",
"72241 @ZeeTamil: Happy birthday @iamsrk #ZeeTamil #... \n",
"160551 @WORLDSTARNOW: Drake looks like he entering h... \n",
"151889 @GenderReveaIs: The moment of truth, hoop sty... \n",
"64998 @LA_Loving: Babys first Halloween \n",
"138015 @MikeChosen1: When you got a girl who doesn't... \n",
"84447 @LiamPayne @pinkskiesljp @BestSimy @LiamPayne ... \n",
"125899 @Oh_Ashlaay We just have to pray to the baseba... \n",
"47898 @JVino_: Never should of gave niggas technolo... \n",
"7597 @junkiejunkrat Were doing goooooood. But fuck... \n",
"173305 Ohh nice. \n",
"133827 @JenBluesnake: This is lovely. Unsworth's app... \n",
"157487 @yesyours: : Vehicle Magnet #VehicleMagnet 🖨 \n",
"142977 @namkook_daily: they are so cosy and soft ☁ \n",
"47506 @Juli_SJ13: ••••••2017 SAVED ••••••#BlackSuit \n",
"164518 @marcogumabao: I hope you like your eggs SUNN... \n",
"96549 @LA_Loving: Babys first Halloween \n",
"69134 Reading this @AshcroftBen with many mixed emot... \n",
"116753 @Caradelevingne: ❤️ NO_ONE EVER REALLY DIES ❤... \n",
"125133 @DannyTu3250187: @Kriszti7504 @filipina_krueg... \n",
"5081 @brysoxox: rt or gay \n",
"62088 Why do say that hondas are for fboys?? \n",
"32769 @menggalurks: Live na ba ang EB today? #ALDU... \n",
"92385 OML, WHAT HAPPENED?!! \n",
"\n",
"[17634 rows x 3 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"pred = clf.predict(vectorizer.transform(Xt1))\n",
"\n",
"# build a dataframe to visualize test results:\n",
"testlist = pd.DataFrame({'text': Xt1, 'teacher': mlb.inverse_transform(yt1), 'predict': mlb.inverse_transform(pred)})\n",
"\n",
"display(testlist)"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [],
"source": [
"testlist.to_csv('test.csv')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* a simple output widget for testing:"
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1e6d156f9beb42ada3790d977beb2b7c",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>Text</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"Text(value='')"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b76a73c0fa7946f28fe91e18240484a4",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(Button(description='get smiley', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"out = widgets.Output()\n",
"\n",
"t = widgets.Text()\n",
"b = widgets.Button(\n",
" description='get smiley',\n",
" disabled=False,\n",
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
" tooltip='Click me',\n",
" icon='check'\n",
")\n",
"\n",
"\n",
"\n",
"def handle_submit(sender):\n",
" with out:\n",
" clear_output()\n",
" with out:\n",
" display(Markdown(\"# \" + str(mlb.inverse_transform(clf.predict(vectorizer.transform([t.value])))[0])))\n",
"\n",
"b.on_click(handle_submit)\n",
" \n",
"display(t)\n",
"display(widgets.VBox([b, out])) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}