nlp-lab/Project/simple_approach/simple_twitter_learning.ipynb

2610 lines
117 KiB
Plaintext
Raw Normal View History

2018-06-05 11:34:14 +02:00
{
"cells": [
{
"cell_type": "code",
2018-06-09 14:54:34 +02:00
"execution_count": 1,
2018-06-05 11:34:14 +02:00
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os\n",
"import glob\n",
2018-06-05 15:51:58 +02:00
"import json\n",
"import numpy as np\n",
"import itertools\n",
"import sklearn.utils as sku\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.preprocessing import MultiLabelBinarizer"
2018-06-05 11:34:14 +02:00
]
},
2018-06-10 14:41:41 +02:00
{
"cell_type": "code",
2018-06-10 15:06:45 +02:00
"execution_count": 2,
2018-06-10 14:41:41 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-10 17:31:43 +02:00
"import sys\n",
"sys.path.append(\"..\")\n",
2018-06-10 14:41:41 +02:00
"\n",
2018-06-10 17:31:43 +02:00
"from Tools.Emoji_Distance import sentiment_vector_to_emoji\n",
"from Tools.Emoji_Distance import emoji_to_sentiment_vector\n",
"\n",
"def emoji2sent(emoji_arr):\n",
" return np.array([emoji_to_sentiment_vector(e) for e in emoji_arr])\n",
"\n",
"def sent2emoji(sent_arr):\n",
" return [sentiment_vector_to_emoji(s) for s in sent_arr]"
2018-06-10 14:41:41 +02:00
]
},
2018-06-09 14:54:34 +02:00
{
"cell_type": "code",
2018-06-10 15:06:45 +02:00
"execution_count": 3,
2018-06-09 14:54:34 +02:00
"metadata": {},
"outputs": [],
"source": [
"SINGLE_LABEL = True"
]
},
2018-06-05 11:34:14 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# simple twitter approach\n",
"*for learning emoji usage by single (in the meaning of unconnected) twitter messages*"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## loading train data"
]
},
{
"cell_type": "code",
2018-06-10 15:06:45 +02:00
"execution_count": 4,
2018-06-05 11:34:14 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-09 14:54:34 +02:00
"data_root_folder = \"./data_en/\" # i created a symlink here"
2018-06-05 11:34:14 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get all json files in `data_root_folder`"
]
},
{
"cell_type": "code",
2018-06-10 15:06:45 +02:00
"execution_count": 5,
2018-06-05 11:34:14 +02:00
"metadata": {},
"outputs": [],
"source": [
"json_files = sorted(glob.glob(data_root_folder + \"/*.json\"))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* so far, only load the first file"
]
},
{
"cell_type": "code",
2018-06-10 15:06:45 +02:00
"execution_count": 6,
2018-06-05 11:34:14 +02:00
"metadata": {},
"outputs": [
{
2018-06-09 14:54:34 +02:00
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>EMOJI</th>\n",
" <th>HASHTAGS</th>\n",
" <th>LINKED_USER</th>\n",
" <th>datetime</th>\n",
" <th>id</th>\n",
" <th>lang</th>\n",
" <th>person</th>\n",
" <th>reply_to</th>\n",
" <th>text</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>[🔥, 👏]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:00</td>\n",
" <td>925716304635547600</td>\n",
" <td>en</td>\n",
" <td>31507978</td>\n",
" <td>NaN</td>\n",
" <td>fashionbombdaily's photo &lt;EMOJI&gt;&lt;EMOJI&gt;🏼</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>[🤦]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:00</td>\n",
" <td>925716304664911900</td>\n",
" <td>en</td>\n",
" <td>231994649</td>\n",
" <td>NaN</td>\n",
" <td>Its scary how on point my horoscope be &lt;EMOJI...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>[😄]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:03</td>\n",
" <td>925716317214089200</td>\n",
" <td>en</td>\n",
" <td>2592765104</td>\n",
" <td>NaN</td>\n",
" <td>Woooaaaahhh &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>[📷]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:04</td>\n",
" <td>925716321416949800</td>\n",
" <td>en</td>\n",
" <td>278737933</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt; vivalcli: Portraits by Zhao Guojing an...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>[😩, 😩]</td>\n",
" <td>[]</td>\n",
" <td>[@hiphopphiIes]</td>\n",
" <td>2017-11-01 13:29:06</td>\n",
" <td>925716329801310200</td>\n",
" <td>en</td>\n",
" <td>824586253634981900</td>\n",
" <td>9.257162e+17</td>\n",
" <td>&lt;USER&gt; i wanna know too&lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>[😭, 💓]</td>\n",
" <td>[]</td>\n",
" <td>[@WizMommma]</td>\n",
" <td>2017-11-01 13:29:02</td>\n",
" <td>925716313019965400</td>\n",
" <td>en</td>\n",
" <td>1581953814</td>\n",
" <td>9.257088e+17</td>\n",
" <td>&lt;USER&gt; veda was yoda too &lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>[😂]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:05</td>\n",
" <td>925716325607133200</td>\n",
" <td>en</td>\n",
" <td>1001999683</td>\n",
" <td>NaN</td>\n",
" <td>Im less stressed about turning 30 now &lt;EMOJI&gt;...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>[💯]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:07</td>\n",
" <td>925716334008082400</td>\n",
" <td>en</td>\n",
" <td>745222369183043600</td>\n",
" <td>NaN</td>\n",
" <td>Full charged. &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>[🙄]</td>\n",
" <td>[]</td>\n",
" <td>[@SeaDimon, @lsarsour]</td>\n",
" <td>2017-11-01 13:29:09</td>\n",
" <td>925716342401052700</td>\n",
" <td>en</td>\n",
" <td>798557155217539100</td>\n",
" <td>9.257147e+17</td>\n",
" <td>&lt;USER&gt; Thats part of the problem, (they) &lt;USE...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>[😟, 😥, 😢]</td>\n",
" <td>[]</td>\n",
" <td>[@Ian_khetye]</td>\n",
" <td>2017-11-01 13:29:10</td>\n",
" <td>925716346570240000</td>\n",
" <td>en</td>\n",
" <td>744396039126421500</td>\n",
" <td>9.250629e+17</td>\n",
" <td>&lt;USER&gt; got me emotional there&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EM...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>[🌻]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:13</td>\n",
" <td>925716359182520300</td>\n",
" <td>en</td>\n",
" <td>721490010118205400</td>\n",
" <td>8.965900e+17</td>\n",
" <td>back to the yellow &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>[🍁, 🌺, 🍂]</td>\n",
" <td>[]</td>\n",
" <td>[@Dimafadma]</td>\n",
" <td>2017-11-01 13:29:15</td>\n",
" <td>925716367558545400</td>\n",
" <td>en</td>\n",
" <td>520536723</td>\n",
" <td>9.257159e+17</td>\n",
" <td>&lt;USER&gt; Happy month to you and your loved ones ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>[🍃]</td>\n",
" <td>[#mortdale, #partofthefamily, #gorgeousboy]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:16</td>\n",
" <td>925716371735900200</td>\n",
" <td>en</td>\n",
" <td>850852815941517300</td>\n",
" <td>NaN</td>\n",
" <td>Maxx and Patricia. Family hangs at For Good He...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>[💭, 🤦]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:20</td>\n",
" <td>925716388513230800</td>\n",
" <td>en</td>\n",
" <td>914145041588867100</td>\n",
" <td>NaN</td>\n",
" <td>I need to STOP beating myself up with my thoug...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>[😍, 😘]</td>\n",
" <td>[#7YearsOfKMH2]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:20</td>\n",
" <td>925716388525645800</td>\n",
" <td>en</td>\n",
" <td>2425405622</td>\n",
" <td>NaN</td>\n",
" <td>Cutest Son &lt;EMOJI&gt;Roll no. 31 &lt;EMOJI&gt; &lt;HASHTAG&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>[😜]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:22</td>\n",
" <td>925716396931240000</td>\n",
" <td>en</td>\n",
" <td>4614871873</td>\n",
" <td>NaN</td>\n",
" <td>By the summer I should have everything up and ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>[😂, 🔥]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:23</td>\n",
" <td>925716401125331000</td>\n",
" <td>en</td>\n",
" <td>2831608345</td>\n",
" <td>NaN</td>\n",
" <td>I know my English is not that good but that do...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>[💕]</td>\n",
" <td>[]</td>\n",
" <td>[@yungbabytate]</td>\n",
" <td>2017-11-01 13:29:23</td>\n",
" <td>925716401133948900</td>\n",
" <td>en</td>\n",
" <td>788571974633009200</td>\n",
" <td>9.255778e+17</td>\n",
" <td>&lt;USER&gt; I &lt;EMOJI&gt; u mama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>[😉]</td>\n",
" <td>[]</td>\n",
" <td>[@cmckenney]</td>\n",
" <td>2017-11-01 13:29:23</td>\n",
" <td>925716401125544000</td>\n",
" <td>en</td>\n",
" <td>218307802</td>\n",
" <td>9.257115e+17</td>\n",
" <td>&lt;USER&gt; That picture was NOT taken this morning...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>[👅]</td>\n",
" <td>[#footfetishnation]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:25</td>\n",
" <td>925716409489002500</td>\n",
" <td>en</td>\n",
" <td>885261166146179100</td>\n",
" <td>NaN</td>\n",
" <td>Welcome to &lt;HASHTAG&gt; &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>[👌, 🙂]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:25</td>\n",
" <td>925716409497272300</td>\n",
" <td>en</td>\n",
" <td>831437760833609700</td>\n",
" <td>NaN</td>\n",
" <td>Awkward &lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>[🤗, 📺]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:26</td>\n",
" <td>925716413699854300</td>\n",
" <td>en</td>\n",
" <td>231664542</td>\n",
" <td>NaN</td>\n",
" <td>back at it with supernatural &lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>[💯]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:26</td>\n",
" <td>925716413679009800</td>\n",
" <td>en</td>\n",
" <td>3196847035</td>\n",
" <td>NaN</td>\n",
" <td>One of the best things I've learned was to sto...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>[👅, 💦, 🍑]</td>\n",
" <td>[]</td>\n",
" <td>[@ctrlpurp]</td>\n",
" <td>2017-11-01 13:29:29</td>\n",
" <td>925716426278735900</td>\n",
" <td>en</td>\n",
" <td>918492858352635900</td>\n",
" <td>9.257161e+17</td>\n",
" <td>&lt;USER&gt; Can I taste?&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>[💔]</td>\n",
" <td>[]</td>\n",
" <td>[@saunders_court1]</td>\n",
" <td>2017-11-01 13:29:30</td>\n",
" <td>925716430473039900</td>\n",
" <td>en</td>\n",
" <td>3471187337</td>\n",
" <td>9.257163e+17</td>\n",
" <td>&lt;USER&gt; we miss you ☹️&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>[🤐, 🤐, 🤐]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:31</td>\n",
" <td>925716434667184100</td>\n",
" <td>en</td>\n",
" <td>780060488600199200</td>\n",
" <td>NaN</td>\n",
" <td>Actually my bias in WJSN are Eunseo &amp;amp; Bona...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>[😴]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:32</td>\n",
" <td>925716438853345300</td>\n",
" <td>en</td>\n",
" <td>388380690</td>\n",
" <td>NaN</td>\n",
" <td>I so cannot be bothered with the rest of the d...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>[😂]</td>\n",
" <td>[]</td>\n",
" <td>[@xxxtentacion]</td>\n",
" <td>2017-11-01 13:29:35</td>\n",
" <td>925716451457163300</td>\n",
" <td>en</td>\n",
" <td>899320696869974000</td>\n",
" <td>NaN</td>\n",
" <td>&lt;USER&gt; 2lit4life&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>[😂, 🙄]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 13:29:35</td>\n",
" <td>925716451461357600</td>\n",
" <td>en</td>\n",
" <td>784790670</td>\n",
" <td>NaN</td>\n",
" <td>Im not stop saying that!&lt;EMOJI&gt;&lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>[🎉, 🎂, 🎈, 🎊, 🎁, 💜]</td>\n",
" <td>[]</td>\n",
" <td>[@justinerooney_]</td>\n",
" <td>2017-11-01 13:29:37</td>\n",
" <td>925716459828936700</td>\n",
" <td>en</td>\n",
" <td>3051266655</td>\n",
" <td>NaN</td>\n",
" <td>&lt;USER&gt; HAPPY BIHDAY &lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68703</th>\n",
" <td>[😕]</td>\n",
" <td>[#halloweencostumes]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:04</td>\n",
" <td>925624214522036200</td>\n",
" <td>en</td>\n",
" <td>1672876458</td>\n",
" <td>NaN</td>\n",
" <td>&lt;HASHTAG&gt; this one falls under the weird crazy...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68704</th>\n",
" <td>[😂, 😩]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:05</td>\n",
" <td>925624218682777600</td>\n",
" <td>en</td>\n",
" <td>382473866</td>\n",
" <td>NaN</td>\n",
" <td>I'm not allowed to have chocolates yet, then I...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68705</th>\n",
" <td>[😂]</td>\n",
" <td>[#MUFC]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:06</td>\n",
" <td>925624222889766900</td>\n",
" <td>en</td>\n",
" <td>893145405457911800</td>\n",
" <td>NaN</td>\n",
" <td>Manchester United manager Mourinho slams 'spec...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68706</th>\n",
" <td>[💖]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:07</td>\n",
" <td>925624227088121900</td>\n",
" <td>en</td>\n",
" <td>240378516</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt; en Bushwhick</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68707</th>\n",
" <td>[🌆, 👉, 🚖, 📞]</td>\n",
" <td>[#BurkeCentre]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:18</td>\n",
" <td>925624273237983200</td>\n",
" <td>en</td>\n",
" <td>784620573209002000</td>\n",
" <td>NaN</td>\n",
" <td>: &lt;HASHTAG&gt; &lt;EMOJI&gt; &lt;EMOJI&gt;&lt;EMOJI&gt; For Taxi &lt;E...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68708</th>\n",
" <td>[😁]</td>\n",
" <td>[]</td>\n",
" <td>[@mychosliaheart, @BarrettoJulia, @iamjoshuaga...</td>\n",
" <td>2017-11-01 07:23:18</td>\n",
" <td>925624273212805100</td>\n",
" <td>en</td>\n",
" <td>170998187</td>\n",
" <td>9.254136e+17</td>\n",
" <td>&lt;USER&gt; &lt;USER&gt; &lt;USER&gt; Look, Mammeh and Daddeh! ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68709</th>\n",
" <td>[😋]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:24</td>\n",
" <td>925624298395533300</td>\n",
" <td>en</td>\n",
" <td>1348667816</td>\n",
" <td>NaN</td>\n",
" <td>Life is so good with you &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68710</th>\n",
" <td>[👌, 🎃, 😘]</td>\n",
" <td>[#portlandoregon, #portlandhalloween, #carrie…]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:24</td>\n",
" <td>925624298378801200</td>\n",
" <td>en</td>\n",
" <td>722481645765300200</td>\n",
" <td>NaN</td>\n",
" <td>Happy Halloween! &lt;EMOJI&gt;🏽&lt;EMOJI&gt;&lt;EMOJI&gt; &lt;HASHT...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68711</th>\n",
" <td>[👍]</td>\n",
" <td>[]</td>\n",
" <td>[@8limbsbondi...]</td>\n",
" <td>2017-11-01 07:23:26</td>\n",
" <td>925624306779897900</td>\n",
" <td>en</td>\n",
" <td>2443251500</td>\n",
" <td>NaN</td>\n",
" <td>Some work on the ropes in todays boxing class...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68712</th>\n",
" <td>[😭]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:27</td>\n",
" <td>925624310974136300</td>\n",
" <td>en</td>\n",
" <td>2406186390</td>\n",
" <td>NaN</td>\n",
" <td>Scotty and Kristens halloween costumes &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68713</th>\n",
" <td>[😂]</td>\n",
" <td>[]</td>\n",
" <td>[@rfrandrea, @AdaaanAndyyy]</td>\n",
" <td>2017-11-01 07:23:30</td>\n",
" <td>925624323557146600</td>\n",
" <td>en</td>\n",
" <td>1039448149</td>\n",
" <td>9.256150e+17</td>\n",
" <td>&lt;USER&gt; &lt;USER&gt; May pre-month celebration sis &lt;E...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68714</th>\n",
" <td>[😭, 😭, 💘]</td>\n",
" <td>[]</td>\n",
" <td>[@peachshua1230]</td>\n",
" <td>2017-11-01 07:23:31</td>\n",
" <td>925624327755591700</td>\n",
" <td>en</td>\n",
" <td>845085544589672400</td>\n",
" <td>9.256009e+17</td>\n",
" <td>&lt;USER&gt; Awww &lt;EMOJI&gt; Ajsksjdjd im smiling like ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68715</th>\n",
" <td>[😫, ✋]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340342812700</td>\n",
" <td>en</td>\n",
" <td>924752524871131100</td>\n",
" <td>NaN</td>\n",
" <td>I hate when I send a text or snap n I'm so anx...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68716</th>\n",
" <td>[😂, 🙁]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340355280900</td>\n",
" <td>en</td>\n",
" <td>419493819</td>\n",
" <td>NaN</td>\n",
" <td>The answer is no I have no plans and I never l...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68717</th>\n",
" <td>[😭]</td>\n",
" <td>[]</td>\n",
" <td>[@BeachBoy_Gab]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340346937300</td>\n",
" <td>en</td>\n",
" <td>2457745952</td>\n",
" <td>9.256219e+17</td>\n",
" <td>&lt;USER&gt; LMAOOO I'm so proud &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68718</th>\n",
" <td>[😢, 💔]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:34</td>\n",
" <td>925624340338507800</td>\n",
" <td>en</td>\n",
" <td>1955767531</td>\n",
" <td>NaN</td>\n",
" <td>My cousin/little sister is leaving to San Fran...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68719</th>\n",
" <td>[🙃]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:35</td>\n",
" <td>925624344524361700</td>\n",
" <td>en</td>\n",
" <td>796490344581898200</td>\n",
" <td>NaN</td>\n",
" <td>Can't be alone w my thoughts tonight so just g...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68720</th>\n",
" <td>[😂, 😂, 😂, 😂, 😂]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:36</td>\n",
" <td>925624348710285300</td>\n",
" <td>en</td>\n",
" <td>907808317124177900</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt; ambot!!!</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68721</th>\n",
" <td>[👌, 👊, 🙌]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:37</td>\n",
" <td>925624352929910800</td>\n",
" <td>en</td>\n",
" <td>262162415</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt;🏽&lt;EMOJI&gt;🏽 1st of the month!!Happy 1st o...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68722</th>\n",
" <td>[😴]</td>\n",
" <td>[#WednesdayWisdom]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:41</td>\n",
" <td>925624369715515400</td>\n",
" <td>en</td>\n",
" <td>574882525</td>\n",
" <td>NaN</td>\n",
" <td>&lt;HASHTAG&gt; ... stay in bed &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68723</th>\n",
" <td>[😂, 😂, 😂, 😂]</td>\n",
" <td>[]</td>\n",
" <td>[@Louis_Tomlinson, @NiallOfficial]</td>\n",
" <td>2017-11-01 07:23:44</td>\n",
" <td>925624382269124600</td>\n",
" <td>en</td>\n",
" <td>556175173</td>\n",
" <td>9.254038e+17</td>\n",
" <td>&lt;USER&gt; &lt;USER&gt; THIS IS GOLD. GOLD. &lt;EMOJI&gt;&lt;EMOJ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68724</th>\n",
" <td>[😀]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:45</td>\n",
" <td>925624386455031800</td>\n",
" <td>en</td>\n",
" <td>1610265588</td>\n",
" <td>NaN</td>\n",
" <td>&lt;EMOJI&gt; thank you for the kind compliment</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68725</th>\n",
" <td>[😎]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:45</td>\n",
" <td>925624386454937600</td>\n",
" <td>en</td>\n",
" <td>4760724450</td>\n",
" <td>NaN</td>\n",
" <td>Enjoyed the silence &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68726</th>\n",
" <td>[✨]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390657572900</td>\n",
" <td>en</td>\n",
" <td>882858115636514800</td>\n",
" <td>NaN</td>\n",
" <td>OS: Spiderman Homecoming &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68727</th>\n",
" <td>[🤷]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390682849300</td>\n",
" <td>en</td>\n",
" <td>188129628</td>\n",
" <td>NaN</td>\n",
" <td>Go to hell &lt;EMOJI&gt;🏽‍♀️</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68728</th>\n",
" <td>[😘]</td>\n",
" <td>[]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390666129400</td>\n",
" <td>en</td>\n",
" <td>2473135939</td>\n",
" <td>NaN</td>\n",
" <td>Thank you Yomi! &lt;EMOJI&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68729</th>\n",
" <td>[😂]</td>\n",
" <td>[]</td>\n",
" <td>[@discopiggu]</td>\n",
" <td>2017-11-01 07:23:46</td>\n",
" <td>925624390670106600</td>\n",
" <td>en</td>\n",
" <td>2373584209</td>\n",
" <td>9.256241e+17</td>\n",
" <td>&lt;USER&gt; Lol. Just enjoy the stars. Music Kidhar...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68730</th>\n",
" <td>[🙏]</td>\n",
" <td>[#NYCStrong]</td>\n",
" <td>[]</td>\n",
" <td>2017-11-01 07:23:50</td>\n",
" <td>925624407459971100</td>\n",
" <td>en</td>\n",
" <td>181689756</td>\n",
" <td>NaN</td>\n",
" <td>Thoughts and prayers for NY&lt;EMOJI&gt;🏻 &lt;HASHTAG&gt;</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68731</th>\n",
" <td>[💁]</td>\n",
" <td>[#GreatSuccess]</td>\n",
" <td>[@BrianyH]</td>\n",
" <td>2017-11-01 07:23:50</td>\n",
" <td>925624407460057100</td>\n",
" <td>en</td>\n",
" <td>60160788</td>\n",
" <td>9.254610e+17</td>\n",
" <td>&lt;USER&gt; I searched COCK, PENIS, SHLONG, WINKY, ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68732</th>\n",
" <td>[🍃, 🌻, 🌻, 🍃, 🍃, 💐, 💐, 🍃, 🙋]</td>\n",
" <td>[]</td>\n",
" <td>[@amitbarman520]</td>\n",
" <td>2017-11-01 07:23:53</td>\n",
" <td>925624420022063100</td>\n",
" <td>en</td>\n",
" <td>3792290725</td>\n",
" <td>9.256215e+17</td>\n",
" <td>&lt;USER&gt; Thank you so much&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;EMOJI&gt;&lt;...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>68733 rows × 9 columns</p>\n",
"</div>"
],
"text/plain": [
" EMOJI \\\n",
"0 [🔥, 👏] \n",
"1 [🤦] \n",
"2 [😄] \n",
"3 [📷] \n",
"4 [😩, 😩] \n",
"5 [😭, 💓] \n",
"6 [😂] \n",
"7 [💯] \n",
"8 [🙄] \n",
"9 [😟, 😥, 😢] \n",
"10 [🌻] \n",
"11 [🍁, 🌺, 🍂] \n",
"12 [🍃] \n",
"13 [💭, 🤦] \n",
"14 [😍, 😘] \n",
"15 [😜] \n",
"16 [😂, 🔥] \n",
"17 [💕] \n",
"18 [😉] \n",
"19 [👅] \n",
"20 [👌, 🙂] \n",
"21 [🤗, 📺] \n",
"22 [💯] \n",
"23 [👅, 💦, 🍑] \n",
"24 [💔] \n",
"25 [🤐, 🤐, 🤐] \n",
"26 [😴] \n",
"27 [😂] \n",
"28 [😂, 🙄] \n",
"29 [🎉, 🎂, 🎈, 🎊, 🎁, 💜] \n",
"... ... \n",
"68703 [😕] \n",
"68704 [😂, 😩] \n",
"68705 [😂] \n",
"68706 [💖] \n",
"68707 [🌆, 👉, 🚖, 📞] \n",
"68708 [😁] \n",
"68709 [😋] \n",
"68710 [👌, 🎃, 😘] \n",
"68711 [👍] \n",
"68712 [😭] \n",
"68713 [😂] \n",
"68714 [😭, 😭, 💘] \n",
"68715 [😫, ✋] \n",
"68716 [😂, 🙁] \n",
"68717 [😭] \n",
"68718 [😢, 💔] \n",
"68719 [🙃] \n",
"68720 [😂, 😂, 😂, 😂, 😂] \n",
"68721 [👌, 👊, 🙌] \n",
"68722 [😴] \n",
"68723 [😂, 😂, 😂, 😂] \n",
"68724 [😀] \n",
"68725 [😎] \n",
"68726 [✨] \n",
"68727 [🤷] \n",
"68728 [😘] \n",
"68729 [😂] \n",
"68730 [🙏] \n",
"68731 [💁] \n",
"68732 [🍃, 🌻, 🌻, 🍃, 🍃, 💐, 💐, 🍃, 🙋] \n",
"\n",
" HASHTAGS \\\n",
"0 [] \n",
"1 [] \n",
"2 [] \n",
"3 [] \n",
"4 [] \n",
"5 [] \n",
"6 [] \n",
"7 [] \n",
"8 [] \n",
"9 [] \n",
"10 [] \n",
"11 [] \n",
"12 [#mortdale, #partofthefamily, #gorgeousboy] \n",
"13 [] \n",
"14 [#7YearsOfKMH2] \n",
"15 [] \n",
"16 [] \n",
"17 [] \n",
"18 [] \n",
"19 [#footfetishnation] \n",
"20 [] \n",
"21 [] \n",
"22 [] \n",
"23 [] \n",
"24 [] \n",
"25 [] \n",
"26 [] \n",
"27 [] \n",
"28 [] \n",
"29 [] \n",
"... ... \n",
"68703 [#halloweencostumes] \n",
"68704 [] \n",
"68705 [#MUFC] \n",
"68706 [] \n",
"68707 [#BurkeCentre] \n",
"68708 [] \n",
"68709 [] \n",
"68710 [#portlandoregon, #portlandhalloween, #carrie…] \n",
"68711 [] \n",
"68712 [] \n",
"68713 [] \n",
"68714 [] \n",
"68715 [] \n",
"68716 [] \n",
"68717 [] \n",
"68718 [] \n",
"68719 [] \n",
"68720 [] \n",
"68721 [] \n",
"68722 [#WednesdayWisdom] \n",
"68723 [] \n",
"68724 [] \n",
"68725 [] \n",
"68726 [] \n",
"68727 [] \n",
"68728 [] \n",
"68729 [] \n",
"68730 [#NYCStrong] \n",
"68731 [#GreatSuccess] \n",
"68732 [] \n",
"\n",
" LINKED_USER datetime \\\n",
"0 [] 2017-11-01 13:29:00 \n",
"1 [] 2017-11-01 13:29:00 \n",
"2 [] 2017-11-01 13:29:03 \n",
"3 [] 2017-11-01 13:29:04 \n",
"4 [@hiphopphiIes] 2017-11-01 13:29:06 \n",
"5 [@WizMommma] 2017-11-01 13:29:02 \n",
"6 [] 2017-11-01 13:29:05 \n",
"7 [] 2017-11-01 13:29:07 \n",
"8 [@SeaDimon, @lsarsour] 2017-11-01 13:29:09 \n",
"9 [@Ian_khetye] 2017-11-01 13:29:10 \n",
"10 [] 2017-11-01 13:29:13 \n",
"11 [@Dimafadma] 2017-11-01 13:29:15 \n",
"12 [] 2017-11-01 13:29:16 \n",
"13 [] 2017-11-01 13:29:20 \n",
"14 [] 2017-11-01 13:29:20 \n",
"15 [] 2017-11-01 13:29:22 \n",
"16 [] 2017-11-01 13:29:23 \n",
"17 [@yungbabytate] 2017-11-01 13:29:23 \n",
"18 [@cmckenney] 2017-11-01 13:29:23 \n",
"19 [] 2017-11-01 13:29:25 \n",
"20 [] 2017-11-01 13:29:25 \n",
"21 [] 2017-11-01 13:29:26 \n",
"22 [] 2017-11-01 13:29:26 \n",
"23 [@ctrlpurp] 2017-11-01 13:29:29 \n",
"24 [@saunders_court1] 2017-11-01 13:29:30 \n",
"25 [] 2017-11-01 13:29:31 \n",
"26 [] 2017-11-01 13:29:32 \n",
"27 [@xxxtentacion] 2017-11-01 13:29:35 \n",
"28 [] 2017-11-01 13:29:35 \n",
"29 [@justinerooney_] 2017-11-01 13:29:37 \n",
"... ... ... \n",
"68703 [] 2017-11-01 07:23:04 \n",
"68704 [] 2017-11-01 07:23:05 \n",
"68705 [] 2017-11-01 07:23:06 \n",
"68706 [] 2017-11-01 07:23:07 \n",
"68707 [] 2017-11-01 07:23:18 \n",
"68708 [@mychosliaheart, @BarrettoJulia, @iamjoshuaga... 2017-11-01 07:23:18 \n",
"68709 [] 2017-11-01 07:23:24 \n",
"68710 [] 2017-11-01 07:23:24 \n",
"68711 [@8limbsbondi...] 2017-11-01 07:23:26 \n",
"68712 [] 2017-11-01 07:23:27 \n",
"68713 [@rfrandrea, @AdaaanAndyyy] 2017-11-01 07:23:30 \n",
"68714 [@peachshua1230] 2017-11-01 07:23:31 \n",
"68715 [] 2017-11-01 07:23:34 \n",
"68716 [] 2017-11-01 07:23:34 \n",
"68717 [@BeachBoy_Gab] 2017-11-01 07:23:34 \n",
"68718 [] 2017-11-01 07:23:34 \n",
"68719 [] 2017-11-01 07:23:35 \n",
"68720 [] 2017-11-01 07:23:36 \n",
"68721 [] 2017-11-01 07:23:37 \n",
"68722 [] 2017-11-01 07:23:41 \n",
"68723 [@Louis_Tomlinson, @NiallOfficial] 2017-11-01 07:23:44 \n",
"68724 [] 2017-11-01 07:23:45 \n",
"68725 [] 2017-11-01 07:23:45 \n",
"68726 [] 2017-11-01 07:23:46 \n",
"68727 [] 2017-11-01 07:23:46 \n",
"68728 [] 2017-11-01 07:23:46 \n",
"68729 [@discopiggu] 2017-11-01 07:23:46 \n",
"68730 [] 2017-11-01 07:23:50 \n",
"68731 [@BrianyH] 2017-11-01 07:23:50 \n",
"68732 [@amitbarman520] 2017-11-01 07:23:53 \n",
"\n",
" id lang person reply_to \\\n",
"0 925716304635547600 en 31507978 NaN \n",
"1 925716304664911900 en 231994649 NaN \n",
"2 925716317214089200 en 2592765104 NaN \n",
"3 925716321416949800 en 278737933 NaN \n",
"4 925716329801310200 en 824586253634981900 9.257162e+17 \n",
"5 925716313019965400 en 1581953814 9.257088e+17 \n",
"6 925716325607133200 en 1001999683 NaN \n",
"7 925716334008082400 en 745222369183043600 NaN \n",
"8 925716342401052700 en 798557155217539100 9.257147e+17 \n",
"9 925716346570240000 en 744396039126421500 9.250629e+17 \n",
"10 925716359182520300 en 721490010118205400 8.965900e+17 \n",
"11 925716367558545400 en 520536723 9.257159e+17 \n",
"12 925716371735900200 en 850852815941517300 NaN \n",
"13 925716388513230800 en 914145041588867100 NaN \n",
"14 925716388525645800 en 2425405622 NaN \n",
"15 925716396931240000 en 4614871873 NaN \n",
"16 925716401125331000 en 2831608345 NaN \n",
"17 925716401133948900 en 788571974633009200 9.255778e+17 \n",
"18 925716401125544000 en 218307802 9.257115e+17 \n",
"19 925716409489002500 en 885261166146179100 NaN \n",
"20 925716409497272300 en 831437760833609700 NaN \n",
"21 925716413699854300 en 231664542 NaN \n",
"22 925716413679009800 en 3196847035 NaN \n",
"23 925716426278735900 en 918492858352635900 9.257161e+17 \n",
"24 925716430473039900 en 3471187337 9.257163e+17 \n",
"25 925716434667184100 en 780060488600199200 NaN \n",
"26 925716438853345300 en 388380690 NaN \n",
"27 925716451457163300 en 899320696869974000 NaN \n",
"28 925716451461357600 en 784790670 NaN \n",
"29 925716459828936700 en 3051266655 NaN \n",
"... ... ... ... ... \n",
"68703 925624214522036200 en 1672876458 NaN \n",
"68704 925624218682777600 en 382473866 NaN \n",
"68705 925624222889766900 en 893145405457911800 NaN \n",
"68706 925624227088121900 en 240378516 NaN \n",
"68707 925624273237983200 en 784620573209002000 NaN \n",
"68708 925624273212805100 en 170998187 9.254136e+17 \n",
"68709 925624298395533300 en 1348667816 NaN \n",
"68710 925624298378801200 en 722481645765300200 NaN \n",
"68711 925624306779897900 en 2443251500 NaN \n",
"68712 925624310974136300 en 2406186390 NaN \n",
"68713 925624323557146600 en 1039448149 9.256150e+17 \n",
"68714 925624327755591700 en 845085544589672400 9.256009e+17 \n",
"68715 925624340342812700 en 924752524871131100 NaN \n",
"68716 925624340355280900 en 419493819 NaN \n",
"68717 925624340346937300 en 2457745952 9.256219e+17 \n",
"68718 925624340338507800 en 1955767531 NaN \n",
"68719 925624344524361700 en 796490344581898200 NaN \n",
"68720 925624348710285300 en 907808317124177900 NaN \n",
"68721 925624352929910800 en 262162415 NaN \n",
"68722 925624369715515400 en 574882525 NaN \n",
"68723 925624382269124600 en 556175173 9.254038e+17 \n",
"68724 925624386455031800 en 1610265588 NaN \n",
"68725 925624386454937600 en 4760724450 NaN \n",
"68726 925624390657572900 en 882858115636514800 NaN \n",
"68727 925624390682849300 en 188129628 NaN \n",
"68728 925624390666129400 en 2473135939 NaN \n",
"68729 925624390670106600 en 2373584209 9.256241e+17 \n",
"68730 925624407459971100 en 181689756 NaN \n",
"68731 925624407460057100 en 60160788 9.254610e+17 \n",
"68732 925624420022063100 en 3792290725 9.256215e+17 \n",
"\n",
" text \n",
"0 fashionbombdaily's photo <EMOJI><EMOJI>🏼 \n",
"1 Its scary how on point my horoscope be <EMOJI... \n",
"2 Woooaaaahhh <EMOJI> \n",
"3 <EMOJI> vivalcli: Portraits by Zhao Guojing an... \n",
"4 <USER> i wanna know too<EMOJI><EMOJI> \n",
"5 <USER> veda was yoda too <EMOJI><EMOJI> \n",
"6 Im less stressed about turning 30 now <EMOJI>... \n",
"7 Full charged. <EMOJI> \n",
"8 <USER> Thats part of the problem, (they) <USE... \n",
"9 <USER> got me emotional there<EMOJI><EMOJI><EM... \n",
"10 back to the yellow <EMOJI> \n",
"11 <USER> Happy month to you and your loved ones ... \n",
"12 Maxx and Patricia. Family hangs at For Good He... \n",
"13 I need to STOP beating myself up with my thoug... \n",
"14 Cutest Son <EMOJI>Roll no. 31 <EMOJI> <HASHTAG> \n",
"15 By the summer I should have everything up and ... \n",
"16 I know my English is not that good but that do... \n",
"17 <USER> I <EMOJI> u mama \n",
"18 <USER> That picture was NOT taken this morning... \n",
"19 Welcome to <HASHTAG> <EMOJI> \n",
"20 Awkward <EMOJI><EMOJI> \n",
"21 back at it with supernatural <EMOJI><EMOJI> \n",
"22 One of the best things I've learned was to sto... \n",
"23 <USER> Can I taste?<EMOJI><EMOJI><EMOJI> \n",
"24 <USER> we miss you ☹️<EMOJI> \n",
"25 Actually my bias in WJSN are Eunseo &amp; Bona... \n",
"26 I so cannot be bothered with the rest of the d... \n",
"27 <USER> 2lit4life<EMOJI> \n",
"28 Im not stop saying that!<EMOJI><EMOJI> \n",
"29 <USER> HAPPY BIHDAY <EMOJI><EMOJI><EMOJI><EMOJ... \n",
"... ... \n",
"68703 <HASHTAG> this one falls under the weird crazy... \n",
"68704 I'm not allowed to have chocolates yet, then I... \n",
"68705 Manchester United manager Mourinho slams 'spec... \n",
"68706 <EMOJI> en Bushwhick \n",
"68707 : <HASHTAG> <EMOJI> <EMOJI><EMOJI> For Taxi <E... \n",
"68708 <USER> <USER> <USER> Look, Mammeh and Daddeh! ... \n",
"68709 Life is so good with you <EMOJI> \n",
"68710 Happy Halloween! <EMOJI>🏽<EMOJI><EMOJI> <HASHT... \n",
"68711 Some work on the ropes in todays boxing class... \n",
"68712 Scotty and Kristens halloween costumes <EMOJI> \n",
"68713 <USER> <USER> May pre-month celebration sis <E... \n",
"68714 <USER> Awww <EMOJI> Ajsksjdjd im smiling like ... \n",
"68715 I hate when I send a text or snap n I'm so anx... \n",
"68716 The answer is no I have no plans and I never l... \n",
"68717 <USER> LMAOOO I'm so proud <EMOJI> \n",
"68718 My cousin/little sister is leaving to San Fran... \n",
"68719 Can't be alone w my thoughts tonight so just g... \n",
"68720 <EMOJI><EMOJI><EMOJI><EMOJI><EMOJI> ambot!!! \n",
"68721 <EMOJI>🏽<EMOJI>🏽 1st of the month!!Happy 1st o... \n",
"68722 <HASHTAG> ... stay in bed <EMOJI> \n",
"68723 <USER> <USER> THIS IS GOLD. GOLD. <EMOJI><EMOJ... \n",
"68724 <EMOJI> thank you for the kind compliment \n",
"68725 Enjoyed the silence <EMOJI> \n",
"68726 OS: Spiderman Homecoming <EMOJI> \n",
"68727 Go to hell <EMOJI>🏽‍♀️ \n",
"68728 Thank you Yomi! <EMOJI> \n",
"68729 <USER> Lol. Just enjoy the stars. Music Kidhar... \n",
"68730 Thoughts and prayers for NY<EMOJI>🏻 <HASHTAG> \n",
"68731 <USER> I searched COCK, PENIS, SHLONG, WINKY, ... \n",
"68732 <USER> Thank you so much<EMOJI><EMOJI><EMOJI><... \n",
"\n",
"[68733 rows x 9 columns]"
]
},
2018-06-10 15:06:45 +02:00
"execution_count": 6,
2018-06-09 14:54:34 +02:00
"metadata": {},
"output_type": "execute_result"
2018-06-05 11:34:14 +02:00
}
],
"source": [
2018-06-05 17:30:41 +02:00
"twitter_data = pd.read_json(json_files[0], encoding=\"utf-8\")\n",
2018-06-05 11:34:14 +02:00
"twitter_data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* extracting emojis and text"
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 7,
2018-06-05 11:34:14 +02:00
"metadata": {},
"outputs": [],
"source": [
"emojis = twitter_data['EMOJI']\n",
"plain_text = twitter_data['text']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* make our plain text more \"plain\":\n",
" * removing the keyword `<EMOJI>` (just for the beginning)\n",
" * removing remaining useless emojis, like skin modifier etc."
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 8,
2018-06-05 11:34:14 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-05 15:51:58 +02:00
"# defining blacklist for modifier emojis:\n",
2018-06-05 11:34:14 +02:00
"emoji_blacklist = set([\n",
" chr(0x1F3FB),\n",
" chr(0x1F3FC),\n",
" chr(0x1F3FD),\n",
" chr(0x1F3FE),\n",
" chr(0x1F3FF),\n",
" chr(0x2642),\n",
" chr(0x2640)\n",
"])"
]
2018-06-05 15:51:58 +02:00
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 9,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
"source": [
"# filtering them and the EMOJI keyword out:\n",
2018-06-11 17:02:54 +02:00
"plain_text = plain_text.str.replace(\"(<EMOJI>|<USER>|<HASHTAG>)\",\"\").str.replace(\"[\" + \"\".join(list(emoji_blacklist)) + \"]\",\"\")"
2018-06-05 15:51:58 +02:00
]
},
2018-06-09 14:54:34 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-06-10 17:31:43 +02:00
"* defining different criterias for choosing a single emoji (currently `latest` is used)"
2018-06-09 14:54:34 +02:00
]
},
2018-06-05 15:51:58 +02:00
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 10,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-09 14:54:34 +02:00
"def latest(lst):\n",
" return lst[-1] if len(lst) > 0 else 'X' \n",
"def most_common(lst):\n",
" # trying to find the most common used emoji in the given lst\n",
" return max(set(lst), key=lst.count) if len(lst) > 0 else \"X\" # setting label to 'X' if there is an empty emoji list"
]
},
2018-06-10 17:31:43 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* convert all emojis to a sentiment vector"
]
},
2018-06-09 14:54:34 +02:00
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 11,
2018-06-09 14:54:34 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-10 17:31:43 +02:00
"labels = emoji2sent([latest(e) for e in emojis])\n"
]
},
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 12,
2018-06-10 17:31:43 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"68733"
]
},
2018-06-10 18:32:41 +02:00
"execution_count": 12,
2018-06-10 17:31:43 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(labels)"
]
},
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 13,
2018-06-10 17:31:43 +02:00
"metadata": {},
"outputs": [],
"source": [
"wrong_labels = np.isnan(np.linalg.norm(labels, axis=1))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* remove all data we have no label for"
]
},
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 14,
2018-06-10 17:31:43 +02:00
"metadata": {},
"outputs": [],
"source": [
"labels = labels[np.invert(wrong_labels)]\n",
"plain_text = plain_text[np.invert(wrong_labels)]\n",
"emojis = emojis[np.invert(wrong_labels)]"
]
},
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 15,
2018-06-10 17:31:43 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
2018-06-10 18:32:41 +02:00
"33368 33368 33368\n"
2018-06-10 17:31:43 +02:00
]
}
],
"source": [
"print(len(labels), len(emojis), len(plain_text))"
2018-06-05 15:51:58 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-06-09 14:54:34 +02:00
"* generate weights:"
2018-06-05 15:51:58 +02:00
]
},
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 16,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-09 14:54:34 +02:00
"# at first count over our table\n",
"emoji_count = {}\n",
"\n",
"\n",
"for e_list in emojis:\n",
" for e in set(e_list):\n",
" if e not in emoji_count:\n",
" emoji_count[e] = 0\n",
" emoji_count[e] += 1\n",
"\n",
"emoji_count\n",
"emoji_sum = sum([emoji_count[e] for e in emoji_count])\n",
"\n",
"emoji_weights = {}\n",
"for e in emoji_count:\n",
" # tfidf for emojis\n",
" emoji_weights[e] = np.log((emoji_sum / emoji_count[e]))\n",
2018-06-05 15:51:58 +02:00
"\n",
2018-06-09 14:54:34 +02:00
"weights_sum= sum([emoji_weights[x] for x in emoji_weights])\n",
" \n",
"# normalize:\n",
"for e in emoji_weights:\n",
" emoji_weights[e] = emoji_weights[e] / weights_sum\n",
"\n",
"emoji_weights['X'] = 0 # dummy values\n",
"emoji_count['X'] = 0"
2018-06-05 15:51:58 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-06-11 18:04:39 +02:00
"* most used emojis in Dataset"
2018-06-05 15:51:58 +02:00
]
},
{
"cell_type": "code",
2018-06-10 18:32:41 +02:00
"execution_count": 17,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
2018-06-11 18:04:39 +02:00
"source": [
"import operator"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('😂', 10182), ('😭', 3893), ('😍', 2866), ('😩', 1647), ('😊', 1450)]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"['😂', '😭', '😍', '😩', '😊']"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"sorted_emoji_count = list(reversed(sorted(emoji_count.items(), key=operator.itemgetter(1))))\n",
"display(sorted_emoji_count[:5])\n",
"\n",
"top_emojis = [x[0] for x in sorted_emoji_count[:5]]\n",
"display(top_emojis)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* only learn the most used ones:"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"17815 17815 17815\n"
]
}
],
"source": [
"in_top = [sentiment_vector_to_emoji(x) in top_emojis for x in labels]\n",
"labels = labels[in_top]\n",
"plain_text = plain_text[in_top]\n",
"emojis = emojis[in_top]\n",
"print(len(labels), len(emojis), len(plain_text))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* generating train and test set:"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
2018-06-05 15:51:58 +02:00
"source": [
"X1, Xt1, y1, yt1 = train_test_split(plain_text, labels, test_size=0.1, random_state=4222)"
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 21,
2018-06-09 14:54:34 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-10 17:31:43 +02:00
"#y1_weights = np.array([(sum([emoji_weights[e] for e in e_list]) / len(e_list)) if len(e_list) > 0 else 0 for e_list in sent2emoji(y1)])"
2018-06-09 14:54:34 +02:00
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 22,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
"source": [
"vectorizer = TfidfVectorizer(stop_words='english')\n",
"vec_train = vectorizer.fit_transform(X1)\n",
"vec_test = vectorizer.transform(Xt1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* train. this can take a very long time..."
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 23,
2018-06-05 15:51:58 +02:00
"metadata": {},
2018-06-11 18:04:39 +02:00
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
2018-06-05 15:51:58 +02:00
"source": [
"from sklearn.neural_network import MLPClassifier as MLP\n",
"from sklearn.multiclass import OneVsRestClassifier as OVRC\n",
2018-06-09 14:54:34 +02:00
"from sklearn.tree import DecisionTreeClassifier as DTC\n",
"\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense"
2018-06-05 15:51:58 +02:00
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 25,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-09 14:54:34 +02:00
"def train(max_size = 10000, layers=[(1024, 'relu'),(y1[0].shape[0],'softmax')], random_state=4222, ovrc=False, n_iter=5):\n",
" \n",
" model = Sequential()\n",
" \n",
" # build mlp layers:\n",
" keras_layers = []\n",
" first_layer = True\n",
" for layer in layers:\n",
" if first_layer:\n",
" model.add(Dense(units=layer[0], activation=layer[1], input_dim=vectorizer.transform([\" \"])[0]._shape[1]))\n",
" first_layer = False\n",
" else:\n",
" model.add(Dense(units=layer[0], activation=layer[1]))\n",
" \n",
" #mlp = MLPClassifier(layers=sknn_layers, random_state=random_state, verbose=True, n_iter=n_iter, batch_size=100)\n",
" \n",
" model.compile(loss='categorical_crossentropy',\n",
" optimizer='sgd',\n",
2018-06-10 18:32:41 +02:00
" metrics=['accuracy'],)\n",
2018-06-09 14:54:34 +02:00
" \n",
" clf = OVRC(model) if ovrc else model\n",
2018-06-05 15:51:58 +02:00
"\n",
2018-06-11 18:04:39 +02:00
" clf.fit(vec_train[:max_size].A, y1[:max_size], validation_split=0.2, epochs=n_iter)#, sample_weight=y1_weights[:max_size])\n",
2018-06-05 15:51:58 +02:00
" \n",
" return clf"
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 27,
2018-06-05 15:51:58 +02:00
"metadata": {},
2018-06-05 17:30:41 +02:00
"outputs": [
{
2018-06-09 14:54:34 +02:00
"name": "stdout",
2018-06-05 17:30:41 +02:00
"output_type": "stream",
"text": [
2018-06-11 18:04:39 +02:00
"Train on 800 samples, validate on 200 samples\n",
"Epoch 1/100\n",
"800/800 [==============================] - 1s 690us/step - loss: 1.0883 - acc: 0.6925 - val_loss: 1.0760 - val_acc: 0.7400\n",
"Epoch 2/100\n",
"800/800 [==============================] - 0s 527us/step - loss: 1.0732 - acc: 0.7150 - val_loss: 1.0628 - val_acc: 0.7400\n",
"Epoch 3/100\n",
"800/800 [==============================] - 0s 528us/step - loss: 1.0651 - acc: 0.7150 - val_loss: 1.0556 - val_acc: 0.7400\n",
"Epoch 4/100\n",
"800/800 [==============================] - 0s 534us/step - loss: 1.0610 - acc: 0.7150 - val_loss: 1.0515 - val_acc: 0.7400\n",
"Epoch 5/100\n",
"800/800 [==============================] - 0s 534us/step - loss: 1.0589 - acc: 0.7150 - val_loss: 1.0493 - val_acc: 0.7400\n",
"Epoch 6/100\n",
"800/800 [==============================] - 0s 511us/step - loss: 1.0579 - acc: 0.7150 - val_loss: 1.0481 - val_acc: 0.7400\n",
"Epoch 7/100\n",
"800/800 [==============================] - 0s 546us/step - loss: 1.0573 - acc: 0.7150 - val_loss: 1.0473 - val_acc: 0.7400\n",
"Epoch 8/100\n",
"800/800 [==============================] - 0s 545us/step - loss: 1.0571 - acc: 0.7150 - val_loss: 1.0469 - val_acc: 0.7400\n",
"Epoch 9/100\n",
"800/800 [==============================] - 0s 522us/step - loss: 1.0570 - acc: 0.7150 - val_loss: 1.0467 - val_acc: 0.7400\n",
"Epoch 10/100\n",
"800/800 [==============================] - 0s 529us/step - loss: 1.0569 - acc: 0.7150 - val_loss: 1.0466 - val_acc: 0.7400\n",
"Epoch 11/100\n",
"800/800 [==============================] - 0s 551us/step - loss: 1.0568 - acc: 0.7150 - val_loss: 1.0465 - val_acc: 0.7400\n",
"Epoch 12/100\n",
"800/800 [==============================] - 0s 529us/step - loss: 1.0568 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 13/100\n",
"800/800 [==============================] - 0s 546us/step - loss: 1.0568 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 14/100\n",
"800/800 [==============================] - 0s 609us/step - loss: 1.0567 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 15/100\n",
"800/800 [==============================] - 0s 610us/step - loss: 1.0567 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 16/100\n",
"800/800 [==============================] - 0s 540us/step - loss: 1.0567 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 17/100\n",
"800/800 [==============================] - 0s 518us/step - loss: 1.0567 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 18/100\n",
"800/800 [==============================] - 0s 547us/step - loss: 1.0567 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 19/100\n",
"800/800 [==============================] - 0s 543us/step - loss: 1.0566 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 20/100\n",
"800/800 [==============================] - 0s 550us/step - loss: 1.0566 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 21/100\n",
"800/800 [==============================] - 0s 551us/step - loss: 1.0566 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 22/100\n",
"800/800 [==============================] - 0s 552us/step - loss: 1.0566 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 23/100\n",
"800/800 [==============================] - 0s 620us/step - loss: 1.0566 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 24/100\n",
"800/800 [==============================] - 0s 587us/step - loss: 1.0566 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 25/100\n",
"800/800 [==============================] - 0s 608us/step - loss: 1.0565 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 26/100\n",
"800/800 [==============================] - 0s 538us/step - loss: 1.0565 - acc: 0.7150 - val_loss: 1.0463 - val_acc: 0.7400\n",
"Epoch 27/100\n",
"800/800 [==============================] - 0s 545us/step - loss: 1.0565 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 28/100\n",
"800/800 [==============================] - 0s 567us/step - loss: 1.0565 - acc: 0.7150 - val_loss: 1.0464 - val_acc: 0.7400\n",
"Epoch 29/100\n",
"256/800 [========>.....................] - ETA: 0s - loss: 1.0609 - acc: 0.6992"
2018-06-10 18:32:41 +02:00
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
2018-06-11 18:04:39 +02:00
"\u001b[0;32m<ipython-input-27-b77d66ff9b63>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'relu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'relu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'linear'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'softmax'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-25-b5cc9918b682>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(max_size, layers, random_state, ovrc, n_iter)\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOVRC\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0movrc\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mmax_size\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mmax_size\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#, sample_weight=y1_weights[:max_size])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mclf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m 1040\u001b[0m \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1041\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1042\u001b[0;31m validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m 1043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1044\u001b[0m def evaluate(self, x=None, y=None,\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 200\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 201\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2659\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2660\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2661\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2662\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2663\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2629\u001b[0m \u001b[0msymbol_vals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2630\u001b[0m session)\n\u001b[0;32m-> 2631\u001b[0;31m \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2632\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1449\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1450\u001b[0m return tf_session.TF_SessionRunCallable(\n\u001b[0;32m-> 1451\u001b[0;31m self._session._session, self._handle, args, status, None)\n\u001b[0m\u001b[1;32m 1452\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1453\u001b[0m return tf_session.TF_DeprecatedSessionRunCallable(\n",
2018-06-10 18:32:41 +02:00
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
2018-06-05 17:30:41 +02:00
]
2018-06-09 14:54:34 +02:00
}
],
"source": [
2018-06-11 18:04:39 +02:00
"clf = train(max_size=1000,layers=[(200, 'relu'),(200, 'relu'),(200, 'linear'),(y1[0].shape[0],'softmax')], n_iter=100)"
2018-06-05 15:51:58 +02:00
]
},
2018-06-05 15:57:26 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* make a prediction and store it in a csv file:"
]
},
2018-06-05 15:51:58 +02:00
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 26,
2018-06-09 14:54:34 +02:00
"metadata": {},
"outputs": [],
"source": [
"pred = clf.predict(vectorizer.transform(Xt1))"
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 36,
2018-06-09 14:54:34 +02:00
"metadata": {},
"outputs": [
{
2018-06-11 18:04:39 +02:00
"name": "stdout",
"output_type": "stream",
"text": [
"8.1356275e-06\n",
"0.02568319204819897\n"
]
2018-06-09 14:54:34 +02:00
}
],
"source": [
2018-06-11 18:04:39 +02:00
"print(np.linalg.norm(np.var(pred, axis=0)))\n",
"print(np.linalg.norm(np.var(labels)))\n"
2018-06-09 14:54:34 +02:00
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 28,
2018-06-05 17:30:41 +02:00
"metadata": {},
"outputs": [],
"source": [
"# build a dataframe to visualize test results:\n",
2018-06-10 15:06:45 +02:00
"testlist = pd.DataFrame({'text': Xt1, \n",
2018-06-10 17:31:43 +02:00
" 'teacher': sent2emoji(yt1),\n",
2018-06-10 18:32:41 +02:00
" 'teacher_sentiment': yt1.tolist(),\n",
" 'predict': sent2emoji(pred),\n",
" 'predicted_sentiment': pred.tolist()})"
2018-06-05 17:30:41 +02:00
]
},
{
"cell_type": "code",
2018-06-11 18:04:39 +02:00
"execution_count": 36,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
2018-06-11 18:04:39 +02:00
" <th>text</th>\n",
2018-06-05 15:51:58 +02:00
" <th>teacher</th>\n",
2018-06-10 18:32:41 +02:00
" <th>teacher_sentiment</th>\n",
2018-06-11 18:04:39 +02:00
" <th>predict</th>\n",
" <th>predicted_sentiment</th>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>7618</th>\n",
2018-06-11 18:04:39 +02:00
" <td>There's fucking snow outside</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😒</td>\n",
" <td>[0.21660649819494585, 0.5913357400722021, 0.19...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46037721633911133, 0.2677566707134247, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>6910</th>\n",
2018-06-11 18:04:39 +02:00
" <td>You look so animated bro! *Rimshot*, Get it? ...</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4636985957622528, 0.26727861166000366, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>35783</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Jon ordered it from a restaurant</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😭</td>\n",
" <td>[0.34310532030401736, 0.4364820846905538, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4635818898677826, 0.2673886716365814, 0.269...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>15623</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I just want to move back in with my mom</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😅</td>\n",
" <td>[0.47186147186147187, 0.2922077922077922, 0.23...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46267321705818176, 0.2673114538192749, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>12023</th>\n",
2018-06-11 18:04:39 +02:00
" <td>guys lets vote!!!</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😭</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.34310532030401736, 0.4364820846905538, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4621191620826721, 0.26737213134765625, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>15763</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Bruhhhh this man need a show ASAP!!!</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46237292885780334, 0.2675153613090515, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>57240</th>\n",
2018-06-11 18:04:39 +02:00
" <td>AND THE RICH SIT IN A LOW PLACE</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😎</td>\n",
" <td>[0.5981432360742706, 0.10477453580901856, 0.29...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4622046649456024, 0.2677682340145111, 0.270...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>2418</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Time to go to bed</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4639696180820465, 0.26709380745887756, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>66384</th>\n",
2018-06-11 18:04:39 +02:00
" <td>May sound like a hillbilly, but girls that can...</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😍</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.7296744771190439, 0.05173769460607014, 0.21...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4639129638671875, 0.26741719245910645, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>44639</th>\n",
2018-06-11 18:04:39 +02:00
" <td>IM DYING</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46023082733154297, 0.27063173055648804, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>54463</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Black forest hot chocolate is coming back to C...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>🙌</td>\n",
" <td>[0.6613545816733067, 0.10092961487383798, 0.23...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46065258979797363, 0.2685313820838928, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>38407</th>\n",
2018-06-11 18:04:39 +02:00
" <td>5 days till my 21st.</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😎</td>\n",
" <td>[0.5981432360742706, 0.10477453580901856, 0.29...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4624767005443573, 0.2678660750389099, 0.269...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>31342</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I'm just going to leave my Twitter acct open ...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😄</td>\n",
" <td>[0.5586552217453505, 0.13662374821173104, 0.30...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4628538191318512, 0.2673165500164032, 0.269...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>41255</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Oh god im so happy</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4628150463104248, 0.26765838265419006, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>19686</th>\n",
2018-06-11 18:04:39 +02:00
" <td>In my head</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😭</td>\n",
" <td>[0.34310532030401736, 0.4364820846905538, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4624774158000946, 0.26805710792541504, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>67193</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Jo, chilllllllll. This kid is mentally beati...</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4607327878475189, 0.26832473278045654, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>40650</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Im ready sis</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😇</td>\n",
" <td>[0.6666666666666666, 0.06666666666666667, 0.26...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4610658884048462, 0.26784956455230713, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>64416</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Sometimes we need to know how to respect our p...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😊</td>\n",
" <td>[0.7040175768989329, 0.059322033898305086, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46121352910995483, 0.27051451802253723, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>19525</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Right?</td>\n",
2018-06-10 17:31:43 +02:00
" <td>🙆</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.5964912280701754, 0.08771929824561403, 0.31...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46745917201042175, 0.265864759683609, 0.266...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>39024</th>\n",
2018-06-11 18:04:39 +02:00
" <td>priceless love how got back at you</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4606868028640747, 0.267996221780777, 0.2713...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>35342</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Right</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😫</td>\n",
" <td>[0.3404710920770878, 0.4860813704496788, 0.173...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46745917201042175, 0.265864759683609, 0.266...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>9715</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Opo pang i will!</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😇</td>\n",
" <td>[0.6666666666666666, 0.06666666666666667, 0.26...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.45438939332962036, 0.2717699408531189, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>40490</th>\n",
2018-06-11 18:04:39 +02:00
" <td>WOW whomever was tasked with vetting applica...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😮</td>\n",
" <td>[0.45555555555555555, 0.17777777777777778, 0.3...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46492910385131836, 0.2654315233230591, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>22064</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Thank you !</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😍</td>\n",
" <td>[0.7296744771190439, 0.05173769460607014, 0.21...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46712905168533325, 0.26472848653793335, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>53563</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Dat daurin Dan kwali from behind is the real ...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😄</td>\n",
" <td>[0.5586552217453505, 0.13662374821173104, 0.30...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4643474817276001, 0.2679416835308075, 0.267...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>67332</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I miss my baby so much ♥️</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😪</td>\n",
" <td>[0.3506224066390041, 0.4315352697095436, 0.217...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4613974094390869, 0.2683691382408142, 0.270...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>15404</th>\n",
2018-06-11 18:04:39 +02:00
" <td>alright i'm gonna go take a shower, let me kno...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😕</td>\n",
" <td>[0.20294117647058824, 0.6029411764705882, 0.19...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46027225255966187, 0.2696966826915741, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>30130</th>\n",
2018-06-11 18:04:39 +02:00
" <td>and neither are buying her music</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😳</td>\n",
" <td>[0.34515366430260047, 0.32742316784869974, 0.3...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46355295181274414, 0.26728588342666626, 0.2...</td>\n",
2018-06-10 18:32:41 +02:00
" </tr>\n",
" <tr>\n",
" <th>37242</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I saw that too as well! I feel to have it dur...</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46367916464805603, 0.26611489057540894, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>30533</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I had a mental break down but Im back like hey</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4640607535839081, 0.26755788922309875, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>...</td>\n",
" <td>...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>16955</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Bless up Esquivel finally update aeries</td>\n",
2018-06-10 18:32:41 +02:00
" <td>🙏</td>\n",
" <td>[0.4983755685510071, 0.08057179987004548, 0.42...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46489447355270386, 0.26702189445495605, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>36805</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Look just like my Mizuno I had in 88</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4630420207977295, 0.26803845167160034, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>56326</th>\n",
2018-06-11 18:04:39 +02:00
" <td>This felt like it happened yesterday.</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😢</td>\n",
" <td>[0.39118825100133514, 0.38451268357810414, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4617263972759247, 0.26921841502189636, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>40306</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Our Guest photo this week highlights the beaut...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😍</td>\n",
" <td>[0.7296744771190439, 0.05173769460607014, 0.21...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46200209856033325, 0.26775091886520386, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>51333</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I need to see what “the stranger things” is ab...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😌</td>\n",
" <td>[0.6240601503759399, 0.13984962406015036, 0.23...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46260306239128113, 0.26784563064575195, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>42339</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Plz can u make me a pc I dont have enough mo...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😔</td>\n",
" <td>[0.31784232365145226, 0.46390041493775935, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4600381553173065, 0.26839619874954224, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>7644</th>\n",
2018-06-11 18:04:39 +02:00
" <td>this is insulting</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😧</td>\n",
" <td>[0.32608695652173914, 0.391304347826087, 0.282...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4657018482685089, 0.2653878927230835, 0.268...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>13953</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I know this might sound crazy but I rather Don...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😐</td>\n",
" <td>[0.16296296296296298, 0.5555555555555556, 0.28...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4625128209590912, 0.26891759037971497, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>26460</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Just a couple of cute pugs hanging out</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😛</td>\n",
" <td>[0.6909090909090909, 0.08181818181818182, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46720531582832336, 0.26648813486099243, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>31571</th>\n",
2018-06-11 18:04:39 +02:00
" <td>She knows nothing will happen to her.. welcom...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😒</td>\n",
" <td>[0.21660649819494585, 0.5913357400722021, 0.19...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4603578746318817, 0.2692199945449829, 0.270...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>9536</th>\n",
2018-06-11 18:04:39 +02:00
" <td>happy raisin</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😭</td>\n",
" <td>[0.34310532030401736, 0.4364820846905538, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4630625247955322, 0.26809123158454895, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>19571</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I'm so disappointed</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😡</td>\n",
" <td>[0.35978835978835977, 0.533068783068783, 0.107...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4639342725276947, 0.26599037647247314, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>35934</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Nailed it.</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46198946237564087, 0.26730334758758545, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>4605</th>\n",
2018-06-11 18:04:39 +02:00
" <td>oh its more than a thing!</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46141430735588074, 0.268981009721756, 0.269...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>1913</th>\n",
2018-06-11 18:04:39 +02:00
" <td>stop it</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😭</td>\n",
" <td>[0.34310532030401736, 0.4364820846905538, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4630756378173828, 0.2665313482284546, 0.270...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>11827</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Have an awesome ❤️</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😄</td>\n",
" <td>[0.5586552217453505, 0.13662374821173104, 0.30...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46398892998695374, 0.26717185974121094, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>15181</th>\n",
2018-06-11 18:04:39 +02:00
" <td>when I see elbows ?</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.45438939332962036, 0.2717699408531189, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>32869</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Damn Jani back at it again with the BB-8 costu...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😹</td>\n",
" <td>[0.4406779661016949, 0.2983050847457627, 0.261...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4616232216358185, 0.2673903703689575, 0.270...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>40784</th>\n",
2018-06-11 18:04:39 +02:00
" <td>I think a big part of the reason Nick Wilde is...</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😻</td>\n",
" <td>[0.6906474820143885, 0.0671462829736211, 0.242...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46204957365989685, 0.26713815331459045, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>18611</th>\n",
2018-06-11 18:04:39 +02:00
" <td>My 12 year old brother on Altuve- “hes like t...</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4634704887866974, 0.2669770121574402, 0.269...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>28341</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Bruh like how she know the exact number?</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46148979663848877, 0.26869532465934753, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>41368</th>\n",
2018-06-11 18:04:39 +02:00
" <td>They starting early</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4605691730976105, 0.26995062828063965, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>35554</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Please come through</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😩</td>\n",
" <td>[0.22289823008849557, 0.5912610619469026, 0.18...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46297168731689453, 0.2675779163837433, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>27968</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Thank you</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😍</td>\n",
" <td>[0.7296744771190439, 0.05173769460607014, 0.21...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46712905168533325, 0.26472848653793335, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>36772</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Imagine if the Vikings had a top 10 QB</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😳</td>\n",
" <td>[0.34515366430260047, 0.32742316784869974, 0.3...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4609829783439636, 0.26704034209251404, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>7568</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Le costume de cupcakke.......</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😭</td>\n",
" <td>[0.34310532030401736, 0.4364820846905538, 0.22...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46219775080680847, 0.26761168241500854, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>62837</th>\n",
2018-06-11 18:04:39 +02:00
" <td>aM FKXNSK SCREAMING ODM</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46401336789131165, 0.2654089033603668, 0.27...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>34105</th>\n",
2018-06-11 18:04:39 +02:00
" <td>My birthday tomorrow</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😇</td>\n",
" <td>[0.6666666666666666, 0.06666666666666667, 0.26...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4665740728378296, 0.26650139689445496, 0.26...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>59069</th>\n",
2018-06-11 18:04:39 +02:00
" <td>im just tryna change your life</td>\n",
2018-06-10 18:32:41 +02:00
" <td>😉</td>\n",
" <td>[0.5634451019066403, 0.0992767915844839, 0.337...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.4622177183628082, 0.268806129693985, 0.2689...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" <tr>\n",
2018-06-10 18:32:41 +02:00
" <th>47088</th>\n",
2018-06-11 18:04:39 +02:00
" <td>Do I look like Im off 79th . Foh</td>\n",
2018-06-10 17:31:43 +02:00
" <td>😂</td>\n",
2018-06-10 18:32:41 +02:00
" <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n",
2018-06-11 18:04:39 +02:00
" <td>😬</td>\n",
" <td>[0.46383899450302124, 0.26802247762680054, 0.2...</td>\n",
2018-06-05 15:51:58 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
2018-06-10 18:32:41 +02:00
"<p>3337 rows × 5 columns</p>\n",
2018-06-05 15:51:58 +02:00
"</div>"
],
"text/plain": [
2018-06-11 18:04:39 +02:00
" text teacher \\\n",
"7618 There's fucking snow outside 😒 \n",
"6910 You look so animated bro! *Rimshot*, Get it? ... 😂 \n",
"35783 Jon ordered it from a restaurant 😭 \n",
"15623 I just want to move back in with my mom 😅 \n",
"12023 guys lets vote!!! 😭 \n",
"15763 Bruhhhh this man need a show ASAP!!! 😂 \n",
"57240 AND THE RICH SIT IN A LOW PLACE 😎 \n",
"2418 Time to go to bed 😂 \n",
"66384 May sound like a hillbilly, but girls that can... 😍 \n",
"44639 IM DYING 😂 \n",
"54463 Black forest hot chocolate is coming back to C... 🙌 \n",
"38407 5 days till my 21st. 😎 \n",
"31342 I'm just going to leave my Twitter acct open ... 😄 \n",
"41255 Oh god im so happy 😂 \n",
"19686 In my head 😭 \n",
"67193 Jo, chilllllllll. This kid is mentally beati... 😂 \n",
"40650 Im ready sis 😇 \n",
"64416 Sometimes we need to know how to respect our p... 😊 \n",
"19525 Right? 🙆 \n",
"39024 priceless love how got back at you 😂 \n",
"35342 Right 😫 \n",
"9715 Opo pang i will! 😇 \n",
"40490 WOW whomever was tasked with vetting applica... 😮 \n",
"22064 Thank you ! 😍 \n",
"53563 Dat daurin Dan kwali from behind is the real ... 😄 \n",
"67332 I miss my baby so much ♥️ 😪 \n",
"15404 alright i'm gonna go take a shower, let me kno... 😕 \n",
"30130 and neither are buying her music 😳 \n",
"37242 I saw that too as well! I feel to have it dur... 😂 \n",
"30533 I had a mental break down but Im back like hey 😂 \n",
"... ... ... \n",
"16955 Bless up Esquivel finally update aeries 🙏 \n",
"36805 Look just like my Mizuno I had in 88 😂 \n",
"56326 This felt like it happened yesterday. 😢 \n",
"40306 Our Guest photo this week highlights the beaut... 😍 \n",
"51333 I need to see what “the stranger things” is ab... 😌 \n",
"42339 Plz can u make me a pc I dont have enough mo... 😔 \n",
"7644 this is insulting 😧 \n",
"13953 I know this might sound crazy but I rather Don... 😐 \n",
"26460 Just a couple of cute pugs hanging out 😛 \n",
"31571 She knows nothing will happen to her.. welcom... 😒 \n",
"9536 happy raisin 😭 \n",
"19571 I'm so disappointed 😡 \n",
"35934 Nailed it. 😂 \n",
"4605 oh its more than a thing! 😂 \n",
"1913 stop it 😭 \n",
"11827 Have an awesome ❤️ 😄 \n",
"15181 when I see elbows ? 😂 \n",
"32869 Damn Jani back at it again with the BB-8 costu... 😹 \n",
"40784 I think a big part of the reason Nick Wilde is... 😻 \n",
"18611 My 12 year old brother on Altuve- “hes like t... 😂 \n",
"28341 Bruh like how she know the exact number? 😂 \n",
"41368 They starting early 😂 \n",
"35554 Please come through 😩 \n",
"27968 Thank you 😍 \n",
"36772 Imagine if the Vikings had a top 10 QB 😳 \n",
"7568 Le costume de cupcakke....... 😭 \n",
"62837 aM FKXNSK SCREAMING ODM 😂 \n",
"34105 My birthday tomorrow 😇 \n",
"59069 im just tryna change your life 😉 \n",
"47088 Do I look like Im off 79th . Foh 😂 \n",
2018-06-10 18:32:41 +02:00
"\n",
2018-06-11 18:04:39 +02:00
" teacher_sentiment predict \\\n",
"7618 [0.21660649819494585, 0.5913357400722021, 0.19... 😬 \n",
"6910 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"35783 [0.34310532030401736, 0.4364820846905538, 0.22... 😬 \n",
"15623 [0.47186147186147187, 0.2922077922077922, 0.23... 😬 \n",
"12023 [0.34310532030401736, 0.4364820846905538, 0.22... 😬 \n",
"15763 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"57240 [0.5981432360742706, 0.10477453580901856, 0.29... 😬 \n",
"2418 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"66384 [0.7296744771190439, 0.05173769460607014, 0.21... 😬 \n",
"44639 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"54463 [0.6613545816733067, 0.10092961487383798, 0.23... 😬 \n",
"38407 [0.5981432360742706, 0.10477453580901856, 0.29... 😬 \n",
"31342 [0.5586552217453505, 0.13662374821173104, 0.30... 😬 \n",
"41255 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"19686 [0.34310532030401736, 0.4364820846905538, 0.22... 😬 \n",
"67193 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"40650 [0.6666666666666666, 0.06666666666666667, 0.26... 😬 \n",
"64416 [0.7040175768989329, 0.059322033898305086, 0.2... 😬 \n",
"19525 [0.5964912280701754, 0.08771929824561403, 0.31... 😬 \n",
"39024 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"35342 [0.3404710920770878, 0.4860813704496788, 0.173... 😬 \n",
"9715 [0.6666666666666666, 0.06666666666666667, 0.26... 😬 \n",
"40490 [0.45555555555555555, 0.17777777777777778, 0.3... 😬 \n",
"22064 [0.7296744771190439, 0.05173769460607014, 0.21... 😬 \n",
"53563 [0.5586552217453505, 0.13662374821173104, 0.30... 😬 \n",
"67332 [0.3506224066390041, 0.4315352697095436, 0.217... 😬 \n",
"15404 [0.20294117647058824, 0.6029411764705882, 0.19... 😬 \n",
"30130 [0.34515366430260047, 0.32742316784869974, 0.3... 😬 \n",
"37242 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"30533 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"... ... ... \n",
"16955 [0.4983755685510071, 0.08057179987004548, 0.42... 😬 \n",
"36805 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"56326 [0.39118825100133514, 0.38451268357810414, 0.2... 😬 \n",
"40306 [0.7296744771190439, 0.05173769460607014, 0.21... 😬 \n",
"51333 [0.6240601503759399, 0.13984962406015036, 0.23... 😬 \n",
"42339 [0.31784232365145226, 0.46390041493775935, 0.2... 😬 \n",
"7644 [0.32608695652173914, 0.391304347826087, 0.282... 😬 \n",
"13953 [0.16296296296296298, 0.5555555555555556, 0.28... 😬 \n",
"26460 [0.6909090909090909, 0.08181818181818182, 0.22... 😬 \n",
"31571 [0.21660649819494585, 0.5913357400722021, 0.19... 😬 \n",
"9536 [0.34310532030401736, 0.4364820846905538, 0.22... 😬 \n",
"19571 [0.35978835978835977, 0.533068783068783, 0.107... 😬 \n",
"35934 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"4605 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"1913 [0.34310532030401736, 0.4364820846905538, 0.22... 😬 \n",
"11827 [0.5586552217453505, 0.13662374821173104, 0.30... 😬 \n",
"15181 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"32869 [0.4406779661016949, 0.2983050847457627, 0.261... 😬 \n",
"40784 [0.6906474820143885, 0.0671462829736211, 0.242... 😬 \n",
"18611 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"28341 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"41368 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"35554 [0.22289823008849557, 0.5912610619469026, 0.18... 😬 \n",
"27968 [0.7296744771190439, 0.05173769460607014, 0.21... 😬 \n",
"36772 [0.34515366430260047, 0.32742316784869974, 0.3... 😬 \n",
"7568 [0.34310532030401736, 0.4364820846905538, 0.22... 😬 \n",
"62837 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
"34105 [0.6666666666666666, 0.06666666666666667, 0.26... 😬 \n",
"59069 [0.5634451019066403, 0.0992767915844839, 0.337... 😬 \n",
"47088 [0.46813021474490496, 0.24716181096977158, 0.2... 😬 \n",
2018-06-10 18:32:41 +02:00
"\n",
2018-06-11 18:04:39 +02:00
" predicted_sentiment \n",
"7618 [0.46037721633911133, 0.2677566707134247, 0.27... \n",
"6910 [0.4636985957622528, 0.26727861166000366, 0.26... \n",
"35783 [0.4635818898677826, 0.2673886716365814, 0.269... \n",
"15623 [0.46267321705818176, 0.2673114538192749, 0.27... \n",
"12023 [0.4621191620826721, 0.26737213134765625, 0.27... \n",
"15763 [0.46237292885780334, 0.2675153613090515, 0.27... \n",
"57240 [0.4622046649456024, 0.2677682340145111, 0.270... \n",
"2418 [0.4639696180820465, 0.26709380745887756, 0.26... \n",
"66384 [0.4639129638671875, 0.26741719245910645, 0.26... \n",
"44639 [0.46023082733154297, 0.27063173055648804, 0.2... \n",
"54463 [0.46065258979797363, 0.2685313820838928, 0.27... \n",
"38407 [0.4624767005443573, 0.2678660750389099, 0.269... \n",
"31342 [0.4628538191318512, 0.2673165500164032, 0.269... \n",
"41255 [0.4628150463104248, 0.26765838265419006, 0.26... \n",
"19686 [0.4624774158000946, 0.26805710792541504, 0.26... \n",
"67193 [0.4607327878475189, 0.26832473278045654, 0.27... \n",
"40650 [0.4610658884048462, 0.26784956455230713, 0.27... \n",
"64416 [0.46121352910995483, 0.27051451802253723, 0.2... \n",
"19525 [0.46745917201042175, 0.265864759683609, 0.266... \n",
"39024 [0.4606868028640747, 0.267996221780777, 0.2713... \n",
"35342 [0.46745917201042175, 0.265864759683609, 0.266... \n",
"9715 [0.45438939332962036, 0.2717699408531189, 0.27... \n",
"40490 [0.46492910385131836, 0.2654315233230591, 0.26... \n",
"22064 [0.46712905168533325, 0.26472848653793335, 0.2... \n",
"53563 [0.4643474817276001, 0.2679416835308075, 0.267... \n",
"67332 [0.4613974094390869, 0.2683691382408142, 0.270... \n",
"15404 [0.46027225255966187, 0.2696966826915741, 0.27... \n",
"30130 [0.46355295181274414, 0.26728588342666626, 0.2... \n",
"37242 [0.46367916464805603, 0.26611489057540894, 0.2... \n",
"30533 [0.4640607535839081, 0.26755788922309875, 0.26... \n",
2018-06-10 18:32:41 +02:00
"... ... \n",
2018-06-11 18:04:39 +02:00
"16955 [0.46489447355270386, 0.26702189445495605, 0.2... \n",
"36805 [0.4630420207977295, 0.26803845167160034, 0.26... \n",
"56326 [0.4617263972759247, 0.26921841502189636, 0.26... \n",
"40306 [0.46200209856033325, 0.26775091886520386, 0.2... \n",
"51333 [0.46260306239128113, 0.26784563064575195, 0.2... \n",
"42339 [0.4600381553173065, 0.26839619874954224, 0.27... \n",
"7644 [0.4657018482685089, 0.2653878927230835, 0.268... \n",
"13953 [0.4625128209590912, 0.26891759037971497, 0.26... \n",
"26460 [0.46720531582832336, 0.26648813486099243, 0.2... \n",
"31571 [0.4603578746318817, 0.2692199945449829, 0.270... \n",
"9536 [0.4630625247955322, 0.26809123158454895, 0.26... \n",
"19571 [0.4639342725276947, 0.26599037647247314, 0.27... \n",
"35934 [0.46198946237564087, 0.26730334758758545, 0.2... \n",
"4605 [0.46141430735588074, 0.268981009721756, 0.269... \n",
"1913 [0.4630756378173828, 0.2665313482284546, 0.270... \n",
"11827 [0.46398892998695374, 0.26717185974121094, 0.2... \n",
"15181 [0.45438939332962036, 0.2717699408531189, 0.27... \n",
"32869 [0.4616232216358185, 0.2673903703689575, 0.270... \n",
"40784 [0.46204957365989685, 0.26713815331459045, 0.2... \n",
"18611 [0.4634704887866974, 0.2669770121574402, 0.269... \n",
"28341 [0.46148979663848877, 0.26869532465934753, 0.2... \n",
"41368 [0.4605691730976105, 0.26995062828063965, 0.26... \n",
"35554 [0.46297168731689453, 0.2675779163837433, 0.26... \n",
"27968 [0.46712905168533325, 0.26472848653793335, 0.2... \n",
"36772 [0.4609829783439636, 0.26704034209251404, 0.27... \n",
"7568 [0.46219775080680847, 0.26761168241500854, 0.2... \n",
"62837 [0.46401336789131165, 0.2654089033603668, 0.27... \n",
"34105 [0.4665740728378296, 0.26650139689445496, 0.26... \n",
"59069 [0.4622177183628082, 0.268806129693985, 0.2689... \n",
"47088 [0.46383899450302124, 0.26802247762680054, 0.2... \n",
2018-06-05 15:51:58 +02:00
"\n",
2018-06-10 18:32:41 +02:00
"[3337 rows x 5 columns]"
2018-06-05 15:51:58 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
2018-06-09 14:54:34 +02:00
"display(testlist)\n"
2018-06-05 15:51:58 +02:00
]
},
{
"cell_type": "code",
2018-06-10 17:31:43 +02:00
"execution_count": 61,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [],
"source": [
"testlist.to_csv('test.csv')"
]
},
2018-06-05 15:57:26 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* a simple output widget for testing:"
]
},
2018-06-05 15:51:58 +02:00
{
"cell_type": "code",
2018-06-10 17:31:43 +02:00
"execution_count": 63,
2018-06-05 15:51:58 +02:00
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
2018-06-10 17:31:43 +02:00
"model_id": "865c37ebeb2443a589ccbaf220456457",
2018-06-05 15:51:58 +02:00
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>Text</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"Text(value='')"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
2018-06-10 17:31:43 +02:00
"model_id": "d34382a5e13c4be396303afc6b138c2a",
2018-06-05 15:51:58 +02:00
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(Button(description='get smiley', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"out = widgets.Output()\n",
"\n",
"t = widgets.Text()\n",
"b = widgets.Button(\n",
" description='get smiley',\n",
" disabled=False,\n",
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
" tooltip='Click me',\n",
" icon='check'\n",
")\n",
"\n",
"\n",
"\n",
"def handle_submit(sender):\n",
" with out:\n",
" clear_output()\n",
" with out:\n",
2018-06-09 14:54:34 +02:00
" pred = clf.predict(vectorizer.transform([t.value]))\n",
" \n",
2018-06-10 17:31:43 +02:00
" display(Markdown(\"# \" + str(sent2emoji(pred))))\n",
2018-06-05 15:51:58 +02:00
"\n",
"b.on_click(handle_submit)\n",
" \n",
"display(t)\n",
"display(widgets.VBox([b, out])) "
]
},
2018-06-05 17:30:41 +02:00
{
"cell_type": "code",
2018-06-10 17:31:43 +02:00
"execution_count": null,
2018-06-05 17:30:41 +02:00
"metadata": {},
2018-06-10 17:31:43 +02:00
"outputs": [],
2018-06-05 17:30:41 +02:00
"source": [
"import numpy as np\n",
"\n",
"y_trans = mlb.inverse_transform(yt1)\n",
"pred_trans = mlb.inverse_transform(yt1)\n",
"\n",
"# evaluate accuracy\n",
"pos = 0\n",
"neg = 0\n",
"all = 0\n",
"for entry in range(len(y_trans)):\n",
" if len(np.intersect1d(y_trans[entry], pred_trans[entry])) > 0:\n",
" pos += 1\n",
" else:\n",
" neg += 1\n",
" all += 1\n",
"print(pos/all)\n",
"print(neg)"
]
},
2018-06-05 15:51:58 +02:00
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
2018-06-05 11:34:14 +02:00
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
2018-06-09 14:54:34 +02:00
"version": "3.6.5"
2018-06-05 11:34:14 +02:00
}
},
"nbformat": 4,
"nbformat_minor": 2
}