{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import glob\n", "import json\n", "import numpy as np\n", "import itertools\n", "import sklearn.utils as sku\n", "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import MultiLabelBinarizer" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "SINGLE_LABEL = True" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# simple twitter approach\n", "*for learning emoji usage by single (in the meaning of unconnected) twitter messages*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## loading train data" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "data_root_folder = \"./data_en/\" # i created a symlink here" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* get all json files in `data_root_folder`" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "json_files = sorted(glob.glob(data_root_folder + \"/*.json\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* so far, only load the first file" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EMOJIHASHTAGSLINKED_USERdatetimeidlangpersonreply_totext
0[๐Ÿ”ฅ, ๐Ÿ‘][][]2017-11-01 13:29:00925716304635547600en31507978NaNfashionbombdaily's photo <EMOJI><EMOJI>๐Ÿผ
1[๐Ÿคฆ][][]2017-11-01 13:29:00925716304664911900en231994649NaNItโ€™s scary how on point my horoscope be <EMOJI...
2[๐Ÿ˜„][][]2017-11-01 13:29:03925716317214089200en2592765104NaNWoooaaaahhh <EMOJI>
3[๐Ÿ“ท][][]2017-11-01 13:29:04925716321416949800en278737933NaN<EMOJI> vivalcli: Portraits by Zhao Guojing an...
4[๐Ÿ˜ฉ, ๐Ÿ˜ฉ][][@hiphopphiIes]2017-11-01 13:29:06925716329801310200en8245862536349819009.257162e+17<USER> i wanna know too<EMOJI><EMOJI>
5[๐Ÿ˜ญ, ๐Ÿ’“][][@WizMommma]2017-11-01 13:29:02925716313019965400en15819538149.257088e+17<USER> veda was yoda too <EMOJI><EMOJI>
6[๐Ÿ˜‚][][]2017-11-01 13:29:05925716325607133200en1001999683NaNIโ€™m less stressed about turning 30 now <EMOJI>...
7[๐Ÿ’ฏ][][]2017-11-01 13:29:07925716334008082400en745222369183043600NaNFull charged. <EMOJI>
8[๐Ÿ™„][][@SeaDimon, @lsarsour]2017-11-01 13:29:09925716342401052700en7985571552175391009.257147e+17<USER> Thatโ€™s part of the problem, (they) <USE...
9[๐Ÿ˜Ÿ, ๐Ÿ˜ฅ, ๐Ÿ˜ข][][@Ian_khetye]2017-11-01 13:29:10925716346570240000en7443960391264215009.250629e+17<USER> got me emotional there<EMOJI><EMOJI><EM...
10[๐ŸŒป][][]2017-11-01 13:29:13925716359182520300en7214900101182054008.965900e+17back to the yellow <EMOJI>
11[๐Ÿ, ๐ŸŒบ, ๐Ÿ‚][][@Dimafadma]2017-11-01 13:29:15925716367558545400en5205367239.257159e+17<USER> Happy month to you and your loved ones ...
12[๐Ÿƒ][#mortdale, #partofthefamily, #gorgeousboy][]2017-11-01 13:29:16925716371735900200en850852815941517300NaNMaxx and Patricia. Family hangs at For Good He...
13[๐Ÿ’ญ, ๐Ÿคฆ][][]2017-11-01 13:29:20925716388513230800en914145041588867100NaNI need to STOP beating myself up with my thoug...
14[๐Ÿ˜, ๐Ÿ˜˜][#7YearsOfKMH2][]2017-11-01 13:29:20925716388525645800en2425405622NaNCutest Son <EMOJI>Roll no. 31 <EMOJI> <HASHTAG>
15[๐Ÿ˜œ][][]2017-11-01 13:29:22925716396931240000en4614871873NaNBy the summer I should have everything up and ...
16[๐Ÿ˜‚, ๐Ÿ”ฅ][][]2017-11-01 13:29:23925716401125331000en2831608345NaNI know my English is not that good but that do...
17[๐Ÿ’•][][@yungbabytate]2017-11-01 13:29:23925716401133948900en7885719746330092009.255778e+17<USER> I <EMOJI> u mama
18[๐Ÿ˜‰][][@cmckenney]2017-11-01 13:29:23925716401125544000en2183078029.257115e+17<USER> That picture was NOT taken this morning...
19[๐Ÿ‘…][#footfetishnation][]2017-11-01 13:29:25925716409489002500en885261166146179100NaNWelcome to <HASHTAG> <EMOJI>
20[๐Ÿ‘Œ, ๐Ÿ™‚][][]2017-11-01 13:29:25925716409497272300en831437760833609700NaNAwkward <EMOJI><EMOJI>
21[๐Ÿค—, ๐Ÿ“บ][][]2017-11-01 13:29:26925716413699854300en231664542NaNback at it with supernatural <EMOJI><EMOJI>
22[๐Ÿ’ฏ][][]2017-11-01 13:29:26925716413679009800en3196847035NaNOne of the best things I've learned was to sto...
23[๐Ÿ‘…, ๐Ÿ’ฆ, ๐Ÿ‘][][@ctrlpurp]2017-11-01 13:29:29925716426278735900en9184928583526359009.257161e+17<USER> Can I taste?<EMOJI><EMOJI><EMOJI>
24[๐Ÿ’”][][@saunders_court1]2017-11-01 13:29:30925716430473039900en34711873379.257163e+17<USER> we miss you โ˜น๏ธ<EMOJI>
25[๐Ÿค, ๐Ÿค, ๐Ÿค][][]2017-11-01 13:29:31925716434667184100en780060488600199200NaNActually my bias in WJSN are Eunseo &amp; Bona...
26[๐Ÿ˜ด][][]2017-11-01 13:29:32925716438853345300en388380690NaNI so cannot be bothered with the rest of the d...
27[๐Ÿ˜‚][][@xxxtentacion]2017-11-01 13:29:35925716451457163300en899320696869974000NaN<USER> 2lit4life<EMOJI>
28[๐Ÿ˜‚, ๐Ÿ™„][][]2017-11-01 13:29:35925716451461357600en784790670NaNIโ€™m not stop saying that!<EMOJI><EMOJI>
29[๐ŸŽ‰, ๐ŸŽ‚, ๐ŸŽˆ, ๐ŸŽŠ, ๐ŸŽ, ๐Ÿ’œ][][@justinerooney_]2017-11-01 13:29:37925716459828936700en3051266655NaN<USER> HAPPY BIHDAY <EMOJI><EMOJI><EMOJI><EMOJ...
..............................
68703[๐Ÿ˜•][#halloweencostumes][]2017-11-01 07:23:04925624214522036200en1672876458NaN<HASHTAG> this one falls under the weird crazy...
68704[๐Ÿ˜‚, ๐Ÿ˜ฉ][][]2017-11-01 07:23:05925624218682777600en382473866NaNI'm not allowed to have chocolates yet, then I...
68705[๐Ÿ˜‚][#MUFC][]2017-11-01 07:23:06925624222889766900en893145405457911800NaNManchester United manager Mourinho slams 'spec...
68706[๐Ÿ’–][][]2017-11-01 07:23:07925624227088121900en240378516NaN<EMOJI> en Bushwhick
68707[๐ŸŒ†, ๐Ÿ‘‰, ๐Ÿš–, ๐Ÿ“ž][#BurkeCentre][]2017-11-01 07:23:18925624273237983200en784620573209002000NaN: <HASHTAG> <EMOJI> <EMOJI><EMOJI> For Taxi <E...
68708[๐Ÿ˜][][@mychosliaheart, @BarrettoJulia, @iamjoshuaga...2017-11-01 07:23:18925624273212805100en1709981879.254136e+17<USER> <USER> <USER> Look, Mammeh and Daddeh! ...
68709[๐Ÿ˜‹][][]2017-11-01 07:23:24925624298395533300en1348667816NaNLife is so good with you <EMOJI>
68710[๐Ÿ‘Œ, ๐ŸŽƒ, ๐Ÿ˜˜][#portlandoregon, #portlandhalloween, #carrieโ€ฆ][]2017-11-01 07:23:24925624298378801200en722481645765300200NaNHappy Halloween! <EMOJI>๐Ÿฝ<EMOJI><EMOJI> <HASHT...
68711[๐Ÿ‘][][@8limbsbondi...]2017-11-01 07:23:26925624306779897900en2443251500NaNSome work on the ropes in todayโ€™s boxing class...
68712[๐Ÿ˜ญ][][]2017-11-01 07:23:27925624310974136300en2406186390NaNScotty and Kristenโ€™s halloween costumes <EMOJI>
68713[๐Ÿ˜‚][][@rfrandrea, @AdaaanAndyyy]2017-11-01 07:23:30925624323557146600en10394481499.256150e+17<USER> <USER> May pre-month celebration sis <E...
68714[๐Ÿ˜ญ, ๐Ÿ˜ญ, ๐Ÿ’˜][][@peachshua1230]2017-11-01 07:23:31925624327755591700en8450855445896724009.256009e+17<USER> Awww <EMOJI> Ajsksjdjd im smiling like ...
68715[๐Ÿ˜ซ, โœ‹][][]2017-11-01 07:23:34925624340342812700en924752524871131100NaNI hate when I send a text or snap n I'm so anx...
68716[๐Ÿ˜‚, ๐Ÿ™][][]2017-11-01 07:23:34925624340355280900en419493819NaNThe answer is no I have no plans and I never l...
68717[๐Ÿ˜ญ][][@BeachBoy_Gab]2017-11-01 07:23:34925624340346937300en24577459529.256219e+17<USER> LMAOOO I'm so proud <EMOJI>
68718[๐Ÿ˜ข, ๐Ÿ’”][][]2017-11-01 07:23:34925624340338507800en1955767531NaNMy cousin/little sister is leaving to San Fran...
68719[๐Ÿ™ƒ][][]2017-11-01 07:23:35925624344524361700en796490344581898200NaNCan't be alone w my thoughts tonight so just g...
68720[๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚][][]2017-11-01 07:23:36925624348710285300en907808317124177900NaN<EMOJI><EMOJI><EMOJI><EMOJI><EMOJI> ambot!!!
68721[๐Ÿ‘Œ, ๐Ÿ‘Š, ๐Ÿ™Œ][][]2017-11-01 07:23:37925624352929910800en262162415NaN<EMOJI>๐Ÿฝ<EMOJI>๐Ÿฝ 1st of the month!!Happy 1st o...
68722[๐Ÿ˜ด][#WednesdayWisdom][]2017-11-01 07:23:41925624369715515400en574882525NaN<HASHTAG> ... stay in bed <EMOJI>
68723[๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚][][@Louis_Tomlinson, @NiallOfficial]2017-11-01 07:23:44925624382269124600en5561751739.254038e+17<USER> <USER> THIS IS GOLD. GOLD. <EMOJI><EMOJ...
68724[๐Ÿ˜€][][]2017-11-01 07:23:45925624386455031800en1610265588NaN<EMOJI> thank you for the kind compliment
68725[๐Ÿ˜Ž][][]2017-11-01 07:23:45925624386454937600en4760724450NaNEnjoyed the silence <EMOJI>
68726[โœจ][][]2017-11-01 07:23:46925624390657572900en882858115636514800NaNOS: Spiderman Homecoming <EMOJI>
68727[๐Ÿคท][][]2017-11-01 07:23:46925624390682849300en188129628NaNGo to hell <EMOJI>๐Ÿฝโ€โ™€๏ธ
68728[๐Ÿ˜˜][][]2017-11-01 07:23:46925624390666129400en2473135939NaNThank you Yomi! <EMOJI>
68729[๐Ÿ˜‚][][@discopiggu]2017-11-01 07:23:46925624390670106600en23735842099.256241e+17<USER> Lol. Just enjoy the stars. Music Kidhar...
68730[๐Ÿ™][#NYCStrong][]2017-11-01 07:23:50925624407459971100en181689756NaNThoughts and prayers for NY<EMOJI>๐Ÿป <HASHTAG>
68731[๐Ÿ’][#GreatSuccess][@BrianyH]2017-11-01 07:23:50925624407460057100en601607889.254610e+17<USER> I searched COCK, PENIS, SHLONG, WINKY, ...
68732[๐Ÿƒ, ๐ŸŒป, ๐ŸŒป, ๐Ÿƒ, ๐Ÿƒ, ๐Ÿ’, ๐Ÿ’, ๐Ÿƒ, ๐Ÿ™‹][][@amitbarman520]2017-11-01 07:23:53925624420022063100en37922907259.256215e+17<USER> Thank you so much<EMOJI><EMOJI><EMOJI><...
\n", "

68733 rows ร— 9 columns

\n", "
" ], "text/plain": [ " EMOJI \\\n", "0 [๐Ÿ”ฅ, ๐Ÿ‘] \n", "1 [๐Ÿคฆ] \n", "2 [๐Ÿ˜„] \n", "3 [๐Ÿ“ท] \n", "4 [๐Ÿ˜ฉ, ๐Ÿ˜ฉ] \n", "5 [๐Ÿ˜ญ, ๐Ÿ’“] \n", "6 [๐Ÿ˜‚] \n", "7 [๐Ÿ’ฏ] \n", "8 [๐Ÿ™„] \n", "9 [๐Ÿ˜Ÿ, ๐Ÿ˜ฅ, ๐Ÿ˜ข] \n", "10 [๐ŸŒป] \n", "11 [๐Ÿ, ๐ŸŒบ, ๐Ÿ‚] \n", "12 [๐Ÿƒ] \n", "13 [๐Ÿ’ญ, ๐Ÿคฆ] \n", "14 [๐Ÿ˜, ๐Ÿ˜˜] \n", "15 [๐Ÿ˜œ] \n", "16 [๐Ÿ˜‚, ๐Ÿ”ฅ] \n", "17 [๐Ÿ’•] \n", "18 [๐Ÿ˜‰] \n", "19 [๐Ÿ‘…] \n", "20 [๐Ÿ‘Œ, ๐Ÿ™‚] \n", "21 [๐Ÿค—, ๐Ÿ“บ] \n", "22 [๐Ÿ’ฏ] \n", "23 [๐Ÿ‘…, ๐Ÿ’ฆ, ๐Ÿ‘] \n", "24 [๐Ÿ’”] \n", "25 [๐Ÿค, ๐Ÿค, ๐Ÿค] \n", "26 [๐Ÿ˜ด] \n", "27 [๐Ÿ˜‚] \n", "28 [๐Ÿ˜‚, ๐Ÿ™„] \n", "29 [๐ŸŽ‰, ๐ŸŽ‚, ๐ŸŽˆ, ๐ŸŽŠ, ๐ŸŽ, ๐Ÿ’œ] \n", "... ... \n", "68703 [๐Ÿ˜•] \n", "68704 [๐Ÿ˜‚, ๐Ÿ˜ฉ] \n", "68705 [๐Ÿ˜‚] \n", "68706 [๐Ÿ’–] \n", "68707 [๐ŸŒ†, ๐Ÿ‘‰, ๐Ÿš–, ๐Ÿ“ž] \n", "68708 [๐Ÿ˜] \n", "68709 [๐Ÿ˜‹] \n", "68710 [๐Ÿ‘Œ, ๐ŸŽƒ, ๐Ÿ˜˜] \n", "68711 [๐Ÿ‘] \n", "68712 [๐Ÿ˜ญ] \n", "68713 [๐Ÿ˜‚] \n", "68714 [๐Ÿ˜ญ, ๐Ÿ˜ญ, ๐Ÿ’˜] \n", "68715 [๐Ÿ˜ซ, โœ‹] \n", "68716 [๐Ÿ˜‚, ๐Ÿ™] \n", "68717 [๐Ÿ˜ญ] \n", "68718 [๐Ÿ˜ข, ๐Ÿ’”] \n", "68719 [๐Ÿ™ƒ] \n", "68720 [๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚] \n", "68721 [๐Ÿ‘Œ, ๐Ÿ‘Š, ๐Ÿ™Œ] \n", "68722 [๐Ÿ˜ด] \n", "68723 [๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚, ๐Ÿ˜‚] \n", "68724 [๐Ÿ˜€] \n", "68725 [๐Ÿ˜Ž] \n", "68726 [โœจ] \n", "68727 [๐Ÿคท] \n", "68728 [๐Ÿ˜˜] \n", "68729 [๐Ÿ˜‚] \n", "68730 [๐Ÿ™] \n", "68731 [๐Ÿ’] \n", "68732 [๐Ÿƒ, ๐ŸŒป, ๐ŸŒป, ๐Ÿƒ, ๐Ÿƒ, ๐Ÿ’, ๐Ÿ’, ๐Ÿƒ, ๐Ÿ™‹] \n", "\n", " HASHTAGS \\\n", "0 [] \n", "1 [] \n", "2 [] \n", "3 [] \n", "4 [] \n", "5 [] \n", "6 [] \n", "7 [] \n", "8 [] \n", "9 [] \n", "10 [] \n", "11 [] \n", "12 [#mortdale, #partofthefamily, #gorgeousboy] \n", "13 [] \n", "14 [#7YearsOfKMH2] \n", "15 [] \n", "16 [] \n", "17 [] \n", "18 [] \n", "19 [#footfetishnation] \n", "20 [] \n", "21 [] \n", "22 [] \n", "23 [] \n", "24 [] \n", "25 [] \n", "26 [] \n", "27 [] \n", "28 [] \n", "29 [] \n", "... ... \n", "68703 [#halloweencostumes] \n", "68704 [] \n", "68705 [#MUFC] \n", "68706 [] \n", "68707 [#BurkeCentre] \n", "68708 [] \n", "68709 [] \n", "68710 [#portlandoregon, #portlandhalloween, #carrieโ€ฆ] \n", "68711 [] \n", "68712 [] \n", "68713 [] \n", "68714 [] \n", "68715 [] \n", "68716 [] \n", "68717 [] \n", "68718 [] \n", "68719 [] \n", "68720 [] \n", "68721 [] \n", "68722 [#WednesdayWisdom] \n", "68723 [] \n", "68724 [] \n", "68725 [] \n", "68726 [] \n", "68727 [] \n", "68728 [] \n", "68729 [] \n", "68730 [#NYCStrong] \n", "68731 [#GreatSuccess] \n", "68732 [] \n", "\n", " LINKED_USER datetime \\\n", "0 [] 2017-11-01 13:29:00 \n", "1 [] 2017-11-01 13:29:00 \n", "2 [] 2017-11-01 13:29:03 \n", "3 [] 2017-11-01 13:29:04 \n", "4 [@hiphopphiIes] 2017-11-01 13:29:06 \n", "5 [@WizMommma] 2017-11-01 13:29:02 \n", "6 [] 2017-11-01 13:29:05 \n", "7 [] 2017-11-01 13:29:07 \n", "8 [@SeaDimon, @lsarsour] 2017-11-01 13:29:09 \n", "9 [@Ian_khetye] 2017-11-01 13:29:10 \n", "10 [] 2017-11-01 13:29:13 \n", "11 [@Dimafadma] 2017-11-01 13:29:15 \n", "12 [] 2017-11-01 13:29:16 \n", "13 [] 2017-11-01 13:29:20 \n", "14 [] 2017-11-01 13:29:20 \n", "15 [] 2017-11-01 13:29:22 \n", "16 [] 2017-11-01 13:29:23 \n", "17 [@yungbabytate] 2017-11-01 13:29:23 \n", "18 [@cmckenney] 2017-11-01 13:29:23 \n", "19 [] 2017-11-01 13:29:25 \n", "20 [] 2017-11-01 13:29:25 \n", "21 [] 2017-11-01 13:29:26 \n", "22 [] 2017-11-01 13:29:26 \n", "23 [@ctrlpurp] 2017-11-01 13:29:29 \n", "24 [@saunders_court1] 2017-11-01 13:29:30 \n", "25 [] 2017-11-01 13:29:31 \n", "26 [] 2017-11-01 13:29:32 \n", "27 [@xxxtentacion] 2017-11-01 13:29:35 \n", "28 [] 2017-11-01 13:29:35 \n", "29 [@justinerooney_] 2017-11-01 13:29:37 \n", "... ... ... \n", "68703 [] 2017-11-01 07:23:04 \n", "68704 [] 2017-11-01 07:23:05 \n", "68705 [] 2017-11-01 07:23:06 \n", "68706 [] 2017-11-01 07:23:07 \n", "68707 [] 2017-11-01 07:23:18 \n", "68708 [@mychosliaheart, @BarrettoJulia, @iamjoshuaga... 2017-11-01 07:23:18 \n", "68709 [] 2017-11-01 07:23:24 \n", "68710 [] 2017-11-01 07:23:24 \n", "68711 [@8limbsbondi...] 2017-11-01 07:23:26 \n", "68712 [] 2017-11-01 07:23:27 \n", "68713 [@rfrandrea, @AdaaanAndyyy] 2017-11-01 07:23:30 \n", "68714 [@peachshua1230] 2017-11-01 07:23:31 \n", "68715 [] 2017-11-01 07:23:34 \n", "68716 [] 2017-11-01 07:23:34 \n", "68717 [@BeachBoy_Gab] 2017-11-01 07:23:34 \n", "68718 [] 2017-11-01 07:23:34 \n", "68719 [] 2017-11-01 07:23:35 \n", "68720 [] 2017-11-01 07:23:36 \n", "68721 [] 2017-11-01 07:23:37 \n", "68722 [] 2017-11-01 07:23:41 \n", "68723 [@Louis_Tomlinson, @NiallOfficial] 2017-11-01 07:23:44 \n", "68724 [] 2017-11-01 07:23:45 \n", "68725 [] 2017-11-01 07:23:45 \n", "68726 [] 2017-11-01 07:23:46 \n", "68727 [] 2017-11-01 07:23:46 \n", "68728 [] 2017-11-01 07:23:46 \n", "68729 [@discopiggu] 2017-11-01 07:23:46 \n", "68730 [] 2017-11-01 07:23:50 \n", "68731 [@BrianyH] 2017-11-01 07:23:50 \n", "68732 [@amitbarman520] 2017-11-01 07:23:53 \n", "\n", " id lang person reply_to \\\n", "0 925716304635547600 en 31507978 NaN \n", "1 925716304664911900 en 231994649 NaN \n", "2 925716317214089200 en 2592765104 NaN \n", "3 925716321416949800 en 278737933 NaN \n", "4 925716329801310200 en 824586253634981900 9.257162e+17 \n", "5 925716313019965400 en 1581953814 9.257088e+17 \n", "6 925716325607133200 en 1001999683 NaN \n", "7 925716334008082400 en 745222369183043600 NaN \n", "8 925716342401052700 en 798557155217539100 9.257147e+17 \n", "9 925716346570240000 en 744396039126421500 9.250629e+17 \n", "10 925716359182520300 en 721490010118205400 8.965900e+17 \n", "11 925716367558545400 en 520536723 9.257159e+17 \n", "12 925716371735900200 en 850852815941517300 NaN \n", "13 925716388513230800 en 914145041588867100 NaN \n", "14 925716388525645800 en 2425405622 NaN \n", "15 925716396931240000 en 4614871873 NaN \n", "16 925716401125331000 en 2831608345 NaN \n", "17 925716401133948900 en 788571974633009200 9.255778e+17 \n", "18 925716401125544000 en 218307802 9.257115e+17 \n", "19 925716409489002500 en 885261166146179100 NaN \n", "20 925716409497272300 en 831437760833609700 NaN \n", "21 925716413699854300 en 231664542 NaN \n", "22 925716413679009800 en 3196847035 NaN \n", "23 925716426278735900 en 918492858352635900 9.257161e+17 \n", "24 925716430473039900 en 3471187337 9.257163e+17 \n", "25 925716434667184100 en 780060488600199200 NaN \n", "26 925716438853345300 en 388380690 NaN \n", "27 925716451457163300 en 899320696869974000 NaN \n", "28 925716451461357600 en 784790670 NaN \n", "29 925716459828936700 en 3051266655 NaN \n", "... ... ... ... ... \n", "68703 925624214522036200 en 1672876458 NaN \n", "68704 925624218682777600 en 382473866 NaN \n", "68705 925624222889766900 en 893145405457911800 NaN \n", "68706 925624227088121900 en 240378516 NaN \n", "68707 925624273237983200 en 784620573209002000 NaN \n", "68708 925624273212805100 en 170998187 9.254136e+17 \n", "68709 925624298395533300 en 1348667816 NaN \n", "68710 925624298378801200 en 722481645765300200 NaN \n", "68711 925624306779897900 en 2443251500 NaN \n", "68712 925624310974136300 en 2406186390 NaN \n", "68713 925624323557146600 en 1039448149 9.256150e+17 \n", "68714 925624327755591700 en 845085544589672400 9.256009e+17 \n", "68715 925624340342812700 en 924752524871131100 NaN \n", "68716 925624340355280900 en 419493819 NaN \n", "68717 925624340346937300 en 2457745952 9.256219e+17 \n", "68718 925624340338507800 en 1955767531 NaN \n", "68719 925624344524361700 en 796490344581898200 NaN \n", "68720 925624348710285300 en 907808317124177900 NaN \n", "68721 925624352929910800 en 262162415 NaN \n", "68722 925624369715515400 en 574882525 NaN \n", "68723 925624382269124600 en 556175173 9.254038e+17 \n", "68724 925624386455031800 en 1610265588 NaN \n", "68725 925624386454937600 en 4760724450 NaN \n", "68726 925624390657572900 en 882858115636514800 NaN \n", "68727 925624390682849300 en 188129628 NaN \n", "68728 925624390666129400 en 2473135939 NaN \n", "68729 925624390670106600 en 2373584209 9.256241e+17 \n", "68730 925624407459971100 en 181689756 NaN \n", "68731 925624407460057100 en 60160788 9.254610e+17 \n", "68732 925624420022063100 en 3792290725 9.256215e+17 \n", "\n", " text \n", "0 fashionbombdaily's photo ๐Ÿผ \n", "1 Itโ€™s scary how on point my horoscope be \n", "3 vivalcli: Portraits by Zhao Guojing an... \n", "4 i wanna know too \n", "5 veda was yoda too \n", "6 Iโ€™m less stressed about turning 30 now ... \n", "7 Full charged. \n", "8 Thatโ€™s part of the problem, (they) got me emotional there \n", "11 Happy month to you and your loved ones ... \n", "12 Maxx and Patricia. Family hangs at For Good He... \n", "13 I need to STOP beating myself up with my thoug... \n", "14 Cutest Son Roll no. 31 \n", "15 By the summer I should have everything up and ... \n", "16 I know my English is not that good but that do... \n", "17 I u mama \n", "18 That picture was NOT taken this morning... \n", "19 Welcome to \n", "20 Awkward \n", "21 back at it with supernatural \n", "22 One of the best things I've learned was to sto... \n", "23 Can I taste? \n", "24 we miss you โ˜น๏ธ \n", "25 Actually my bias in WJSN are Eunseo & Bona... \n", "26 I so cannot be bothered with the rest of the d... \n", "27 2lit4life \n", "28 Iโ€™m not stop saying that! \n", "29 HAPPY BIHDAY this one falls under the weird crazy... \n", "68704 I'm not allowed to have chocolates yet, then I... \n", "68705 Manchester United manager Mourinho slams 'spec... \n", "68706 en Bushwhick \n", "68707 : For Taxi Look, Mammeh and Daddeh! ... \n", "68709 Life is so good with you \n", "68710 Happy Halloween! ๐Ÿฝ \n", "68713 May pre-month celebration sis Awww Ajsksjdjd im smiling like ... \n", "68715 I hate when I send a text or snap n I'm so anx... \n", "68716 The answer is no I have no plans and I never l... \n", "68717 LMAOOO I'm so proud \n", "68718 My cousin/little sister is leaving to San Fran... \n", "68719 Can't be alone w my thoughts tonight so just g... \n", "68720 ambot!!! \n", "68721 ๐Ÿฝ๐Ÿฝ 1st of the month!!Happy 1st o... \n", "68722 ... stay in bed \n", "68723 THIS IS GOLD. GOLD. thank you for the kind compliment \n", "68725 Enjoyed the silence \n", "68726 OS: Spiderman Homecoming \n", "68727 Go to hell ๐Ÿฝโ€โ™€๏ธ \n", "68728 Thank you Yomi! \n", "68729 Lol. Just enjoy the stars. Music Kidhar... \n", "68730 Thoughts and prayers for NY๐Ÿป \n", "68731 I searched COCK, PENIS, SHLONG, WINKY, ... \n", "68732 Thank you so much<... \n", "\n", "[68733 rows x 9 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "twitter_data = pd.read_json(json_files[0], encoding=\"utf-8\")\n", "twitter_data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* extracting emojis and text" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "emojis = twitter_data['EMOJI']\n", "plain_text = twitter_data['text']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* make our plain text more \"plain\":\n", " * removing the keyword `` (just for the beginning)\n", " * removing remaining useless emojis, like skin modifier etc." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# defining blacklist for modifier emojis:\n", "emoji_blacklist = set([\n", " chr(0x1F3FB),\n", " chr(0x1F3FC),\n", " chr(0x1F3FD),\n", " chr(0x1F3FE),\n", " chr(0x1F3FF),\n", " chr(0x2642),\n", " chr(0x2640)\n", "])" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# filtering them and the EMOJI keyword out:\n", "plain_text = plain_text.str.replace(\"\",\"\").str.replace(\"[\" + \"\".join(list(emoji_blacklist)) + \"]\",\"\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* trying a multi label vectorizer" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def latest(lst):\n", " return lst[-1] if len(lst) > 0 else 'X' \n", "def most_common(lst):\n", " # trying to find the most common used emoji in the given lst\n", " return max(set(lst), key=lst.count) if len(lst) > 0 else \"X\" # setting label to 'X' if there is an empty emoji list" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "mlb_multi = MultiLabelBinarizer()\n", "mlb_single = MultiLabelBinarizer()\n", "\n", "multi_labels = mlb_multi.fit_transform(emojis)\n", "single_labels = mlb_single.fit_transform(np.array([latest(x) for x in emojis]))\n", "\n", "mlb = mlb_single if SINGLE_LABEL else mlb_multi\n", "labels = single_labels if SINGLE_LABEL else multi_labels" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* generate weights:" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "# at first count over our table\n", "emoji_count = {}\n", "\n", "\n", "for e_list in emojis:\n", " for e in set(e_list):\n", " if e not in emoji_count:\n", " emoji_count[e] = 0\n", " emoji_count[e] += 1\n", "\n", "emoji_count\n", "emoji_sum = sum([emoji_count[e] for e in emoji_count])\n", "\n", "emoji_weights = {}\n", "for e in emoji_count:\n", " # tfidf for emojis\n", " emoji_weights[e] = np.log((emoji_sum / emoji_count[e]))\n", "\n", "weights_sum= sum([emoji_weights[x] for x in emoji_weights])\n", " \n", "# normalize:\n", "for e in emoji_weights:\n", " emoji_weights[e] = emoji_weights[e] / weights_sum\n", "\n", "emoji_weights['X'] = 0 # dummy values\n", "emoji_count['X'] = 0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* generating train and test set:" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "X1, Xt1, y1, yt1 = train_test_split(plain_text, labels, test_size=0.1, random_state=4222)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "y1_weights = np.array([(sum([emoji_weights[e] for e in e_list]) / len(e_list)) if len(e_list) > 0 else 0 for e_list in mlb.inverse_transform(y1)])" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "vectorizer = TfidfVectorizer(stop_words='english')\n", "vec_train = vectorizer.fit_transform(X1)\n", "vec_test = vectorizer.transform(Xt1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* train. this can take a very long time..." ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", " from ._conv import register_converters as _register_converters\n", "Using TensorFlow backend.\n" ] } ], "source": [ "from sklearn.neural_network import MLPClassifier as MLP\n", "from sklearn.multiclass import OneVsRestClassifier as OVRC\n", "from sklearn.tree import DecisionTreeClassifier as DTC\n", "\n", "from keras.models import Sequential\n", "from keras.layers import Dense" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "def train(max_size = 10000, layers=[(1024, 'relu'),(y1[0].shape[0],'softmax')], random_state=4222, ovrc=False, n_iter=5):\n", " \n", " model = Sequential()\n", " \n", " # build mlp layers:\n", " keras_layers = []\n", " first_layer = True\n", " for layer in layers:\n", " if first_layer:\n", " model.add(Dense(units=layer[0], activation=layer[1], input_dim=vectorizer.transform([\" \"])[0]._shape[1]))\n", " first_layer = False\n", " else:\n", " model.add(Dense(units=layer[0], activation=layer[1]))\n", " \n", " #mlp = MLPClassifier(layers=sknn_layers, random_state=random_state, verbose=True, n_iter=n_iter, batch_size=100)\n", " \n", " model.compile(loss='categorical_crossentropy',\n", " optimizer='sgd',\n", " metrics=['accuracy'])\n", " \n", " clf = OVRC(model) if ovrc else model\n", "\n", " clf.fit(vec_train[:max_size].A, y1[:max_size], validation_split=0.2, sample_weight=y1_weights[:max_size])\n", " \n", " return clf" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train on 8000 samples, validate on 2000 samples\n", "Epoch 1/1\n", "8000/8000 [==============================] - 109s 14ms/step - loss: 0.0042 - acc: 0.0012 - val_loss: 0.0042 - val_acc: 0.0020\n" ] } ], "source": [ "clf = train(max_size=10000,layers=[(1200, 'relu'),(1200,'relu'),(y1[0].shape[0],'softmax')])" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "62691 I just finished mine I was extremely ne...\n", "9959 Kinda Gross but yup it's ...\n", "39726 Treating myself to a manicure because even une...\n", "20883 Sarfraz Done it.Congrats.Pakistan Rules the Wo...\n", "2705 I turn 20 in Just over a month and Iโ€™ve never ...\n", "20766 The coolest & beautiful duo ever โคโค\n", "13242 Studying abroad in Italy for 3 weeks i...\n", "32951 You're nice to meeeeee thanks I appreciate that \n", "7297 hahahha it kinda does look like me \n", "10913 Ya Rasulullah.,You are the Messenger...\n", "58664 crying\n", "28113 Man my Dad crazy but I miss his Bok Choi soup ...\n", "20698 Well done!!\n", "45787 Mi mess\n", "36076 schooling all the \n", "66970 Awww but Iโ€™m sure your man can \n", "34787 TWEET using the hashtag... is the rel...\n", "9746 Nation is lost y NS boosting tat he will appea...\n", "64458 Video dinner medic \n", "63868 He wouldn't expect anything less \n", "7878 Looking for an ASAP?๐Ÿ•ฐ๏ธCheck out our ...\n", "42253 Cuddled up in a hoodie and fuzzy socks. \n", "32650 Too true the Buble album is out and...\n", "58945 Woke up late again \n", "60382 Damn g well mfs glad you out take adva...\n", "37224 + SATURDAY OFFICIAL CAU VS MOREHOUSE ...\n", "19488 My back is sore\n", "36042 Actually, it should be October 31st๐Ÿ•ธ vs. Novem...\n", " ... \n", "4414 Absolutely mate \n", "21768 Running late to pick up the kids, and this tra...\n", "5359 Lmao well if you say so \n", "22550 Hardcore \n", "55252 Click here to watch it: Kane hot office sex i...\n", "42040 And brother โ€๏ธ thanks\n", "19063 yes Haley \n", "477 Our differences are the real treasur...\n", "41045 I just took a two hour nap instead of d...\n", "52270 Some guy full on grabbed my ass last night whe...\n", "52102 Can't ever do anything for myself. \n", "29210 OMFG SHES SO BEAUTIFUL \n", "2131 Check out what I'm selling on my shop \n", "2912 My shordy b boostin \n", "64797 I pray my nieces NEVER feel this type of hurt ...\n", "63632 I should definitely go to sleep and rea...\n", "30426 Texas asf \n", "68567 annyeong! if you have time kindly check...\n", "61825 He melts me! I want him to stay this small for...\n", "8057 Sorry Not Sorry By is Track 13 on I liked it too \n", "18808 A mansion gap, maybe \n", "64643 Happy halloween \n", "34804 Where is a Civil Rights Lawyer ready to help t...\n", "62123 I keep having dreams that my family come and v...\n", "18325 # May this day bring countless happines...\n", "66916 Happy Halloween \n", "63618 This is what happens when I finish a painting \n", "Name: text, Length: 61859, dtype: object" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X1" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* make a prediction and store it in a csv file:" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "pred = clf.predict(vectorizer.transform(Xt1))" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[0.00124256, 0.00124216, 0.00123592, ..., 0.00123677, 0.00123611,\n", " 0.00123534],\n", " [0.00124442, 0.00123547, 0.00123502, ..., 0.00123892, 0.00123958,\n", " 0.00123556],\n", " [0.00124413, 0.00123538, 0.00124574, ..., 0.00123829, 0.00123556,\n", " 0.00123374],\n", " ...,\n", " [0.00123915, 0.00123915, 0.00123916, ..., 0.00123915, 0.00123916,\n", " 0.00123915],\n", " [0.00124084, 0.00124438, 0.00123832, ..., 0.00123602, 0.00125055,\n", " 0.00123963],\n", " [0.00124321, 0.0012361 , 0.00123488, ..., 0.00123589, 0.00124259,\n", " 0.00123691]], dtype=float32)" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pred" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "binary_pred = np.zeros_like(pred)\n", "binary_pred[np.arange(len(pred)), pred.argmax(1)] = 1 " ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "binary_pred[[0,1,2]]" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "# build a dataframe to visualize test results:\n", "testlist = pd.DataFrame({'text': Xt1, 'teacher': (mlb_single if SINGLE_LABEL else mlb).inverse_transform(yt1), 'predict': (mlb_single if SINGLE_LABEL else mlb).inverse_transform(binary_pred)})" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
predictteachertext
15278(๐ŸŒ ,)(๐Ÿ˜…,)<USER> <USER> 2 years? โœŒ
36882(๐Ÿ•”,)(๐Ÿ˜‚,)tell me why this is me everyday
65543(๐ŸŒ,)(๐Ÿ“Œ,)please follow us ::Twitter โ–ถ๏ธ โ–ถ๏ธ โ–ถ๏ธMARSHMALL...
65893(๐Ÿ˜‚,)(๐Ÿ”,)This
55496(๐ŸŽณ,)(๐Ÿ˜,)<USER> Im in love
61997(๐ŸŒ,)(๐ŸŒธ,)<USER> Miss you gemmthankyou sooo muchโคโค
15819(๐Ÿ”น,)(๐Ÿ˜‚,)Me as a parent
21340(๐ŸŒ‘,)(๐Ÿ˜‚,)Really betting my manager rn for tonightโ€™s game
26691(๐ŸŽณ,)(๐Ÿ˜ฉ,)Y ES IM A BAD BOY SO I LIKE BAD GIRL
57025(๐ŸŒท,)(๐Ÿ˜ƒ,)<USER> Just downloaded to my fire stick over ...
17967(๐Ÿ”,)(๐Ÿ˜,)<USER> Love this ! Hollywood is the man
68555(๐ŸŒ‘,)(๐Ÿ˜‰,)<USER> Absolutely but if it had been a banana ...
20784(๐Ÿ’,)(๐Ÿป,)La Grasaaaaaaaaa Reverse Shatter Back Board S...
68300(๐Ÿฅฃ,)(๐Ÿ˜,)I go to New York next month ahhhhhh
4659(๐Ÿ’,)(๐Ÿ˜ญ,)<USER> Iโ€™m trying to enjoy it as much as the f...
66706(๐ŸŒ•,)(๐Ÿ“ž,): Salamander Resort <HASHTAG> ๐Ÿฝ๏ธ For Taxi 70...
47510(๐ŸŽ„,)(๐Ÿ‘‰,)For your actions there WILL be consequences. Y...
55294(๐Ÿ˜พ,)(๐Ÿ˜‚,)<USER> Omg my cousin charging 40
621(๐ŸŒ,)(๐Ÿคฆ,)Zoey daycare lets her do whatever she want, I ...
31011(๐Ÿ˜Œ,)(๐Ÿ˜€,)yeah! โœŒ๏ธ
30659(๐ŸŽ,)(๐Ÿ’ช,)This was so much I mean SO MUCH over due for t...
66284(๐Ÿค”,)(๐Ÿ™‚,)Parent contact booking is now open! Please fol...
66758(๐ŸŽ„,)(๐Ÿ˜ฅ,)<USER> OCONUS we donโ€™t have the Turkey Day buf...
13456(๐ŸŒ ,)(๐Ÿ’•,)Dems realtor business partners have lots of ni...
15057(๐Ÿฅฃ,)(๐Ÿ˜‚,)Ready for the dodgers fans excuses
188(๐ŸŽ„,)(๐ŸŽ‰,)<USER> Happy Birthday Angge
5054(๐ŸŽ„,)(๐Ÿ˜‚,)<USER> <USER> Lawsuit
18099(๐Ÿ‘ƒ,)(๐Ÿ˜ค,)Finna roll this up no excuses
17120(๐ŸŒ,)(โšก,)But itโ€™s forever fuck em and keep it gutta ๏ธ
14552(๐Ÿ“–,)(๐Ÿ˜,)I just need to hubby a hockey player up
............
21819(๐ŸŽ„,)(๐Ÿ˜‚,)Niggas funny af
18460(โ™ฟ,)(๐Ÿ’,)1. Donโ€™t EVER disrespect Thanksgiving cuz yo f...
60291(๐Ÿšฎ,)(๐Ÿ’˜,)happy bday lexi <USER>
22623(๐Ÿ”–,)(๐Ÿ˜ฉ,)Mr.Lazaro said โ€œthereโ€™s my girlsโ€ when he seen...
87(๐ŸŒ,)(๐ŸŽˆ,)Happy Birthday Blake!!!! I hope you have an am...
2884(๐Ÿ„,)(๐Ÿ˜‰,)<USER> This would be a good time to do a Week ...
30212(๐Ÿง˜,)(๐Ÿ“Œ,)<USER> Done, pls โค๏ธ y FAV
55813(๐Ÿ˜พ,)(๐Ÿ˜,)There is just something about <USER> <HASHTAG>
11657(๐Ÿ’š,)(๐Ÿ˜ด,)<USER> you're brudda!
18215(๐Ÿน,)(๐Ÿคท,)Not being depressed โ€๏ธ
48609(๐Ÿง˜,)(๐Ÿค‘,)<USER> I โค you. My woman crush everyday as alw...
16079(๐Ÿฅฃ,)(โš“,)I added a video to a <USER> playlist ๐Ÿ›ณ ๏ธ The ...
28380(๐Ÿฅฃ,)(๐Ÿ˜‚,)Dodgers
41212(๐ŸŒ,)(๐Ÿ”ด,)[LIVE] on <USER>
67555(๐Ÿ”—,)(๐Ÿ’ž,)โ€ข <USER> my precious Niya always checking up o...
45490(๐Ÿ•”,)(๐Ÿคท,)If you wanna make things right it go both ways โ€๏ธ
54052(๐ŸŒ,)(๐Ÿ˜ข,)Poor baby!!!
54517(๐Ÿน,)(๐Ÿ˜š,)<USER> happy birthday, hope you have a good day
24047(๐ŸŒท,)(๐Ÿ˜Œ,)Donโ€™t get mad when them haters stare let em lo...
253(๐Ÿฅ„,)(๐Ÿ˜Ž,)Sag season right around the corner. 25 on the ...
64707(๐ŸŽธ,)(๐Ÿ˜‡,)<USER> Letโ€™s see his costume
34279(๐ŸŒฏ,)(๐Ÿ˜ฅ,)<USER> I AM NERVOUS!!
58342(๐Ÿค,)(๐Ÿ™Œ,)<USER> This looks amazing
16361(๐ŸŽ„,)(๐Ÿ™Œ,)<USER> <USER> <USER> <USER> <USER> <USER> Shit...
39069(๐Ÿ˜‚,)(๐Ÿ‘‡,)THIS
59739(๐ŸŒท,)(๐Ÿ†,)I've never consumed a reindeer.
49140(๐ŸŒ•,)(๐Ÿค,)Sales is the heart of business.
47901(๐Ÿ˜‚,)(๐Ÿ˜ญ,)bruuuuhhh
4241(๐Ÿ’ฒ,)(๐ŸŽ“,)<USER> Congratulations ! youโ€™ve graduated with...
24123(โž—,)(๐Ÿ‘,)<USER> Your A Rock Star! How You Put Up With ...
\n", "

6874 rows ร— 3 columns

\n", "
" ], "text/plain": [ " predict teacher text\n", "15278 (๐ŸŒ ,) (๐Ÿ˜…,) 2 years? โœŒ\n", "36882 (๐Ÿ•”,) (๐Ÿ˜‚,) tell me why this is me everyday \n", "65543 (๐ŸŒ,) (๐Ÿ“Œ,) please follow us ::Twitter โ–ถ๏ธ โ–ถ๏ธ โ–ถ๏ธMARSHMALL...\n", "65893 (๐Ÿ˜‚,) (๐Ÿ”,) This \n", "55496 (๐ŸŽณ,) (๐Ÿ˜,) Im in love \n", "61997 (๐ŸŒ,) (๐ŸŒธ,) Miss you gemmthankyou sooo muchโคโค\n", "15819 (๐Ÿ”น,) (๐Ÿ˜‚,) Me as a parent \n", "21340 (๐ŸŒ‘,) (๐Ÿ˜‚,) Really betting my manager rn for tonightโ€™s game \n", "26691 (๐ŸŽณ,) (๐Ÿ˜ฉ,) Y ES IM A BAD BOY SO I LIKE BAD GIRL \n", "57025 (๐ŸŒท,) (๐Ÿ˜ƒ,) Just downloaded to my fire stick over ...\n", "17967 (๐Ÿ”,) (๐Ÿ˜,) Love this ! Hollywood is the man \n", "68555 (๐ŸŒ‘,) (๐Ÿ˜‰,) Absolutely but if it had been a banana ...\n", "20784 (๐Ÿ’,) (๐Ÿป,) La Grasaaaaaaaaa Reverse Shatter Back Board S...\n", "68300 (๐Ÿฅฃ,) (๐Ÿ˜,) I go to New York next month ahhhhhh \n", "4659 (๐Ÿ’,) (๐Ÿ˜ญ,) Iโ€™m trying to enjoy it as much as the f...\n", "66706 (๐ŸŒ•,) (๐Ÿ“ž,) : Salamander Resort ๐Ÿฝ๏ธ For Taxi 70...\n", "47510 (๐ŸŽ„,) (๐Ÿ‘‰,) For your actions there WILL be consequences. Y...\n", "55294 (๐Ÿ˜พ,) (๐Ÿ˜‚,) Omg my cousin charging 40 \n", "621 (๐ŸŒ,) (๐Ÿคฆ,) Zoey daycare lets her do whatever she want, I ...\n", "31011 (๐Ÿ˜Œ,) (๐Ÿ˜€,) yeah! โœŒ๏ธ \n", "30659 (๐ŸŽ,) (๐Ÿ’ช,) This was so much I mean SO MUCH over due for t...\n", "66284 (๐Ÿค”,) (๐Ÿ™‚,) Parent contact booking is now open! Please fol...\n", "66758 (๐ŸŽ„,) (๐Ÿ˜ฅ,) OCONUS we donโ€™t have the Turkey Day buf...\n", "13456 (๐ŸŒ ,) (๐Ÿ’•,) Dems realtor business partners have lots of ni...\n", "15057 (๐Ÿฅฃ,) (๐Ÿ˜‚,) Ready for the dodgers fans excuses \n", "188 (๐ŸŽ„,) (๐ŸŽ‰,) Happy Birthday Angge \n", "5054 (๐ŸŽ„,) (๐Ÿ˜‚,) Lawsuit \n", "18099 (๐Ÿ‘ƒ,) (๐Ÿ˜ค,) Finna roll this up no excuses \n", "17120 (๐ŸŒ,) (โšก,) But itโ€™s forever fuck em and keep it gutta ๏ธ\n", "14552 (๐Ÿ“–,) (๐Ÿ˜,) I just need to hubby a hockey player up \n", "... ... ... ...\n", "21819 (๐ŸŽ„,) (๐Ÿ˜‚,) Niggas funny af \n", "18460 (โ™ฟ,) (๐Ÿ’,) 1. Donโ€™t EVER disrespect Thanksgiving cuz yo f...\n", "60291 (๐Ÿšฎ,) (๐Ÿ’˜,) happy bday lexi \n", "22623 (๐Ÿ”–,) (๐Ÿ˜ฉ,) Mr.Lazaro said โ€œthereโ€™s my girlsโ€ when he seen...\n", "87 (๐ŸŒ,) (๐ŸŽˆ,) Happy Birthday Blake!!!! I hope you have an am...\n", "2884 (๐Ÿ„,) (๐Ÿ˜‰,) This would be a good time to do a Week ...\n", "30212 (๐Ÿง˜,) (๐Ÿ“Œ,) Done, pls โค๏ธ y FAV\n", "55813 (๐Ÿ˜พ,) (๐Ÿ˜,) There is just something about \n", "11657 (๐Ÿ’š,) (๐Ÿ˜ด,) you're brudda!\n", "18215 (๐Ÿน,) (๐Ÿคท,) Not being depressed โ€๏ธ \n", "48609 (๐Ÿง˜,) (๐Ÿค‘,) I โค you. My woman crush everyday as alw...\n", "16079 (๐Ÿฅฃ,) (โš“,) I added a video to a playlist ๐Ÿ›ณ ๏ธ The ...\n", "28380 (๐Ÿฅฃ,) (๐Ÿ˜‚,) Dodgers \n", "41212 (๐ŸŒ,) (๐Ÿ”ด,) [LIVE] on \n", "67555 (๐Ÿ”—,) (๐Ÿ’ž,) โ€ข my precious Niya always checking up o...\n", "45490 (๐Ÿ•”,) (๐Ÿคท,) If you wanna make things right it go both ways โ€๏ธ\n", "54052 (๐ŸŒ,) (๐Ÿ˜ข,) Poor baby!!! \n", "54517 (๐Ÿน,) (๐Ÿ˜š,) happy birthday, hope you have a good day\n", "24047 (๐ŸŒท,) (๐Ÿ˜Œ,) Donโ€™t get mad when them haters stare let em lo...\n", "253 (๐Ÿฅ„,) (๐Ÿ˜Ž,) Sag season right around the corner. 25 on the ...\n", "64707 (๐ŸŽธ,) (๐Ÿ˜‡,) Letโ€™s see his costume \n", "34279 (๐ŸŒฏ,) (๐Ÿ˜ฅ,) I AM NERVOUS!!\n", "58342 (๐Ÿค,) (๐Ÿ™Œ,) This looks amazing \n", "16361 (๐ŸŽ„,) (๐Ÿ™Œ,) Shit...\n", "39069 (๐Ÿ˜‚,) (๐Ÿ‘‡,) THIS \n", "59739 (๐ŸŒท,) (๐Ÿ†,) I've never consumed a reindeer. \n", "49140 (๐ŸŒ•,) (๐Ÿค,) Sales is the heart of business. \n", "47901 (๐Ÿ˜‚,) (๐Ÿ˜ญ,) bruuuuhhh \n", "4241 (๐Ÿ’ฒ,) (๐ŸŽ“,) Congratulations ! youโ€™ve graduated with...\n", "24123 (โž—,) (๐Ÿ‘,) Your A Rock Star! How You Put Up With ...\n", "\n", "[6874 rows x 3 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(testlist)\n" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "testlist.to_csv('test.csv')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* a simple output widget for testing:" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ba20ba113dff4af0b14ea6a6e666e521", "version_major": 2, "version_minor": 0 }, "text/html": [ "

Failed to display Jupyter Widget of type Text.

\n", "

\n", " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", " that the widgets JavaScript is still loading. If this message persists, it\n", " likely means that the widgets JavaScript library is either not installed or\n", " not enabled. See the Jupyter\n", " Widgets Documentation for setup instructions.\n", "

\n", "

\n", " If you're reading this message in another frontend (for example, a static\n", " rendering on GitHub or NBViewer),\n", " it may mean that your frontend doesn't currently support widgets.\n", "

\n" ], "text/plain": [ "Text(value='')" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f3af634eb2e64822921b57848e032dcf", "version_major": 2, "version_minor": 0 }, "text/html": [ "

Failed to display Jupyter Widget of type VBox.

\n", "

\n", " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", " that the widgets JavaScript is still loading. If this message persists, it\n", " likely means that the widgets JavaScript library is either not installed or\n", " not enabled. See the Jupyter\n", " Widgets Documentation for setup instructions.\n", "

\n", "

\n", " If you're reading this message in another frontend (for example, a static\n", " rendering on GitHub or NBViewer),\n", " it may mean that your frontend doesn't currently support widgets.\n", "

\n" ], "text/plain": [ "VBox(children=(Button(description='get smiley', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "out = widgets.Output()\n", "\n", "t = widgets.Text()\n", "b = widgets.Button(\n", " description='get smiley',\n", " disabled=False,\n", " button_style='', # 'success', 'info', 'warning', 'danger' or ''\n", " tooltip='Click me',\n", " icon='check'\n", ")\n", "\n", "\n", "\n", "def handle_submit(sender):\n", " with out:\n", " clear_output()\n", " with out:\n", " pred = clf.predict(vectorizer.transform([t.value]))\n", " \n", " #binary_pred = np.zeros_like(pred)\n", " #binary_pred[np.arange(len(pred)), pred.argmax(1)] = 1\n", " \n", " binary_pred = []\n", " \n", " s = np.argsort(pred[0])[-5:]\n", " \n", " for i in s:\n", " b = np.zeros_like(pred[0])\n", " b[i] = 1\n", " binary_pred.append(b)\n", " \n", " binary_pred = np.array(binary_pred)\n", " \n", " display(Markdown(\"# \" + str((mlb_single if SINGLE_LABEL else mlb).inverse_transform(binary_pred))))\n", "\n", "b.on_click(handle_submit)\n", " \n", "display(t)\n", "display(widgets.VBox([b, out])) " ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.9991493705341953\n", "15\n" ] } ], "source": [ "import numpy as np\n", "\n", "y_trans = mlb.inverse_transform(yt1)\n", "pred_trans = mlb.inverse_transform(yt1)\n", "\n", "# evaluate accuracy\n", "pos = 0\n", "neg = 0\n", "all = 0\n", "for entry in range(len(y_trans)):\n", " if len(np.intersect1d(y_trans[entry], pred_trans[entry])) > 0:\n", " pos += 1\n", " else:\n", " neg += 1\n", " all += 1\n", "print(pos/all)\n", "print(neg)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }