{ "cells": [ { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import glob\n", "import json" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# simple twitter approach\n", "*for learning emoji usage by single (in the meaning of unconnected) twitter messages*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## loading train data" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "data_root_folder = \"./data_en/\" # i created a symlink here" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* get all json files in `data_root_folder`" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "json_files = sorted(glob.glob(data_root_folder + \"/*.json\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* so far, only load the first file" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | EMOJI | \n", "datetime | \n", "id | \n", "lang | \n", "person | \n", "reply_to | \n", "text | \n", "
---|---|---|---|---|---|---|---|
0 | \n", "[π₯, π] | \n", "2017-11-01 13:29:00 | \n", "925716304635547600 | \n", "en | \n", "31507978 | \n", "NaN | \n", "fashionbombdaily's photo <EMOJI><EMOJI>πΌ | \n", "
1 | \n", "[π€¦] | \n", "2017-11-01 13:29:00 | \n", "925716304664911900 | \n", "en | \n", "231994649 | \n", "NaN | \n", "Itβs scary how on point my horoscope be <EMOJI... | \n", "
2 | \n", "[π€§] | \n", "2017-11-01 13:29:00 | \n", "925716304652308500 | \n", "en | \n", "78123421 | \n", "NaN | \n", "@mvyn_: Them boring ass licks, you supposed t... | \n", "
3 | \n", "[π―, π₯, π] | \n", "2017-11-01 13:29:01 | \n", "925716308846661600 | \n", "en | \n", "860396470540705800 | \n", "NaN | \n", "@taehyungpic: #vmin βs unit shoot <EMOJI><EM... | \n", "
4 | \n", "[π, π, π] | \n", "2017-11-01 13:29:02 | \n", "925716313015586800 | \n", "en | \n", "775381892321333200 | \n", "NaN | \n", "@risa_del: Can I just say how much Meng respe... | \n", "
5 | \n", "[π] | \n", "2017-11-01 13:29:02 | \n", "925716313032364000 | \n", "en | \n", "3244140547 | \n", "NaN | \n", "@Melodia_Muse: omg did y'all just cut off Red... | \n", "
6 | \n", "[β , π] | \n", "2017-11-01 13:29:02 | \n", "925716313036722200 | \n", "en | \n", "4891498023 | \n", "NaN | \n", "@Harrys1DEmpire: Follow everyone who LIKES th... | \n", "
7 | \n", "[πΆ] | \n", "2017-11-01 13:29:03 | \n", "925716317243555800 | \n", "en | \n", "2660072274 | \n", "NaN | \n", "@taehyungpic: he is so smollll <EMOJI>Β©rocket... | \n", "
8 | \n", "[π] | \n", "2017-11-01 13:29:03 | \n", "925716317214089200 | \n", "en | \n", "2592765104 | \n", "NaN | \n", "Woooaaaahhh <EMOJI> | \n", "
9 | \n", "[π, π] | \n", "2017-11-01 13:29:03 | \n", "925716317218508800 | \n", "en | \n", "3011085514 | \n", "NaN | \n", "@mymermaidIagoon: <EMOJI> HURRY! For your cha... | \n", "
10 | \n", "[π·] | \n", "2017-11-01 13:29:04 | \n", "925716321416949800 | \n", "en | \n", "278737933 | \n", "NaN | \n", "<EMOJI> vivalcli: Portraits by Zhao Guojing an... | \n", "
11 | \n", "[π¨, π¨, π¨, π¨, π¨, π¨] | \n", "2017-11-01 13:29:05 | \n", "925716325615439900 | \n", "en | \n", "876587129077477400 | \n", "NaN | \n", "@BillRatchet: <EMOJI><EMOJI><EMOJI>NO NUT NOV... | \n", "
12 | \n", "[π] | \n", "2017-11-01 13:29:05 | \n", "925716325615505400 | \n", "en | \n", "2402588155 | \n", "NaN | \n", "@mysweetapplexxx: β€οΈHappy Birthday, @GoodGalC... | \n", "
13 | \n", "[π] | \n", "2017-11-01 13:29:06 | \n", "925716329797054500 | \n", "en | \n", "1546678166 | \n", "NaN | \n", "@pourlui: @realDonaldTrump @SSNjl @foxandfrie... | \n", "
14 | \n", "[π©, π©] | \n", "2017-11-01 13:29:06 | \n", "925716329801310200 | \n", "en | \n", "824586253634981900 | \n", "9.257162e+17 | \n", "@hiphopphiIes i wanna know too<EMOJI><EMOJI> | \n", "
15 | \n", "[π, π] | \n", "2017-11-01 13:29:02 | \n", "925716313019965400 | \n", "en | \n", "1581953814 | \n", "9.257088e+17 | \n", "@WizMommma veda was yoda too <EMOJI><EMOJI> | \n", "
16 | \n", "[π, π€, π] | \n", "2017-11-01 13:29:06 | \n", "925716329801371600 | \n", "en | \n", "1737878390 | \n", "NaN | \n", "@TiagoIlori4: Me and my family are humbled by... | \n", "
17 | \n", "[π] | \n", "2017-11-01 13:29:05 | \n", "925716325607133200 | \n", "en | \n", "1001999683 | \n", "NaN | \n", "Iβm less stressed about turning 30 now <EMOJI>... | \n", "
18 | \n", "[π―] | \n", "2017-11-01 13:29:07 | \n", "925716334008082400 | \n", "en | \n", "745222369183043600 | \n", "NaN | \n", "Full charged. <EMOJI> | \n", "
19 | \n", "[π] | \n", "2017-11-01 13:29:09 | \n", "925716342401052700 | \n", "en | \n", "798557155217539100 | \n", "9.257147e+17 | \n", "@SeaDimon Thatβs part of the problem, (they) @... | \n", "
20 | \n", "[π, π₯, π’] | \n", "2017-11-01 13:29:10 | \n", "925716346570240000 | \n", "en | \n", "744396039126421500 | \n", "9.250629e+17 | \n", "@Ian_khetye got me emotional there<EMOJI><EMOJ... | \n", "
21 | \n", "[π, π, π] | \n", "2017-11-01 13:29:10 | \n", "925716346603753500 | \n", "en | \n", "2428265479 | \n", "NaN | \n", "@30SECONDFlGHTS: BRUHHH <EMOJI><EMOJI><EMOJI> | \n", "
22 | \n", "[π] | \n", "2017-11-01 13:29:10 | \n", "925716346595434500 | \n", "en | \n", "2523596372 | \n", "NaN | \n", "@BasebaIINation: Game 7 who you got?<EMOJI> f... | \n", "
23 | \n", "[π] | \n", "2017-11-01 13:29:11 | \n", "925716350764331000 | \n", "en | \n", "2315896393 | \n", "NaN | \n", "@JudicialWatch: JW says DOJ & Congress ar... | \n", "
24 | \n", "[π£, π] | \n", "2017-11-01 13:29:12 | \n", "925716354988236800 | \n", "en | \n", "2316464875 | \n", "NaN | \n", "@cssconfau: βπ» <EMOJI> Ever thought about spe... | \n", "
25 | \n", "[β¨, π] | \n", "2017-11-01 13:29:12 | \n", "925716354975465500 | \n", "en | \n", "2614916743 | \n", "NaN | \n", "@btspat_: Deal for Young Forever: Night Vers... | \n", "
26 | \n", "[πͺ] | \n", "2017-11-01 13:29:13 | \n", "925716359153041400 | \n", "en | \n", "2887539044 | \n", "NaN | \n", "@DFC_SARAVANA: @dhanushkraja Reached 6 Millio... | \n", "
27 | \n", "[π»] | \n", "2017-11-01 13:29:13 | \n", "925716359182520300 | \n", "en | \n", "721490010118205400 | \n", "8.965900e+17 | \n", "back to the yellow <EMOJI> | \n", "
28 | \n", "[π, πΊ, π] | \n", "2017-11-01 13:29:15 | \n", "925716367558545400 | \n", "en | \n", "520536723 | \n", "9.257159e+17 | \n", "@Dimafadma Happy month to you and your loved o... | \n", "
29 | \n", "[π] | \n", "2017-11-01 13:29:16 | \n", "925716371735900200 | \n", "en | \n", "850852815941517300 | \n", "NaN | \n", "Maxx and Patricia. Family hangs at For Good He... | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
176302 | \n", "[π, π, π] | \n", "2017-11-01 07:23:31 | \n", "925624327755591700 | \n", "en | \n", "845085544589672400 | \n", "9.256009e+17 | \n", "@peachshua1230 Awww <EMOJI> Ajsksjdjd im smili... | \n", "
176303 | \n", "[π€£, π€£, π€£, π€£] | \n", "2017-11-01 07:23:33 | \n", "925624336156938200 | \n", "en | \n", "1154786048 | \n", "NaN | \n", "@playboinathann: MY LIL BRO ATE AN EDIBLE THI... | \n", "
176304 | \n", "[π«, β] | \n", "2017-11-01 07:23:34 | \n", "925624340342812700 | \n", "en | \n", "924752524871131100 | \n", "NaN | \n", "I hate when I send a text or snap n I'm so anx... | \n", "
176305 | \n", "[π», π] | \n", "2017-11-01 07:23:34 | \n", "925624340334501900 | \n", "en | \n", "868681065308631000 | \n", "NaN | \n", "@LDNNOISE: Happy Halloween <EMOJI><EMOJI> | \n", "
176306 | \n", "[π, π] | \n", "2017-11-01 07:23:34 | \n", "925624340355280900 | \n", "en | \n", "419493819 | \n", "NaN | \n", "The answer is no I have no plans and I never l... | \n", "
176307 | \n", "[π] | \n", "2017-11-01 07:23:34 | \n", "925624340346937300 | \n", "en | \n", "2457745952 | \n", "9.256219e+17 | \n", "@BeachBoy_Gab LMAOOO I'm so proud <EMOJI> | \n", "
176308 | \n", "[π’, π] | \n", "2017-11-01 07:23:34 | \n", "925624340338507800 | \n", "en | \n", "1955767531 | \n", "NaN | \n", "My cousin/little sister is leaving to San Fran... | \n", "
176309 | \n", "[π] | \n", "2017-11-01 07:23:35 | \n", "925624344524361700 | \n", "en | \n", "796490344581898200 | \n", "NaN | \n", "Can't be alone w my thoughts tonight so just g... | \n", "
176310 | \n", "[β¨] | \n", "2017-11-01 07:23:35 | \n", "925624344511897600 | \n", "en | \n", "825346617620328400 | \n", "NaN | \n", "@ohmygirlpcs: Jine <EMOJI> | \n", "
176311 | \n", "[π, π, π, π, π] | \n", "2017-11-01 07:23:36 | \n", "925624348710285300 | \n", "en | \n", "907808317124177900 | \n", "NaN | \n", "<EMOJI><EMOJI><EMOJI><EMOJI><EMOJI> ambot!!! | \n", "
176312 | \n", "[π] | \n", "2017-11-01 07:23:37 | \n", "925624352921608200 | \n", "en | \n", "1605371160 | \n", "NaN | \n", "@WestHamUtd: It wasn't the result we wanted, ... | \n", "
176313 | \n", "[π, π, π] | \n", "2017-11-01 07:23:37 | \n", "925624352929910800 | \n", "en | \n", "262162415 | \n", "NaN | \n", "<EMOJI>π½<EMOJI>π½ 1st of the month!!Happy 1st o... | \n", "
176314 | \n", "[π΄] | \n", "2017-11-01 07:23:41 | \n", "925624369715515400 | \n", "en | \n", "574882525 | \n", "NaN | \n", "#WednesdayWisdom ... stay in bed <EMOJI> | \n", "
176315 | \n", "[π, π€] | \n", "2017-11-01 07:23:42 | \n", "925624373905707000 | \n", "en | \n", "2694136560 | \n", "NaN | \n", "@Brandonwoelfel: Happy Halloween friends!<EMO... | \n", "
176316 | \n", "[π΄] | \n", "2017-11-01 07:23:43 | \n", "925624378078851100 | \n", "en | \n", "799982767886872600 | \n", "NaN | \n", "@carras16: Good win tonight and great to see ... | \n", "
176317 | \n", "[π, π, π, π] | \n", "2017-11-01 07:23:44 | \n", "925624382269124600 | \n", "en | \n", "556175173 | \n", "9.254038e+17 | \n", "@Louis_Tomlinson @NiallOfficial THIS IS GOLD. ... | \n", "
176318 | \n", "[π, π, π, π, π] | \n", "2017-11-01 07:23:44 | \n", "925624382281547800 | \n", "en | \n", "837324291637051400 | \n", "NaN | \n", "@95duet: i love them smβΉοΈ<EMOJI><EMOJI><EMOJI... | \n", "
176319 | \n", "[π] | \n", "2017-11-01 07:23:45 | \n", "925624386455031800 | \n", "en | \n", "1610265588 | \n", "NaN | \n", "<EMOJI> thank you for the kind compliment | \n", "
176320 | \n", "[π] | \n", "2017-11-01 07:23:45 | \n", "925624386454937600 | \n", "en | \n", "4760724450 | \n", "NaN | \n", "Enjoyed the silence <EMOJI> | \n", "
176321 | \n", "[β¨] | \n", "2017-11-01 07:23:46 | \n", "925624390657572900 | \n", "en | \n", "882858115636514800 | \n", "NaN | \n", "OS: Spiderman Homecoming <EMOJI> | \n", "
176322 | \n", "[π€·] | \n", "2017-11-01 07:23:46 | \n", "925624390682849300 | \n", "en | \n", "188129628 | \n", "NaN | \n", "Go to hell <EMOJI>π½ββοΈ | \n", "
176323 | \n", "[π] | \n", "2017-11-01 07:23:46 | \n", "925624390666129400 | \n", "en | \n", "2473135939 | \n", "NaN | \n", "Thank you Yomi! <EMOJI> | \n", "
176324 | \n", "[π] | \n", "2017-11-01 07:23:46 | \n", "925624390670106600 | \n", "en | \n", "2373584209 | \n", "9.256241e+17 | \n", "@discopiggu Lol. Just enjoy the stars. Music K... | \n", "
176325 | \n", "[π] | \n", "2017-11-01 07:23:47 | \n", "925624394860453900 | \n", "en | \n", "40709255 | \n", "NaN | \n", "@INS_Ecosystem: We are honored to have @eyal ... | \n", "
176326 | \n", "[π₯] | \n", "2017-11-01 07:23:48 | \n", "925624399063117800 | \n", "en | \n", "889503742474768400 | \n", "NaN | \n", "@EmiNiPr: I just love twitterMan's not hot <E... | \n", "
176327 | \n", "[π] | \n", "2017-11-01 07:23:50 | \n", "925624407459971100 | \n", "en | \n", "181689756 | \n", "NaN | \n", "Thoughts and prayers for NY<EMOJI>π» #NYCStrong | \n", "
176328 | \n", "[π] | \n", "2017-11-01 07:23:50 | \n", "925624407460057100 | \n", "en | \n", "60160788 | \n", "9.254610e+17 | \n", "@BrianyH I searched COCK, PENIS, SHLONG, WINKY... | \n", "
176329 | \n", "[π , π, π , π] | \n", "2017-11-01 07:23:51 | \n", "925624411620646900 | \n", "en | \n", "4915042321 | \n", "NaN | \n", "@eyesnoselips99: 170930 KMF <EMOJI>β<EMOJI> ... | \n", "
176330 | \n", "[π, π», π», π, π, π, π, π, π] | \n", "2017-11-01 07:23:53 | \n", "925624420022063100 | \n", "en | \n", "3792290725 | \n", "9.256215e+17 | \n", "@amitbarman520 Thank you so much<EMOJI><EMOJI>... | \n", "
176331 | \n", "[π», π«, π€, π¦, π¦, π¦] | \n", "2017-11-01 07:23:57 | \n", "925624436799049700 | \n", "en | \n", "612733752 | \n", "NaN | \n", "@DarrionDaDon_: Vocals ? <EMOJI>High Notes ? ... | \n", "
176332 rows Γ 7 columns
\n", "