nlp-lab/Project/Tools/EmojiCounting.ipynb
2018-07-19 17:53:16 +02:00

205 lines
12 KiB
Plaintext

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib ipympl"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Count emoji occurences"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import json\n",
"import glob\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"from __future__ import unicode_literals\n",
"matplotlib.rc('font', family='symbola')\n",
"from matplotlib.font_manager import FontProperties"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from matplotlib import font_manager as fm\n",
"\n",
"#fm.findfont(prop)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"json_root = \"../simple_approach/\""
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"json_files = sorted(glob.glob(json_root + \"/*.json\"))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['../simple_approach/count_from_read_progress_2018-07-19 17:49:53.844311.json']"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"json_files"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"json_lists = []"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"for path in json_files:\n",
" with open(path) as f:\n",
" data = json.load(f)\n",
" json_lists.append(data)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"merged_dict = {}\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"for j in json_lists:\n",
" for emoji in j.keys():\n",
" if emoji in merged_dict:\n",
" merged_dict[emoji] = merged_dict[emoji] + j[emoji]\n",
" else:\n",
" merged_dict[emoji] = j[emoji]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"n_top = 30"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"keysort = np.argsort(list(merged_dict.values()))[-n_top:]"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFy5JREFUeJzt3Xu8XFV99/HPLyTcKgYwh0uNIcHWSxFo8YhGeCqo2IJURFEQ1LaAlJvWipYKjw+NV7yhaKUYEIsiqA2g1UAprfKillJJ5GkRsUIrPNT0EhW5tA8K4dc/1hoymU5yJpNzziRnfd6v17yyZ806e6+1Z+/9XXvvmUlkJpKk9swadQMkSaNhAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaNXvUDdiQefPm5cKFC0fdDEnaoqxcufKHmTk2Ub3NOgAWLlzIihUrRt0MSdqiRMQ9g9TzEpAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDVqs/4msCTNVLEkNvh6npNT3oYJzwAiYseI+LWI+Iv6fKuIOCsiDo2Is+vzocumvIeSpL4mPAPIzJ8A10XE22vRy4F7M/PaiJgHHAHEJpRdNfndkiRNZJh7AIuBVXV6VX2+KWXriIiTImJFRKxYvXr1EM2TJA1imACYC6yp02uAHTexbB2ZuTQzxzNzfGxswl8zlSQNaZibwPcDc7r+/gEgN6FMkjQCwwTA3wHz6/R84Oau6WHLJEnTbJBPAW0fEa8CFkXEy4HrgN0j4iBgV8pN3Ks3oUySNAKRueHPmkZEAGRmdk9PQ9sYHx9P/0cwSTPRVH4PICJWZub4RPUG+Rho9puWJG3Z/CkISWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaNXuYP4qIrYEjgB8DzwQ+C5wG3ArsB5xbq545UVlmrhm28ZKk4Q0VAMBhwKrM/JuI+BHwYuDezLw2IuZRwiEGLLtq07shSdpYw14Cug34dES8DHgBNRDqa6uAxfUxSJkkaQSGOgPIzH+KiI8AbwfuBh4COpdy1gA7dk0PUva4iDgJOAlgwYIFwzRPkjSAoc4AIuJ1wDXA84FvAScAc+rLs4EHgPsHLFtHZi7NzPHMHB8bGxumeZKkAQx7D2AhcHlmJvDBiNgWmF9fmw/c3DU9SJkkaZoNew/gEuDUiDg8Io4DPgnsHhEHAbtSbuxePWCZJGkEogziN0/j4+O5YsWKUTdDkiZdLIkNvp7nDH9sjoiVmTk+UT2/CCZJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY2aPewfRsTBwM7AvsBy4EXArcB+wLm12pkTlWXmmmHbIEka3lABEBELgQMy893AlRHxSuDezLw2IuYBRwAxYNlVk9APSdJGGvYS0LHAnIg4JCLeDBwIrKqvrQIW18cgZZKkERg2ABYC38vM64F7gNOBzqWcNcCOwNwBy9YRESdFxIqIWLF69eohmydJmsiwAXA/8O91+j7gM8Cc+nw28ECtM0jZOjJzaWaOZ+b42NjYkM2TJE1k2JvANwI/X6d3Bu4G5tfn84Gbu6YHKZMkTbNhzwCWA3tGxJHALwLvB3aPiIOAXSk3dq8esEySNAKRmaNuw3qNj4/nihUrRt0MSZp0sSQ2+HqeM/yxOSJWZub4RPX8IpgkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDVqkwIgIp4QEUvqv2dHxGERcUbXaxOWSZJGY1PPAI4A9gB+B7gxM68Bto+I/TaiTJI0AkMHQEQsAv6lPl0MrKrTq+rzQct653tSRKyIiBWrV68etnmSpAlsyhnAIuCeOj0XWFOn1wA7bkTZOjJzaWaOZ+b42NjYJjRPkrQhQwVARDwPuKWr6H5gTp2eDTywEWWSpBGYPeTfLQJ2A3YBFgDXAfOBO+u/y4FtByyTpBkhlsSEdfKcnIaWDGaoAMjMKyJiW+AEykH9b4EDIyKAn2TmLRHxj8DpE5VNUj8kSRtp2DMAMvPhiLggMz8REbMy88b60tfq6w8A752oTJI0Gpv0MdDMzPrvY5PTHEnSdPGbwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSoof9LSElqxUT/2fvm9B+9bwzPACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkho11K+BRsSTgEOAbYFnA28FzgBuBfYDzq1Vz5yoLDPXDNt4SdLwhv056GOByMyPRcTuwOnAvZl5bUTMA44AYsCyqza9G5KkjTXsJaCLgKV1+qfA2cCq+nwVsLg+BimTJI3AUGcAmfkwQERsBewNXAl0LuWsAXbsmh6k7HERcRJwEsCCBQuGaZ4kaQCbehP4N4HfB+4H5tSy2cADG1G2jsxcmpnjmTk+Nja2ic2TJK3P0P8lZES8EFiemasj4kFgfn1pPnBz1/QgZZKkaTbUGUBEHAxcAvxVRNwBrAR2j4iDgF0pN3avHrBMkjQCw94D+DqwsKf4q/XfG7rK3jtgmSRpmvlFMElqlAEgSY0a+iawJG3pYkls8PU8J6epJaPhGYAkNcoAkKRGGQCS1CjvAUiacVq/tj8ozwAkqVEGgCQ1ygCQpEYZAJLUKANAkhrlp4AkbTH8dM/k8gxAkhrlGYCkkZpoVA+O7KeKASBpSni5ZvPnJSBJapQBIEmNMgAkqVEGgCQ1ygCQpEb5KSBJG8VP98wcBoAkwAN7iwwAaQbzS1baEANA2gI5WtdkMACkaTDoAdsDu6aTASBtAg/Y2pIZAGrGZI3Cu+tKWzIDQJNiYw6ak305xFG4NBwDYAvgwVXSVJixAbAlHDQ9uEoaJX8KQpIaZQBIUqOmPQAiYquIOCsiDo2IsyNiq+lugyRpNGcALwfuzcxrgf8HHDGCNkhS80YRAIuBVXV6VX0uSZpmkTm9nzSJiIuAz2XmDRFxEHBcZr6h6/WTgJPq06cD/ziJi58H/HAG1Bvlsu3L5rnsmdRG+7Lp9sjMsQlrZea0PoAPAYfU6RcDH57GZa+YCfW2hDbal82z3pbQRvsyfY9RXAL6O2B+nZ4P3DyCNkhS80YRAFcDu9fLP7sCV42gDZLUvGn/JnBmPgq8tz69YZoXv3SG1Bvlsu3L5rnsmdRG+zJNpv0msCRp8+A3gSWpUQaAJDXKAKgi4uCIeGJPWUTEy0bVpk0VEb9Qf3Jj3/p8p4hYGRGviIhvjbht+0TExyPi8xHxyYg4cJTtkSZTRCyKiHdFxOci4oMRsc+o29TPjP056I6IeAlwcn06m/LzE3+Ymb1fuHgB8ImI+DSwD3APsBewE/BnPfM8BTgYWEMJ0duAc+sN7t56BwGPDVBv0PkNUm8ucCZwO3AHsDAiXgt8ifLR223qv0TEbOAYYE/g+8AXMvNnfdbjpLWxtuUY4J2Z+c2ImA9cGBFfy8zzpnLZU1FvM2jjQUDWurcDH9ic+tJV/8XAzZn5UJ/Xjs/MSyLiVZR9cXVmLqkDsK91/80U9uUgJmFfjYjfAF4BfBj4HvAU4I0R8Z3MXNozv8XAG1k7GP8v4H2ZeWe/dTjppvuLB9P5AI4F3kn5ZvEOwK8BhwNXADv31P1D4HLKQX9rygHxfODrPfXeAxzZU3YgcCH1pnotezfwsp56zwf+uKfeoPMbtF4A7wO276k7q85jbk/5HwN3A/8K3AJc3Gc9vgf4AnBwff4s4LPApza2jcAi4P11Hb+1lp1M+Rbkm4ADh1zf/eqtb30fDcypz+dQDjgXDbO+p+g9HLTeWcALeur9ImVb3iz6UsuPB04FrgMOAC7oeT2A44AXUvbBfwD+HNiP8sXRXaa4L++m/EbZyfX5a4CXrGfbmWj73h34OGUQOburf78CnA3s1/W3hwHvArbuKptL+TTQL/Xuh1PxmLGXgCJiB+C5wHnAgsx8kDI62gs4A3hLV93tgYeBDwJ75toR8ArgSxHx5FrvOcBdwOKI+FAte0ud3zLg1V317gTuj4gT66WkIymj7iuAV/XM70f1l1FnRcSpwBjw+Z753ZWZV3f3MTO/0V2vOrS25XURcXL9+1MoG9sVwG/VsidGxGeAnwdOAP6SsoGvjIjPRsROPW28nTJKnwUsoYw4L+9tI3B9ROxdy55G2aC72/hq4FzgKODAiOgcgI8F/qi2n555zoqIv6/r8Rpg/37rp+d9OQN4GyW4etf3MykHT4B3AL8AfKPP/P42InapZWOUnyX50+71Peh70zXPjIhjI+IZ9Uxo+/XUe3JEvLGW/S7lTPSbPfUOAW6IiOw8KCPOQ/r0eY+IuKquwwsoA6Er+/UFeG5EfKyWrW/7vgu4LSJOrGWnATv3vNdExC8Bj1K2/f8NPBW4pZ4NdBxV27MGeCLwReCvKfvvo3X53cu9te4nRMQbKIOH3uVu7L56B/Af9c/vrW3pt6/uExHfj4ibImIVcFrPsl9D2Y+eAFxfyy6nDHw+QDkz6Jx5vxD4NvDliJgbEc8AvgJ8grJPTrkZGwCUg8qXKBvc2RGxCDgHODQzV1F2vI7XUEazCSytb84fAT9HeTOOrvUOAS4DvkvZYAB+Qkn8OykHlk69y+v8FlJGAIuAxzLzxp56l9XX76v176OMJm7oqffFemC+ACAiTo2IZZQNqFMP4Jczc2WdV+czvj+gjBS/XfsE5cD/TcqlmPmUs4C5wKXAlykjyc6ylwHfApZk5mOUkdGf17/v7ctrgL+oZe8CftbTl9mZeV9mXg6cAjwJODMzP1bn/f+7+tKZ503AOVmGSOcBX+mzfnrfl/so78v3+9S7m7KTQ/lm+p6ZeWnv+qYEyIm17DDKSPHvWXd9HwJcFhFHRcTnAaLc2zhlA238FPC/gCdTAuUbferNoZwlQRmcnAr8SVe9l1DOaP+5Pr+p/vtT4DeBZ/TM7wHKWc4Cyrfv76SMsJ/d25faxu/VsvVt35dRfsjxd2vZLOAJ3X2OiE6wPwYsz8xb6nz/GXh+lN/9gnLQOxbYhXIwfbj2Z05m/gHljKZ7uc8B3hzlp+TnUM52H19un74Msq/OomyLUC4V77CeffUrlMvIR1NC4rKeZW+fmf9GeX8uqmXXAM/NzEdq36CcEVxP2R/fBjwN2LvO9+nADyKis69OmZl8D2AR5beGjqRsMD+gjEC2i4hXAK+IiLdn5k+B3TJzdUQsoBzoAvgO8GhmPhpr/8+CWZn5SEQ8F5gbEYd0Le/NlNHK4/WAGyNiN8op4krWfut5Tc/8TqRsyL/amVlEfK1nfv8JvK7zemZeAHTCoPs65ez6+tKuun/G2vsYnff8gbpOnk3ZQZYBe1AOTH/D2gPDLODXKSOSH1B2lgOB51F2mN4+XxQRn6vLPToiOjtVp95DdcP+L8r7sTfw44g4qq6Xx7r60lk/7wO2jYhjutbPqb3Ljogf1r6S5Xry/pRAWWd9Uw6knXWzHFje08bO+j6jq96llHDsXd+deS6rDzLzjV2v966f70bETpn5cJ3X+XUb6633kfp6UEa4t1MCoVMvMvNnEfErlIP67ZRt/pHMvDciHuueX0QcTNnGur2fMvJ+vC91XR0CbBXr3pj/H+ubchnws7XPH+/XZ8pBE+CArvldRnlftqt/+3BEbE25tn4acEZmfj0iXhgR83Lt/brOcv+0PoiIT1FG5ft1Lbe77iWdgsx8fLrP+r4jIvaOiKWUfbVz8O7ddr5FGVwSEXtn5k96+vxwPau9i3KMuQT4EeWeHKwdlG1Led92BV5U//5nlAB4jBKGWwP/yRSayWcAO1MOZDtk5grKQe7QzLybcqD9AGXEC+USwzaZuTIzX5qZj2TmWzPz4vp6p96D9ZLAvwDHZuYxmXkM5ZrlA6w9uD4YEbtHxJsob/TvUUYN76kb+tY987urM686vzsoB8ju+e0SEa+PiN/vevxO3di6g3x1ROzZeRLlkk1nepvaHiinn8uB11MubRxHOcCfTdlg53WWDfwT5azg8lr21fr8xt421um/jIgrI+KbwDY9bbwS+G3Khr4qM3+1zutk4JWUS1H0zPO79fFtyrfH/52yY3Qve1fKQeapta87AP8HeKjP+iYivhARy+pjrKeNnfV9aD3173yi6tg+67u73531PBYRr1zPPF9NOWNYVs/gTl5PvadHxHmUS1V71H7v31NvN0qAPlrPju4HnrMR29jtQPd/yPQga3+R8ttd6/vf+qzvXSLipbH2ct9TIuK1PX3ZLjO/SxkwjAF/RbkXdzxlgNXZFqFseyeyrsXAqoj45X7rOspluQ9TzkSvoM/7EhEHRMTTa/0nRcSRfdb3wPtq7fNetWyHiDimZ37LKdsxlIP7fZSDORFxAHBrfe0fKPvcLsC+lPfur4HfqGX7ZeZ9TLXpuNEwigdwOiWpL6zPH6Ic3J5HuUb3DmCr+tr+lAPQ4ZSD01zKJwLeRjnNPrrW25VyCWJPSoAcR7ke/iJKwu/TVe9CymWEl3c9vkC5qbVPz/yeSRmNHUu5bHJgn/mdB3ySciOt81jWXa/W3bHW69yUupl6k4lyQNy3a/1sQ7nEdRPltPw6yo3YfYDju5ddpy9l7Uh3rF8b6/QxlLOVt9TnvW08EvgYa28q70+5cX1yz3vY6fdCyojsGsoNsuf0WfZHKdePP0q5jHAb5QMAV/VrY5/tpd/63o6yrVxDOdtap8/95knZgS+tbe67fgZc9m6s3T5PpIxkl/TUO59yCeFDlDO8y6iXPPvMb4PbWE/dpwEXUw5mn1zP+j6vLvs8ymWRS2qbu+s9pa63DwO/V8sOp+w759P1IQlgNWV0/GzKZZqtKZdsEthrPet6fu37hZRBRL++bE+5t/flrvWzKfvqEyhnZ50+795nPZ5G2deOqc9P6ZR11Vlc57Gw9ndZne+bKAPULwLjU36cHPWBeso6VkYab6XswC+i3Pzs3DzcHjirp/47KaPh11FOa8+iJHnvJwEOqzvHPpQbynvVDfz4nvkdRhlhnVLf0NdTbiqf0KfexZRRwLMoO+r65ndxXd4cSgj9j3q17gtqH/btqvtR4MSuOgvreuj9NNRsyoFu4cYuu6vesyjh+Kz11Hsp8FrKpZiz6450GnB4n750L3veBMv+dF3m1pRrsKs2sL4H7csg63tS57mR9S7pqfeRYbexIdb3QOtnwP31RsrB/v9SBged5z8Ftl3Pcnei7IeT8b5szL46yDZxYu3HOygH9xNY9ziyJyU8LqTcBN+lPj+dcsxaBMyb6uPkjP4toHoNGMrKfgnwY8r1va9QPhn0nZ76r6eM4NZQTo1vAs7PcnOyu95Cyhv6VMqp62cy87Y+yx9JvVr3ycAbKDdz/xW4dH11BzGZbYyI7Sg7x22UEeZvU3bk92fmPVO57KmotyW0cZR9GVS9XLl1rr0n8Fj2/2z/yPoy4PZ9IuWz/dtTQuVTlMuWX83Md/bp81GU49MjwFWZeT3TZEYHgCRp/WbyTWBJ0gYYAJLUKANAkhplAEhSowwASWrUfwNba3+z2RZGZgAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.bar(np.array(list(merged_dict.keys()))[keysort], np.array(list(merged_dict.values()))[keysort], color='g')\n",
"plt.savefig(\"histogram.png\", bbox_inches='tight')\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}