nlp-lab/Project/Tools/EmojiCounting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Count emoji occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import glob\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from __future__ import unicode_literals\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "json_root = \"./emoji_counts/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "json_files = sorted(glob.glob(json_root + \"/*.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./emoji_counts/twitter_emoji_count_2017-12.json',\n",
       " './emoji_counts/twitter_emoji_count_2017_11.json']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "json_lists = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for path in json_files:\n",
    "    with open(path) as f:\n",
    "        data = json.load(f)\n",
    "        json_lists.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merged_dict = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for j in json_lists:\n",
    "    for emoji in j.keys():\n",
    "        if emoji in merged_dict:\n",
    "            merged_dict[emoji] = merged_dict[emoji] + j[emoji]\n",
    "        else:\n",
    "            merged_dict[emoji] = j[emoji]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_top = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "keysort = np.argsort(list(merged_dict.values()))[-n_top:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for apple:\n",
    "from matplotlib.font_manager import FontProperties\n",
    "\n",
    "prop = FontProperties(fname='/System/Library/Fonts/Apple Color Emoji.ttc')\n",
    "plt.rcParams['font.family'] = prop.get_family()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEzCAYAAACMpV+aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm4XFWZ7/HvCwHEAcMQkZtgJ0KuGm0b5DTEqZ0aiCMqitAtRBuNA9g4dQtOYbBttVtpUOSKguKI6FVBRDEiXrTvgxKcELhqHElECDO2gg2894+1yuyUdc6pk5yTs8L5fp6nnlO1atVee+1ae9ev9lAnMhNJkiS1a4vpngFJkiSNzcAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktS4WdM9A5Ntp512yvnz50/3bEiSJI3rsssuuz4z54xXb6jAFhG/BG4D7gLuzMyRiNgB+DQwH/glcFBm3hQRAZwEPA34PfCizPxunc5S4M11sm/LzDNr+V7AR4BtgfOBozIzR2tjrHmdP38+K1euHKZbkiRJ0yoifjVMvYkcEn1SZu6RmSP18dHAhZm5ELiwPgZ4KrCw3pYBp9YZ2gFYDuwD7A0sj4jt62tOBV7aed2ScdqQJEmaMTbmHLYDgDPr/TOBZ3fKP5rFJcDsiNgF2B9YkZk31r1kK4Al9bntMvOSzEzgo33TGtSGJEnSjDFsYEvgqxFxWUQsq2U7Z+Y19f5vgZ3r/bnA1Z3Xrq5lY5WvHlA+VhvriYhlEbEyIlauXbt2yC5JkiRtHoa96OBxmbkmIh4ArIiI/9d9sp5vlpM/e8O1kZmnAacBjIyMTOl8SJIkbWpD7WHLzDX173XA5ynnoF1bD2dS/15Xq68Bdu28fF4tG6t83oByxmhDkiRpxhg3sEXEfSLifr37wH7Aj4BzgaW12lLgnHr/XOCwKBYDt9TDmhcA+0XE9vVig/2AC+pzt0bE4nqF6WF90xrUhiRJ0owxzCHRnYHPlyzFLOCTmfmViLgUODsiDgd+BRxU659P+UmPVZSf9XgxQGbeGBEnAJfWesdn5o31/itZ97MeX643gHeM0oYkSdKMEeXCzHuOkZGR9HfYJEnS5iAiLuv8ZNqo/NdUkiRJjTOwSZIkNc7AJkmS1Lh73D9/lyRJGk8cF2M+n8vbOsffPWySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS44YObBGxZUR8LyLOq48XRMS3I2JVRHw6Irau5dvUx6vq8/M70zimlv84IvbvlC+pZasi4uhO+cA2JEmSZpKJ7GE7Criq8/idwImZuTtwE3B4LT8cuKmWn1jrERGLgIOBhwNLgPfXELglcArwVGARcEitO1YbkiRJM8ZQgS0i5gFPBz5UHwfwZOCztcqZwLPr/QPqY+rzT6n1DwDOysw7MvMXwCpg73pblZk/z8w/AmcBB4zThiRJ0owx7B62/wD+Gbi7Pt4RuDkz76yPVwNz6/25wNUA9flbav0/lfe9ZrTysdqQJEmaMcYNbBHxDOC6zLxsE8zPBomIZRGxMiJWrl27drpnR5IkaVINs4ftscCzIuKXlMOVTwZOAmZHxKxaZx6wpt5fA+wKUJ+/P3BDt7zvNaOV3zBGG+vJzNMycyQzR+bMmTNElyRJkjYf4wa2zDwmM+dl5nzKRQNfz8y/By4CnlerLQXOqffPrY+pz389M7OWH1yvIl0ALAS+A1wKLKxXhG5d2zi3vma0NiRJkmaMjfkdtjcAr42IVZTzzU6v5acDO9by1wJHA2TmFcDZwJXAV4AjMvOueo7akcAFlKtQz651x2pDkiRpxoiyI+ueY2RkJFeuXDndsyFJkhoWx8WYz+fyTZOPIuKyzBwZr57/6UCSJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJaty4gS0i7hUR34mIH0TEFRFxXC1fEBHfjohVEfHpiNi6lm9TH6+qz8/vTOuYWv7jiNi/U76klq2KiKM75QPbkCRJmkmG2cN2B/DkzPwrYA9gSUQsBt4JnJiZuwM3AYfX+ocDN9XyE2s9ImIRcDDwcGAJ8P6I2DIitgROAZ4KLAIOqXUZow1JkqQZY9zAlsXv6sOt6i2BJwOfreVnAs+u9w+oj6nPPyUiopaflZl3ZOYvgFXA3vW2KjN/npl/BM4CDqivGa0NSZKkGWOoc9jqnrDvA9cBK4CfATdn5p21ympgbr0/F7gaoD5/C7Bjt7zvNaOV7zhGG5IkSTPGUIEtM+/KzD2AeZQ9Yg+d0rmaoIhYFhErI2Ll2rVrp3t2JEmSJtWErhLNzJuBi4BHA7MjYlZ9ah6wpt5fA+wKUJ+/P3BDt7zvNaOV3zBGG/3zdVpmjmTmyJw5cybSJUmSpOYNc5XonIiYXe9vC+wLXEUJbs+r1ZYC59T759bH1Oe/nplZyw+uV5EuABYC3wEuBRbWK0K3plyYcG59zWhtSJIkzRizxq/CLsCZ9WrOLYCzM/O8iLgSOCsi3gZ8Dzi91j8d+FhErAJupAQwMvOKiDgbuBK4EzgiM+8CiIgjgQuALYEzMvOKOq03jNKGJEnSjBFlR9Y9x8jISK5cuXK6Z0OSJDUsjosxn8/lmyYfRcRlmTkyXj3/04EkSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuPGDWwRsWtEXBQRV0bEFRFxVC3fISJWRMRP69/ta3lExMkRsSoifhgRj+pMa2mt/9OIWNop3ysiLq+vOTkiYqw2JEmSZpJh9rDdCbwuMxcBi4EjImIRcDRwYWYuBC6sjwGeCiyst2XAqVDCF7Ac2AfYG1jeCWCnAi/tvG5JLR+tDUmSpBlj3MCWmddk5nfr/duAq4C5wAHAmbXamcCz6/0DgI9mcQkwOyJ2AfYHVmTmjZl5E7ACWFKf2y4zL8nMBD7aN61BbUiSJM0YEzqHLSLmA3sC3wZ2zsxr6lO/BXau9+cCV3detrqWjVW+ekA5Y7QhSZI0Ywwd2CLivsD/Bl6dmbd2n6t7xnKS5209Y7UREcsiYmVErFy7du1UzoYkSdImN1Rgi4itKGHtE5n5uVp
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#matplotlib.rc('font', family='Arial')\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.bar(np.array(list(merged_dict.keys()))[keysort], np.array(list(merged_dict.values()))[keysort], color='g')\n",
    "plt.savefig(\"histogram.png\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([106,  14,  46,  41,  74,  91,  87, 104,  69,  11,  52,  81,  15,\n",
       "        38,  16,  27,  65,  66,  28,   1,  57,  31,  55,  53,  33,  21,\n",
       "         2,  59,  29,   6,  32,  34,  13,  49,  37,  54,  40,  10,   4,\n",
       "        23,  50,  25,   8,  17,   5,  24,   7,   9,   3,   0])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keysort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['🤔', '🤣', '💕', '😚', '🤦', '😓', '😥', '🙄', '😻', '😣', '😛', '😝', '😃',\n",
       "       '😐', '😈', '😡', '😞', '😪', '😑', '😕', '😬', '😱', '😇', '😄', '😆', '😴',\n",
       "       '🙈', '😜', '😤', '😫', '😳', '😋', '😀', '😌', '😏', '😔', '😒', '😎', '😢',\n",
       "       '😅', '😁', '😉', '🙌', '🙏', '😘', '😊', '😩', '😍', '😭', '😂'], dtype='<U1')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(list(merged_dict.keys()))[keysort]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'🤔🤣💕😚🤦😓😥🙄😻😣😛😝😃😐😈😡😞😪😑😕😬😱😇😄😆😴🙈😜😤😫😳😋😀😌😏😔😒😎😢😅😁😉🙌🙏😘😊😩😍😭😂'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(np.array(list(merged_dict.keys()))[keysort])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}