nlp-lab/Project/Tools/EmojiCounting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Count emoji occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import glob\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from __future__ import unicode_literals\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "json_root = \"./emoji_counts/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "json_files = sorted(glob.glob(json_root + \"/*.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./emoji_counts/twitter_emoji_count_2017-12.json',\n",
       " './emoji_counts/twitter_emoji_count_2017_11.json']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "json_lists = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for path in json_files:\n",
    "    with open(path) as f:\n",
    "        data = json.load(f)\n",
    "        json_lists.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merged_dict = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for j in json_lists:\n",
    "    for emoji in j.keys():\n",
    "        if emoji in merged_dict:\n",
    "            merged_dict[emoji] = merged_dict[emoji] + j[emoji]\n",
    "        else:\n",
    "            merged_dict[emoji] = j[emoji]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_top = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "keysort = np.argsort(list(merged_dict.values()))[-n_top:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for apple:\n",
    "from matplotlib.font_manager import FontProperties\n",
    "\n",
    "prop = FontProperties(fname='/System/Library/Fonts/Apple Color Emoji.ttc')\n",
    "plt.rcParams['font.family'] = prop.get_family()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEyCAYAAABH+Yw/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF0ZJREFUeJzt3X+wZOVd5/H3F4ZfRSDDj4GwM+gQM1qQGElyi4wVdzUhgQGjQ5lkC1bDmMKdLQQlkt2E6O5OBtQK1q5EDEFRZjNYUTIbdRlTJLMjxF9VkHDHZAk/ZOeaKNwdhCEDiCZLFvLdP85ztbnpub8Ypr/d/X5V3eo+3/OcPs9T3bfv555znu7ITCRJklTXIYPugCRJkuZmYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVt2zQHTjQTjzxxFy9evWguyFJkjSvXbt2PZGZK+ZrN3KBbfXq1UxOTg66G5IkSfOKiL9dSLsFnRKNiL+JiC9HxJciYrLVjo+InRGxu90e1+oREddHxFRE3BsRr+95nA2t/e6I2NBTf0N7/Km2bcy1D0mSpHGymGvY3pyZZ2bmRFu+CrgjM9cAd7RlgPOANe1nI3AjdOEL2AS8ETgL2NQTwG5sbWe2WzfPPiRJksbGi5l0sB7Y2u5vBS7oqd+SnbuB5RFxCnAusDMz92Xmk8BOYF1bd2xm3pWZCdwy67H67UOSJGlsLDSwJfA/I2JXRGxstZMz81GAdntSq68EHunZdrrV5qpP96nPtY8XiIiNETEZEZN79+5d4JAkSZKGw0InHbwpM/dExEnAzoj4qznaRp9aLqG+YJl5E3ATwMTExKK2lSRJqm5BR9gyc0+7fRz4Q7pr0B5rpzNpt4+35tPAqT2brwL2zFNf1afOHPuQJEkaG/MGtog4OiKOmbkPnAPcB2wHZmZ6bgBua/e3Axe32aJrgafb6cwdwDkRcVybbHAOsKOteyYi1rbZoRfPeqx++5AkSRobCzklejLwh+2TNpYBv5uZn42Ie4BtEXEJ8DDwrtb+duB8YAr4OvAegMzcFxHXAPe0dldn5r52/1Lg48BRwGfaD8CH97MPSZKksRHdxMzRMTExkX5wriRJGgYRsavnI9P2y+8SlSRJKs7AJkmSVNzIfZeoNEpic79PvaknN43WpRWSVI1H2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSiltwYIuIQyPiixHx6bZ8WkR8PiJ2R8QnI+LwVj+iLU+19at7HuODrf5QRJzbU1/XalMRcVVPve8+JEmSxslijrBdATzYs3wtcF1mrgGeBC5p9UuAJzPzVcB1rR0RcQZwIfBqYB3wsRYCDwVuAM4DzgAuam3n2ockSdLYWFBgi4hVwA8Dv92WA3gL8KnWZCtwQbu/vi3T1p/d2q8Hbs3MZzPzq8AUcFb7mcrMr2TmN4FbgfXz7EOSJGlsLPQI20eA9wPfassnAE9l5nNteRpY2e6vBB4BaOufbu3/qT5rm/3V59rHC0TExoiYjIjJvXv3LnBIkiRJw2HewBYRbwcez8xdveU+TXOedQeq/u3FzJsycyIzJ1asWNGviSRJ0tBatoA2bwJ+NCLOB44EjqU74rY8Ipa1I2CrgD2t/TRwKjAdEcuAlwP7euozerfpV39ijn1IkiSNjXmPsGXmBzNzVWaupps0cGdm/jjwOeCdrdkG4LZ2f3tbpq2/MzOz1S9ss0hPA9YAXwDuAda0GaGHt31sb9vsbx+SJElj48V8DtsHgCsjYoruerObW/1m4IRWvxK4CiAz7we2AQ8AnwUuy8zn29Gzy4EddLNQt7W2c+1DkiRpbER3IGt0TExM5OTk5KC7IR0QsbnfpZz15KbReh+RpIMlInZl5sR87fymA0mSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFTdvYIuIIyPiCxHxvyLi/ojY3OqnRcTnI2J3RHwyIg5v9SPa8lRbv7rnsT7Y6g9FxLk99XWtNhURV/XU++5DkiRpnCzkCNuzwFsy8/uAM4F1EbEWuBa4LjPXAE8Cl7T2lwBPZuargOtaOyLiDOBC4NXAOuBjEXFoRBwK3ACcB5wBXNTaMsc+JEmSxsa8gS07/9AWD2s/CbwF+FSrbwUuaPfXt2Xa+rMjIlr91sx8NjO/CkwBZ7Wfqcz8SmZ+E7gVWN+22d8+JEmSxsaCrmFrR8K+BDwO7AT+GngqM59rTaaBle3+SuARgLb+aeCE3vqsbfZXP2GOfUiSJI2NBQW2zHw+M88EVtEdETu9X7N2G/tZd6Dq3yYiNkbEZERM7t27t18TSZKkobWoWaKZ+RTwJ8BaYHlELGurVgF72v1p4FSAtv7lwL7e+qxt9ld/Yo59zO7XTZk5kZkTK1asWMyQJEmSylvILNEVEbG83T8KeCvwIPA54J2t2QbgtnZ/e1umrb8zM7PVL2yzSE8D1gBfAO4B1rQZoYfTTUzY3rbZ3z4kSZLGxrL5m3AKsLXN5jwE2JaZn46IB4BbI+IXgS8CN7f2NwO/ExFTdEfWLgTIzPsjYhvwAPAccFlmPg8QEZcDO4BDgS2ZeX97rA/sZx+SJEljI7oDWaNjYmIiJycnB90N6YCIzf0u5awnN43W+4gkHSwRsSszJ+Zr5zcdSJIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScfMGtog4NSI+FxEPRsT9EXFFqx8fETsjYne7Pa7VIyKuj4ipiLg3Il7f81gbWvvdEbGhp/6GiPhy2+b6iIi59iFJkjROFnKE7TngfZl5OrAWuCwizgCuAu7IzDXAHW0Z4DxgTfvZCNwIXfgCNgFvBM4CNvUEsBtb25nt1rX6/vYhSZI0NuYNbJn5aGb+Zbv/DPAgsBJYD2xtzbYCF7T764FbsnM3sDwiTgHOBXZm5r7MfBLYCaxr647NzLsyM4FbZj1Wv31IkiSNjUVdwxYRq4HXAZ8HTs7MR6ELdcBJrdlK4JGezaZbba76dJ86c+xjdr82RsRkREzu3bt3MUOSJEkqb8GBLSJeBvw+8N7M/Pu5mvap5RLqC5aZN2XmRGZOrFixYjGbSpIklbegwBY
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10d80ce48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#matplotlib.rc('font', family='Arial')\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.bar(np.array(list(merged_dict.keys()))[keysort], np.array(list(merged_dict.values()))[keysort], color='g')\n",
    "plt.savefig(\"histogram.png\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([106,  14,  46,  41,  74,  91,  87, 104,  69,  11,  52,  81,  15,\n",
       "        38,  16,  27,  65,  66,  28,   1,  57,  31,  55,  53,  33,  21,\n",
       "         2,  59,  29,   6,  32,  34,  13,  49,  37,  54,  40,  10,   4,\n",
       "        23,  50,  25,   8,  17,   5,  24,   7,   9,   3,   0])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keysort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['🤔', '🤣', '💕', '😚', '🤦', '😓', '😥', '🙄', '😻', '😣', '😛', '😝', '😃',\n",
       "       '😐', '😈', '😡', '😞', '😪', '😑', '😕', '😬', '😱', '😇', '😄', '😆', '😴',\n",
       "       '🙈', '😜', '😤', '😫', '😳', '😋', '😀', '😌', '😏', '😔', '😒', '😎', '😢',\n",
       "       '😅', '😁', '😉', '🙌', '🙏', '😘', '😊', '😩', '😍', '😭', '😂'],\n",
       "      dtype='<U1')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(list(merged_dict.keys()))[keysort]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'🤔🤣💕😚🤦😓😥🙄😻😣😛😝😃😐😈😡😞😪😑😕😬😱😇😄😆😴🙈😜😤😫😳😋😀😌😏😔😒😎😢😅😁😉🙌🙏😘😊😩😍😭😂'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(np.array(list(merged_dict.keys()))[keysort])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}