nlp-lab/Project/Tools/EmojiCounting.ipynb

280 lines
17 KiB
Plaintext
Raw Normal View History

2018-07-19 17:49:00 +02:00
{
"cells": [
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 1,
2018-07-19 17:49:00 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-07-20 13:46:12 +02:00
"%matplotlib inline"
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Count emoji occurences"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 2,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:53:16 +02:00
"outputs": [],
2018-07-19 17:49:00 +02:00
"source": [
"import numpy as np\n",
"import json\n",
"import glob\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
2018-07-20 11:30:28 +02:00
"from __future__ import unicode_literals\n"
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 3,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
2018-07-20 11:30:28 +02:00
"json_root = \"./emoji_counts/\""
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 4,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"json_files = sorted(glob.glob(json_root + \"/*.json\"))"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 5,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 11:54:14 +02:00
"outputs": [
{
"data": {
"text/plain": [
"['./emoji_counts/twitter_emoji_count_2017-12.json',\n",
" './emoji_counts/twitter_emoji_count_2017_11.json']"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
2018-07-19 17:49:00 +02:00
"source": [
"json_files"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 6,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"json_lists = []"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 7,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"for path in json_files:\n",
" with open(path) as f:\n",
" data = json.load(f)\n",
" json_lists.append(data)"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 8,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"merged_dict = {}\n"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 9,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"for j in json_lists:\n",
" for emoji in j.keys():\n",
" if emoji in merged_dict:\n",
" merged_dict[emoji] = merged_dict[emoji] + j[emoji]\n",
" else:\n",
" merged_dict[emoji] = j[emoji]"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 10,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
2018-07-20 11:30:28 +02:00
"n_top = 50"
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 11,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"keysort = np.argsort(list(merged_dict.values()))[-n_top:]"
]
},
2018-07-20 13:46:12 +02:00
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# for apple:\n",
"from matplotlib.font_manager import FontProperties\n",
"\n",
"prop = FontProperties(fname='/System/Library/Fonts/Apple Color Emoji.ttc')\n",
"plt.rcParams['font.family'] = prop.get_family()"
]
},
2018-07-19 17:49:00 +02:00
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 13,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 11:54:14 +02:00
"outputs": [
{
"data": {
2018-07-27 15:18:34 +02:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEzCAYAAACMpV+aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm4XFWZ7/HvCwHEAcMQkZtgJ0KuGm0b5DTEqZ0aiCMqitAtRBuNA9g4dQtOYbBttVtpUOSKguKI6FVBRDEiXrTvgxKcELhqHElECDO2gg2894+1yuyUdc6pk5yTs8L5fp6nnlO1atVee+1ae9ev9lAnMhNJkiS1a4vpngFJkiSNzcAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktS4WdM9A5Ntp512yvnz50/3bEiSJI3rsssuuz4z54xXb6jAFhG/BG4D7gLuzMyRiNgB+DQwH/glcFBm3hQRAZwEPA34PfCizPxunc5S4M11sm/LzDNr+V7AR4BtgfOBozIzR2tjrHmdP38+K1euHKZbkiRJ0yoifjVMvYkcEn1SZu6RmSP18dHAhZm5ELiwPgZ4KrCw3pYBp9YZ2gFYDuwD7A0sj4jt62tOBV7aed2ScdqQJEmaMTbmHLYDgDPr/TOBZ3fKP5rFJcDsiNgF2B9YkZk31r1kK4Al9bntMvOSzEzgo33TGtSGJEnSjDFsYEvgqxFxWUQsq2U7Z+Y19f5vgZ3r/bnA1Z3Xrq5lY5WvHlA+VhvriYhlEbEyIlauXbt2yC5JkiRtHoa96OBxmbkmIh4ArIiI/9d9sp5vlpM/e8O1kZmnAacBjIyMTOl8SJIkbWpD7WHLzDX173XA5ynnoF1bD2dS/15Xq68Bdu28fF4tG6t83oByxmhDkiRpxhg3sEXEfSLifr37wH7Aj4BzgaW12lLgnHr/XOCwKBYDt9TDmhcA+0XE9vVig/2AC+pzt0bE4nqF6WF90xrUhiRJ0owxzCHRnYHPlyzFLOCTmfmViLgUODsiDgd+BRxU659P+UmPVZSf9XgxQGbeGBEnAJfWesdn5o31/itZ97MeX643gHeM0oYkSdKMEeXCzHuOkZGR9HfYJEnS5iAiLuv8ZNqo/NdUkiRJjTOwSZIkNc7AJkmS1Lh73D9/lyRJGk8cF2M+n8vbOsffPWySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS44YObBGxZUR8LyLOq48XRMS3I2JVRHw6Irau5dvUx6vq8/M70zimlv84IvbvlC+pZasi4uhO+cA2JEmSZpKJ7GE7Criq8/idwImZuTtwE3B4LT8cuKmWn1jrERGLgIOBhwNLgPfXELglcArwVGARcEitO1YbkiRJM8ZQgS0i5gFPBz5UHwfwZOCztcqZwLPr/QPqY+rzT6n1DwDOysw7MvMXwCpg73pblZk/z8w/AmcBB4zThiRJ0owx7B62/wD+Gbi7Pt4RuDkz76yPVwNz6/25wNUA9flbav0/lfe9ZrTysdqQJEmaMcYNbBHxDOC6zLxsE8zPBomIZRGxMiJWrl27drpnR5IkaVINs4ftscCzIuKXlMOVTwZOAmZHxKxaZx6wpt5fA+wKUJ+/P3BDt7zvNaOV3zBGG+vJzNMycyQzR+bMmTNElyRJkjYf4wa2zDwmM+dl5nzKRQNfz8y/By4CnlerLQXOqffPrY+pz389M7OWH1yvIl0ALAS+A1wKLKxXhG5d2zi3vma0NiRJkmaMjfkdtjcAr42IVZTzzU6v5acDO9by1wJHA2TmFcDZwJXAV4AjMvOueo7akcAFlKtQz651x2pDkiRpxoiyI+ueY2RkJFeuXDndsyFJkhoWx8WYz+fyTZOPIuKyzBwZr57/6UCSJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJaty4gS0i7hUR34mIH0TEFRFxXC1fEBHfjohVEfHpiNi6lm9TH6+qz8/vTOuYWv7jiNi/U76klq2KiKM75QPbkCRJmkmG2cN2B/DkzPwrYA9gSUQsBt4JnJiZuwM3AYfX+ocDN9XyE2s9ImIRcDDwcGAJ8P6I2DIitgROAZ4KLAIOqXUZow1JkqQZY9zAlsXv6sOt6i2BJwOfreVnAs+u9w+oj6nPPyUiopaflZl3ZOYvgFXA3vW2KjN/npl/BM4CDqivGa0NSZKkGWOoc9jqnrDvA9cBK4CfATdn5p21ympgbr0/F7gaoD5/C7Bjt7zvNaOV7zhGG5IkSTPGUIEtM+/KzD2AeZQ9Yg+d0rmaoIhYFhErI2Ll2rVrp3t2JEmSJtWErhLNzJuBi4BHA7MjYlZ9ah6wpt5fA+wKUJ+/P3BDt7zvNaOV3zBGG/3zdVpmjmTmyJw5cybSJUmSpOYNc5XonIiYXe9vC+wLXEUJbs+r1ZYC59T759bH1Oe/nplZyw+uV5EuABYC3wEuBRbWK0K3plyYcG59zWhtSJIkzRizxq/CLsCZ9WrOLYCzM/O8iLgSOCsi3gZ8Dzi91j8d+FhErAJupAQwMvOKiDgbuBK4EzgiM+8CiIgjgQuALYEzMvOKOq03jNKGJEnSjBFlR9Y9x8jISK5cuXK6Z0OSJDUsjosxn8/lmyYfRcRlmTkyXj3/04EkSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuPGDWwRsWtEXBQRV0bEFRFxVC3fISJWRMRP69/ta3lExMkRsSoifhgRj+pMa2mt/9OIWNop3ysiLq+vOTkiYqw2JEmSZpJh9rDdCbwuMxcBi4EjImIRcDRwYWYuBC6sjwGeCiyst2XAqVDCF7Ac2AfYG1jeCWCnAi/tvG5JLR+tDUmSpBlj3MCWmddk5nfr/duAq4C5wAHAmbXamcCz6/0DgI9mcQkwOyJ2AfYHVmTmjZl5E7ACWFKf2y4zL8nMBD7aN61BbUiSJM0YEzqHLSLmA3sC3wZ2zsxr6lO/BXau9+cCV3detrqWjVW+ekA5Y7QhSZI0Ywwd2CLivsD/Bl6dmbd2n6t7xnKS5209Y7UREcsiYmVErFy7du1UzoYkSdImN1Rgi4itKGHtE5n5uVp
2018-07-20 11:54:14 +02:00
"text/plain": [
2018-07-27 15:18:34 +02:00
"<Figure size 720x360 with 1 Axes>"
2018-07-20 11:54:14 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
2018-07-19 17:49:00 +02:00
"source": [
2018-07-20 13:46:12 +02:00
"#matplotlib.rc('font', family='Arial')\n",
2018-07-20 11:30:28 +02:00
"plt.figure(figsize=(10,5))\n",
2018-07-19 17:49:00 +02:00
"plt.bar(np.array(list(merged_dict.keys()))[keysort], np.array(list(merged_dict.values()))[keysort], color='g')\n",
"plt.savefig(\"histogram.png\", bbox_inches='tight')\n",
"plt.show()"
]
},
{
"cell_type": "code",
2018-07-20 13:46:12 +02:00
"execution_count": 14,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 13:46:12 +02:00
"outputs": [
{
"data": {
"text/plain": [
"array([106, 14, 46, 41, 74, 91, 87, 104, 69, 11, 52, 81, 15,\n",
" 38, 16, 27, 65, 66, 28, 1, 57, 31, 55, 53, 33, 21,\n",
" 2, 59, 29, 6, 32, 34, 13, 49, 37, 54, 40, 10, 4,\n",
" 23, 50, 25, 8, 17, 5, 24, 7, 9, 3, 0])"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"keysort"
]
2018-07-19 17:49:00 +02:00
},
{
"cell_type": "code",
2018-07-20 13:46:12 +02:00
"execution_count": 15,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 13:46:12 +02:00
"outputs": [
{
"data": {
"text/plain": [
"array(['🤔', '🤣', '💕', '😚', '🤦', '😓', '😥', '🙄', '😻', '😣', '😛', '😝', '😃',\n",
" '😐', '😈', '😡', '😞', '😪', '😑', '😕', '😬', '😱', '😇', '😄', '😆', '😴',\n",
" '🙈', '😜', '😤', '😫', '😳', '😋', '😀', '😌', '😏', '😔', '😒', '😎', '😢',\n",
2018-07-27 15:18:34 +02:00
" '😅', '😁', '😉', '🙌', '🙏', '😘', '😊', '😩', '😍', '😭', '😂'], dtype='<U1')"
2018-07-20 13:46:12 +02:00
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.array(list(merged_dict.keys()))[keysort]"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'🤔🤣💕😚🤦😓😥🙄😻😣😛😝😃😐😈😡😞😪😑😕😬😱😇😄😆😴🙈😜😤😫😳😋😀😌😏😔😒😎😢😅😁😉🙌🙏😘😊😩😍😭😂'"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"''.join(np.array(list(merged_dict.keys()))[keysort])"
]
2018-07-19 17:49:00 +02:00
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
2018-07-27 15:18:34 +02:00
"version": "3.6.5"
2018-07-19 17:49:00 +02:00
}
},
"nbformat": 4,
"nbformat_minor": 2
}