nlp-lab/Project/Tools/EmojiCounting.ipynb

290 lines
14 KiB
Plaintext
Raw Normal View History

2018-07-19 17:49:00 +02:00
{
"cells": [
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 1,
2018-07-19 17:49:00 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-07-20 13:46:12 +02:00
"%matplotlib inline"
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Count emoji occurences"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 2,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:53:16 +02:00
"outputs": [],
2018-07-19 17:49:00 +02:00
"source": [
"import numpy as np\n",
"import json\n",
"import glob\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
2018-07-20 11:30:28 +02:00
"from __future__ import unicode_literals\n"
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 3,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
2018-07-20 11:30:28 +02:00
"json_root = \"./emoji_counts/\""
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 4,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"json_files = sorted(glob.glob(json_root + \"/*.json\"))"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 5,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 11:54:14 +02:00
"outputs": [
{
"data": {
"text/plain": [
"['./emoji_counts/twitter_emoji_count_2017-12.json',\n",
" './emoji_counts/twitter_emoji_count_2017_11.json']"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
2018-07-19 17:49:00 +02:00
"source": [
"json_files"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 6,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"json_lists = []"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 7,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"for path in json_files:\n",
" with open(path) as f:\n",
" data = json.load(f)\n",
" json_lists.append(data)"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 8,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"merged_dict = {}\n"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 9,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"for j in json_lists:\n",
" for emoji in j.keys():\n",
" if emoji in merged_dict:\n",
" merged_dict[emoji] = merged_dict[emoji] + j[emoji]\n",
" else:\n",
" merged_dict[emoji] = j[emoji]"
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 10,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
2018-07-20 11:30:28 +02:00
"n_top = 50"
2018-07-19 17:49:00 +02:00
]
},
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 11,
2018-07-20 13:46:12 +02:00
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": [
"keysort = np.argsort(list(merged_dict.values()))[-n_top:]"
]
},
2018-07-20 13:46:12 +02:00
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# for apple:\n",
"from matplotlib.font_manager import FontProperties\n",
"\n",
"prop = FontProperties(fname='/System/Library/Fonts/Apple Color Emoji.ttc')\n",
"plt.rcParams['font.family'] = prop.get_family()"
]
},
2018-07-19 17:49:00 +02:00
{
"cell_type": "code",
2018-07-20 11:54:14 +02:00
"execution_count": 13,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 11:54:14 +02:00
"outputs": [
{
"data": {
2018-07-20 13:46:12 +02:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEyCAYAAABH+Yw/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF0ZJREFUeJzt3X+wZOVd5/H3F4ZfRSDDj4GwM+gQM1qQGElyi4wVdzUhgQGjQ5lkC1bDmMKdLQQlkt2E6O5OBtQK1q5EDEFRZjNYUTIbdRlTJLMjxF9VkHDHZAk/ZOeaKNwdhCEDiCZLFvLdP85ztbnpub8Ypr/d/X5V3eo+3/OcPs9T3bfv555znu7ITCRJklTXIYPugCRJkuZmYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVt2zQHTjQTjzxxFy9evWguyFJkjSvXbt2PZGZK+ZrN3KBbfXq1UxOTg66G5IkSfOKiL9dSLsFnRKNiL+JiC9HxJciYrLVjo+InRGxu90e1+oREddHxFRE3BsRr+95nA2t/e6I2NBTf0N7/Km2bcy1D0mSpHGymGvY3pyZZ2bmRFu+CrgjM9cAd7RlgPOANe1nI3AjdOEL2AS8ETgL2NQTwG5sbWe2WzfPPiRJksbGi5l0sB7Y2u5vBS7oqd+SnbuB5RFxCnAusDMz92Xmk8BOYF1bd2xm3pWZCdwy67H67UOSJGlsLDSwJfA/I2JXRGxstZMz81GAdntSq68EHunZdrrV5qpP96nPtY8XiIiNETEZEZN79+5d4JAkSZKGw0InHbwpM/dExEnAzoj4qznaRp9aLqG+YJl5E3ATwMTExKK2lSRJqm5BR9gyc0+7fRz4Q7pr0B5rpzNpt4+35tPAqT2brwL2zFNf1afOHPuQJEkaG/MGtog4OiKOmbkPnAPcB2wHZmZ6bgBua/e3Axe32aJrgafb6cwdwDkRcVybbHAOsKOteyYi1rbZoRfPeqx++5AkSRobCzklejLwh+2TNpYBv5uZn42Ie4BtEXEJ8DDwrtb+duB8YAr4OvAegMzcFxHXAPe0dldn5r52/1Lg48BRwGfaD8CH97MPSZKksRHdxMzRMTExkX5wriRJGgYRsavnI9P2y+8SlSRJKs7AJkmSVNzIfZeoNEpic79PvaknN43WpRWSVI1H2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSiltwYIuIQyPiixHx6bZ8WkR8PiJ2R8QnI+LwVj+iLU+19at7HuODrf5QRJzbU1/XalMRcVVPve8+JEmSxslijrBdATzYs3wtcF1mrgGeBC5p9UuAJzPzVcB1rR0RcQZwIfBqYB3wsRYCDwVuAM4DzgAuam3n2ockSdLYWFBgi4hVwA8Dv92WA3gL8KnWZCtwQbu/vi3T1p/d2q8Hbs3MZzPzq8AUcFb7mcrMr2TmN4FbgfXz7EOSJGlsLPQI20eA9wPfassnAE9l5nNteRpY2e6vBB4BaOufbu3/qT5rm/3V59rHC0TExoiYjIjJvXv3LnBIkiRJw2HewBYRbwcez8xdveU+TXOedQeq/u3FzJsycyIzJ1asWNGviSRJ0tBatoA2bwJ+NCLOB44EjqU74rY8Ipa1I2CrgD2t/TRwKjAdEcuAlwP7euozerfpV39ijn1IkiSNjXmPsGXmBzNzVWaupps0cGdm/jjwOeCdrdkG4LZ2f3tbpq2/MzOz1S9ss0hPA9YAXwDuAda0GaGHt31sb9vsbx+SJElj48V8DtsHgCsjYoruerObW/1m4IRWvxK4CiAz7we2AQ8AnwUuy8zn29Gzy4EddLNQt7W2c+1DkiRpbER3IGt0TExM5OTk5KC7IR0QsbnfpZz15KbReh+RpIMlInZl5sR87fymA0mSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFTdvYIuIIyPiCxHxvyLi/ojY3OqnRcTnI2J3RHwyIg5v9SPa8lRbv7rnsT7Y6g9FxLk99XWtNhURV/XU++5DkiRpnCzkCNuzwFsy8/uAM4F1EbEWuBa4LjPXAE8Cl7T2lwBPZuargOtaOyLiDOBC4NXAOuBjEXFoRBwK3ACcB5wBXNTaMsc+JEmSxsa8gS07/9AWD2s/CbwF+FSrbwUuaPfXt2Xa+rMjIlr91sx8NjO/CkwBZ7Wfqcz8SmZ+E7gVWN+22d8+JEmSxsaCrmFrR8K+BDwO7AT+GngqM59rTaaBle3+SuARgLb+aeCE3vqsbfZXP2GOfUiSJI2NBQW2zHw+M88EVtEdETu9X7N2G/tZd6Dq3yYiNkbEZERM7t27t18TSZKkobWoWaKZ+RTwJ8BaYHlELGurVgF72v1p4FSAtv7lwL7e+qxt9ld/Yo59zO7XTZk5kZkTK1asWMyQJEmSylvILNEVEbG83T8KeCvwIPA54J2t2QbgtnZ/e1umrb8zM7PVL2yzSE8D1gBfAO4B1rQZoYfTTUzY3rbZ3z4kSZLGxrL5m3AKsLXN5jwE2JaZn46IB4BbI+IXgS8CN7f2NwO/ExFTdEfWLgTIzPsjYhvwAPAccFlmPg8QEZcDO4BDgS2ZeX97rA/sZx+SJEljI7oDWaNjYmIiJycnB90N6YCIzf0u5awnN43W+4gkHSwRsSszJ+Zr5zcdSJIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScQY2SZKk4gxskiRJxRnYJEmSijOwSZIkFWdgkyRJKs7AJkmSVJyBTZIkqTgDmyRJUnEGNkmSpOIMbJIkScUZ2CRJkoozsEmSJBVnYJMkSSrOwCZJklScgU2SJKk4A5skSVJxBjZJkqTiDGySJEnFGdgkSZKKM7BJkiQVZ2CTJEkqzsAmSZJUnIFNkiSpOAObJElScfMGtog4NSI+FxEPRsT9EXFFqx8fETsjYne7Pa7VIyKuj4ipiLg3Il7f81gbWvvdEbGhp/6GiPhy2+b6iIi59iFJkjROFnKE7TngfZl5OrAWuCwizgCuAu7IzDXAHW0Z4DxgTfvZCNwIXfgCNgFvBM4CNvUEsBtb25nt1rX6/vYhSZI0NuYNbJn5aGb+Zbv/DPAgsBJYD2xtzbYCF7T764FbsnM3sDwiTgHOBXZm5r7MfBLYCaxr647NzLsyM4FbZj1Wv31IkiSNjUVdwxYRq4HXAZ8HTs7MR6ELdcBJrdlK4JGezaZbba76dJ86c+xjdr82RsRkREzu3bt3MUOSJEkqb8GBLSJeBvw+8N7M/Pu5mvap5RLqC5aZN2XmRGZOrFixYjGbSpIklbegwBY
2018-07-20 11:54:14 +02:00
"text/plain": [
2018-07-20 13:46:12 +02:00
"<matplotlib.figure.Figure at 0x10d80ce48>"
2018-07-20 11:54:14 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
2018-07-19 17:49:00 +02:00
"source": [
2018-07-20 13:46:12 +02:00
"#matplotlib.rc('font', family='Arial')\n",
2018-07-20 11:30:28 +02:00
"plt.figure(figsize=(10,5))\n",
2018-07-19 17:49:00 +02:00
"plt.bar(np.array(list(merged_dict.keys()))[keysort], np.array(list(merged_dict.values()))[keysort], color='g')\n",
"plt.savefig(\"histogram.png\", bbox_inches='tight')\n",
"plt.show()"
]
},
{
"cell_type": "code",
2018-07-20 13:46:12 +02:00
"execution_count": 14,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 13:46:12 +02:00
"outputs": [
{
"data": {
"text/plain": [
"array([106, 14, 46, 41, 74, 91, 87, 104, 69, 11, 52, 81, 15,\n",
" 38, 16, 27, 65, 66, 28, 1, 57, 31, 55, 53, 33, 21,\n",
" 2, 59, 29, 6, 32, 34, 13, 49, 37, 54, 40, 10, 4,\n",
" 23, 50, 25, 8, 17, 5, 24, 7, 9, 3, 0])"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"keysort"
]
2018-07-19 17:49:00 +02:00
},
{
"cell_type": "code",
2018-07-20 13:46:12 +02:00
"execution_count": 15,
2018-07-19 17:49:00 +02:00
"metadata": {},
2018-07-20 13:46:12 +02:00
"outputs": [
{
"data": {
"text/plain": [
"array(['🤔', '🤣', '💕', '😚', '🤦', '😓', '😥', '🙄', '😻', '😣', '😛', '😝', '😃',\n",
" '😐', '😈', '😡', '😞', '😪', '😑', '😕', '😬', '😱', '😇', '😄', '😆', '😴',\n",
" '🙈', '😜', '😤', '😫', '😳', '😋', '😀', '😌', '😏', '😔', '😒', '😎', '😢',\n",
" '😅', '😁', '😉', '🙌', '🙏', '😘', '😊', '😩', '😍', '😭', '😂'],\n",
" dtype='<U1')"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.array(list(merged_dict.keys()))[keysort]"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'🤔🤣💕😚🤦😓😥🙄😻😣😛😝😃😐😈😡😞😪😑😕😬😱😇😄😆😴🙈😜😤😫😳😋😀😌😏😔😒😎😢😅😁😉🙌🙏😘😊😩😍😭😂'"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"''.join(np.array(list(merged_dict.keys()))[keysort])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
2018-07-19 17:49:00 +02:00
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
2018-07-20 13:46:12 +02:00
"version": "3.6.3"
2018-07-19 17:49:00 +02:00
}
},
"nbformat": 4,
"nbformat_minor": 2
}