{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import clear_output, Markdown, Math\n",
    "import ipywidgets as widgets\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "## file input stuff:\n",
    "\n",
    "* replace `test.txt` with yout whatsapp log file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "./whatsapp2csv.sh test.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* read table `test.csv` exported by `whatsapp2csv.sh`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = pd.read_csv('test.txt.csv', delimiter='\\t')\n",
    "messages.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "if [ ! -e emoji-data.txt ]\n",
    "then\n",
    "    echo \"downloading emoji specification\"\n",
    "    wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n",
    "else\n",
    "    echo \"found existing emoji specification\"\n",
    "fi\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_blacklist = set([\n",
    "    0x1F3FB,\n",
    "    0x1F3FC,\n",
    "    0x1F3FD,\n",
    "    0x1F3FE,\n",
    "    0x1F3FF\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n",
    "emoji_data['type'] = emoji_data['type'].str.strip()\n",
    "emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n",
    "emoji_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* now build a set out of the unicode types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ord(\"😀\") == int('0x1f600',16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_codes = emoji_data['unicode']\n",
    "emoji_codes.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* we have to iterate over the whole list and extract all given ranges:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_code_list = []\n",
    "for entry in emoji_codes:\n",
    "    # testing whether we have an entry or a range:\n",
    "    if '.' in entry:\n",
    "        # range\n",
    "        a,b = entry.split(\"..\")\n",
    "        for i in range(int(a,16),int(b,16) +1):\n",
    "            if i not in emoji_blacklist:\n",
    "                emoji_code_list.append(i)\n",
    "    else:\n",
    "        # single entry\n",
    "        if i not in emoji_blacklist:\n",
    "            emoji_code_list.append(int(entry,16))\n",
    "emoji_code_set = set(emoji_code_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# simple test:\n",
    "print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* expanding column and fill new emojis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages[\"emojis\"] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in messages.index:\n",
    "    emoji_list = []\n",
    "    to_remove = []\n",
    "    m = messages.iloc[i]['message']\n",
    "    for c in str(m):\n",
    "        if ord(c) in emoji_code_set:\n",
    "            emoji_list.append(c)\n",
    "        elif ord(c) in emoji_blacklist:\n",
    "            to_remove.append(c)\n",
    "            \n",
    "    messages.loc[i,'emojis'] = emoji_list\n",
    "    #remove emiójis from message\n",
    "    for e in (emoji_list + to_remove):\n",
    "        m = m.replace(e,\"\")\n",
    "    messages.loc[i,'message'] = m\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* get a list only containing messaged with emojis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n",
    "emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(emoji_messages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "## learning part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import itertools\n",
    "import sklearn.utils as sku\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=[e[-1] for e in emoji_messages['emojis']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(stop_words='english')\n",
    "vec_train = vectorizer.fit_transform(X1)\n",
    "vec_test = vectorizer.transform(Xt1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier as RFC\n",
    "from sklearn.neural_network import MLPClassifier as MLP\n",
    "from sklearn.naive_bayes import MultinomialNB as MNB\n",
    "#clf_a = RFC(criterion='entropy', random_state=4222)\n",
    "clf_a = MLP()\n",
    "clf_a.fit(vec_train, y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = clf_a.predict(vectorizer.transform(Xt1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n",
    "testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n",
    "testlist.to_csv('export.csv')\n",
    "testlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(chr(0x1F3F))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "out = widgets.Output()\n",
    "\n",
    "t = widgets.Text()\n",
    "b = widgets.Button(\n",
    "    description='get smiley',\n",
    "    disabled=False,\n",
    "    button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    tooltip='Click me',\n",
    "    icon='check'\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "def handle_submit(sender):\n",
    "    with out:\n",
    "        clear_output()\n",
    "    with out:\n",
    "        display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n",
    "\n",
    "b.on_click(handle_submit)\n",
    "    \n",
    "display(t)\n",
    "display(widgets.VBox([b, out]))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
