{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## file input stuff:\n", "\n", "* replace `test.txt` with yout whatsapp log file" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "./whatsapp2csv.sh test.txt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read table `test.csv` exported by `whatsapp2csv.sh`" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "messages = pd.read_csv('test.txt.csv', delimiter='\\t')\n", "messages.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "if [ ! -e emoji-data.txt ]\n", "then\n", " echo \"downloading emoji specification\"\n", " wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n", "else\n", " echo \"found existing emoji specification\"\n", "fi\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "emoji_blacklist = set([\n", " 0x1F3FB,\n", " 0x1F3FC,\n", " 0x1F3FD,\n", " 0x1F3FE,\n", " 0x1F3FF\n", "])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n", "emoji_data['type'] = emoji_data['type'].str.strip()\n", "emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n", "emoji_data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* now build a set out of the unicode types" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ord(\"😀\") == int('0x1f600',16)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "emoji_codes = emoji_data['unicode']\n", "emoji_codes.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* we have to iterate over the whole list and extract all given ranges:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "emoji_code_list = []\n", "for entry in emoji_codes:\n", " # testing whether we have an entry or a range:\n", " if '.' in entry:\n", " # range\n", " a,b = entry.split(\"..\")\n", " for i in range(int(a,16),int(b,16) +1):\n", " if i not in emoji_blacklist:\n", " emoji_code_list.append(i)\n", " else:\n", " # single entry\n", " if i not in emoji_blacklist:\n", " emoji_code_list.append(int(entry,16))\n", "emoji_code_set = set(emoji_code_list)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# simple test:\n", "print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* expanding column and fill new emojis" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "messages[\"emojis\"] = None" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for i in messages.index:\n", " emoji_list = []\n", " to_remove = []\n", " m = messages.iloc[i]['message']\n", " for c in str(m):\n", " if ord(c) in emoji_code_set:\n", " emoji_list.append(c)\n", " elif ord(c) in emoji_blacklist:\n", " to_remove.append(c)\n", " \n", " messages.loc[i,'emojis'] = emoji_list\n", " #remove emiójis from message\n", " for e in (emoji_list + to_remove):\n", " m = m.replace(e,\"\")\n", " messages.loc[i,'message'] = m\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "messages[:20]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* get a list only containing messaged with emojis" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n", "emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "display(emoji_messages)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## learning part" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import itertools\n", "import sklearn.utils as sku\n", "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "labels=[e[-1] for e in emoji_messages['emojis']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "labels[:10]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "vectorizer = TfidfVectorizer(stop_words='english')\n", "vec_train = vectorizer.fit_transform(X1)\n", "vec_test = vectorizer.transform(Xt1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.ensemble import RandomForestClassifier as RFC\n", "from sklearn.neural_network import MLPClassifier as MLP\n", "from sklearn.naive_bayes import MultinomialNB as MNB\n", "#clf_a = RFC(criterion='entropy', random_state=4222)\n", "clf_a = MLP()\n", "clf_a.fit(vec_train, y1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pred = clf_a.predict(vectorizer.transform(Xt1))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n", "testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n", "testlist.to_csv('export.csv')\n", "testlist" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(chr(0x1F3F))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "vec_train[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "out = widgets.Output()\n", "\n", "t = widgets.Text()\n", "b = widgets.Button(\n", " description='get smiley',\n", " disabled=False,\n", " button_style='', # 'success', 'info', 'warning', 'danger' or ''\n", " tooltip='Click me',\n", " icon='check'\n", ")\n", "\n", "\n", "\n", "def handle_submit(sender):\n", " with out:\n", " clear_output()\n", " with out:\n", " display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n", "\n", "b.on_click(handle_submit)\n", " \n", "display(t)\n", "display(widgets.VBox([b, out])) " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }