424 lines
9.3 KiB
Plaintext
424 lines
9.3 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"from IPython.display import clear_output, Markdown, Math\n",
|
|
"import ipywidgets as widgets\n",
|
|
"import os"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----\n",
|
|
"## file input stuff:\n",
|
|
"\n",
|
|
"* replace `test.txt` with yout whatsapp log file"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%bash\n",
|
|
"./whatsapp2csv.sh test.txt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* read table `test.csv` exported by `whatsapp2csv.sh`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"messages = pd.read_csv('test.txt.csv', delimiter='\\t')\n",
|
|
"messages.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%bash\n",
|
|
"if [ ! -e emoji-data.txt ]\n",
|
|
"then\n",
|
|
" echo \"downloading emoji specification\"\n",
|
|
" wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n",
|
|
"else\n",
|
|
" echo \"found existing emoji specification\"\n",
|
|
"fi\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emoji_blacklist = set([\n",
|
|
" 0x1F3FB,\n",
|
|
" 0x1F3FC,\n",
|
|
" 0x1F3FD,\n",
|
|
" 0x1F3FE,\n",
|
|
" 0x1F3FF\n",
|
|
"])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n",
|
|
"emoji_data['type'] = emoji_data['type'].str.strip()\n",
|
|
"emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n",
|
|
"emoji_data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* now build a set out of the unicode types"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ord(\"😀\") == int('0x1f600',16)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emoji_codes = emoji_data['unicode']\n",
|
|
"emoji_codes.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* we have to iterate over the whole list and extract all given ranges:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emoji_code_list = []\n",
|
|
"for entry in emoji_codes:\n",
|
|
" # testing whether we have an entry or a range:\n",
|
|
" if '.' in entry:\n",
|
|
" # range\n",
|
|
" a,b = entry.split(\"..\")\n",
|
|
" for i in range(int(a,16),int(b,16) +1):\n",
|
|
" if i not in emoji_blacklist:\n",
|
|
" emoji_code_list.append(i)\n",
|
|
" else:\n",
|
|
" # single entry\n",
|
|
" if i not in emoji_blacklist:\n",
|
|
" emoji_code_list.append(int(entry,16))\n",
|
|
"emoji_code_set = set(emoji_code_list)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# simple test:\n",
|
|
"print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* expanding column and fill new emojis"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"messages[\"emojis\"] = None"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"for i in messages.index:\n",
|
|
" emoji_list = []\n",
|
|
" to_remove = []\n",
|
|
" m = messages.iloc[i]['message']\n",
|
|
" for c in str(m):\n",
|
|
" if ord(c) in emoji_code_set:\n",
|
|
" emoji_list.append(c)\n",
|
|
" elif ord(c) in emoji_blacklist:\n",
|
|
" to_remove.append(c)\n",
|
|
" \n",
|
|
" messages.loc[i,'emojis'] = emoji_list\n",
|
|
" #remove emiójis from message\n",
|
|
" for e in (emoji_list + to_remove):\n",
|
|
" m = m.replace(e,\"\")\n",
|
|
" messages.loc[i,'message'] = m\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"messages[:20]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* get a list only containing messaged with emojis"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n",
|
|
"emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"display(emoji_messages)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----\n",
|
|
"## learning part"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import itertools\n",
|
|
"import sklearn.utils as sku\n",
|
|
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
|
|
"from sklearn.model_selection import train_test_split"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"labels=[e[-1] for e in emoji_messages['emojis']]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"labels[:10]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vectorizer = TfidfVectorizer(stop_words='english')\n",
|
|
"vec_train = vectorizer.fit_transform(X1)\n",
|
|
"vec_test = vectorizer.transform(Xt1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.ensemble import RandomForestClassifier as RFC\n",
|
|
"from sklearn.neural_network import MLPClassifier as MLP\n",
|
|
"from sklearn.naive_bayes import MultinomialNB as MNB\n",
|
|
"#clf_a = RFC(criterion='entropy', random_state=4222)\n",
|
|
"clf_a = MLP()\n",
|
|
"clf_a.fit(vec_train, y1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"pred = clf_a.predict(vectorizer.transform(Xt1))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n",
|
|
"testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n",
|
|
"testlist.to_csv('export.csv')\n",
|
|
"testlist"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"print(chr(0x1F3F))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vec_train[0]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"out = widgets.Output()\n",
|
|
"\n",
|
|
"t = widgets.Text()\n",
|
|
"b = widgets.Button(\n",
|
|
" description='get smiley',\n",
|
|
" disabled=False,\n",
|
|
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
|
|
" tooltip='Click me',\n",
|
|
" icon='check'\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"def handle_submit(sender):\n",
|
|
" with out:\n",
|
|
" clear_output()\n",
|
|
" with out:\n",
|
|
" display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n",
|
|
"\n",
|
|
"b.on_click(handle_submit)\n",
|
|
" \n",
|
|
"display(t)\n",
|
|
"display(widgets.VBox([b, out])) "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|