nlp-lab/Project/Tools/emoji tester old.ipynb
Jonas Weinz 4b31d44b87 blubb
2018-05-20 10:38:14 +02:00

424 lines
9.3 KiB
Plaintext

{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## file input stuff:\n",
"\n",
"* replace `test.txt` with yout whatsapp log file"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%bash\n",
"./whatsapp2csv.sh test.txt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read table `test.csv` exported by `whatsapp2csv.sh`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"messages = pd.read_csv('test.txt.csv', delimiter='\\t')\n",
"messages.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%bash\n",
"if [ ! -e emoji-data.txt ]\n",
"then\n",
" echo \"downloading emoji specification\"\n",
" wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n",
"else\n",
" echo \"found existing emoji specification\"\n",
"fi\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_blacklist = set([\n",
" 0x1F3FB,\n",
" 0x1F3FC,\n",
" 0x1F3FD,\n",
" 0x1F3FE,\n",
" 0x1F3FF\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n",
"emoji_data['type'] = emoji_data['type'].str.strip()\n",
"emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n",
"emoji_data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* now build a set out of the unicode types"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ord(\"😀\") == int('0x1f600',16)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_codes = emoji_data['unicode']\n",
"emoji_codes.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* we have to iterate over the whole list and extract all given ranges:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_code_list = []\n",
"for entry in emoji_codes:\n",
" # testing whether we have an entry or a range:\n",
" if '.' in entry:\n",
" # range\n",
" a,b = entry.split(\"..\")\n",
" for i in range(int(a,16),int(b,16) +1):\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(i)\n",
" else:\n",
" # single entry\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(int(entry,16))\n",
"emoji_code_set = set(emoji_code_list)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# simple test:\n",
"print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* expanding column and fill new emojis"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"messages[\"emojis\"] = None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for i in messages.index:\n",
" emoji_list = []\n",
" to_remove = []\n",
" m = messages.iloc[i]['message']\n",
" for c in str(m):\n",
" if ord(c) in emoji_code_set:\n",
" emoji_list.append(c)\n",
" elif ord(c) in emoji_blacklist:\n",
" to_remove.append(c)\n",
" \n",
" messages.loc[i,'emojis'] = emoji_list\n",
" #remove emiójis from message\n",
" for e in (emoji_list + to_remove):\n",
" m = m.replace(e,\"\")\n",
" messages.loc[i,'message'] = m\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"messages[:20]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get a list only containing messaged with emojis"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n",
"emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(emoji_messages)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## learning part"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import itertools\n",
"import sklearn.utils as sku\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"labels=[e[-1] for e in emoji_messages['emojis']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"labels[:10]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vectorizer = TfidfVectorizer(stop_words='english')\n",
"vec_train = vectorizer.fit_transform(X1)\n",
"vec_test = vectorizer.transform(Xt1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import RandomForestClassifier as RFC\n",
"from sklearn.neural_network import MLPClassifier as MLP\n",
"from sklearn.naive_bayes import MultinomialNB as MNB\n",
"#clf_a = RFC(criterion='entropy', random_state=4222)\n",
"clf_a = MLP()\n",
"clf_a.fit(vec_train, y1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pred = clf_a.predict(vectorizer.transform(Xt1))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n",
"testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n",
"testlist.to_csv('export.csv')\n",
"testlist"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(chr(0x1F3F))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vec_train[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"out = widgets.Output()\n",
"\n",
"t = widgets.Text()\n",
"b = widgets.Button(\n",
" description='get smiley',\n",
" disabled=False,\n",
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
" tooltip='Click me',\n",
" icon='check'\n",
")\n",
"\n",
"\n",
"\n",
"def handle_submit(sender):\n",
" with out:\n",
" clear_output()\n",
" with out:\n",
" display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n",
"\n",
"b.on_click(handle_submit)\n",
" \n",
"display(t)\n",
"display(widgets.VBox([b, out])) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}