TWITTER! yeah!

This commit is contained in:
Jonas Weinz 2018-05-21 22:46:36 +02:00
parent 4b31d44b87
commit 1eb68b1bbc
3 changed files with 1372 additions and 14 deletions

View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -14,7 +14,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -55,9 +55,256 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"data": {
"text/markdown": [
"## converting plain whatsapp export to csv"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8c94a8d3b3724ad08359817b2086cf85",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='test.txt'), Button(description='convert whatsapp file to csv', icon='check', style=ButtonStyle(), tooltip='convert whatsapp file to csv'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## read csv and build database"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ee22e300097e49f1ac24b11662f7dc69",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='test.txt.csv'), Checkbox(value=False, description='using only last emoji'), Button(description='read', icon='check', style=ButtonStyle(), tooltip='read'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## Train"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "de06136ee346492d80ef39d304cbc31c",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Dropdown(description='Learning Method', index=1, options=('DecisionTree', 'MLP', 'RandomForest'), value='MLP'), Checkbox(value=False, description='Using one vs all (very slow, only with multi-label!)'), Button(description='train', icon='check', style=ButtonStyle(), tooltip='train'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## save trained classifier"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b046f3ada7ec4ba7a17965d718552d21",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='save classifier', icon='check', style=ButtonStyle(), tooltip='save classifier'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## import trained classifier"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ac4054e75d8a4214ad040b1b21d6c925",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='import classifier', icon='check', style=ButtonStyle(), tooltip='import classifier'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## predict emoji on custom text"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "db631acbe9c94cac907efaf501a69c6a",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value=''), Checkbox(value=False, description='Show probabilities (only on trees)'), Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='get emoji'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"mp(\"## converting plain whatsapp export to csv\")\n", "mp(\"## converting plain whatsapp export to csv\")\n",
"t_convert, b_convert = create_widgets([\"test.txt\"], \"convert whatsapp file to csv\", out_convert)\n", "t_convert, b_convert = create_widgets([\"test.txt\"], \"convert whatsapp file to csv\", out_convert)\n",
@ -88,7 +335,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -113,9 +360,17 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"found existing emoji specification\n"
]
}
],
"source": [ "source": [
"%%bash\n", "%%bash\n",
"if [ ! -e emoji-data.txt ]\n", "if [ ! -e emoji-data.txt ]\n",
@ -136,7 +391,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -177,7 +432,11 @@
" emoji_code_list.append(int(entry,16))\n", " emoji_code_list.append(int(entry,16))\n",
" emoji_code_set = set(emoji_code_list)\n", " emoji_code_set = set(emoji_code_list)\n",
" display(Markdown(\"**imported Emojis** (without modifier):\\n>\" + \"\".join([chr(x) for x in emoji_code_set])))\n", " display(Markdown(\"**imported Emojis** (without modifier):\\n>\" + \"\".join([chr(x) for x in emoji_code_set])))\n",
" display(Markdown(\"**blacklisted Emojis:**\\n>\" + \"\".join([chr(x) for x in emoji_blacklist])))" " display(Markdown(\"**blacklisted Emojis:**\\n>\" + \"\".join([chr(x) for x in emoji_blacklist])))\n",
" f = open('emoji-list.txt', 'w')\n",
" for e in emoji_code_set:\n",
" f.write(chr(e) + \"\\n\")\n",
" f.close()"
] ]
}, },
{ {
@ -189,7 +448,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -203,7 +462,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -322,7 +581,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -345,7 +604,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 10,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -360,6 +619,8 @@
" clf_a = joblib.load(t_read_c.value)\n", " clf_a = joblib.load(t_read_c.value)\n",
" if t_read_m.value != \"\":\n", " if t_read_m.value != \"\":\n",
" mlb = joblib.load(t_read_m.value)\n", " mlb = joblib.load(t_read_m.value)\n",
" else:\n",
" mlb = None\n",
" vectorizer = joblib.load(t_read_v.value)\n", " vectorizer = joblib.load(t_read_v.value)\n",
" mp(\"**done**\")\n", " mp(\"**done**\")\n",
"b_read.on_click(read_from_file)" "b_read.on_click(read_from_file)"
@ -367,7 +628,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 11,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [

1027
Project/Tools/emoji-list.txt Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,70 @@
#!/usr/bin/env bash
SCRIPT=`realpath $0`
SCRIPTPATH=`dirname $SCRIPT`
# toolset:---------------------------------------------------------------------
command 2> >(while read line; do echo -e "\e[01;31m$line\e[0m" >&2; done)
function lineprint {
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' =
}
function message {
lineprint
printf "$1\n"
lineprint
}
function error_message {
lineprint
printf "$1\n" >&2
lineprint
}
current_action="IDLE"
function confirm_action {
message "successfully finished action: $current_action"
}
function set_action {
current_action="$1"
message "$1"
}
function perform {
"$@"
local status=$?
if [ $status -ne 0 ]
then
error_message "$current_action failed!"
fi
return $status
}
function perform_and_exit {
perform "$@" || exit 1
}
# -----------------------------------------------------------------------------
INPUT=$1
OUTPUT=$2
if [ $# -ne 2 ]
then
error_message "Error: no input file given. Usage: $0 <Filepath> <outputfile>"
exit 1
fi
set_action "processing all files in $INPUT and write to $OUTPUT"
perform_and_exit export elist=\"`head -c -1 "$SCRIPTPATH/emoji-list.txt" | tr '\n' ',' | sed 's/,/\",\"/g'`\"
perform_and_exit echo "filter by emoji list:"
perform_and_exit echo $elist | tr -d '"' | tr -d ','
perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
# ↑ such obvious, much selfexplaining 💁😈
confirm_action