TWITTER! yeah!
This commit is contained in:
Jonas Weinz
parent
4b31d44b87
commit
1eb68b1bbc
@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -14,7 +14,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -55,9 +55,256 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"## converting plain whatsapp export to csv"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "8c94a8d3b3724ad08359817b2086cf85",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
|
||||
" that the widgets JavaScript is still loading. If this message persists, it\n",
|
||||
" likely means that the widgets JavaScript library is either not installed or\n",
|
||||
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
|
||||
" Widgets Documentation</a> for setup instructions.\n",
|
||||
"</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in another frontend (for example, a static\n",
|
||||
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
|
||||
" it may mean that your frontend doesn't currently support widgets.\n",
|
||||
"</p>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"VBox(children=(HBox(children=(Text(value='test.txt'), Button(description='convert whatsapp file to csv', icon='check', style=ButtonStyle(), tooltip='convert whatsapp file to csv'))), Output()))"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"## read csv and build database"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ee22e300097e49f1ac24b11662f7dc69",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
|
||||
" that the widgets JavaScript is still loading. If this message persists, it\n",
|
||||
" likely means that the widgets JavaScript library is either not installed or\n",
|
||||
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
|
||||
" Widgets Documentation</a> for setup instructions.\n",
|
||||
"</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in another frontend (for example, a static\n",
|
||||
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
|
||||
" it may mean that your frontend doesn't currently support widgets.\n",
|
||||
"</p>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"VBox(children=(HBox(children=(Text(value='test.txt.csv'), Checkbox(value=False, description='using only last emoji'), Button(description='read', icon='check', style=ButtonStyle(), tooltip='read'))), Output()))"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"## Train"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "de06136ee346492d80ef39d304cbc31c",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
|
||||
" that the widgets JavaScript is still loading. If this message persists, it\n",
|
||||
" likely means that the widgets JavaScript library is either not installed or\n",
|
||||
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
|
||||
" Widgets Documentation</a> for setup instructions.\n",
|
||||
"</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in another frontend (for example, a static\n",
|
||||
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
|
||||
" it may mean that your frontend doesn't currently support widgets.\n",
|
||||
"</p>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"VBox(children=(HBox(children=(Dropdown(description='Learning Method', index=1, options=('DecisionTree', 'MLP', 'RandomForest'), value='MLP'), Checkbox(value=False, description='Using one vs all (very slow, only with multi-label!)'), Button(description='train', icon='check', style=ButtonStyle(), tooltip='train'))), Output()))"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"## save trained classifier"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b046f3ada7ec4ba7a17965d718552d21",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
|
||||
" that the widgets JavaScript is still loading. If this message persists, it\n",
|
||||
" likely means that the widgets JavaScript library is either not installed or\n",
|
||||
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
|
||||
" Widgets Documentation</a> for setup instructions.\n",
|
||||
"</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in another frontend (for example, a static\n",
|
||||
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
|
||||
" it may mean that your frontend doesn't currently support widgets.\n",
|
||||
"</p>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='save classifier', icon='check', style=ButtonStyle(), tooltip='save classifier'))), Output()))"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"## import trained classifier"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ac4054e75d8a4214ad040b1b21d6c925",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
|
||||
" that the widgets JavaScript is still loading. If this message persists, it\n",
|
||||
" likely means that the widgets JavaScript library is either not installed or\n",
|
||||
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
|
||||
" Widgets Documentation</a> for setup instructions.\n",
|
||||
"</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in another frontend (for example, a static\n",
|
||||
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
|
||||
" it may mean that your frontend doesn't currently support widgets.\n",
|
||||
"</p>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='import classifier', icon='check', style=ButtonStyle(), tooltip='import classifier'))), Output()))"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"## predict emoji on custom text"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "db631acbe9c94cac907efaf501a69c6a",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
|
||||
" that the widgets JavaScript is still loading. If this message persists, it\n",
|
||||
" likely means that the widgets JavaScript library is either not installed or\n",
|
||||
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
|
||||
" Widgets Documentation</a> for setup instructions.\n",
|
||||
"</p>\n",
|
||||
"<p>\n",
|
||||
" If you're reading this message in another frontend (for example, a static\n",
|
||||
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
|
||||
" it may mean that your frontend doesn't currently support widgets.\n",
|
||||
"</p>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"VBox(children=(HBox(children=(Text(value=''), Checkbox(value=False, description='Show probabilities (only on trees)'), Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='get emoji'))), Output()))"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"mp(\"## converting plain whatsapp export to csv\")\n",
|
||||
"t_convert, b_convert = create_widgets([\"test.txt\"], \"convert whatsapp file to csv\", out_convert)\n",
|
||||
@ -88,7 +335,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -113,9 +360,17 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"found existing emoji specification\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%bash\n",
|
||||
"if [ ! -e emoji-data.txt ]\n",
|
||||
@ -136,7 +391,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -177,7 +432,11 @@
|
||||
" emoji_code_list.append(int(entry,16))\n",
|
||||
" emoji_code_set = set(emoji_code_list)\n",
|
||||
" display(Markdown(\"**imported Emojis** (without modifier):\\n>\" + \"\".join([chr(x) for x in emoji_code_set])))\n",
|
||||
" display(Markdown(\"**blacklisted Emojis:**\\n>\" + \"\".join([chr(x) for x in emoji_blacklist])))"
|
||||
" display(Markdown(\"**blacklisted Emojis:**\\n>\" + \"\".join([chr(x) for x in emoji_blacklist])))\n",
|
||||
" f = open('emoji-list.txt', 'w')\n",
|
||||
" for e in emoji_code_set:\n",
|
||||
" f.write(chr(e) + \"\\n\")\n",
|
||||
" f.close()"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -189,7 +448,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -203,7 +462,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -322,7 +581,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -345,7 +604,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -360,6 +619,8 @@
|
||||
" clf_a = joblib.load(t_read_c.value)\n",
|
||||
" if t_read_m.value != \"\":\n",
|
||||
" mlb = joblib.load(t_read_m.value)\n",
|
||||
" else:\n",
|
||||
" mlb = None\n",
|
||||
" vectorizer = joblib.load(t_read_v.value)\n",
|
||||
" mp(\"**done**\")\n",
|
||||
"b_read.on_click(read_from_file)"
|
||||
@ -367,7 +628,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
||||
1027
Project/Tools/emoji-list.txt
Normal file
1027
Project/Tools/emoji-list.txt
Normal file
File diff suppressed because it is too large
Load Diff
70
Project/Tools/twitter2messages.sh
Executable file
70
Project/Tools/twitter2messages.sh
Executable file
@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
SCRIPT=`realpath $0`
|
||||
SCRIPTPATH=`dirname $SCRIPT`
|
||||
|
||||
# toolset:---------------------------------------------------------------------
|
||||
|
||||
command 2> >(while read line; do echo -e "\e[01;31m$line\e[0m" >&2; done)
|
||||
|
||||
function lineprint {
|
||||
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' =
|
||||
}
|
||||
|
||||
function message {
|
||||
lineprint
|
||||
printf "$1\n"
|
||||
lineprint
|
||||
}
|
||||
|
||||
function error_message {
|
||||
lineprint
|
||||
printf "$1\n" >&2
|
||||
lineprint
|
||||
}
|
||||
|
||||
current_action="IDLE"
|
||||
|
||||
function confirm_action {
|
||||
message "successfully finished action: $current_action"
|
||||
}
|
||||
|
||||
function set_action {
|
||||
current_action="$1"
|
||||
message "$1"
|
||||
}
|
||||
|
||||
function perform {
|
||||
"$@"
|
||||
local status=$?
|
||||
if [ $status -ne 0 ]
|
||||
then
|
||||
error_message "$current_action failed!"
|
||||
fi
|
||||
return $status
|
||||
}
|
||||
|
||||
function perform_and_exit {
|
||||
perform "$@" || exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
INPUT=$1
|
||||
OUTPUT=$2
|
||||
if [ $# -ne 2 ]
|
||||
then
|
||||
error_message "Error: no input file given. Usage: $0 <Filepath> <outputfile>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set_action "processing all files in $INPUT and write to $OUTPUT"
|
||||
|
||||
perform_and_exit export elist=\"`head -c -1 "$SCRIPTPATH/emoji-list.txt" | tr '\n' ',' | sed 's/,/\",\"/g'`\"
|
||||
perform_and_exit echo "filter by emoji list:"
|
||||
perform_and_exit echo $elist | tr -d '"' | tr -d ','
|
||||
|
||||
perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
|
||||
# ↑ such obvious, much selfexplaining 💁😈
|
||||
|
||||
confirm_action
|
||||
Loading…
Reference in New Issue
Block a user