jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimierungen für observe und vorbereitung für parallele ansätze der

Browse Source 
prediction
This commit is contained in:
Carsten 2018-06-26 15:02:51 +02:00
parent 43e9ace028
commit d6e24c2065
1 changed files with 158 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
Project/Tools/User_Interface.ipynb
View File

@ -26,7 +26,9 @@
"source": [ "source": [
"import random\n", "import random\n",
"import ipywidgets as widgets\n", "import ipywidgets as widgets\n",
"from IPython.display import display, clear_output" "from IPython.display import display, clear_output\n",
"import math\n",
"import datetime"
] ]
}, },
{ {
@ -37,9 +39,38 @@
"each action of typing and sending should yield a new updated prediction for best fitting emojis" "each action of typing and sending should yield a new updated prediction for best fitting emojis"
] ]
}, },
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"Initial definition of emojis used later"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"top_emojis = ['😂','😭','😍','😩','😊','😘','🙏','🙌','😉','😁','😅','😎','😢','😒','😏','😌','😔','😋','😀','😤']\n",
"predictions = [\"🤐\",\"🤑\",\"🤒\",\"🤓\",\"🤔\",\"🤕\",\"🤗\",\"🤘\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Advanced Approach\n",
"define the classifier for advanced prediction, used for the sentiment prediction"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -69,53 +100,114 @@
"sys.path.append(\"..\")\n", "sys.path.append(\"..\")\n",
"\n", "\n",
"import simple_approach.simple_twitter_learning as stl\n", "import simple_approach.simple_twitter_learning as stl\n",
"clf = stl.pipeline_manager.load_pipeline_from_files( '../simple_approach/custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n", "clf_advanced = stl.pipeline_manager.load_pipeline_from_files( '../simple_approach/custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n",
"\n", "\n",
"import Tools.Emoji_Distance as ed" "import Tools.Emoji_Distance as ed"
] ]
}, },
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Generate new Sample for online learning / reinforcement learning"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 4,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
"outputs": [], "outputs": [],
"source": [] "source": [
"def generate_new_training_sample (msg, emoji):\n",
" sentiment = ed.emoji_to_sentiment_vector(emoji)\n",
" \n",
" #TODO message msg could be filtred\n",
" text = msg\n",
" return text, sentiment"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Naive Approach\n",
"for topic related emoji prediction"
]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"#TODO"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Merge Predictions\n",
"combine the predictions of both approaches"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"top_emojis = ['😂',\n", "def merged_prediction(msg , split = 1 , number = 8, target_emojis = top_emojis):\n",
" '😭',\n",
" '😍',\n",
" '😩',\n",
" '😊',\n",
" '😘',\n",
" '🙏',\n",
" '🙌',\n",
" '😉',\n",
" '😁',\n",
" '😅',\n",
" '😎',\n",
" '😢',\n",
" '😒',\n",
" '😏',\n",
" '😌',\n",
" '😔',\n",
" '😋',\n",
" '😀',\n",
" '😤']\n",
"predictions = [\"🤐\",\"🤑\",\"🤒\",\"🤓\",\"🤔\",\"🤕\",\"🤗\",\"🤘\"]\n",
" \n", " \n",
" #calc ratio of prediction splitted between advanced aprroach and naive approach\n",
" number_advanced = round(split*number)\n",
" number_naive = round(1-split)*number\n",
" \n",
" #predict the advanced approach\n",
" sentiment = clf_advanced.predict([msg])\n",
" prediction_advanced = ed.sentiment_vector_to_emoji(sentiment,n_results = number_advanced, custom_target_emojis=target_emojis)\n",
" \n",
" #predict emojis with the naive approach\n",
" #prediction_naive = clf_naive.predict(msg, target_emojis)\n",
" \n",
" #concat both predictions\n",
" prediction = prediction_advanced#.append(prediction_naive)\n",
" \n",
" return prediction[:number]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Actions triggered when something is changed"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def trigger_new_prediction(all_chat, current_message):\n", "def trigger_new_prediction(all_chat, current_message):\n",
" global predictions\n", " global predictions\n",
" \n",
" #random prediction for initial test\n",
" #random.shuffle(predictions)\n", " #random.shuffle(predictions)\n",
" sent = clf.predict([current_message])\n", " \n",
" p = ed.sentiment_vector_to_emoji(sent,n_results = 8, custom_target_emojis=top_emojis)\n", " #first prediction only using advanced approach\n",
" #sent = clf_advanced.predict([current_message])\n",
" #p = ed.sentiment_vector_to_emoji(sent,n_results = 8, custom_target_emojis=top_emojis)\n",
" \n",
" #merged prediction\n",
" p = merged_prediction(msg = current_message, target_emojis=top_emojis)\n",
" \n",
" predictions = p\n", " predictions = p\n",
" update_descriptions()" " update_descriptions()"
] ]
@ -130,7 +222,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 8,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -159,18 +251,27 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 9,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"text_input = widgets.Text()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"text_input = widgets.Text()\n",
"\n",
"def submit_new_message(p):\n", "def submit_new_message(p):\n",
" global all_text\n", " global all_text\n",
" bar = \"----------- \\n\"\n", " bar = \"----------- \\n\"\n",
" time = \"12:00 \\n\"\n", " time = str(datetime.datetime.now())+\"\\n\"\n",
" msg = text_input.value +\"\\n\"\n", " msg = text_input.value +\"\\n\"\n",
" new_message = bar + time + msg\n", " new_message = bar + time + msg\n",
" all_text += new_message \n", " all_text += new_message \n",
@ -181,12 +282,19 @@
" \n", " \n",
" trigger_new_prediction(all_text, text_input.value)\n", " trigger_new_prediction(all_text, text_input.value)\n",
" update_descriptions()\n", " update_descriptions()\n",
" text_input.description = \"\"\n", " text_input.value = \"\""
" \n", ]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"text_input.on_submit(submit_new_message)\n", "text_input.on_submit(submit_new_message)\n",
"\n", "\n",
"\n",
"#TODO\n",
"text_input.observe(lambda b: trigger_new_prediction(None, text_input.value))" "text_input.observe(lambda b: trigger_new_prediction(None, text_input.value))"
] ]
}, },
@ -200,7 +308,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 12,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -221,7 +329,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 13,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -239,7 +347,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 14,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -257,7 +365,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 15,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -275,7 +383,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 16,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -293,7 +401,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 17,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -311,7 +419,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 18,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -329,7 +437,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 19,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -347,7 +455,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 20,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -365,7 +473,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 21,
"metadata": { "metadata": {
"collapsed": true "collapsed": true
}, },
@ -403,13 +511,13 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 16, "execution_count": 22,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"application/vnd.jupyter.widget-view+json": { "application/vnd.jupyter.widget-view+json": {
"model_id": "29e1cbab23c841a8b201b95c3d75da01", "model_id": "dc2c580ed3224ae7b2582c7c039813dd",
"version_major": 2, "version_major": 2,
"version_minor": 0 "version_minor": 0
}, },