optimierungen für observe und vorbereitung für parallele ansätze der
prediction
This commit is contained in:
		| @ -26,7 +26,9 @@ | ||||
|    "source": [ | ||||
|     "import random\n", | ||||
|     "import ipywidgets as widgets\n", | ||||
|     "from IPython.display import display, clear_output" | ||||
|     "from IPython.display import display, clear_output\n", | ||||
|     "import math\n", | ||||
|     "import datetime" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
| @ -37,9 +39,38 @@ | ||||
|     "each action of typing and sending should yield a new updated prediction for best fitting emojis" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "source": [ | ||||
|     "Initial definition of emojis used later" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "top_emojis = ['😂','😭','😍','😩','😊','😘','🙏','🙌','😉','😁','😅','😎','😢','😒','😏','😌','😔','😋','😀','😤']\n", | ||||
|     "predictions = [\"🤐\",\"🤑\",\"🤒\",\"🤓\",\"🤔\",\"🤕\",\"🤗\",\"🤘\"]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "#### Advanced Approach\n", | ||||
|     "define the classifier for advanced prediction, used for the sentiment prediction" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -69,53 +100,114 @@ | ||||
|     "sys.path.append(\"..\")\n", | ||||
|     "\n", | ||||
|     "import simple_approach.simple_twitter_learning as stl\n", | ||||
|     "clf = stl.pipeline_manager.load_pipeline_from_files( '../simple_approach/custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n", | ||||
|     "clf_advanced = stl.pipeline_manager.load_pipeline_from_files( '../simple_approach/custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n", | ||||
|     "\n", | ||||
|     "import Tools.Emoji_Distance as ed" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "#### Generate new Sample for online learning / reinforcement learning" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "execution_count": 4, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|    "source": [ | ||||
|     "def generate_new_training_sample (msg, emoji):\n", | ||||
|     "    sentiment = ed.emoji_to_sentiment_vector(emoji)\n", | ||||
|     "    \n", | ||||
|     "    #TODO message msg could be filtred\n", | ||||
|     "    text = msg\n", | ||||
|     "    return text, sentiment" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "#### Naive Approach\n", | ||||
|     "for topic related emoji prediction" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "execution_count": 5, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "#TODO" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "#### Merge Predictions\n", | ||||
|     "combine the predictions of both approaches" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "top_emojis = ['😂',\n", | ||||
|     " '😭',\n", | ||||
|     " '😍',\n", | ||||
|     " '😩',\n", | ||||
|     " '😊',\n", | ||||
|     " '😘',\n", | ||||
|     " '🙏',\n", | ||||
|     " '🙌',\n", | ||||
|     " '😉',\n", | ||||
|     " '😁',\n", | ||||
|     " '😅',\n", | ||||
|     " '😎',\n", | ||||
|     " '😢',\n", | ||||
|     " '😒',\n", | ||||
|     " '😏',\n", | ||||
|     " '😌',\n", | ||||
|     " '😔',\n", | ||||
|     " '😋',\n", | ||||
|     " '😀',\n", | ||||
|     " '😤']\n", | ||||
|     "predictions = [\"🤐\",\"🤑\",\"🤒\",\"🤓\",\"🤔\",\"🤕\",\"🤗\",\"🤘\"]\n", | ||||
|     "\n", | ||||
|     "def merged_prediction(msg , split = 1 , number = 8, target_emojis = top_emojis):\n", | ||||
|     "    \n", | ||||
|     "    #calc ratio of prediction splitted between advanced aprroach and naive approach\n", | ||||
|     "    number_advanced = round(split*number)\n", | ||||
|     "    number_naive = round(1-split)*number\n", | ||||
|     "    \n", | ||||
|     "    #predict the advanced approach\n", | ||||
|     "    sentiment = clf_advanced.predict([msg])\n", | ||||
|     "    prediction_advanced = ed.sentiment_vector_to_emoji(sentiment,n_results = number_advanced, custom_target_emojis=target_emojis)\n", | ||||
|     "    \n", | ||||
|     "    #predict emojis with the naive approach\n", | ||||
|     "    #prediction_naive = clf_naive.predict(msg, target_emojis)\n", | ||||
|     "    \n", | ||||
|     "    #concat both predictions\n", | ||||
|     "    prediction = prediction_advanced#.append(prediction_naive)\n", | ||||
|     "    \n", | ||||
|     "    return prediction[:number]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "Actions triggered when something is changed" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def trigger_new_prediction(all_chat, current_message):\n", | ||||
|     "    global predictions\n", | ||||
|     "    \n", | ||||
|     "    #random prediction for  initial test\n", | ||||
|     "    #random.shuffle(predictions)\n", | ||||
|     "    sent = clf.predict([current_message])\n", | ||||
|     "    p = ed.sentiment_vector_to_emoji(sent,n_results = 8, custom_target_emojis=top_emojis)\n", | ||||
|     "    \n", | ||||
|     "    #first prediction only using advanced approach\n", | ||||
|     "    #sent = clf_advanced.predict([current_message])\n", | ||||
|     "    #p = ed.sentiment_vector_to_emoji(sent,n_results = 8, custom_target_emojis=top_emojis)\n", | ||||
|     "    \n", | ||||
|     "    #merged prediction\n", | ||||
|     "    p = merged_prediction(msg = current_message, target_emojis=top_emojis)\n", | ||||
|     "    \n", | ||||
|     "    predictions = p\n", | ||||
|     "    update_descriptions()" | ||||
|    ] | ||||
| @ -130,7 +222,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "execution_count": 8, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -159,18 +251,27 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "execution_count": 9, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "text_input = widgets.Text()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 23, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "text_input = widgets.Text()\n", | ||||
|     "\n", | ||||
|     "def submit_new_message(p):\n", | ||||
|     "    global all_text\n", | ||||
|     "    bar = \"----------- \\n\"\n", | ||||
|     "    time = \"12:00 \\n\"\n", | ||||
|     "    time = str(datetime.datetime.now())+\"\\n\"\n", | ||||
|     "    msg = text_input.value +\"\\n\"\n", | ||||
|     "    new_message = bar + time + msg\n", | ||||
|     "    all_text += new_message \n", | ||||
| @ -181,12 +282,19 @@ | ||||
|     "        \n", | ||||
|     "    trigger_new_prediction(all_text, text_input.value)\n", | ||||
|     "    update_descriptions()\n", | ||||
|     "    text_input.description = \"\"\n", | ||||
|     "        \n", | ||||
|     "    text_input.value = \"\"" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "text_input.on_submit(submit_new_message)\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "#TODO\n", | ||||
|     "text_input.observe(lambda b: trigger_new_prediction(None, text_input.value))" | ||||
|    ] | ||||
|   }, | ||||
| @ -200,7 +308,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "execution_count": 12, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -221,7 +329,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "execution_count": 13, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -239,7 +347,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "execution_count": 14, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -257,7 +365,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "execution_count": 15, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -275,7 +383,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "execution_count": 16, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -293,7 +401,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "execution_count": 17, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -311,7 +419,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 12, | ||||
|    "execution_count": 18, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -329,7 +437,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 13, | ||||
|    "execution_count": 19, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -347,7 +455,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 14, | ||||
|    "execution_count": 20, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -365,7 +473,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 15, | ||||
|    "execution_count": 21, | ||||
|    "metadata": { | ||||
|     "collapsed": true | ||||
|    }, | ||||
| @ -403,13 +511,13 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 16, | ||||
|    "execution_count": 22, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "29e1cbab23c841a8b201b95c3d75da01", | ||||
|        "model_id": "dc2c580ed3224ae7b2582c7c039813dd", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|  | ||||
		Reference in New Issue
	
	Block a user