498 lines
14 KiB
Plaintext
498 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Evaluation\n",
|
|
"We want to evaluate our approach"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Needed\n",
|
|
"We want to define needed components for this UI"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import random\n",
|
|
"import ipywidgets as widgets\n",
|
|
"from IPython.display import display, clear_output\n",
|
|
"import math\n",
|
|
"import datetime"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Trigger refresh of prediction\n",
|
|
"each action of typing and sending should yield a new updated prediction for best fitting emojis"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Initial definition of emojis used later"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"#locally defined based on the first analysis of parts of our twitter data: resulting in the 20 most used emojis\n",
|
|
"#we used them for our first approaches of prediction\n",
|
|
"top_emojis = list(\"😳😋😀😌😏😔😒😎😢😅😁😉🙌🙏😘😊😩😍😭😂\")\n",
|
|
"#possible initial set of predictions, only used in naive test cases\n",
|
|
"predictions = [\"🤐\",\"🤑\",\"🤒\",\"🤓\",\"🤔\",\"🤕\",\"🤗\",\"🤘\"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Advanced Approach\n",
|
|
"define the classifier for advanced prediction, used for the sentiment prediction"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Using TensorFlow backend.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[nltk_data] Downloading package punkt to /Users/Carsten/nltk_data...\n",
|
|
"[nltk_data] Package punkt is already up-to-date!\n",
|
|
"[nltk_data] Downloading package averaged_perceptron_tagger to\n",
|
|
"[nltk_data] /Users/Carsten/nltk_data...\n",
|
|
"[nltk_data] Package averaged_perceptron_tagger is already up-to-\n",
|
|
"[nltk_data] date!\n",
|
|
"[nltk_data] Downloading package wordnet to /Users/Carsten/nltk_data...\n",
|
|
"[nltk_data] Package wordnet is already up-to-date!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#navigation into right path and generating classifier\n",
|
|
"import sys\n",
|
|
"sys.path.append(\"..\")\n",
|
|
"sys.path.append(\"../naive_approach\")\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"import simple_approach.simple_twitter_learning as stl\n",
|
|
"clf_advanced = stl.pipeline_manager.load_from_pipeline_file(\"/Users/Carsten/DataSets/NLP_LAB/d2v_final/test_d2v_e2.pipeline\")\n",
|
|
"\n",
|
|
"import Tools.Emoji_Distance as ed"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Generate new Sample for online learning / reinforcement learning"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def generate_new_training_sample (msg, emoji):\n",
|
|
" sentiment = ed.emoji_to_sentiment_vector(emoji)\n",
|
|
" \n",
|
|
" #TODO message msg could be filtred\n",
|
|
" text = msg\n",
|
|
" return text, sentiment"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Naive Approach\n",
|
|
"for topic related emoji prediction"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#sys.path.append(\"..\")\n",
|
|
"#print(sys.path)\n",
|
|
"\n",
|
|
"import naive_approach as clf_naive"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"tmp_dict = clf_naive.prepareData(stem=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Merge Predictions\n",
|
|
"combine the predictions of both approaches"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def merged_prediction(msg , split = 0.5 , number = 8, target_emojis = top_emojis):\n",
|
|
" \n",
|
|
" #calc ratio of prediction splitted between advanced aprroach and naive approach\n",
|
|
" number_advanced = round(split*number)\n",
|
|
" number_naive = round((1-split)*number)\n",
|
|
" \n",
|
|
" #predict emojis with the naive approach\n",
|
|
" prediction_naive , prediction_naive_values = clf_naive.predict(sentence = msg, lookup= tmp_dict, n = number_naive, embeddings = \"word2Vec\", stem = True)\n",
|
|
"\n",
|
|
" #filter 0 values\n",
|
|
" tmp1 = []\n",
|
|
" tmp2 = []\n",
|
|
" epsilon = 0.0001\n",
|
|
"\n",
|
|
" for i in range(len(prediction_naive)):\n",
|
|
" if(abs(prediction_naive_values[i]) > epsilon):\n",
|
|
" tmp1.append(prediction_naive[i])\n",
|
|
" tmp2.append(prediction_naive[i])\n",
|
|
"\n",
|
|
" prediction_naive = tmp1\n",
|
|
" prediction_naive_values = tmp2\n",
|
|
" \n",
|
|
" if(len(prediction_naive) < number_naive):\n",
|
|
" #print(\"only few matches\")\n",
|
|
" number_advanced = number - len(prediction_naive)\n",
|
|
" \n",
|
|
" #print(number, number_advanced, number_naive)\n",
|
|
" \n",
|
|
" #predict the advanced approach\n",
|
|
" sentiment = clf_advanced.predict([msg])\n",
|
|
" prediction_advanced = ed.sentiment_vector_to_emoji(sentiment,n_results = number_advanced, custom_target_emojis=target_emojis)\n",
|
|
" \n",
|
|
" #concat both predictions\n",
|
|
" prediction = list(prediction_advanced)+list(prediction_naive)\n",
|
|
" \n",
|
|
" return prediction[:number]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Actions triggered when something is changed"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def trigger_new_prediction(all_chat, current_message):\n",
|
|
" global predictions\n",
|
|
" \n",
|
|
" #random prediction for initial test\n",
|
|
" #random.shuffle(predictions)\n",
|
|
" \n",
|
|
" #first prediction only using advanced approach\n",
|
|
" #sent = clf_advanced.predict([current_message])\n",
|
|
" #p = ed.sentiment_vector_to_emoji(sent,n_results = 8, custom_target_emojis=top_emojis)\n",
|
|
" \n",
|
|
" #merged prediction\n",
|
|
" if(current_message != \"\"):\n",
|
|
" p = merged_prediction(msg = current_message, target_emojis=top_emojis)\n",
|
|
"\n",
|
|
" predictions = p"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Trigger Prediction for CSV Table"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style>\n",
|
|
" .dataframe thead tr:only-child th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: left;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>Sentence</th>\n",
|
|
" <th>prediction</th>\n",
|
|
" <th>label</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>I am so happy</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>i love my life</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>i really like this sunshine</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>while doing sport i feel free</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>i is terrible to learn when the weather is thi...</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>n</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" Sentence prediction label\n",
|
|
"0 I am so happy NaN p\n",
|
|
"1 i love my life NaN p\n",
|
|
"2 i really like this sunshine NaN p\n",
|
|
"3 while doing sport i feel free NaN p\n",
|
|
"4 i is terrible to learn when the weather is thi... NaN n"
|
|
]
|
|
},
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# get table\n",
|
|
"import pandas as pd\n",
|
|
"df = pd.read_csv(\"Evaluation Sentences - Sentiment related sentences.csv\")#, sep=\"\\t\")\n",
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"all_predictions = []\n",
|
|
"\n",
|
|
"for index, row in df.iterrows():\n",
|
|
" sentence = row[\"Sentence\"]\n",
|
|
" #print(sentence)\n",
|
|
"\n",
|
|
" trigger_new_prediction(all_chat=\"\", current_message = sentence)\n",
|
|
" #print(predictions)\n",
|
|
" \n",
|
|
" #prediction to string\n",
|
|
" tmp_prediction = \"\".join(predictions)\n",
|
|
" \n",
|
|
" #construct the preediction column\n",
|
|
" all_predictions.append(tmp_prediction)\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style>\n",
|
|
" .dataframe thead tr:only-child th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: left;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>Sentence</th>\n",
|
|
" <th>prediction</th>\n",
|
|
" <th>label</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>I am so happy</td>\n",
|
|
" <td>😂😅😢😳😁😌😉😎</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>i love my life</td>\n",
|
|
" <td>😅😂😢😳😁🏩💌🤟</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>i really like this sunshine</td>\n",
|
|
" <td>😅😂😢😳😭😁😌😔</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>while doing sport i feel free</td>\n",
|
|
" <td>😂😅😁😌😉😎😳😢</td>\n",
|
|
" <td>p</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>i is terrible to learn when the weather is thi...</td>\n",
|
|
" <td>😂😅😁😉😌😎😳🙅</td>\n",
|
|
" <td>n</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" Sentence prediction label\n",
|
|
"0 I am so happy 😂😅😢😳😁😌😉😎 p\n",
|
|
"1 i love my life 😅😂😢😳😁🏩💌🤟 p\n",
|
|
"2 i really like this sunshine 😅😂😢😳😭😁😌😔 p\n",
|
|
"3 while doing sport i feel free 😂😅😁😌😉😎😳😢 p\n",
|
|
"4 i is terrible to learn when the weather is thi... 😂😅😁😉😌😎😳🙅 n"
|
|
]
|
|
},
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df[\"prediction\"] = all_predictions\n",
|
|
"\n",
|
|
"df.head()\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"df.to_csv(\"E_S - sentiment - d2v - w2v - no stemming.csv\", sep='\\t', encoding='utf-8')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|