nlp-lab/Project/Tools/Evaluation_with_csv.ipynb

381 lines
16 KiB
Plaintext
Raw Normal View History

2018-07-19 13:00:57 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Evaluation\n",
"We want to evaluate our approach"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Needed\n",
"We want to define needed components for this UI"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import random\n",
"import ipywidgets as widgets\n",
"from IPython.display import display, clear_output\n",
"import math\n",
"import datetime"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Trigger refresh of prediction\n",
"each action of typing and sending should yield a new updated prediction for best fitting emojis"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"Initial definition of emojis used later"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"#locally defined based on the first analysis of parts of our twitter data: resulting in the 20 most used emojis\n",
"#we used them for our first approaches of prediction\n",
"top_emojis = ['😂','😭','😍','😩','😊','😘','🙏','🙌','😉','😁','😅','😎','😢','😒','😏','😌','😔','😋','😀','😤']\n",
"#possible initial set of predictions, only used in naive test cases\n",
"predictions = [\"🤐\",\"🤑\",\"🤒\",\"🤓\",\"🤔\",\"🤕\",\"🤗\",\"🤘\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Advanced Approach\n",
"define the classifier for advanced prediction, used for the sentiment prediction"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[nltk_data] Downloading package punkt to /Users/Carsten/nltk_data...\n",
"[nltk_data] Package punkt is already up-to-date!\n",
"[nltk_data] Downloading package averaged_perceptron_tagger to\n",
"[nltk_data] /Users/Carsten/nltk_data...\n",
"[nltk_data] Package averaged_perceptron_tagger is already up-to-\n",
"[nltk_data] date!\n",
"[nltk_data] Downloading package wordnet to /Users/Carsten/nltk_data...\n",
"[nltk_data] Package wordnet is already up-to-date!\n"
]
}
],
"source": [
"#navigation into right path and generating classifier\n",
"import sys\n",
"sys.path.append(\"..\")\n",
2018-07-20 13:46:12 +02:00
"sys.path.append(\"../naive_approach\")\n",
"\n",
"\n",
2018-07-19 13:00:57 +02:00
"\n",
"import simple_approach.simple_twitter_learning as stl\n",
2018-07-23 10:22:34 +02:00
"clf_advanced = stl.pipeline_manager.load_from_pipeline_file(\"/Users/Carsten/DataSets/NLP_LAB/tfidf_final/final_epoch01.pipeline\")\n",
2018-07-19 13:00:57 +02:00
"\n",
"import Tools.Emoji_Distance as ed"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Generate new Sample for online learning / reinforcement learning"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def generate_new_training_sample (msg, emoji):\n",
" sentiment = ed.emoji_to_sentiment_vector(emoji)\n",
" \n",
" #TODO message msg could be filtred\n",
" text = msg\n",
" return text, sentiment"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Naive Approach\n",
"for topic related emoji prediction"
]
},
{
"cell_type": "code",
"execution_count": 5,
2018-07-20 13:46:12 +02:00
"metadata": {},
2018-07-23 10:22:34 +02:00
"outputs": [
{
"ename": "KeyError",
"evalue": "'character'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_value\u001b[0;34m(self, series, key)\u001b[0m\n\u001b[1;32m 2482\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2483\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mlibts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value_box\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2484\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32mpandas/_libs/tslib.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslib.get_value_box (pandas/_libs/tslib.c:18843)\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mpandas/_libs/tslib.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslib.get_value_box (pandas/_libs/tslib.c:18477)\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: 'str' object cannot be interpreted as an integer",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-5-2e408a3beaf0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;31m#print(sys.path)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mnaive_approach\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mclf_naive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/GitRepos/NLP-LAB/Project/naive_approach/naive_approach.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0mtableDict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mtableDict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'character'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'description'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m#######################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 599\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 601\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 602\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 603\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_value\u001b[0;34m(self, series, key)\u001b[0m\n\u001b[1;32m 2489\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mInvalidIndexError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2490\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2491\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2492\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pragma: no cover\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2493\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0me1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_value\u001b[0;34m(self, series, key)\u001b[0m\n\u001b[1;32m 2475\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2476\u001b[0m return self._engine.get_value(s, k,\n\u001b[0;32m-> 2477\u001b[0;31m tz=getattr(series.dtype, 'tz', None))\n\u001b[0m\u001b[1;32m 2478\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2479\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minferred_type\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'integer'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'boolean'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: 'character'"
]
}
],
2018-07-19 13:00:57 +02:00
"source": [
"#sys.path.append(\"..\")\n",
"#print(sys.path)\n",
"\n",
2018-07-20 13:46:12 +02:00
"import naive_approach as clf_naive"
2018-07-19 13:00:57 +02:00
]
},
{
"cell_type": "code",
2018-07-23 10:22:34 +02:00
"execution_count": null,
2018-07-19 13:00:57 +02:00
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"tmp_dict = clf_naive.prepareData()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Merge Predictions\n",
"combine the predictions of both approaches"
]
},
{
"cell_type": "code",
2018-07-23 10:22:34 +02:00
"execution_count": null,
2018-07-19 13:00:57 +02:00
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def merged_prediction(msg , split = 0.5 , number = 8, target_emojis = top_emojis):\n",
" \n",
" #calc ratio of prediction splitted between advanced aprroach and naive approach\n",
" number_advanced = round(split*number)\n",
" number_naive = round((1-split)*number)\n",
" \n",
" #predict emojis with the naive approach\n",
2018-07-23 10:22:34 +02:00
" prediction_naive , prediction_naive_values = clf_naive.predict(sentence = msg, lookup= tmp_dict, n = number_naive, em)\n",
2018-07-19 13:00:57 +02:00
"\n",
" #filter 0 values\n",
" tmp1 = []\n",
" tmp2 = []\n",
" epsilon = 0.0001\n",
"\n",
" for i in range(len(prediction_naive)):\n",
" if(abs(prediction_naive_values[i]) > epsilon):\n",
" tmp1.append(prediction_naive[i])\n",
" tmp2.append(prediction_naive[i])\n",
"\n",
" prediction_naive = tmp1\n",
" prediction_naive_values = tmp2\n",
" \n",
" if(len(prediction_naive) < number_naive):\n",
" #print(\"only few matches\")\n",
" number_advanced = number - len(prediction_naive)\n",
" \n",
" #print(number, number_advanced, number_naive)\n",
" \n",
" #predict the advanced approach\n",
" sentiment = clf_advanced.predict([msg])\n",
" prediction_advanced = ed.sentiment_vector_to_emoji(sentiment,n_results = number_advanced, custom_target_emojis=target_emojis)\n",
" \n",
" #concat both predictions\n",
" prediction = list(prediction_advanced)+list(prediction_naive)\n",
" \n",
" return prediction[:number]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Actions triggered when something is changed"
]
},
{
"cell_type": "code",
2018-07-23 10:22:34 +02:00
"execution_count": null,
2018-07-19 13:00:57 +02:00
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def trigger_new_prediction(all_chat, current_message):\n",
" global predictions\n",
" \n",
" #random prediction for initial test\n",
" #random.shuffle(predictions)\n",
" \n",
" #first prediction only using advanced approach\n",
" #sent = clf_advanced.predict([current_message])\n",
" #p = ed.sentiment_vector_to_emoji(sent,n_results = 8, custom_target_emojis=top_emojis)\n",
" \n",
" #merged prediction\n",
" if(current_message != \"\"):\n",
" p = merged_prediction(msg = current_message, target_emojis=top_emojis)\n",
"\n",
2018-07-20 13:46:12 +02:00
" predictions = p"
2018-07-19 13:00:57 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Trigger Prediction for CSV Table"
]
},
{
"cell_type": "code",
2018-07-23 10:22:34 +02:00
"execution_count": null,
2018-07-19 13:00:57 +02:00
"metadata": {},
2018-07-23 10:22:34 +02:00
"outputs": [],
2018-07-19 13:00:57 +02:00
"source": [
"# get table\n",
"import pandas as pd\n",
2018-07-23 10:22:34 +02:00
"df = pd.read_csv(\"Evaluation Sentences - Tabellenblatt1.csv\", sep=\"\\t\")\n",
2018-07-19 13:00:57 +02:00
"df.head()"
]
},
{
"cell_type": "code",
2018-07-20 13:46:12 +02:00
"execution_count": null,
2018-07-23 10:22:34 +02:00
"metadata": {
"collapsed": true
},
2018-07-20 13:46:12 +02:00
"outputs": [],
2018-07-19 13:00:57 +02:00
"source": [
2018-07-20 13:46:12 +02:00
"all_predictions = []\n",
"\n",
2018-07-19 13:00:57 +02:00
"for index, row in df.iterrows():\n",
" sentence = row[\"Sentence\"]\n",
2018-07-20 13:46:12 +02:00
" #print(sentence)\n",
2018-07-19 13:00:57 +02:00
"\n",
" trigger_new_prediction(all_chat=\"\", current_message = sentence)\n",
2018-07-20 13:46:12 +02:00
" #print(predictions)\n",
" \n",
" #prediction to string\n",
" tmp_prediction = \"\".join(predictions)\n",
" \n",
" #construct the preediction column\n",
" all_predictions.append(tmp_prediction)\n",
2018-07-19 13:00:57 +02:00
" "
]
},
2018-07-20 13:46:12 +02:00
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df[\"prediction\"] = all_predictions\n",
"\n",
"df.head()\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
2018-07-23 10:22:34 +02:00
"df.to_csv(\"Evaluation Sentences - Wordnet - newClf.csv\", sep='\\t', encoding='utf-8')"
2018-07-20 13:46:12 +02:00
]
},
2018-07-19 13:00:57 +02:00
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}