jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved output and threshold

Browse Source
This commit is contained in:
Jonas Weinz 2018-05-31 15:23:57 +02:00
parent 9bac276770
commit 48e89cda68
1 changed files with 77 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
Project/naive_approach/naive_approach.ipynb
View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 42, "execution_count": 149,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -15,7 +15,10 @@
"from nltk.stem import PorterStemmer\n", "from nltk.stem import PorterStemmer\n",
"from nltk.tokenize import sent_tokenize, word_tokenize\n", "from nltk.tokenize import sent_tokenize, word_tokenize\n",
"from nltk.corpus import wordnet\n", "from nltk.corpus import wordnet\n",
"import math" "import math\n",
"import pprint\n",
"\n",
"pp=pprint.PrettyPrinter(indent=4)"
] ]
}, },
{ {
@ -270,11 +273,11 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 106, "execution_count": 130,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"result = evaluate_sentence(\"car soccer surf\")" "result = evaluate_sentence(\"I like playing soccer\")"
] ]
}, },
{ {
@ -286,7 +289,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 107, "execution_count": 131,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -304,7 +307,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 108, "execution_count": 139,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -312,33 +315,86 @@
"max_val = np.argsort([-np.max(x) for x in result])\n", "max_val = np.argsort([-np.max(x) for x in result])\n",
"avg = np.argsort([-np.mean(x) for x in result])\n", "avg = np.argsort([-np.mean(x) for x in result])\n",
"\n", "\n",
"t = 0.7\n", "t = 0.9\n",
"threshold = np.argsort([-len(np.where(x>t)[0]) for x in result])\n" "threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 109, "execution_count": 156,
"metadata": {},
"outputs": [],
"source": [
"def print_best_results(sorted_indices, n=10):\n",
" pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n",
" pp.pprint([result[x] for x in sorted_indices[:10]])"
]
},
{
"cell_type": "code",
"execution_count": 157,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "name": "stdout",
"text/markdown": [ "output_type": "stream",
"# 🏉⚾🎳🔥🏐🎱💏🧾🚗🚘" "text": [
], "[ '⚽ -- SOCCER BALL',\n",
"text/plain": [ " '🏉 -- RUGBY FOOTBALL',\n",
"<IPython.core.display.Markdown object>" " '🏈 -- AMERICAN FOOTBALL',\n",
" '🎴 -- FLOWER PLAYING CARDS',\n",
" '🃏 -- PLAYING CARD BLACK JOKER',\n",
" '🇮 -- REGIONAL INDICATOR SYMBOL LETTER I',\n",
" '\\U0001f91f -- I LOVE YOU HAND SIGN',\n",
" '📧 -- E-MAIL SYMBOL',\n",
" '📭 -- OPEN MAILBOX WITH LOWERED FLAG',\n",
" '📥 -- INBOX TRAY']\n",
"[ array([[0.25 , 0.28571429, 0.58333333, 1. ],\n",
" [0.26666667, 0.10526316, 0.1 , 0.1 ]]),\n",
" array([[0.25 , 0.28571429, 0.58333333, 0.84615385],\n",
" [0.26666667, 0.3 , 0.60869565, 0.96 ]]),\n",
" array([[0.33333333, 0.125 , 0.11764706, 0.11764706],\n",
" [0.26666667, 0.3 , 0.60869565, 0.96 ]]),\n",
" array([[0.23529412, 0.0952381 , 0.09090909, 0.09090909],\n",
" [0.25 , 0.47619048, 1. , 0.58333333],\n",
" [0.30769231, 0.33333333, 0.57142857, 0.7 ]]),\n",
" array([[0.25 , 0.47619048, 1. , 0.58333333],\n",
" [0.53333333, 0.22222222, 0.21052632, 0.21052632],\n",
" [0.30769231, 0.22222222, 0.21052632, 0.21052632],\n",
" [0.28571429, 0.11111111, 0.10526316, 0.10526316]]),\n",
" array([[0. , 0. , 0. , 0. ],\n",
" [0.33333333, 0.23529412, 0.22222222, 0.22222222],\n",
" [0.4 , 0.26666667, 0.25 , 0.25 ],\n",
" [0.30769231, 0.22222222, 0.21052632, 0.21052632],\n",
" [1. , 0.26666667, 0.25 , 0.25 ]]),\n",
" array([[1. , 0.26666667, 0.25 , 0.25 ],\n",
" [0.33333333, 0.23529412, 0.22222222, 0.22222222],\n",
" [0. , 0. , 0. , 0. ],\n",
" [0.28571429, 0.11111111, 0.10526316, 0.10526316],\n",
" [0.33333333, 0.23529412, 0.22222222, 0.22222222]]),\n",
" array([[0.28571429, 0.31578947, 0.45454545, 0.5 ],\n",
" [0.4 , 0.26666667, 0.25 , 0.25 ]]),\n",
" array([[0.30769231, 0.11764706, 0.11111111, 0.11111111],\n",
" [0.26666667, 0.10526316, 0.1 , 0.1 ],\n",
" [0. , 0. , 0. , 0. ],\n",
" [0.22222222, 0.14285714, 0.13333333, 0.13333333],\n",
" [0.26666667, 0.10526316, 0.1 , 0.1 ]]),\n",
" array([[0. , 0. , 0. , 0. ],\n",
" [0.26666667, 0.10526316, 0.1 , 0.1 ]])]\n"
] ]
},
"metadata": {},
"output_type": "display_data"
} }
], ],
"source": [ "source": [
"def print_best_results(sorted_indices, n=10):\n", "print_best_results(threshold)"
" print([lookup[x] + \" -- \" + table.iloc[]])"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,