From 12f99af9a964952ca57d16ce2d1d6f1688fbfd5d Mon Sep 17 00:00:00 2001 From: Jonas Weinz Date: Tue, 5 Jun 2018 10:02:35 +0200 Subject: [PATCH] better naive approach --- Project/naive_approach/naive_approach.ipynb | 319 +++++++++++++++----- 1 file changed, 237 insertions(+), 82 deletions(-) diff --git a/Project/naive_approach/naive_approach.ipynb b/Project/naive_approach/naive_approach.ipynb index b365d5c..0e4205e 100644 --- a/Project/naive_approach/naive_approach.ipynb +++ b/Project/naive_approach/naive_approach.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 149, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -65,7 +65,7 @@ " code\n", " character\n", " description\n", - " Unnamed: 4\n", + " description_de\n", " \n", " \n", " \n", @@ -75,7 +75,7 @@ " 126980\n", " ๐Ÿ€„\n", " MAHJONG TILE RED DRAGON\n", - " NaN\n", + " MAHJONG FLIESE ROT DRACHE\n", " \n", " \n", " 1\n", @@ -83,7 +83,7 @@ " 129525\n", " ๐Ÿงต\n", " SPOOL OF THREAD\n", - " NaN\n", + " Spool Gewinde\n", " \n", " \n", " 2\n", @@ -91,7 +91,7 @@ " 129526\n", " ๐Ÿงถ\n", " BALL OF YARN\n", - " NaN\n", + " BALL OF YARN\n", " \n", " \n", " 3\n", @@ -99,7 +99,7 @@ " 127183\n", " ๐Ÿƒ\n", " PLAYING CARD BLACK JOKER\n", - " NaN\n", + " SPIELKARTE BLACK JOKER\n", " \n", " \n", " 4\n", @@ -107,19 +107,26 @@ " 129296\n", " ๐Ÿค\n", " ZIPPER-MOUTH FACE\n", - " NaN\n", + " ZIPPER-MUND Gesicht\n", " \n", " \n", "\n", "" ], "text/plain": [ - " Unnamed: 0 code character description Unnamed: 4\n", - "0 0 126980 ๐Ÿ€„ MAHJONG TILE RED DRAGON NaN\n", - "1 1 129525 ๐Ÿงต SPOOL OF THREAD NaN\n", - "2 2 129526 ๐Ÿงถ BALL OF YARN NaN\n", - "3 3 127183 ๐Ÿƒ PLAYING CARD BLACK JOKER NaN\n", - "4 4 129296 ๐Ÿค ZIPPER-MOUTH FACE NaN" + " Unnamed: 0 code character description \\\n", + "0 0 126980 ๐Ÿ€„ MAHJONG TILE RED DRAGON \n", + "1 1 129525 ๐Ÿงต SPOOL OF THREAD \n", + "2 2 129526 ๐Ÿงถ BALL OF YARN \n", + "3 3 127183 ๐Ÿƒ PLAYING CARD BLACK JOKER \n", + "4 4 129296 ๐Ÿค ZIPPER-MOUTH FACE \n", + "\n", + " description_de \n", + "0 MAHJONG FLIESE ROT DRACHE \n", + "1 Spool Gewinde \n", + "2 BALL OF YARN \n", + "3 SPIELKARTE BLACK JOKER \n", + "4 ZIPPER-MUND Gesicht " ] }, "execution_count": 2, @@ -148,6 +155,13 @@ "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* using a Stemmer to get the main 'Part' of each word" + ] + }, { "cell_type": "code", "execution_count": 4, @@ -233,18 +247,18 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "def evaluate_sentence(sentence):\n", + "def evaluate_sentence(sentence, description_key = 'description'):\n", " tokenized_sentence = word_tokenize(sentence)\n", " n = len(tokenized_sentence)\n", " l = table.shape[0]\n", " matrix_list = []\n", " \n", " for index, row in table.iterrows():\n", - " emoji_tokens = word_tokenize(row['description'])\n", + " emoji_tokens = word_tokenize(row[description_key])\n", " m = len(emoji_tokens)\n", "\n", " mat = np.zeros(shape=(m,n))\n", @@ -271,15 +285,6 @@ " " ] }, - { - "cell_type": "code", - "execution_count": 177, - "metadata": {}, - "outputs": [], - "source": [ - "result = evaluate_sentence(\"horse back riding\")" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -289,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -311,76 +316,226 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "summed = np.argsort([-np.sum(x) for x in result])\n", - "max_val = np.argsort([-np.max(x) for x in result])\n", - "avg = np.argsort([-np.mean(x) for x in result])\n", + "def predict(sentence, description_key='description', n=10, t=0.9):\n", "\n", - "t = 0.9\n", - "threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])\n" + " result = evaluate_sentence(sentence, description_key)\n", + " \n", + " summed = np.argsort([-np.sum(x) for x in result])[0:n]\n", + " max_val = np.argsort([-np.max(x) for x in result])[0:n]\n", + " avg = np.argsort([-np.mean(x) for x in result])[0:n]\n", + " threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n", + " \n", + " # build a result table\n", + " table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n", + " lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n", + " lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n", + " lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n", + " \n", + " \n", + " table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n", + " \n", + " display(table_frame)\n" ] }, { "cell_type": "code", - "execution_count": 180, - "metadata": {}, - "outputs": [], - "source": [ - "def print_best_results(sorted_indices, n=10):\n", - " pp.pprint([lookup[x] + \" -- \" + str(table.iloc[x]['description']) for x in sorted_indices[:10]])\n", - " pp.pprint([result[x] for x in sorted_indices[:10]])" - ] - }, - { - "cell_type": "code", - "execution_count": 181, + "execution_count": 11, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "[ '๐ŸŽ -- HORSE',\n", - " '๐Ÿ‡ -- HORSE RACING',\n", - " '๐Ÿด -- HORSE FACE',\n", - " '\\U0001f993 -- ZEBRA FACE',\n", - " '๐Ÿ… -- SPORTS MEDAL',\n", - " '๐Ÿ—ป -- MOUNT FUJI',\n", - " '๐ŸŽ  -- CAROUSEL HORSE',\n", - " '๐Ÿคš -- RAISED BACK OF HAND',\n", - " '๐Ÿ”™ -- BACK WITH LEFTWARDS ARROW ABOVE',\n", - " '๐Ÿ“ฎ -- POSTBOX']\n", - "[ array([[1. , 0.19047619, 0.08333333]]),\n", - " array([[1. , 0.19047619, 0.08333333],\n", - " [0.08333333, 0.13333333, 0.88888889]]),\n", - " array([[1. , 0.19047619, 0.08333333],\n", - " [0.18181818, 0.76923077, 0.125 ]]),\n", - " array([[0.93333333, 0.19047619, 0.08333333],\n", - " [0.18181818, 0.76923077, 0.125 ]]),\n", - " array([[0.08695652, 0.14285714, 0.94117647],\n", - " [0.09090909, 0.15384615, 0.25 ]]),\n", - " array([[0.96774194, 0.18181818, 0.08 ],\n", - " [0.4 , 0.19047619, 0.08333333]]),\n", - " array([[0.07407407, 0.11111111, 0.19047619],\n", - " [1. , 0.19047619, 0.08333333]]),\n", - " array([[0.1 , 0.18181818, 0.14285714],\n", - " [0.19047619, 1. , 0.13333333],\n", - " [0. , 0. , 0. ],\n", - " [0.17391304, 0.71428571, 0.11764706]]),\n", - " array([[0.19047619, 1. , 0.13333333],\n", - " [0. , 0. , 0. ],\n", - " [0. , 0. , 0. ],\n", - " [0.08695652, 0.14285714, 0.23529412],\n", - " [0.09090909, 0.15384615, 0.25 ]]),\n", - " array([[0.32 , 0.25 , 0.10526316]])]\n" - ] + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
summedsummed_descriptionmax_valmax_val_descriptionavgavg_descriptionthresholdthreshold_description
0๐ŸคชGRINNING FACE WITH ONE LARGE AND ONE SMALL EYE๐ŸคŸI LOVE YOU HAND SIGN๐Ÿ’ขANGER SYMBOL๐Ÿ’ŒLOVE LETTER
1๐ŸšฎPUT LITTER IN ITS PLACE SYMBOL๐Ÿ’ŒLOVE LETTER๐Ÿ“ฆPACKAGE๐ŸฉLOVE HOTEL
2๐Ÿ––RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I๐Ÿ’ŒLOVE LETTER๐ŸคŸI LOVE YOU HAND SIGN
3๐ŸฅฐSMILING FACE WITH SMILING EYES AND THREE HEARTS๐ŸฉLOVE HOTEL๐Ÿ†‘SQUARED CL๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I
4๐ŸคญSMILING FACE WITH SMILING EYES AND HAND COVERI...๐Ÿ˜ฑFACE SCREAMING IN FEAR๐ŸŒ‰BRIDGE AT NIGHT๐Ÿ“จINCOMING ENVELOPE
5๐Ÿ”‚CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...๐Ÿ‡ญREGIONAL INDICATOR SYMBOL LETTER H๐Ÿ’BOUQUET๐Ÿ“ฉENVELOPE WITH DOWNWARDS ARROW ABOVE
6๐Ÿ”›ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...๐Ÿ‡ณREGIONAL INDICATOR SYMBOL LETTER N๐Ÿ“œSCROLL๐Ÿ“ชCLOSED MAILBOX WITH LOWERED FLAG
7๐Ÿ” INPUT SYMBOL FOR LATIN CAPITAL LETTERS๐Ÿ‡ดREGIONAL INDICATOR SYMBOL LETTER O๐ŸŽWRAPPED PRESENT๐Ÿ“ซCLOSED MAILBOX WITH RAISED FLAG
8๐ŸคŸI LOVE YOU HAND SIGN๐Ÿ“ฒMOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT๐Ÿ’จDASH SYMBOL๐Ÿ“ฐNEWSPAPER
9๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I๐ŸŒ†CITYSCAPE AT DUSK๐Ÿ’คSLEEPING SYMBOL๐Ÿ“ฌOPEN MAILBOX WITH RAISED FLAG
\n", + "
" + ], + "text/plain": [ + " summed summed_description max_val \\\n", + "0 ๐Ÿคช GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE ๐ŸคŸ \n", + "1 ๐Ÿšฎ PUT LITTER IN ITS PLACE SYMBOL ๐Ÿ’Œ \n", + "2 ๐Ÿ–– RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... ๐Ÿ‡ฎ \n", + "3 ๐Ÿฅฐ SMILING FACE WITH SMILING EYES AND THREE HEARTS ๐Ÿฉ \n", + "4 ๐Ÿคญ SMILING FACE WITH SMILING EYES AND HAND COVERI... ๐Ÿ˜ฑ \n", + "5 ๐Ÿ”‚ CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... ๐Ÿ‡ญ \n", + "6 ๐Ÿ”› ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... ๐Ÿ‡ณ \n", + "7 ๐Ÿ”  INPUT SYMBOL FOR LATIN CAPITAL LETTERS ๐Ÿ‡ด \n", + "8 ๐ŸคŸ I LOVE YOU HAND SIGN ๐Ÿ“ฒ \n", + "9 ๐Ÿ‡ฎ REGIONAL INDICATOR SYMBOL LETTER I ๐ŸŒ† \n", + "\n", + " max_val_description avg avg_description threshold \\\n", + "0 I LOVE YOU HAND SIGN ๐Ÿ’ข ANGER SYMBOL ๐Ÿ’Œ \n", + "1 LOVE LETTER ๐Ÿ“ฆ PACKAGE ๐Ÿฉ \n", + "2 REGIONAL INDICATOR SYMBOL LETTER I ๐Ÿ’Œ LOVE LETTER ๐ŸคŸ \n", + "3 LOVE HOTEL ๐Ÿ†‘ SQUARED CL ๐Ÿ‡ฎ \n", + "4 FACE SCREAMING IN FEAR ๐ŸŒ‰ BRIDGE AT NIGHT ๐Ÿ“จ \n", + "5 REGIONAL INDICATOR SYMBOL LETTER H ๐Ÿ’ BOUQUET ๐Ÿ“ฉ \n", + "6 REGIONAL INDICATOR SYMBOL LETTER N ๐Ÿ“œ SCROLL ๐Ÿ“ช \n", + "7 REGIONAL INDICATOR SYMBOL LETTER O ๐ŸŽ WRAPPED PRESENT ๐Ÿ“ซ \n", + "8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT ๐Ÿ’จ DASH SYMBOL ๐Ÿ“ฐ \n", + "9 CITYSCAPE AT DUSK ๐Ÿ’ค SLEEPING SYMBOL ๐Ÿ“ฌ \n", + "\n", + " threshold_description \n", + "0 LOVE LETTER \n", + "1 LOVE HOTEL \n", + "2 I LOVE YOU HAND SIGN \n", + "3 REGIONAL INDICATOR SYMBOL LETTER I \n", + "4 INCOMING ENVELOPE \n", + "5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n", + "6 CLOSED MAILBOX WITH LOWERED FLAG \n", + "7 CLOSED MAILBOX WITH RAISED FLAG \n", + "8 NEWSPAPER \n", + "9 OPEN MAILBOX WITH RAISED FLAG " + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "print_best_results(threshold)" + "predict(\"I love sunny days!\", description_key='description' )" ] }, {