From c4a67c8e849205d7a80689b8b7e8f11d5b8c00ed Mon Sep 17 00:00:00 2001 From: Jonas Weinz Date: Thu, 31 May 2018 15:06:33 +0200 Subject: [PATCH] naive approach --- Project/naive_approach/naive_approach.ipynb | 371 ++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 Project/naive_approach/naive_approach.ipynb diff --git a/Project/naive_approach/naive_approach.ipynb b/Project/naive_approach/naive_approach.ipynb new file mode 100644 index 0000000..c3b8249 --- /dev/null +++ b/Project/naive_approach/naive_approach.ipynb @@ -0,0 +1,371 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from IPython.display import clear_output, Markdown, Math\n", + "import ipywidgets as widgets\n", + "import os\n", + "import unicodedata as uni\n", + "import numpy as np\n", + "from nltk.stem import PorterStemmer\n", + "from nltk.tokenize import sent_tokenize, word_tokenize\n", + "from nltk.corpus import wordnet\n", + "import math" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Naive Approach" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* read in table" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0codecharacterdescriptionUnnamed: 4
00126980πŸ€„MAHJONG TILE RED DRAGONNaN
11129525🧡SPOOL OF THREADNaN
22129526🧢BALL OF YARNNaN
33127183πŸƒPLAYING CARD BLACK JOKERNaN
44129296🀐ZIPPER-MOUTH FACENaN
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 code character description Unnamed: 4\n", + "0 0 126980 πŸ€„ MAHJONG TILE RED DRAGON NaN\n", + "1 1 129525 🧡 SPOOL OF THREAD NaN\n", + "2 2 129526 🧢 BALL OF YARN NaN\n", + "3 3 127183 πŸƒ PLAYING CARD BLACK JOKER NaN\n", + "4 4 129296 🀐 ZIPPER-MOUTH FACE NaN" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n", + "table.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* todo: read in a lot of messages" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ps = PorterStemmer()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "stemmed_messages = []\n", + "for m in messages:\n", + " words = word_tokenize(m)\n", + " sm = []\n", + " for w in words:\n", + " sm.append(ps.stem(w))\n", + " stemmed_messages.append(sm)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n", + " ['thi',\n", + " 'is',\n", + " 'a',\n", + " 'veri',\n", + " 'sunni',\n", + " 'day',\n", + " 'today',\n", + " ',',\n", + " 'i',\n", + " 'am',\n", + " 'veri',\n", + " 'happi']]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stemmed_messages" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1027, 5)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* compare words to emoji descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_sentence(sentence):\n", + " tokenized_sentence = word_tokenize(sentence)\n", + " n = len(tokenized_sentence)\n", + " l = table.shape[0]\n", + " matrix_list = []\n", + " \n", + " for index, row in table.iterrows():\n", + " emoji_tokens = word_tokenize(row['description'])\n", + " m = len(emoji_tokens)\n", + "\n", + " mat = np.zeros(shape=(m,n))\n", + " for i in range(len(emoji_tokens)):\n", + " for j in range(len(tokenized_sentence)):\n", + " syn1 = wordnet.synsets(emoji_tokens[i])\n", + " if len(syn1) == 0:\n", + " continue\n", + " w1 = syn1[0]\n", + " #print(j, tokenized_sentence)\n", + " syn2 = wordnet.synsets(tokenized_sentence[j])\n", + " if len(syn2) == 0:\n", + " continue\n", + " w2 = syn2[0]\n", + " val = w1.wup_similarity(w2)\n", + " if val is None:\n", + " continue\n", + " mat[i,j] = val\n", + " #print(row['character'], mat)\n", + " matrix_list.append(mat)\n", + " \n", + " return matrix_list\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [], + "source": [ + "result = evaluate_sentence(\"car soccer surf\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* building a lookup table:" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [], + "source": [ + "lookup = {}\n", + "for index, row in table.iterrows():\n", + " lookup[index] = row['character']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* sorting" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [], + "source": [ + "summed = np.argsort([-np.sum(x) for x in result])\n", + "max_val = np.argsort([-np.max(x) for x in result])\n", + "avg = np.argsort([-np.mean(x) for x in result])\n", + "\n", + "t = 0.7\n", + "threshold = np.argsort([-len(np.where(x>t)[0]) for x in result])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "# πŸ‰βšΎπŸŽ³πŸ”₯πŸπŸŽ±πŸ’πŸ§ΎπŸš—πŸš˜" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_best_results(sorted_indices, n=10):\n", + " print([lookup[x] + \" -- \" + table.iloc[]])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}