{ "cells": [ { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import unicodedata as uni\n", "import numpy as np\n", "from nltk.stem import PorterStemmer\n", "from nltk.tokenize import sent_tokenize, word_tokenize\n", "from nltk.corpus import wordnet\n", "import math" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Naive Approach" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read in table" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptionUnnamed: 4
00126980πŸ€„MAHJONG TILE RED DRAGONNaN
11129525🧡SPOOL OF THREADNaN
22129526🧢BALL OF YARNNaN
33127183πŸƒPLAYING CARD BLACK JOKERNaN
44129296🀐ZIPPER-MOUTH FACENaN
\n", "
" ], "text/plain": [ " Unnamed: 0 code character description Unnamed: 4\n", "0 0 126980 πŸ€„ MAHJONG TILE RED DRAGON NaN\n", "1 1 129525 🧡 SPOOL OF THREAD NaN\n", "2 2 129526 🧢 BALL OF YARN NaN\n", "3 3 127183 πŸƒ PLAYING CARD BLACK JOKER NaN\n", "4 4 129296 🀐 ZIPPER-MOUTH FACE NaN" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n", "table.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* todo: read in a lot of messages" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "ps = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "stemmed_messages = []\n", "for m in messages:\n", " words = word_tokenize(m)\n", " sm = []\n", " for w in words:\n", " sm.append(ps.stem(w))\n", " stemmed_messages.append(sm)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n", " ['thi',\n", " 'is',\n", " 'a',\n", " 'veri',\n", " 'sunni',\n", " 'day',\n", " 'today',\n", " ',',\n", " 'i',\n", " 'am',\n", " 'veri',\n", " 'happi']]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "stemmed_messages" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1027, 5)" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table.shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* compare words to emoji descriptions" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "def evaluate_sentence(sentence):\n", " tokenized_sentence = word_tokenize(sentence)\n", " n = len(tokenized_sentence)\n", " l = table.shape[0]\n", " matrix_list = []\n", " \n", " for index, row in table.iterrows():\n", " emoji_tokens = word_tokenize(row['description'])\n", " m = len(emoji_tokens)\n", "\n", " mat = np.zeros(shape=(m,n))\n", " for i in range(len(emoji_tokens)):\n", " for j in range(len(tokenized_sentence)):\n", " syn1 = wordnet.synsets(emoji_tokens[i])\n", " if len(syn1) == 0:\n", " continue\n", " w1 = syn1[0]\n", " #print(j, tokenized_sentence)\n", " syn2 = wordnet.synsets(tokenized_sentence[j])\n", " if len(syn2) == 0:\n", " continue\n", " w2 = syn2[0]\n", " val = w1.wup_similarity(w2)\n", " if val is None:\n", " continue\n", " mat[i,j] = val\n", " #print(row['character'], mat)\n", " matrix_list.append(mat)\n", " \n", " return matrix_list\n", " \n", " " ] }, { "cell_type": "code", "execution_count": 106, "metadata": {}, "outputs": [], "source": [ "result = evaluate_sentence(\"car soccer surf\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* building a lookup table:" ] }, { "cell_type": "code", "execution_count": 107, "metadata": {}, "outputs": [], "source": [ "lookup = {}\n", "for index, row in table.iterrows():\n", " lookup[index] = row['character']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* sorting" ] }, { "cell_type": "code", "execution_count": 108, "metadata": {}, "outputs": [], "source": [ "summed = np.argsort([-np.sum(x) for x in result])\n", "max_val = np.argsort([-np.max(x) for x in result])\n", "avg = np.argsort([-np.mean(x) for x in result])\n", "\n", "t = 0.7\n", "threshold = np.argsort([-len(np.where(x>t)[0]) for x in result])\n" ] }, { "cell_type": "code", "execution_count": 109, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "# πŸ‰βšΎπŸŽ³πŸ”₯πŸπŸŽ±πŸ’πŸ§ΎπŸš—πŸš˜" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def print_best_results(sorted_indices, n=10):\n", " print([lookup[x] + \" -- \" + table.iloc[]])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }