{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from IPython.display import clear_output, Markdown, Math\n", "import ipywidgets as widgets\n", "import os\n", "import unicodedata as uni\n", "import numpy as np\n", "from nltk.stem import PorterStemmer\n", "from nltk.tokenize import sent_tokenize, word_tokenize\n", "from nltk.corpus import wordnet\n", "import math\n", "import pprint\n", "\n", "pp=pprint.PrettyPrinter(indent=4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Naive Approach" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* read in table" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0codecharacterdescriptiondescription_de
00126980๐Ÿ€„MAHJONG TILE RED DRAGONMAHJONG FLIESE ROT DRACHE
11129525๐ŸงตSPOOL OF THREADSpool Gewinde
22129526๐ŸงถBALL OF YARNBALL OF YARN
33127183๐ŸƒPLAYING CARD BLACK JOKERSPIELKARTE BLACK JOKER
44129296๐ŸคZIPPER-MOUTH FACEZIPPER-MUND Gesicht
\n", "
" ], "text/plain": [ " Unnamed: 0 code character description \\\n", "0 0 126980 ๐Ÿ€„ MAHJONG TILE RED DRAGON \n", "1 1 129525 ๐Ÿงต SPOOL OF THREAD \n", "2 2 129526 ๐Ÿงถ BALL OF YARN \n", "3 3 127183 ๐Ÿƒ PLAYING CARD BLACK JOKER \n", "4 4 129296 ๐Ÿค ZIPPER-MOUTH FACE \n", "\n", " description_de \n", "0 MAHJONG FLIESE ROT DRACHE \n", "1 Spool Gewinde \n", "2 BALL OF YARN \n", "3 SPIELKARTE BLACK JOKER \n", "4 ZIPPER-MUND Gesicht " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n", "table.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* todo: read in a lot of messages" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* using a Stemmer to get the main 'Part' of each word" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "ps = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "stemmed_messages = []\n", "for m in messages:\n", " words = word_tokenize(m)\n", " sm = []\n", " for w in words:\n", " sm.append(ps.stem(w))\n", " stemmed_messages.append(sm)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n", " ['thi',\n", " 'is',\n", " 'a',\n", " 'veri',\n", " 'sunni',\n", " 'day',\n", " 'today',\n", " ',',\n", " 'i',\n", " 'am',\n", " 'veri',\n", " 'happi']]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "stemmed_messages" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1027, 5)" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "table.shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* compare words to emoji descriptions" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "def evaluate_sentence(sentence, description_key = 'description', lang = 'eng'):\n", " \n", " tokenized_sentence = word_tokenize(sentence)\n", " n = len(tokenized_sentence)\n", " l = table.shape[0]\n", " matrix_list = []\n", " \n", " for index, row in table.iterrows():\n", " emoji_tokens = word_tokenize(row[description_key])\n", " m = len(emoji_tokens)\n", "\n", " mat = np.zeros(shape=(m,n))\n", " for i in range(len(emoji_tokens)):\n", " for j in range(len(tokenized_sentence)):\n", " syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)\n", " if len(syn1) == 0:\n", " continue\n", " w1 = syn1[0]\n", " #print(j, tokenized_sentence)\n", " syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)\n", " if len(syn2) == 0:\n", " continue\n", " w2 = syn2[0]\n", " val = w1.wup_similarity(w2)\n", " if val is None:\n", " continue\n", " mat[i,j] = val\n", " #print(row['character'], mat)\n", " matrix_list.append(mat)\n", " \n", " return matrix_list\n", " \n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* building a lookup table:" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "lookup = {}\n", "emoji_set = []\n", "for index, row in table.iterrows():\n", " lookup[index] = row['character']\n", " emoji_set.append(row['character'])\n", "\n", "emoji_set = set(emoji_set)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* sorting" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def predict(sentence, description_key='description', lang = 'eng', n=10, t=0.9):\n", "\n", " result = evaluate_sentence(sentence, description_key, lang)\n", " \n", " summed = np.argsort([-np.sum(x) for x in result])[0:n]\n", " max_val = np.argsort([-np.max(x) for x in result])[0:n]\n", " avg = np.argsort([-np.mean(x) for x in result])[0:n]\n", " threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n", " \n", " # build a result table\n", " table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n", " lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n", " lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n", " lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n", " \n", " \n", " table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n", " \n", " display(table_frame)\n" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
summedsummed_descriptionmax_valmax_val_descriptionavgavg_descriptionthresholdthreshold_description
0๐ŸคญSMILING FACE WITH SMILING EYES AND HAND COVERI...๐ŸคŸI LOVE YOU HAND SIGN๐Ÿ’BOUQUET๐Ÿš†TRAIN
1๐Ÿ”›ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I๐ŸณCOOKING๐ŸšธCHILDREN CROSSING
2๐ŸšฎPUT LITTER IN ITS PLACE SYMBOL๐Ÿš†TRAIN๐Ÿš†TRAIN๐Ÿš„HIGH-SPEED TRAIN
3๐ŸคชGRINNING FACE WITH ONE LARGE AND ONE SMALL EYE๐Ÿš…HIGH-SPEED TRAIN WITH BULLET NOSE๐ŸŽฅMOVIE CAMERA๐Ÿ‡ฎREGIONAL INDICATOR SYMBOL LETTER I
4๐ŸฅฐSMILING FACE WITH SMILING EYES AND THREE HEARTS๐Ÿš„HIGH-SPEED TRAIN๐ŸŽญPERFORMING ARTS๐ŸคŸI LOVE YOU HAND SIGN
5๐Ÿ™ŒPERSON RAISING BOTH HANDS IN CELEBRATION๐ŸšธCHILDREN CROSSING๐ŸŽนMUSICAL KEYBOARD๐Ÿš…HIGH-SPEED TRAIN WITH BULLET NOSE
6๐Ÿ––RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...๐Ÿ›ธFLYING SAUCER๐ŸงพRECEIPT๐Ÿ”นSMALL BLUE DIAMOND
7๐Ÿ”‚CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...๐ŸฅพHIKING BOOT๐Ÿ’KISS๐Ÿ“ FAX MACHINE
8๐Ÿ˜„SMILING FACE WITH OPEN MOUTH AND SMILING EYES๐ŸฅFLYING DISC๐Ÿ”ฅFIRE๐Ÿ“กSATELLITE ANTENNA
9๐Ÿ‘‰WHITE RIGHT POINTING BACKHAND INDEX๐ŸšBUS STOP๐ŸŽผMUSICAL SCORE๐Ÿ“ขPUBLIC ADDRESS LOUDSPEAKER
\n", "
" ], "text/plain": [ " summed summed_description max_val \\\n", "0 ๐Ÿคญ SMILING FACE WITH SMILING EYES AND HAND COVERI... ๐ŸคŸ \n", "1 ๐Ÿ”› ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... ๐Ÿ‡ฎ \n", "2 ๐Ÿšฎ PUT LITTER IN ITS PLACE SYMBOL ๐Ÿš† \n", "3 ๐Ÿคช GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE ๐Ÿš… \n", "4 ๐Ÿฅฐ SMILING FACE WITH SMILING EYES AND THREE HEARTS ๐Ÿš„ \n", "5 ๐Ÿ™Œ PERSON RAISING BOTH HANDS IN CELEBRATION ๐Ÿšธ \n", "6 ๐Ÿ–– RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... ๐Ÿ›ธ \n", "7 ๐Ÿ”‚ CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... ๐Ÿฅพ \n", "8 ๐Ÿ˜„ SMILING FACE WITH OPEN MOUTH AND SMILING EYES ๐Ÿฅ \n", "9 ๐Ÿ‘‰ WHITE RIGHT POINTING BACKHAND INDEX ๐Ÿš \n", "\n", " max_val_description avg avg_description threshold \\\n", "0 I LOVE YOU HAND SIGN ๐Ÿ’ BOUQUET ๐Ÿš† \n", "1 REGIONAL INDICATOR SYMBOL LETTER I ๐Ÿณ COOKING ๐Ÿšธ \n", "2 TRAIN ๐Ÿš† TRAIN ๐Ÿš„ \n", "3 HIGH-SPEED TRAIN WITH BULLET NOSE ๐ŸŽฅ MOVIE CAMERA ๐Ÿ‡ฎ \n", "4 HIGH-SPEED TRAIN ๐ŸŽญ PERFORMING ARTS ๐ŸคŸ \n", "5 CHILDREN CROSSING ๐ŸŽน MUSICAL KEYBOARD ๐Ÿš… \n", "6 FLYING SAUCER ๐Ÿงพ RECEIPT ๐Ÿ”น \n", "7 HIKING BOOT ๐Ÿ’ KISS ๐Ÿ“  \n", "8 FLYING DISC ๐Ÿ”ฅ FIRE ๐Ÿ“ก \n", "9 BUS STOP ๐ŸŽผ MUSICAL SCORE ๐Ÿ“ข \n", "\n", " threshold_description \n", "0 TRAIN \n", "1 CHILDREN CROSSING \n", "2 HIGH-SPEED TRAIN \n", "3 REGIONAL INDICATOR SYMBOL LETTER I \n", "4 I LOVE YOU HAND SIGN \n", "5 HIGH-SPEED TRAIN WITH BULLET NOSE \n", "6 SMALL BLUE DIAMOND \n", "7 FAX MACHINE \n", "8 SATELLITE ANTENNA \n", "9 PUBLIC ADDRESS LOUDSPEAKER " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "predict(\"I like to travel by train\", description_key='description' , lang='eng')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }