{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import clear_output, Markdown, Math\n",
    "import ipywidgets as widgets\n",
    "import os\n",
    "import unicodedata as uni\n",
    "import numpy as np\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.corpus import wordnet\n",
    "import math\n",
    "import pprint\n",
    "\n",
    "pp=pprint.PrettyPrinter(indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Approach"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* read in table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>code</th>\n",
       "      <th>character</th>\n",
       "      <th>description</th>\n",
       "      <th>description_de</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>126980</td>\n",
       "      <td>🀄</td>\n",
       "      <td>MAHJONG TILE RED DRAGON</td>\n",
       "      <td>MAHJONG FLIESE ROT DRACHE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>129525</td>\n",
       "      <td>🧵</td>\n",
       "      <td>SPOOL OF THREAD</td>\n",
       "      <td>Spool Gewinde</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>129526</td>\n",
       "      <td>🧶</td>\n",
       "      <td>BALL OF YARN</td>\n",
       "      <td>BALL OF YARN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>127183</td>\n",
       "      <td>🃏</td>\n",
       "      <td>PLAYING CARD BLACK JOKER</td>\n",
       "      <td>SPIELKARTE BLACK JOKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>129296</td>\n",
       "      <td>🤐</td>\n",
       "      <td>ZIPPER-MOUTH FACE</td>\n",
       "      <td>ZIPPER-MUND Gesicht</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0    code character               description  \\\n",
       "0           0  126980         🀄   MAHJONG TILE RED DRAGON   \n",
       "1           1  129525         🧵           SPOOL OF THREAD   \n",
       "2           2  129526         🧶              BALL OF YARN   \n",
       "3           3  127183         🃏  PLAYING CARD BLACK JOKER   \n",
       "4           4  129296         🤐         ZIPPER-MOUTH FACE   \n",
       "\n",
       "              description_de  \n",
       "0  MAHJONG FLIESE ROT DRACHE  \n",
       "1              Spool Gewinde  \n",
       "2               BALL OF YARN  \n",
       "3     SPIELKARTE BLACK JOKER  \n",
       "4        ZIPPER-MUND Gesicht  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pd.read_csv('../Tools/emoji_descriptions.csv')\n",
    "table.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* todo: read in a lot of messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* using a Stemmer to get the main 'Part' of each word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmed_messages = []\n",
    "for m in messages:\n",
    "    words = word_tokenize(m)\n",
    "    sm = []\n",
    "    for w in words:\n",
    "        sm.append(ps.stem(w))\n",
    "    stemmed_messages.append(sm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n",
       " ['thi',\n",
       "  'is',\n",
       "  'a',\n",
       "  'veri',\n",
       "  'sunni',\n",
       "  'day',\n",
       "  'today',\n",
       "  ',',\n",
       "  'i',\n",
       "  'am',\n",
       "  'veri',\n",
       "  'happi']]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stemmed_messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1027, 5)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* compare words to emoji descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_sentence(sentence, description_key = 'description', lang = 'eng'):\n",
    "    \n",
    "    tokenized_sentence = word_tokenize(sentence)\n",
    "    n = len(tokenized_sentence)\n",
    "    l = table.shape[0]\n",
    "    matrix_list = []\n",
    "    \n",
    "    for index, row in table.iterrows():\n",
    "        emoji_tokens = word_tokenize(row[description_key])\n",
    "        m = len(emoji_tokens)\n",
    "\n",
    "        mat = np.zeros(shape=(m,n))\n",
    "        for i in range(len(emoji_tokens)):\n",
    "            for j in range(len(tokenized_sentence)):\n",
    "                syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)\n",
    "                if len(syn1) == 0:\n",
    "                    continue\n",
    "                w1 = syn1[0]\n",
    "                #print(j, tokenized_sentence)\n",
    "                syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)\n",
    "                if len(syn2) == 0:\n",
    "                    continue\n",
    "                w2 = syn2[0]\n",
    "                val = w1.wup_similarity(w2)\n",
    "                if val is None:\n",
    "                    continue\n",
    "                mat[i,j] = val\n",
    "        #print(row['character'], mat)\n",
    "        matrix_list.append(mat)\n",
    "            \n",
    "    return matrix_list\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* building a lookup table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "lookup = {}\n",
    "emoji_set = []\n",
    "for index, row in table.iterrows():\n",
    "    lookup[index] = row['character']\n",
    "    emoji_set.append(row['character'])\n",
    "\n",
    "emoji_set = set(emoji_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* sorting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(sentence, description_key='description', lang = 'eng', n=10, t=0.9):\n",
    "\n",
    "    result = evaluate_sentence(sentence, description_key, lang)\n",
    "    \n",
    "    summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
    "    max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
    "    avg =  np.argsort([-np.mean(x) for x in result])[0:n]\n",
    "    threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n",
    "    \n",
    "    # build a result table\n",
    "    table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n",
    "                    lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n",
    "                    lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n",
    "                    lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n",
    "    \n",
    "    \n",
    "    table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n",
    "    \n",
    "    display(table_frame)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summed</th>\n",
       "      <th>summed_description</th>\n",
       "      <th>max_val</th>\n",
       "      <th>max_val_description</th>\n",
       "      <th>avg</th>\n",
       "      <th>avg_description</th>\n",
       "      <th>threshold</th>\n",
       "      <th>threshold_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>🤭</td>\n",
       "      <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
       "      <td>🤟</td>\n",
       "      <td>I LOVE YOU HAND SIGN</td>\n",
       "      <td>💐</td>\n",
       "      <td>BOUQUET</td>\n",
       "      <td>🚆</td>\n",
       "      <td>TRAIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>🔛</td>\n",
       "      <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
       "      <td>🇮</td>\n",
       "      <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
       "      <td>🍳</td>\n",
       "      <td>COOKING</td>\n",
       "      <td>🚸</td>\n",
       "      <td>CHILDREN CROSSING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>🚮</td>\n",
       "      <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
       "      <td>🚆</td>\n",
       "      <td>TRAIN</td>\n",
       "      <td>🚆</td>\n",
       "      <td>TRAIN</td>\n",
       "      <td>🚄</td>\n",
       "      <td>HIGH-SPEED TRAIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>🤪</td>\n",
       "      <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
       "      <td>🚅</td>\n",
       "      <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
       "      <td>🎥</td>\n",
       "      <td>MOVIE CAMERA</td>\n",
       "      <td>🇮</td>\n",
       "      <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>🥰</td>\n",
       "      <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
       "      <td>🚄</td>\n",
       "      <td>HIGH-SPEED TRAIN</td>\n",
       "      <td>🎭</td>\n",
       "      <td>PERFORMING ARTS</td>\n",
       "      <td>🤟</td>\n",
       "      <td>I LOVE YOU HAND SIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>🙌</td>\n",
       "      <td>PERSON RAISING BOTH HANDS IN CELEBRATION</td>\n",
       "      <td>🚸</td>\n",
       "      <td>CHILDREN CROSSING</td>\n",
       "      <td>🎹</td>\n",
       "      <td>MUSICAL KEYBOARD</td>\n",
       "      <td>🚅</td>\n",
       "      <td>HIGH-SPEED TRAIN WITH BULLET NOSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>🖖</td>\n",
       "      <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
       "      <td>🛸</td>\n",
       "      <td>FLYING SAUCER</td>\n",
       "      <td>🧾</td>\n",
       "      <td>RECEIPT</td>\n",
       "      <td>🔹</td>\n",
       "      <td>SMALL BLUE DIAMOND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>🔂</td>\n",
       "      <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
       "      <td>🥾</td>\n",
       "      <td>HIKING BOOT</td>\n",
       "      <td>💏</td>\n",
       "      <td>KISS</td>\n",
       "      <td>📠</td>\n",
       "      <td>FAX MACHINE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>😄</td>\n",
       "      <td>SMILING FACE WITH OPEN MOUTH AND SMILING EYES</td>\n",
       "      <td>🥏</td>\n",
       "      <td>FLYING DISC</td>\n",
       "      <td>🔥</td>\n",
       "      <td>FIRE</td>\n",
       "      <td>📡</td>\n",
       "      <td>SATELLITE ANTENNA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>👉</td>\n",
       "      <td>WHITE RIGHT POINTING BACKHAND INDEX</td>\n",
       "      <td>🚏</td>\n",
       "      <td>BUS STOP</td>\n",
       "      <td>🎼</td>\n",
       "      <td>MUSICAL SCORE</td>\n",
       "      <td>📢</td>\n",
       "      <td>PUBLIC ADDRESS LOUDSPEAKER</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  summed                                 summed_description max_val  \\\n",
       "0      🤭  SMILING FACE WITH SMILING EYES AND HAND COVERI...       🤟   \n",
       "1      🔛  ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...       🇮   \n",
       "2      🚮                     PUT LITTER IN ITS PLACE SYMBOL       🚆   \n",
       "3      🤪     GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE       🚅   \n",
       "4      🥰    SMILING FACE WITH SMILING EYES AND THREE HEARTS       🚄   \n",
       "5      🙌           PERSON RAISING BOTH HANDS IN CELEBRATION       🚸   \n",
       "6      🖖  RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...       🛸   \n",
       "7      🔂  CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...       🥾   \n",
       "8      😄      SMILING FACE WITH OPEN MOUTH AND SMILING EYES       🥏   \n",
       "9      👉                WHITE RIGHT POINTING BACKHAND INDEX       🚏   \n",
       "\n",
       "                  max_val_description avg   avg_description threshold  \\\n",
       "0                I LOVE YOU HAND SIGN   💐           BOUQUET         🚆   \n",
       "1  REGIONAL INDICATOR SYMBOL LETTER I   🍳           COOKING         🚸   \n",
       "2                               TRAIN   🚆             TRAIN         🚄   \n",
       "3   HIGH-SPEED TRAIN WITH BULLET NOSE   🎥      MOVIE CAMERA         🇮   \n",
       "4                    HIGH-SPEED TRAIN   🎭   PERFORMING ARTS         🤟   \n",
       "5                   CHILDREN CROSSING   🎹  MUSICAL KEYBOARD         🚅   \n",
       "6                       FLYING SAUCER   🧾           RECEIPT         🔹   \n",
       "7                         HIKING BOOT   💏              KISS         📠   \n",
       "8                         FLYING DISC   🔥              FIRE         📡   \n",
       "9                            BUS STOP   🎼     MUSICAL SCORE         📢   \n",
       "\n",
       "                threshold_description  \n",
       "0                               TRAIN  \n",
       "1                   CHILDREN CROSSING  \n",
       "2                    HIGH-SPEED TRAIN  \n",
       "3  REGIONAL INDICATOR SYMBOL LETTER I  \n",
       "4                I LOVE YOU HAND SIGN  \n",
       "5   HIGH-SPEED TRAIN WITH BULLET NOSE  \n",
       "6                  SMALL BLUE DIAMOND  \n",
       "7                         FAX MACHINE  \n",
       "8                   SATELLITE ANTENNA  \n",
       "9          PUBLIC ADDRESS LOUDSPEAKER  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predict(\"I like to travel by train\", description_key='description' , lang='eng')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
