nlp-lab/Project/naive_approach/naive_approach.ipynb

571 lines
18 KiB
Plaintext
Raw Normal View History

2018-05-31 15:06:33 +02:00
{
"cells": [
{
"cell_type": "code",
2018-06-05 10:02:35 +02:00
"execution_count": 1,
2018-05-31 15:06:33 +02:00
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os\n",
"import unicodedata as uni\n",
"import numpy as np\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.tokenize import sent_tokenize, word_tokenize\n",
"from nltk.corpus import wordnet\n",
2018-05-31 15:23:57 +02:00
"import math\n",
"import pprint\n",
"\n",
"pp=pprint.PrettyPrinter(indent=4)"
2018-05-31 15:06:33 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Naive Approach"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read in table"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Unnamed: 0</th>\n",
" <th>code</th>\n",
" <th>character</th>\n",
" <th>description</th>\n",
2018-06-05 10:02:35 +02:00
" <th>description_de</th>\n",
2018-05-31 15:06:33 +02:00
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>126980</td>\n",
" <td>🀄</td>\n",
" <td>MAHJONG TILE RED DRAGON</td>\n",
2018-06-05 10:02:35 +02:00
" <td>MAHJONG FLIESE ROT DRACHE</td>\n",
2018-05-31 15:06:33 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>129525</td>\n",
" <td>🧵</td>\n",
" <td>SPOOL OF THREAD</td>\n",
2018-06-05 10:02:35 +02:00
" <td>Spool Gewinde</td>\n",
2018-05-31 15:06:33 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>129526</td>\n",
" <td>🧶</td>\n",
" <td>BALL OF YARN</td>\n",
2018-06-05 10:02:35 +02:00
" <td>BALL OF YARN</td>\n",
2018-05-31 15:06:33 +02:00
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>127183</td>\n",
" <td>🃏</td>\n",
" <td>PLAYING CARD BLACK JOKER</td>\n",
2018-06-05 10:02:35 +02:00
" <td>SPIELKARTE BLACK JOKER</td>\n",
2018-05-31 15:06:33 +02:00
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>129296</td>\n",
" <td>🤐</td>\n",
" <td>ZIPPER-MOUTH FACE</td>\n",
2018-06-05 10:02:35 +02:00
" <td>ZIPPER-MUND Gesicht</td>\n",
2018-05-31 15:06:33 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
2018-06-05 10:02:35 +02:00
" Unnamed: 0 code character description \\\n",
"0 0 126980 🀄 MAHJONG TILE RED DRAGON \n",
"1 1 129525 🧵 SPOOL OF THREAD \n",
"2 2 129526 🧶 BALL OF YARN \n",
"3 3 127183 🃏 PLAYING CARD BLACK JOKER \n",
"4 4 129296 🤐 ZIPPER-MOUTH FACE \n",
"\n",
" description_de \n",
"0 MAHJONG FLIESE ROT DRACHE \n",
"1 Spool Gewinde \n",
"2 BALL OF YARN \n",
"3 SPIELKARTE BLACK JOKER \n",
"4 ZIPPER-MUND Gesicht "
2018-05-31 15:06:33 +02:00
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"table = pd.read_csv('../Tools/emoji_descriptions.csv')\n",
"table.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* todo: read in a lot of messages"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
]
},
2018-06-05 10:02:35 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* using a Stemmer to get the main 'Part' of each word"
]
},
2018-05-31 15:06:33 +02:00
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"ps = PorterStemmer()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"stemmed_messages = []\n",
"for m in messages:\n",
" words = word_tokenize(m)\n",
" sm = []\n",
" for w in words:\n",
" sm.append(ps.stem(w))\n",
" stemmed_messages.append(sm)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n",
" ['thi',\n",
" 'is',\n",
" 'a',\n",
" 'veri',\n",
" 'sunni',\n",
" 'day',\n",
" 'today',\n",
" ',',\n",
" 'i',\n",
" 'am',\n",
" 'veri',\n",
" 'happi']]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"stemmed_messages"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1027, 5)"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"table.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* compare words to emoji descriptions"
]
},
{
"cell_type": "code",
2018-06-05 10:02:35 +02:00
"execution_count": 8,
2018-05-31 15:06:33 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-05 10:02:35 +02:00
"def evaluate_sentence(sentence, description_key = 'description'):\n",
2018-05-31 15:06:33 +02:00
" tokenized_sentence = word_tokenize(sentence)\n",
" n = len(tokenized_sentence)\n",
" l = table.shape[0]\n",
" matrix_list = []\n",
" \n",
" for index, row in table.iterrows():\n",
2018-06-05 10:02:35 +02:00
" emoji_tokens = word_tokenize(row[description_key])\n",
2018-05-31 15:06:33 +02:00
" m = len(emoji_tokens)\n",
"\n",
" mat = np.zeros(shape=(m,n))\n",
" for i in range(len(emoji_tokens)):\n",
" for j in range(len(tokenized_sentence)):\n",
" syn1 = wordnet.synsets(emoji_tokens[i])\n",
" if len(syn1) == 0:\n",
" continue\n",
" w1 = syn1[0]\n",
" #print(j, tokenized_sentence)\n",
" syn2 = wordnet.synsets(tokenized_sentence[j])\n",
" if len(syn2) == 0:\n",
" continue\n",
" w2 = syn2[0]\n",
" val = w1.wup_similarity(w2)\n",
" if val is None:\n",
" continue\n",
" mat[i,j] = val\n",
" #print(row['character'], mat)\n",
" matrix_list.append(mat)\n",
" \n",
" return matrix_list\n",
" \n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* building a lookup table:"
]
},
{
"cell_type": "code",
2018-06-05 10:02:35 +02:00
"execution_count": 9,
2018-05-31 15:06:33 +02:00
"metadata": {},
"outputs": [],
"source": [
"lookup = {}\n",
2018-06-04 20:37:18 +02:00
"emoji_set = []\n",
2018-05-31 15:06:33 +02:00
"for index, row in table.iterrows():\n",
2018-06-04 20:37:18 +02:00
" lookup[index] = row['character']\n",
" emoji_set.append(row['character'])\n",
"\n",
"emoji_set = set(emoji_set)"
2018-05-31 15:06:33 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* sorting"
]
},
{
"cell_type": "code",
2018-06-05 10:02:35 +02:00
"execution_count": 10,
2018-05-31 15:06:33 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-06-05 10:02:35 +02:00
"def predict(sentence, description_key='description', n=10, t=0.9):\n",
2018-05-31 15:06:33 +02:00
"\n",
2018-06-05 10:02:35 +02:00
" result = evaluate_sentence(sentence, description_key)\n",
" \n",
" summed = np.argsort([-np.sum(x) for x in result])[0:n]\n",
" max_val = np.argsort([-np.max(x) for x in result])[0:n]\n",
" avg = np.argsort([-np.mean(x) for x in result])[0:n]\n",
" threshold = np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]\n",
" \n",
" # build a result table\n",
" table_array = [[lookup[summed[i]], str(table.iloc[summed[i]][description_key]), \n",
" lookup[max_val[i]], str(table.iloc[max_val[i]][description_key]),\n",
" lookup[avg[i]], str(table.iloc[avg[i]][description_key]),\n",
" lookup[threshold[i]], str(table.iloc[threshold[i]][description_key])] for i in range(n) ]\n",
" \n",
" \n",
" table_frame = pd.DataFrame(table_array, columns=['summed', 'summed_description','max_val', 'max_val_description','avg', 'avg_description','threshold', 'threshold_description'])\n",
" \n",
" display(table_frame)\n"
2018-05-31 15:06:33 +02:00
]
},
{
"cell_type": "code",
2018-06-05 10:02:35 +02:00
"execution_count": 11,
2018-05-31 15:06:33 +02:00
"metadata": {},
"outputs": [
{
2018-06-05 10:02:35 +02:00
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>summed</th>\n",
" <th>summed_description</th>\n",
" <th>max_val</th>\n",
" <th>max_val_description</th>\n",
" <th>avg</th>\n",
" <th>avg_description</th>\n",
" <th>threshold</th>\n",
" <th>threshold_description</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>🤪</td>\n",
" <td>GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE</td>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" <td>💢</td>\n",
" <td>ANGER SYMBOL</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>🚮</td>\n",
" <td>PUT LITTER IN ITS PLACE SYMBOL</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>📦</td>\n",
" <td>PACKAGE</td>\n",
" <td>🏩</td>\n",
" <td>LOVE HOTEL</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>🖖</td>\n",
" <td>RAISED HAND WITH PART BETWEEN MIDDLE AND RING ...</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>💌</td>\n",
" <td>LOVE LETTER</td>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>🥰</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND THREE HEARTS</td>\n",
" <td>🏩</td>\n",
" <td>LOVE HOTEL</td>\n",
" <td>🆑</td>\n",
" <td>SQUARED CL</td>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>🤭</td>\n",
" <td>SMILING FACE WITH SMILING EYES AND HAND COVERI...</td>\n",
" <td>😱</td>\n",
" <td>FACE SCREAMING IN FEAR</td>\n",
" <td>🌉</td>\n",
" <td>BRIDGE AT NIGHT</td>\n",
" <td>📨</td>\n",
" <td>INCOMING ENVELOPE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>🔂</td>\n",
" <td>CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE...</td>\n",
" <td>🇭</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER H</td>\n",
" <td>💐</td>\n",
" <td>BOUQUET</td>\n",
" <td>📩</td>\n",
" <td>ENVELOPE WITH DOWNWARDS ARROW ABOVE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>🔛</td>\n",
" <td>ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW...</td>\n",
" <td>🇳</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER N</td>\n",
" <td>📜</td>\n",
" <td>SCROLL</td>\n",
" <td>📪</td>\n",
" <td>CLOSED MAILBOX WITH LOWERED FLAG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>🔠</td>\n",
" <td>INPUT SYMBOL FOR LATIN CAPITAL LETTERS</td>\n",
" <td>🇴</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER O</td>\n",
" <td>🎁</td>\n",
" <td>WRAPPED PRESENT</td>\n",
" <td>📫</td>\n",
" <td>CLOSED MAILBOX WITH RAISED FLAG</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>🤟</td>\n",
" <td>I LOVE YOU HAND SIGN</td>\n",
" <td>📲</td>\n",
" <td>MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT</td>\n",
" <td>💨</td>\n",
" <td>DASH SYMBOL</td>\n",
" <td>📰</td>\n",
" <td>NEWSPAPER</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>🇮</td>\n",
" <td>REGIONAL INDICATOR SYMBOL LETTER I</td>\n",
" <td>🌆</td>\n",
" <td>CITYSCAPE AT DUSK</td>\n",
" <td>💤</td>\n",
" <td>SLEEPING SYMBOL</td>\n",
" <td>📬</td>\n",
" <td>OPEN MAILBOX WITH RAISED FLAG</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" summed summed_description max_val \\\n",
"0 🤪 GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE 🤟 \n",
"1 🚮 PUT LITTER IN ITS PLACE SYMBOL 💌 \n",
"2 🖖 RAISED HAND WITH PART BETWEEN MIDDLE AND RING ... 🇮 \n",
"3 🥰 SMILING FACE WITH SMILING EYES AND THREE HEARTS 🏩 \n",
"4 🤭 SMILING FACE WITH SMILING EYES AND HAND COVERI... 😱 \n",
"5 🔂 CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE... 🇭 \n",
"6 🔛 ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW... 🇳 \n",
"7 🔠 INPUT SYMBOL FOR LATIN CAPITAL LETTERS 🇴 \n",
"8 🤟 I LOVE YOU HAND SIGN 📲 \n",
"9 🇮 REGIONAL INDICATOR SYMBOL LETTER I 🌆 \n",
"\n",
" max_val_description avg avg_description threshold \\\n",
"0 I LOVE YOU HAND SIGN 💢 ANGER SYMBOL 💌 \n",
"1 LOVE LETTER 📦 PACKAGE 🏩 \n",
"2 REGIONAL INDICATOR SYMBOL LETTER I 💌 LOVE LETTER 🤟 \n",
"3 LOVE HOTEL 🆑 SQUARED CL 🇮 \n",
"4 FACE SCREAMING IN FEAR 🌉 BRIDGE AT NIGHT 📨 \n",
"5 REGIONAL INDICATOR SYMBOL LETTER H 💐 BOUQUET 📩 \n",
"6 REGIONAL INDICATOR SYMBOL LETTER N 📜 SCROLL 📪 \n",
"7 REGIONAL INDICATOR SYMBOL LETTER O 🎁 WRAPPED PRESENT 📫 \n",
"8 MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT 💨 DASH SYMBOL 📰 \n",
"9 CITYSCAPE AT DUSK 💤 SLEEPING SYMBOL 📬 \n",
"\n",
" threshold_description \n",
"0 LOVE LETTER \n",
"1 LOVE HOTEL \n",
"2 I LOVE YOU HAND SIGN \n",
"3 REGIONAL INDICATOR SYMBOL LETTER I \n",
"4 INCOMING ENVELOPE \n",
"5 ENVELOPE WITH DOWNWARDS ARROW ABOVE \n",
"6 CLOSED MAILBOX WITH LOWERED FLAG \n",
"7 CLOSED MAILBOX WITH RAISED FLAG \n",
"8 NEWSPAPER \n",
"9 OPEN MAILBOX WITH RAISED FLAG "
]
},
"metadata": {},
"output_type": "display_data"
2018-05-31 15:06:33 +02:00
}
],
"source": [
2018-06-05 10:02:35 +02:00
"predict(\"I love sunny days!\", description_key='description' )"
2018-05-31 15:06:33 +02:00
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}