naive approach
This commit is contained in:
Jonas Weinz
parent
bbc880f56c
commit
c4a67c8e84
371
Project/naive_approach/naive_approach.ipynb
Normal file
371
Project/naive_approach/naive_approach.ipynb
Normal file
@ -0,0 +1,371 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 42,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"from IPython.display import clear_output, Markdown, Math\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"import os\n",
|
||||
"import unicodedata as uni\n",
|
||||
"import numpy as np\n",
|
||||
"from nltk.stem import PorterStemmer\n",
|
||||
"from nltk.tokenize import sent_tokenize, word_tokenize\n",
|
||||
"from nltk.corpus import wordnet\n",
|
||||
"import math"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Naive Approach"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* read in table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>Unnamed: 0</th>\n",
|
||||
" <th>code</th>\n",
|
||||
" <th>character</th>\n",
|
||||
" <th>description</th>\n",
|
||||
" <th>Unnamed: 4</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>126980</td>\n",
|
||||
" <td>🀄</td>\n",
|
||||
" <td>MAHJONG TILE RED DRAGON</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>129525</td>\n",
|
||||
" <td>🧵</td>\n",
|
||||
" <td>SPOOL OF THREAD</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>2</td>\n",
|
||||
" <td>129526</td>\n",
|
||||
" <td>🧶</td>\n",
|
||||
" <td>BALL OF YARN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>3</td>\n",
|
||||
" <td>127183</td>\n",
|
||||
" <td>🃏</td>\n",
|
||||
" <td>PLAYING CARD BLACK JOKER</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>4</td>\n",
|
||||
" <td>129296</td>\n",
|
||||
" <td>🤐</td>\n",
|
||||
" <td>ZIPPER-MOUTH FACE</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Unnamed: 0 code character description Unnamed: 4\n",
|
||||
"0 0 126980 🀄 MAHJONG TILE RED DRAGON NaN\n",
|
||||
"1 1 129525 🧵 SPOOL OF THREAD NaN\n",
|
||||
"2 2 129526 🧶 BALL OF YARN NaN\n",
|
||||
"3 3 127183 🃏 PLAYING CARD BLACK JOKER NaN\n",
|
||||
"4 4 129296 🤐 ZIPPER-MOUTH FACE NaN"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"table = pd.read_csv('../Tools/emoji_descriptions.csv')\n",
|
||||
"table.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* todo: read in a lot of messages"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"messages = [\"Hello, this is a testing message\", \"this is a very sunny day today, i am very happy\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ps = PorterStemmer()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"stemmed_messages = []\n",
|
||||
"for m in messages:\n",
|
||||
" words = word_tokenize(m)\n",
|
||||
" sm = []\n",
|
||||
" for w in words:\n",
|
||||
" sm.append(ps.stem(w))\n",
|
||||
" stemmed_messages.append(sm)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[['hello', ',', 'thi', 'is', 'a', 'test', 'messag'],\n",
|
||||
" ['thi',\n",
|
||||
" 'is',\n",
|
||||
" 'a',\n",
|
||||
" 'veri',\n",
|
||||
" 'sunni',\n",
|
||||
" 'day',\n",
|
||||
" 'today',\n",
|
||||
" ',',\n",
|
||||
" 'i',\n",
|
||||
" 'am',\n",
|
||||
" 'veri',\n",
|
||||
" 'happi']]"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"stemmed_messages"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(1027, 5)"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"table.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* compare words to emoji descriptions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 59,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def evaluate_sentence(sentence):\n",
|
||||
" tokenized_sentence = word_tokenize(sentence)\n",
|
||||
" n = len(tokenized_sentence)\n",
|
||||
" l = table.shape[0]\n",
|
||||
" matrix_list = []\n",
|
||||
" \n",
|
||||
" for index, row in table.iterrows():\n",
|
||||
" emoji_tokens = word_tokenize(row['description'])\n",
|
||||
" m = len(emoji_tokens)\n",
|
||||
"\n",
|
||||
" mat = np.zeros(shape=(m,n))\n",
|
||||
" for i in range(len(emoji_tokens)):\n",
|
||||
" for j in range(len(tokenized_sentence)):\n",
|
||||
" syn1 = wordnet.synsets(emoji_tokens[i])\n",
|
||||
" if len(syn1) == 0:\n",
|
||||
" continue\n",
|
||||
" w1 = syn1[0]\n",
|
||||
" #print(j, tokenized_sentence)\n",
|
||||
" syn2 = wordnet.synsets(tokenized_sentence[j])\n",
|
||||
" if len(syn2) == 0:\n",
|
||||
" continue\n",
|
||||
" w2 = syn2[0]\n",
|
||||
" val = w1.wup_similarity(w2)\n",
|
||||
" if val is None:\n",
|
||||
" continue\n",
|
||||
" mat[i,j] = val\n",
|
||||
" #print(row['character'], mat)\n",
|
||||
" matrix_list.append(mat)\n",
|
||||
" \n",
|
||||
" return matrix_list\n",
|
||||
" \n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 106,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result = evaluate_sentence(\"car soccer surf\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* building a lookup table:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 107,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lookup = {}\n",
|
||||
"for index, row in table.iterrows():\n",
|
||||
" lookup[index] = row['character']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* sorting"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 108,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summed = np.argsort([-np.sum(x) for x in result])\n",
|
||||
"max_val = np.argsort([-np.max(x) for x in result])\n",
|
||||
"avg = np.argsort([-np.mean(x) for x in result])\n",
|
||||
"\n",
|
||||
"t = 0.7\n",
|
||||
"threshold = np.argsort([-len(np.where(x>t)[0]) for x in result])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 109,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/markdown": [
|
||||
"# 🏉⚾🎳🔥🏐🎱💏🧾🚗🚘"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def print_best_results(sorted_indices, n=10):\n",
|
||||
" print([lookup[x] + \" -- \" + table.iloc[]])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user