nlp-lab/Project/Tools/Emoji_Distance.py

260 lines
7.4 KiB
Python

# coding: utf-8
# # Emoji Distance
# a notebook dealing witch emoji distance measures. Uses an external csv with labeled data to compare arbitriary emojis related to sentiment
# Autor = Carsten Draschner
# Version = 0.1
# ## Used Ressources
# https://www.clarin.si/repository/xmlui/handle/11356/1048
# https://github.com/words/emoji-emotion
# In[34]:
import pandas as pd
import math
import numpy as np
# In[35]:
N=3
# In[2]:
#read in csv as panda file
df = pd.read_csv("../Tools/Emoji_Sentiment_Data_v1.0.csv", delimiter=";")
#df.head()
# In[3]:
def dataframe_to_dictionary():
data = {}
data_only_emoticons = {}
list_sentiment_vectors = []
list_emojis = []
list_sentiment_emoticon_vectors = []
list_emoticon_emojis = []
for index, row in df.iterrows():
emo = row["Emoji"]
occ = row["Occurrences"]
pos = row["Positive"]
neg = row["Negative"]
neu = row["Neutral"]
data.update({emo:[pos/occ,neg/occ,neu/occ]})
list_sentiment_vectors.append(np.array([pos/occ,neg/occ,neu/occ]))
list_emojis.append(emo)
if(row["Unicode block"]=="Emoticons"):
data_only_emoticons.update({emo:[pos/occ,neg/occ,neu/occ]})
list_sentiment_emoticon_vectors.append(np.array([pos/occ,neg/occ,neu/occ]))
list_emoticon_emojis.append(emo)
return data,data_only_emoticons,np.array(list_sentiment_vectors), np.array(list_emojis), np.array(list_sentiment_emoticon_vectors),np.array(list_emoticon_emojis)
#d , doe = dataframe_to_dictionary()
# In[4]:
# create global emoji lists and datasets
data , data_only_emoticons, list_sentiment_vectors , list_emojis , list_sentiment_emoticon_vectors , list_emoticon_emojis = dataframe_to_dictionary()
# In[5]:
#calculates vector distance between 2 3-dim sentiment representations of emojis
def sentiment_vector_dist(v1,v2):
#calculates vector distance between 2 3-dim sentiment representations of emojis consisting of positive neutral and negative probabilistic occuring
tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2))
return tmp_dist
# In[6]:
#calculates vector representation in a 3dim 0 to 1space of dimension: positive,negative,neutral
def emoji_to_sentiment_vector(e, only_emoticons=True):
"""tmp = df[df["Emoji"]==e]
#calculate by espacial labeled occurences devided by sum of overall occurences
pos = tmp["Positive"].values[0]/tmp["Occurrences"].values[0]
neg = tmp["Negative"].values[0]/tmp["Occurrences"].values[0]
neu = tmp["Neutral"].values[0]/tmp["Occurrences"].values[0]
#return as np array
return np.array([pos,neg,neu])"""
if e in (data_only_emoticons if only_emoticons else data):
return np.array((data_only_emoticons if only_emoticons else data)[e])
return np.array([float('NaN')]*N)
# In[7]:
#function to call for evaluating two emojis in its sentimental distance
def emoji_distance(e1,e2):
sent_v1 = emoji_to_sentiment_vector(e1)
sent_v2 = emoji_to_sentiment_vector(e2)
d = sentiment_vector_dist(sent_v1,sent_v2)
return d
# In[27]:
def sentiment_vector_to_emoji(v1, only_emoticons=True, custom_target_emojis=None, n_results=1):
target_sentiment_emojis = (list_sentiment_emoticon_vectors if only_emoticons else list_sentiment_vectors)
target_emojis = (list_emoticon_emojis if only_emoticons else list_emojis)
# filter target emojis by custom emojis, if some are given:
if custom_target_emojis is not None:
binary_filter_mask = np.isin(target_emojis, custom_target_emojis)
target_sentiment_emojis = target_sentiment_emojis[binary_filter_mask]
target_emojis = target_emojis[binary_filter_mask]
#more efficient approach for min distance
distances = target_sentiment_emojis - v1
distances = np.linalg.norm(distances, axis=1)
#find min entry
sorted_entrys = np.argsort(distances)
min_entry = np.argmin(distances)
#print(distances[sorted_entrys[:n_results]])
return target_emojis[min_entry] if n_results == 1 else target_emojis[sorted_entrys[:n_results]]
#version for dics
"""#set initial values to compare with
best_emoji = "😐"
min_distance = 10000
#compare only with filtred emoticons not containing other elements like cars etc.
#compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match
for e,v2 in doe.items():
#v2 = emoji_to_sentiment_vector(e)
d = sentiment_vector_dist(v1,v2)
if(d < min_distance):
min_distance = d
best_emoji = e
#print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
return best_emoji"""
#old version
"""#set initial values to compare with
best_emoji = "😐"
min_distance = 10000
#compare only with filtred emoticons not containing other elements like cars etc.
df_filtered = df[df["Unicode block"]=="Emoticons"]
all_smilies = list(df_filtered["Emoji"])
#compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match
for e in all_smilies:
v2 = emoji_to_sentiment_vector(e)
d = sentiment_vector_dist(v1,v2)
if(d < min_distance):
min_distance = d
best_emoji = e
#print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
return best_emoji"""
# In[28]:
def show_demo_min_distances(only_emoticons = True):
#df_filtered = df[df["Unicode block"]=="Emoticons"]
all_smilies = list_emoticon_emojis if only_emoticons else list_emojis
d_m = np.zeros(shape=(len(all_smilies),len(all_smilies)))
for c1 in range(len(all_smilies)):
for c2 in range(len(all_smilies)):
e1 = all_smilies[c1]
e2 = all_smilies[c2]
d = emoji_distance(e1,e2)
d_m[c1,c2] = d
for c in range(len(d_m[0])):
emoji = all_smilies[c]
row = d_m[c]
row_sorted = np.argsort(row)
#closest 5
r = row_sorted[0:10]
#print()
closest = ""
for i in r:
closest+=all_smilies[i]
print(emoji+": "+closest)
"""df_filtered = df[df["Unicode block"]=="Emoticons"]
all_smilies = list(df_filtered["Emoji"])
d_m = np.zeros(shape=(len(all_smilies),len(all_smilies)))
for c1 in range(len(all_smilies)):
for c2 in range(len(all_smilies)):
e1 = all_smilies[c1]
e2 = all_smilies[c2]
d = emoji_distance(e1,e2)
d_m[c1,c2] = d
for c in range(len(d_m[0])):
emoji = all_smilies[c]
row = d_m[c]
row_sorted = np.argsort(row)
#closest 5
r = row_sorted[0:10]
#print()
closest = ""
for i in r:
closest+=all_smilies[i]
print(emoji+": "+closest)"""
# In[29]:
#show_demo_min_distances()
# In[30]:
#test bipolar matching entiment vector vs. emoji
#def show_demo_matching_bipolar
# df_filtered = df[df["Unicode block"]=="Emoticons"]
# all_smilies = list(df_filtered["Emoji"])
# for e in all_smilies:
# v2 = emoji_to_sentiment_vector(e)
# sentiment_vector_to_emoji(v2)
# In[36]:
#[(e,sentiment_vector_to_emoji(emoji_to_sentiment_vector(e,only_emoticons=False))) for e in list_emojis]
# In[26]:
#sentiment_vector_to_emoji(np.array([ 0.72967448, 0.05173769, 0.21858783]))