performance optimization over np function
This commit is contained in:
Carsten
parent
2cbb06ea94
commit
2c93c98269
@ -9,15 +9,21 @@
|
||||
# https://www.clarin.si/repository/xmlui/handle/11356/1048
|
||||
# https://github.com/words/emoji-emotion
|
||||
|
||||
# In[1]:
|
||||
# In[34]:
|
||||
|
||||
|
||||
import pandas as pd
|
||||
import math
|
||||
import numpy as np
|
||||
|
||||
|
||||
# In[35]:
|
||||
|
||||
|
||||
N=3
|
||||
# In[53]:
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
#read in csv as panda file
|
||||
@ -25,41 +31,72 @@ df = pd.read_csv("/Users/Carsten/GitRepos/NLP-LAB/Project/Tools/Emoji_Sentiment_
|
||||
#df.head()
|
||||
|
||||
|
||||
# In[54]:
|
||||
# In[3]:
|
||||
|
||||
|
||||
def dataframe_to_dictionary():
|
||||
data = {}
|
||||
data_only_emoticons = {}
|
||||
list_sentiment_vectors = []
|
||||
list_emojis = []
|
||||
list_sentiment_emoticon_vectors = []
|
||||
list_emoticon_emojis = []
|
||||
for index, row in df.iterrows():
|
||||
emo = row["Emoji"]
|
||||
occ = row["Occurrences"]
|
||||
pos = row["Positive"]
|
||||
neg = row["Negative"]
|
||||
neu = row["Neutral"]
|
||||
data.update({emo:[pos/occ,neg/occ,neu/occ]})
|
||||
|
||||
list_sentiment_vectors.append(np.array([pos/occ,neg/occ,neu/occ]))
|
||||
list_emojis.append(emo)
|
||||
|
||||
if(row["Unicode block"]=="Emoticons"):
|
||||
data_only_emoticons.update({emo:[pos/occ,neg/occ,neu/occ]})
|
||||
|
||||
list_sentiment_emoticon_vectors.append(np.array([pos/occ,neg/occ,neu/occ]))
|
||||
list_emoticon_emojis.append(emo)
|
||||
|
||||
|
||||
return data,data_only_emoticons,np.array(list_sentiment_vectors), np.array(list_emojis), np.array(list_sentiment_emoticon_vectors),np.array(list_emoticon_emojis)
|
||||
#d , doe = dataframe_to_dictionary()
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
data , data_only_emoticons, list_sentiment_vectors , list_emojis , list_sentiment_emoticon_vectors , list_emoticon_emojis = dataframe_to_dictionary()
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
#calculates vector distance between 2 3-dim sentiment representations of emojis
|
||||
def sentiment_vector_dist(v1,v2):
|
||||
#pos_v1 = v1[0]
|
||||
#neg_v1 = v1[1]
|
||||
#neu_v1 = v1[2]
|
||||
|
||||
#pos_v2 = v2[0]
|
||||
#neg_v2 = v2[1]
|
||||
#neu_v2 = v2[2]
|
||||
|
||||
#tmp_dist = float(np.abs(pos_v1-pos_v2))+float(np.abs(neg_v1-neg_v2))+float(np.abs(neu_v1-neu_v2))
|
||||
|
||||
#calculates vector distance between 2 3-dim sentiment representations of emojis consisting of positive neutral and negative probabilistic occuring
|
||||
tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2))
|
||||
return tmp_dist
|
||||
|
||||
|
||||
# In[55]:
|
||||
# In[6]:
|
||||
|
||||
|
||||
#calculates vector representation in a 3dim 0 to 1space of dimension: positive,negative,neutral
|
||||
def emoji_to_sentiment_vector(e):
|
||||
tmp = df[df["Emoji"]==e]
|
||||
def emoji_to_sentiment_vector(e, only_emoticons=True):
|
||||
"""tmp = df[df["Emoji"]==e]
|
||||
#calculate by espacial labeled occurences devided by sum of overall occurences
|
||||
pos = tmp["Positive"].values[0]/tmp["Occurrences"].values[0]
|
||||
neg = tmp["Negative"].values[0]/tmp["Occurrences"].values[0]
|
||||
neu = tmp["Neutral"].values[0]/tmp["Occurrences"].values[0]
|
||||
#return as np array
|
||||
return np.array([pos,neg,neu])
|
||||
return np.array([pos,neg,neu])"""
|
||||
if e in (data_only_emoticons if only_emoticons else data):
|
||||
return np.array((data_only_emoticons if only_emoticons else data)[e])
|
||||
return np.array([float('NaN')]*N)
|
||||
|
||||
|
||||
# In[56]:
|
||||
# In[7]:
|
||||
|
||||
|
||||
#function to call for evaluating two emojis in its sentimental distance
|
||||
@ -71,40 +108,65 @@ def emoji_distance(e1,e2):
|
||||
return d
|
||||
|
||||
|
||||
# In[57]:
|
||||
# In[27]:
|
||||
|
||||
|
||||
def sentiment_vector_to_emoji(v1):
|
||||
#if(len(v1) == 3):
|
||||
#set initial values to compare with
|
||||
best_emoji = "😐"
|
||||
min_distance = 10000
|
||||
def sentiment_vector_to_emoji(v1, only_emoticons=True):
|
||||
#more efficient approach for min distance
|
||||
distances = (list_sentiment_emoticon_vectors if only_emoticons else list_sentiment_vectors) - v1
|
||||
distances = np.linalg.norm(distances, axis=1)
|
||||
#find min entry
|
||||
min_entry = np.argmin(distances)
|
||||
|
||||
#compare only with filtred emoticons
|
||||
df_filtered = df[df["Unicode block"]=="Emoticons"]
|
||||
all_smilies = list(df_filtered["Emoji"])
|
||||
for e in all_smilies:
|
||||
v2 = emoji_to_sentiment_vector(e)
|
||||
d = sentiment_vector_dist(v1,v2)
|
||||
if(d < min_distance):
|
||||
min_distance = d
|
||||
best_emoji = e
|
||||
#print(str(v1),str(v2),str(min_distance),str(type(v1)),str(type(v2)),e)
|
||||
return (list_emoticon_emojis if only_emoticons else list_emojis)[min_entry]
|
||||
|
||||
#version for dics
|
||||
|
||||
"""#set initial values to compare with
|
||||
best_emoji = "😐"
|
||||
min_distance = 10000
|
||||
|
||||
#compare only with filtred emoticons not containing other elements like cars etc.
|
||||
#compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match
|
||||
for e,v2 in doe.items():
|
||||
#v2 = emoji_to_sentiment_vector(e)
|
||||
d = sentiment_vector_dist(v1,v2)
|
||||
if(d < min_distance):
|
||||
min_distance = d
|
||||
best_emoji = e
|
||||
|
||||
|
||||
#print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
|
||||
return best_emoji
|
||||
#print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
|
||||
return best_emoji"""
|
||||
|
||||
#else:
|
||||
#print("WRONG SENTIMENT VECTOR")
|
||||
#old version
|
||||
|
||||
"""#set initial values to compare with
|
||||
best_emoji = "😐"
|
||||
min_distance = 10000
|
||||
|
||||
# In[58]:
|
||||
|
||||
|
||||
def show_demo():
|
||||
#compare only with filtred emoticons not containing other elements like cars etc.
|
||||
df_filtered = df[df["Unicode block"]=="Emoticons"]
|
||||
all_smilies = list(df_filtered["Emoji"])
|
||||
#compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match
|
||||
for e in all_smilies:
|
||||
v2 = emoji_to_sentiment_vector(e)
|
||||
d = sentiment_vector_dist(v1,v2)
|
||||
if(d < min_distance):
|
||||
min_distance = d
|
||||
best_emoji = e
|
||||
|
||||
|
||||
#print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
|
||||
return best_emoji"""
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
def show_demo_min_distances(only_emoticons = True):
|
||||
#df_filtered = df[df["Unicode block"]=="Emoticons"]
|
||||
all_smilies = list_emoticon_emojis if only_emoticons else list_emojis
|
||||
|
||||
d_m = np.zeros(shape=(len(all_smilies),len(all_smilies)))
|
||||
|
||||
@ -128,19 +190,58 @@ def show_demo():
|
||||
closest+=all_smilies[i]
|
||||
print(emoji+": "+closest)
|
||||
|
||||
"""df_filtered = df[df["Unicode block"]=="Emoticons"]
|
||||
all_smilies = list(df_filtered["Emoji"])
|
||||
|
||||
# In[60]:
|
||||
d_m = np.zeros(shape=(len(all_smilies),len(all_smilies)))
|
||||
|
||||
for c1 in range(len(all_smilies)):
|
||||
for c2 in range(len(all_smilies)):
|
||||
e1 = all_smilies[c1]
|
||||
e2 = all_smilies[c2]
|
||||
|
||||
d = emoji_distance(e1,e2)
|
||||
d_m[c1,c2] = d
|
||||
|
||||
for c in range(len(d_m[0])):
|
||||
emoji = all_smilies[c]
|
||||
row = d_m[c]
|
||||
row_sorted = np.argsort(row)
|
||||
#closest 5
|
||||
r = row_sorted[0:10]
|
||||
#print()
|
||||
closest = ""
|
||||
for i in r:
|
||||
closest+=all_smilies[i]
|
||||
print(emoji+": "+closest)"""
|
||||
|
||||
|
||||
#show_demo()
|
||||
# In[29]:
|
||||
|
||||
|
||||
# In[61]:
|
||||
show_demo_min_distances()
|
||||
|
||||
|
||||
# In[30]:
|
||||
|
||||
|
||||
#test bipolar matching entiment vector vs. emoji
|
||||
#df_filtered = df[df["Unicode block"]=="Emoticons"]
|
||||
#all_smilies = list(df_filtered["Emoji"])
|
||||
#for e in all_smilies:
|
||||
# v2 = emoji_to_sentiment_vector(e)
|
||||
# sentiment_vector_to_emoji(v2)
|
||||
#def show_demo_matching_bipolar
|
||||
# df_filtered = df[df["Unicode block"]=="Emoticons"]
|
||||
# all_smilies = list(df_filtered["Emoji"])
|
||||
# for e in all_smilies:
|
||||
# v2 = emoji_to_sentiment_vector(e)
|
||||
# sentiment_vector_to_emoji(v2)
|
||||
|
||||
|
||||
# In[36]:
|
||||
|
||||
|
||||
[(e,sentiment_vector_to_emoji(emoji_to_sentiment_vector(e,only_emoticons=False))) for e in list_emojis]
|
||||
|
||||
|
||||
# In[26]:
|
||||
|
||||
|
||||
sentiment_vector_to_emoji(np.array([ 0.72967448, 0.05173769, 0.21858783]))
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user