From 2c93c9826921818942ec368e177ed8eba1d57a25 Mon Sep 17 00:00:00 2001 From: Carsten Date: Sun, 10 Jun 2018 17:53:26 +0200 Subject: [PATCH] performance optimization over np function --- Project/Tools/Emoji_Distance.py | 207 ++++++++++++++++++++++++-------- 1 file changed, 154 insertions(+), 53 deletions(-) diff --git a/Project/Tools/Emoji_Distance.py b/Project/Tools/Emoji_Distance.py index 6aebde8..775d935 100644 --- a/Project/Tools/Emoji_Distance.py +++ b/Project/Tools/Emoji_Distance.py @@ -9,15 +9,21 @@ # https://www.clarin.si/repository/xmlui/handle/11356/1048 # https://github.com/words/emoji-emotion -# In[1]: +# In[34]: import pandas as pd import math import numpy as np + +# In[35]: + + N=3 -# In[53]: + + +# In[2]: #read in csv as panda file @@ -25,86 +31,142 @@ df = pd.read_csv("/Users/Carsten/GitRepos/NLP-LAB/Project/Tools/Emoji_Sentiment_ #df.head() -# In[54]: +# In[3]: + + +def dataframe_to_dictionary(): + data = {} + data_only_emoticons = {} + list_sentiment_vectors = [] + list_emojis = [] + list_sentiment_emoticon_vectors = [] + list_emoticon_emojis = [] + for index, row in df.iterrows(): + emo = row["Emoji"] + occ = row["Occurrences"] + pos = row["Positive"] + neg = row["Negative"] + neu = row["Neutral"] + data.update({emo:[pos/occ,neg/occ,neu/occ]}) + + list_sentiment_vectors.append(np.array([pos/occ,neg/occ,neu/occ])) + list_emojis.append(emo) + + if(row["Unicode block"]=="Emoticons"): + data_only_emoticons.update({emo:[pos/occ,neg/occ,neu/occ]}) + + list_sentiment_emoticon_vectors.append(np.array([pos/occ,neg/occ,neu/occ])) + list_emoticon_emojis.append(emo) + + + return data,data_only_emoticons,np.array(list_sentiment_vectors), np.array(list_emojis), np.array(list_sentiment_emoticon_vectors),np.array(list_emoticon_emojis) +#d , doe = dataframe_to_dictionary() + + +# In[4]: + + +data , data_only_emoticons, list_sentiment_vectors , list_emojis , list_sentiment_emoticon_vectors , list_emoticon_emojis = dataframe_to_dictionary() + + +# In[5]: #calculates vector distance between 2 3-dim sentiment representations of emojis def sentiment_vector_dist(v1,v2): - #pos_v1 = v1[0] - #neg_v1 = v1[1] - #neu_v1 = v1[2] - - #pos_v2 = v2[0] - #neg_v2 = v2[1] - #neu_v2 = v2[2] - - #tmp_dist = float(np.abs(pos_v1-pos_v2))+float(np.abs(neg_v1-neg_v2))+float(np.abs(neu_v1-neu_v2)) - #calculates vector distance between 2 3-dim sentiment representations of emojis consisting of positive neutral and negative probabilistic occuring - tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2)) + tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2)) return tmp_dist -# In[55]: +# In[6]: #calculates vector representation in a 3dim 0 to 1space of dimension: positive,negative,neutral -def emoji_to_sentiment_vector(e): - tmp = df[df["Emoji"]==e] +def emoji_to_sentiment_vector(e, only_emoticons=True): + """tmp = df[df["Emoji"]==e] #calculate by espacial labeled occurences devided by sum of overall occurences pos = tmp["Positive"].values[0]/tmp["Occurrences"].values[0] neg = tmp["Negative"].values[0]/tmp["Occurrences"].values[0] neu = tmp["Neutral"].values[0]/tmp["Occurrences"].values[0] #return as np array - return np.array([pos,neg,neu]) + return np.array([pos,neg,neu])""" + if e in (data_only_emoticons if only_emoticons else data): + return np.array((data_only_emoticons if only_emoticons else data)[e]) + return np.array([float('NaN')]*N) -# In[56]: +# In[7]: #function to call for evaluating two emojis in its sentimental distance def emoji_distance(e1,e2): sent_v1 = emoji_to_sentiment_vector(e1) sent_v2 = emoji_to_sentiment_vector(e2) - + d = sentiment_vector_dist(sent_v1,sent_v2) return d -# In[57]: +# In[27]: -def sentiment_vector_to_emoji(v1): - #if(len(v1) == 3): - #set initial values to compare with - best_emoji = "😐" - min_distance = 10000 +def sentiment_vector_to_emoji(v1, only_emoticons=True): + #more efficient approach for min distance + distances = (list_sentiment_emoticon_vectors if only_emoticons else list_sentiment_vectors) - v1 + distances = np.linalg.norm(distances, axis=1) + #find min entry + min_entry = np.argmin(distances) + + return (list_emoticon_emojis if only_emoticons else list_emojis)[min_entry] - #compare only with filtred emoticons - df_filtered = df[df["Unicode block"]=="Emoticons"] - all_smilies = list(df_filtered["Emoji"]) - for e in all_smilies: - v2 = emoji_to_sentiment_vector(e) - d = sentiment_vector_dist(v1,v2) - if(d < min_distance): - min_distance = d - best_emoji = e - #print(str(v1),str(v2),str(min_distance),str(type(v1)),str(type(v2)),e) + #version for dics + + """#set initial values to compare with + best_emoji = "😐" + min_distance = 10000 + + #compare only with filtred emoticons not containing other elements like cars etc. + #compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match + for e,v2 in doe.items(): + #v2 = emoji_to_sentiment_vector(e) + d = sentiment_vector_dist(v1,v2) + if(d < min_distance): + min_distance = d + best_emoji = e - #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!") - return best_emoji + #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!") + return best_emoji""" - #else: - #print("WRONG SENTIMENT VECTOR") + #old version + """#set initial values to compare with + best_emoji = "😐" + min_distance = 10000 -# In[58]: - - -def show_demo(): + #compare only with filtred emoticons not containing other elements like cars etc. df_filtered = df[df["Unicode block"]=="Emoticons"] all_smilies = list(df_filtered["Emoji"]) + #compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match + for e in all_smilies: + v2 = emoji_to_sentiment_vector(e) + d = sentiment_vector_dist(v1,v2) + if(d < min_distance): + min_distance = d + best_emoji = e + + + #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!") + return best_emoji""" + + +# In[28]: + + +def show_demo_min_distances(only_emoticons = True): + #df_filtered = df[df["Unicode block"]=="Emoticons"] + all_smilies = list_emoticon_emojis if only_emoticons else list_emojis d_m = np.zeros(shape=(len(all_smilies),len(all_smilies))) @@ -115,7 +177,7 @@ def show_demo(): d = emoji_distance(e1,e2) d_m[c1,c2] = d - + for c in range(len(d_m[0])): emoji = all_smilies[c] row = d_m[c] @@ -127,20 +189,59 @@ def show_demo(): for i in r: closest+=all_smilies[i] print(emoji+": "+closest) + + """df_filtered = df[df["Unicode block"]=="Emoticons"] + all_smilies = list(df_filtered["Emoji"]) + + d_m = np.zeros(shape=(len(all_smilies),len(all_smilies))) + + for c1 in range(len(all_smilies)): + for c2 in range(len(all_smilies)): + e1 = all_smilies[c1] + e2 = all_smilies[c2] + + d = emoji_distance(e1,e2) + d_m[c1,c2] = d + + for c in range(len(d_m[0])): + emoji = all_smilies[c] + row = d_m[c] + row_sorted = np.argsort(row) + #closest 5 + r = row_sorted[0:10] + #print() + closest = "" + for i in r: + closest+=all_smilies[i] + print(emoji+": "+closest)""" -# In[60]: +# In[29]: -#show_demo() +show_demo_min_distances() -# In[61]: +# In[30]: #test bipolar matching entiment vector vs. emoji -#df_filtered = df[df["Unicode block"]=="Emoticons"] -#all_smilies = list(df_filtered["Emoji"]) -#for e in all_smilies: -# v2 = emoji_to_sentiment_vector(e) -# sentiment_vector_to_emoji(v2) +#def show_demo_matching_bipolar +# df_filtered = df[df["Unicode block"]=="Emoticons"] +# all_smilies = list(df_filtered["Emoji"]) +# for e in all_smilies: +# v2 = emoji_to_sentiment_vector(e) +# sentiment_vector_to_emoji(v2) + + +# In[36]: + + +[(e,sentiment_vector_to_emoji(emoji_to_sentiment_vector(e,only_emoticons=False))) for e in list_emojis] + + +# In[26]: + + +sentiment_vector_to_emoji(np.array([ 0.72967448, 0.05173769, 0.21858783])) +