performance optimization over np function
This commit is contained in:
		| @ -9,15 +9,21 @@ | ||||
| # https://www.clarin.si/repository/xmlui/handle/11356/1048 | ||||
| # https://github.com/words/emoji-emotion | ||||
|  | ||||
| # In[1]: | ||||
| # In[34]: | ||||
|  | ||||
|  | ||||
| import pandas as pd | ||||
| import math | ||||
| import numpy as np | ||||
|  | ||||
|  | ||||
| # In[35]: | ||||
|  | ||||
|  | ||||
| N=3 | ||||
| # In[53]: | ||||
|  | ||||
|  | ||||
| # In[2]: | ||||
|  | ||||
|  | ||||
| #read in csv as panda file | ||||
| @ -25,86 +31,142 @@ df = pd.read_csv("/Users/Carsten/GitRepos/NLP-LAB/Project/Tools/Emoji_Sentiment_ | ||||
| #df.head() | ||||
|  | ||||
|  | ||||
| # In[54]: | ||||
| # In[3]: | ||||
|  | ||||
|  | ||||
| def dataframe_to_dictionary(): | ||||
|     data = {} | ||||
|     data_only_emoticons = {} | ||||
|     list_sentiment_vectors = [] | ||||
|     list_emojis = [] | ||||
|     list_sentiment_emoticon_vectors = [] | ||||
|     list_emoticon_emojis = [] | ||||
|     for index, row in df.iterrows(): | ||||
|         emo = row["Emoji"] | ||||
|         occ = row["Occurrences"] | ||||
|         pos = row["Positive"] | ||||
|         neg = row["Negative"] | ||||
|         neu = row["Neutral"] | ||||
|         data.update({emo:[pos/occ,neg/occ,neu/occ]}) | ||||
|          | ||||
|         list_sentiment_vectors.append(np.array([pos/occ,neg/occ,neu/occ])) | ||||
|         list_emojis.append(emo) | ||||
|          | ||||
|         if(row["Unicode block"]=="Emoticons"): | ||||
|             data_only_emoticons.update({emo:[pos/occ,neg/occ,neu/occ]}) | ||||
|              | ||||
|             list_sentiment_emoticon_vectors.append(np.array([pos/occ,neg/occ,neu/occ])) | ||||
|             list_emoticon_emojis.append(emo) | ||||
|  | ||||
|  | ||||
|     return data,data_only_emoticons,np.array(list_sentiment_vectors), np.array(list_emojis), np.array(list_sentiment_emoticon_vectors),np.array(list_emoticon_emojis) | ||||
| #d , doe = dataframe_to_dictionary() | ||||
|  | ||||
|  | ||||
| # In[4]: | ||||
|  | ||||
|  | ||||
| data , data_only_emoticons, list_sentiment_vectors , list_emojis , list_sentiment_emoticon_vectors , list_emoticon_emojis = dataframe_to_dictionary() | ||||
|  | ||||
|  | ||||
| # In[5]: | ||||
|  | ||||
|  | ||||
| #calculates vector distance between 2 3-dim sentiment representations of emojis | ||||
| def sentiment_vector_dist(v1,v2): | ||||
|     #pos_v1 = v1[0] | ||||
|     #neg_v1 = v1[1] | ||||
|     #neu_v1 = v1[2] | ||||
|  | ||||
|     #pos_v2 = v2[0] | ||||
|     #neg_v2 = v2[1] | ||||
|     #neu_v2 = v2[2] | ||||
|  | ||||
|     #tmp_dist = float(np.abs(pos_v1-pos_v2))+float(np.abs(neg_v1-neg_v2))+float(np.abs(neu_v1-neu_v2)) | ||||
|  | ||||
|     #calculates vector distance between 2 3-dim sentiment representations of emojis consisting of positive neutral and negative probabilistic occuring | ||||
|     tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2)) | ||||
|     tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2))   | ||||
|     return tmp_dist | ||||
|  | ||||
|  | ||||
| # In[55]: | ||||
| # In[6]: | ||||
|  | ||||
|  | ||||
| #calculates vector representation in a 3dim 0 to 1space of dimension: positive,negative,neutral | ||||
| def emoji_to_sentiment_vector(e): | ||||
|     tmp = df[df["Emoji"]==e] | ||||
| def emoji_to_sentiment_vector(e, only_emoticons=True): | ||||
|     """tmp = df[df["Emoji"]==e]     | ||||
|     #calculate by espacial labeled occurences devided by sum of overall occurences | ||||
|     pos = tmp["Positive"].values[0]/tmp["Occurrences"].values[0] | ||||
|     neg = tmp["Negative"].values[0]/tmp["Occurrences"].values[0] | ||||
|     neu = tmp["Neutral"].values[0]/tmp["Occurrences"].values[0] | ||||
|     #return as np array | ||||
|     return np.array([pos,neg,neu]) | ||||
|     return np.array([pos,neg,neu])""" | ||||
|     if e in (data_only_emoticons if only_emoticons else data): | ||||
|         return np.array((data_only_emoticons if only_emoticons else data)[e]) | ||||
|     return np.array([float('NaN')]*N)  | ||||
|  | ||||
|  | ||||
| # In[56]: | ||||
| # In[7]: | ||||
|  | ||||
|  | ||||
| #function to call for evaluating two emojis in its sentimental distance | ||||
| def emoji_distance(e1,e2): | ||||
|     sent_v1 = emoji_to_sentiment_vector(e1) | ||||
|     sent_v2 = emoji_to_sentiment_vector(e2) | ||||
|  | ||||
|      | ||||
|     d = sentiment_vector_dist(sent_v1,sent_v2) | ||||
|     return d | ||||
|  | ||||
|  | ||||
| # In[57]: | ||||
| # In[27]: | ||||
|  | ||||
|  | ||||
| def sentiment_vector_to_emoji(v1): | ||||
|     #if(len(v1) == 3): | ||||
|         #set initial values to compare with | ||||
|         best_emoji = "😐" | ||||
|         min_distance = 10000 | ||||
| def sentiment_vector_to_emoji(v1, only_emoticons=True): | ||||
|     #more efficient approach for min distance | ||||
|     distances = (list_sentiment_emoticon_vectors if only_emoticons else list_sentiment_vectors) - v1 | ||||
|     distances = np.linalg.norm(distances, axis=1) | ||||
|     #find min entry | ||||
|     min_entry = np.argmin(distances) | ||||
|      | ||||
|     return (list_emoticon_emojis if only_emoticons else list_emojis)[min_entry] | ||||
|  | ||||
|         #compare only with filtred emoticons | ||||
|         df_filtered = df[df["Unicode block"]=="Emoticons"] | ||||
|         all_smilies = list(df_filtered["Emoji"]) | ||||
|         for e in all_smilies: | ||||
|             v2 = emoji_to_sentiment_vector(e) | ||||
|             d = sentiment_vector_dist(v1,v2) | ||||
|             if(d < min_distance): | ||||
|                 min_distance = d | ||||
|                 best_emoji = e | ||||
|                 #print(str(v1),str(v2),str(min_distance),str(type(v1)),str(type(v2)),e) | ||||
|     #version for dics | ||||
|  | ||||
|     """#set initial values to compare with | ||||
|     best_emoji = "😐" | ||||
|     min_distance = 10000 | ||||
|  | ||||
|     #compare only with filtred emoticons not containing other elements like cars etc. | ||||
|     #compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match | ||||
|     for e,v2 in doe.items(): | ||||
|         #v2 = emoji_to_sentiment_vector(e) | ||||
|         d = sentiment_vector_dist(v1,v2) | ||||
|         if(d < min_distance): | ||||
|             min_distance = d | ||||
|             best_emoji = e | ||||
|  | ||||
|  | ||||
|         #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!") | ||||
|         return best_emoji | ||||
|     #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!") | ||||
|     return best_emoji""" | ||||
|  | ||||
|     #else: | ||||
|         #print("WRONG SENTIMENT VECTOR") | ||||
|     #old version | ||||
|  | ||||
|     """#set initial values to compare with | ||||
|     best_emoji = "😐" | ||||
|     min_distance = 10000 | ||||
|  | ||||
| # In[58]: | ||||
|  | ||||
|  | ||||
| def show_demo(): | ||||
|     #compare only with filtred emoticons not containing other elements like cars etc. | ||||
|     df_filtered = df[df["Unicode block"]=="Emoticons"] | ||||
|     all_smilies = list(df_filtered["Emoji"]) | ||||
|     #compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match | ||||
|     for e in all_smilies: | ||||
|         v2 = emoji_to_sentiment_vector(e) | ||||
|         d = sentiment_vector_dist(v1,v2) | ||||
|         if(d < min_distance): | ||||
|             min_distance = d | ||||
|             best_emoji = e | ||||
|  | ||||
|  | ||||
|     #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!") | ||||
|     return best_emoji""" | ||||
|  | ||||
|  | ||||
| # In[28]: | ||||
|  | ||||
|  | ||||
| def show_demo_min_distances(only_emoticons = True): | ||||
|     #df_filtered = df[df["Unicode block"]=="Emoticons"] | ||||
|     all_smilies = list_emoticon_emojis if only_emoticons else list_emojis | ||||
|  | ||||
|     d_m = np.zeros(shape=(len(all_smilies),len(all_smilies))) | ||||
|  | ||||
| @ -115,7 +177,7 @@ def show_demo(): | ||||
|  | ||||
|             d = emoji_distance(e1,e2) | ||||
|             d_m[c1,c2] = d | ||||
|  | ||||
|              | ||||
|     for c in range(len(d_m[0])): | ||||
|         emoji = all_smilies[c] | ||||
|         row = d_m[c] | ||||
| @ -127,20 +189,59 @@ def show_demo(): | ||||
|         for i in r: | ||||
|             closest+=all_smilies[i] | ||||
|         print(emoji+": "+closest) | ||||
|      | ||||
|     """df_filtered = df[df["Unicode block"]=="Emoticons"] | ||||
|     all_smilies = list(df_filtered["Emoji"]) | ||||
|  | ||||
|     d_m = np.zeros(shape=(len(all_smilies),len(all_smilies))) | ||||
|  | ||||
|     for c1 in range(len(all_smilies)): | ||||
|         for c2 in range(len(all_smilies)): | ||||
|             e1 = all_smilies[c1] | ||||
|             e2 = all_smilies[c2] | ||||
|  | ||||
|             d = emoji_distance(e1,e2) | ||||
|             d_m[c1,c2] = d | ||||
|              | ||||
|     for c in range(len(d_m[0])): | ||||
|         emoji = all_smilies[c] | ||||
|         row = d_m[c] | ||||
|         row_sorted = np.argsort(row) | ||||
|         #closest 5 | ||||
|         r = row_sorted[0:10] | ||||
|         #print() | ||||
|         closest = "" | ||||
|         for i in r: | ||||
|             closest+=all_smilies[i] | ||||
|         print(emoji+": "+closest)""" | ||||
|  | ||||
|  | ||||
| # In[60]: | ||||
| # In[29]: | ||||
|  | ||||
|  | ||||
| #show_demo() | ||||
| show_demo_min_distances() | ||||
|  | ||||
|  | ||||
| # In[61]: | ||||
| # In[30]: | ||||
|  | ||||
|  | ||||
| #test bipolar matching entiment vector vs. emoji | ||||
| #df_filtered = df[df["Unicode block"]=="Emoticons"] | ||||
| #all_smilies = list(df_filtered["Emoji"]) | ||||
| #for e in all_smilies: | ||||
| #    v2 = emoji_to_sentiment_vector(e) | ||||
| #    sentiment_vector_to_emoji(v2) | ||||
| #def show_demo_matching_bipolar | ||||
| #    df_filtered = df[df["Unicode block"]=="Emoticons"] | ||||
| #    all_smilies = list(df_filtered["Emoji"]) | ||||
| #    for e in all_smilies: | ||||
| #        v2 = emoji_to_sentiment_vector(e) | ||||
| #        sentiment_vector_to_emoji(v2) | ||||
|  | ||||
|  | ||||
| # In[36]: | ||||
|  | ||||
|  | ||||
| [(e,sentiment_vector_to_emoji(emoji_to_sentiment_vector(e,only_emoticons=False))) for e in list_emojis] | ||||
|  | ||||
|  | ||||
| # In[26]: | ||||
|  | ||||
|  | ||||
| sentiment_vector_to_emoji(np.array([ 0.72967448,  0.05173769,  0.21858783])) | ||||
|  | ||||
|  | ||||
		Reference in New Issue
	
	Block a user