performance optimization over np function

Project/Tools/Emoji_Distance.py

@@ -9,15 +9,21 @@
 # https://www.clarin.si/repository/xmlui/handle/11356/1048
 # https://github.com/words/emoji-emotion
 
-# In[1]:
+# In[34]:
 
 
 import pandas as pd
 import math
 import numpy as np
 
+
+# In[35]:
+
+
 N=3
-# In[53]:
+
+
+# In[2]:
 
 
 #read in csv as panda file
@@ -25,86 +31,142 @@ df = pd.read_csv("/Users/Carsten/GitRepos/NLP-LAB/Project/Tools/Emoji_Sentiment_
 #df.head()
 
 
-# In[54]:
+# In[3]:
+
+
+def dataframe_to_dictionary():
+    data = {}
+    data_only_emoticons = {}
+    list_sentiment_vectors = []
+    list_emojis = []
+    list_sentiment_emoticon_vectors = []
+    list_emoticon_emojis = []
+    for index, row in df.iterrows():
+        emo = row["Emoji"]
+        occ = row["Occurrences"]
+        pos = row["Positive"]
+        neg = row["Negative"]
+        neu = row["Neutral"]
+        data.update({emo:[pos/occ,neg/occ,neu/occ]})
+        
+        list_sentiment_vectors.append(np.array([pos/occ,neg/occ,neu/occ]))
+        list_emojis.append(emo)
+        
+        if(row["Unicode block"]=="Emoticons"):
+            data_only_emoticons.update({emo:[pos/occ,neg/occ,neu/occ]})
+            
+            list_sentiment_emoticon_vectors.append(np.array([pos/occ,neg/occ,neu/occ]))
+            list_emoticon_emojis.append(emo)
+
+
+    return data,data_only_emoticons,np.array(list_sentiment_vectors), np.array(list_emojis), np.array(list_sentiment_emoticon_vectors),np.array(list_emoticon_emojis)
+#d , doe = dataframe_to_dictionary()
+
+
+# In[4]:
+
+
+data , data_only_emoticons, list_sentiment_vectors , list_emojis , list_sentiment_emoticon_vectors , list_emoticon_emojis = dataframe_to_dictionary()
+
+
+# In[5]:
 
 
 #calculates vector distance between 2 3-dim sentiment representations of emojis
 def sentiment_vector_dist(v1,v2):
-    #pos_v1 = v1[0]
-    #neg_v1 = v1[1]
-    #neu_v1 = v1[2]
-
-    #pos_v2 = v2[0]
-    #neg_v2 = v2[1]
-    #neu_v2 = v2[2]
-
-    #tmp_dist = float(np.abs(pos_v1-pos_v2))+float(np.abs(neg_v1-neg_v2))+float(np.abs(neu_v1-neu_v2))
-
     #calculates vector distance between 2 3-dim sentiment representations of emojis consisting of positive neutral and negative probabilistic occuring
-    tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2))
+    tmp_dist = np.linalg.norm(np.array(v1)-np.array(v2))  
     return tmp_dist
 
 
-# In[55]:
+# In[6]:
 
 
 #calculates vector representation in a 3dim 0 to 1space of dimension: positive,negative,neutral
-def emoji_to_sentiment_vector(e):
+def emoji_to_sentiment_vector(e, only_emoticons=True):
-    tmp = df[df["Emoji"]==e]
+    """tmp = df[df["Emoji"]==e]    
     #calculate by espacial labeled occurences devided by sum of overall occurences
     pos = tmp["Positive"].values[0]/tmp["Occurrences"].values[0]
     neg = tmp["Negative"].values[0]/tmp["Occurrences"].values[0]
     neu = tmp["Neutral"].values[0]/tmp["Occurrences"].values[0]
     #return as np array
-    return np.array([pos,neg,neu])
+    return np.array([pos,neg,neu])"""
+    if e in (data_only_emoticons if only_emoticons else data):
+        return np.array((data_only_emoticons if only_emoticons else data)[e])
+    return np.array([float('NaN')]*N) 
 
 
-# In[56]:
+# In[7]:
 
 
 #function to call for evaluating two emojis in its sentimental distance
 def emoji_distance(e1,e2):
     sent_v1 = emoji_to_sentiment_vector(e1)
     sent_v2 = emoji_to_sentiment_vector(e2)
-
+    
     d = sentiment_vector_dist(sent_v1,sent_v2)
     return d
 
 
-# In[57]:
+# In[27]:
 
 
-def sentiment_vector_to_emoji(v1):
+def sentiment_vector_to_emoji(v1, only_emoticons=True):
-    #if(len(v1) == 3):
+    #more efficient approach for min distance
-        #set initial values to compare with
+    distances = (list_sentiment_emoticon_vectors if only_emoticons else list_sentiment_vectors) - v1
-        best_emoji = "😐"
+    distances = np.linalg.norm(distances, axis=1)
-        min_distance = 10000
+    #find min entry
+    min_entry = np.argmin(distances)
+    
+    return (list_emoticon_emojis if only_emoticons else list_emojis)[min_entry]
 
-        #compare only with filtred emoticons
+    #version for dics
-        df_filtered = df[df["Unicode block"]=="Emoticons"]
+
-        all_smilies = list(df_filtered["Emoji"])
+    """#set initial values to compare with
-        for e in all_smilies:
+    best_emoji = "😐"
-            v2 = emoji_to_sentiment_vector(e)
+    min_distance = 10000
-            d = sentiment_vector_dist(v1,v2)
+
-            if(d < min_distance):
+    #compare only with filtred emoticons not containing other elements like cars etc.
-                min_distance = d
+    #compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match
-                best_emoji = e
+    for e,v2 in doe.items():
-                #print(str(v1),str(v2),str(min_distance),str(type(v1)),str(type(v2)),e)
+        #v2 = emoji_to_sentiment_vector(e)
+        d = sentiment_vector_dist(v1,v2)
+        if(d < min_distance):
+            min_distance = d
+            best_emoji = e
 
 
-        #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
+    #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
-        return best_emoji
+    return best_emoji"""
 
-    #else:
+    #old version
-        #print("WRONG SENTIMENT VECTOR")
 
+    """#set initial values to compare with
+    best_emoji = "😐"
+    min_distance = 10000
 
-# In[58]:
+    #compare only with filtred emoticons not containing other elements like cars etc.
-
-
-def show_demo():
     df_filtered = df[df["Unicode block"]=="Emoticons"]
     all_smilies = list(df_filtered["Emoji"])
+    #compare for each existing emoticons sentment vector to find the minimal distance equivalent to the best match
+    for e in all_smilies:
+        v2 = emoji_to_sentiment_vector(e)
+        d = sentiment_vector_dist(v1,v2)
+        if(d < min_distance):
+            min_distance = d
+            best_emoji = e
+
+
+    #print("for sentiment vector: "+str(v1)+" the emoji is : "+str(best_emoji)+" with distance of "+str(min_distance)+"!")
+    return best_emoji"""
+
+
+# In[28]:
+
+
+def show_demo_min_distances(only_emoticons = True):
+    #df_filtered = df[df["Unicode block"]=="Emoticons"]
+    all_smilies = list_emoticon_emojis if only_emoticons else list_emojis
 
     d_m = np.zeros(shape=(len(all_smilies),len(all_smilies)))
 
@@ -115,7 +177,7 @@ def show_demo():
 
             d = emoji_distance(e1,e2)
             d_m[c1,c2] = d
-
+            
     for c in range(len(d_m[0])):
         emoji = all_smilies[c]
         row = d_m[c]
@@ -127,20 +189,59 @@ def show_demo():
         for i in r:
             closest+=all_smilies[i]
         print(emoji+": "+closest)
+    
+    """df_filtered = df[df["Unicode block"]=="Emoticons"]
+    all_smilies = list(df_filtered["Emoji"])
+
+    d_m = np.zeros(shape=(len(all_smilies),len(all_smilies)))
+
+    for c1 in range(len(all_smilies)):
+        for c2 in range(len(all_smilies)):
+            e1 = all_smilies[c1]
+            e2 = all_smilies[c2]
+
+            d = emoji_distance(e1,e2)
+            d_m[c1,c2] = d
+            
+    for c in range(len(d_m[0])):
+        emoji = all_smilies[c]
+        row = d_m[c]
+        row_sorted = np.argsort(row)
+        #closest 5
+        r = row_sorted[0:10]
+        #print()
+        closest = ""
+        for i in r:
+            closest+=all_smilies[i]
+        print(emoji+": "+closest)"""
 
 
-# In[60]:
+# In[29]:
 
 
-#show_demo()
+show_demo_min_distances()
 
 
-# In[61]:
+# In[30]:
 
 
 #test bipolar matching entiment vector vs. emoji
-#df_filtered = df[df["Unicode block"]=="Emoticons"]
+#def show_demo_matching_bipolar
-#all_smilies = list(df_filtered["Emoji"])
+#    df_filtered = df[df["Unicode block"]=="Emoticons"]
-#for e in all_smilies:
+#    all_smilies = list(df_filtered["Emoji"])
-#    v2 = emoji_to_sentiment_vector(e)
+#    for e in all_smilies:
-#    sentiment_vector_to_emoji(v2)
+#        v2 = emoji_to_sentiment_vector(e)
+#        sentiment_vector_to_emoji(v2)
+
+
+# In[36]:
+
+
+[(e,sentiment_vector_to_emoji(emoji_to_sentiment_vector(e,only_emoticons=False))) for e in list_emojis]
+
+
+# In[26]:
+
+
+sentiment_vector_to_emoji(np.array([ 0.72967448,  0.05173769,  0.21858783]))
+