Naive approach mit word2Vec similarities

This commit is contained in:
aikira7 2018-07-07 18:25:17 +02:00
parent abd61c35d1
commit bc94107353
2 changed files with 21 additions and 26 deletions

View File

@ -15,6 +15,7 @@ from nltk.corpus import wordnet
import math
import pprint
from gensim.models import Word2Vec, KeyedVectors
# # Naive Approach
table = pd.read_csv('../Tools/emoji_descriptions.csv')
@ -29,25 +30,25 @@ for index, row in table.iterrows():
# Helper functions
#######################
def stemming(messages):
stemmed_messages = []
def stemming(message):
ps = PorterStemmer()
for m in messages:
words = word_tokenize(m)
sm = []
for w in words:
sm.append(ps.stem(w))
m = (" ").join(sm)
stemmed_messages.append(m)
return stemmed_messages
words = word_tokenize(message)
sm = []
for w in words:
sm.append(ps.stem(w))
stemmed_message = (" ").join(sm)
return stemmed_message
# * compare words to emoji descriptions
def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all"):
def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all", stem=True):
# assumes there is a trained w2v model stored in the same directory!
wv = KeyedVectors.load("word2vec.model", mmap='r')
if (stem):
sentence = stemming(sentence)
tokenized_sentence = word_tokenize(sentence)
n = len(tokenized_sentence)
l = table.shape[0]
matrix_list = []
for index in tableDict.keys():
@ -57,20 +58,11 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e
mat = np.zeros(shape=(m,n))
for i in range(len(emoji_tokens)):
for j in range(len(tokenized_sentence)):
syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)
if len(syn1) == 0:
continue
w1 = syn1[0]
#print(j, tokenized_sentence)
syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)
if len(syn2) == 0:
continue
w2 = syn2[0]
val = w1.wup_similarity(w2)
if val is None:
try:
val = wv.similarity(emoji_tokens[i], tokenized_sentence[j])
except KeyError:
continue
mat[i,j] = val
#print(row['character'], mat)
matrix_list.append(mat)
return matrix_list
@ -83,10 +75,13 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e
# load and preprocess data
# emojis_to_consider can be either a list or "all"
def prepareData(stemming=False):
if(stemming):
def prepareData(stem=True, lower=True):
if(stem):
for index in tableDict.keys():
tableDict[index][1] = stemming(tableDict[index][1])
if(lower):
for index in tableDict.keys():
tableDict[index][1] = tableDict[index][1].lower()
#collect the emojis
lookup = {}

Binary file not shown.