diff --git a/Project/naive_approach/naive_approach.py b/Project/naive_approach/naive_approach.py index 809a2f2..3c911a5 100644 --- a/Project/naive_approach/naive_approach.py +++ b/Project/naive_approach/naive_approach.py @@ -15,6 +15,7 @@ from nltk.corpus import wordnet import math import pprint +from gensim.models import Word2Vec, KeyedVectors # # Naive Approach table = pd.read_csv('../Tools/emoji_descriptions.csv') @@ -29,25 +30,25 @@ for index, row in table.iterrows(): # Helper functions ####################### -def stemming(messages): - stemmed_messages = [] +def stemming(message): ps = PorterStemmer() - for m in messages: - words = word_tokenize(m) - sm = [] - for w in words: - sm.append(ps.stem(w)) - m = (" ").join(sm) - stemmed_messages.append(m) - return stemmed_messages + words = word_tokenize(message) + sm = [] + for w in words: + sm.append(ps.stem(w)) + stemmed_message = (" ").join(sm) + return stemmed_message # * compare words to emoji descriptions -def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all"): +def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all", stem=True): + # assumes there is a trained w2v model stored in the same directory! + wv = KeyedVectors.load("word2vec.model", mmap='r') + if (stem): + sentence = stemming(sentence) tokenized_sentence = word_tokenize(sentence) n = len(tokenized_sentence) - l = table.shape[0] matrix_list = [] for index in tableDict.keys(): @@ -57,20 +58,11 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e mat = np.zeros(shape=(m,n)) for i in range(len(emoji_tokens)): for j in range(len(tokenized_sentence)): - syn1 = wordnet.synsets(emoji_tokens[i],lang=lang) - if len(syn1) == 0: - continue - w1 = syn1[0] - #print(j, tokenized_sentence) - syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang) - if len(syn2) == 0: - continue - w2 = syn2[0] - val = w1.wup_similarity(w2) - if val is None: + try: + val = wv.similarity(emoji_tokens[i], tokenized_sentence[j]) + except KeyError: continue mat[i,j] = val - #print(row['character'], mat) matrix_list.append(mat) return matrix_list @@ -83,10 +75,13 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e # load and preprocess data # emojis_to_consider can be either a list or "all" -def prepareData(stemming=False): - if(stemming): +def prepareData(stem=True, lower=True): + if(stem): for index in tableDict.keys(): tableDict[index][1] = stemming(tableDict[index][1]) + if(lower): + for index in tableDict.keys(): + tableDict[index][1] = tableDict[index][1].lower() #collect the emojis lookup = {} diff --git a/Project/naive_approach/word2vec.model b/Project/naive_approach/word2vec.model new file mode 100644 index 0000000..9ce773d Binary files /dev/null and b/Project/naive_approach/word2vec.model differ