From 7bc2215ec960041b7ae63ec3824800e72bc2eb4f Mon Sep 17 00:00:00 2001 From: Maren Date: Sat, 21 Jul 2018 10:40:29 +0200 Subject: [PATCH] =?UTF-8?q?M=C3=B6glichkeit=20zwischen=20wordnet=20und=20W?= =?UTF-8?q?ord2Vec=20zu=20wechseln?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Project/naive_approach/naive_approach.py | 36 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/Project/naive_approach/naive_approach.py b/Project/naive_approach/naive_approach.py index e08cd47..efea7fe 100644 --- a/Project/naive_approach/naive_approach.py +++ b/Project/naive_approach/naive_approach.py @@ -18,7 +18,7 @@ import pprint from gensim.models import Word2Vec, KeyedVectors # # Naive Approach -table = pd.read_csv('../Tools/emoji_descriptions.csv') +table = pd.read_csv('../Tools/emoji_descriptions_preprocessed.csv') ##Store table in the format: ## { index: [emoji, description]} @@ -41,10 +41,12 @@ def stemming(message): # * compare words to emoji descriptions -def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all", stem=True): +def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all",\ + stem=True, use_wordnet=True): # assumes there is a trained w2v model stored in the same directory! __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) - wv = KeyedVectors.load(str(__location__)+"/word2vec.model", mmap='r') + if use_wordnet==False: + wv = KeyedVectors.load(str(__location__)+"/word2vec.model", mmap='r') if (stem): sentence = stemming(sentence) @@ -59,10 +61,24 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e mat = np.zeros(shape=(m,n)) for i in range(len(emoji_tokens)): for j in range(len(tokenized_sentence)): - try: - val = wv.similarity(emoji_tokens[i], tokenized_sentence[j]) - except KeyError: - continue + if use_wordnet: + syn1 = wordnet.synsets(emoji_tokens[i],lang=lang) + if len(syn1) == 0: + continue + w1 = syn1[0] + #print(j, tokenized_sentence) + syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang) + if len(syn2) == 0: + continue + w2 = syn2[0] + val = w1.wup_similarity(w2) + if val is None: + continue + else: + try: + val = wv.similarity(emoji_tokens[i], tokenized_sentence[j]) + except KeyError: + continue mat[i,j] = val matrix_list.append(mat) @@ -96,9 +112,11 @@ def prepareData(stem=True, lower=True): return lookup # make a prediction for an input sentence -def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", lang = 'eng', n=10, t=0.9): +# use_wordnet=True --> use wordnet similarites, otherwise use Word2Vec +def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", lang = 'eng',\ + use_wordnet=True, n=10, t=0.9): - result = evaluate_sentence(sentence, lang, emojis_to_consider=emojis_to_consider) + result = evaluate_sentence(sentence, lang, emojis_to_consider=emojis_to_consider, use_wordnet=use_wordnet) try: if(criteria=="summed"):