stemming als parameter und anpassungen für evaluation

This commit is contained in:
Carsten
2018-07-23 12:09:36 +02:00
parent b99b2006f7
commit d899e2f069
2 changed files with 158 additions and 51 deletions

View File

@ -18,7 +18,7 @@ import pprint
from gensim.models import Word2Vec, KeyedVectors
# # Naive Approach
table = pd.read_csv('../Tools/emoji_descriptions_preprocessed.csv')
table = pd.read_csv('../Tools/emoji_descriptions_preprocessed.csv', delimiter = ";")
##Store table in the format:
## { index: [emoji, description]}
@ -49,7 +49,7 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e
if embeddings=="word2Vec":
wv = KeyedVectors.load(str(__location__)+"/word2vec.model", mmap='r')
elif embeddings=="fastText":
wv = KeyedVectors.load("/fastTextVectors.kv", mmap='r')
wv = KeyedVectors.load(str(__location__)+"/fastTextVectors.kv", mmap='r')
if (stem):
sentence = stemming(sentence)
@ -118,9 +118,9 @@ def prepareData(stem=True, lower=True):
# make a prediction for an input sentence
# embeddings = ["wordnet", "word2Vec", "fastText"]
def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", lang = 'eng',\
embeddings="wordnet", n=10, t=0.9):
embeddings="wordnet", n=10, t=0.9, stem = True):
result = evaluate_sentence(sentence, lang, emojis_to_consider=emojis_to_consider, embeddings=embeddings)
result = evaluate_sentence(sentence, lang, emojis_to_consider=emojis_to_consider, embeddings=embeddings, stem = stem)
try:
if(criteria=="summed"):