jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Naive Approach updated (emojis_to_consider, gibt Scores zurück)

Browse Source
This commit is contained in:
Maren 2018-06-26 18:32:04 +02:00
parent 60bac91655
commit 2706e19aec
1 changed files with 40 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
Project/naive_approach/naive_approach.py
View File

@ -37,7 +37,7 @@ def stemming(messages):
# * compare words to emoji descriptions # * compare words to emoji descriptions
def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng'): def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng', emojis_to_consider="all"):
tokenized_sentence = word_tokenize(sentence) tokenized_sentence = word_tokenize(sentence)
n = len(tokenized_sentence) n = len(tokenized_sentence)
@ -77,10 +77,7 @@ def evaluate_sentence(sentence, table, description_key = 'description', lang = '
# load and preprocess data # load and preprocess data
# emojis_to_consider can be either a list or "all" # emojis_to_consider can be either a list or "all"
def prepareData(stemming=False, emojis_to_consider="all"): def prepareData(stemming=False):
table.head()
if(stemming): if(stemming):
table['description'] = stemming(table['description']) table['description'] = stemming(table['description'])
@ -88,9 +85,8 @@ def prepareData(stemming=False, emojis_to_consider="all"):
lookup = {} lookup = {}
emoji_set = [] emoji_set = []
for index, row in table.iterrows(): for index, row in table.iterrows():
if(emojis_to_consider=="all" or (type(emojis_to_consider)==list and row['character'] in emojis_to_consider)): lookup[index] = row['character']
lookup[index] = row['character'] emoji_set.append(row['character'])
emoji_set.append(row['character'])
emoji_set = set(emoji_set) emoji_set = set(emoji_set)
@ -99,30 +95,44 @@ def prepareData(stemming=False, emojis_to_consider="all"):
# make a prediction for an input sentence # make a prediction for an input sentence
def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", description_key='description', lang = 'eng', n=10, t=0.9): def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", description_key='description', lang = 'eng', n=10, t=0.9):
result = evaluate_sentence(sentence, table, description_key, lang) result = evaluate_sentence(sentence, table, description_key, lang, emojis_to_consider=emojis_to_consider)
if(criteria=="summed"): try:
indexes = np.argsort([-np.sum(x) for x in result])[0:n] if(criteria=="summed"):
elif (criteria=="max_val"): resultValues = [-np.sum(x) for x in result]
indexes = np.argsort([-np.max(x) for x in result])[0:n] elif (criteria=="max_val"):
elif(criteria=="avg"): resultValues = [-np.max(x) for x in result]
indexes = np.argsort([-np.mean(x) for x in result])[0:n] elif(criteria=="avg"):
else: resultValues = [-np.mean(x) for x in result]
indexes= np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n] else:
resultValues = [-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result]
indexes = np.argsort(resultValues)
results = np.sort(resultValues)
if(emojis_to_consider!="all"): if (emojis_to_consider != "all" and type(emojis_to_consider) == list):
for i in indexes: indexes2 = []
if (i not in lookup): results2 = []
indexes = np.delete(indexes, [i]) for i in range(len(indexes)):
if lookup[indexes[i]] in emojis_to_consider:
indexes2.append(indexes[i])
results2.append(results[i])
indexes = indexes2
results = results2
indexes = indexes[0:n]
results = results[0:n]
# build a result table # build a result table
table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ]
table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) table_frame = pd.DataFrame(table_array, columns=[criteria, 'description'])
#display(table_frame) #display(table_frame)
return list(table_frame[criteria]), results
except ZeroDivisionError as err:
print("There seems to be a problem with the input format. Please enter a nonempty string")
return list(table_frame[criteria])
#predict("I like to travel by train", description_key='description' , lang='eng') #predict("I like to travel by train", description_key='description' , lang='eng')