From 2706e19aec1f94ca3ca57c0f5071ec0fd866ddfb Mon Sep 17 00:00:00 2001 From: Maren Date: Tue, 26 Jun 2018 18:32:04 +0200 Subject: [PATCH] =?UTF-8?q?Naive=20Approach=20updated=20(emojis=5Fto=5Fcon?= =?UTF-8?q?sider,=20gibt=20Scores=20zur=C3=BCck)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Project/naive_approach/naive_approach.py | 70 ++++++++++++++---------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/Project/naive_approach/naive_approach.py b/Project/naive_approach/naive_approach.py index bf38f8a..59ca5e7 100644 --- a/Project/naive_approach/naive_approach.py +++ b/Project/naive_approach/naive_approach.py @@ -37,7 +37,7 @@ def stemming(messages): # * compare words to emoji descriptions -def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng'): +def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng', emojis_to_consider="all"): tokenized_sentence = word_tokenize(sentence) n = len(tokenized_sentence) @@ -77,10 +77,7 @@ def evaluate_sentence(sentence, table, description_key = 'description', lang = ' # load and preprocess data # emojis_to_consider can be either a list or "all" -def prepareData(stemming=False, emojis_to_consider="all"): - - table.head() - +def prepareData(stemming=False): if(stemming): table['description'] = stemming(table['description']) @@ -88,9 +85,8 @@ def prepareData(stemming=False, emojis_to_consider="all"): lookup = {} emoji_set = [] for index, row in table.iterrows(): - if(emojis_to_consider=="all" or (type(emojis_to_consider)==list and row['character'] in emojis_to_consider)): - lookup[index] = row['character'] - emoji_set.append(row['character']) + lookup[index] = row['character'] + emoji_set.append(row['character']) emoji_set = set(emoji_set) @@ -99,30 +95,44 @@ def prepareData(stemming=False, emojis_to_consider="all"): # make a prediction for an input sentence def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", description_key='description', lang = 'eng', n=10, t=0.9): - result = evaluate_sentence(sentence, table, description_key, lang) + result = evaluate_sentence(sentence, table, description_key, lang, emojis_to_consider=emojis_to_consider) - if(criteria=="summed"): - indexes = np.argsort([-np.sum(x) for x in result])[0:n] - elif (criteria=="max_val"): - indexes = np.argsort([-np.max(x) for x in result])[0:n] - elif(criteria=="avg"): - indexes = np.argsort([-np.mean(x) for x in result])[0:n] - else: - indexes= np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n] + try: + if(criteria=="summed"): + resultValues = [-np.sum(x) for x in result] + elif (criteria=="max_val"): + resultValues = [-np.max(x) for x in result] + elif(criteria=="avg"): + resultValues = [-np.mean(x) for x in result] + else: + resultValues = [-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result] + indexes = np.argsort(resultValues) + results = np.sort(resultValues) + + if (emojis_to_consider != "all" and type(emojis_to_consider) == list): + indexes2 = [] + results2 = [] + for i in range(len(indexes)): + if lookup[indexes[i]] in emojis_to_consider: + indexes2.append(indexes[i]) + results2.append(results[i]) + indexes = indexes2 + results = results2 + indexes = indexes[0:n] + results = results[0:n] + + # build a result table + table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] + + table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) + + #display(table_frame) + + return list(table_frame[criteria]), results - if(emojis_to_consider!="all"): - for i in indexes: - if (i not in lookup): - indexes = np.delete(indexes, [i]) - - # build a result table - table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] - - table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) - - #display(table_frame) - - return list(table_frame[criteria]) + except ZeroDivisionError as err: + print("There seems to be a problem with the input format. Please enter a nonempty string") + #predict("I like to travel by train", description_key='description' , lang='eng')