Naive Approach updated (emojis_to_consider, gibt Scores zurück)
This commit is contained in:
		| @ -37,7 +37,7 @@ def stemming(messages): | ||||
|  | ||||
|  | ||||
| # * compare words to emoji descriptions | ||||
| def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng'): | ||||
| def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng', emojis_to_consider="all"): | ||||
|      | ||||
|     tokenized_sentence = word_tokenize(sentence) | ||||
|     n = len(tokenized_sentence) | ||||
| @ -77,10 +77,7 @@ def evaluate_sentence(sentence, table, description_key = 'description', lang = ' | ||||
|  | ||||
| # load and preprocess data | ||||
| # emojis_to_consider can be either a list or "all" | ||||
| def prepareData(stemming=False, emojis_to_consider="all"): | ||||
|  | ||||
|     table.head() | ||||
|      | ||||
| def prepareData(stemming=False): | ||||
|     if(stemming): | ||||
|         table['description'] = stemming(table['description']) | ||||
|      | ||||
| @ -88,9 +85,8 @@ def prepareData(stemming=False, emojis_to_consider="all"): | ||||
|     lookup = {} | ||||
|     emoji_set = [] | ||||
|     for index, row in table.iterrows(): | ||||
|         if(emojis_to_consider=="all" or (type(emojis_to_consider)==list and row['character'] in emojis_to_consider)): | ||||
|             lookup[index] = row['character'] | ||||
|             emoji_set.append(row['character']) | ||||
|         lookup[index] = row['character'] | ||||
|         emoji_set.append(row['character']) | ||||
|  | ||||
|     emoji_set = set(emoji_set) | ||||
|      | ||||
| @ -99,30 +95,44 @@ def prepareData(stemming=False, emojis_to_consider="all"): | ||||
| # make a prediction for an input sentence | ||||
| def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", description_key='description', lang = 'eng', n=10, t=0.9): | ||||
|  | ||||
|     result = evaluate_sentence(sentence, table, description_key, lang) | ||||
|     result = evaluate_sentence(sentence, table, description_key, lang, emojis_to_consider=emojis_to_consider) | ||||
|      | ||||
|     if(criteria=="summed"): | ||||
|         indexes = np.argsort([-np.sum(x) for x in result])[0:n] | ||||
|     elif (criteria=="max_val"): | ||||
|         indexes = np.argsort([-np.max(x) for x in result])[0:n] | ||||
|     elif(criteria=="avg"): | ||||
|         indexes = np.argsort([-np.mean(x) for x in result])[0:n] | ||||
|     else: | ||||
|         indexes= np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]  | ||||
|     try: | ||||
|         if(criteria=="summed"): | ||||
|             resultValues = [-np.sum(x) for x in result] | ||||
|         elif (criteria=="max_val"): | ||||
|             resultValues = [-np.max(x) for x in result] | ||||
|         elif(criteria=="avg"): | ||||
|             resultValues = [-np.mean(x) for x in result] | ||||
|         else: | ||||
|             resultValues = [-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result] | ||||
|         indexes = np.argsort(resultValues) | ||||
|         results = np.sort(resultValues) | ||||
|          | ||||
|         if (emojis_to_consider != "all" and type(emojis_to_consider) == list): | ||||
|             indexes2 = [] | ||||
|             results2 = [] | ||||
|             for i in range(len(indexes)): | ||||
|                 if lookup[indexes[i]] in emojis_to_consider: | ||||
|                     indexes2.append(indexes[i]) | ||||
|                     results2.append(results[i]) | ||||
|             indexes = indexes2 | ||||
|             results = results2 | ||||
|         indexes = indexes[0:n] | ||||
|         results = results[0:n] | ||||
|          | ||||
|         # build a result table | ||||
|         table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] | ||||
|            | ||||
|         table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) | ||||
|          | ||||
|         #display(table_frame) | ||||
|          | ||||
|         return list(table_frame[criteria]), results | ||||
|      | ||||
|     if(emojis_to_consider!="all"): | ||||
|         for i in indexes: | ||||
|             if (i not in lookup): | ||||
|                 indexes = np.delete(indexes, [i]) | ||||
|      | ||||
|     # build a result table | ||||
|     table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] | ||||
|        | ||||
|     table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) | ||||
|      | ||||
|     #display(table_frame) | ||||
|      | ||||
|     return list(table_frame[criteria]) | ||||
|     except ZeroDivisionError as err: | ||||
|         print("There seems to be a problem with the input format. Please enter a nonempty string") | ||||
|  | ||||
|  | ||||
| #predict("I like to travel by train", description_key='description' , lang='eng') | ||||
|  | ||||
|  | ||||
		Reference in New Issue
	
	Block a user