Naive approach mit word2Vec similarities
This commit is contained in:
		| @ -15,6 +15,7 @@ from nltk.corpus import wordnet | ||||
| import math | ||||
| import pprint | ||||
|  | ||||
| from gensim.models import Word2Vec, KeyedVectors | ||||
|  | ||||
| # # Naive Approach | ||||
| table = pd.read_csv('../Tools/emoji_descriptions.csv') | ||||
| @ -29,25 +30,25 @@ for index, row in table.iterrows(): | ||||
| # Helper functions | ||||
| ####################### | ||||
|  | ||||
| def stemming(messages): | ||||
|     stemmed_messages = [] | ||||
| def stemming(message): | ||||
|     ps = PorterStemmer() | ||||
|     for m in messages: | ||||
|         words = word_tokenize(m) | ||||
|         sm = [] | ||||
|         for w in words: | ||||
|             sm.append(ps.stem(w)) | ||||
|         m = (" ").join(sm) | ||||
|         stemmed_messages.append(m) | ||||
|     return stemmed_messages | ||||
|     words = word_tokenize(message) | ||||
|     sm = [] | ||||
|     for w in words: | ||||
|         sm.append(ps.stem(w)) | ||||
|     stemmed_message = (" ").join(sm) | ||||
|     return stemmed_message | ||||
|  | ||||
|  | ||||
| # * compare words to emoji descriptions | ||||
| def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all"): | ||||
| def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all", stem=True): | ||||
|     # assumes there is a trained w2v model stored in the same directory! | ||||
|     wv = KeyedVectors.load("word2vec.model", mmap='r') | ||||
|      | ||||
|     if (stem): | ||||
|         sentence = stemming(sentence) | ||||
|     tokenized_sentence = word_tokenize(sentence) | ||||
|     n = len(tokenized_sentence) | ||||
|     l = table.shape[0] | ||||
|     matrix_list = [] | ||||
|      | ||||
|     for index in tableDict.keys(): | ||||
| @ -57,20 +58,11 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e | ||||
|         mat = np.zeros(shape=(m,n)) | ||||
|         for i in range(len(emoji_tokens)): | ||||
|             for j in range(len(tokenized_sentence)): | ||||
|                 syn1 = wordnet.synsets(emoji_tokens[i],lang=lang) | ||||
|                 if len(syn1) == 0: | ||||
|                     continue | ||||
|                 w1 = syn1[0] | ||||
|                 #print(j, tokenized_sentence) | ||||
|                 syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang) | ||||
|                 if len(syn2) == 0: | ||||
|                     continue | ||||
|                 w2 = syn2[0] | ||||
|                 val = w1.wup_similarity(w2) | ||||
|                 if val is None: | ||||
|                 try: | ||||
|                     val = wv.similarity(emoji_tokens[i], tokenized_sentence[j]) | ||||
|                 except KeyError: | ||||
|                     continue | ||||
|                 mat[i,j] = val | ||||
|         #print(row['character'], mat) | ||||
|         matrix_list.append(mat) | ||||
|              | ||||
|     return matrix_list | ||||
| @ -83,10 +75,13 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e | ||||
|  | ||||
| # load and preprocess data | ||||
| # emojis_to_consider can be either a list or "all" | ||||
| def prepareData(stemming=False): | ||||
|     if(stemming): | ||||
| def prepareData(stem=True, lower=True): | ||||
|     if(stem): | ||||
|         for index in tableDict.keys(): | ||||
|             tableDict[index][1] = stemming(tableDict[index][1]) | ||||
|     if(lower): | ||||
|         for index in tableDict.keys(): | ||||
|             tableDict[index][1] = tableDict[index][1].lower() | ||||
|      | ||||
|     #collect the emojis | ||||
|     lookup = {} | ||||
|  | ||||
							
								
								
									
										
											BIN
										
									
								
								Project/naive_approach/word2vec.model
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										
											BIN
										
									
								
								Project/naive_approach/word2vec.model
									
									
									
									
									
										Normal file
									
								
							
										
											Binary file not shown.
										
									
								
							
		Reference in New Issue
	
	Block a user