jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Möglichkeit zwischen wordnet und Word2Vec zu wechseln

Browse Source
This commit is contained in:
Maren 2018-07-21 10:40:29 +02:00
parent 96e4606880
commit 7bc2215ec9
1 changed files with 27 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Project/naive_approach/naive_approach.py
View File

@ -18,7 +18,7 @@ import pprint
from gensim.models import Word2Vec, KeyedVectors
# # Naive Approach
table = pd.read_csv('../Tools/emoji_descriptions.csv')
table = pd.read_csv('../Tools/emoji_descriptions_preprocessed.csv')
##Store table in the format:
## { index: [emoji, description]}
@ -41,9 +41,11 @@ def stemming(message):
# * compare words to emoji descriptions
def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all", stem=True):
def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', emojis_to_consider="all",\
stem=True, use_wordnet=True):
# assumes there is a trained w2v model stored in the same directory!
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
if use_wordnet==False:
wv = KeyedVectors.load(str(__location__)+"/word2vec.model", mmap='r')
if (stem):
@ -59,6 +61,20 @@ def evaluate_sentence(sentence, description_key = 'description', lang = 'eng', e
mat = np.zeros(shape=(m,n))
for i in range(len(emoji_tokens)):
for j in range(len(tokenized_sentence)):
if use_wordnet:
syn1 = wordnet.synsets(emoji_tokens[i],lang=lang)
if len(syn1) == 0:
continue
w1 = syn1[0]
#print(j, tokenized_sentence)
syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang)
if len(syn2) == 0:
continue
w2 = syn2[0]
val = w1.wup_similarity(w2)
if val is None:
continue
else:
try:
val = wv.similarity(emoji_tokens[i], tokenized_sentence[j])
except KeyError:
@ -96,9 +112,11 @@ def prepareData(stem=True, lower=True):
return lookup
# make a prediction for an input sentence
def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", lang = 'eng', n=10, t=0.9):
# use_wordnet=True --> use wordnet similarites, otherwise use Word2Vec
def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", lang = 'eng',\
use_wordnet=True, n=10, t=0.9):
result = evaluate_sentence(sentence, lang, emojis_to_consider=emojis_to_consider)
result = evaluate_sentence(sentence, lang, emojis_to_consider=emojis_to_consider, use_wordnet=use_wordnet)
try:
if(criteria=="summed"):