jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filtered out empty labels

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-19 15:58:06 +02:00
parent 406460d031
commit 8300ed4510
1 changed files with 28 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Project/simple_approach/simple_twitter_learning.py
View File

@ -235,12 +235,38 @@ class sample_data_manager(object):
# replacing keywords. TODO: maybe these information can be extracted and used
plain_text_i = plain_text_i.str.replace("(<EMOJI>|<USER>|<HASHTAG>)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","")
# filter empty labels
empty_labels = []
for e in emojis_i:
if len(e) < 1:
empty_labels.append(True)
else:
empty_labels.append(False)
empty_labels = np.array(empty_labels, dtype=np.bool_)
plain_text_i = plain_text_i[np.invert(empty_labels)]
emojis_i = emojis_i[np.invert(empty_labels)]
print("ignored " + str(np.sum(empty_labels)) + " empty labels")
if not emoji_mean:
# so far filtering for the latest emoji. TODO: maybe there are also better approaches
labels_i = emoji2sent([latest(e) for e in emojis_i], only_emoticons=only_emoticons )
else:
labels_i = np.array([np.mean(emoji2sent(e, only_emoticons=only_emoticons), axis=0).tolist() for e in emojis_i])
tmp = [np.nanmean(emoji2sent(e, only_emoticons=only_emoticons), axis=0, dtype=float) for e in emojis_i]
c = 0
for t in tmp:
if str(type(t)) != "<class 'numpy.ndarray'>":
print(t, type(t))
print(emojis_i[c])
print(emoji2sent(emojis_i[c], only_emoticons=only_emoticons))
c += 1
labels_i = np.array(tmp, dtype=float)
# and filter out all samples we have no label for:
wrong_labels = np.isnan(np.linalg.norm(labels_i, axis=1))
@ -678,4 +704,4 @@ class trainer(object):
self.sdm.create_train_test_split()
return self.pm.predict(self.sdm.Xt, use_lemmatization=False, use_stemming=False), self.sdm.yt