diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index 102b790..8bfedae 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -235,12 +235,38 @@ class sample_data_manager(object): # replacing keywords. TODO: maybe these information can be extracted and used plain_text_i = plain_text_i.str.replace("(||)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","") + + # filter empty labels + empty_labels = [] + + for e in emojis_i: + if len(e) < 1: + empty_labels.append(True) + else: + empty_labels.append(False) + + empty_labels = np.array(empty_labels, dtype=np.bool_) + + plain_text_i = plain_text_i[np.invert(empty_labels)] + emojis_i = emojis_i[np.invert(empty_labels)] + + print("ignored " + str(np.sum(empty_labels)) + " empty labels") if not emoji_mean: # so far filtering for the latest emoji. TODO: maybe there are also better approaches labels_i = emoji2sent([latest(e) for e in emojis_i], only_emoticons=only_emoticons ) else: - labels_i = np.array([np.mean(emoji2sent(e, only_emoticons=only_emoticons), axis=0).tolist() for e in emojis_i]) + tmp = [np.nanmean(emoji2sent(e, only_emoticons=only_emoticons), axis=0, dtype=float) for e in emojis_i] + c = 0 + for t in tmp: + + if str(type(t)) != "": + print(t, type(t)) + print(emojis_i[c]) + print(emoji2sent(emojis_i[c], only_emoticons=only_emoticons)) + c += 1 + + labels_i = np.array(tmp, dtype=float) # and filter out all samples we have no label for: wrong_labels = np.isnan(np.linalg.norm(labels_i, axis=1)) @@ -678,4 +704,4 @@ class trainer(object): self.sdm.create_train_test_split() return self.pm.predict(self.sdm.Xt, use_lemmatization=False, use_stemming=False), self.sdm.yt - \ No newline at end of file +