filtered out empty labels
This commit is contained in:
		| @ -235,12 +235,38 @@ class sample_data_manager(object): | ||||
|  | ||||
|              # replacing keywords. TODO: maybe these information can be extracted and used | ||||
|             plain_text_i = plain_text_i.str.replace("(<EMOJI>|<USER>|<HASHTAG>)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","") | ||||
|              | ||||
|             # filter empty labels | ||||
|             empty_labels = [] | ||||
|              | ||||
|             for e in emojis_i: | ||||
|                 if len(e) < 1: | ||||
|                     empty_labels.append(True) | ||||
|                 else: | ||||
|                     empty_labels.append(False) | ||||
|                      | ||||
|             empty_labels = np.array(empty_labels, dtype=np.bool_) | ||||
|              | ||||
|             plain_text_i = plain_text_i[np.invert(empty_labels)] | ||||
|             emojis_i = emojis_i[np.invert(empty_labels)] | ||||
|              | ||||
|             print("ignored " + str(np.sum(empty_labels)) + " empty labels") | ||||
|  | ||||
|             if not emoji_mean: | ||||
|                 # so far filtering for the latest emoji. TODO: maybe there are also better approaches | ||||
|                 labels_i = emoji2sent([latest(e) for e in emojis_i], only_emoticons=only_emoticons ) | ||||
|             else: | ||||
|                 labels_i = np.array([np.mean(emoji2sent(e, only_emoticons=only_emoticons), axis=0).tolist() for e in emojis_i]) | ||||
|                 tmp = [np.nanmean(emoji2sent(e, only_emoticons=only_emoticons), axis=0, dtype=float) for e in emojis_i] | ||||
|                 c = 0 | ||||
|                 for t in tmp: | ||||
|                      | ||||
|                     if str(type(t)) != "<class 'numpy.ndarray'>": | ||||
|                         print(t, type(t)) | ||||
|                         print(emojis_i[c]) | ||||
|                         print(emoji2sent(emojis_i[c], only_emoticons=only_emoticons)) | ||||
|                     c += 1 | ||||
|  | ||||
|                 labels_i = np.array(tmp, dtype=float) | ||||
|  | ||||
|             # and filter out all samples we have no label for: | ||||
|             wrong_labels = np.isnan(np.linalg.norm(labels_i, axis=1)) | ||||
| @ -678,4 +704,4 @@ class trainer(object): | ||||
|             self.sdm.create_train_test_split() | ||||
|         return self.pm.predict(self.sdm.Xt, use_lemmatization=False, use_stemming=False), self.sdm.yt | ||||
|  | ||||
|      | ||||
|      | ||||
|  | ||||
		Reference in New Issue
	
	Block a user