added possibility to filter by emoji list
This commit is contained in:
		| @ -145,7 +145,7 @@ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "43f80c7d9c024a57b4009b77296f1ab0", | ||||
|        "model_id": "31e69854333f4c599b037b6c27f30f20", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
| @ -175,7 +175,8 @@ | ||||
|     "               [\n", | ||||
|     "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
|     "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\"),\n", | ||||
|     "                   (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\")\n", | ||||
|     "                   (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\"),\n", | ||||
|     "                   (widgets.Text(value=\"\"), \"custom_emojis\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Button(disabled=True),\"load_data\")\n", | ||||
| @ -449,6 +450,8 @@ | ||||
|     "        \n", | ||||
|     "        emoji_mean = shown_widgets[\"label_criteria\"].value == \"mean\"\n", | ||||
|     "        \n", | ||||
|     "        custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n", | ||||
|     "        \n", | ||||
|     "        sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", | ||||
|     "                                                    n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", | ||||
|     "                                                    file_range=range(r[0], r[1]),\n", | ||||
| @ -456,7 +459,8 @@ | ||||
|     "                                                    read_progress_callback=p_r.update,\n", | ||||
|     "                                                    stem_progress_callback=p_s.update if lemm_and_stemm else None,\n", | ||||
|     "                                                    apply_stemming = lemm_and_stemm,\n", | ||||
|     "                                                    emoji_mean=emoji_mean)\n", | ||||
|     "                                                    emoji_mean=emoji_mean,\n", | ||||
|     "                                                    custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None)\n", | ||||
|     "        shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", | ||||
|     "        \n", | ||||
|     "        \n", | ||||
| @ -679,6 +683,15 @@ | ||||
|     "shown_widgets[\"test_input\"].observe(test_input)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "sdm" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|  | ||||
| @ -164,7 +164,7 @@ def batch_lemm(sentences): | ||||
|  | ||||
| class sample_data_manager(object): | ||||
|     @staticmethod | ||||
|     def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False): | ||||
|     def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None): | ||||
|         """ | ||||
|         generate, read and process train data in one step. | ||||
|          | ||||
| @ -184,7 +184,10 @@ class sample_data_manager(object): | ||||
|          | ||||
|         sdm.generate_emoji_count_and_weights() | ||||
|          | ||||
|         if n_top_emojis > 0: | ||||
|         if custom_target_emojis is not None: | ||||
|             sdm.filter_by_emoji_list(custom_target_emojis) | ||||
|  | ||||
|         elif n_top_emojis > 0: | ||||
|             sdm.filter_by_top_emojis(n_top=n_top_emojis) | ||||
|          | ||||
|         if n_kmeans_cluster > 0: | ||||
| @ -393,6 +396,16 @@ class sample_data_manager(object): | ||||
|         self.emojis = self.emojis[in_top] | ||||
|         print("remaining samples after top emoji filtering: ", len(self.labels)) | ||||
|      | ||||
|     def filter_by_emoji_list(self, custom_target_emojis): | ||||
|  | ||||
|         assert self.labels is not None | ||||
|  | ||||
|         in_list = [edist.sentiment_vector_to_emoji(x) in custom_target_emojis for x in self.labels] | ||||
|         self.labels = self.labels[in_list] | ||||
|         self.plain_text = self.plain_text[in_list] | ||||
|         self.emojis = self.emojis[in_list] | ||||
|         print("remaining samples after custom emoji filtering: ", len(self.labels)) | ||||
|  | ||||
|     def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5): | ||||
|         """ | ||||
|         generate binary labels using kmeans. | ||||
|  | ||||
		Reference in New Issue
	
	Block a user