diff --git a/Project/simple_approach/Continous_Learner.ipynb b/Project/simple_approach/Continous_Learner.ipynb index 337cc31..d561cc8 100644 --- a/Project/simple_approach/Continous_Learner.ipynb +++ b/Project/simple_approach/Continous_Learner.ipynb @@ -145,7 +145,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "43f80c7d9c024a57b4009b77296f1ab0", + "model_id": "31e69854333f4c599b037b6c27f30f20", "version_major": 2, "version_minor": 0 }, @@ -175,7 +175,8 @@ " [\n", " (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", " (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\"),\n", - " (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\")\n", + " (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\"),\n", + " (widgets.Text(value=\"\"), \"custom_emojis\")\n", " ],\n", " [\n", " (widgets.Button(disabled=True),\"load_data\")\n", @@ -449,6 +450,8 @@ " \n", " emoji_mean = shown_widgets[\"label_criteria\"].value == \"mean\"\n", " \n", + " custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n", + " \n", " sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", " n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", " file_range=range(r[0], r[1]),\n", @@ -456,7 +459,8 @@ " read_progress_callback=p_r.update,\n", " stem_progress_callback=p_s.update if lemm_and_stemm else None,\n", " apply_stemming = lemm_and_stemm,\n", - " emoji_mean=emoji_mean)\n", + " emoji_mean=emoji_mean,\n", + " custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None)\n", " shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", " \n", " \n", @@ -679,6 +683,15 @@ "shown_widgets[\"test_input\"].observe(test_input)" ] }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "sdm" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index 8c4d9e9..f81dc08 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -164,7 +164,7 @@ def batch_lemm(sentences): class sample_data_manager(object): @staticmethod - def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False): + def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None): """ generate, read and process train data in one step. @@ -184,7 +184,10 @@ class sample_data_manager(object): sdm.generate_emoji_count_and_weights() - if n_top_emojis > 0: + if custom_target_emojis is not None: + sdm.filter_by_emoji_list(custom_target_emojis) + + elif n_top_emojis > 0: sdm.filter_by_top_emojis(n_top=n_top_emojis) if n_kmeans_cluster > 0: @@ -393,6 +396,16 @@ class sample_data_manager(object): self.emojis = self.emojis[in_top] print("remaining samples after top emoji filtering: ", len(self.labels)) + def filter_by_emoji_list(self, custom_target_emojis): + + assert self.labels is not None + + in_list = [edist.sentiment_vector_to_emoji(x) in custom_target_emojis for x in self.labels] + self.labels = self.labels[in_list] + self.plain_text = self.plain_text[in_list] + self.emojis = self.emojis[in_list] + print("remaining samples after custom emoji filtering: ", len(self.labels)) + def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5): """ generate binary labels using kmeans.