added possibility to filter by emoji list
This commit is contained in:
		@ -145,7 +145,7 @@
 | 
			
		||||
    {
 | 
			
		||||
     "data": {
 | 
			
		||||
      "application/vnd.jupyter.widget-view+json": {
 | 
			
		||||
       "model_id": "43f80c7d9c024a57b4009b77296f1ab0",
 | 
			
		||||
       "model_id": "31e69854333f4c599b037b6c27f30f20",
 | 
			
		||||
       "version_major": 2,
 | 
			
		||||
       "version_minor": 0
 | 
			
		||||
      },
 | 
			
		||||
@ -175,7 +175,8 @@
 | 
			
		||||
    "               [\n",
 | 
			
		||||
    "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
 | 
			
		||||
    "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\"),\n",
 | 
			
		||||
    "                   (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\")\n",
 | 
			
		||||
    "                   (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\"),\n",
 | 
			
		||||
    "                   (widgets.Text(value=\"\"), \"custom_emojis\")\n",
 | 
			
		||||
    "               ],\n",
 | 
			
		||||
    "               [\n",
 | 
			
		||||
    "                   (widgets.Button(disabled=True),\"load_data\")\n",
 | 
			
		||||
@ -449,6 +450,8 @@
 | 
			
		||||
    "        \n",
 | 
			
		||||
    "        emoji_mean = shown_widgets[\"label_criteria\"].value == \"mean\"\n",
 | 
			
		||||
    "        \n",
 | 
			
		||||
    "        custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n",
 | 
			
		||||
    "        \n",
 | 
			
		||||
    "        sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n",
 | 
			
		||||
    "                                                    n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n",
 | 
			
		||||
    "                                                    file_range=range(r[0], r[1]),\n",
 | 
			
		||||
@ -456,7 +459,8 @@
 | 
			
		||||
    "                                                    read_progress_callback=p_r.update,\n",
 | 
			
		||||
    "                                                    stem_progress_callback=p_s.update if lemm_and_stemm else None,\n",
 | 
			
		||||
    "                                                    apply_stemming = lemm_and_stemm,\n",
 | 
			
		||||
    "                                                    emoji_mean=emoji_mean)\n",
 | 
			
		||||
    "                                                    emoji_mean=emoji_mean,\n",
 | 
			
		||||
    "                                                    custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None)\n",
 | 
			
		||||
    "        shown_widgets[\"batch_size\"].max = len(sdm.labels)\n",
 | 
			
		||||
    "        \n",
 | 
			
		||||
    "        \n",
 | 
			
		||||
@ -679,6 +683,15 @@
 | 
			
		||||
    "shown_widgets[\"test_input\"].observe(test_input)"
 | 
			
		||||
   ]
 | 
			
		||||
  },
 | 
			
		||||
  {
 | 
			
		||||
   "cell_type": "code",
 | 
			
		||||
   "execution_count": 11,
 | 
			
		||||
   "metadata": {},
 | 
			
		||||
   "outputs": [],
 | 
			
		||||
   "source": [
 | 
			
		||||
    "sdm"
 | 
			
		||||
   ]
 | 
			
		||||
  },
 | 
			
		||||
  {
 | 
			
		||||
   "cell_type": "code",
 | 
			
		||||
   "execution_count": null,
 | 
			
		||||
 | 
			
		||||
@ -164,7 +164,7 @@ def batch_lemm(sentences):
 | 
			
		||||
 | 
			
		||||
class sample_data_manager(object):
 | 
			
		||||
    @staticmethod
 | 
			
		||||
    def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False):
 | 
			
		||||
    def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None):
 | 
			
		||||
        """
 | 
			
		||||
        generate, read and process train data in one step.
 | 
			
		||||
        
 | 
			
		||||
@ -184,7 +184,10 @@ class sample_data_manager(object):
 | 
			
		||||
        
 | 
			
		||||
        sdm.generate_emoji_count_and_weights()
 | 
			
		||||
        
 | 
			
		||||
        if n_top_emojis > 0:
 | 
			
		||||
        if custom_target_emojis is not None:
 | 
			
		||||
            sdm.filter_by_emoji_list(custom_target_emojis)
 | 
			
		||||
 | 
			
		||||
        elif n_top_emojis > 0:
 | 
			
		||||
            sdm.filter_by_top_emojis(n_top=n_top_emojis)
 | 
			
		||||
        
 | 
			
		||||
        if n_kmeans_cluster > 0:
 | 
			
		||||
@ -393,6 +396,16 @@ class sample_data_manager(object):
 | 
			
		||||
        self.emojis = self.emojis[in_top]
 | 
			
		||||
        print("remaining samples after top emoji filtering: ", len(self.labels))
 | 
			
		||||
    
 | 
			
		||||
    def filter_by_emoji_list(self, custom_target_emojis):
 | 
			
		||||
 | 
			
		||||
        assert self.labels is not None
 | 
			
		||||
 | 
			
		||||
        in_list = [edist.sentiment_vector_to_emoji(x) in custom_target_emojis for x in self.labels]
 | 
			
		||||
        self.labels = self.labels[in_list]
 | 
			
		||||
        self.plain_text = self.plain_text[in_list]
 | 
			
		||||
        self.emojis = self.emojis[in_list]
 | 
			
		||||
        print("remaining samples after custom emoji filtering: ", len(self.labels))
 | 
			
		||||
 | 
			
		||||
    def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5):
 | 
			
		||||
        """
 | 
			
		||||
        generate binary labels using kmeans.
 | 
			
		||||
 | 
			
		||||
		Reference in New Issue
	
	Block a user