continous learning with gui works for keras models

Project/simple_approach/Continous_Learner.ipynb

@@ -144,7 +144,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3c11801d12b643d9b059ba1058d66d5e",
+       "model_id": "5ac970d7d7cf4849b4f5adfb80a820c0",
        "version_major": 2,
        "version_minor": 0
       },
@@ -168,11 +168,11 @@
     "               ],\n",
     "               [\n",
     "                   (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n",
-    "                   (widgets.Checkbox(disabled=True), \"only_emoticons\")\n",
+    "                   (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\")\n",
     "               ],\n",
     "               [\n",
-    "                   (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
+    "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
-    "                   (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"n_top_emojis\")\n",
+    "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n",
     "               ],\n",
     "               [\n",
     "                   (widgets.Button(disabled=True),\"load_data\")\n",
@@ -197,7 +197,7 @@
     "           None,\n",
     "           classifier_tab)\n",
     "\n",
-    "create_area(\"create classifier\",\n",
+    "create_area(\"create/save/load classifier\",\n",
     "           [\n",
     "               [\n",
     "                   (classifier_tab, \"classifier_tab\")\n",
@@ -206,8 +206,19 @@
     "                   (widgets.Button(), \"create_classifier\")\n",
     "               ],\n",
     "               [\n",
-    "                   (widgets.Text(), \"classifier name\"),\n",
+    "                   (widgets.Label(\"save_area:\"), \"save_area:\")\n",
-    "                   (widgets.Button(), \"save classifier\")\n",
+    "               ],\n",
+    "               [\n",
+    "                   (widgets.Text(), \"classifier_name\"),\n",
+    "                   (widgets.Button(), \"save_classifier\")\n",
+    "               ],\n",
+    "               [\n",
+    "                   (widgets.Label(\"load_area:\"), \"load_area:\")\n",
+    "               ],\n",
+    "               [\n",
+    "                   (widgets.Select(options=sorted(glob.glob(\"./*.pipeline\"))), \"clf_file_selector\"),\n",
+    "                   (widgets.Text(), \"clf_file\"),\n",
+    "                   (widgets.Button(), \"load_classifier\")\n",
     "               ]\n",
     "           ],\n",
     "           \"create\")\n",
@@ -541,9 +552,54 @@
     "            pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n",
     "                                                           layers=layers, sdm=sdm)\n",
     "\n",
+    "def save_classifier(b):\n",
+    "    global sdm\n",
+    "    global pm\n",
+    "    global tr\n",
+    "    with out_areas[\"create\"]:\n",
+    "        clear_output()\n",
+    "        mp(\"----\")\n",
+    "        if pm is None:\n",
+    "            sys.stderr.write(\"ERROR: create classifier first\")\n",
+    "            return\n",
+    "        \n",
+    "        pm.save(shown_widgets[\"classifier_name\"].value)\n",
+    "\n",
+    "def load_classifier(b):\n",
+    "    global sdm\n",
+    "    global pm\n",
+    "    global tr\n",
+    "    with out_areas[\"create\"]:\n",
+    "        clear_output()\n",
+    "        mp(\"----\")\n",
+    "\n",
+    "def update_file_selector(b):\n",
+    "    shown_widgets[\"clf_file_selector\"].options = sorted(glob.glob(\"./*.pipeline\"))\n",
+    "\n",
+    "def clf_file_selector(b):\n",
+    "    shown_widgets[\"clf_file\"].value = shown_widgets[\"clf_file_selector\"].value\n",
+    "    update_file_selector(b)\n",
+    "\n",
+    "def load_classifier(b):\n",
+    "    global sdm\n",
+    "    global pm\n",
+    "    global tr\n",
+    "    with out_areas[\"create\"]:\n",
+    "        clear_output()\n",
+    "        mp(\"----\")\n",
+    "        clf_file = shown_widgets[\"clf_file\"].value\n",
+    "        pm = stl.pipeline_manager.load_from_pipeline_file(clf_file)\n",
+    "    \n",
+    "\n",
     "# link\n",
     "shown_widgets[\"n_keras_layer\"].observe(populate_keras_options)\n",
-    "shown_widgets[\"create_classifier\"].on_click(create_classifier)"
+    "shown_widgets[\"create_classifier\"].on_click(create_classifier)\n",
+    "shown_widgets[\"save_classifier\"].on_click(save_classifier)\n",
+    "shown_widgets[\"load_classifier\"].on_click(load_classifier)\n",
+    "shown_widgets[\"clf_file_selector\"].observe(clf_file_selector)\n",
+    "\n",
+    "\n",
+    "\n"
    ]
   }
  ],

Project/simple_approach/simple_twitter_learning.py

@@ -23,6 +23,7 @@ from sklearn.externals import joblib
 import pickle
 import operator
 from sklearn.pipeline import Pipeline
+import json
 nltk.download('punkt')
 nltk.download('averaged_perceptron_tagger')
 nltk.download('wordnet')
@@ -329,6 +330,20 @@ class sample_data_manager(object):
 
 
 class pipeline_manager(object):
+    @staticmethod
+    def load_from_pipeline_file(pipeline_file:str):
+        """
+        loading a json configuration file and using it's paramters to call 'load_pipeline_from_files'
+        """
+        with open(pipeline_file, 'r') as f:
+            d = json.load(f)
+        
+        keras_models = d['keras_models']
+        all_models = d['all_models']
+        
+        return pipeline_manager.load_pipeline_from_files(pipeline_file.rsplit('.',1)[0], keras_models, all_models)
+
+
     @staticmethod
     def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []):
         """
@@ -442,6 +457,7 @@ class pipeline_manager(object):
         @param prefix: file prefix for all models
         """
         
+
         print(self.keras_models)
         # doing this like explained here: https://stackoverflow.com/a/43415459
         for step in self.pipeline.named_steps:
@@ -454,6 +470,9 @@ class pipeline_manager(object):
         load_command += prefix + "', " + str(self.keras_models) + ", "
         load_command += str(list(self.pipeline.named_steps.keys())) + ")"
 
+        with open(prefix + '.pipeline', 'w') as outfile:
+            json.dump({'keras_models': self.keras_models, 'all_models': [step for step in self.pipeline.named_steps]}, outfile)
+        
         import __main__ as main
         if not hasattr(main, '__file__'):
             display("saved pipeline. It can be loaded the following way:")