continous learning with gui works for keras models
This commit is contained in:
		| @ -144,7 +144,7 @@ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "3c11801d12b643d9b059ba1058d66d5e", | ||||
|        "model_id": "5ac970d7d7cf4849b4f5adfb80a820c0", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
| @ -168,11 +168,11 @@ | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n", | ||||
|     "                   (widgets.Checkbox(disabled=True), \"only_emoticons\")\n", | ||||
|     "                   (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
|     "                   (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"n_top_emojis\")\n", | ||||
|     "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
|     "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Button(disabled=True),\"load_data\")\n", | ||||
| @ -197,7 +197,7 @@ | ||||
|     "           None,\n", | ||||
|     "           classifier_tab)\n", | ||||
|     "\n", | ||||
|     "create_area(\"create classifier\",\n", | ||||
|     "create_area(\"create/save/load classifier\",\n", | ||||
|     "           [\n", | ||||
|     "               [\n", | ||||
|     "                   (classifier_tab, \"classifier_tab\")\n", | ||||
| @ -206,8 +206,19 @@ | ||||
|     "                   (widgets.Button(), \"create_classifier\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Text(), \"classifier name\"),\n", | ||||
|     "                   (widgets.Button(), \"save classifier\")\n", | ||||
|     "                   (widgets.Label(\"save_area:\"), \"save_area:\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Text(), \"classifier_name\"),\n", | ||||
|     "                   (widgets.Button(), \"save_classifier\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Label(\"load_area:\"), \"load_area:\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Select(options=sorted(glob.glob(\"./*.pipeline\"))), \"clf_file_selector\"),\n", | ||||
|     "                   (widgets.Text(), \"clf_file\"),\n", | ||||
|     "                   (widgets.Button(), \"load_classifier\")\n", | ||||
|     "               ]\n", | ||||
|     "           ],\n", | ||||
|     "           \"create\")\n", | ||||
| @ -541,9 +552,54 @@ | ||||
|     "            pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", | ||||
|     "                                                           layers=layers, sdm=sdm)\n", | ||||
|     "\n", | ||||
|     "def save_classifier(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"create\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "        if pm is None:\n", | ||||
|     "            sys.stderr.write(\"ERROR: create classifier first\")\n", | ||||
|     "            return\n", | ||||
|     "        \n", | ||||
|     "        pm.save(shown_widgets[\"classifier_name\"].value)\n", | ||||
|     "\n", | ||||
|     "def load_classifier(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"create\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "\n", | ||||
|     "def update_file_selector(b):\n", | ||||
|     "    shown_widgets[\"clf_file_selector\"].options = sorted(glob.glob(\"./*.pipeline\"))\n", | ||||
|     "\n", | ||||
|     "def clf_file_selector(b):\n", | ||||
|     "    shown_widgets[\"clf_file\"].value = shown_widgets[\"clf_file_selector\"].value\n", | ||||
|     "    update_file_selector(b)\n", | ||||
|     "\n", | ||||
|     "def load_classifier(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"create\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "        clf_file = shown_widgets[\"clf_file\"].value\n", | ||||
|     "        pm = stl.pipeline_manager.load_from_pipeline_file(clf_file)\n", | ||||
|     "    \n", | ||||
|     "\n", | ||||
|     "# link\n", | ||||
|     "shown_widgets[\"n_keras_layer\"].observe(populate_keras_options)\n", | ||||
|     "shown_widgets[\"create_classifier\"].on_click(create_classifier)" | ||||
|     "shown_widgets[\"create_classifier\"].on_click(create_classifier)\n", | ||||
|     "shown_widgets[\"save_classifier\"].on_click(save_classifier)\n", | ||||
|     "shown_widgets[\"load_classifier\"].on_click(load_classifier)\n", | ||||
|     "shown_widgets[\"clf_file_selector\"].observe(clf_file_selector)\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "\n" | ||||
|    ] | ||||
|   } | ||||
|  ], | ||||
|  | ||||
| @ -23,6 +23,7 @@ from sklearn.externals import joblib | ||||
| import pickle | ||||
| import operator | ||||
| from sklearn.pipeline import Pipeline | ||||
| import json | ||||
| nltk.download('punkt') | ||||
| nltk.download('averaged_perceptron_tagger') | ||||
| nltk.download('wordnet') | ||||
| @ -329,6 +330,20 @@ class sample_data_manager(object): | ||||
|  | ||||
|  | ||||
| class pipeline_manager(object): | ||||
|     @staticmethod | ||||
|     def load_from_pipeline_file(pipeline_file:str): | ||||
|         """ | ||||
|         loading a json configuration file and using it's paramters to call 'load_pipeline_from_files' | ||||
|         """ | ||||
|         with open(pipeline_file, 'r') as f: | ||||
|             d = json.load(f) | ||||
|          | ||||
|         keras_models = d['keras_models'] | ||||
|         all_models = d['all_models'] | ||||
|          | ||||
|         return pipeline_manager.load_pipeline_from_files(pipeline_file.rsplit('.',1)[0], keras_models, all_models) | ||||
|  | ||||
|  | ||||
|     @staticmethod | ||||
|     def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []): | ||||
|         """ | ||||
| @ -442,6 +457,7 @@ class pipeline_manager(object): | ||||
|         @param prefix: file prefix for all models | ||||
|         """ | ||||
|          | ||||
|  | ||||
|         print(self.keras_models) | ||||
|         # doing this like explained here: https://stackoverflow.com/a/43415459 | ||||
|         for step in self.pipeline.named_steps: | ||||
| @ -453,6 +469,9 @@ class pipeline_manager(object): | ||||
|         load_command = "pipeline_manager.load_pipeline_from_files( '" | ||||
|         load_command += prefix + "', " + str(self.keras_models) + ", " | ||||
|         load_command += str(list(self.pipeline.named_steps.keys())) + ")" | ||||
|  | ||||
|         with open(prefix + '.pipeline', 'w') as outfile: | ||||
|             json.dump({'keras_models': self.keras_models, 'all_models': [step for step in self.pipeline.named_steps]}, outfile) | ||||
|          | ||||
|         import __main__ as main | ||||
|         if not hasattr(main, '__file__'): | ||||
|  | ||||
		Reference in New Issue
	
	Block a user