From df811196e587149173bac6faac7de05a045c052b Mon Sep 17 00:00:00 2001 From: Jonas Weinz Date: Tue, 26 Jun 2018 14:44:05 +0200 Subject: [PATCH] continous learning with gui works for keras models --- .../simple_approach/Continous_Learner.ipynb | 72 ++++++++++++++++--- .../simple_twitter_learning.py | 19 +++++ 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/Project/simple_approach/Continous_Learner.ipynb b/Project/simple_approach/Continous_Learner.ipynb index 49c586a..10ee731 100644 --- a/Project/simple_approach/Continous_Learner.ipynb +++ b/Project/simple_approach/Continous_Learner.ipynb @@ -144,7 +144,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3c11801d12b643d9b059ba1058d66d5e", + "model_id": "5ac970d7d7cf4849b4f5adfb80a820c0", "version_major": 2, "version_minor": 0 }, @@ -168,11 +168,11 @@ " ],\n", " [\n", " (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n", - " (widgets.Checkbox(disabled=True), \"only_emoticons\")\n", + " (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\")\n", " ],\n", " [\n", - " (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", - " (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"n_top_emojis\")\n", + " (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", + " (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n", " ],\n", " [\n", " (widgets.Button(disabled=True),\"load_data\")\n", @@ -197,7 +197,7 @@ " None,\n", " classifier_tab)\n", "\n", - "create_area(\"create classifier\",\n", + "create_area(\"create/save/load classifier\",\n", " [\n", " [\n", " (classifier_tab, \"classifier_tab\")\n", @@ -206,8 +206,19 @@ " (widgets.Button(), \"create_classifier\")\n", " ],\n", " [\n", - " (widgets.Text(), \"classifier name\"),\n", - " (widgets.Button(), \"save classifier\")\n", + " (widgets.Label(\"save_area:\"), \"save_area:\")\n", + " ],\n", + " [\n", + " (widgets.Text(), \"classifier_name\"),\n", + " (widgets.Button(), \"save_classifier\")\n", + " ],\n", + " [\n", + " (widgets.Label(\"load_area:\"), \"load_area:\")\n", + " ],\n", + " [\n", + " (widgets.Select(options=sorted(glob.glob(\"./*.pipeline\"))), \"clf_file_selector\"),\n", + " (widgets.Text(), \"clf_file\"),\n", + " (widgets.Button(), \"load_classifier\")\n", " ]\n", " ],\n", " \"create\")\n", @@ -541,9 +552,54 @@ " pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", " layers=layers, sdm=sdm)\n", "\n", + "def save_classifier(b):\n", + " global sdm\n", + " global pm\n", + " global tr\n", + " with out_areas[\"create\"]:\n", + " clear_output()\n", + " mp(\"----\")\n", + " if pm is None:\n", + " sys.stderr.write(\"ERROR: create classifier first\")\n", + " return\n", + " \n", + " pm.save(shown_widgets[\"classifier_name\"].value)\n", + "\n", + "def load_classifier(b):\n", + " global sdm\n", + " global pm\n", + " global tr\n", + " with out_areas[\"create\"]:\n", + " clear_output()\n", + " mp(\"----\")\n", + "\n", + "def update_file_selector(b):\n", + " shown_widgets[\"clf_file_selector\"].options = sorted(glob.glob(\"./*.pipeline\"))\n", + "\n", + "def clf_file_selector(b):\n", + " shown_widgets[\"clf_file\"].value = shown_widgets[\"clf_file_selector\"].value\n", + " update_file_selector(b)\n", + "\n", + "def load_classifier(b):\n", + " global sdm\n", + " global pm\n", + " global tr\n", + " with out_areas[\"create\"]:\n", + " clear_output()\n", + " mp(\"----\")\n", + " clf_file = shown_widgets[\"clf_file\"].value\n", + " pm = stl.pipeline_manager.load_from_pipeline_file(clf_file)\n", + " \n", + "\n", "# link\n", "shown_widgets[\"n_keras_layer\"].observe(populate_keras_options)\n", - "shown_widgets[\"create_classifier\"].on_click(create_classifier)" + "shown_widgets[\"create_classifier\"].on_click(create_classifier)\n", + "shown_widgets[\"save_classifier\"].on_click(save_classifier)\n", + "shown_widgets[\"load_classifier\"].on_click(load_classifier)\n", + "shown_widgets[\"clf_file_selector\"].observe(clf_file_selector)\n", + "\n", + "\n", + "\n" ] } ], diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index 5f1cf26..d551df2 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -23,6 +23,7 @@ from sklearn.externals import joblib import pickle import operator from sklearn.pipeline import Pipeline +import json nltk.download('punkt') nltk.download('averaged_perceptron_tagger') nltk.download('wordnet') @@ -329,6 +330,20 @@ class sample_data_manager(object): class pipeline_manager(object): + @staticmethod + def load_from_pipeline_file(pipeline_file:str): + """ + loading a json configuration file and using it's paramters to call 'load_pipeline_from_files' + """ + with open(pipeline_file, 'r') as f: + d = json.load(f) + + keras_models = d['keras_models'] + all_models = d['all_models'] + + return pipeline_manager.load_pipeline_from_files(pipeline_file.rsplit('.',1)[0], keras_models, all_models) + + @staticmethod def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []): """ @@ -442,6 +457,7 @@ class pipeline_manager(object): @param prefix: file prefix for all models """ + print(self.keras_models) # doing this like explained here: https://stackoverflow.com/a/43415459 for step in self.pipeline.named_steps: @@ -453,6 +469,9 @@ class pipeline_manager(object): load_command = "pipeline_manager.load_pipeline_from_files( '" load_command += prefix + "', " + str(self.keras_models) + ", " load_command += str(list(self.pipeline.named_steps.keys())) + ")" + + with open(prefix + '.pipeline', 'w') as outfile: + json.dump({'keras_models': self.keras_models, 'all_models': [step for step in self.pipeline.named_steps]}, outfile) import __main__ as main if not hasattr(main, '__file__'):