jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

continous learning with gui works for keras models

Browse Source
This commit is contained in:
Jonas Weinz 2018-06-26 14:44:05 +02:00
parent 43e9ace028
commit df811196e5
2 changed files with 83 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
Project/simple_approach/Continous_Learner.ipynb
View File

@ -144,7 +144,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3c11801d12b643d9b059ba1058d66d5e",
"model_id": "5ac970d7d7cf4849b4f5adfb80a820c0",
"version_major": 2,
"version_minor": 0
},
@ -168,11 +168,11 @@
" ],\n",
" [\n",
" (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n",
" (widgets.Checkbox(disabled=True), \"only_emoticons\")\n",
" (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\")\n",
" ],\n",
" [\n",
" (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
" (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"n_top_emojis\")\n",
" (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
" (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n",
" ],\n",
" [\n",
" (widgets.Button(disabled=True),\"load_data\")\n",
@ -197,7 +197,7 @@
" None,\n",
" classifier_tab)\n",
"\n",
"create_area(\"create classifier\",\n",
"create_area(\"create/save/load classifier\",\n",
" [\n",
" [\n",
" (classifier_tab, \"classifier_tab\")\n",
@ -206,8 +206,19 @@
" (widgets.Button(), \"create_classifier\")\n",
" ],\n",
" [\n",
" (widgets.Text(), \"classifier name\"),\n",
" (widgets.Button(), \"save classifier\")\n",
" (widgets.Label(\"save_area:\"), \"save_area:\")\n",
" ],\n",
" [\n",
" (widgets.Text(), \"classifier_name\"),\n",
" (widgets.Button(), \"save_classifier\")\n",
" ],\n",
" [\n",
" (widgets.Label(\"load_area:\"), \"load_area:\")\n",
" ],\n",
" [\n",
" (widgets.Select(options=sorted(glob.glob(\"./*.pipeline\"))), \"clf_file_selector\"),\n",
" (widgets.Text(), \"clf_file\"),\n",
" (widgets.Button(), \"load_classifier\")\n",
" ]\n",
" ],\n",
" \"create\")\n",
@ -541,9 +552,54 @@
" pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n",
" layers=layers, sdm=sdm)\n",
"\n",
"def save_classifier(b):\n",
" global sdm\n",
" global pm\n",
" global tr\n",
" with out_areas[\"create\"]:\n",
" clear_output()\n",
" mp(\"----\")\n",
" if pm is None:\n",
" sys.stderr.write(\"ERROR: create classifier first\")\n",
" return\n",
" \n",
" pm.save(shown_widgets[\"classifier_name\"].value)\n",
"\n",
"def load_classifier(b):\n",
" global sdm\n",
" global pm\n",
" global tr\n",
" with out_areas[\"create\"]:\n",
" clear_output()\n",
" mp(\"----\")\n",
"\n",
"def update_file_selector(b):\n",
" shown_widgets[\"clf_file_selector\"].options = sorted(glob.glob(\"./*.pipeline\"))\n",
"\n",
"def clf_file_selector(b):\n",
" shown_widgets[\"clf_file\"].value = shown_widgets[\"clf_file_selector\"].value\n",
" update_file_selector(b)\n",
"\n",
"def load_classifier(b):\n",
" global sdm\n",
" global pm\n",
" global tr\n",
" with out_areas[\"create\"]:\n",
" clear_output()\n",
" mp(\"----\")\n",
" clf_file = shown_widgets[\"clf_file\"].value\n",
" pm = stl.pipeline_manager.load_from_pipeline_file(clf_file)\n",
" \n",
"\n",
"# link\n",
"shown_widgets[\"n_keras_layer\"].observe(populate_keras_options)\n",
"shown_widgets[\"create_classifier\"].on_click(create_classifier)"
"shown_widgets[\"create_classifier\"].on_click(create_classifier)\n",
"shown_widgets[\"save_classifier\"].on_click(save_classifier)\n",
"shown_widgets[\"load_classifier\"].on_click(load_classifier)\n",
"shown_widgets[\"clf_file_selector\"].observe(clf_file_selector)\n",
"\n",
"\n",
"\n"
]
}
],

19
Project/simple_approach/simple_twitter_learning.py
View File

@ -23,6 +23,7 @@ from sklearn.externals import joblib
import pickle
import operator
from sklearn.pipeline import Pipeline
import json
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
@ -329,6 +330,20 @@ class sample_data_manager(object):
class pipeline_manager(object):
@staticmethod
def load_from_pipeline_file(pipeline_file:str):
"""
loading a json configuration file and using it's paramters to call 'load_pipeline_from_files'
"""
with open(pipeline_file, 'r') as f:
d = json.load(f)
keras_models = d['keras_models']
all_models = d['all_models']
return pipeline_manager.load_pipeline_from_files(pipeline_file.rsplit('.',1)[0], keras_models, all_models)
@staticmethod
def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []):
"""
@ -442,6 +457,7 @@ class pipeline_manager(object):
@param prefix: file prefix for all models
"""
print(self.keras_models)
# doing this like explained here: https://stackoverflow.com/a/43415459
for step in self.pipeline.named_steps:
@ -454,6 +470,9 @@ class pipeline_manager(object):
load_command += prefix + "', " + str(self.keras_models) + ", "
load_command += str(list(self.pipeline.named_steps.keys())) + ")"
with open(prefix + '.pipeline', 'w') as outfile:
json.dump({'keras_models': self.keras_models, 'all_models': [step for step in self.pipeline.named_steps]}, outfile)
import __main__ as main
if not hasattr(main, '__file__'):
display("saved pipeline. It can be loaded the following way:")