removed interactive parts from simple_twitter_learning python module
This commit is contained in:
		| @ -538,151 +538,4 @@ class trainer(object): | ||||
|             self.sdm.create_train_test_split() | ||||
|         return self.pm.predict(self.sdm.Xt), self.sdm.yt | ||||
|  | ||||
|      | ||||
|  | ||||
|  | ||||
| # ---- | ||||
| # ## Train | ||||
|  | ||||
| # * when in notebook environment: run the stuff below: | ||||
|  | ||||
| # In[10]: | ||||
|  | ||||
|  | ||||
| import __main__ as main | ||||
| if not hasattr(main, '__file__'): | ||||
|     # we are in an interactive environment (probably in jupyter) | ||||
|     # load data: | ||||
|      | ||||
|     # setting n_kmeans_clusters to a value > 0 activates binarized labeling automatically!  | ||||
|     # set to -1 to disable kmeans clustering and generating labels in plain sentiment space | ||||
|      | ||||
|     #n_kmeans_cluster = 5 | ||||
|     n_kmeans_cluster = -1 | ||||
|     sdm = sample_data_manager.generate_and_read(path="./data_en/", n_top_emojis=20, file_range=range(1), n_kmeans_cluster=n_kmeans_cluster) | ||||
|     sdm.create_train_test_split() | ||||
|     #pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", | ||||
|     #                                                           layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\n", | ||||
|     pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'), | ||||
|                                                            layers=[(2500, 'relu'),(sdm.y.shape[1],None)], sdm=sdm) | ||||
|     tr = trainer(sdm=sdm, pm=pm) | ||||
|     tr.fit(100) | ||||
|  | ||||
|  | ||||
| # ---- | ||||
| # ## save classifier | ||||
|  | ||||
| # In[11]: | ||||
|  | ||||
|  | ||||
| import __main__ as main | ||||
| if not hasattr(main, '__file__'): | ||||
|     pm.save('custom_classifier') | ||||
|  | ||||
|  | ||||
| # ---- | ||||
| # ## Prediction | ||||
| #  | ||||
| # * predict and save to `test.csv` | ||||
|  | ||||
| # In[12]: | ||||
|  | ||||
|  | ||||
| import __main__ as main | ||||
| if not hasattr(main, '__file__'): | ||||
|     pred, teacher = tr.test() | ||||
|      | ||||
|     display(pred) | ||||
|     display(teacher) | ||||
|      | ||||
|     print('prediction variance: ', np.linalg.norm(np.var(pred, axis=0))) | ||||
|     print('teacher variance: ', np.linalg.norm(np.var(teacher, axis=0))) | ||||
|      | ||||
|     # build a dataframe to visualize test results: | ||||
|     testlist = pd.DataFrame({'text': sdm.Xt,  | ||||
|                          'teacher': sent2emoji(sdm.yt), | ||||
|                          'teacher_sentiment': sdm.yt.tolist(), | ||||
|                          'predict': sent2emoji(pred, custom_target_emojis=sdm.top_emojis), | ||||
|                          'predicted_sentiment': pred.tolist()}) | ||||
|     # display: | ||||
|     display(testlist.head()) | ||||
|      | ||||
|     # mean squared error: | ||||
|     teacher_sentiments = np.array([sample[1]['teacher_sentiment'] for sample in testlist.iterrows()]) | ||||
|     predicted_sentiments = np.array([sample[1]['predicted_sentiment'] for sample in testlist.iterrows()]) | ||||
|  | ||||
|     mean_squared_error = ((teacher_sentiments - predicted_sentiments)**2).mean(axis=0) | ||||
|     print("Mean Squared Error: ", mean_squared_error) | ||||
|     print("Variance teacher: ", np.var(teacher_sentiments, axis=0)) | ||||
|     print("Variance prediction: ", np.var(predicted_sentiments, axis=0)) | ||||
|      | ||||
|     # save to csv: | ||||
|     testlist.to_csv('test.csv') | ||||
|  | ||||
|  | ||||
| # ---- | ||||
| # ## Load classifier | ||||
| #  | ||||
| # * loading classifier and show a test widget | ||||
|  | ||||
| # In[13]: | ||||
|  | ||||
|  | ||||
| import __main__ as main | ||||
| if not hasattr(main, '__file__'): | ||||
|     try: | ||||
|         pm | ||||
|     except NameError: | ||||
|         pass | ||||
|     else: | ||||
|         del pm # delete existing pipeline manager if ther is one | ||||
|  | ||||
|     pm = pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model']) | ||||
|     lookup_emojis = [#'😂', | ||||
|          '😭', | ||||
|          '😍', | ||||
|          '😩', | ||||
|          '😊', | ||||
|          '😘', | ||||
|          '🙏', | ||||
|          '🙌', | ||||
|          '😉', | ||||
|          '😁', | ||||
|          '😅', | ||||
|          '😎', | ||||
|          '😢', | ||||
|          '😒', | ||||
|          '😏', | ||||
|          '😌', | ||||
|          '😔', | ||||
|          '😋', | ||||
|          '😀', | ||||
|          '😤'] | ||||
|     out = widgets.Output() | ||||
|  | ||||
|     t = widgets.Text() | ||||
|     b = widgets.Button( | ||||
|         description='get emoji', | ||||
|         disabled=False, | ||||
|         button_style='', # 'success', 'info', 'warning', 'danger' or '' | ||||
|         tooltip='Click me', | ||||
|         icon='check' | ||||
|     ) | ||||
|  | ||||
|  | ||||
|  | ||||
|     def handle_submit(sender): | ||||
|         with out: | ||||
|             clear_output() | ||||
|         with out: | ||||
|             pred = pm.predict([t.value]) | ||||
|  | ||||
|             display(Markdown("# Predicted Emoji " + str(sent2emoji(pred, lookup_emojis)[0]))) | ||||
|             display(Markdown("# Sentiment Vector: $$ \pmatrix{" + str(pred[0,0]) + | ||||
|                              "\\\\" + str(pred[0,1]) + "\\\\" + str(pred[0,2]) + "}$$")) | ||||
|  | ||||
|     b.on_click(handle_submit) | ||||
|  | ||||
|     display(t) | ||||
|     display(widgets.VBox([b, out]))   | ||||
|  | ||||
|      | ||||
		Reference in New Issue
	
	Block a user