diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index efd8ac1..8d13eee 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -538,151 +538,4 @@ class trainer(object): self.sdm.create_train_test_split() return self.pm.predict(self.sdm.Xt), self.sdm.yt - - - -# ---- -# ## Train - -# * when in notebook environment: run the stuff below: - -# In[10]: - - -import __main__ as main -if not hasattr(main, '__file__'): - # we are in an interactive environment (probably in jupyter) - # load data: - - # setting n_kmeans_clusters to a value > 0 activates binarized labeling automatically! - # set to -1 to disable kmeans clustering and generating labels in plain sentiment space - - #n_kmeans_cluster = 5 - n_kmeans_cluster = -1 - sdm = sample_data_manager.generate_and_read(path="./data_en/", n_top_emojis=20, file_range=range(1), n_kmeans_cluster=n_kmeans_cluster) - sdm.create_train_test_split() - #pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", - # layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\n", - pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'), - layers=[(2500, 'relu'),(sdm.y.shape[1],None)], sdm=sdm) - tr = trainer(sdm=sdm, pm=pm) - tr.fit(100) - - -# ---- -# ## save classifier - -# In[11]: - - -import __main__ as main -if not hasattr(main, '__file__'): - pm.save('custom_classifier') - - -# ---- -# ## Prediction -# -# * predict and save to `test.csv` - -# In[12]: - - -import __main__ as main -if not hasattr(main, '__file__'): - pred, teacher = tr.test() - - display(pred) - display(teacher) - - print('prediction variance: ', np.linalg.norm(np.var(pred, axis=0))) - print('teacher variance: ', np.linalg.norm(np.var(teacher, axis=0))) - - # build a dataframe to visualize test results: - testlist = pd.DataFrame({'text': sdm.Xt, - 'teacher': sent2emoji(sdm.yt), - 'teacher_sentiment': sdm.yt.tolist(), - 'predict': sent2emoji(pred, custom_target_emojis=sdm.top_emojis), - 'predicted_sentiment': pred.tolist()}) - # display: - display(testlist.head()) - - # mean squared error: - teacher_sentiments = np.array([sample[1]['teacher_sentiment'] for sample in testlist.iterrows()]) - predicted_sentiments = np.array([sample[1]['predicted_sentiment'] for sample in testlist.iterrows()]) - - mean_squared_error = ((teacher_sentiments - predicted_sentiments)**2).mean(axis=0) - print("Mean Squared Error: ", mean_squared_error) - print("Variance teacher: ", np.var(teacher_sentiments, axis=0)) - print("Variance prediction: ", np.var(predicted_sentiments, axis=0)) - - # save to csv: - testlist.to_csv('test.csv') - - -# ---- -# ## Load classifier -# -# * loading classifier and show a test widget - -# In[13]: - - -import __main__ as main -if not hasattr(main, '__file__'): - try: - pm - except NameError: - pass - else: - del pm # delete existing pipeline manager if ther is one - - pm = pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model']) - lookup_emojis = [#'😂', - '😭', - '😍', - '😩', - '😊', - '😘', - '🙏', - '🙌', - '😉', - '😁', - '😅', - '😎', - '😢', - '😒', - '😏', - '😌', - '😔', - '😋', - '😀', - '😤'] - out = widgets.Output() - - t = widgets.Text() - b = widgets.Button( - description='get emoji', - disabled=False, - button_style='', # 'success', 'info', 'warning', 'danger' or '' - tooltip='Click me', - icon='check' - ) - - - - def handle_submit(sender): - with out: - clear_output() - with out: - pred = pm.predict([t.value]) - - display(Markdown("# Predicted Emoji " + str(sent2emoji(pred, lookup_emojis)[0]))) - display(Markdown("# Sentiment Vector: $$ \pmatrix{" + str(pred[0,0]) + - "\\\\" + str(pred[0,1]) + "\\\\" + str(pred[0,2]) + "}$$")) - - b.on_click(handle_submit) - - display(t) - display(widgets.VBox([b, out])) - + \ No newline at end of file