removed interactive parts from simple_twitter_learning python module
This commit is contained in:
Jonas Weinz
parent
919eb6d4a5
commit
7e3b5be4fe
@ -538,151 +538,4 @@ class trainer(object):
|
|||||||
self.sdm.create_train_test_split()
|
self.sdm.create_train_test_split()
|
||||||
return self.pm.predict(self.sdm.Xt), self.sdm.yt
|
return self.pm.predict(self.sdm.Xt), self.sdm.yt
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ----
|
|
||||||
# ## Train
|
|
||||||
|
|
||||||
# * when in notebook environment: run the stuff below:
|
|
||||||
|
|
||||||
# In[10]:
|
|
||||||
|
|
||||||
|
|
||||||
import __main__ as main
|
|
||||||
if not hasattr(main, '__file__'):
|
|
||||||
# we are in an interactive environment (probably in jupyter)
|
|
||||||
# load data:
|
|
||||||
|
|
||||||
# setting n_kmeans_clusters to a value > 0 activates binarized labeling automatically!
|
|
||||||
# set to -1 to disable kmeans clustering and generating labels in plain sentiment space
|
|
||||||
|
|
||||||
#n_kmeans_cluster = 5
|
|
||||||
n_kmeans_cluster = -1
|
|
||||||
sdm = sample_data_manager.generate_and_read(path="./data_en/", n_top_emojis=20, file_range=range(1), n_kmeans_cluster=n_kmeans_cluster)
|
|
||||||
sdm.create_train_test_split()
|
|
||||||
#pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n",
|
|
||||||
# layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\n",
|
|
||||||
pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),
|
|
||||||
layers=[(2500, 'relu'),(sdm.y.shape[1],None)], sdm=sdm)
|
|
||||||
tr = trainer(sdm=sdm, pm=pm)
|
|
||||||
tr.fit(100)
|
|
||||||
|
|
||||||
|
|
||||||
# ----
|
|
||||||
# ## save classifier
|
|
||||||
|
|
||||||
# In[11]:
|
|
||||||
|
|
||||||
|
|
||||||
import __main__ as main
|
|
||||||
if not hasattr(main, '__file__'):
|
|
||||||
pm.save('custom_classifier')
|
|
||||||
|
|
||||||
|
|
||||||
# ----
|
|
||||||
# ## Prediction
|
|
||||||
#
|
|
||||||
# * predict and save to `test.csv`
|
|
||||||
|
|
||||||
# In[12]:
|
|
||||||
|
|
||||||
|
|
||||||
import __main__ as main
|
|
||||||
if not hasattr(main, '__file__'):
|
|
||||||
pred, teacher = tr.test()
|
|
||||||
|
|
||||||
display(pred)
|
|
||||||
display(teacher)
|
|
||||||
|
|
||||||
print('prediction variance: ', np.linalg.norm(np.var(pred, axis=0)))
|
|
||||||
print('teacher variance: ', np.linalg.norm(np.var(teacher, axis=0)))
|
|
||||||
|
|
||||||
# build a dataframe to visualize test results:
|
|
||||||
testlist = pd.DataFrame({'text': sdm.Xt,
|
|
||||||
'teacher': sent2emoji(sdm.yt),
|
|
||||||
'teacher_sentiment': sdm.yt.tolist(),
|
|
||||||
'predict': sent2emoji(pred, custom_target_emojis=sdm.top_emojis),
|
|
||||||
'predicted_sentiment': pred.tolist()})
|
|
||||||
# display:
|
|
||||||
display(testlist.head())
|
|
||||||
|
|
||||||
# mean squared error:
|
|
||||||
teacher_sentiments = np.array([sample[1]['teacher_sentiment'] for sample in testlist.iterrows()])
|
|
||||||
predicted_sentiments = np.array([sample[1]['predicted_sentiment'] for sample in testlist.iterrows()])
|
|
||||||
|
|
||||||
mean_squared_error = ((teacher_sentiments - predicted_sentiments)**2).mean(axis=0)
|
|
||||||
print("Mean Squared Error: ", mean_squared_error)
|
|
||||||
print("Variance teacher: ", np.var(teacher_sentiments, axis=0))
|
|
||||||
print("Variance prediction: ", np.var(predicted_sentiments, axis=0))
|
|
||||||
|
|
||||||
# save to csv:
|
|
||||||
testlist.to_csv('test.csv')
|
|
||||||
|
|
||||||
|
|
||||||
# ----
|
|
||||||
# ## Load classifier
|
|
||||||
#
|
|
||||||
# * loading classifier and show a test widget
|
|
||||||
|
|
||||||
# In[13]:
|
|
||||||
|
|
||||||
|
|
||||||
import __main__ as main
|
|
||||||
if not hasattr(main, '__file__'):
|
|
||||||
try:
|
|
||||||
pm
|
|
||||||
except NameError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
del pm # delete existing pipeline manager if ther is one
|
|
||||||
|
|
||||||
pm = pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])
|
|
||||||
lookup_emojis = [#'😂',
|
|
||||||
'😭',
|
|
||||||
'😍',
|
|
||||||
'😩',
|
|
||||||
'😊',
|
|
||||||
'😘',
|
|
||||||
'🙏',
|
|
||||||
'🙌',
|
|
||||||
'😉',
|
|
||||||
'😁',
|
|
||||||
'😅',
|
|
||||||
'😎',
|
|
||||||
'😢',
|
|
||||||
'😒',
|
|
||||||
'😏',
|
|
||||||
'😌',
|
|
||||||
'😔',
|
|
||||||
'😋',
|
|
||||||
'😀',
|
|
||||||
'😤']
|
|
||||||
out = widgets.Output()
|
|
||||||
|
|
||||||
t = widgets.Text()
|
|
||||||
b = widgets.Button(
|
|
||||||
description='get emoji',
|
|
||||||
disabled=False,
|
|
||||||
button_style='', # 'success', 'info', 'warning', 'danger' or ''
|
|
||||||
tooltip='Click me',
|
|
||||||
icon='check'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def handle_submit(sender):
|
|
||||||
with out:
|
|
||||||
clear_output()
|
|
||||||
with out:
|
|
||||||
pred = pm.predict([t.value])
|
|
||||||
|
|
||||||
display(Markdown("# Predicted Emoji " + str(sent2emoji(pred, lookup_emojis)[0])))
|
|
||||||
display(Markdown("# Sentiment Vector: $$ \pmatrix{" + str(pred[0,0]) +
|
|
||||||
"\\\\" + str(pred[0,1]) + "\\\\" + str(pred[0,2]) + "}$$"))
|
|
||||||
|
|
||||||
b.on_click(handle_submit)
|
|
||||||
|
|
||||||
display(t)
|
|
||||||
display(widgets.VBox([b, out]))
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user