jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removed interactive parts from simple_twitter_learning python module

Browse Source
This commit is contained in:
Jonas Weinz 2018-06-25 11:03:12 +02:00
parent 919eb6d4a5
commit 7e3b5be4fe
1 changed files with 1 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
Project/simple_approach/simple_twitter_learning.py
View File

@ -538,151 +538,4 @@ class trainer(object):
self.sdm.create_train_test_split()
return self.pm.predict(self.sdm.Xt), self.sdm.yt
# ----
# ## Train
# * when in notebook environment: run the stuff below:
# In[10]:
import __main__ as main
if not hasattr(main, '__file__'):
# we are in an interactive environment (probably in jupyter)
# load data:
# setting n_kmeans_clusters to a value > 0 activates binarized labeling automatically!
# set to -1 to disable kmeans clustering and generating labels in plain sentiment space
#n_kmeans_cluster = 5
n_kmeans_cluster = -1
sdm = sample_data_manager.generate_and_read(path="./data_en/", n_top_emojis=20, file_range=range(1), n_kmeans_cluster=n_kmeans_cluster)
sdm.create_train_test_split()
#pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n",
# layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\n",
pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),
layers=[(2500, 'relu'),(sdm.y.shape[1],None)], sdm=sdm)
tr = trainer(sdm=sdm, pm=pm)
tr.fit(100)
# ----
# ## save classifier
# In[11]:
import __main__ as main
if not hasattr(main, '__file__'):
pm.save('custom_classifier')
# ----
# ## Prediction
#
# * predict and save to `test.csv`
# In[12]:
import __main__ as main
if not hasattr(main, '__file__'):
pred, teacher = tr.test()
display(pred)
display(teacher)
print('prediction variance: ', np.linalg.norm(np.var(pred, axis=0)))
print('teacher variance: ', np.linalg.norm(np.var(teacher, axis=0)))
# build a dataframe to visualize test results:
testlist = pd.DataFrame({'text': sdm.Xt,
'teacher': sent2emoji(sdm.yt),
'teacher_sentiment': sdm.yt.tolist(),
'predict': sent2emoji(pred, custom_target_emojis=sdm.top_emojis),
'predicted_sentiment': pred.tolist()})
# display:
display(testlist.head())
# mean squared error:
teacher_sentiments = np.array([sample[1]['teacher_sentiment'] for sample in testlist.iterrows()])
predicted_sentiments = np.array([sample[1]['predicted_sentiment'] for sample in testlist.iterrows()])
mean_squared_error = ((teacher_sentiments - predicted_sentiments)**2).mean(axis=0)
print("Mean Squared Error: ", mean_squared_error)
print("Variance teacher: ", np.var(teacher_sentiments, axis=0))
print("Variance prediction: ", np.var(predicted_sentiments, axis=0))
# save to csv:
testlist.to_csv('test.csv')
# ----
# ## Load classifier
#
# * loading classifier and show a test widget
# In[13]:
import __main__ as main
if not hasattr(main, '__file__'):
try:
pm
except NameError:
pass
else:
del pm # delete existing pipeline manager if ther is one
pm = pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])
lookup_emojis = [#'😂',
'😭',
'😍',
'😩',
'😊',
'😘',
'🙏',
'🙌',
'😉',
'😁',
'😅',
'😎',
'😢',
'😒',
'😏',
'😌',
'😔',
'😋',
'😀',
'😤']
out = widgets.Output()
t = widgets.Text()
b = widgets.Button(
description='get emoji',
disabled=False,
button_style='', # 'success', 'info', 'warning', 'danger' or ''
tooltip='Click me',
icon='check'
)
def handle_submit(sender):
with out:
clear_output()
with out:
pred = pm.predict([t.value])
display(Markdown("# Predicted Emoji " + str(sent2emoji(pred, lookup_emojis)[0])))
display(Markdown("# Sentiment Vector: $$ \pmatrix{" + str(pred[0,0]) +
"\\\\" + str(pred[0,1]) + "\\\\" + str(pred[0,2]) + "}$$"))
b.on_click(handle_submit)
display(t)
display(widgets.VBox([b, out]))