jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

integrated word to vec

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-03 14:20:31 +02:00
parent bde4216707
commit 8abebc5957
2 changed files with 47 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
Project/simple_approach/Continous_Learner.ipynb
View File

@ -144,7 +144,7 @@
{ {
"data": { "data": {
"application/vnd.jupyter.widget-view+json": { "application/vnd.jupyter.widget-view+json": {
"model_id": "4fd5552e6a024dcaa0f35a594c77ae99", "model_id": "d018a59d95fe45f2ae7be013a49b5900",
"version_major": 2, "version_major": 2,
"version_minor": 0 "version_minor": 0
}, },
@ -168,7 +168,8 @@
" ],\n", " ],\n",
" [\n", " [\n",
" (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n", " (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n",
" (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\")\n", " (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\"),\n",
" (widgets.Checkbox(value=False,disabled=True), \"apply_lemmatization_and_stemming\")\n",
" ],\n", " ],\n",
" [\n", " [\n",
" (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", " (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
@ -203,6 +204,12 @@
" (classifier_tab, \"classifier_tab\")\n", " (classifier_tab, \"classifier_tab\")\n",
" ],\n", " ],\n",
" [\n", " [\n",
" (widgets.Checkbox(value=True),\"use_doc2vec\"),\n",
" (widgets.IntText(value=100),\"d2v_size\"),\n",
" (widgets.IntText(value=8), \"d2v_window\"),\n",
" (widgets.IntSlider(value=5, min=0, max=32), \"d2v_min_count\")\n",
" ],\n",
" [\n",
" (widgets.Button(), \"create_classifier\")\n", " (widgets.Button(), \"create_classifier\")\n",
" ],\n", " ],\n",
" [\n", " [\n",
@ -406,6 +413,7 @@
" \"only_emoticons\",\n", " \"only_emoticons\",\n",
" \"k_means_cluster\",\n", " \"k_means_cluster\",\n",
" \"n_top_emojis\",\n", " \"n_top_emojis\",\n",
" \"apply_lemmatization_and_stemming\",\n",
" \"load_data\"], False)\n", " \"load_data\"], False)\n",
" return\n", " return\n",
" \n", " \n",
@ -415,6 +423,7 @@
" \"only_emoticons\",\n", " \"only_emoticons\",\n",
" \"k_means_cluster\",\n", " \"k_means_cluster\",\n",
" \"n_top_emojis\",\n", " \"n_top_emojis\",\n",
" \"apply_lemmatization_and_stemming\",\n",
" \"load_data\"], True)\n", " \"load_data\"], True)\n",
" shown_widgets[\"file_range\"].min=0\n", " shown_widgets[\"file_range\"].min=0\n",
" shown_widgets[\"file_range\"].max=len(files) -1\n", " shown_widgets[\"file_range\"].max=len(files) -1\n",
@ -429,14 +438,19 @@
" r = (r[0], r[1] + 1) # range has to be exclusive according to the last element!\n", " r = (r[0], r[1] + 1) # range has to be exclusive according to the last element!\n",
" \n", " \n",
" p_r = progress_indicator(\"reading progress\")\n", " p_r = progress_indicator(\"reading progress\")\n",
" p_s = progress_indicator(\"stemming progress\")\n", " \n",
" lemm_and_stemm = shown_widgets[\"apply_lemmatization_and_stemming\"].value\n",
" \n",
" if lemm_and_stemm:\n",
" p_s = progress_indicator(\"stemming progress\")\n",
" \n", " \n",
" sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", " sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n",
" n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", " n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n",
" file_range=range(r[0], r[1]),\n", " file_range=range(r[0], r[1]),\n",
" n_kmeans_cluster=shown_widgets[\"k_means_cluster\"].value,\n", " n_kmeans_cluster=shown_widgets[\"k_means_cluster\"].value,\n",
" read_progress_callback=p_r.update,\n", " read_progress_callback=p_r.update,\n",
" stem_progress_callback=p_s.update)\n", " stem_progress_callback=p_s.update if lemm_and_stemm else None,\n",
" apply_stemming = lemm_and_stemm)\n",
" shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", " shown_widgets[\"batch_size\"].max = len(sdm.labels)\n",
" \n", " \n",
" \n", " \n",
@ -541,6 +555,15 @@
" \n", " \n",
" mp(\"**chosen classifier**: `\" + chosen_classifier + \"`\")\n", " mp(\"**chosen classifier**: `\" + chosen_classifier + \"`\")\n",
" \n", " \n",
" # creating the vectorizer\n",
" vectorizer = None\n",
" if shown_widgets[\"use_doc2vec\"].value:\n",
" vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n",
" window=shown_widgets[\"d2v_window\"].value,\n",
" min_count=shown_widgets[\"d2v_min_count\"].value)\n",
" else:\n",
" vectorizer=TfidfVectorizer(stop_words='english')\n",
" \n",
" # TODO: add more classifier options here:\n", " # TODO: add more classifier options here:\n",
" if chosen_classifier is 'keras':\n", " if chosen_classifier is 'keras':\n",
" sdm.create_train_test_split()\n", " sdm.create_train_test_split()\n",
@ -562,8 +585,7 @@
" mp(\"**layers:** \")\n", " mp(\"**layers:** \")\n",
" jp(layers, headers=['#neurons', 'activation_func'])\n", " jp(layers, headers=['#neurons', 'activation_func'])\n",
"\n", "\n",
" pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", " pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer, layers=layers, sdm=sdm)\n",
" layers=layers, sdm=sdm)\n",
"\n", "\n",
"def save_classifier(b):\n", "def save_classifier(b):\n",
" global sdm\n", " global sdm\n",

23
Project/simple_approach/simple_twitter_learning.py
View File

@ -42,6 +42,7 @@ import sys
sys.path.append("..") sys.path.append("..")
import Tools.Emoji_Distance as edist import Tools.Emoji_Distance as edist
import Tools.sklearn_doc2vec as skd2v
def emoji2sent(emoji_arr, only_emoticons=True): def emoji2sent(emoji_arr, only_emoticons=True):
return np.array([edist.emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr]) return np.array([edist.emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr])
@ -49,7 +50,6 @@ def emoji2sent(emoji_arr, only_emoticons=True):
def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True): def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True):
return [edist.sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr] return [edist.sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr]
# In[3]: # In[3]:
@ -440,7 +440,7 @@ class pipeline_manager(object):
@param sdm: sample data manager to get data for the vectorizer @param sdm: sample data manager to get data for the vectorizer
@param loss: set keras loss function. Depending whether sdm use multiclass labels `categorical_crossentropy` or `mean_squared_error` is used as default @param loss: set keras loss function. Depending whether sdm use multiclass labels `categorical_crossentropy` or `mean_squared_error` is used as default
@param optimizer: set keras optimizer. Depending whether sdm use multiclass labels `sgd` or `adam` is used as default @param optimizer: set keras optimizer. Depending whether sdm use multiclass labels `sgd` or `adam` is used as default
@return: a pipeline manager object @return: a pipeline manager object
''' '''
@ -459,7 +459,12 @@ class pipeline_manager(object):
first_layer = True first_layer = True
for layer in layers: for layer in layers:
if first_layer: if first_layer:
model.add(Dense(units=layer[0], activation=layer[1], input_dim=vectorizer.transform([" "])[0]._shape[1])) size = None
if "size" in dir(vectorizer):
size = vectorizer.size
else:
size = vectorizer.transform([" "])[0]._shape[1]
model.add(Dense(units=layer[0], activation=layer[1], input_dim=size))
first_layer = False first_layer = False
else: else:
model.add(Dense(units=layer[0], activation=layer[1])) model.add(Dense(units=layer[0], activation=layer[1]))
@ -587,6 +592,15 @@ class pipeline_manager(object):
# In[9]: # In[9]:
def to_dense_if_sparse(X):
"""
little hepler function to make data dense (if it is sparse).
is used in trainer.fit function
"""
if "todense" in dir(X):
return X.todense()
return X
class trainer(object): class trainer(object):
def __init__(self, sdm:sample_data_manager, pm:pipeline_manager): def __init__(self, sdm:sample_data_manager, pm:pipeline_manager):
@ -622,7 +636,8 @@ class trainer(object):
for k in keras_batch_fitting_layer: for k in keras_batch_fitting_layer:
# forcing batch fitting on keras # forcing batch fitting on keras
disabled_keras_fits[k]=named_steps[k].fit disabled_keras_fits[k]=named_steps[k].fit
named_steps[k].fit = lambda X, y: named_steps[k].train_on_batch(X.todense(), y) # ← why has keras no sparse support on batch progressing!?!?!
named_steps[k].fit = lambda X, y: named_steps[k].train_on_batch(to_dense_if_sparse(X), y) # ← why has keras no sparse support on batch progressing!?!?!
if batch_size is None: if batch_size is None:
self.pm.fit(X = self.sdm.X[:max_size], y = self.sdm.y[:max_size]) self.pm.fit(X = self.sdm.X[:max_size], y = self.sdm.y[:max_size])