jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added possibility to load pretrained doc2vecD

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-10 13:48:51 +02:00
parent 951b552f61
commit b4ae0b033e
2 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Project/simple_approach/Continous_Learner.ipynb
View File

@ -40,7 +40,8 @@
"import simple_twitter_learning as stl\n",
"import glob\n",
"import sys\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer"
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"import pickle"
]
},
{
@ -144,7 +145,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d304cda50752491da1637b292a9367e8",
"model_id": "5a488abefd074719adb15425714a076f",
"version_major": 2,
"version_minor": 0
},
@ -205,6 +206,7 @@
" ],\n",
" [\n",
" (widgets.Checkbox(value=True),\"use_doc2vec\"),\n",
" (widgets.Checkbox(value=True),\"d2v_use_pretrained\"),\n",
" (widgets.IntText(value=100),\"d2v_size\"),\n",
" (widgets.IntText(value=8), \"d2v_window\"),\n",
" (widgets.IntSlider(value=5, min=0, max=32), \"d2v_min_count\")\n",
@ -558,9 +560,12 @@
" # creating the vectorizer\n",
" vectorizer = None\n",
" if shown_widgets[\"use_doc2vec\"].value:\n",
" vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n",
" window=shown_widgets[\"d2v_window\"].value,\n",
" min_count=shown_widgets[\"d2v_min_count\"].value)\n",
" if shown_widgets[\"d2v_use_pretrained\"].value:\n",
" vectorizer = pickle.load( open( \"doc2VecModel.p\", \"rb\" ) )\n",
" else:\n",
" vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n",
" window=shown_widgets[\"d2v_window\"].value,\n",
" min_count=shown_widgets[\"d2v_min_count\"].value)\n",
" else:\n",
" vectorizer=TfidfVectorizer(stop_words='english')\n",
" \n",

16
Project/simple_approach/simple_twitter_learning.py
View File

@ -52,7 +52,6 @@ def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True):
# In[3]:
SINGLE_LABEL = True
@ -431,7 +430,7 @@ class pipeline_manager(object):
return pm
@staticmethod
def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None):
def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None, fit_vectorizer=True):
'''
creates pipeline with vectorizer and keras classifier
@ -447,11 +446,12 @@ class pipeline_manager(object):
from keras.models import Sequential
from keras.layers import Dense
if sdm.X is None:
sdm.create_train_test_split()
if fit_vectorizer:
if sdm.X is None:
sdm.create_train_test_split()
vec_train = vectorizer.fit_transform(sdm.X)
vec_test = vectorizer.transform(sdm.Xt)
vec_train = vectorizer.fit_transform(sdm.X)
vec_test = vectorizer.transform(sdm.Xt)
# creating keras model:
model=Sequential()
@ -578,7 +578,7 @@ class pipeline_manager(object):
"""fitting the pipeline"""
self.pipeline.fit(X,y)
def predict(self,X, use_stemming=True, use_lemmatization=True):
def predict(self,X, use_stemming=False, use_lemmatization=False):
"""predict"""
if use_stemming:
X = np.array(batch_stem(X))
@ -608,7 +608,7 @@ class trainer(object):
self.sdm = sdm
self.pm = pm
def fit(self, max_size=10000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None):
def fit(self, max_size=1000000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None):
"""
fitting data in the pipeline. Because we don't want to refit the vectorizer, the pipeline models containing the vectorizer have to be named explicitly