jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added possibility to load pretrained doc2vecD

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-10 13:48:51 +02:00
parent 951b552f61
commit b4ae0b033e
2 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Project/simple_approach/Continous_Learner.ipynb
View File

@ -40,7 +40,8 @@
"import simple_twitter_learning as stl\n", "import simple_twitter_learning as stl\n",
"import glob\n", "import glob\n",
"import sys\n", "import sys\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer" "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"import pickle"
] ]
}, },
{ {
@ -144,7 +145,7 @@
{ {
"data": { "data": {
"application/vnd.jupyter.widget-view+json": { "application/vnd.jupyter.widget-view+json": {
"model_id": "d304cda50752491da1637b292a9367e8", "model_id": "5a488abefd074719adb15425714a076f",
"version_major": 2, "version_major": 2,
"version_minor": 0 "version_minor": 0
}, },
@ -205,6 +206,7 @@
" ],\n", " ],\n",
" [\n", " [\n",
" (widgets.Checkbox(value=True),\"use_doc2vec\"),\n", " (widgets.Checkbox(value=True),\"use_doc2vec\"),\n",
" (widgets.Checkbox(value=True),\"d2v_use_pretrained\"),\n",
" (widgets.IntText(value=100),\"d2v_size\"),\n", " (widgets.IntText(value=100),\"d2v_size\"),\n",
" (widgets.IntText(value=8), \"d2v_window\"),\n", " (widgets.IntText(value=8), \"d2v_window\"),\n",
" (widgets.IntSlider(value=5, min=0, max=32), \"d2v_min_count\")\n", " (widgets.IntSlider(value=5, min=0, max=32), \"d2v_min_count\")\n",
@ -558,9 +560,12 @@
" # creating the vectorizer\n", " # creating the vectorizer\n",
" vectorizer = None\n", " vectorizer = None\n",
" if shown_widgets[\"use_doc2vec\"].value:\n", " if shown_widgets[\"use_doc2vec\"].value:\n",
" vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n", " if shown_widgets[\"d2v_use_pretrained\"].value:\n",
" window=shown_widgets[\"d2v_window\"].value,\n", " vectorizer = pickle.load( open( \"doc2VecModel.p\", \"rb\" ) )\n",
" min_count=shown_widgets[\"d2v_min_count\"].value)\n", " else:\n",
" vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n",
" window=shown_widgets[\"d2v_window\"].value,\n",
" min_count=shown_widgets[\"d2v_min_count\"].value)\n",
" else:\n", " else:\n",
" vectorizer=TfidfVectorizer(stop_words='english')\n", " vectorizer=TfidfVectorizer(stop_words='english')\n",
" \n", " \n",

18
Project/simple_approach/simple_twitter_learning.py
View File

@ -52,7 +52,6 @@ def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True):
# In[3]: # In[3]:
SINGLE_LABEL = True SINGLE_LABEL = True
@ -431,7 +430,7 @@ class pipeline_manager(object):
return pm return pm
@staticmethod @staticmethod
def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None): def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None, fit_vectorizer=True):
''' '''
creates pipeline with vectorizer and keras classifier creates pipeline with vectorizer and keras classifier
@ -447,11 +446,12 @@ class pipeline_manager(object):
from keras.models import Sequential from keras.models import Sequential
from keras.layers import Dense from keras.layers import Dense
if sdm.X is None: if fit_vectorizer:
sdm.create_train_test_split() if sdm.X is None:
sdm.create_train_test_split()
vec_train = vectorizer.fit_transform(sdm.X)
vec_test = vectorizer.transform(sdm.Xt) vec_train = vectorizer.fit_transform(sdm.X)
vec_test = vectorizer.transform(sdm.Xt)
# creating keras model: # creating keras model:
model=Sequential() model=Sequential()
@ -578,7 +578,7 @@ class pipeline_manager(object):
"""fitting the pipeline""" """fitting the pipeline"""
self.pipeline.fit(X,y) self.pipeline.fit(X,y)
def predict(self,X, use_stemming=True, use_lemmatization=True): def predict(self,X, use_stemming=False, use_lemmatization=False):
"""predict""" """predict"""
if use_stemming: if use_stemming:
X = np.array(batch_stem(X)) X = np.array(batch_stem(X))
@ -608,7 +608,7 @@ class trainer(object):
self.sdm = sdm self.sdm = sdm
self.pm = pm self.pm = pm
def fit(self, max_size=10000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None): def fit(self, max_size=1000000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None):
""" """
fitting data in the pipeline. Because we don't want to refit the vectorizer, the pipeline models containing the vectorizer have to be named explicitly fitting data in the pipeline. Because we don't want to refit the vectorizer, the pipeline models containing the vectorizer have to be named explicitly