diff --git a/Project/simple_approach/Continous_Learner.ipynb b/Project/simple_approach/Continous_Learner.ipynb index 59996ec..2cafe29 100644 --- a/Project/simple_approach/Continous_Learner.ipynb +++ b/Project/simple_approach/Continous_Learner.ipynb @@ -40,7 +40,8 @@ "import simple_twitter_learning as stl\n", "import glob\n", "import sys\n", - "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer" + "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", + "import pickle" ] }, { @@ -144,7 +145,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d304cda50752491da1637b292a9367e8", + "model_id": "5a488abefd074719adb15425714a076f", "version_major": 2, "version_minor": 0 }, @@ -205,6 +206,7 @@ " ],\n", " [\n", " (widgets.Checkbox(value=True),\"use_doc2vec\"),\n", + " (widgets.Checkbox(value=True),\"d2v_use_pretrained\"),\n", " (widgets.IntText(value=100),\"d2v_size\"),\n", " (widgets.IntText(value=8), \"d2v_window\"),\n", " (widgets.IntSlider(value=5, min=0, max=32), \"d2v_min_count\")\n", @@ -558,9 +560,12 @@ " # creating the vectorizer\n", " vectorizer = None\n", " if shown_widgets[\"use_doc2vec\"].value:\n", - " vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n", - " window=shown_widgets[\"d2v_window\"].value,\n", - " min_count=shown_widgets[\"d2v_min_count\"].value)\n", + " if shown_widgets[\"d2v_use_pretrained\"].value:\n", + " vectorizer = pickle.load( open( \"doc2VecModel.p\", \"rb\" ) )\n", + " else:\n", + " vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n", + " window=shown_widgets[\"d2v_window\"].value,\n", + " min_count=shown_widgets[\"d2v_min_count\"].value)\n", " else:\n", " vectorizer=TfidfVectorizer(stop_words='english')\n", " \n", diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index 88267b3..c9b82c7 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -52,7 +52,6 @@ def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True): # In[3]: - SINGLE_LABEL = True @@ -431,7 +430,7 @@ class pipeline_manager(object): return pm @staticmethod - def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None): + def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None, fit_vectorizer=True): ''' creates pipeline with vectorizer and keras classifier @@ -447,11 +446,12 @@ class pipeline_manager(object): from keras.models import Sequential from keras.layers import Dense - if sdm.X is None: - sdm.create_train_test_split() - - vec_train = vectorizer.fit_transform(sdm.X) - vec_test = vectorizer.transform(sdm.Xt) + if fit_vectorizer: + if sdm.X is None: + sdm.create_train_test_split() + + vec_train = vectorizer.fit_transform(sdm.X) + vec_test = vectorizer.transform(sdm.Xt) # creating keras model: model=Sequential() @@ -578,7 +578,7 @@ class pipeline_manager(object): """fitting the pipeline""" self.pipeline.fit(X,y) - def predict(self,X, use_stemming=True, use_lemmatization=True): + def predict(self,X, use_stemming=False, use_lemmatization=False): """predict""" if use_stemming: X = np.array(batch_stem(X)) @@ -608,7 +608,7 @@ class trainer(object): self.sdm = sdm self.pm = pm - def fit(self, max_size=10000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None): + def fit(self, max_size=1000000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None): """ fitting data in the pipeline. Because we don't want to refit the vectorizer, the pipeline models containing the vectorizer have to be named explicitly