added possibility to load pretrained doc2vecD
This commit is contained in:
		| @ -40,7 +40,8 @@ | ||||
|     "import simple_twitter_learning as stl\n", | ||||
|     "import glob\n", | ||||
|     "import sys\n", | ||||
|     "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer" | ||||
|     "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", | ||||
|     "import pickle" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
| @ -144,7 +145,7 @@ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "d304cda50752491da1637b292a9367e8", | ||||
|        "model_id": "5a488abefd074719adb15425714a076f", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
| @ -205,6 +206,7 @@ | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Checkbox(value=True),\"use_doc2vec\"),\n", | ||||
|     "                   (widgets.Checkbox(value=True),\"d2v_use_pretrained\"),\n", | ||||
|     "                   (widgets.IntText(value=100),\"d2v_size\"),\n", | ||||
|     "                   (widgets.IntText(value=8), \"d2v_window\"),\n", | ||||
|     "                   (widgets.IntSlider(value=5, min=0, max=32), \"d2v_min_count\")\n", | ||||
| @ -558,9 +560,12 @@ | ||||
|     "        # creating the vectorizer\n", | ||||
|     "        vectorizer = None\n", | ||||
|     "        if shown_widgets[\"use_doc2vec\"].value:\n", | ||||
|     "            vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n", | ||||
|     "                                                     window=shown_widgets[\"d2v_window\"].value,\n", | ||||
|     "                                                     min_count=shown_widgets[\"d2v_min_count\"].value)\n", | ||||
|     "            if shown_widgets[\"d2v_use_pretrained\"].value:\n", | ||||
|     "                vectorizer = pickle.load( open( \"doc2VecModel.p\", \"rb\" ) )\n", | ||||
|     "            else:\n", | ||||
|     "                vectorizer = stl.skd2v.Doc2VecTransformer(size=shown_widgets[\"d2v_size\"].value,\n", | ||||
|     "                                                         window=shown_widgets[\"d2v_window\"].value,\n", | ||||
|     "                                                         min_count=shown_widgets[\"d2v_min_count\"].value)\n", | ||||
|     "        else:\n", | ||||
|     "            vectorizer=TfidfVectorizer(stop_words='english')\n", | ||||
|     "        \n", | ||||
|  | ||||
| @ -52,7 +52,6 @@ def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True): | ||||
|  | ||||
| # In[3]: | ||||
|  | ||||
|  | ||||
| SINGLE_LABEL = True | ||||
|  | ||||
|  | ||||
| @ -431,7 +430,7 @@ class pipeline_manager(object): | ||||
|         return pm | ||||
|      | ||||
|     @staticmethod | ||||
|     def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None): | ||||
|     def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None, fit_vectorizer=True): | ||||
|         ''' | ||||
|         creates pipeline with vectorizer and keras classifier | ||||
|          | ||||
| @ -447,11 +446,12 @@ class pipeline_manager(object): | ||||
|         from keras.models import Sequential | ||||
|         from keras.layers import Dense | ||||
|          | ||||
|         if sdm.X is None: | ||||
|             sdm.create_train_test_split() | ||||
|          | ||||
|         vec_train = vectorizer.fit_transform(sdm.X) | ||||
|         vec_test = vectorizer.transform(sdm.Xt) | ||||
|         if fit_vectorizer: | ||||
|             if sdm.X is None: | ||||
|                 sdm.create_train_test_split() | ||||
|  | ||||
|             vec_train = vectorizer.fit_transform(sdm.X) | ||||
|             vec_test = vectorizer.transform(sdm.Xt) | ||||
|         # creating keras model: | ||||
|         model=Sequential() | ||||
|          | ||||
| @ -578,7 +578,7 @@ class pipeline_manager(object): | ||||
|         """fitting the pipeline""" | ||||
|         self.pipeline.fit(X,y) | ||||
|      | ||||
|     def predict(self,X, use_stemming=True, use_lemmatization=True): | ||||
|     def predict(self,X, use_stemming=False, use_lemmatization=False): | ||||
|         """predict""" | ||||
|         if use_stemming: | ||||
|             X = np.array(batch_stem(X)) | ||||
| @ -608,7 +608,7 @@ class trainer(object): | ||||
|         self.sdm = sdm | ||||
|         self.pm = pm | ||||
|      | ||||
|     def fit(self, max_size=10000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None): | ||||
|     def fit(self, max_size=1000000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None): | ||||
|         """ | ||||
|         fitting data in the pipeline. Because we don't want to refit the vectorizer, the pipeline models containing the vectorizer have to be named explicitly | ||||
|          | ||||
|  | ||||
		Reference in New Issue
	
	Block a user