Merge branch 'master' of ssh://gogs@the-cake-is-a-lie.net:20022/jonas/NLP-LAB.git
This commit is contained in:
		
							
								
								
									
										44
									
								
								Project/Tools/sklearn_doc2vec.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										44
									
								
								Project/Tools/sklearn_doc2vec.py
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,44 @@ | ||||
| #!/usr/bin/env python3 | ||||
| from gensim.models import doc2vec | ||||
| from collections import namedtuple | ||||
| from gensim.utils import to_unicode | ||||
| from sklearn.base import BaseEstimator, TransformerMixin | ||||
| import numpy as np | ||||
|  | ||||
| """ | ||||
| This is a litte helper module providing a doc2vec class | ||||
| which can be thrown into a sklearn pipeline. A little bit modified taken from: | ||||
| https://github.com/fanta-mnix/sklearn-doc2vec/blob/master/word_embeddings.py | ||||
| """ | ||||
|  | ||||
| def documentize(X): | ||||
|     docs = [] | ||||
|     analyzedDocument = namedtuple('AnalyzedDocument', 'words tags') | ||||
|     for i, text in enumerate(X): | ||||
|         words = text.lower().split() | ||||
|         tags = [i] | ||||
|         docs.append(analyzedDocument(words, tags)) | ||||
|     return docs | ||||
|  | ||||
| class Doc2VecTransformer(BaseEstimator, TransformerMixin): | ||||
|  | ||||
|     def __init__(self, size=300, window=8, min_count=5): | ||||
|         self.size = size | ||||
|         self.window = window | ||||
|         self.min_count = min_count | ||||
|         self._model = None | ||||
|  | ||||
|     def fit(self, X, y=None): | ||||
|         model = doc2vec.Doc2Vec(documentize(X), size=self.size, window=self.window, min_count=self.min_count) | ||||
|  | ||||
|         self._model = model | ||||
|         return self | ||||
|  | ||||
|     def fit_transform(self, X, y=None, **fit_params): | ||||
|         self.fit(X, y) | ||||
|         return self._model.docvecs | ||||
|  | ||||
|     def transform(self, X, copy=True): | ||||
|         assert self._model is not None, 'model is not fitted' | ||||
|         return np.array([self._model.infer_vector(document.words) for document in documentize(X)]) | ||||
|  | ||||
		Reference in New Issue
	
	Block a user