diff --git a/Project/simple_approach/simple_twitter_learning.ipynb b/Project/simple_approach/simple_twitter_learning.ipynb index c2e623e..277e2b8 100644 --- a/Project/simple_approach/simple_twitter_learning.ipynb +++ b/Project/simple_approach/simple_twitter_learning.ipynb @@ -51,7 +51,8 @@ "import sklearn.utils as sku\n", "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import MultiLabelBinarizer\n", + "from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer\n", + "from sklearn.cluster import KMeans\n", "import nltk\n", "from keras.models import load_model\n", "from sklearn.externals import joblib\n", @@ -72,14 +73,13 @@ "import sys\n", "sys.path.append(\"..\")\n", "\n", - "from Tools.Emoji_Distance import sentiment_vector_to_emoji\n", - "from Tools.Emoji_Distance import emoji_to_sentiment_vector\n", + "import Tools.Emoji_Distance as edist\n", "\n", "def emoji2sent(emoji_arr, only_emoticons=True):\n", - " return np.array([emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr])\n", + " return np.array([edist.emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr])\n", "\n", "def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True):\n", - " return [sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr]" + " return [edist.sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr]" ] }, { @@ -200,7 +200,7 @@ "source": [ "class sample_data_manager(object):\n", " @staticmethod\n", - " def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None):\n", + " def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1):\n", " \"\"\"\n", " generate, read and process train data in one step.\n", " \n", @@ -209,6 +209,8 @@ " @param apply_stemming: apply stemming and lemmatization on dataset\n", " @param n_top_emojis: only use messages containing one of <`n_top_emojis`>-top emojis. set to `-1` to prevent top emoji filtering\n", " @param file_range: range of file's indices to read (eg `range(3)` to read the first three files). If `None`: all files are read\n", + " @param n_kmeans_cluster: generating multilabeled labels with kmeans with these number of clusters. Set to -1 to use the plain sentiment space as label\n", + " \n", " @return: sample_data_manager object\n", " \"\"\"\n", " sdm = sample_data_manager(path)\n", @@ -221,6 +223,9 @@ " if n_top_emojis > 0:\n", " sdm.filter_by_top_emojis(n_top=n_top_emojis)\n", " \n", + " if n_kmeans_cluster > 0:\n", + " sdm.generate_kmeans_binary_label(only_emoticons=only_emoticons, n_clusters=n_kmeans_cluster)\n", + " \n", " return sdm\n", " \n", " \n", @@ -244,6 +249,10 @@ " self.Xt = None\n", " self.yt = None\n", " self.top_emojis = None\n", + " self.binary_labels = None\n", + " self.use_binary_labels = False\n", + " self.kmeans_cluster = None\n", + " self.label_binarizer = None\n", " \n", " def read_files(self, file_index_range:list, only_emoticons=True):\n", " \"\"\"\n", @@ -348,16 +357,46 @@ " assert self.labels is not None # ← messages are already read in\n", " \n", " self.top_emojis = [x[0] for x in self.get_emoji_count()[:n_top]]\n", - " in_top = [sentiment_vector_to_emoji(x) in self.top_emojis for x in self.labels]\n", + " in_top = [edist.sentiment_vector_to_emoji(x) in self.top_emojis for x in self.labels]\n", " self.labels = self.labels[in_top]\n", " self.plain_text = self.plain_text[in_top]\n", " self.emojis = self.emojis[in_top]\n", " print(\"remaining samples after top emoji filtering: \", len(self.labels))\n", " \n", + " def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5):\n", + " \"\"\"\n", + " generate binary labels using kmeans.\n", + " \n", + " @param only_emoticons: set whether we're using the full emoji set or only emoticons\n", + " @param n_clusters: number of cluster we're generating in emoji's sentiment space\n", + " \"\"\"\n", + " assert self.labels is not None\n", + " array_sentiment_vectors = edist.list_sentiment_emoticon_vectors if only_emoticons else edist.list_sentiment_vectors\n", + " array_sentiment_vectors = np.array(array_sentiment_vectors)\n", + " \n", + " list_emojis = edist.list_emoticon_emojis if only_emoticons else edist.list_emojis\n", + " self.use_binary_labels = True\n", + " print(\"clustering following emojis: \" + \"\".join(list_emojis) + \"...\")\n", + " self.kmeans_cluster = KMeans(n_clusters=n_clusters).fit(array_sentiment_vectors)\n", + " print(\"clustering done\")\n", + " self.label_binarizer = LabelBinarizer()\n", + " \n", + " multiclass_labels = self.kmeans_cluster.predict(self.labels)\n", + " \n", + " # FIXME: we have to guarantee that in every dataset all classes occur.\n", + " # otherwise batch fitting is not possible!\n", + " # (or we have to precompute the mlb fitting process somewhere...)\n", + " self.binary_labels = self.label_binarizer.fit_transform(multiclass_labels)\n", + " \n", + " \n", " def create_train_test_split(self, split = 0.1, random_state = 4222):\n", + " assert self.plain_text is not None and self.labels is not None\n", " if self.X is not None:\n", " sys.stderr.write(\"WARNING: overwriting existing train/test split \\n\")\n", - " self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, self.labels, test_size=split, random_state=random_state)\n", + " \n", + " labels = self.binary_labels if self.use_binary_labels else self.labels\n", + " assert labels is not None\n", + " self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, labels, test_size=split, random_state=random_state)\n", "\n" ] }, @@ -392,13 +431,15 @@ " return pm\n", " \n", " @staticmethod\n", - " def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager):\n", + " def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None):\n", " '''\n", " creates pipeline with vectorizer and keras classifier\n", " \n", " @param vectorizer: Vectorizer object. will be fitted with data provided by sdm\n", " @param layers: list of keras layers. One keras layer is a tuple in form: (<#neurons:int>, )\n", " @param sdm: sample data manager to get data for the vectorizer\n", + " @param loss: set keras loss function. Depending whether sdm use multiclass labels `categorical_crossentropy` or `mean_squared_error` is used as default\n", + " @param optimizer: set keras optimizer. Depending whether sdm use multiclass labels `sgd` or `adam` is used as default\n", " \n", " @return: a pipeline manager object\n", " \n", @@ -423,8 +464,17 @@ " else:\n", " model.add(Dense(units=layer[0], activation=layer[1]))\n", " \n", - " model.compile(loss='mean_squared_error',\n", - " optimizer='adam')\n", + " if sdm.use_binary_labels: \n", + " loss_function = loss if loss is not None else 'categorical_crossentropy'\n", + " optimizer_function = optimizer if optimizer is not None else 'sgd'\n", + " model.compile(loss=loss_function,\n", + " optimizer=optimizer_function,\n", + " metrics=['accuracy'])\n", + " else:\n", + " loss_function = loss if loss is not None else 'mean_squared_error'\n", + " optimizer_function = optimizer if optimizer is not None else 'adam'\n", + " model.compile(loss=loss_function,\n", + " optimizer=optimizer_function)\n", " \n", " pipeline = Pipeline([\n", " ('vectorizer',vectorizer),\n", @@ -613,7 +663,7 @@ "imported 33368 samples\n", "remaining samples after top emoji filtering: 26197\n", "Epoch 1/1\n", - "100/100 [==============================] - 3s 27ms/step - loss: 0.1227\n" + "100/100 [==============================] - 3s 28ms/step - loss: 0.1230\n" ] } ], @@ -622,11 +672,18 @@ "if not hasattr(main, '__file__'):\n", " # we are in an interactive environment (probably in jupyter)\n", " # load data:\n", - " sdm = sample_data_manager.generate_and_read(path=\"./data_en/\", n_top_emojis=20, file_range=range(1))\n", + " \n", + " # setting n_kmeans_clusters to a value > 0 activates binarized labeling automatically! \n", + " # set to -1 to disable kmeans clustering and generating labels in plain sentiment space\n", + " \n", + " #n_kmeans_cluster = 5\n", + " n_kmeans_cluster = -1\n", + " sdm = sample_data_manager.generate_and_read(path=\"./data_en/\", n_top_emojis=20, file_range=range(1), n_kmeans_cluster=n_kmeans_cluster)\n", + " sdm.create_train_test_split()\n", " #pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\\n\",\n", " # layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\\n\",\n", " pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", - " layers=[(2500, 'relu'),(3,None)], sdm=sdm)\n", + " layers=[(2500, 'relu'),(sdm.y.shape[1],None)], sdm=sdm)\n", " tr = trainer(sdm=sdm, pm=pm)\n", " tr.fit(100)" ] @@ -641,7 +698,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -651,14 +708,6 @@ "['keras_model']\n" ] }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/jonas/.local/lib/python3.6/site-packages/keras/engine/sequential.py:109: UserWarning: `Sequential.model` is deprecated. `Sequential` is a subclass of `Model`, you can just use your `Sequential` instance directly.\n", - " warnings.warn('`Sequential.model` is deprecated. '\n" - ] - }, { "data": { "text/plain": [ @@ -701,19 +750,19 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[0.4423941 , 0.22976081, 0.26076168],\n", - " [0.75167173, 0.2919423 , 0.3423372 ],\n", - " [0.48943695, 0.21931192, 0.22773138],\n", + "array([[0.16062996, 0.08324276, 0.09433182],\n", + " [0.16413 , 0.09421383, 0.07578427],\n", + " [0.11994962, 0.05705731, 0.06310127],\n", " ...,\n", - " [0.51003224, 0.26002786, 0.25588542],\n", - " [0.5808168 , 0.30632192, 0.2964917 ],\n", - " [0.39000767, 0.31723523, 0.24713083]], dtype=float32)" + " [0.13887292, 0.08502828, 0.08176519],\n", + " [0.18185864, 0.09223703, 0.10704609],\n", + " [0.17687687, 0.09147045, 0.10650696]], dtype=float32)" ] }, "metadata": {}, @@ -738,7 +787,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "prediction variance: 0.009700283\n", + "prediction variance: 0.0005431187\n", "teacher variance: 0.03341702104519965\n" ] }, @@ -763,79 +812,79 @@ " \n", " \n", " \n", - " text\n", - " teacher\n", - " teacher_sentiment\n", " predict\n", " predicted_sentiment\n", + " teacher\n", + " teacher_sentiment\n", + " text\n", " \n", " \n", " \n", " \n", " 35671\n", - " i feel like i care so much more in everi situat\n", + " πŸ˜‚\n", + " [0.16062995791435242, 0.0832427591085434, 0.09...\n", " πŸ˜‚\n", " [0.46813021474490496, 0.24716181096977158, 0.2...\n", - " πŸ˜‚\n", - " [0.44239410758018494, 0.2297608107328415, 0.26...\n", + " i feel like i care so much more in everi situat\n", " \n", " \n", " 25683\n", - " i did not meat to add that 2 there ... hav see...\n", + " 😒\n", + " [0.16413000226020813, 0.0942138284444809, 0.07...\n", " πŸ˜‚\n", " [0.46813021474490496, 0.24716181096977158, 0.2...\n", - " 😌\n", - " [0.7516717314720154, 0.291942298412323, 0.3423...\n", + " i did not meat to add that 2 there ... hav see...\n", " \n", " \n", " 8985\n", - " never…\n", + " πŸ˜‚\n", + " [0.11994962394237518, 0.05705730617046356, 0.0...\n", " 😊\n", " [0.7040175768989329, 0.059322033898305086, 0.2...\n", - " πŸ˜‚\n", - " [0.48943695425987244, 0.21931192278862, 0.2277...\n", + " never…\n", " \n", " \n", " 5410\n", - " lmao on me ! ! ! wtf wa he suppos to say\n", + " πŸ˜‚\n", + " [0.18114930391311646, 0.10199417173862457, 0.1...\n", " πŸ˜‚\n", " [0.46813021474490496, 0.24716181096977158, 0.2...\n", - " 😒\n", - " [0.3661550283432007, 0.32579296827316284, 0.23...\n", + " lmao on me ! ! ! wtf wa he suppos to say\n", " \n", " \n", " 62611\n", - " this dude alway help me get through my school ...\n", + " πŸ˜‚\n", + " [0.16997836530208588, 0.08633847534656525, 0.0...\n", " 😊\n", " [0.7040175768989329, 0.059322033898305086, 0.2...\n", - " πŸ˜‚\n", - " [0.48689204454421997, 0.20729433000087738, 0.2...\n", + " this dude alway help me get through my school ...\n", " \n", " \n", "\n", "" ], "text/plain": [ - " text teacher \\\n", - "35671 i feel like i care so much more in everi situat πŸ˜‚ \n", - "25683 i did not meat to add that 2 there ... hav see... πŸ˜‚ \n", - "8985 never… 😊 \n", - "5410 lmao on me ! ! ! wtf wa he suppos to say πŸ˜‚ \n", - "62611 this dude alway help me get through my school ... 😊 \n", + " predict predicted_sentiment teacher \\\n", + "35671 πŸ˜‚ [0.16062995791435242, 0.0832427591085434, 0.09... πŸ˜‚ \n", + "25683 😒 [0.16413000226020813, 0.0942138284444809, 0.07... πŸ˜‚ \n", + "8985 πŸ˜‚ [0.11994962394237518, 0.05705730617046356, 0.0... 😊 \n", + "5410 πŸ˜‚ [0.18114930391311646, 0.10199417173862457, 0.1... πŸ˜‚ \n", + "62611 πŸ˜‚ [0.16997836530208588, 0.08633847534656525, 0.0... 😊 \n", "\n", - " teacher_sentiment predict \\\n", - "35671 [0.46813021474490496, 0.24716181096977158, 0.2... πŸ˜‚ \n", - "25683 [0.46813021474490496, 0.24716181096977158, 0.2... 😌 \n", - "8985 [0.7040175768989329, 0.059322033898305086, 0.2... πŸ˜‚ \n", - "5410 [0.46813021474490496, 0.24716181096977158, 0.2... 😒 \n", - "62611 [0.7040175768989329, 0.059322033898305086, 0.2... πŸ˜‚ \n", + " teacher_sentiment \\\n", + "35671 [0.46813021474490496, 0.24716181096977158, 0.2... \n", + "25683 [0.46813021474490496, 0.24716181096977158, 0.2... \n", + "8985 [0.7040175768989329, 0.059322033898305086, 0.2... \n", + "5410 [0.46813021474490496, 0.24716181096977158, 0.2... \n", + "62611 [0.7040175768989329, 0.059322033898305086, 0.2... \n", "\n", - " predicted_sentiment \n", - "35671 [0.44239410758018494, 0.2297608107328415, 0.26... \n", - "25683 [0.7516717314720154, 0.291942298412323, 0.3423... \n", - "8985 [0.48943695425987244, 0.21931192278862, 0.2277... \n", - "5410 [0.3661550283432007, 0.32579296827316284, 0.23... \n", - "62611 [0.48689204454421997, 0.20729433000087738, 0.2... " + " text \n", + "35671 i feel like i care so much more in everi situat \n", + "25683 i did not meat to add that 2 there ... hav see... \n", + "8985 never… \n", + "5410 lmao on me ! ! ! wtf wa he suppos to say \n", + "62611 this dude alway help me get through my school ... " ] }, "metadata": {}, @@ -845,9 +894,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean Squared Error: [0.02340565 0.02344435 0.00374819]\n", + "Mean Squared Error: [0.13877691 0.04682433 0.02937794]\n", "Variance teacher: [0.02183094 0.02513847 0.00285735]\n", - "Variance prediction: [0.0083875 0.00473354 0.00115709]\n" + "Variance prediction: [0.00046378 0.00019441 0.00020516]\n" ] } ], @@ -896,16 +945,31 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2ca4e06fcd4f41c2bfd161f9f16ca594", + "model_id": "003ae16760b04c25bdc9f2fe2193747a", "version_major": 2, "version_minor": 0 }, + "text/html": [ + "

Failed to display Jupyter Widget of type Text.

\n", + "

\n", + " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", + " that the widgets JavaScript is still loading. If this message persists, it\n", + " likely means that the widgets JavaScript library is either not installed or\n", + " not enabled. See the Jupyter\n", + " Widgets Documentation for setup instructions.\n", + "

\n", + "

\n", + " If you're reading this message in another frontend (for example, a static\n", + " rendering on GitHub or NBViewer),\n", + " it may mean that your frontend doesn't currently support widgets.\n", + "

\n" + ], "text/plain": [ "Text(value='')" ] @@ -916,12 +980,27 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a39abb79d70e4ae1952b2d928cfab174", + "model_id": "4580af82b30545f197a41e4359010556", "version_major": 2, "version_minor": 0 }, + "text/html": [ + "

Failed to display Jupyter Widget of type VBox.

\n", + "

\n", + " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", + " that the widgets JavaScript is still loading. If this message persists, it\n", + " likely means that the widgets JavaScript library is either not installed or\n", + " not enabled. See the Jupyter\n", + " Widgets Documentation for setup instructions.\n", + "

\n", + "

\n", + " If you're reading this message in another frontend (for example, a static\n", + " rendering on GitHub or NBViewer),\n", + " it may mean that your frontend doesn't currently support widgets.\n", + "

\n" + ], "text/plain": [ - "VBox(children=(Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='Click me'), Output(…" + "VBox(children=(Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))" ] }, "metadata": {}, @@ -987,13 +1066,6 @@ " display(t)\n", " display(widgets.VBox([b, out])) " ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index 0f0b944..efd8ac1 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -15,7 +15,8 @@ import itertools import sklearn.utils as sku from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer from sklearn.model_selection import train_test_split -from sklearn.preprocessing import MultiLabelBinarizer +from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer +from sklearn.cluster import KMeans import nltk from keras.models import load_model from sklearn.externals import joblib @@ -33,14 +34,13 @@ nltk.download('wordnet') import sys sys.path.append("..") -from Tools.Emoji_Distance import sentiment_vector_to_emoji -from Tools.Emoji_Distance import emoji_to_sentiment_vector +import Tools.Emoji_Distance as edist def emoji2sent(emoji_arr, only_emoticons=True): - return np.array([emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr]) + return np.array([edist.emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr]) def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True): - return [sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr] + return [edist.sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr] # In[3]: @@ -122,7 +122,7 @@ def get_wordnet_pos(treebank_tag): class sample_data_manager(object): @staticmethod - def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None): + def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1): """ generate, read and process train data in one step. @@ -131,6 +131,8 @@ class sample_data_manager(object): @param apply_stemming: apply stemming and lemmatization on dataset @param n_top_emojis: only use messages containing one of <`n_top_emojis`>-top emojis. set to `-1` to prevent top emoji filtering @param file_range: range of file's indices to read (eg `range(3)` to read the first three files). If `None`: all files are read + @param n_kmeans_cluster: generating multilabeled labels with kmeans with these number of clusters. Set to -1 to use the plain sentiment space as label + @return: sample_data_manager object """ sdm = sample_data_manager(path) @@ -143,6 +145,9 @@ class sample_data_manager(object): if n_top_emojis > 0: sdm.filter_by_top_emojis(n_top=n_top_emojis) + if n_kmeans_cluster > 0: + sdm.generate_kmeans_binary_label(only_emoticons=only_emoticons, n_clusters=n_kmeans_cluster) + return sdm @@ -166,6 +171,10 @@ class sample_data_manager(object): self.Xt = None self.yt = None self.top_emojis = None + self.binary_labels = None + self.use_binary_labels = False + self.kmeans_cluster = None + self.label_binarizer = None def read_files(self, file_index_range:list, only_emoticons=True): """ @@ -270,16 +279,46 @@ class sample_data_manager(object): assert self.labels is not None # ← messages are already read in self.top_emojis = [x[0] for x in self.get_emoji_count()[:n_top]] - in_top = [sentiment_vector_to_emoji(x) in self.top_emojis for x in self.labels] + in_top = [edist.sentiment_vector_to_emoji(x) in self.top_emojis for x in self.labels] self.labels = self.labels[in_top] self.plain_text = self.plain_text[in_top] self.emojis = self.emojis[in_top] print("remaining samples after top emoji filtering: ", len(self.labels)) + def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5): + """ + generate binary labels using kmeans. + + @param only_emoticons: set whether we're using the full emoji set or only emoticons + @param n_clusters: number of cluster we're generating in emoji's sentiment space + """ + assert self.labels is not None + array_sentiment_vectors = edist.list_sentiment_emoticon_vectors if only_emoticons else edist.list_sentiment_vectors + array_sentiment_vectors = np.array(array_sentiment_vectors) + + list_emojis = edist.list_emoticon_emojis if only_emoticons else edist.list_emojis + self.use_binary_labels = True + print("clustering following emojis: " + "".join(list_emojis) + "...") + self.kmeans_cluster = KMeans(n_clusters=n_clusters).fit(array_sentiment_vectors) + print("clustering done") + self.label_binarizer = LabelBinarizer() + + multiclass_labels = self.kmeans_cluster.predict(self.labels) + + # FIXME: we have to guarantee that in every dataset all classes occur. + # otherwise batch fitting is not possible! + # (or we have to precompute the mlb fitting process somewhere...) + self.binary_labels = self.label_binarizer.fit_transform(multiclass_labels) + + def create_train_test_split(self, split = 0.1, random_state = 4222): + assert self.plain_text is not None and self.labels is not None if self.X is not None: sys.stderr.write("WARNING: overwriting existing train/test split \n") - self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, self.labels, test_size=split, random_state=random_state) + + labels = self.binary_labels if self.use_binary_labels else self.labels + assert labels is not None + self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, labels, test_size=split, random_state=random_state) @@ -306,13 +345,15 @@ class pipeline_manager(object): return pm @staticmethod - def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager): + def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager, loss=None, optimizer=None): ''' creates pipeline with vectorizer and keras classifier @param vectorizer: Vectorizer object. will be fitted with data provided by sdm @param layers: list of keras layers. One keras layer is a tuple in form: (<#neurons:int>, ) @param sdm: sample data manager to get data for the vectorizer + @param loss: set keras loss function. Depending whether sdm use multiclass labels `categorical_crossentropy` or `mean_squared_error` is used as default + @param optimizer: set keras optimizer. Depending whether sdm use multiclass labels `sgd` or `adam` is used as default @return: a pipeline manager object @@ -337,8 +378,17 @@ class pipeline_manager(object): else: model.add(Dense(units=layer[0], activation=layer[1])) - model.compile(loss='mean_squared_error', - optimizer='adam') + if sdm.use_binary_labels: + loss_function = loss if loss is not None else 'categorical_crossentropy' + optimizer_function = optimizer if optimizer is not None else 'sgd' + model.compile(loss=loss_function, + optimizer=optimizer_function, + metrics=['accuracy']) + else: + loss_function = loss if loss is not None else 'mean_squared_error' + optimizer_function = optimizer if optimizer is not None else 'adam' + model.compile(loss=loss_function, + optimizer=optimizer_function) pipeline = Pipeline([ ('vectorizer',vectorizer), @@ -503,11 +553,18 @@ import __main__ as main if not hasattr(main, '__file__'): # we are in an interactive environment (probably in jupyter) # load data: - sdm = sample_data_manager.generate_and_read(path="./data_en/", n_top_emojis=20, file_range=range(1)) + + # setting n_kmeans_clusters to a value > 0 activates binarized labeling automatically! + # set to -1 to disable kmeans clustering and generating labels in plain sentiment space + + #n_kmeans_cluster = 5 + n_kmeans_cluster = -1 + sdm = sample_data_manager.generate_and_read(path="./data_en/", n_top_emojis=20, file_range=range(1), n_kmeans_cluster=n_kmeans_cluster) + sdm.create_train_test_split() #pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", # layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\n", pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'), - layers=[(2500, 'relu'),(3,None)], sdm=sdm) + layers=[(2500, 'relu'),(sdm.y.shape[1],None)], sdm=sdm) tr = trainer(sdm=sdm, pm=pm) tr.fit(100) @@ -515,7 +572,7 @@ if not hasattr(main, '__file__'): # ---- # ## save classifier -# In[13]: +# In[11]: import __main__ as main @@ -528,7 +585,7 @@ if not hasattr(main, '__file__'): # # * predict and save to `test.csv` -# In[14]: +# In[12]: import __main__ as main @@ -568,7 +625,7 @@ if not hasattr(main, '__file__'): # # * loading classifier and show a test widget -# In[15]: +# In[13]: import __main__ as main