diff --git a/Project/simple_approach/simple_twitter_learning.ipynb b/Project/simple_approach/simple_twitter_learning.ipynb index 1dd7ab5..de24a90 100644 --- a/Project/simple_approach/simple_twitter_learning.ipynb +++ b/Project/simple_approach/simple_twitter_learning.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -25,7 +25,7 @@ "True" ] }, - "execution_count": 1, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -44,6 +44,10 @@ "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import MultiLabelBinarizer\n", "import nltk\n", + "from keras.models import load_model\n", + "from sklearn.externals import joblib\n", + "import operator\n", + "from sklearn.pipeline import Pipeline\n", "nltk.download('punkt')\n", "nltk.download('averaged_perceptron_tagger')\n", "nltk.download('wordnet')" @@ -51,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -61,11 +65,11 @@ "from Tools.Emoji_Distance import sentiment_vector_to_emoji\n", "from Tools.Emoji_Distance import emoji_to_sentiment_vector\n", "\n", - "def emoji2sent(emoji_arr):\n", - " return np.array([emoji_to_sentiment_vector(e) for e in emoji_arr])\n", + "def emoji2sent(emoji_arr, only_emoticons=True):\n", + " return np.array([emoji_to_sentiment_vector(e, only_emoticons=only_emoticons) for e in emoji_arr])\n", "\n", - "def sent2emoji(sent_arr, custom_target_emojis=None):\n", - " return [sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis) for s in sent_arr]" + "def sent2emoji(sent_arr, custom_target_emojis=None, only_emoticons=True):\n", + " return [sentiment_vector_to_emoji(s, custom_target_emojis=custom_target_emojis, only_emoticons=only_emoticons) for s in sent_arr]" ] }, { @@ -77,6 +81,311 @@ "SINGLE_LABEL = True" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "----\n", + "## classes and functions we are using later:\n", + "----" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* functions for selecting items from a set / list" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def latest(lst):\n", + " return lst[-1] if len(lst) > 0 else 'X' \n", + "def most_common(lst):\n", + " # trying to find the most common used emoji in the given lst\n", + " return max(set(lst), key=lst.count) if len(lst) > 0 else \"X\" # setting label to 'X' if there is an empty emoji list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* our emoji blacklist (skin and sex modifiers)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# defining blacklist for modifier emojis:\n", + "emoji_blacklist = set([\n", + " chr(0x1F3FB),\n", + " chr(0x1F3FC),\n", + " chr(0x1F3FD),\n", + " chr(0x1F3FE),\n", + " chr(0x1F3FF),\n", + " chr(0x2642),\n", + " chr(0x2640)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* lemmatization helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from nltk.stem.snowball import SnowballStemmer\n", + "from nltk.stem import WordNetLemmatizer\n", + "from nltk import pos_tag\n", + "from nltk import word_tokenize\n", + "from nltk.corpus import wordnet\n", + "\n", + "def get_wordnet_pos(treebank_tag):\n", + "\n", + " if treebank_tag.startswith('J'):\n", + " return wordnet.ADJ\n", + " elif treebank_tag.startswith('V'):\n", + " return wordnet.VERB\n", + " elif treebank_tag.startswith('N'):\n", + " return wordnet.NOUN\n", + " elif treebank_tag.startswith('R'):\n", + " return wordnet.ADV\n", + " else:\n", + " return wordnet.NOUN" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* the pipeline manager saves and stores sklearn pipelines. Keras models are handled differently, so the have to be named explicitly during save and load operations" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "class pipeline_manager(object):\n", + " @staticmethod\n", + " def load_pipeline_from_files(file_prefix:str, keras_models = []):\n", + " pm = pipeline_manager()\n", + " pm.load(file_prefix, keras_models=keras_models)\n", + " return pm\n", + " \n", + " def __init__(self, pipeline = None):\n", + " self.pipeline = pipeline\n", + " self.additional_objects = {}\n", + " \n", + " def save(prefix:str, keras_models = []):\n", + " # doing this like explained here: https://stackoverflow.com/a/43415459\n", + " for km in keras_models:\n", + " self.pipeline.named_steps[km].model.save(prefix + \".\" + km)\n", + " # setting this part to None:\n", + " self.pipeline.named_steps[km].model = None\n", + " \n", + " # now we can save the pipeline:\n", + " joblib.dump(self.pipeline, prefix + \".pipeline\")\n", + " \n", + " def load(prefix:str, keras_models=[]):\n", + " self.pipeline = joblib.load(prefix + \".pipeline\")\n", + " for km in keras_models:\n", + " self.pipeline.named_steps[km].model = load_model(prefix + \".\" + km)\n", + " \n", + " def fit(self,X,y):\n", + " self.pipeline.fit(X,y)\n", + " \n", + " def predict(self,X):\n", + " self.pipeline.predict(X)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* the sample data manager loads and preprocesses data" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "class sample_data_manager(object):\n", + " @staticmethod\n", + " def generate_and_read(path:str):\n", + " sdm = sample_data_manager(path):\n", + " \n", + " def __init__(self, data_root_folder:str):\n", + " self.data_root_folder = data_root_folder\n", + " self.json_files = sorted(glob.glob(self.data_root_folder + \"/*.json\"))\n", + " self.n_files = len(self.json_files)\n", + " self.raw_data = None\n", + " self.emojis = None\n", + " self.plain_text = None\n", + " self.labels = None\n", + " self.emoji_count = None\n", + " self.emoji_weights = None\n", + " self.X = None\n", + " self.y = None\n", + " self.Xt = None\n", + " self.yt = None\n", + " \n", + " def read_files(self, file_index_range:list, only_emoticons=True):\n", + " assert np.min(file_index_range) >= 0 and np.max(file_index_range) < self.n_files\n", + " for i in file_index_range:\n", + " print(\"reaing file: \" + self.json_files[i] + \"...\")\n", + " if self.raw_data is None:\n", + " self.raw_data = pd.read_json(self.json_files[i], encoding=\"utf-8\")\n", + " else:\n", + " self.raw_data = self.raw_data.append(pd.read_json(self.json_files[i], encoding=\"utf-8\"))\n", + " \n", + " self.emojis = self.raw_data['EMOJI']\n", + " self.plain_text = self.raw_data['text']\n", + " \n", + " # replacing keywords. TODO: maybe these information can be extracted and used\n", + " self.plain_text = self.plain_text.str.replace(\"(||)\",\"\").str.replace(\"[\" + \"\".join(list(emoji_blacklist)) + \"]\",\"\")\n", + " \n", + " # so far filtering for the latest emoji. TODO: maybe there are also better approaches\n", + " self.labels = emoji2sent([latest(e) for e in emojis], only_emoticons=only_emoticons )\n", + " \n", + " # and filter out all samples we have no label for:\n", + " wrong_labels = np.isnan(np.linalg.norm(self.labels, axis=1)) \n", + "\n", + " self.labels = self.labels[np.invert(wrong_labels)]\n", + " self.plain_text = self.plain_text[np.invert(wrong_labels)]\n", + " self.emojis = self.emojis[np.invert(wrong_labels)]\n", + " \n", + " print(\"imported \" + len(self.labels) + \" samples\")\n", + " \n", + " def apply_stemming_and_lemmatization(self):\n", + " stemmer = SnowballStemmer(\"english\")\n", + " for key in self.plain_text.keys():\n", + " stemmed_sent = []\n", + " for word in self.plain_text[key].split(\" \"):\n", + " word_stemmed = stemmer.stem(word)\n", + " stemmed_sent.append(word_stemmed)\n", + " stemmed_sent = (\" \").join(stemmed_sent)\n", + " self.plain_text[key] = stemmed_sent\n", + " \n", + " lemmatizer = WordNetLemmatizer()\n", + " for key in self.plain_text.keys():\n", + " lemmatized_sent = []\n", + " sent_pos = pos_tag(word_tokenize(self.plain_text[key]))\n", + " for word in sent_pos:\n", + " wordnet_pos = get_wordnet_pos(word[1].lower())\n", + " word_lemmatized = lemmatizer.lemmatize(word[0], pos=wordnet_pos)\n", + " lemmatized_sent.append(word_lemmatized)\n", + " lemmatized_sent = (\" \").join(lemmatized_sent)\n", + " self.plain_text[key] = lemmatized_sent\n", + " \n", + " def generate_emoji_count_and_weights(self):\n", + " self.emoji_count = {}\n", + " for e_list in self.emojis:\n", + " for e in set(e_list):\n", + " if e not in self.emoji_count:\n", + " self.emoji_count[e] = 0\n", + " self.emoji_count[e] += 1\n", + " \n", + " emoji_sum = sum([self.emoji_count[e] for e in self.emoji_count])\n", + "\n", + " self.emoji_weights = {}\n", + " for e in self.emoji_count:\n", + " # tfidf for emojis\n", + " self.emoji_weights[e] = np.log((emoji_sum / self.emoji_count[e]))\n", + "\n", + " weights_sum= sum([self.emoji_weights[x] for x in self.emoji_weights])\n", + "\n", + " # normalize:\n", + " for e in self.emoji_weights:\n", + " self.emoji_weights[e] = self.emoji_weights[e] / weights_sum\n", + "\n", + " self.emoji_weights['X'] = 0 # dummy values\n", + " self.emoji_count['X'] = 0\n", + " \n", + " def get_emoji_count(self):\n", + " sorted_emoji_count = list(reversed(sorted(self.emoji_count.items(), key=operator.itemgetter(1))))\n", + " return sorted_emoji_count\n", + " \n", + " def filter_by_top_emojis(self,n_top = 20):\n", + " in_top = [sentiment_vector_to_emoji(x) in self.get_emoji_count()[:n_top] for x in self.labels]\n", + " self.labels = self.labels[in_top]\n", + " self.plain_text = self.plain_text[in_top]\n", + " self.emojis = self.emojis[in_top]\n", + " print(\"remaining samples after top emoji filtering: \", len(labels))\n", + " \n", + " def create_train_test_split(self, split = 0.1, random_state = 4222):\n", + " self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, self.labels, test_size=split, random_state=random_state)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* the trainer class passes Data from the sample manager to the pipeline manager" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "class trainer(object):\n", + " def __init__(self, sdm:sample_data_manager, pm:pipeline_manager):\n", + " self.sdm = sdm\n", + " self.pm = pm\n", + " \n", + " def fit(self):\n", + " # TODO: make batch fitting available here\n", + " self.pm.fit(X = self.sdm.X, y = self.sdm.y)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* when in notebook environment: run the stuff below:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "you are in a notebook\n" + ] + } + ], + "source": [ + "import __main__ as main\n", + "if not hasattr(main, '__file__'):\n", + " print(\"you are in a notebook\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1298,26 +1607,6 @@ "plain_text = plain_text.str.replace(\"(||)\",\"\").str.replace(\"[\" + \"\".join(list(emoji_blacklist)) + \"]\",\"\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* defining different criterias for choosing a single emoji (currently `latest` is used)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def latest(lst):\n", - " return lst[-1] if len(lst) > 0 else 'X' \n", - "def most_common(lst):\n", - " # trying to find the most common used emoji in the given lst\n", - " return max(set(lst), key=lst.count) if len(lst) > 0 else \"X\" # setting label to 'X' if there is an empty emoji list" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -2722,7 +3011,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -2743,17 +3032,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using TensorFlow backend.\n" - ] - } - ], + "outputs": [], "source": [ "import keras\n", "import pickle\n",