diff --git a/Project/simple_approach/Continous_Learner.ipynb b/Project/simple_approach/Continous_Learner.ipynb index 10ee731..1738d1a 100644 --- a/Project/simple_approach/Continous_Learner.ipynb +++ b/Project/simple_approach/Continous_Learner.ipynb @@ -11,31 +11,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using TensorFlow backend.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[nltk_data] Downloading package punkt to /home/jonas/nltk_data...\n", - "[nltk_data] Package punkt is already up-to-date!\n", - "[nltk_data] Downloading package averaged_perceptron_tagger to\n", - "[nltk_data] /home/jonas/nltk_data...\n", - "[nltk_data] Package averaged_perceptron_tagger is already up-to-\n", - "[nltk_data] date!\n", - "[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n", - "[nltk_data] Package wordnet is already up-to-date!\n" - ] - } - ], + "outputs": [], "source": [ "import simple_twitter_learning as stl\n", "import glob\n", @@ -59,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -114,48 +92,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "----" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "## User Interface" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5ac970d7d7cf4849b4f5adfb80a820c0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tab(children=(VBox(children=(HBox(children=(Text(value='./data_en/', description='root_path'), Button(descript…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "mp(\"----\")\n", "mp(\"## User Interface\")\n", @@ -172,7 +111,7 @@ " ],\n", " [\n", " (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", - " (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n", + " (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\")\n", " ],\n", " [\n", " (widgets.Button(disabled=True),\"load_data\")\n", @@ -248,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -266,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -354,17 +293,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class progress_indicator(object):\n", " \n", - " def __init__(self, n, description=\"progress\"):\n", - " self.w = widgets.IntProgress(value=0, min=0,max=n, description = description)\n", + " def __init__(self, description=\"progress\"):\n", + " self.w = widgets.FloatProgress(value=0, min=0,max=1, description = description)\n", " display(self.w)\n", - " def update(self, dn=1):\n", - " self.w.value += dn\n", + " def update(self, val):\n", + " self.w.value = val\n", " " ] }, @@ -378,7 +317,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -416,13 +355,15 @@ " r = shown_widgets[\"file_range\"].value\n", " r = (r[0], r[1] + 1) # range has to be exclusive according to the last element!\n", " \n", - " p = progress_indicator(r[1] - r[0], \"reading progress\")\n", + " p_r = progress_indicator(\"reading progress\")\n", + " p_s = progress_indicator(\"stemming progress\")\n", " \n", " sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", " n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", " file_range=range(r[0], r[1]),\n", " n_kmeans_cluster=shown_widgets[\"k_means_cluster\"].value,\n", - " progress_callback=p.update)\n", + " read_progress_callback=p_r.update,\n", + " stem_progress_callback=p_s.update)\n", " shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", " \n", " \n", @@ -440,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -461,11 +402,10 @@ " \n", " print(\"update train test split:\")\n", " sdm.create_train_test_split(split=val_split)\n", - " batch_n = len(sdm.X) // batch_size\n", " \n", " print(\"fit\")\n", " \n", - " p = progress_indicator(batch_n)\n", + " p = progress_indicator()\n", " \n", " tr = stl.trainer(sdm=sdm, pm=pm)\n", " tr.fit(progress_callback=p.update, batch_size=batch_size, n_epochs=n_epochs)\n", @@ -485,7 +425,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/Project/simple_approach/simple_twitter_learning.py b/Project/simple_approach/simple_twitter_learning.py index d551df2..9a8e6c6 100644 --- a/Project/simple_approach/simple_twitter_learning.py +++ b/Project/simple_approach/simple_twitter_learning.py @@ -123,7 +123,7 @@ def get_wordnet_pos(treebank_tag): class sample_data_manager(object): @staticmethod - def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, progress_callback=None): + def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None): """ generate, read and process train data in one step. @@ -137,9 +137,9 @@ class sample_data_manager(object): @return: sample_data_manager object """ sdm = sample_data_manager(path) - sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons, progress_callback=progress_callback) + sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons, progress_callback=read_progress_callback) if apply_stemming: - sdm.apply_stemming_and_lemmatization() + sdm.apply_stemming_and_lemmatization(progress_callback=stem_progress_callback) sdm.generate_emoji_count_and_weights() @@ -161,7 +161,6 @@ class sample_data_manager(object): self.data_root_folder = data_root_folder self.json_files = sorted(glob.glob(self.data_root_folder + "/*.json")) self.n_files = len(self.json_files) - self.raw_data = None self.emojis = None self.plain_text = None self.labels = None @@ -177,45 +176,65 @@ class sample_data_manager(object): self.kmeans_cluster = None self.label_binarizer = None - def read_files(self, file_index_range:list, only_emoticons=True, progress_callback=None): + def read_files(self, file_index_range:list, only_emoticons=True, emoji_mean=False ,progress_callback=None): """ reading (multiple) files to one panda table. @param file_index_range: range of file's indices to read (eg `range(3)` to read the first three files) @param only_emoticons: if True, only messages containing emoticons (aka smileys) are used. This classification is derived from Tools.Emoji_Distance + @param emoji_mean: if True, using mean of all emojis instead of the last one """ assert np.min(file_index_range) >= 0 and np.max(file_index_range) < self.n_files + n = len(file_index_range) for i in file_index_range: print("reading file: " + self.json_files[i] + "...") - if self.raw_data is None: - self.raw_data = pd.read_json(self.json_files[i], encoding="utf-8") - else: - self.raw_data = self.raw_data.append(pd.read_json(self.json_files[i], encoding="utf-8")) - if progress_callback is not None: - progress_callback() - self.emojis = self.raw_data['EMOJI'] - self.plain_text = self.raw_data['text'] - - # replacing keywords. TODO: maybe these information can be extracted and used - self.plain_text = self.plain_text.str.replace("(||)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","") - - # so far filtering for the latest emoji. TODO: maybe there are also better approaches - self.labels = emoji2sent([latest(e) for e in self.emojis], only_emoticons=only_emoticons ) - - # and filter out all samples we have no label for: - wrong_labels = np.isnan(np.linalg.norm(self.labels, axis=1)) + raw_data_i = pd.read_json(self.json_files[i], encoding="utf-8") + emojis_i = raw_data_i['EMOJI'] + plain_text_i = raw_data_i['text'] - self.labels = self.labels[np.invert(wrong_labels)] - self.plain_text = self.plain_text[np.invert(wrong_labels)] - self.emojis = self.emojis[np.invert(wrong_labels)] + # replacing keywords. TODO: maybe these information can be extracted and used + plain_text_i = plain_text_i.str.replace("(||)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","") + + if not emoji_mean: + # so far filtering for the latest emoji. TODO: maybe there are also better approaches + labels_i = emoji2sent([latest(e) for e in emojis_i], only_emoticons=only_emoticons ) + else: + labels_i = np.array([np.mean(emoji2sent(e, only_emoticons=only_emoticons), axis=0) for e in emojis_i]) + + # and filter out all samples we have no label for: + wrong_labels = np.isnan(np.linalg.norm(labels_i, axis=1)) + labels_i = labels_i[np.invert(wrong_labels)] + plain_text_i = plain_text_i[np.invert(wrong_labels)] + emojis_i = emojis_i[np.invert(wrong_labels)] + print("imported " + str(len(labels_i)) + " samples") + + if self.labels is None: + self.labels = labels_i + else: + self.labels = np.append(self.labels, labels_i, axis=0) + + if self.emojis is None: + self.emojis = emojis_i + else: + self.emojis = pd.concat([self.emojis,emojis_i],ignore_index=True) + + if self.plain_text is None: + self.plain_text = plain_text_i + else: + self.plain_text = pd.concat([self.plain_text,plain_text_i],ignore_index=True) + + if progress_callback is not None: + progress_callback((i+1)/n) - print("imported " + str(len(self.labels)) + " samples") - def apply_stemming_and_lemmatization(self): + def apply_stemming_and_lemmatization(self, progress_callback = None): """ apply stemming and lemmatization to plain text samples """ + print("apply stemming and lemmatization...") stemmer = SnowballStemmer("english") + n = self.plain_text.shape[0] * 2 # 2 for loops + i = 0 for key in self.plain_text.keys(): stemmed_sent = [] for word in self.plain_text[key].split(" "): @@ -223,6 +242,11 @@ class sample_data_manager(object): stemmed_sent.append(word_stemmed) stemmed_sent = (" ").join(stemmed_sent) self.plain_text[key] = stemmed_sent + i += 1 + if progress_callback is not None and i % 1024 == 0: + progress_callback(i / n) + + lemmatizer = WordNetLemmatizer() for key in self.plain_text.keys(): @@ -234,6 +258,10 @@ class sample_data_manager(object): lemmatized_sent.append(word_lemmatized) lemmatized_sent = (" ").join(lemmatized_sent) self.plain_text[key] = lemmatized_sent + i += 1 + if progress_callback is not None and i % 1024 == 0: + progress_callback(i / n) + print("stemming and lemmatization done") def generate_emoji_count_and_weights(self): """ @@ -558,7 +586,7 @@ class trainer(object): for j in range(n): self.pm.fit(X = np.array(self.sdm.X[j*batch_size:(j+1)*batch_size]), y = np.array(self.sdm.y[j*batch_size:(j+1)*batch_size])) if progress_callback is not None: - progress_callback() + progress_callback(j / n) pred, yt = self.test() mean_squared_error = ((pred - yt)**2).mean(axis=0) print("#" + str(j) + ": loss: ", mean_squared_error)