Merge branch 'master' of ssh://gogs@the-cake-is-a-lie.net:20022/jonas/NLP-LAB.git
This commit is contained in:
		
							
								
								
									
										128
									
								
								Project/naive_approach/naive_approach.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										128
									
								
								Project/naive_approach/naive_approach.py
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,128 @@ | ||||
| # coding: utf-8 | ||||
|  | ||||
| # In[1]: | ||||
|  | ||||
|  | ||||
| import pandas as pd | ||||
| from IPython.display import clear_output, Markdown, Math | ||||
| import ipywidgets as widgets | ||||
| import os | ||||
| import unicodedata as uni | ||||
| import numpy as np | ||||
| from nltk.stem import PorterStemmer | ||||
| from nltk.tokenize import sent_tokenize, word_tokenize | ||||
| from nltk.corpus import wordnet | ||||
| import math | ||||
| import pprint | ||||
|  | ||||
|  | ||||
| # # Naive Approach | ||||
| table = pd.read_csv('../Tools/emoji_descriptions.csv') | ||||
|  | ||||
| ####################### | ||||
| # Helper functions | ||||
| ####################### | ||||
|  | ||||
| def stemming(messages): | ||||
|     stemmed_messages = [] | ||||
|     ps = PorterStemmer() | ||||
|     for m in messages: | ||||
|         words = word_tokenize(m) | ||||
|         sm = [] | ||||
|         for w in words: | ||||
|             sm.append(ps.stem(w)) | ||||
|         m = (" ").join(sm) | ||||
|         stemmed_messages.append(m) | ||||
|     return stemmed_messages | ||||
|  | ||||
|  | ||||
| # * compare words to emoji descriptions | ||||
| def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng'): | ||||
|      | ||||
|     tokenized_sentence = word_tokenize(sentence) | ||||
|     n = len(tokenized_sentence) | ||||
|     l = table.shape[0] | ||||
|     matrix_list = [] | ||||
|      | ||||
|     for index, row in table.iterrows(): | ||||
|         emoji_tokens = word_tokenize(row[description_key]) | ||||
|         m = len(emoji_tokens) | ||||
|  | ||||
|         mat = np.zeros(shape=(m,n)) | ||||
|         for i in range(len(emoji_tokens)): | ||||
|             for j in range(len(tokenized_sentence)): | ||||
|                 syn1 = wordnet.synsets(emoji_tokens[i],lang=lang) | ||||
|                 if len(syn1) == 0: | ||||
|                     continue | ||||
|                 w1 = syn1[0] | ||||
|                 #print(j, tokenized_sentence) | ||||
|                 syn2 = wordnet.synsets(tokenized_sentence[j], lang=lang) | ||||
|                 if len(syn2) == 0: | ||||
|                     continue | ||||
|                 w2 = syn2[0] | ||||
|                 val = w1.wup_similarity(w2) | ||||
|                 if val is None: | ||||
|                     continue | ||||
|                 mat[i,j] = val | ||||
|         #print(row['character'], mat) | ||||
|         matrix_list.append(mat) | ||||
|              | ||||
|     return matrix_list | ||||
|      | ||||
|      | ||||
| ########################### | ||||
| #Functions to be called from main script | ||||
| ########################### | ||||
|      | ||||
|  | ||||
| # load and preprocess data | ||||
| # emojis_to_consider can be either a list or "all" | ||||
| def prepareData(stemming=False, emojis_to_consider="all"): | ||||
|  | ||||
|     table.head() | ||||
|      | ||||
|     if(stemming): | ||||
|         table['description'] = stemming(table['description']) | ||||
|      | ||||
|     #collect the emojis | ||||
|     lookup = {} | ||||
|     emoji_set = [] | ||||
|     for index, row in table.iterrows(): | ||||
|         if(emojis_to_consider=="all" or (type(emojis_to_consider)==list and row['character'] in emojis_to_consider)): | ||||
|             lookup[index] = row['character'] | ||||
|             emoji_set.append(row['character']) | ||||
|  | ||||
|     emoji_set = set(emoji_set) | ||||
|      | ||||
|     return lookup | ||||
|  | ||||
| # make a prediction for an input sentence | ||||
| def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", description_key='description', lang = 'eng', n=10, t=0.9): | ||||
|  | ||||
|     result = evaluate_sentence(sentence, table, description_key, lang) | ||||
|      | ||||
|     if(criteria=="summed"): | ||||
|         indexes = np.argsort([-np.sum(x) for x in result])[0:n] | ||||
|     elif (criteria=="max_val"): | ||||
|         indexes = np.argsort([-np.max(x) for x in result])[0:n] | ||||
|     elif(criteria=="avg"): | ||||
|         indexes = np.argsort([-np.mean(x) for x in result])[0:n] | ||||
|     else: | ||||
|         indexes= np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]  | ||||
|      | ||||
|     if(emojis_to_consider!="all"): | ||||
|         for i in indexes: | ||||
|             if (i not in lookup): | ||||
|                 indexes = np.delete(indexes, [i]) | ||||
|      | ||||
|     # build a result table | ||||
|     table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] | ||||
|        | ||||
|     table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) | ||||
|      | ||||
|     #display(table_frame) | ||||
|      | ||||
|     return list(table_frame[criteria]) | ||||
|  | ||||
| #predict("I like to travel by train", description_key='description' , lang='eng') | ||||
|  | ||||
| @ -144,7 +144,7 @@ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "3c11801d12b643d9b059ba1058d66d5e", | ||||
|        "model_id": "5ac970d7d7cf4849b4f5adfb80a820c0", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
| @ -168,11 +168,11 @@ | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n", | ||||
|     "                   (widgets.Checkbox(disabled=True), \"only_emoticons\")\n", | ||||
|     "                   (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
|     "                   (widgets.BoundedIntText(disabled=True,min=-1, max=10), \"n_top_emojis\")\n", | ||||
|     "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
|     "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Button(disabled=True),\"load_data\")\n", | ||||
| @ -197,7 +197,7 @@ | ||||
|     "           None,\n", | ||||
|     "           classifier_tab)\n", | ||||
|     "\n", | ||||
|     "create_area(\"create classifier\",\n", | ||||
|     "create_area(\"create/save/load classifier\",\n", | ||||
|     "           [\n", | ||||
|     "               [\n", | ||||
|     "                   (classifier_tab, \"classifier_tab\")\n", | ||||
| @ -206,8 +206,19 @@ | ||||
|     "                   (widgets.Button(), \"create_classifier\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Text(), \"classifier name\"),\n", | ||||
|     "                   (widgets.Button(), \"save classifier\")\n", | ||||
|     "                   (widgets.Label(\"save_area:\"), \"save_area:\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Text(), \"classifier_name\"),\n", | ||||
|     "                   (widgets.Button(), \"save_classifier\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Label(\"load_area:\"), \"load_area:\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Select(options=sorted(glob.glob(\"./*.pipeline\"))), \"clf_file_selector\"),\n", | ||||
|     "                   (widgets.Text(), \"clf_file\"),\n", | ||||
|     "                   (widgets.Button(), \"load_classifier\")\n", | ||||
|     "               ]\n", | ||||
|     "           ],\n", | ||||
|     "           \"create\")\n", | ||||
| @ -541,9 +552,54 @@ | ||||
|     "            pm = stl.pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", | ||||
|     "                                                           layers=layers, sdm=sdm)\n", | ||||
|     "\n", | ||||
|     "def save_classifier(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"create\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "        if pm is None:\n", | ||||
|     "            sys.stderr.write(\"ERROR: create classifier first\")\n", | ||||
|     "            return\n", | ||||
|     "        \n", | ||||
|     "        pm.save(shown_widgets[\"classifier_name\"].value)\n", | ||||
|     "\n", | ||||
|     "def load_classifier(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"create\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "\n", | ||||
|     "def update_file_selector(b):\n", | ||||
|     "    shown_widgets[\"clf_file_selector\"].options = sorted(glob.glob(\"./*.pipeline\"))\n", | ||||
|     "\n", | ||||
|     "def clf_file_selector(b):\n", | ||||
|     "    shown_widgets[\"clf_file\"].value = shown_widgets[\"clf_file_selector\"].value\n", | ||||
|     "    update_file_selector(b)\n", | ||||
|     "\n", | ||||
|     "def load_classifier(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"create\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "        clf_file = shown_widgets[\"clf_file\"].value\n", | ||||
|     "        pm = stl.pipeline_manager.load_from_pipeline_file(clf_file)\n", | ||||
|     "    \n", | ||||
|     "\n", | ||||
|     "# link\n", | ||||
|     "shown_widgets[\"n_keras_layer\"].observe(populate_keras_options)\n", | ||||
|     "shown_widgets[\"create_classifier\"].on_click(create_classifier)" | ||||
|     "shown_widgets[\"create_classifier\"].on_click(create_classifier)\n", | ||||
|     "shown_widgets[\"save_classifier\"].on_click(save_classifier)\n", | ||||
|     "shown_widgets[\"load_classifier\"].on_click(load_classifier)\n", | ||||
|     "shown_widgets[\"clf_file_selector\"].observe(clf_file_selector)\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "\n" | ||||
|    ] | ||||
|   } | ||||
|  ], | ||||
|  | ||||
| @ -2,43 +2,9 @@ | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", | ||||
|       "  from ._conv import register_converters as _register_converters\n", | ||||
|       "Using TensorFlow backend.\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "[nltk_data] Downloading package punkt to /home/jonas/nltk_data...\n", | ||||
|       "[nltk_data]   Package punkt is already up-to-date!\n", | ||||
|       "[nltk_data] Downloading package averaged_perceptron_tagger to\n", | ||||
|       "[nltk_data]     /home/jonas/nltk_data...\n", | ||||
|       "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n", | ||||
|       "[nltk_data]       date!\n", | ||||
|       "[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n", | ||||
|       "[nltk_data]   Package wordnet is already up-to-date!\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "True" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 1, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import pandas as pd\n", | ||||
|     "from IPython.display import clear_output, Markdown, Math\n", | ||||
| @ -59,6 +25,7 @@ | ||||
|     "import pickle\n", | ||||
|     "import operator\n", | ||||
|     "from sklearn.pipeline import Pipeline\n", | ||||
|     "import json\n", | ||||
|     "nltk.download('punkt')\n", | ||||
|     "nltk.download('averaged_perceptron_tagger')\n", | ||||
|     "nltk.download('wordnet')" | ||||
| @ -66,7 +33,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -84,7 +51,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -109,7 +76,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -129,7 +96,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -154,7 +121,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -194,13 +161,13 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "class sample_data_manager(object):\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1):\n", | ||||
|     "    def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, progress_callback=None):\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        generate, read and process train data in one step.\n", | ||||
|     "        \n", | ||||
| @ -214,7 +181,7 @@ | ||||
|     "        @return: sample_data_manager object\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        sdm = sample_data_manager(path)\n", | ||||
|     "        sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons)\n", | ||||
|     "        sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons, progress_callback=progress_callback)\n", | ||||
|     "        if apply_stemming:\n", | ||||
|     "            sdm.apply_stemming_and_lemmatization()\n", | ||||
|     "        \n", | ||||
| @ -254,7 +221,7 @@ | ||||
|     "        self.kmeans_cluster = None\n", | ||||
|     "        self.label_binarizer = None\n", | ||||
|     "    \n", | ||||
|     "    def read_files(self, file_index_range:list, only_emoticons=True):\n", | ||||
|     "    def read_files(self, file_index_range:list, only_emoticons=True, progress_callback=None):\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        reading (multiple) files to one panda table.\n", | ||||
|     "        \n", | ||||
| @ -268,7 +235,8 @@ | ||||
|     "                self.raw_data = pd.read_json(self.json_files[i], encoding=\"utf-8\")\n", | ||||
|     "            else:\n", | ||||
|     "                self.raw_data = self.raw_data.append(pd.read_json(self.json_files[i], encoding=\"utf-8\"))\n", | ||||
|     "        \n", | ||||
|     "            if progress_callback is not None:\n", | ||||
|     "                progress_callback()\n", | ||||
|     "        self.emojis = self.raw_data['EMOJI']\n", | ||||
|     "        self.plain_text = self.raw_data['text']\n", | ||||
|     "        \n", | ||||
| @ -396,8 +364,7 @@ | ||||
|     "        \n", | ||||
|     "        labels = self.binary_labels if self.use_binary_labels else self.labels\n", | ||||
|     "        assert labels is not None\n", | ||||
|     "        self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, labels, test_size=split, random_state=random_state)\n", | ||||
|     "\n" | ||||
|     "        self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, labels, test_size=split, random_state=random_state)\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
| @ -409,12 +376,26 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "class pipeline_manager(object):\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def load_from_pipeline_file(pipeline_file:str):\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        loading a json configuration file and using it's paramters to call 'load_pipeline_from_files'\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        with open(pipeline_file, 'r') as f:\n", | ||||
|     "            d = json.load(f)\n", | ||||
|     "        \n", | ||||
|     "        keras_models = d['keras_models']\n", | ||||
|     "        all_models = d['all_models']\n", | ||||
|     "        \n", | ||||
|     "        return pipeline_manager.load_pipeline_from_files(pipeline_file.rsplit('.',1)[0], keras_models, all_models)\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []):\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        load a pipeline from files. A pipeline should be represented by multiple model files in the form '<file_prefix>.<model_name>'\n", | ||||
| @ -527,6 +508,7 @@ | ||||
|     "        @param prefix: file prefix for all models\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        \n", | ||||
|     "\n", | ||||
|     "        print(self.keras_models)\n", | ||||
|     "        # doing this like explained here: https://stackoverflow.com/a/43415459\n", | ||||
|     "        for step in self.pipeline.named_steps:\n", | ||||
| @ -538,6 +520,9 @@ | ||||
|     "        load_command = \"pipeline_manager.load_pipeline_from_files( '\"\n", | ||||
|     "        load_command += prefix + \"', \" + str(self.keras_models) + \", \"\n", | ||||
|     "        load_command += str(list(self.pipeline.named_steps.keys())) + \")\"\n", | ||||
|     "\n", | ||||
|     "        with open(prefix + '.pipeline', 'w') as outfile:\n", | ||||
|     "            json.dump({'keras_models': self.keras_models, 'all_models': [step for step in self.pipeline.named_steps]}, outfile)\n", | ||||
|     "        \n", | ||||
|     "        import __main__ as main\n", | ||||
|     "        if not hasattr(main, '__file__'):\n", | ||||
| @ -584,7 +569,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -594,7 +579,7 @@ | ||||
|     "        self.sdm = sdm\n", | ||||
|     "        self.pm = pm\n", | ||||
|     "    \n", | ||||
|     "    def fit(self, max_size=10000, disabled_fit_steps=['vectorizer']):\n", | ||||
|     "    def fit(self, max_size=10000, disabled_fit_steps=['vectorizer'], keras_batch_fitting_layer=['keras_model'], batch_size=None, n_epochs=1, progress_callback=None):\n", | ||||
|     "        \"\"\"\n", | ||||
|     "        fitting data in the pipeline. Because we don't want to refit the vectorizer, the pipeline models containing the vectorizer have to be named explicitly\n", | ||||
|     "        \n", | ||||
| @ -607,22 +592,44 @@ | ||||
|     "        disabled_fits = {}\n", | ||||
|     "        disabled_fit_transforms = {}\n", | ||||
|     "        \n", | ||||
|     "        disabled_keras_fits = {}\n", | ||||
|     "        \n", | ||||
|     "        named_steps = self.pm.pipeline.named_steps\n", | ||||
|     "        \n", | ||||
|     "        for s in disabled_fit_steps:\n", | ||||
|     "            # now it gets a little bit dirty:\n", | ||||
|     "            # now it gets really dirty:\n", | ||||
|     "            # replace fit functions we don't want to call again (e.g. for vectorizers)\n", | ||||
|     "            disabled_fits[s] = named_steps[s].fit\n", | ||||
|     "            disabled_fit_transforms[s] = named_steps[s].fit_transform\n", | ||||
|     "            named_steps[s].fit = lambda self, X, y=None: self\n", | ||||
|     "            named_steps[s].fit_transform = named_steps[s].transform\n", | ||||
|     "            \n", | ||||
|     "        self.pm.fit(X = self.sdm.X[:max_size], y = self.sdm.y[:max_size])\n", | ||||
|     "        \n", | ||||
|     "        for k in keras_batch_fitting_layer:\n", | ||||
|     "            # forcing batch fitting on keras\n", | ||||
|     "            disabled_keras_fits[k]=named_steps[k].fit\n", | ||||
|     "            named_steps[k].fit = lambda X, y: named_steps[k].train_on_batch(X.todense(), y) # ← why has keras no sparse support on batch progressing!?!?!\n", | ||||
|     "        \n", | ||||
|     "        if batch_size is None:\n", | ||||
|     "            self.pm.fit(X = self.sdm.X[:max_size], y = self.sdm.y[:max_size])\n", | ||||
|     "        else:\n", | ||||
|     "            n = len(self.sdm.X) // batch_size\n", | ||||
|     "            for i in range(n_epochs):\n", | ||||
|     "                for j in range(n):\n", | ||||
|     "                    self.pm.fit(X = np.array(self.sdm.X[j*batch_size:(j+1)*batch_size]), y = np.array(self.sdm.y[j*batch_size:(j+1)*batch_size]))\n", | ||||
|     "                    if progress_callback is not None:\n", | ||||
|     "                        progress_callback()\n", | ||||
|     "                    pred, yt = self.test()\n", | ||||
|     "                    mean_squared_error = ((pred - yt)**2).mean(axis=0)\n", | ||||
|     "                    print(\"#\" + str(j) + \": loss: \", mean_squared_error)\n", | ||||
|     "\n", | ||||
|     "        \n", | ||||
|     "        # restore replaced fit functions:\n", | ||||
|     "        for s in disabled_fit_steps:\n", | ||||
|     "            named_steps[s].fit = disabled_fits[s]\n", | ||||
|     "            named_steps[s].fit_transform = disabled_fit_transforms[s]\n", | ||||
|     "        \n", | ||||
|     "        for k in keras_batch_fitting_layer:\n", | ||||
|     "            named_steps[k].fit = disabled_keras_fits[k]\n", | ||||
|     "    \n", | ||||
|     "    def test(self):\n", | ||||
|     "        '''\n", | ||||
| @ -630,9 +637,7 @@ | ||||
|     "        '''\n", | ||||
|     "        if self.sdm.X is None:\n", | ||||
|     "            self.sdm.create_train_test_split()\n", | ||||
|     "        return self.pm.predict(self.sdm.Xt), self.sdm.yt\n", | ||||
|     "\n", | ||||
|     "    " | ||||
|     "        return self.pm.predict(self.sdm.Xt), self.sdm.yt\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
| @ -652,21 +657,9 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "reading file: ./data_en/2017-11-01.json...\n", | ||||
|       "imported 33368 samples\n", | ||||
|       "remaining samples after top emoji filtering:  26197\n", | ||||
|       "Epoch 1/1\n", | ||||
|       "100/100 [==============================] - 3s 28ms/step - loss: 0.1230\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
| @ -698,40 +691,9 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "['keras_model']\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "'saved pipeline. It can be loaded the following way:'" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "> ```\n", | ||||
|        "pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n", | ||||
|        "```" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     } | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
| @ -750,156 +712,9 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 12, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "array([[0.16062996, 0.08324276, 0.09433182],\n", | ||||
|        "       [0.16413   , 0.09421383, 0.07578427],\n", | ||||
|        "       [0.11994962, 0.05705731, 0.06310127],\n", | ||||
|        "       ...,\n", | ||||
|        "       [0.13887292, 0.08502828, 0.08176519],\n", | ||||
|        "       [0.18185864, 0.09223703, 0.10704609],\n", | ||||
|        "       [0.17687687, 0.09147045, 0.10650696]], dtype=float32)" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "array([[0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.70401758, 0.05932203, 0.23666039],\n", | ||||
|        "       ...,\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797]])" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "prediction variance:  0.0005431187\n", | ||||
|       "teacher variance:  0.03341702104519965\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/html": [ | ||||
|        "<div>\n", | ||||
|        "<style scoped>\n", | ||||
|        "    .dataframe tbody tr th:only-of-type {\n", | ||||
|        "        vertical-align: middle;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe tbody tr th {\n", | ||||
|        "        vertical-align: top;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe thead th {\n", | ||||
|        "        text-align: right;\n", | ||||
|        "    }\n", | ||||
|        "</style>\n", | ||||
|        "<table border=\"1\" class=\"dataframe\">\n", | ||||
|        "  <thead>\n", | ||||
|        "    <tr style=\"text-align: right;\">\n", | ||||
|        "      <th></th>\n", | ||||
|        "      <th>predict</th>\n", | ||||
|        "      <th>predicted_sentiment</th>\n", | ||||
|        "      <th>teacher</th>\n", | ||||
|        "      <th>teacher_sentiment</th>\n", | ||||
|        "      <th>text</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>35671</th>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.16062995791435242, 0.0832427591085434, 0.09...</td>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n", | ||||
|        "      <td>i feel like i care so much more in everi situat</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>25683</th>\n", | ||||
|        "      <td>😢</td>\n", | ||||
|        "      <td>[0.16413000226020813, 0.0942138284444809, 0.07...</td>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n", | ||||
|        "      <td>i did not meat to add that 2 there ... hav see...</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>8985</th>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.11994962394237518, 0.05705730617046356, 0.0...</td>\n", | ||||
|        "      <td>😊</td>\n", | ||||
|        "      <td>[0.7040175768989329, 0.059322033898305086, 0.2...</td>\n", | ||||
|        "      <td>never…</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>5410</th>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.18114930391311646, 0.10199417173862457, 0.1...</td>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n", | ||||
|        "      <td>lmao on me ! ! ! wtf wa he suppos to say</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>62611</th>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.16997836530208588, 0.08633847534656525, 0.0...</td>\n", | ||||
|        "      <td>😊</td>\n", | ||||
|        "      <td>[0.7040175768989329, 0.059322033898305086, 0.2...</td>\n", | ||||
|        "      <td>this dude alway help me get through my school ...</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "      predict                                predicted_sentiment teacher  \\\n", | ||||
|        "35671       😂  [0.16062995791435242, 0.0832427591085434, 0.09...       😂   \n", | ||||
|        "25683       😢  [0.16413000226020813, 0.0942138284444809, 0.07...       😂   \n", | ||||
|        "8985        😂  [0.11994962394237518, 0.05705730617046356, 0.0...       😊   \n", | ||||
|        "5410        😂  [0.18114930391311646, 0.10199417173862457, 0.1...       😂   \n", | ||||
|        "62611       😂  [0.16997836530208588, 0.08633847534656525, 0.0...       😊   \n", | ||||
|        "\n", | ||||
|        "                                       teacher_sentiment  \\\n", | ||||
|        "35671  [0.46813021474490496, 0.24716181096977158, 0.2...   \n", | ||||
|        "25683  [0.46813021474490496, 0.24716181096977158, 0.2...   \n", | ||||
|        "8985   [0.7040175768989329, 0.059322033898305086, 0.2...   \n", | ||||
|        "5410   [0.46813021474490496, 0.24716181096977158, 0.2...   \n", | ||||
|        "62611  [0.7040175768989329, 0.059322033898305086, 0.2...   \n", | ||||
|        "\n", | ||||
|        "                                                    text  \n", | ||||
|        "35671    i feel like i care so much more in everi situat  \n", | ||||
|        "25683  i did not meat to add that 2 there ... hav see...  \n", | ||||
|        "8985                                              never…  \n", | ||||
|        "5410            lmao on me ! ! ! wtf wa he suppos to say  \n", | ||||
|        "62611  this dude alway help me get through my school ...  " | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Mean Squared Error:  [0.13877691 0.04682433 0.02937794]\n", | ||||
|       "Variance teacher:  [0.02183094 0.02513847 0.00285735]\n", | ||||
|       "Variance prediction:  [0.00046378 0.00019441 0.00020516]\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
| @ -945,68 +760,9 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 13, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "003ae16760b04c25bdc9f2fe2193747a", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>Text</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "Text(value='')" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "4580af82b30545f197a41e4359010556", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='Click me'), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     } | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
|  | ||||
| @ -23,6 +23,7 @@ from sklearn.externals import joblib | ||||
| import pickle | ||||
| import operator | ||||
| from sklearn.pipeline import Pipeline | ||||
| import json | ||||
| nltk.download('punkt') | ||||
| nltk.download('averaged_perceptron_tagger') | ||||
| nltk.download('wordnet') | ||||
| @ -329,6 +330,20 @@ class sample_data_manager(object): | ||||
|  | ||||
|  | ||||
| class pipeline_manager(object): | ||||
|     @staticmethod | ||||
|     def load_from_pipeline_file(pipeline_file:str): | ||||
|         """ | ||||
|         loading a json configuration file and using it's paramters to call 'load_pipeline_from_files' | ||||
|         """ | ||||
|         with open(pipeline_file, 'r') as f: | ||||
|             d = json.load(f) | ||||
|          | ||||
|         keras_models = d['keras_models'] | ||||
|         all_models = d['all_models'] | ||||
|          | ||||
|         return pipeline_manager.load_pipeline_from_files(pipeline_file.rsplit('.',1)[0], keras_models, all_models) | ||||
|  | ||||
|  | ||||
|     @staticmethod | ||||
|     def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []): | ||||
|         """ | ||||
| @ -442,6 +457,7 @@ class pipeline_manager(object): | ||||
|         @param prefix: file prefix for all models | ||||
|         """ | ||||
|          | ||||
|  | ||||
|         print(self.keras_models) | ||||
|         # doing this like explained here: https://stackoverflow.com/a/43415459 | ||||
|         for step in self.pipeline.named_steps: | ||||
| @ -453,6 +469,9 @@ class pipeline_manager(object): | ||||
|         load_command = "pipeline_manager.load_pipeline_from_files( '" | ||||
|         load_command += prefix + "', " + str(self.keras_models) + ", " | ||||
|         load_command += str(list(self.pipeline.named_steps.keys())) + ")" | ||||
|  | ||||
|         with open(prefix + '.pipeline', 'w') as outfile: | ||||
|             json.dump({'keras_models': self.keras_models, 'all_models': [step for step in self.pipeline.named_steps]}, outfile) | ||||
|          | ||||
|         import __main__ as main | ||||
|         if not hasattr(main, '__file__'): | ||||
|  | ||||
		Reference in New Issue
	
	Block a user