first refactored twitter_learning version
This commit is contained in:
		| @ -2,9 +2,18 @@ | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 15, | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", | ||||
|       "  from ._conv import register_converters as _register_converters\n", | ||||
|       "Using TensorFlow backend.\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
| @ -25,7 +34,7 @@ | ||||
|        "True" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 15, | ||||
|      "execution_count": 1, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -46,6 +55,7 @@ | ||||
|     "import nltk\n", | ||||
|     "from keras.models import load_model\n", | ||||
|     "from sklearn.externals import joblib\n", | ||||
|     "import pickle\n", | ||||
|     "import operator\n", | ||||
|     "from sklearn.pipeline import Pipeline\n", | ||||
|     "nltk.download('punkt')\n", | ||||
| @ -55,7 +65,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -168,53 +178,6 @@ | ||||
|     "        return wordnet.NOUN" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* the pipeline manager saves and stores sklearn pipelines. Keras models are handled differently, so the have to be named explicitly during save and load operations" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 16, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "class pipeline_manager(object):\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def load_pipeline_from_files(file_prefix:str, keras_models = []):\n", | ||||
|     "        pm = pipeline_manager()\n", | ||||
|     "        pm.load(file_prefix, keras_models=keras_models)\n", | ||||
|     "        return pm\n", | ||||
|     "    \n", | ||||
|     "    def __init__(self, pipeline = None):\n", | ||||
|     "        self.pipeline = pipeline\n", | ||||
|     "        self.additional_objects = {}\n", | ||||
|     "    \n", | ||||
|     "    def save(prefix:str, keras_models = []):\n", | ||||
|     "        # doing this like explained here: https://stackoverflow.com/a/43415459\n", | ||||
|     "        for km in keras_models:\n", | ||||
|     "            self.pipeline.named_steps[km].model.save(prefix + \".\" + km)\n", | ||||
|     "            # setting this part to None:\n", | ||||
|     "            self.pipeline.named_steps[km].model = None\n", | ||||
|     "        \n", | ||||
|     "        # now we can save the pipeline:\n", | ||||
|     "        joblib.dump(self.pipeline, prefix + \".pipeline\")\n", | ||||
|     "    \n", | ||||
|     "    def load(prefix:str, keras_models=[]):\n", | ||||
|     "        self.pipeline = joblib.load(prefix + \".pipeline\")\n", | ||||
|     "        for km in keras_models:\n", | ||||
|     "            self.pipeline.named_steps[km].model = load_model(prefix + \".\" + km)\n", | ||||
|     "    \n", | ||||
|     "    def fit(self,X,y):\n", | ||||
|     "        self.pipeline.fit(X,y)\n", | ||||
|     "    \n", | ||||
|     "    def predict(self,X):\n", | ||||
|     "        self.pipeline.predict(X)\n", | ||||
|     "    " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
| @ -224,14 +187,25 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 17, | ||||
|    "execution_count": 11, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "class sample_data_manager(object):\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def generate_and_read(path:str):\n", | ||||
|     "        sdm = sample_data_manager(path):\n", | ||||
|     "    def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None):\n", | ||||
|     "        sdm = sample_data_manager(path)\n", | ||||
|     "        sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons)\n", | ||||
|     "        if apply_stemming:\n", | ||||
|     "            sdm.apply_stemming_and_lemmatization()\n", | ||||
|     "        \n", | ||||
|     "        sdm.generate_emoji_count_and_weights()\n", | ||||
|     "        \n", | ||||
|     "        if n_top_emojis > 0:\n", | ||||
|     "            sdm.filter_by_top_emojis(n_top=n_top_emojis)\n", | ||||
|     "        \n", | ||||
|     "        return sdm\n", | ||||
|     "        \n", | ||||
|     "    \n", | ||||
|     "    def __init__(self, data_root_folder:str):\n", | ||||
|     "        self.data_root_folder = data_root_folder\n", | ||||
| @ -247,11 +221,12 @@ | ||||
|     "        self.y = None\n", | ||||
|     "        self.Xt = None\n", | ||||
|     "        self.yt = None\n", | ||||
|     "        self.top_emojis = None\n", | ||||
|     "    \n", | ||||
|     "    def read_files(self, file_index_range:list, only_emoticons=True):\n", | ||||
|     "        assert np.min(file_index_range) >= 0 and np.max(file_index_range) < self.n_files\n", | ||||
|     "        for i in file_index_range:\n", | ||||
|     "            print(\"reaing file: \" + self.json_files[i] + \"...\")\n", | ||||
|     "            print(\"reading file: \" + self.json_files[i] + \"...\")\n", | ||||
|     "            if self.raw_data is None:\n", | ||||
|     "                self.raw_data = pd.read_json(self.json_files[i], encoding=\"utf-8\")\n", | ||||
|     "            else:\n", | ||||
| @ -264,7 +239,7 @@ | ||||
|     "        self.plain_text = self.plain_text.str.replace(\"(<EMOJI>|<USER>|<HASHTAG>)\",\"\").str.replace(\"[\" + \"\".join(list(emoji_blacklist)) + \"]\",\"\")\n", | ||||
|     "        \n", | ||||
|     "        # so far filtering for the latest emoji. TODO: maybe there are also better approaches\n", | ||||
|     "        self.labels = emoji2sent([latest(e) for e in emojis], only_emoticons=only_emoticons )\n", | ||||
|     "        self.labels = emoji2sent([latest(e) for e in self.emojis], only_emoticons=only_emoticons )\n", | ||||
|     "        \n", | ||||
|     "        # and filter out all samples we have no label for:\n", | ||||
|     "        wrong_labels = np.isnan(np.linalg.norm(self.labels, axis=1))    \n", | ||||
| @ -273,7 +248,7 @@ | ||||
|     "        self.plain_text = self.plain_text[np.invert(wrong_labels)]\n", | ||||
|     "        self.emojis = self.emojis[np.invert(wrong_labels)]\n", | ||||
|     "        \n", | ||||
|     "        print(\"imported \" + len(self.labels) + \" samples\")\n", | ||||
|     "        print(\"imported \" + str(len(self.labels)) + \" samples\")\n", | ||||
|     "    \n", | ||||
|     "    def apply_stemming_and_lemmatization(self):\n", | ||||
|     "        stemmer = SnowballStemmer(\"english\")\n", | ||||
| @ -322,20 +297,142 @@ | ||||
|     "    \n", | ||||
|     "    def get_emoji_count(self):\n", | ||||
|     "        sorted_emoji_count = list(reversed(sorted(self.emoji_count.items(), key=operator.itemgetter(1))))\n", | ||||
|     "        #display(sorted_emoji_count)\n", | ||||
|     "        return sorted_emoji_count\n", | ||||
|     "    \n", | ||||
|     "    def filter_by_top_emojis(self,n_top = 20):\n", | ||||
|     "        in_top = [sentiment_vector_to_emoji(x) in self.get_emoji_count()[:n_top] for x in self.labels]\n", | ||||
|     "        self.top_emojis = [x[0] for x in self.get_emoji_count()[:n_top]]\n", | ||||
|     "        in_top = [sentiment_vector_to_emoji(x) in self.top_emojis for x in self.labels]\n", | ||||
|     "        self.labels = self.labels[in_top]\n", | ||||
|     "        self.plain_text = self.plain_text[in_top]\n", | ||||
|     "        self.emojis = self.emojis[in_top]\n", | ||||
|     "        print(\"remaining samples after top emoji filtering: \", len(labels))\n", | ||||
|     "        print(\"remaining samples after top emoji filtering: \", len(self.labels))\n", | ||||
|     "    \n", | ||||
|     "    def create_train_test_split(self, split = 0.1, random_state = 4222):\n", | ||||
|     "        self.X, self.Xt, self.y, self.yt = train_test_split(self.plain_text, self.labels, test_size=split, random_state=random_state)\n", | ||||
|     "\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* the pipeline manager saves and stores sklearn pipelines. Keras models are handled differently, so the have to be named explicitly during save and load operations" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 21, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "class pipeline_manager(object):\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def load_pipeline_from_files(file_prefix:str, keras_models = [], all_models = []):\n", | ||||
|     "        pm = pipeline_manager(keras_models=keras_models)\n", | ||||
|     "        pm.load(file_prefix, all_models)\n", | ||||
|     "        return pm\n", | ||||
|     "    \n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def create_keras_pipeline_with_vectorizer(vectorizer, layers, sdm:sample_data_manager):\n", | ||||
|     "        '''\n", | ||||
|     "        creates pipeline with vectorizer and keras classifier\n", | ||||
|     "        '''\n", | ||||
|     "        from keras.models import Sequential\n", | ||||
|     "        from keras.layers import Dense\n", | ||||
|     "        \n", | ||||
|     "        if sdm.X is None:\n", | ||||
|     "            sdm.create_train_test_split()\n", | ||||
|     "        \n", | ||||
|     "        vec_train = vectorizer.fit_transform(sdm.X)\n", | ||||
|     "        vec_test = vectorizer.transform(sdm.Xt)\n", | ||||
|     "        # creating keras model:\n", | ||||
|     "        model=Sequential()\n", | ||||
|     "        \n", | ||||
|     "        keras_layers = []\n", | ||||
|     "        first_layer = True\n", | ||||
|     "        for layer in layers:\n", | ||||
|     "            if first_layer:\n", | ||||
|     "                model.add(Dense(units=layer[0], activation=layer[1], input_dim=vectorizer.transform([\" \"])[0]._shape[1]))\n", | ||||
|     "                first_layer = False\n", | ||||
|     "            else:\n", | ||||
|     "                model.add(Dense(units=layer[0], activation=layer[1]))\n", | ||||
|     "        \n", | ||||
|     "        model.compile(loss='mean_squared_error',\n", | ||||
|     "                  optimizer='adam')\n", | ||||
|     "        \n", | ||||
|     "        pipeline = Pipeline([\n", | ||||
|     "            ('vectorizer',vectorizer),\n", | ||||
|     "            ('keras_model', model)\n", | ||||
|     "        ])\n", | ||||
|     "        \n", | ||||
|     "        return pipeline_manager(pipeline=pipeline, keras_models=['keras_model'])\n", | ||||
|     "    \n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def create_pipeline_with_classifier_and_vectorizer(vectorizer, classifier, sdm:sample_data_manager = None):\n", | ||||
|     "        '''\n", | ||||
|     "        creates a pipeline with vectorizer and classifier for non keras classifiers\n", | ||||
|     "        if sample data manager is given, the vectorizer will be also fitted!\n", | ||||
|     "        '''\n", | ||||
|     "        if sdm is not None:\n", | ||||
|     "            if sdm.X is None:\n", | ||||
|     "                sdm.create_train_test_split()\n", | ||||
|     "\n", | ||||
|     "            vec_train = vectorizer.fit_transform(sdm.X)\n", | ||||
|     "            vec_test = vectorizer.transform(sdm.Xt)\n", | ||||
|     "        \n", | ||||
|     "        pipeline = Pipeline([\n", | ||||
|     "            ('vectorizer',vectorizer),\n", | ||||
|     "            ('classifier', classifier)\n", | ||||
|     "        ])\n", | ||||
|     "        \n", | ||||
|     "        return pipeline_manager(pipeline=pipeline, keras_models=[])\n", | ||||
|     "    \n", | ||||
|     "    def __init__(self, pipeline = None, keras_models = []):\n", | ||||
|     "        self.pipeline = pipeline\n", | ||||
|     "        self.additional_objects = {}\n", | ||||
|     "        self.keras_models = keras_models\n", | ||||
|     "    \n", | ||||
|     "    def save(self, prefix:str):\n", | ||||
|     "        print(self.keras_models)\n", | ||||
|     "        # doing this like explained here: https://stackoverflow.com/a/43415459\n", | ||||
|     "        for step in self.pipeline.named_steps:\n", | ||||
|     "            if step in self.keras_models:\n", | ||||
|     "                self.pipeline.named_steps[step].model.save(prefix + \".\" + step)\n", | ||||
|     "            else:\n", | ||||
|     "                joblib.dump(self.pipeline.named_steps[step], prefix + \".\" + str(step))\n", | ||||
|     "        \n", | ||||
|     "        load_command = \"pipeline_manager.load_pipeline_from_files( '\"\n", | ||||
|     "        load_command += prefix + \"', \" + str(self.keras_models) + \", \"\n", | ||||
|     "        load_command += str(list(self.pipeline.named_steps.keys())) + \")\"\n", | ||||
|     "        \n", | ||||
|     "        import __main__ as main\n", | ||||
|     "        if not hasattr(main, '__file__'):\n", | ||||
|     "            display(\"saved pipeline. It can be loaded the following way:\")\n", | ||||
|     "            display(Markdown(\"> ```\\n\"+load_command+\"\\n```\"))\n", | ||||
|     "        else:\n", | ||||
|     "            print(\"saved pipeline. It can be loaded the following way:\")\n", | ||||
|     "            print(load_command)\n", | ||||
|     "        \n", | ||||
|     "    \n", | ||||
|     "    def load(self, prefix:str, models = []):\n", | ||||
|     "        self.pipeline = None\n", | ||||
|     "        model_list = []\n", | ||||
|     "        for model in models:\n", | ||||
|     "            if model in self.keras_models:\n", | ||||
|     "                model_list.append((model, load_model(prefix + \".\" + model)))\n", | ||||
|     "            else:\n", | ||||
|     "                model_list.append((model, joblib.load(prefix+\".\" + model)))\n", | ||||
|     "        self.pipeline = Pipeline(model_list)\n", | ||||
|     "    \n", | ||||
|     "    def fit(self,X,y):\n", | ||||
|     "        self.pipeline.fit(X,y)\n", | ||||
|     "    \n", | ||||
|     "    def predict(self,X):\n", | ||||
|     "        return self.pipeline.predict(X)\n", | ||||
|     "    " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
| @ -345,7 +442,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 23, | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -354,12 +451,49 @@ | ||||
|     "        self.sdm = sdm\n", | ||||
|     "        self.pm = pm\n", | ||||
|     "    \n", | ||||
|     "    def fit(self):\n", | ||||
|     "        # TODO: make batch fitting available here\n", | ||||
|     "        self.pm.fit(X = self.sdm.X, y = self.sdm.y)\n", | ||||
|     "    def fit(self, max_size=10000, disabled_fit_steps=['vectorizer']):\n", | ||||
|     "        # TODO: make batch fitting available here (eg: continous waiting for data and fitting them)\n", | ||||
|     "        if self.sdm.X is None:\n", | ||||
|     "            self.sdm.create_train_test_split()\n", | ||||
|     "        disabled_fits = {}\n", | ||||
|     "        disabled_fit_transforms = {}\n", | ||||
|     "        \n", | ||||
|     "        named_steps = self.pm.pipeline.named_steps\n", | ||||
|     "        \n", | ||||
|     "        for s in disabled_fit_steps:\n", | ||||
|     "            # now it gets a little bit dirty:\n", | ||||
|     "            # replace fit functions we don't want to call again (e.g. for vectorizers)\n", | ||||
|     "            disabled_fits[s] = named_steps[s].fit\n", | ||||
|     "            disabled_fit_transforms[s] = named_steps[s].fit_transform\n", | ||||
|     "            named_steps[s].fit = lambda self, X, y=None: self\n", | ||||
|     "            named_steps[s].fit_transform = named_steps[s].transform\n", | ||||
|     "            \n", | ||||
|     "        self.pm.fit(X = self.sdm.X[:max_size], y = self.sdm.y[:max_size])\n", | ||||
|     "        \n", | ||||
|     "        # restore replaced fit functions:\n", | ||||
|     "        for s in disabled_fit_steps:\n", | ||||
|     "            named_steps[s].fit = disabled_fits[s]\n", | ||||
|     "            named_steps[s].fit_transform = disabled_fit_transforms[s]\n", | ||||
|     "    \n", | ||||
|     "    def test(self):\n", | ||||
|     "        '''\n", | ||||
|     "        return: prediction:list, teacher:list\n", | ||||
|     "        '''\n", | ||||
|     "        if self.sdm.X is None:\n", | ||||
|     "            self.sdm.create_train_test_split()\n", | ||||
|     "        return self.pm.predict(self.sdm.Xt), self.sdm.yt\n", | ||||
|     "\n", | ||||
|     "    " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## Train" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
| @ -369,21 +503,371 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 25, | ||||
|    "execution_count": 12, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "you are in a notebook\n" | ||||
|       "reading file: ./data_en/2017-11-01.json...\n", | ||||
|       "imported 33368 samples\n", | ||||
|       "remaining samples after top emoji filtering:  26197\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
|     "    print(\"you are in a notebook\")" | ||||
|     "    # we are in an interactive environment (probably in jupyter)\n", | ||||
|     "    # load data:\n", | ||||
|     "    sdm = sample_data_manager.generate_and_read(path=\"./data_en/\", n_top_emojis=20, file_range=range(1))\n", | ||||
|     "    " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 22, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Epoch 1/1\n", | ||||
|       "100/100 [==============================] - 3s 27ms/step - loss: 0.1225\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "    #pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", | ||||
|     "    #                                                           layers=[(10000, 'relu'),(5000, 'relu'),(2500, 'relu'),(y1[0].shape[0],None)], sdm=sdm)\n", | ||||
|     "    pm = pipeline_manager.create_keras_pipeline_with_vectorizer(vectorizer=TfidfVectorizer(stop_words='english'),\n", | ||||
|     "                                                           layers=[(2500, 'relu'),(3,None)], sdm=sdm)\n", | ||||
|     "    \n", | ||||
|     "    tr = trainer(sdm=sdm, pm=pm)\n", | ||||
|     "    tr.fit(100)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## save classifier" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 23, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "['keras_model']\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "'saved pipeline. It can be loaded the following way:'" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "> ```\n", | ||||
|        "pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n", | ||||
|        "```" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "pm.save('custom_classifier')" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## Prediction" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 33, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "array([[0.15801723, 0.11859037, 0.10975348],\n", | ||||
|        "       [0.17035495, 0.10913695, 0.09354854],\n", | ||||
|        "       [0.11777218, 0.06569621, 0.06620223],\n", | ||||
|        "       ...,\n", | ||||
|        "       [0.14746301, 0.09480572, 0.08052498],\n", | ||||
|        "       [0.15932804, 0.11895895, 0.10343507],\n", | ||||
|        "       [0.17135939, 0.1061406 , 0.09402546]], dtype=float32)" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "array([[0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.70401758, 0.05932203, 0.23666039],\n", | ||||
|        "       ...,\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797],\n", | ||||
|        "       [0.46813021, 0.24716181, 0.28470797]])" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "prediction variance:  0.0006294687\n", | ||||
|       "teacher variance:  0.03341702104519965\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/html": [ | ||||
|        "<div>\n", | ||||
|        "<style scoped>\n", | ||||
|        "    .dataframe tbody tr th:only-of-type {\n", | ||||
|        "        vertical-align: middle;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe tbody tr th {\n", | ||||
|        "        vertical-align: top;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe thead th {\n", | ||||
|        "        text-align: right;\n", | ||||
|        "    }\n", | ||||
|        "</style>\n", | ||||
|        "<table border=\"1\" class=\"dataframe\">\n", | ||||
|        "  <thead>\n", | ||||
|        "    <tr style=\"text-align: right;\">\n", | ||||
|        "      <th></th>\n", | ||||
|        "      <th>predict</th>\n", | ||||
|        "      <th>predicted_sentiment</th>\n", | ||||
|        "      <th>teacher</th>\n", | ||||
|        "      <th>teacher_sentiment</th>\n", | ||||
|        "      <th>text</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>35671</th>\n", | ||||
|        "      <td>😢</td>\n", | ||||
|        "      <td>[0.15801723301410675, 0.11859036982059479, 0.1...</td>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n", | ||||
|        "      <td>i feel like i care so much more in everi situat</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>25683</th>\n", | ||||
|        "      <td>😢</td>\n", | ||||
|        "      <td>[0.1703549474477768, 0.10913695394992828, 0.09...</td>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n", | ||||
|        "      <td>i did not meat to add that 2 there ... hav see...</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>8985</th>\n", | ||||
|        "      <td>😢</td>\n", | ||||
|        "      <td>[0.1177721843123436, 0.06569620966911316, 0.06...</td>\n", | ||||
|        "      <td>😊</td>\n", | ||||
|        "      <td>[0.7040175768989329, 0.059322033898305086, 0.2...</td>\n", | ||||
|        "      <td>never…</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>5410</th>\n", | ||||
|        "      <td>😢</td>\n", | ||||
|        "      <td>[0.18182337284088135, 0.12382747232913971, 0.0...</td>\n", | ||||
|        "      <td>😂</td>\n", | ||||
|        "      <td>[0.46813021474490496, 0.24716181096977158, 0.2...</td>\n", | ||||
|        "      <td>lmao on me ! ! ! wtf wa he suppos to say</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>62611</th>\n", | ||||
|        "      <td>😢</td>\n", | ||||
|        "      <td>[0.1786666363477707, 0.11502400785684586, 0.10...</td>\n", | ||||
|        "      <td>😊</td>\n", | ||||
|        "      <td>[0.7040175768989329, 0.059322033898305086, 0.2...</td>\n", | ||||
|        "      <td>this dude alway help me get through my school ...</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "      predict                                predicted_sentiment teacher  \\\n", | ||||
|        "35671       😢  [0.15801723301410675, 0.11859036982059479, 0.1...       😂   \n", | ||||
|        "25683       😢  [0.1703549474477768, 0.10913695394992828, 0.09...       😂   \n", | ||||
|        "8985        😢  [0.1177721843123436, 0.06569620966911316, 0.06...       😊   \n", | ||||
|        "5410        😢  [0.18182337284088135, 0.12382747232913971, 0.0...       😂   \n", | ||||
|        "62611       😢  [0.1786666363477707, 0.11502400785684586, 0.10...       😊   \n", | ||||
|        "\n", | ||||
|        "                                       teacher_sentiment  \\\n", | ||||
|        "35671  [0.46813021474490496, 0.24716181096977158, 0.2...   \n", | ||||
|        "25683  [0.46813021474490496, 0.24716181096977158, 0.2...   \n", | ||||
|        "8985   [0.7040175768989329, 0.059322033898305086, 0.2...   \n", | ||||
|        "5410   [0.46813021474490496, 0.24716181096977158, 0.2...   \n", | ||||
|        "62611  [0.7040175768989329, 0.059322033898305086, 0.2...   \n", | ||||
|        "\n", | ||||
|        "                                                    text  \n", | ||||
|        "35671    i feel like i care so much more in everi situat  \n", | ||||
|        "25683  i did not meat to add that 2 there ... hav see...  \n", | ||||
|        "8985                                              never…  \n", | ||||
|        "5410            lmao on me ! ! ! wtf wa he suppos to say  \n", | ||||
|        "62611  this dude alway help me get through my school ...  " | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Mean Squared Error:  [0.14140389 0.04240099 0.02944344]\n", | ||||
|       "Variance teacher:  [0.02183094 0.02513847 0.00285735]\n", | ||||
|       "Variance prediction:  [0.00053908 0.00024232 0.00021658]\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
|     "    pred, teacher = tr.test()\n", | ||||
|     "    \n", | ||||
|     "    display(pred)\n", | ||||
|     "    display(teacher)\n", | ||||
|     "    \n", | ||||
|     "    print('prediction variance: ', np.linalg.norm(np.var(pred, axis=0)))\n", | ||||
|     "    print('teacher variance: ', np.linalg.norm(np.var(teacher, axis=0)))\n", | ||||
|     "    \n", | ||||
|     "    # build a dataframe to visualize test results:\n", | ||||
|     "    testlist = pd.DataFrame({'text': sdm.Xt, \n", | ||||
|     "                         'teacher': sent2emoji(sdm.yt),\n", | ||||
|     "                         'teacher_sentiment': sdm.yt.tolist(),\n", | ||||
|     "                         'predict': sent2emoji(pred, custom_target_emojis=sdm.top_emojis),\n", | ||||
|     "                         'predicted_sentiment': pred.tolist()})\n", | ||||
|     "    # display:\n", | ||||
|     "    display(testlist.head())\n", | ||||
|     "    \n", | ||||
|     "    # mean squared error:\n", | ||||
|     "    teacher_sentiments = np.array([sample[1]['teacher_sentiment'] for sample in testlist.iterrows()])\n", | ||||
|     "    predicted_sentiments = np.array([sample[1]['predicted_sentiment'] for sample in testlist.iterrows()])\n", | ||||
|     "\n", | ||||
|     "    mean_squared_error = ((teacher_sentiments - predicted_sentiments)**2).mean(axis=0)\n", | ||||
|     "    print(\"Mean Squared Error: \", mean_squared_error)\n", | ||||
|     "    print(\"Variance teacher: \", np.var(teacher_sentiments, axis=0))\n", | ||||
|     "    print(\"Variance prediction: \", np.var(predicted_sentiments, axis=0))\n", | ||||
|     "    \n", | ||||
|     "    # save to csv:\n", | ||||
|     "    testlist.to_csv('test.csv')" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## Load classifier" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import __main__ as main\n", | ||||
|     "if not hasattr(main, '__file__'):\n", | ||||
|     "    try:\n", | ||||
|     "        pm\n", | ||||
|     "    except NameError:\n", | ||||
|     "        pass\n", | ||||
|     "    else:\n", | ||||
|     "        del pm # delete existing pipeline manager if ther is one\n", | ||||
|     "\n", | ||||
|     "    pm = pipeline_manager.load_pipeline_from_files( 'custom_classifier', ['keras_model'], ['vectorizer', 'keras_model'])\n", | ||||
|     "    lookup_emojis = [#'😂',\n", | ||||
|     "         '😭',\n", | ||||
|     "         '😍',\n", | ||||
|     "         '😩',\n", | ||||
|     "         '😊',\n", | ||||
|     "         '😘',\n", | ||||
|     "         '🙏',\n", | ||||
|     "         '🙌',\n", | ||||
|     "         '😉',\n", | ||||
|     "         '😁',\n", | ||||
|     "         '😅',\n", | ||||
|     "         '😎',\n", | ||||
|     "         '😢',\n", | ||||
|     "         '😒',\n", | ||||
|     "         '😏',\n", | ||||
|     "         '😌',\n", | ||||
|     "         '😔',\n", | ||||
|     "         '😋',\n", | ||||
|     "         '😀',\n", | ||||
|     "         '😤']\n", | ||||
|     "    out = widgets.Output()\n", | ||||
|     "\n", | ||||
|     "    t = widgets.Text()\n", | ||||
|     "    b = widgets.Button(\n", | ||||
|     "        description='get emoji',\n", | ||||
|     "        disabled=False,\n", | ||||
|     "        button_style='', # 'success', 'info', 'warning', 'danger' or ''\n", | ||||
|     "        tooltip='Click me',\n", | ||||
|     "        icon='check'\n", | ||||
|     "    )\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "    def handle_submit(sender):\n", | ||||
|     "        with out:\n", | ||||
|     "            clear_output()\n", | ||||
|     "        with out:\n", | ||||
|     "            pred = pm.predict([t.value])\n", | ||||
|     "\n", | ||||
|     "            display(Markdown(\"# Predicted Emoji \" + str(sent2emoji(pred, lookup_emojis)[0])))\n", | ||||
|     "            display(Markdown(\"# Sentiment Vector: $$ \\pmatrix{\" + str(pred[0,0]) +\n", | ||||
|     "                             \"\\\\\\\\\" + str(pred[0,1]) + \"\\\\\\\\\" + str(pred[0,2]) + \"}$$\"))\n", | ||||
|     "\n", | ||||
|     "    b.on_click(handle_submit)\n", | ||||
|     "\n", | ||||
|     "    display(t)\n", | ||||
|     "    display(widgets.VBox([b, out]))  " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|  | ||||
		Reference in New Issue
	
	Block a user