added ability to filter by sentence length
This commit is contained in:
		| @ -11,7 +11,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 14, | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -43,6 +43,19 @@ | ||||
|       "[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n", | ||||
|       "[nltk_data]   Package wordnet is already up-to-date!\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "ename": "NameError", | ||||
|      "evalue": "name 'min_words' is not defined", | ||||
|      "output_type": "error", | ||||
|      "traceback": [ | ||||
|       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | ||||
|       "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)", | ||||
|       "\u001b[0;32m<ipython-input-2-ce00b6a80bda>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0msimple_twitter_learning\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_extraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCountVectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTfidfVectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHashingVectorizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[0;32m~/Dokumente/gitRepos/NLP-LAB/Project/simple_approach/simple_twitter_learning.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 166\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0msample_data_manager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    167\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgenerate_and_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0monly_emoticons\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapply_stemming\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_top_emojis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_range\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_kmeans_cluster\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mread_progress_callback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstem_progress_callback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0memoji_mean\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_target_emojis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[0;32m~/Dokumente/gitRepos/NLP-LAB/Project/simple_approach/simple_twitter_learning.py\u001b[0m in \u001b[0;36msample_data_manager\u001b[0;34m()\u001b[0m\n\u001b[1;32m    412\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"remaining samples after custom emoji filtering: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    413\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 414\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0mfilter_by_sentence_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmin_words\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    415\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplain_text\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[0;31mNameError\u001b[0m: name 'min_words' is not defined" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
| @ -69,7 +82,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -124,48 +137,9 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "----" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## User Interface" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "3e7d23dfb4b24f888d95bbd416565026", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/plain": [ | ||||
|        "Tab(children=(VBox(children=(HBox(children=(Text(value='./data_en/', description='root_path'), Button(descript…" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     } | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "mp(\"----\")\n", | ||||
|     "mp(\"## User Interface\")\n", | ||||
| @ -179,7 +153,8 @@ | ||||
|     "               [\n", | ||||
|     "                   (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n", | ||||
|     "                   (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\"),\n", | ||||
|     "                   (widgets.Checkbox(value=False,disabled=True), \"apply_lemmatization_and_stemming\")\n", | ||||
|     "                   (widgets.Checkbox(value=False,disabled=True), \"apply_lemmatization_and_stemming\"),\n", | ||||
|     "                   (widgets.BoundedIntText(value=5,min=0, max=10), \"min_words\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
| @ -280,7 +255,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -298,7 +273,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -386,7 +361,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -410,7 +385,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -461,6 +436,8 @@ | ||||
|     "        \n", | ||||
|     "        custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n", | ||||
|     "        \n", | ||||
|     "        min_words = shown_widgets[\"min_words\"].value\n", | ||||
|     "        \n", | ||||
|     "        sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", | ||||
|     "                                                    n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", | ||||
|     "                                                    file_range=range(r[0], r[1]),\n", | ||||
| @ -469,7 +446,8 @@ | ||||
|     "                                                    stem_progress_callback=p_s.update if lemm_and_stemm else None,\n", | ||||
|     "                                                    apply_stemming = lemm_and_stemm,\n", | ||||
|     "                                                    emoji_mean=emoji_mean,\n", | ||||
|     "                                                    custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None)\n", | ||||
|     "                                                    custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None,\n", | ||||
|     "                                                    min_words=min_words)\n", | ||||
|     "        shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", | ||||
|     "        \n", | ||||
|     "        \n", | ||||
| @ -487,7 +465,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -531,7 +509,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -668,7 +646,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -692,45 +670,6 @@ | ||||
|     "shown_widgets[\"test_input\"].observe(test_input)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 12, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "sdm" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 13, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "ename": "AttributeError", | ||||
|      "evalue": "'NoneType' object has no attribute 'pipeline'", | ||||
|      "output_type": "error", | ||||
|      "traceback": [ | ||||
|       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | ||||
|       "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)", | ||||
|       "\u001b[0;32m<ipython-input-13-beaf1df9153b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_steps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'vectorizer'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"I am sad\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", | ||||
|       "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'pipeline'" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "v = pm.pipeline.named_steps['vectorizer'].transform([\"I am sad\"])" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "pm.pipeline.named_steps['keras_model'].predict([v])" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|  | ||||
| @ -165,7 +165,7 @@ def batch_lemm(sentences): | ||||
|  | ||||
| class sample_data_manager(object): | ||||
|     @staticmethod | ||||
|     def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None): | ||||
|     def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None, min_words=0): | ||||
|         """ | ||||
|         generate, read and process train data in one step. | ||||
|          | ||||
| @ -194,6 +194,10 @@ class sample_data_manager(object): | ||||
|         if n_kmeans_cluster > 0: | ||||
|             sdm.generate_kmeans_binary_label(only_emoticons=only_emoticons, n_clusters=n_kmeans_cluster) | ||||
|          | ||||
|         if min_words > 0: | ||||
|             sdm.filter_by_sentence_length(min_words=min_words) | ||||
|  | ||||
|  | ||||
|         return sdm | ||||
|          | ||||
|      | ||||
| @ -407,6 +411,17 @@ class sample_data_manager(object): | ||||
|         self.emojis = self.emojis[in_list] | ||||
|         print("remaining samples after custom emoji filtering: ", len(self.labels)) | ||||
|  | ||||
|     def filter_by_sentence_length(self, min_words): | ||||
|         assert self.plain_text is not None | ||||
|  | ||||
|         is_long = [True if len(x.split()) >= min_words else False for x in self.plain_text] | ||||
|  | ||||
|         self.labels = self.labels[is_long] | ||||
|         self.plain_text = self.plain_text[is_long] | ||||
|         self.emojis = self.emojis[is_long] | ||||
|  | ||||
|         print("remaining samples after sentence length filtering: ", len(self.labels)) | ||||
|  | ||||
|     def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5): | ||||
|         """ | ||||
|         generate binary labels using kmeans. | ||||
|  | ||||
		Reference in New Issue
	
	Block a user