Merge branch 'master' of ssh://gogs@the-cake-is-a-lie.net:20022/jonas/NLP-LAB.git
This commit is contained in:
		
							
								
								
									
										130
									
								
								Project/naive_approach/naiveApproachTest.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										130
									
								
								Project/naive_approach/naiveApproachTest.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,130 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import naive_approach" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "top_emojis = ['😂',\n", | ||||
|     "         '😭',\n", | ||||
|     "         '😍',\n", | ||||
|     "         '😩',\n", | ||||
|     "         '😊',\n", | ||||
|     "         '😘',\n", | ||||
|     "         '🙏',\n", | ||||
|     "         '🙌',\n", | ||||
|     "         '😉',\n", | ||||
|     "         '😁',\n", | ||||
|     "         '😅',\n", | ||||
|     "         '😎',\n", | ||||
|     "         '😢',\n", | ||||
|     "         '😒',\n", | ||||
|     "         '😏',\n", | ||||
|     "         '😌',\n", | ||||
|     "         '😔',\n", | ||||
|     "         '😋',\n", | ||||
|     "         '😀',\n", | ||||
|     "         '😤']" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "lookup = naive_approach.prepareData(emojis_to_consider=top_emojis)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "sentence=\"I am very happy today\"" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "ename": "KeyError", | ||||
|      "evalue": "357", | ||||
|      "output_type": "error", | ||||
|      "traceback": [ | ||||
|       "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", | ||||
|       "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)", | ||||
|       "\u001b[1;32m<ipython-input-7-a7b8b0832a7d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnaive_approach\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlookup\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0memojis_to_consider\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtop_emojis\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0mn\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", | ||||
|       "\u001b[1;32m~\\Desktop\\NLP-LAB\\Project\\naive_approach\\naive_approach.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(sentence, lookup, emojis_to_consider, criteria, description_key, lang, n, t)\u001b[0m\n\u001b[0;32m    117\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    118\u001b[0m     \u001b[1;31m# build a result table\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 119\u001b[1;33m     \u001b[0mtable_array\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlookup\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindexes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtable\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindexes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdescription_key\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m     \u001b[0mtable_frame\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtable_array\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcriteria\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'description'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[1;32m~\\Desktop\\NLP-LAB\\Project\\naive_approach\\naive_approach.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    117\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    118\u001b[0m     \u001b[1;31m# build a result table\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 119\u001b[1;33m     \u001b[0mtable_array\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlookup\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindexes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtable\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindexes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdescription_key\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m     \u001b[0mtable_frame\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtable_array\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcriteria\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'description'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[1;31mKeyError\u001b[0m: 357" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "pred = naive_approach.predict(sentence, lookup, emojis_to_consider=top_emojis,  n=3)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "['🎁', '🙋', '\\U0001f91f']" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 9, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "pred" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.6.4" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 2 | ||||
| } | ||||
| @ -37,7 +37,7 @@ def stemming(messages): | ||||
|  | ||||
|  | ||||
| # * compare words to emoji descriptions | ||||
| def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng'): | ||||
| def evaluate_sentence(sentence, table, description_key = 'description', lang = 'eng', emojis_to_consider="all"): | ||||
|      | ||||
|     tokenized_sentence = word_tokenize(sentence) | ||||
|     n = len(tokenized_sentence) | ||||
| @ -77,10 +77,7 @@ def evaluate_sentence(sentence, table, description_key = 'description', lang = ' | ||||
|  | ||||
| # load and preprocess data | ||||
| # emojis_to_consider can be either a list or "all" | ||||
| def prepareData(stemming=False, emojis_to_consider="all"): | ||||
|  | ||||
|     table.head() | ||||
|      | ||||
| def prepareData(stemming=False): | ||||
|     if(stemming): | ||||
|         table['description'] = stemming(table['description']) | ||||
|      | ||||
| @ -88,9 +85,8 @@ def prepareData(stemming=False, emojis_to_consider="all"): | ||||
|     lookup = {} | ||||
|     emoji_set = [] | ||||
|     for index, row in table.iterrows(): | ||||
|         if(emojis_to_consider=="all" or (type(emojis_to_consider)==list and row['character'] in emojis_to_consider)): | ||||
|             lookup[index] = row['character'] | ||||
|             emoji_set.append(row['character']) | ||||
|         lookup[index] = row['character'] | ||||
|         emoji_set.append(row['character']) | ||||
|  | ||||
|     emoji_set = set(emoji_set) | ||||
|      | ||||
| @ -99,30 +95,44 @@ def prepareData(stemming=False, emojis_to_consider="all"): | ||||
| # make a prediction for an input sentence | ||||
| def predict(sentence, lookup, emojis_to_consider="all", criteria="threshold", description_key='description', lang = 'eng', n=10, t=0.9): | ||||
|  | ||||
|     result = evaluate_sentence(sentence, table, description_key, lang) | ||||
|     result = evaluate_sentence(sentence, table, description_key, lang, emojis_to_consider=emojis_to_consider) | ||||
|      | ||||
|     if(criteria=="summed"): | ||||
|         indexes = np.argsort([-np.sum(x) for x in result])[0:n] | ||||
|     elif (criteria=="max_val"): | ||||
|         indexes = np.argsort([-np.max(x) for x in result])[0:n] | ||||
|     elif(criteria=="avg"): | ||||
|         indexes = np.argsort([-np.mean(x) for x in result])[0:n] | ||||
|     else: | ||||
|         indexes= np.argsort([-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result])[0:n]  | ||||
|     try: | ||||
|         if(criteria=="summed"): | ||||
|             resultValues = [-np.sum(x) for x in result] | ||||
|         elif (criteria=="max_val"): | ||||
|             resultValues = [-np.max(x) for x in result] | ||||
|         elif(criteria=="avg"): | ||||
|             resultValues = [-np.mean(x) for x in result] | ||||
|         else: | ||||
|             resultValues = [-len(np.where(x>t)[0]) / (x.shape[0] * x.shape[1]) for x in result] | ||||
|         indexes = np.argsort(resultValues) | ||||
|         results = np.sort(resultValues) | ||||
|          | ||||
|         if (emojis_to_consider != "all" and type(emojis_to_consider) == list): | ||||
|             indexes2 = [] | ||||
|             results2 = [] | ||||
|             for i in range(len(indexes)): | ||||
|                 if lookup[indexes[i]] in emojis_to_consider: | ||||
|                     indexes2.append(indexes[i]) | ||||
|                     results2.append(results[i]) | ||||
|             indexes = indexes2 | ||||
|             results = results2 | ||||
|         indexes = indexes[0:n] | ||||
|         results = results[0:n] | ||||
|          | ||||
|         # build a result table | ||||
|         table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] | ||||
|            | ||||
|         table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) | ||||
|          | ||||
|         #display(table_frame) | ||||
|          | ||||
|         return list(table_frame[criteria]), results | ||||
|      | ||||
|     if(emojis_to_consider!="all"): | ||||
|         for i in indexes: | ||||
|             if (i not in lookup): | ||||
|                 indexes = np.delete(indexes, [i]) | ||||
|      | ||||
|     # build a result table | ||||
|     table_array = [[lookup[indexes[i]], str(table.iloc[indexes[i]][description_key])] for i in range(n) ] | ||||
|        | ||||
|     table_frame = pd.DataFrame(table_array, columns=[criteria, 'description']) | ||||
|      | ||||
|     #display(table_frame) | ||||
|      | ||||
|     return list(table_frame[criteria]) | ||||
|     except ZeroDivisionError as err: | ||||
|         print("There seems to be a problem with the input format. Please enter a nonempty string") | ||||
|  | ||||
|  | ||||
| #predict("I like to travel by train", description_key='description' , lang='eng') | ||||
|  | ||||
|  | ||||
| @ -144,7 +144,7 @@ | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "5ac970d7d7cf4849b4f5adfb80a820c0", | ||||
|        "model_id": "4fd5552e6a024dcaa0f35a594c77ae99", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
| @ -172,7 +172,7 @@ | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", | ||||
|     "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=10), \"n_top_emojis\")\n", | ||||
|     "                   (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Button(disabled=True),\"load_data\")\n", | ||||
| @ -235,6 +235,18 @@ | ||||
|     "                ]\n", | ||||
|     "            ], \n", | ||||
|     "            \"train\" )\n", | ||||
|     "create_area(\"playground 😎\",\n", | ||||
|     "           [\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Text(),\"test_input\"),\n", | ||||
|     "                   (widgets.HTML(),\"prediction\")\n", | ||||
|     "               ],\n", | ||||
|     "               [\n", | ||||
|     "                   (widgets.Checkbox(),\"show_sorted_list\")\n", | ||||
|     "               ]\n", | ||||
|     "           ],\n", | ||||
|     "           \"playground\")\n", | ||||
|     "\n", | ||||
|     "tab_manager" | ||||
|    ] | ||||
|   }, | ||||
| @ -360,11 +372,11 @@ | ||||
|    "source": [ | ||||
|     "class progress_indicator(object):\n", | ||||
|     "    \n", | ||||
|     "    def __init__(self, n, description=\"progress\"):\n", | ||||
|     "        self.w = widgets.IntProgress(value=0, min=0,max=n, description = description)\n", | ||||
|     "    def __init__(self, description=\"progress\"):\n", | ||||
|     "        self.w = widgets.FloatProgress(value=0, min=0,max=1, description = description)\n", | ||||
|     "        display(self.w)\n", | ||||
|     "    def update(self, dn=1):\n", | ||||
|     "        self.w.value += dn\n", | ||||
|     "    def update(self, val):\n", | ||||
|     "        self.w.value = val\n", | ||||
|     "        " | ||||
|    ] | ||||
|   }, | ||||
| @ -416,13 +428,15 @@ | ||||
|     "        r = shown_widgets[\"file_range\"].value\n", | ||||
|     "        r = (r[0], r[1] + 1) # range has to be exclusive according to the last element!\n", | ||||
|     "        \n", | ||||
|     "        p = progress_indicator(r[1] - r[0], \"reading progress\")\n", | ||||
|     "        p_r = progress_indicator(\"reading progress\")\n", | ||||
|     "        p_s = progress_indicator(\"stemming progress\")\n", | ||||
|     "        \n", | ||||
|     "        sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", | ||||
|     "                                                    n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", | ||||
|     "                                                    file_range=range(r[0], r[1]),\n", | ||||
|     "                                                    n_kmeans_cluster=shown_widgets[\"k_means_cluster\"].value,\n", | ||||
|     "                                                    progress_callback=p.update)\n", | ||||
|     "                                                    read_progress_callback=p_r.update,\n", | ||||
|     "                                                    stem_progress_callback=p_s.update)\n", | ||||
|     "        shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", | ||||
|     "        \n", | ||||
|     "        \n", | ||||
| @ -461,11 +475,10 @@ | ||||
|     "        \n", | ||||
|     "        print(\"update train test split:\")\n", | ||||
|     "        sdm.create_train_test_split(split=val_split)\n", | ||||
|     "        batch_n = len(sdm.X) // batch_size\n", | ||||
|     "        \n", | ||||
|     "        print(\"fit\")\n", | ||||
|     "        \n", | ||||
|     "        p = progress_indicator(batch_n)\n", | ||||
|     "        p = progress_indicator()\n", | ||||
|     "        \n", | ||||
|     "        tr = stl.trainer(sdm=sdm, pm=pm)\n", | ||||
|     "        tr.fit(progress_callback=p.update, batch_size=batch_size, n_epochs=n_epochs)\n", | ||||
| @ -601,6 +614,46 @@ | ||||
|     "\n", | ||||
|     "\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "## testing area" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def test_input(b):\n", | ||||
|     "    global sdm\n", | ||||
|     "    global pm\n", | ||||
|     "    global tr\n", | ||||
|     "    with out_areas[\"playground\"]:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"----\")\n", | ||||
|     "        if pm is None:\n", | ||||
|     "            sys.stderr.write(\"ERROR: load or create classifier first\")\n", | ||||
|     "            return\n", | ||||
|     "        X = shown_widgets[\"test_input\"].value\n", | ||||
|     "        pred = pm.predict([X])\n", | ||||
|     "        shown_widgets[\"prediction\"].value = \"<h1> \" + str(stl.sent2emoji(pred)[0]) + \"</h1>\"\n", | ||||
|     "        if shown_widgets[\"show_sorted_list\"].value:\n", | ||||
|     "            mp(\"## \" + \"\".join(stl.edist.sentiment_vector_to_emoji(pred, only_emoticons=True, n_results=100)))\n", | ||||
|     "\n", | ||||
|     "#link\n", | ||||
|     "shown_widgets[\"test_input\"].observe(test_input)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|  | ||||
| @ -28,6 +28,12 @@ nltk.download('punkt') | ||||
| nltk.download('averaged_perceptron_tagger') | ||||
| nltk.download('wordnet') | ||||
|  | ||||
| # check whether the display function exists: | ||||
| try: | ||||
|     display | ||||
| except NameError: | ||||
|     print("no fancy display function found... using print instead") | ||||
|     display = print | ||||
|  | ||||
| # In[2]: | ||||
|  | ||||
| @ -108,6 +114,38 @@ def get_wordnet_pos(treebank_tag): | ||||
|         return wordnet.NOUN | ||||
|  | ||||
|  | ||||
| # global stemmer and lemmatizer function | ||||
| stemmer = SnowballStemmer("english") | ||||
|  | ||||
| def stem(s): | ||||
|     stemmed_sent = [] | ||||
|     for word in s.split(" "): | ||||
|         word_stemmed = stemmer.stem(word) | ||||
|         stemmed_sent.append(word_stemmed) | ||||
|     stemmed_sent = (" ").join(stemmed_sent) | ||||
|     return stemmed_sent | ||||
|  | ||||
|  | ||||
| lemmatizer = WordNetLemmatizer() | ||||
|  | ||||
| def lemm(s): | ||||
|     lemmatized_sent = [] | ||||
|     sent_pos = pos_tag(word_tokenize(s)) | ||||
|     for word in sent_pos: | ||||
|         wordnet_pos = get_wordnet_pos(word[1].lower()) | ||||
|         word_lemmatized = lemmatizer.lemmatize(word[0], pos=wordnet_pos) | ||||
|         lemmatized_sent.append(word_lemmatized) | ||||
|     lemmatized_sent = (" ").join(lemmatized_sent) | ||||
|     return lemmatized_sent | ||||
|  | ||||
|  | ||||
| def batch_stem(sentences): | ||||
|     return [stem(s) for s in sentences] | ||||
|  | ||||
| def batch_lemm(sentences): | ||||
|     return [lemm(s) for s in sentences] | ||||
|  | ||||
|  | ||||
| # ### sample data manager | ||||
| # the sample data manager loads and preprocesses data | ||||
| # most common way to use: | ||||
| @ -123,7 +161,7 @@ def get_wordnet_pos(treebank_tag): | ||||
|  | ||||
| class sample_data_manager(object): | ||||
|     @staticmethod | ||||
|     def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, progress_callback=None): | ||||
|     def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None): | ||||
|         """ | ||||
|         generate, read and process train data in one step. | ||||
|          | ||||
| @ -137,9 +175,9 @@ class sample_data_manager(object): | ||||
|         @return: sample_data_manager object | ||||
|         """ | ||||
|         sdm = sample_data_manager(path) | ||||
|         sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons, progress_callback=progress_callback) | ||||
|         sdm.read_files(file_index_range=range(sdm.n_files) if file_range is None else file_range, only_emoticons=only_emoticons, progress_callback=read_progress_callback) | ||||
|         if apply_stemming: | ||||
|             sdm.apply_stemming_and_lemmatization() | ||||
|             sdm.apply_stemming_and_lemmatization(progress_callback=stem_progress_callback) | ||||
|          | ||||
|         sdm.generate_emoji_count_and_weights() | ||||
|          | ||||
| @ -161,7 +199,6 @@ class sample_data_manager(object): | ||||
|         self.data_root_folder = data_root_folder | ||||
|         self.json_files = sorted(glob.glob(self.data_root_folder + "/*.json")) | ||||
|         self.n_files = len(self.json_files) | ||||
|         self.raw_data = None | ||||
|         self.emojis = None | ||||
|         self.plain_text = None | ||||
|         self.labels = None | ||||
| @ -176,46 +213,70 @@ class sample_data_manager(object): | ||||
|         self.use_binary_labels = False | ||||
|         self.kmeans_cluster = None | ||||
|         self.label_binarizer = None | ||||
|         self.use_stemming = False | ||||
|         self.use_lemmatization = False | ||||
|      | ||||
|     def read_files(self, file_index_range:list, only_emoticons=True, progress_callback=None): | ||||
|     def read_files(self, file_index_range:list, only_emoticons=True, emoji_mean=False ,progress_callback=None): | ||||
|         """ | ||||
|         reading (multiple) files to one panda table. | ||||
|          | ||||
|         @param file_index_range: range of file's indices to read (eg `range(3)` to read the first three files) | ||||
|         @param only_emoticons: if True, only messages containing emoticons (aka smileys) are used. This classification is derived from Tools.Emoji_Distance | ||||
|         @param emoji_mean: if True, using mean of all emojis instead of the last one | ||||
|         """ | ||||
|         assert np.min(file_index_range) >= 0 and np.max(file_index_range) < self.n_files | ||||
|         n = len(file_index_range) | ||||
|         for i in file_index_range: | ||||
|             print("reading file: " + self.json_files[i] + "...") | ||||
|             if self.raw_data is None: | ||||
|                 self.raw_data = pd.read_json(self.json_files[i], encoding="utf-8") | ||||
|             else: | ||||
|                 self.raw_data = self.raw_data.append(pd.read_json(self.json_files[i], encoding="utf-8")) | ||||
|             if progress_callback is not None: | ||||
|                 progress_callback() | ||||
|         self.emojis = self.raw_data['EMOJI'] | ||||
|         self.plain_text = self.raw_data['text'] | ||||
|          | ||||
|         # replacing keywords. TODO: maybe these information can be extracted and used | ||||
|         self.plain_text = self.plain_text.str.replace("(<EMOJI>|<USER>|<HASHTAG>)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","") | ||||
|          | ||||
|         # so far filtering for the latest emoji. TODO: maybe there are also better approaches | ||||
|         self.labels = emoji2sent([latest(e) for e in self.emojis], only_emoticons=only_emoticons ) | ||||
|          | ||||
|         # and filter out all samples we have no label for: | ||||
|         wrong_labels = np.isnan(np.linalg.norm(self.labels, axis=1))     | ||||
|             raw_data_i = pd.read_json(self.json_files[i], encoding="utf-8") | ||||
|             emojis_i = raw_data_i['EMOJI'] | ||||
|             plain_text_i = raw_data_i['text'] | ||||
|  | ||||
|         self.labels = self.labels[np.invert(wrong_labels)] | ||||
|         self.plain_text = self.plain_text[np.invert(wrong_labels)] | ||||
|         self.emojis = self.emojis[np.invert(wrong_labels)] | ||||
|              # replacing keywords. TODO: maybe these information can be extracted and used | ||||
|             plain_text_i = plain_text_i.str.replace("(<EMOJI>|<USER>|<HASHTAG>)","").str.replace("[" + "".join(list(emoji_blacklist)) + "]","") | ||||
|  | ||||
|             if not emoji_mean: | ||||
|                 # so far filtering for the latest emoji. TODO: maybe there are also better approaches | ||||
|                 labels_i = emoji2sent([latest(e) for e in emojis_i], only_emoticons=only_emoticons ) | ||||
|             else: | ||||
|                 labels_i = np.array([np.mean(emoji2sent(e, only_emoticons=only_emoticons), axis=0) for e in emojis_i]) | ||||
|  | ||||
|             # and filter out all samples we have no label for: | ||||
|             wrong_labels = np.isnan(np.linalg.norm(labels_i, axis=1)) | ||||
|             labels_i = labels_i[np.invert(wrong_labels)] | ||||
|             plain_text_i = plain_text_i[np.invert(wrong_labels)] | ||||
|             emojis_i = emojis_i[np.invert(wrong_labels)] | ||||
|             print("imported " + str(len(labels_i)) + " samples") | ||||
|  | ||||
|             if self.labels is None: | ||||
|                 self.labels = labels_i | ||||
|             else: | ||||
|                 self.labels = np.append(self.labels, labels_i, axis=0) | ||||
|              | ||||
|             if self.emojis is None: | ||||
|                 self.emojis = emojis_i | ||||
|             else: | ||||
|                 self.emojis = pd.concat([self.emojis,emojis_i],ignore_index=True) | ||||
|              | ||||
|             if self.plain_text is None: | ||||
|                 self.plain_text = plain_text_i | ||||
|             else: | ||||
|                 self.plain_text = pd.concat([self.plain_text,plain_text_i],ignore_index=True) | ||||
|  | ||||
|             if progress_callback is not None: | ||||
|                 progress_callback((i+1)/n) | ||||
|          | ||||
|         print("imported " + str(len(self.labels)) + " samples") | ||||
|      | ||||
|     def apply_stemming_and_lemmatization(self): | ||||
|     def apply_stemming_and_lemmatization(self, progress_callback = None): | ||||
|         """ | ||||
|         apply stemming and lemmatization to plain text samples | ||||
|         """ | ||||
|         self.use_stemming = True | ||||
|         self.use_lemmatization = True | ||||
|         print("apply stemming and lemmatization...") | ||||
|         stemmer = SnowballStemmer("english") | ||||
|         n = self.plain_text.shape[0] * 2 # 2 for loops | ||||
|         i = 0 | ||||
|         for key in self.plain_text.keys(): | ||||
|             stemmed_sent = [] | ||||
|             for word in self.plain_text[key].split(" "): | ||||
| @ -223,6 +284,11 @@ class sample_data_manager(object): | ||||
|                 stemmed_sent.append(word_stemmed) | ||||
|             stemmed_sent = (" ").join(stemmed_sent) | ||||
|             self.plain_text[key] = stemmed_sent | ||||
|             i += 1 | ||||
|             if progress_callback is not None and i % 1024 == 0: | ||||
|                 progress_callback(i / n) | ||||
|                  | ||||
|  | ||||
|              | ||||
|         lemmatizer = WordNetLemmatizer() | ||||
|         for key in self.plain_text.keys(): | ||||
| @ -234,6 +300,10 @@ class sample_data_manager(object): | ||||
|                 lemmatized_sent.append(word_lemmatized) | ||||
|             lemmatized_sent = (" ").join(lemmatized_sent) | ||||
|             self.plain_text[key] = lemmatized_sent | ||||
|             i += 1 | ||||
|             if progress_callback is not None and i % 1024 == 0: | ||||
|                 progress_callback(i / n) | ||||
|         print("stemming and lemmatization done") | ||||
|      | ||||
|     def generate_emoji_count_and_weights(self): | ||||
|         """ | ||||
| @ -503,8 +573,12 @@ class pipeline_manager(object): | ||||
|         """fitting the pipeline""" | ||||
|         self.pipeline.fit(X,y) | ||||
|      | ||||
|     def predict(self,X): | ||||
|     def predict(self,X, use_stemming=True, use_lemmatization=True): | ||||
|         """predict""" | ||||
|         if use_stemming: | ||||
|             X = np.array(batch_stem(X)) | ||||
|         if use_lemmatization: | ||||
|             X = np.array(batch_lemm(X)) | ||||
|         return self.pipeline.predict(X) | ||||
|      | ||||
|  | ||||
| @ -558,7 +632,7 @@ class trainer(object): | ||||
|                 for j in range(n): | ||||
|                     self.pm.fit(X = np.array(self.sdm.X[j*batch_size:(j+1)*batch_size]), y = np.array(self.sdm.y[j*batch_size:(j+1)*batch_size])) | ||||
|                     if progress_callback is not None: | ||||
|                         progress_callback() | ||||
|                         progress_callback(j / n) | ||||
|                     pred, yt = self.test() | ||||
|                     mean_squared_error = ((pred - yt)**2).mean(axis=0) | ||||
|                     print("#" + str(j) + ": loss: ", mean_squared_error) | ||||
| @ -578,6 +652,6 @@ class trainer(object): | ||||
|         ''' | ||||
|         if self.sdm.X is None: | ||||
|             self.sdm.create_train_test_split() | ||||
|         return self.pm.predict(self.sdm.Xt), self.sdm.yt | ||||
|         return self.pm.predict(self.sdm.Xt, use_lemmatization=False, use_stemming=False), self.sdm.yt | ||||
|  | ||||
|      | ||||
		Reference in New Issue
	
	Block a user