Merge branch 'master' of ssh://the-cake-is-a-lie.net:20022/jonas/NLP-LAB
This commit is contained in:
		
							
								
								
									
										291
									
								
								Project/Tools/Preprocessing.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										291
									
								
								Project/Tools/Preprocessing.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,291 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 36, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "[nltk_data] Downloading package stopwords to\n", | ||||
|       "[nltk_data]     C:\\Users\\Maren\\AppData\\Roaming\\nltk_data...\n", | ||||
|       "[nltk_data]   Unzipping corpora\\stopwords.zip.\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "True" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 36, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import pandas as pd\n", | ||||
|     "import nltk\n", | ||||
|     "nltk.download('stopwords')" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 33, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "table = pd.read_csv(\"emoji_descriptions.csv\")" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 15, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/html": [ | ||||
|        "<div>\n", | ||||
|        "<style scoped>\n", | ||||
|        "    .dataframe tbody tr th:only-of-type {\n", | ||||
|        "        vertical-align: middle;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe tbody tr th {\n", | ||||
|        "        vertical-align: top;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe thead th {\n", | ||||
|        "        text-align: right;\n", | ||||
|        "    }\n", | ||||
|        "</style>\n", | ||||
|        "<table border=\"1\" class=\"dataframe\">\n", | ||||
|        "  <thead>\n", | ||||
|        "    <tr style=\"text-align: right;\">\n", | ||||
|        "      <th></th>\n", | ||||
|        "      <th>Unnamed: 0</th>\n", | ||||
|        "      <th>code</th>\n", | ||||
|        "      <th>character</th>\n", | ||||
|        "      <th>description</th>\n", | ||||
|        "      <th>Unnamed: 4</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>0</th>\n", | ||||
|        "      <td>0</td>\n", | ||||
|        "      <td>126980</td>\n", | ||||
|        "      <td>🀄</td>\n", | ||||
|        "      <td>MAHJONG TILE RED DRAGON</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1</th>\n", | ||||
|        "      <td>1</td>\n", | ||||
|        "      <td>129525</td>\n", | ||||
|        "      <td>🧵</td>\n", | ||||
|        "      <td>SPOOL OF THREAD</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>2</th>\n", | ||||
|        "      <td>2</td>\n", | ||||
|        "      <td>129526</td>\n", | ||||
|        "      <td>🧶</td>\n", | ||||
|        "      <td>BALL OF YARN</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>3</th>\n", | ||||
|        "      <td>3</td>\n", | ||||
|        "      <td>127183</td>\n", | ||||
|        "      <td>🃏</td>\n", | ||||
|        "      <td>PLAYING CARD BLACK JOKER</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>4</th>\n", | ||||
|        "      <td>4</td>\n", | ||||
|        "      <td>129296</td>\n", | ||||
|        "      <td>🤐</td>\n", | ||||
|        "      <td>ZIPPER-MOUTH FACE</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "   Unnamed: 0    code character               description  Unnamed: 4\n", | ||||
|        "0           0  126980         🀄   MAHJONG TILE RED DRAGON         NaN\n", | ||||
|        "1           1  129525         🧵           SPOOL OF THREAD         NaN\n", | ||||
|        "2           2  129526         🧶              BALL OF YARN         NaN\n", | ||||
|        "3           3  127183         🃏  PLAYING CARD BLACK JOKER         NaN\n", | ||||
|        "4           4  129296         🤐         ZIPPER-MOUTH FACE         NaN" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 15, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "table.head()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 46, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "  \"\"\"\n", | ||||
|       "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "  # Remove the CWD from sys.path while we load stuff.\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/html": [ | ||||
|        "<div>\n", | ||||
|        "<style scoped>\n", | ||||
|        "    .dataframe tbody tr th:only-of-type {\n", | ||||
|        "        vertical-align: middle;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe tbody tr th {\n", | ||||
|        "        vertical-align: top;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe thead th {\n", | ||||
|        "        text-align: right;\n", | ||||
|        "    }\n", | ||||
|        "</style>\n", | ||||
|        "<table border=\"1\" class=\"dataframe\">\n", | ||||
|        "  <thead>\n", | ||||
|        "    <tr style=\"text-align: right;\">\n", | ||||
|        "      <th></th>\n", | ||||
|        "      <th>Unnamed: 0</th>\n", | ||||
|        "      <th>code</th>\n", | ||||
|        "      <th>character</th>\n", | ||||
|        "      <th>description</th>\n", | ||||
|        "      <th>Unnamed: 4</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>0</th>\n", | ||||
|        "      <td>0</td>\n", | ||||
|        "      <td>126980</td>\n", | ||||
|        "      <td>🀄</td>\n", | ||||
|        "      <td>mahjong tile red dragon</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1</th>\n", | ||||
|        "      <td>1</td>\n", | ||||
|        "      <td>129525</td>\n", | ||||
|        "      <td>🧵</td>\n", | ||||
|        "      <td>spool thread</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>2</th>\n", | ||||
|        "      <td>2</td>\n", | ||||
|        "      <td>129526</td>\n", | ||||
|        "      <td>🧶</td>\n", | ||||
|        "      <td>ball yarn</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>3</th>\n", | ||||
|        "      <td>3</td>\n", | ||||
|        "      <td>127183</td>\n", | ||||
|        "      <td>🃏</td>\n", | ||||
|        "      <td>playing card black joker</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>4</th>\n", | ||||
|        "      <td>4</td>\n", | ||||
|        "      <td>129296</td>\n", | ||||
|        "      <td>🤐</td>\n", | ||||
|        "      <td>zipper-mouth face</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "   Unnamed: 0    code character               description  Unnamed: 4\n", | ||||
|        "0           0  126980         🀄   mahjong tile red dragon         NaN\n", | ||||
|        "1           1  129525         🧵              spool thread         NaN\n", | ||||
|        "2           2  129526         🧶                 ball yarn         NaN\n", | ||||
|        "3           3  127183         🃏  playing card black joker         NaN\n", | ||||
|        "4           4  129296         🤐         zipper-mouth face         NaN" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 46, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "prepStep1Table = table\n", | ||||
|     "\n", | ||||
|     "# lowercasing and stopword removal\n", | ||||
|     "for entry in range(len(table[\"description\"][1:])):\n", | ||||
|     "    table[\"description\"][entry] = table[\"description\"][entry].lower()\n", | ||||
|     "    newEntry = []\n", | ||||
|     "    for word in table[\"description\"][entry].split(\" \"):\n", | ||||
|     "        if word not in set(stopwords.words('english')):\n", | ||||
|     "            newEntry.append(word)\n", | ||||
|     "    table[\"description\"][entry] = (\" \").join(newEntry).lower()\n", | ||||
|     "        \n", | ||||
|     "table.head()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.6.4" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 2 | ||||
| } | ||||
							
								
								
									
										1028
									
								
								Project/Tools/emoji_descriptions_preprocessed.csv
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1028
									
								
								Project/Tools/emoji_descriptions_preprocessed.csv
									
									
									
									
									
										Normal file
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							
		Reference in New Issue
	
	Block a user