preprocessing finished and edited file
This commit is contained in:
		| @ -2,26 +2,47 @@ | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "execution_count": 36, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "[nltk_data] Downloading package stopwords to\n", | ||||
|       "[nltk_data]     C:\\Users\\Maren\\AppData\\Roaming\\nltk_data...\n", | ||||
|       "[nltk_data]   Unzipping corpora\\stopwords.zip.\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "True" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 36, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import pandas as pd\n", | ||||
|     "from nltk.corpus import stopwords" | ||||
|     "import nltk\n", | ||||
|     "nltk.download('stopwords')" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "execution_count": 33, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "table = pd.read_csv(\"emoji_descriptions.csv\", names=(\"id\",\"char\", \"description\"))" | ||||
|     "table = pd.read_csv(\"emoji_descriptions.csv\")" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "execution_count": 15, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -45,56 +66,68 @@ | ||||
|        "  <thead>\n", | ||||
|        "    <tr style=\"text-align: right;\">\n", | ||||
|        "      <th></th>\n", | ||||
|        "      <th>id</th>\n", | ||||
|        "      <th>char</th>\n", | ||||
|        "      <th>Unnamed: 0</th>\n", | ||||
|        "      <th>code</th>\n", | ||||
|        "      <th>character</th>\n", | ||||
|        "      <th>description</th>\n", | ||||
|        "      <th>Unnamed: 4</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>NaN</th>\n", | ||||
|        "      <td>code</td>\n", | ||||
|        "      <td>character</td>\n", | ||||
|        "      <td>descripion,</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>0.0</th>\n", | ||||
|        "      <th>0</th>\n", | ||||
|        "      <td>0</td>\n", | ||||
|        "      <td>126980</td>\n", | ||||
|        "      <td>🀄</td>\n", | ||||
|        "      <td>MAHJONG TILE RED DRAGON</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1.0</th>\n", | ||||
|        "      <th>1</th>\n", | ||||
|        "      <td>1</td>\n", | ||||
|        "      <td>129525</td>\n", | ||||
|        "      <td>🧵</td>\n", | ||||
|        "      <td>SPOOL OF THREAD</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>2.0</th>\n", | ||||
|        "      <th>2</th>\n", | ||||
|        "      <td>2</td>\n", | ||||
|        "      <td>129526</td>\n", | ||||
|        "      <td>🧶</td>\n", | ||||
|        "      <td>BALL OF YARN</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>3.0</th>\n", | ||||
|        "      <th>3</th>\n", | ||||
|        "      <td>3</td>\n", | ||||
|        "      <td>127183</td>\n", | ||||
|        "      <td>🃏</td>\n", | ||||
|        "      <td>PLAYING CARD BLACK JOKER</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>4</th>\n", | ||||
|        "      <td>4</td>\n", | ||||
|        "      <td>129296</td>\n", | ||||
|        "      <td>🤐</td>\n", | ||||
|        "      <td>ZIPPER-MOUTH FACE</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "          id       char               description\n", | ||||
|        "NaN     code  character               descripion,\n", | ||||
|        " 0.0  126980          🀄   MAHJONG TILE RED DRAGON\n", | ||||
|        " 1.0  129525          🧵           SPOOL OF THREAD\n", | ||||
|        " 2.0  129526          🧶              BALL OF YARN\n", | ||||
|        " 3.0  127183          🃏  PLAYING CARD BLACK JOKER" | ||||
|        "   Unnamed: 0    code character               description  Unnamed: 4\n", | ||||
|        "0           0  126980         🀄   MAHJONG TILE RED DRAGON         NaN\n", | ||||
|        "1           1  129525         🧵           SPOOL OF THREAD         NaN\n", | ||||
|        "2           2  129526         🧶              BALL OF YARN         NaN\n", | ||||
|        "3           3  127183         🃏  PLAYING CARD BLACK JOKER         NaN\n", | ||||
|        "4           4  129296         🤐         ZIPPER-MOUTH FACE         NaN" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 4, | ||||
|      "execution_count": 15, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -105,18 +138,125 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "execution_count": 46, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "  \"\"\"\n", | ||||
|       "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n", | ||||
|       "A value is trying to be set on a copy of a slice from a DataFrame\n", | ||||
|       "\n", | ||||
|       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", | ||||
|       "  # Remove the CWD from sys.path while we load stuff.\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/html": [ | ||||
|        "<div>\n", | ||||
|        "<style scoped>\n", | ||||
|        "    .dataframe tbody tr th:only-of-type {\n", | ||||
|        "        vertical-align: middle;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe tbody tr th {\n", | ||||
|        "        vertical-align: top;\n", | ||||
|        "    }\n", | ||||
|        "\n", | ||||
|        "    .dataframe thead th {\n", | ||||
|        "        text-align: right;\n", | ||||
|        "    }\n", | ||||
|        "</style>\n", | ||||
|        "<table border=\"1\" class=\"dataframe\">\n", | ||||
|        "  <thead>\n", | ||||
|        "    <tr style=\"text-align: right;\">\n", | ||||
|        "      <th></th>\n", | ||||
|        "      <th>Unnamed: 0</th>\n", | ||||
|        "      <th>code</th>\n", | ||||
|        "      <th>character</th>\n", | ||||
|        "      <th>description</th>\n", | ||||
|        "      <th>Unnamed: 4</th>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </thead>\n", | ||||
|        "  <tbody>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>0</th>\n", | ||||
|        "      <td>0</td>\n", | ||||
|        "      <td>126980</td>\n", | ||||
|        "      <td>🀄</td>\n", | ||||
|        "      <td>mahjong tile red dragon</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>1</th>\n", | ||||
|        "      <td>1</td>\n", | ||||
|        "      <td>129525</td>\n", | ||||
|        "      <td>🧵</td>\n", | ||||
|        "      <td>spool thread</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>2</th>\n", | ||||
|        "      <td>2</td>\n", | ||||
|        "      <td>129526</td>\n", | ||||
|        "      <td>🧶</td>\n", | ||||
|        "      <td>ball yarn</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>3</th>\n", | ||||
|        "      <td>3</td>\n", | ||||
|        "      <td>127183</td>\n", | ||||
|        "      <td>🃏</td>\n", | ||||
|        "      <td>playing card black joker</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "    <tr>\n", | ||||
|        "      <th>4</th>\n", | ||||
|        "      <td>4</td>\n", | ||||
|        "      <td>129296</td>\n", | ||||
|        "      <td>🤐</td>\n", | ||||
|        "      <td>zipper-mouth face</td>\n", | ||||
|        "      <td>NaN</td>\n", | ||||
|        "    </tr>\n", | ||||
|        "  </tbody>\n", | ||||
|        "</table>\n", | ||||
|        "</div>" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "   Unnamed: 0    code character               description  Unnamed: 4\n", | ||||
|        "0           0  126980         🀄   mahjong tile red dragon         NaN\n", | ||||
|        "1           1  129525         🧵              spool thread         NaN\n", | ||||
|        "2           2  129526         🧶                 ball yarn         NaN\n", | ||||
|        "3           3  127183         🃏  playing card black joker         NaN\n", | ||||
|        "4           4  129296         🤐         zipper-mouth face         NaN" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 46, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "prepStep1Table = table\n", | ||||
|     "print(table[3])\n", | ||||
|     "# lowercasing\n", | ||||
|     "#for entry in table[\"description\"][1:]:\n", | ||||
|     "   # prepStep1Table[\"description\"][entry] = table[\"description\"][entry].toLower()\n", | ||||
|     "#print(prepStep1Table)\n", | ||||
|     "#stopword removal\n", | ||||
|     "#for entry in prepStep1Table[\"description\"][1:]:" | ||||
|     "\n", | ||||
|     "# lowercasing and stopword removal\n", | ||||
|     "for entry in range(len(table[\"description\"][1:])):\n", | ||||
|     "    table[\"description\"][entry] = table[\"description\"][entry].lower()\n", | ||||
|     "    newEntry = []\n", | ||||
|     "    for word in table[\"description\"][entry].split(\" \"):\n", | ||||
|     "        if word not in set(stopwords.words('english')):\n", | ||||
|     "            newEntry.append(word)\n", | ||||
|     "    table[\"description\"][entry] = (\" \").join(newEntry).lower()\n", | ||||
|     "        \n", | ||||
|     "table.head()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|  | ||||
							
								
								
									
										1028
									
								
								Project/Tools/emoji_descriptions_preprocessed.csv
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1028
									
								
								Project/Tools/emoji_descriptions_preprocessed.csv
									
									
									
									
									
										Normal file
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							
		Reference in New Issue
	
	Block a user