diff --git a/Project/simple_approach/simple_twitter_learning.ipynb b/Project/simple_approach/simple_twitter_learning.ipynb index b32de54..5a25358 100644 --- a/Project/simple_approach/simple_twitter_learning.ipynb +++ b/Project/simple_approach/simple_twitter_learning.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -74,11 +74,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "data_root_folder = \"./data_en/\" # i created a symlink here" + "data_root_folder = \"../data/en/\" # i created a symlink here" ] }, { @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -113,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -1205,7 +1205,7 @@ "[68733 rows x 9 columns]" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -1224,7 +1224,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -1243,7 +1243,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -1261,7 +1261,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -1278,7 +1278,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -1298,7 +1298,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -1307,7 +1307,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -1316,7 +1316,7 @@ "68733" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -1327,7 +1327,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -1343,7 +1343,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -1354,7 +1354,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -1369,6 +1369,150 @@ "print(len(labels), len(emojis), len(plain_text))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* Apply stemming and lemmatization (if needed)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "from nltk.stem.snowball import SnowballStemmer\n", + "from nltk.stem import WordNetLemmatizer\n", + "from nltk import pos_tag\n", + "from nltk import word_tokenize\n", + "from nltk.corpus import wordnet" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "def get_wordnet_pos(treebank_tag):\n", + "\n", + " if treebank_tag.startswith('J'):\n", + " return wordnet.ADJ\n", + " elif treebank_tag.startswith('V'):\n", + " return wordnet.VERB\n", + " elif treebank_tag.startswith('N'):\n", + " return wordnet.NOUN\n", + " elif treebank_tag.startswith('R'):\n", + " return wordnet.ADV\n", + " else:\n", + " return wordnet.NOUN" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "stemmer = SnowballStemmer(\"english\")\n", + "for key in plain_text.keys():\n", + " stemmed_sent = []\n", + " for word in plain_text[key].split(\" \"):\n", + " word_stemmed = stemmer.stem(word)\n", + " stemmed_sent.append(word_stemmed)\n", + " stemmed_sent = (\" \").join(stemmed_sent)\n", + " plain_text[key] = stemmed_sent" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2 woooaaaahhh\n", + "4 i wan na know too\n", + "6 i 'm le stress about turn 30 now i think i'v r...\n", + "9 got me emot there\n", + "14 cutest son roll no . 31\n", + "15 by the summer i should have everyth up and run...\n", + "18 that pictur wa not taken this morning !\n", + "26 i so can not be bother with the rest of the da...\n", + "27 2lit4lif\n", + "35 hate fall asleep befor i put my phone on the c...\n", + "36 unexpect saw two of my crush today . this day ...\n", + "40 elvi whi o whi ? our girl wa such a love stori...\n", + "42 you'r late i ate them all\n", + "43 me toooo\n", + "47 the pressur is just too much\n", + "51 i broke grammar\n", + "52 have not desir to go to work today\n", + "53 omg do n't it scari all i know is that i do no...\n", + "56 achoo mr. fuck nigga you , you done caught cau...\n", + "58 i can never catch a dang break !\n", + "59 pa my p on two hour of sleep\n", + "60 i 'm realli not amus\n", + "65 i can help you\n", + "71 whew i slept good af last night\n", + "74 this would be epic . pizza and play perfect gi...\n", + "76 hey , it 1st novemb\n", + "80 u is to press bitch for me to have been speak ...\n", + "88 lmfao thought it wa just me be bitter\n", + "89 yupp yuppp . super prettttyyy , my heart cant ...\n", + "90 bakit halo halong seri binanggit mo be ? none ...\n", + " ... \n", + "68675 go back to dark hair tomorrow , mhmm yasss\n", + "68677 i miss them so much\n", + "68678 i wan na feel your gut too\n", + "68683 everi time\n", + "68687 i neither own nor watch tv . now go watch cnn\n", + "68688 revolutionari love\n", + "68694 ear worm is run in the famili after sing an aw...\n", + "68696 ill never look at you the same . yeah you got ...\n", + "68699 it our 3 year anniversari today to celebrate ,...\n", + "68700 person that scare me\n", + "68701 damn girl . can u look ani hotter than this ? ...\n", + "68703 this one fall under the weird crazi one .\n", + "68704 i 'm not allow to have chocol yet , then i uni...\n", + "68705 manchest unit manag mourinho slam specialists'...\n", + "68708 look , mammeh and daddeh ! cuuutee..\n", + "68709 life is so good with you\n", + "68710 happi halloween !\n", + "68712 scotti and kristen halloween costum\n", + "68713 may pre-month celebr si\n", + "68717 lmaooo i 'm so proud\n", + "68720 ambot ! ! !\n", + "68721 1st of the month ! ! happi 1st of novemb *53 d...\n", + "68722 ... stay in bed\n", + "68723 this is gold . gold .\n", + "68724 thank you for the kind compliment\n", + "68725 enjoy the silenc\n", + "68728 thank you yomi !\n", + "68729 lol . just enjoy the star . music kidhar aur b...\n", + "68730 thought and prayer for ny\n", + "68732 thank you so muchhav a happi wednesday and a g...\n", + "Name: text, Length: 33368, dtype: object\n" + ] + } + ], + "source": [ + "lemmatizer = WordNetLemmatizer()\n", + "for key in plain_text.keys():\n", + " lemmatized_sent = []\n", + " sent_pos = pos_tag(word_tokenize(plain_text[key]))\n", + " for word in sent_pos:\n", + " wordnet_pos = get_wordnet_pos(word[1].lower())\n", + " word_lemmatized = lemmatizer.lemmatize(word[0], pos=wordnet_pos)\n", + " lemmatized_sent.append(word_lemmatized)\n", + " lemmatized_sent = (\" \").join(lemmatized_sent)\n", + " plain_text[key] = lemmatized_sent\n", + "print(plain_text)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1378,7 +1522,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -1419,7 +1563,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -1428,7 +1572,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -1505,7 +1649,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -1533,7 +1677,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -1551,7 +1695,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -1569,13 +1713,15 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ + "C:\\Users\\Maren\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", + " from ._conv import register_converters as _register_converters\n", "Using TensorFlow backend.\n" ] } @@ -1591,7 +1737,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -1623,7 +1769,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1631,12 +1777,7 @@ "output_type": "stream", "text": [ "Train on 18861 samples, validate on 4716 samples\n", - "Epoch 1/3\n", - "18861/18861 [==============================] - 1288s 68ms/step - loss: 0.0185 - val_loss: 0.0164\n", - "Epoch 2/3\n", - "18861/18861 [==============================] - 1282s 68ms/step - loss: 0.0103 - val_loss: 0.0164\n", - "Epoch 3/3\n", - "18861/18861 [==============================] - 1333s 71ms/step - loss: 0.0063 - val_loss: 0.0166\n" + "Epoch 1/3\n" ] } ], @@ -2739,7 +2880,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.4" } }, "nbformat": 4,