nlp-lab/Jonas_Solutions/Task_02_JonasWeinz.ipynb

1394 lines
113 KiB
Plaintext
Raw Normal View History

2018-05-09 10:47:51 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-05-13 10:12:19 +02:00
"# NLP-LAB Exercise 02 by Jonas Weinz (2571421)\n",
2018-05-09 16:50:59 +02:00
"## links:\n",
"\n",
"* Article: https://miguelmalvarez.com/2017/03/23/how-can-machine-learning-and-ai-help-solving-the-fake-news-problem/\n",
" * corresponding code: https://github.com/kjam/random_hackery/blob/master/Attempting%20to%20detect%20fake%20news.ipynb\n",
"\n",
"* Tutorial on Datacamp: https://www.datacamp.com/community/tutorials/scikit-learn-fake-news\n",
"\n",
"* liar dataset paper: https://www.cs.ucsb.edu/~william/papers/acl2017.pdf\n",
2018-05-09 19:13:08 +02:00
" * dataset: https://www.cs.ucsb.edu/~william/data/liar_dataset.zip"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-05-09 16:50:59 +02:00
"## Dependencies for this Notebook:\n",
2018-05-09 10:47:51 +02:00
"* library [rdflib](https://github.com/RDFLib/rdflib)\n",
2018-05-09 19:13:08 +02:00
" * install: `pip3 install rdflib`\n"
2018-05-09 10:47:51 +02:00
]
},
{
"cell_type": "code",
2018-05-09 19:13:08 +02:00
"execution_count": 1,
2018-05-09 10:47:51 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
2018-05-11 12:19:53 +02:00
"%pylab inline"
2018-05-09 16:50:59 +02:00
]
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 2,
2018-05-09 16:50:59 +02:00
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import itertools\n",
"import sklearn.utils as sku\n",
2018-05-09 19:13:08 +02:00
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.linear_model import PassiveAggressiveClassifier\n",
"from sklearn.naive_bayes import MultinomialNB\n",
"from sklearn import metrics\n",
"import matplotlib.pyplot as plt\n",
2018-05-09 16:50:59 +02:00
"import os"
]
},
2018-05-09 19:13:08 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tools used later"
]
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 3,
2018-05-09 19:13:08 +02:00
"metadata": {},
"outputs": [],
"source": [
"def plot_confusion_matrix(cm, classes,\n",
" title,\n",
" normalize=False,\n",
" cmap=plt.cm.Blues):\n",
" fig_1, ax_1 = plt.subplots()\n",
" \"\"\"\n",
" See full source and example: \n",
" http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
" \n",
" This function prints and plots the confusion matrix.\n",
" Normalization can be applied by setting `normalize=True`.\n",
" \"\"\"\n",
" plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
" plt.title('Confusion Matrix for: ' + title)\n",
" plt.colorbar()\n",
" tick_marks = np.arange(len(classes))\n",
" plt.xticks(tick_marks, classes, rotation=45)\n",
" plt.yticks(tick_marks, classes)\n",
"\n",
" if normalize:\n",
" cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
" print(\"Normalized confusion matrix\")\n",
" else:\n",
" print('Confusion matrix, without normalization')\n",
"\n",
" thresh = cm.max() / 2.\n",
" for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
" plt.text(j, i, cm[i, j],\n",
" horizontalalignment=\"center\",\n",
" color=\"white\" if cm[i, j] > thresh else \"black\")\n",
"\n",
" plt.tight_layout()\n",
" plt.ylabel('True label')\n",
" plt.xlabel('Predicted label')"
]
},
2018-05-13 10:12:19 +02:00
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"from pprint import pprint as pp\n",
"from IPython.display import display, Markdown, Latex\n",
"import collections\n",
"import traceback\n",
"\n",
"def jupyter_print(obj, cell_w = 10, headers=None, p_type=True, ret_mdown=False, index_offset=0, list_horizontal=False):\n",
" try:\n",
" ts = \"**Type:** \" + str(type(obj)).strip(\"<>\") + \"\\n\\n\"\n",
" if type(obj) == str:\n",
" display(Markdown(obj))\n",
" elif isinstance(obj, collections.Iterable):\n",
" if isinstance(obj[0], collections.Iterable) and type(obj[0]) is not str:\n",
" # we have a table\n",
" \n",
" if headers is None:\n",
" headers = [str(i) for i in range(len(obj[0]))]\n",
" \n",
" if len(headers) < len(obj[0]):\n",
" headers += [\" \" for i in range(len(obj[0]) - len(headers))]\n",
" \n",
" s = \"|\" + \" \" * cell_w + \"|\"\n",
" \n",
" for h in headers:\n",
" s += str(h) + \" \" * (cell_w - len(h)) + \"|\"\n",
" s += \"\\n|\" + \"-\" * (len(headers) + (len(headers) + 1) * cell_w) + \"|\\n\"\n",
" \n",
" #s = (\"|\" + (\" \" * (cell_w))) * len(obj[0]) + \"|\\n\" + \"|\" + (\"-\" * (cell_w + 1)) * len(obj[0])\n",
" #s += '|\\n'\n",
" \n",
" row = index_offset\n",
" \n",
" for o in obj:\n",
" s += \"|**\" + str(row) + \"**\" + \" \" * (cell_w - (len(str(row))+4))\n",
" for i in o:\n",
" s += \"|\" + str(i) + \" \" * (cell_w - len(str(i)))\n",
" s+=\"|\" + '\\n'\n",
" s += ts\n",
" display(Markdown(s))\n",
" return s if ret_mdown else None\n",
" else:\n",
" # we have a list\n",
" \n",
" \n",
" if headers is None:\n",
" headers = [\"index\",\"value\"]\n",
" \n",
" index_title = headers[0]\n",
" value_title = headers[1]\n",
" \n",
" s = \"|\" + index_title + \" \" * (cell_w - len(value_title)) + \"|\" + value_title + \" \" * (cell_w - len(value_title)) + \"|\" + '\\n'\n",
" s += \"|\" + \"-\" * (1 + 2 * cell_w) + '|\\n'\n",
" i = index_offset\n",
" for o in obj:\n",
" s_i = str(i)\n",
" s_o = str(o)\n",
" s += \"|\" + s_i + \" \" * (cell_w - len(s_i)) + \"|\" + s_o + \" \" * (cell_w - len(s_o)) + \"|\" + '\\n'\n",
" i+=1\n",
" s += ts\n",
" #print(s)\n",
" display(Markdown(s))\n",
" return s if ret_mdown else None\n",
" else:\n",
" jupyter_print([obj])\n",
" except Exception as e:\n",
" print(ts)\n",
" pp(obj) \n",
"\n",
"jp = jupyter_print\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"| |0 |1 |2 |\n",
"|-------------------------------------------|\n",
"|**0** |1 |2000 |3 |\n",
"|**0** |4 |5 |6 |\n",
"**Type:** class 'list'\n",
"\n"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"jp([[1,2000,3],[4,5,6]])"
]
},
2018-05-09 16:50:59 +02:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Generate/Download Datasets we are working on\n",
"\n",
"* running bash script to download all needed data and store it into the `data` subfolder"
]
},
{
"cell_type": "code",
2018-05-13 10:12:19 +02:00
"execution_count": 6,
2018-05-09 16:50:59 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"================================================================================\n",
2018-05-11 12:19:53 +02:00
"checking whether unzip is installed\n",
2018-05-09 16:50:59 +02:00
"================================================================================\n",
"UnZip 6.00 of 20 April 2009, by Debian. Original by Info-ZIP.\n",
"\n",
"Latest sources and executables are at ftp://ftp.info-zip.org/pub/infozip/ ;\n",
"see ftp://ftp.info-zip.org/pub/infozip/UnZip.html for other sites.\n",
"\n",
"Compiled with gcc 6.3.0 20170415 for Unix (Linux ELF).\n",
"\n",
"UnZip special compilation options:\n",
" ACORN_FTYPE_NFS\n",
" COPYRIGHT_CLEAN (PKZIP 0.9x unreducing method not supported)\n",
" SET_DIR_ATTRIB\n",
" SYMLINKS (symbolic links supported, if RTL and file system permit)\n",
" TIMESTAMP\n",
" UNIXBACKUP\n",
" USE_EF_UT_TIME\n",
" USE_UNSHRINK (PKZIP/Zip 1.x unshrinking method supported)\n",
" USE_DEFLATE64 (PKZIP 4.x Deflate64(tm) supported)\n",
" UNICODE_SUPPORT [wide-chars, char coding: UTF-8] (handle UTF-8 paths)\n",
" LARGE_FILE_SUPPORT (large files over 2 GiB supported)\n",
" ZIP64_SUPPORT (archives using Zip64 for large files supported)\n",
" USE_BZIP2 (PKZIP 4.6+, using bzip2 lib version 1.0.6, 6-Sept-2010)\n",
" VMS_TEXT_CONV\n",
" WILD_STOP_AT_DIR\n",
" [decryption, version 2.11 of 05 Jan 2007]\n",
"\n",
"UnZip and ZipInfo environment options:\n",
" UNZIP: [none]\n",
" UNZIPOPT: [none]\n",
" ZIPINFO: [none]\n",
" ZIPINFOOPT: [none]\n",
"================================================================================\n",
2018-05-11 12:19:53 +02:00
"successfully finished action: checking whether unzip is installed\n",
"================================================================================\n",
"================================================================================\n",
"downloading and unpacking https://www.cs.ucsb.edu/~william/data/liar_dataset.zip if not already existing\n",
"================================================================================\n",
"================================================================================\n",
2018-05-09 16:50:59 +02:00
"successfully finished action: downloading and unpacking https://www.cs.ucsb.edu/~william/data/liar_dataset.zip if not already existing\n",
"================================================================================\n",
"================================================================================\n",
"downloading and unpacking https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip if not already existing\n",
"================================================================================\n",
"================================================================================\n",
"successfully finished action: downloading and unpacking https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip if not already existing\n",
2018-05-13 10:12:19 +02:00
"================================================================================\n",
"================================================================================\n",
"downloading Helper script: script_dataset3.py\n",
"================================================================================\n",
"================================================================================\n",
"successfully finished action: downloading Helper script: script_dataset3.py\n",
2018-05-09 16:50:59 +02:00
"================================================================================\n"
]
}
],
"source": [
"%%bash\n",
"./Task_2_gen_data.sh"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-05-13 10:12:19 +02:00
"----\n",
"## configuration 1"
2018-05-09 16:50:59 +02:00
]
},
{
"cell_type": "code",
2018-05-13 10:12:19 +02:00
"execution_count": 7,
2018-05-09 16:50:59 +02:00
"metadata": {},
"outputs": [],
"source": [
"df_1 = pd.read_csv('data/fake_or_real_news.csv').set_index('Unnamed: 0')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* display first 10 entries"
]
},
{
"cell_type": "code",
2018-05-13 10:12:19 +02:00
"execution_count": 8,
2018-05-09 16:50:59 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(6335, 3)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>title</th>\n",
" <th>text</th>\n",
" <th>label</th>\n",
" </tr>\n",
" <tr>\n",
" <th>Unnamed: 0</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>8476</th>\n",
" <td>You Can Smell Hillarys Fear</td>\n",
" <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
" <td>FAKE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10294</th>\n",
" <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
" <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
" <td>FAKE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3608</th>\n",
" <td>Kerry to go to Paris in gesture of sympathy</td>\n",
" <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
" <td>REAL</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10142</th>\n",
" <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
" <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
" <td>FAKE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>875</th>\n",
" <td>The Battle of New York: Why This Primary Matters</td>\n",
" <td>It's primary day in New York and front-runners...</td>\n",
" <td>REAL</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6903</th>\n",
" <td>Tehran, USA</td>\n",
" <td>\\nIm not an immigrant, but my grandparents ...</td>\n",
" <td>FAKE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7341</th>\n",
" <td>Girl Horrified At What She Watches Boyfriend D...</td>\n",
" <td>Share This Baylee Luciani (left), Screenshot o...</td>\n",
" <td>FAKE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>Britains Schindler Dies at 106</td>\n",
" <td>A Czech stockbroker who saved more than 650 Je...</td>\n",
" <td>REAL</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4869</th>\n",
" <td>Fact check: Trump and Clinton at the 'commande...</td>\n",
" <td>Hillary Clinton and Donald Trump made some ina...</td>\n",
" <td>REAL</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2909</th>\n",
" <td>Iran reportedly makes new push for uranium con...</td>\n",
" <td>Iranian negotiators reportedly have made a las...</td>\n",
" <td>REAL</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" title \\\n",
"Unnamed: 0 \n",
"8476 You Can Smell Hillarys Fear \n",
"10294 Watch The Exact Moment Paul Ryan Committed Pol... \n",
"3608 Kerry to go to Paris in gesture of sympathy \n",
"10142 Bernie supporters on Twitter erupt in anger ag... \n",
"875 The Battle of New York: Why This Primary Matters \n",
"6903 Tehran, USA \n",
"7341 Girl Horrified At What She Watches Boyfriend D... \n",
"95 Britains Schindler Dies at 106 \n",
"4869 Fact check: Trump and Clinton at the 'commande... \n",
"2909 Iran reportedly makes new push for uranium con... \n",
"\n",
" text label \n",
"Unnamed: 0 \n",
"8476 Daniel Greenfield, a Shillman Journalism Fello... FAKE \n",
"10294 Google Pinterest Digg Linkedin Reddit Stumbleu... FAKE \n",
"3608 U.S. Secretary of State John F. Kerry said Mon... REAL \n",
"10142 — Kaydee King (@KaydeeKing) November 9, 2016 T... FAKE \n",
"875 It's primary day in New York and front-runners... REAL \n",
"6903 \\nIm not an immigrant, but my grandparents ... FAKE \n",
"7341 Share This Baylee Luciani (left), Screenshot o... FAKE \n",
"95 A Czech stockbroker who saved more than 650 Je... REAL \n",
"4869 Hillary Clinton and Donald Trump made some ina... REAL \n",
"2909 Iranian negotiators reportedly have made a las... REAL "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display(df_1.shape)\n",
"display(df_1[:10])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create test dataset"
2018-05-09 10:47:51 +02:00
]
},
{
"cell_type": "code",
2018-05-13 10:12:19 +02:00
"execution_count": 9,
2018-05-09 10:47:51 +02:00
"metadata": {},
"outputs": [],
2018-05-09 16:50:59 +02:00
"source": [
2018-05-09 19:13:08 +02:00
"def create_training_and_test_set(dset, cutoff=0.7):\n",
" shuffled = sku.shuffle(dset) # shuffle dataset\n",
2018-05-09 16:50:59 +02:00
" y = shuffled.label\n",
2018-05-09 19:13:08 +02:00
" shuffled = shuffled.drop('label', axis=1)['text']\n",
" size = int(cutoff * shuffled.shape[0])\n",
" return shuffled[:size], y[:size], shuffled[size:], y[size:]\n",
2018-05-09 16:50:59 +02:00
" "
]
},
{
"cell_type": "code",
2018-05-13 10:12:19 +02:00
"execution_count": 10,
2018-05-09 16:50:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-05-13 10:35:15 +02:00
"X1,y1, Xt1,yt1 = create_training_and_test_set(df_1)"
2018-05-09 19:13:08 +02:00
]
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 11,
2018-05-09 19:13:08 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-05-13 10:12:19 +02:00
"count_vectorizer_1 = CountVectorizer(stop_words='english')\n",
2018-05-13 10:35:15 +02:00
"count_train_1 = count_vectorizer_1.fit_transform(X1)\n",
"count_test_1 = count_vectorizer_1.transform(Xt1)"
2018-05-09 19:13:08 +02:00
]
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 12,
2018-05-09 19:13:08 +02:00
"metadata": {},
"outputs": [],
"source": [
2018-05-13 10:12:19 +02:00
"tfidf_vectorizer_1 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
2018-05-13 10:35:15 +02:00
"tfidf_train_1 = tfidf_vectorizer_1.fit_transform(X1)\n",
"tfidf_test_1 = tfidf_vectorizer_1.transform(Xt1)"
2018-05-09 16:50:59 +02:00
]
2018-05-09 19:13:08 +02:00
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 13,
2018-05-09 19:13:08 +02:00
"metadata": {},
2018-05-13 10:12:19 +02:00
"outputs": [],
2018-05-09 19:13:08 +02:00
"source": [
2018-05-13 10:12:19 +02:00
"#jupyter_print(count_vectorizer.get_feature_names()[0:10])\n",
"#jupyter_print(count_vectorizer.get_feature_names()[-10:])\n"
2018-05-09 19:13:08 +02:00
]
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 14,
2018-05-09 19:13:08 +02:00
"metadata": {},
2018-05-13 10:12:19 +02:00
"outputs": [],
2018-05-09 19:13:08 +02:00
"source": [
2018-05-13 10:12:19 +02:00
"#jupyter_print(tfidf_vectorizer.get_feature_names()[:10])\n",
"#jupyter_print(tfidf_vectorizer.get_feature_names()[-10:])"
2018-05-09 19:13:08 +02:00
]
},
{
"cell_type": "code",
2018-05-11 12:19:53 +02:00
"execution_count": 15,
2018-05-09 19:13:08 +02:00
"metadata": {},
"outputs": [],
"source": [
"#count_df = pd.DataFrame(count_train.A, columns=count_vectorizer.get_feature_names())\n",
"#tfidf_df = pd.DataFrame(count_train.A, columns=tfidf_vectorizer.get_feature_names())\n",
"#diff = set(count_df.columns) - set(tfidf_df.columns)\n",
"#pp(count_df.equals(tfidf_df))"
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 17,
2018-05-09 19:13:08 +02:00
"metadata": {},
"outputs": [
2018-05-13 10:12:19 +02:00
{
"data": {
"text/markdown": [
2018-05-13 10:35:15 +02:00
"score: 0.849026827985271"
2018-05-13 10:12:19 +02:00
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
2018-05-09 19:13:08 +02:00
{
"name": "stdout",
"output_type": "stream",
"text": [
2018-05-11 12:19:53 +02:00
"Confusion matrix, without normalization\n"
2018-05-09 19:13:08 +02:00
]
},
{
"data": {
2018-05-13 10:35:15 +02:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEmCAYAAABf+4ZQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XecHVX5x/HPNw0C6YRmCIRepfcaadIJIF1qMHRULD8QCyIgggoiIqJIVYqA9CKKoCAgJaH3moSQQgmpkPL8/jhnk5t1szuT7O7dvft95zWv7JR75rkzc+8z55y5M4oIzMzMmlunagdgZma1yQnGzMxahBOMmZm1CCcYMzNrEU4wZmbWIpxgzMysRVQ1wUjqLulOSRMl/WUhyjlU0t+aM7ZqkHSvpCMW8LVnS5og6YPmjstaxsLs72qTNFjSqEbmLy9psqTOrRxX4fU29R6aIZbLJP2gpcpvFyKiyQE4BHgKmAyMAe4Fti7y2ibKPQz4L9BlYctqiQEYDATw13rT18vTHypYzpnAdS0Y5/LANGCpFip7csUQwJSK8W2Aq4DP6y13YH79O8CO+e8jgVkVy7wNXAmsVrG+QXkdlWU920h8m+d4ejQwbzhwUgtt84eAY6p9jC5A3HXbd3i96f3zPnynYDmDgVEV43P2c3sZ6r+HBuYH8DzQqWLa2cBVVY77nfx5nwx8DNwNDKz29mxoaLIGI+lU4CLgXGDp/IVzKbB3U68tYAXgtYiY2QxltZTxwBaSlqiYdgTwWnOtQMnC1CaXBz6MiHELsO4ujc2PiPciokfdkCevVzHt33na+ZXLRcSN8ynysVxOb2BH0gflaUnr1FuuT0VZ6zUS3+PAKOAr9d7XOsBawPWNvb9qaIb93RzlLlZvmx9CSvg2ry8AB1U7iAbsmT9HywJjgV9XOZ6GNZEpe5Oy5P6NLLMIKQG9n4eLgEUqzxCAbwHjSLWfo/K8H5POmGbkdQyl3pk+c8+2uuTxI4G3gEmkD8OhFdMfqXjdlsCTwMT8/5YV8x4CfgI8msv5G9C/sTMc4DLgxDytMzAa+CEVNRjgV8BI4FPgaWCbPH2Xeu/z2Yo4zslxTANWoeKsGPgtcEtF+T8D/gGoXox1X9Kzc/lX5el7AS8Cn+Ry16x3BvR/wHPAZ5SoQeb9sUq9aVcBZzdytlVZg3mkgWXuAm5uaJ8XjOl7wIP1pp1PRc2TVNP5T94ezwKDK+b1I9Wk3iedEd5WMW9vYETer2/m/XkOqSY2PW/zSwoed43t72f535ri4AKx/0+5TWyruu37feCCiulPAWdQUYOpv68r9zMVZ//Atfn4qzur/m79/UgTnzuaPl6/QzpepwBXkE52781l/R3oO5/vjKOAl/NybwHH1v98N3Gs/x/wekV589RggL8AH+R9/i9g7flsr5eBPSrmdSGdvG7Y1D5u7DOVx3cjnajXje9Oqr1/SvpOOrNi3t3AyfXKew7YJ/+9BvAA8BHwKnBAvfW8lLflaODbTX42mzgYdwFm0siHHTgLeBxYClgyb6SfVOzAmXmZrjnAqRUHw5nMm1Dqj885WIDF8wZbPc9btm5nUvHFRfqy+JjU/NYFODiPL1FxoL8JrAZ0z+Pnzee9DSYlmC2BJyo28v3AMcybYL4KLJHX+a180C3a0PuqiOM9YO38mq7M+4WzGKmWdCSpGWoCsFxjcVaMr0b6IO6Uy/0u8AbQreIAHQEMBLrnaZcClzZ5wLRMgjkaGNvQF0SRIb+PmeRmAlLf4ihgSB4fAHyY912nvF0+BJas+NDdCPTN22u7PH1T0hfHTvl1A4A1KvbfMRUxFDnu5ru/672fYcArQK8Csf9PuU1sq7rtO4j05dOZVNN7hXSyUjrBzOdLb579SCOfO4odr4+TksoA0snqM8AGwKLAg8CP5rPe3YGVAQHbkb5/NmzoPcznWF+VdMJY97msn2COBnoy90R7xHy21w+BP1XM2x14ucjx2cRnajHgauCaet8HX8xlrUuq4dR9Fg4gf5fl8fXyurqRvmNHkpJyl7x9JwBr5WXHMPfEuW/ddmxsaKo6vQQwIRpvwjoUOCsixkXEeFLN5LCK+TPy/BkRcQ/pDGf1JtY7P7OBdSR1j4gxEfFiA8vsDrweEddGxMyIuJ704dmzYpkrI+K1iJgG3ASs39hKI+I/QD9JqwOHA9c0sMx1EfFhXucvSAdcU+/zqoh4Mb9mRr3yppK24y+B60hnHUU7JA8E7o6IB3K5Pyd9qLesWObiiBiZtwERcUJEnFCw/IZ8W9IneZhQ8rXvk76gK02oKO/bjb04IkaSvrDqjrsdSNv/7jz+VeCeiLgnImZHxAOkM/bdJC0L7AocFxEf5+P04fy6ocAf83acHRGjI+KV+YRR5Lib7/6uI2lr0pfYXhHxaWOxlym3AaNIZ6g7ko7pawu+bmHM73NX5Hj9dUSMjYjRwL9JX5LDI2I68FfSl+H/iIi7I+LNSB4m1Zy2KRFzAD8AfiCpWwPl/zEiJkXEZ6QTyfUk9W6gnD8De0laLI8fwtzm2yL7uL7bJH3C3BOgCypieigins9lPZfXs12efQewmqRV8/hhwI0R8TmwB+kE48p8LA0HbgH2z8vOANaS1Ct/Vp5pJD6g6avIPgT6N9FO/wXg3Yrxd/O0OWXUS1BTgR6UFBFTSAficcAYSXdLWqNAPHUxDagYr7zSqmg81wInAV8iHdDzkPRtSS/nK+I+ITUv9m+izJGNzYyIJ0jVepE+kEXNsw0iYnZeV+U2aHTdC+DnEdEnD0297/oGkKrklfpXlPfzAmVczdwEcxhwQ8WX7QrA/hUJ6xNga1IteCDwUUR83ECZA0ln3UUUOe4a3eaSBpL28xERUdfH11jshcptxDWkWuXBtE6Cmd/nrsjxOrbi72kNjDf4GZa0q6THJX2Ut91uNP25nEc+MR4FHFuv7M6SzpP0pqRPSTULGio/It4gNZPtmZPMXqSkA8X2cX1DIqIPqQZ3EvCwpGVyXJtJ+qek8ZImkr4z++c4ppNq61/N/XWV+34FYLN6cRwKLJPn70fafu9KeljSFk1tu6YSzGOkNvohjSzzfg6szvJ52oKYQqry1VmmcmZE3B8RO5E2/CvA7wvEUxfT6AWMqc61wAmkM42plTMkbUOq1h9Aav7rQzqzUF3o8ylzftPryj2RdCb+fi6/qHm2gSSRviwrt0Gj625l+5DOShfGrcBykr4E7EtKOHVGAtdWJKw+EbF4RJyX5/WT1KeBMkeSmlcaUn/7FTnu5rvNJXUHbgMuioh7C8beZLlNuIVU83orIt5rYP5UGvk81rMwx1OR47U0SYuQ3uPPgaXz5/Ie5n4uyziD1NdXuT0OIfXR7Ug6oRxUt+r5lHE96Qt9b+ClnHSg2D5uUETMiohbSX2CW+fJfybVVAZGRG9SH3JlTFeTEscOwNSIeKwijofrxdEjIo7P63oyIvYmdYfcRoGT3kYTTERMJLUd/kbSEEmLSeqazwrOz4tdD3xf0pKS+uflr2tqxfMxAtg2X8veGzi9boakpSXtLWlxUtKbTGoyq+8eUhXwEEldJB1IamO+awFjAiAi3iZVM89oYHZPUh/AeKCLpB+S2s/rjAUGlblySNJqpKaSr5LOyL8rqdGmvAo3AbtL2kFSV1Kf0Gek/rE2IZ/9rSjp16Q24x8vTHm5hnszqbP+3Yh4qmL2daQzxy/n9S6afwOxXETUXXZ/qaS++fjeNr/uCuCovB07SRpQUWseC6xUsY6FPe7+CLwSEefXmz7f2OdXkKQzJT3U1ArzNtue1J/YkBHAIXm9uzC3maUh9bdHGS11vHYjnaCNB2ZK2hXYeUEKioiHgBdIV5DW6UmK80NS4jm3iWJuyOs/nrm1F1iAfVxHyd6kPpGXK+L6KCKmS9qUlAgr38tjpO/OXzBvzfUu0jF8WP4cdJW0iaQ1JXVT+r1h79wy8CkNf//Oo8kvvEj9CaeSrjoZT8pyJ5EyGKQvwadIVyI8T+p8O7upcuezrgdI1bfnSB1rlR/OTjmO90nNKduRdlT9Mj4ktSV+i7T
2018-05-09 19:13:08 +02:00
"text/plain": [
2018-05-13 10:35:15 +02:00
"<matplotlib.figure.Figure at 0x7f4c9f545978>"
2018-05-09 19:13:08 +02:00
]
},
"metadata": {},
"output_type": "display_data"
2018-05-11 12:19:53 +02:00
}
],
"source": [
"clf = MultinomialNB()\n",
2018-05-13 10:35:15 +02:00
"clf.fit(tfidf_train_1, y1)\n",
2018-05-13 10:12:19 +02:00
"pred = clf.predict(tfidf_test_1)\n",
2018-05-13 10:35:15 +02:00
"score = metrics.accuracy_score(yt1, pred)\n",
2018-05-13 10:12:19 +02:00
"jupyter_print(\"score: \" + str(score))\n",
2018-05-13 10:35:15 +02:00
"cm = metrics.confusion_matrix(yt1, pred, labels=[\"FAKE\", \"REAL\"])\n",
2018-05-11 12:19:53 +02:00
"plot_confusion_matrix(cm, classes=[\"FAKE\", \"REAL\"], title= \"TFIDF_Vecctorizer, Multinomial Naive Bayes\")"
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 18,
2018-05-11 12:19:53 +02:00
"metadata": {},
"outputs": [
2018-05-09 19:13:08 +02:00
{
"name": "stdout",
"output_type": "stream",
"text": [
2018-05-13 10:35:15 +02:00
"'score: 0.9021567596002105'\n",
2018-05-09 19:13:08 +02:00
"Confusion matrix, without normalization\n"
]
2018-05-11 12:19:53 +02:00
},
{
"data": {
2018-05-13 10:35:15 +02:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEmCAYAAACj7q2aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xe8HFX9//HX+yYkhARIwwBJICBVUAKELgoEkCpFaSIBBAJIUeGrYvnZC1aKqBQpAaV3qSJFAWkBQpMWahICKUAghJbk8/vjnA2b693ZvTd7s7e8n3nMIzNnZmfO7Mydz5yyM4oIzMzM2qKp0RkwM7POy0HEzMzazEHEzMzazEHEzMzazEHEzMzazEHEzMzarKFBRFIfSX+XNEvSZYuwnv0l/aOeeWsESTdKOrCNn/2ZpBmSXq13vqztOvu5KelFSdsWzG/zObsoWrPdavuwiPnYUtLT7bHuTiMiqg7Al4DxwGxgKnAj8OlaPltlvQcA9wM9F3Vd7TEAWwEBXNUsfb2cfkeN6/kR8Nd2zOdKwLvAx9pxGwKOBR4H3gEmA5cBn2znYzAif9ctniPAvsCLgJql9wSmAbsswrYPAu5anOdcHb+3O/L3tl6z9Kty+lY1rudFYNs83q7ncTt+Fwv2oYV55+XvY+OytNXSpbGhef4R8GG+5s4GngS+0OjvsqWhaklE0nHAycAvgCH5gvUnYLdqn63BysAzETG3DutqL9OBzSQNKks7EHimXhtQsiilwpWAmRExrQ3b7lnjoqcAXyMFkoHAGsDVwM6t3WadXQ30Bz7bLH0H0sXhpsWeo6wV3217rfcZYEzZ5wYBm5HOafvI68DPGp2JFlwSEf0ioh/wdeCvkoY0OlP/o0o0XJYUBfcqWKY3Kci8koeTgd553lakO9bjSXeFU4GD87wfAx/wUbQ9hGZ3OjS7CyXdGT4PvA28AOxfln5X2ec2Bx4AZuX/Ny+bdwfwU+DuvJ5/AIMr7Fsp/6cDR+W0HsAU4AeUlURIF9lJwFvAg8CWOX2HZvv5SFk+fp7z8S7p7ucO4NA8/8/AFWXr/xVwK/97x71t/vz8vP7zcvrngSeAN/N61252Z/Zt4FHgfaqUBIHVgXmU3a1VOFfOJ12gXgK+DzSV3VUVHdeKxwR4OS9buiPbrIVtnwmc0yztUuCksuldgAn5+/gP8KmyecOBK3PeZwKnAWsD7+X9ng28WcN+HpT34aS8np9Rdm4C3yrbj9n5nDivbL1nk/5GpuTP9qi03hruZO8gnaOTy9ZzNOm8mkwuiZDuxH9W9rmtgMnNzpVtKT6PDy3L513Ab4E3SH+jO5ata0XgWtJFeyJwWNm8H5FKtn/N58BjpBuV75CuHZOA7ZvtX2m7Hwduy9/NDOBvQP/m+1DhezoP+D3wKvDZnLZQSQQ4mFQSeJt0/Tm8pe+L9Dd1ebP1nwKcWu0Yt5CvH9Gs1Je/h83z+ADgOtJ5+EYeH5bn7QU82OyzxwHXlF2zf0v623qNdH3rk+cNzut6Mx+nO8nnd8VzrcqJuAMwl4KLDPAT4F7gY8BypD/Qn5Z9wXPzMksAOwFzgAEVLi7Np0eQLzZAX9IFes08bwVgnfKTN48PzF/qAflz++XpQWUn33OkE7RPnj6xwr5tRfqD2xy4L6ftBNwMHMrCQeTLwKC8zeNJJ+WSBSfEHfkgrpM/swQL/2EsRbqTPAjYkvTHMawon2XTa5CqnLbL6/0W6Y+2V9kf1QTSxbN08vwJ+FOF9R8BvFTlXDkfuAZYOh+3Z4BDqh3Xasek+bIVtr1FPjdK+7IsKbCOzNPrk/4ANyHdBByYv4PeefoR0gW6L7AkuaqWFqqzquznQaTz/Zh8TPu0tI687HDSTdeOefoq4Iych4+RqnkPr7TeomNR9p0eSgrIpW3cTyqJtDqIVDmPy4PIh8Bh+Xs9Mu+j8vx/k86zJYGRpAvgNmXrfg/4XN7H80lB6Hukc/gw4IUK212NdK73Jl2D/g2c3NI+tPA9nUe6mB/LR9eQ5kFkZ1KgEqnEOwfYoPn3RapZmQMsnad7kALGptWOcQv5WvBd5+3uTLqw989pg4AvkK4TS5MC8NV5Xm9SACi/cXyYXB1GOtevJV0rlwb+Dvwyz/slKagskYctaXbj2nyoVoUyCJgRxdVN+wM/iYhpETGdVMI4oGz+h3n+hxFxA+kuZs0q261kPrCupD4RMTUinmhhmZ2BZyPigoiYGxEXAU8Bu5Ytc25EPBMR75LuWEcWbTQi/gMMlLQmqXrg/BaW+WtEzMzb/B3pQFbbz/Mi4on8mQ+brW8O6Xv8Penu7JiImFxlfSX7ANdHxC15vb8lXdA2L1vm1IiYlL8DIuKrEfHVCusbRPpjaJGkHqS2ie9ExNsR8SLwOxY+D6pp1TEpFxF3k+6o9shJe5OqSSfk6bHAGRFxX0TMi4hxpBLYpsDGpDvkb0bEOxHxXkTctQj7+UpE/CEf03crrKcPqRrulIi4MVdR7AR8PedhGukPfd/WrLeC84ExktYiXYDuacVn2+KliDgrIuYB40g3e0MkDScF+2/n73gC8BfKqtuAOyPi5ny9uYwUEE7M5/DFwAhJ/ZtvMCIm5nP9/XwN+j3/W71ZzRnASpJ2bGH910fEc5H8ixSYt2xhuZeAh/joPNwGmBMR99Z4jJvbW9KbpGvmtcAvIuLNvK2ZEXFFRMyJiLdJtRqfzfPeBy4h3dgiaR3SDc91kkT6e/hGRLyeP/uLsnx8SDpmK+dr9p2Ro0sl1YLITGBwlTrYFUnF+pKXctqCdTQLQnOAflW2+z8i4h3SxfEIYKqk6/MfRrX8lPI0tGy6vAdTrfm5gFQdsDXpjmIhkv5P0pO5p9mbpLvhwVXWOaloZkTcRyo+i3RhrdVC30FEzM/bKv8OCrfdzEzSiVXJYNJdS/PzYGjLi7eoLcek3Pl8dEE6gIUD/crA8ZLeLA2kksCK+f+XqtwoldSyn7V8r2cDT0fEr8rytwTpvC7l7wzS3Wpr1tuSK0kXs6NJ53B7W3Ac840QpGO5IlC6aJU0/+5eKxt/l3QDO69surSuhUgaIuliSVMkvUW66ar2t7eQfOH9aR6ar39HSfdKej0fm50K1n8hqfYDUoekC/N4Lce4uUsjon9E9CWVhMZIOjznaSlJZ0h6Ke/zv4H++UYHUgD/Ug4aB+R1vU8KzEsBD5bl46acDvAbUq3FPyQ9L+mEgvwB1YPIPaQ7tt0LlnmF9AWVrJTT2uId0g6WLF8+M9+lbEe6oD0FnFVDfkp5mtLGPJVcAHwVuKHsjwNI3fxIVUZ7k6rq+pPaY1TKeoV1FkZ4SUeRSjSv5PXXaqHvIJ9Iw1n4OyjcdjO3AsMkjaowfwbpDqb5eVDaXuFxraLWfF4AjJa0GamE8beyeZOAn+c/yNKwVC6lTiLdgbZ0o9R829X2s2p+8x/lGqQ2wPL8vU9qByrlb5mIWKfW9VaSz9UbSVVLLQWR1hybNuUhe4VUml+6LK0ef5eQ7qSD1FNwGdIduIo/0qJzSZ009iwlSOoNXEEqzQ/Jf9s3FKz/MmArScNIJZJSEKnlGFeUS7038lGNyvGkmo5N8j5/ppTlvPy9pDasLUnBrHTsZ5AC8jpl+Vg2UuM9uYR9fESsSmpXPU7S6KK8FQaRiJhFapz7o6Tdc/RbIkfmX+fFLgK+L2k5SYPz8n+t4XtpyQTgM5JWkrQsqVENWHC3sZukvqSDMZtUvdXcDcAakr4kqaekfYBPkBqL2iwiXiAVF7/XwuylSXXW04Gekn4ALFM2/zVSUbzmHliS1iDV1X6ZdCfxLUm1VvFcCuwsabSkJUgn3Puk9qpWi4hnSXXZF0naSlIvSUtK2lfSCflu8VLg55KWlrQyqSGvdB5UPK41mE46zqtWyeOLpEbdi4BbIqK8ZHMWcISkTXJPuL6Sds4XtPtJVXUn5vQlJW2RP/caKXj2ytuotp+FclXJscAe5VVSETGVVEXyO0nLSGqS9HFJFatkJI2QFJJG1LDp75IajV9sYd4EYCdJAyUtT+oFVEmrz+OSiJhEOv9+mb/
2018-05-11 12:19:53 +02:00
"text/plain": [
2018-05-13 10:35:15 +02:00
"<matplotlib.figure.Figure at 0x7f4c9f4a0048>"
2018-05-11 12:19:53 +02:00
]
},
"metadata": {},
"output_type": "display_data"
2018-05-09 19:13:08 +02:00
}
],
"source": [
2018-05-13 10:35:15 +02:00
"def model_a(labels,title,X,Xt,y,yt):\n",
" clf = MultinomialNB()\n",
" clf.fit(X, y)\n",
" pred = clf.predict(Xt)\n",
" score = metrics.accuracy_score(yt, pred)\n",
" pp(\"score: \" + str(score))\n",
" cm = metrics.confusion_matrix(yt, pred, labels=labels)\n",
" plot_confusion_matrix(cm, classes=labels, title=title)\n",
"model_a(labels=[\"FAKE\",\"REAL\"], title=\"Count Vectorizer, Multinomial Naive Bayes\", X=count_train_1, Xt=count_test_1, y=y1,yt=yt1)"
2018-05-11 12:19:53 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* comparing to PassiveAggresiveClassifier"
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 19,
2018-05-11 12:19:53 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
" DeprecationWarning)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
2018-05-13 10:35:15 +02:00
"accuracy: 0.935\n",
2018-05-11 12:19:53 +02:00
"Confusion matrix, without normalization\n"
]
},
{
"data": {
2018-05-13 10:35:15 +02:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ8AAAEmCAYAAAC9J50pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XecFdX9//HXe+kCgooVK2JL4k/sJUUsiYgFTawxir3GFE2iKUZjNGrK195iTGyxRYMSY4k90dgVW2zYIogURRAVC3x+f5yzMtws7C27e9nL+7mPeezMmbkzZ8qdz5wz584oIjAzM+tITfXOgJmZLXwcfMzMrMM5+JiZWYdz8DEzsw7n4GNmZh3OwcfMzDpcXYOPpF6S/iZpmqS/1DCfvST9oy3zVg+SbpE0ssrPniRpiqS32jpfVrlGOSYBJD0raWi981Fv9dqnkoZKGteO879A0nGF4cMkTZQ0Q9IS+f+gNl9wRLTaAd8EHgVmABOAW4AvlfPZVua7N/Aw0LXWebVHBwwFAhhVkr5OTr+nzPmcAFzRjvlcEfgQWKqd5j2j0AXwfmH4y8AlwMcl0+2eP/8asHXu3xeYVZjmVeBPwOqF5a2cl1Gc15Mt5GuPPG+VpHcFJgHb17DO+wL3tfF2DGBwGx6Xs/O2eQ94AdivvY6vNt4OAl4B/lPvvCxIHbARcDPwLvBOPi/uV9jf4zooH93yuWSd9l5WqyUfSUcBZwC/ApbOJ6PzgBGtfbYMKwEvRsSnbTCv9jIZ2FTSEoW0kcCLbbUAJbWUQlcE3o6ISVUsu+v8xkfEfyOiT3OXk9cppP0rp/26OF1EXDOPWT6Q59MP2Jp0oD8m6Qsl0/UvzGudFuZzA9Af2LwkfRjpRH/r/NarPbW2TdvIm3k7LgocA1wk6XMdsNxafQVYChgkacP2WkgH7YM2IWlT4C7gXmAwsARwGLBtHbKzNNATeLbWGbW6D1qJgv1IV1e7zmeaHqTg9GbuzgB6FCM2cDTpanQCc6L5L0hXy5/kZRxASQmBOVfBXfPwvqSrpvdIV817FdLvK3xuM+ARYFr+v1lh3D3AL4H783z+AQyYx7o15/8C4Iic1gUYD/ycQskHOBN4A5gOPAZ8OacPK1nPJwv5ODnn40PSQXcPcGAefz5wfWH+pwF38r9X+s0n8OYr4Uty+o6kA+jdPN+1Cp95jXTCegr4iApKnrRwBU8q+Zw0j+lfY+6Sz/+UKICbgOta2uet5OX3wB9L0q4FTi8Mbw+Mydvh38D/K4xbAfgr6QLjbeAcYC1gJnNKaO8WvguX5WlfB34GNBXW637g9Dyfk4rrCvyTuUuMu7eWt1bWeyglV8I5X7vk/r8Ab5GO/38Cny9MNxz4D+nYHw/8IKcPyPuh+cr7X4X1ey0fZ8vlY23xwvzWBaYA3fLw/sBzwFTgNmClknz+Efhz3u7nlIxbJef3PeAO4FzmPh/sk7f928BxJcfWCcB1wBWk7+CBpNsKxwIv589c25x30gn2ipz+Luk8sXS55xnS9/O3Jfm/ETgq9y8HXJ/3y6vAd+azP+8Dzi13fxfW6b28L3cujBtMCmLT8n65JqeLdHxOytvnaeALxe8vsDrpGG2uebir9DtPOt//FvgvMJF0buxVcr48hnT8XT7f47iVg3wY8CnzOREAJwIPkq5mliR9iX5ZyMyneZpupAP/A2CxwgFTPLhKh1fOK94V6J032hp53LLkL1XJQbE46cDfO39uzzy8RB5/T95xqwO98vCp89vppGD2UOHLexvp4C4Gn2+Rrli6koLtW0DPltarkI//Ap/Pn+nG3MFnEVLpal9S1dYUYPkyD87mg+ireb4/AsYC3QsnkzGkk2/zgXMecF4ZJ772CD77AxNL93kZefliPiaa16Ef6eQ4pHBinARsTLpoGJnz0yMPP0n6QvYmnYy+NK98kgLPjUDfnMcXgQMK038KHJn3Za/SeZRut/nlrYz1/mx/k06wO5MubtYobM++zLkwHFP47ATmXBgtBqyX+08hnUi65e7L5Audkn14F3BQYX6/AS7I/SNIx9laeTv8DPh3YdpF8v4aDnyDdEx3L4x/gHRi6w58KU97RR73OdIJ8Ut5/G/zOheDzyfATnmb9AK+Szo3LZ+3xYXAVXn6Q4C/5Tx1AdYnlSLLPc98hXSxqcK2/JAUdJpIF6A/z3kdRApm27SwLxchXehsUc7+zsO7FpazO+m7vmwedxXw0zyueExvk/PUnxSI1ip85hLy95cWvn/MHXxOB0aTzrN98zY8peR8f1re3r3mexy3cpDvBbzVyjQvA8MLw9sArxUy82HJikwCNikcMJUEn3dJB22vkjwUD4q9gYdLxj8A7Jv77wF+Vhh3OHBrGV/yl4A1gKvzdpkr+LTw2anketPS9Srk48QW0g4sDG9Mugp9HdizgoPzOODawnAT6Sp3aOFksn9rJ7l5LGtewWdm3j/vAlMK416j9eAzDPikZJ+/W+h+MJ/8vAR8M/cfROH+EOnq9Jcl079AqqrblHRV+j9BrjSfpJPTx8DnCmmHNO//PP1/W5lHafCZZ97K2AdDSSXd5lLKGGCPeUzbPy+7Xx7+b877oiXTnUgKrv9zX6pkHx7InCtikU7AX8nDt5ADcuG4+4Bc+iFdoE0mfZ97kq7Od87jViSduBYpfP4K5gSfn5MDRx5eJO+TYvD5Z0m+nwO2KgwvSwpQXUkB+n9Km5R/nlHels3rflBhu2zcwvHwY+BPLWzbgXn/rFnu97uF8WOAEbn/MlKNwPIl02xJumDahFyiLfn+thp88jq/D6xaGLcp8Gohnx+TL7pb61q7z/A2MKCVurvlSCfHZq/ntM/mEXPf0/kA6EOFIuJ9UpQ/FJgg6e+S1iwjP815GlgYLrYIKzc/lwPfBrYARpWOlPQDSc/llnvvkq7CB7QyzzfmNzIiHiJdMYlUZVCuubZBRMzOyypug/kuuwq/jYj+uWttvUsNJJ1EiwYU5vfb+Xz2MlJ1DKQLj8sK41YCjpb0bnNHKu0tl/+/HuXdbxxAKg2UHue1bM/55a0cb+Zts3hEDImIqwEkdZF0qqSXJU0nBY7mdYB0Uh0OvC7p3ny/AVIJZizwD0mvSDp2Hsu9nnQPdFnS1f9sUhVd8zqdWVifd0jHbvN2Gkm6KPo0ImbmeY3M45YD3omIDwrLKm7T5YrDebq3S/JWug9WAkYV8vMcqZSxNOn7fBtwtaQ3Jf1aUrdyzzORzrZXk2pWIDXK+nNhucuV7Nuf5OWWmkrahsu2MK5FkvaRNKYw7y8wZ//+iLTNH86tFPfP+b2LVK18LjBJ0u8lLVruMrMlSUH/scKyb83pzSbnfduq1oLPA6R7AjvNZ5o3SRu72Yo5rRrvk1au2TLFkRFxW0R8lbSjngcuKiM/zXkaX2Weml1OKiXdXPIFQdKXSTt9N1KVYn/SVZ2asz6Pec4rvXm+R5CKr2/m+Zdrrm0gSaQTW3EbzHfZHWxn5pzAKnU5sFU+iW7CnBMApJPRyYUg1j8iFomIq/K4FedxYVW6baaQrphLj/Natuf88laLb5Kqv7YmXQCtnNMFEBGPRMQIUjX5DeSLmoh4LyKOjohBpPuFR0naqnTmETGVdJ9097ysq/OJuHmdDilZp14R8W9Jy5Ouvr8l6a38k4BdgOGSBpCqAxeXVPz+r1Don0CqPksrI/UiVXPPlb2S4TeAbUvy0zMixkfEJxHxi4j4HKlafXvyRUyZ5xlIVVy7SFqJVNq5vrDcV0uW2zcihrewPT8gnWe/MY9lzCUv6yLShfAS+VzzDHP271sRcVBELEcq4Z4naXAed1ZErE+qwlwd+GE5yyyYQqrJ+nxhvfrFnIZIUMH3YL7BJyKmkYq750raSdIikrpJ2lbSr/NkVwE/k7RkPoh+TiouV2MM8BVJK0rqRyqqAiBpaUkjJPUmBcQZpCuGUjcDq0v6pqSuknYnbeybqswTABHxKqm65qctjO5LqjKYDHSV9HNS/XGzicDKlbRok7Q66Sbgt0hX9D+SNKTMj18LbCdpK0ndSPegPiJVMyw
2018-05-11 12:19:53 +02:00
"text/plain": [
2018-05-13 10:35:15 +02:00
"<matplotlib.figure.Figure at 0x7f4c9f313400>"
2018-05-11 12:19:53 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"linear_clf = PassiveAggressiveClassifier(n_iter=50)\n",
"\n",
2018-05-13 10:35:15 +02:00
"linear_clf.fit(tfidf_train_1, y1)\n",
2018-05-13 10:12:19 +02:00
"pred = linear_clf.predict(tfidf_test_1)\n",
2018-05-13 10:35:15 +02:00
"score = metrics.accuracy_score(yt1, pred)\n",
2018-05-11 12:19:53 +02:00
"print(\"accuracy: %0.3f\" % score)\n",
2018-05-13 10:35:15 +02:00
"cm = metrics.confusion_matrix(yt1, pred, labels=['FAKE', 'REAL'])\n",
2018-05-11 12:19:53 +02:00
"plot_confusion_matrix(cm, classes=['FAKE', 'REAL'], title= \"TFIDF Vectorite, PassiveAggressive Classifier\")"
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 20,
2018-05-11 12:19:53 +02:00
"metadata": {},
2018-05-13 10:12:19 +02:00
"outputs": [],
2018-05-11 12:19:53 +02:00
"source": [
2018-05-13 10:12:19 +02:00
"#clf = MultinomialNB(alpha=0.1)\n",
"#last_score = 0\n",
"#for alpha in np.arange(0,1,.1):\n",
"# nb_classifier = MultinomialNB(alpha=alpha)\n",
2018-05-13 10:35:15 +02:00
"# nb_classifier.fit(tfidf_train_1, y1)\n",
2018-05-13 10:12:19 +02:00
"# pred = nb_classifier.predict(tfidf_test_1)\n",
2018-05-13 10:35:15 +02:00
"# score = metrics.accuracy_score(yt1, pred)\n",
2018-05-13 10:12:19 +02:00
"# if score > last_score:\n",
"# clf = nb_classifier\n",
"# print(\"Alpha: {:.2f} Score: {:.5f}\".format(alpha, score))"
2018-05-11 12:19:53 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* try to get most important features"
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 21,
2018-05-11 12:19:53 +02:00
"metadata": {},
"outputs": [
{
2018-05-13 10:12:19 +02:00
"data": {
"text/markdown": [
"| |0 |1 |2 |\n",
"|-------------------------------------------|\n",
2018-05-13 10:35:15 +02:00
"|**0** |FAKE |-5.288437174814149|2016 |\n",
"|**0** |FAKE |-4.173273952371509|october |\n",
"|**0** |FAKE |-3.1422466009406578|election |\n",
"|**0** |FAKE |-3.0941004148189317|hillary |\n",
"|**0** |FAKE |-3.057544511024138|article |\n",
"|**0** |FAKE |-2.895415419748362|november |\n",
"|**0** |FAKE |-2.71535929690965|advertisement|\n",
"|**0** |FAKE |-2.5250775142313033|share |\n",
"|**0** |FAKE |-2.4821900429193917|print |\n",
"|**0** |FAKE |-2.293722835404013|source |\n",
"|**0** |FAKE |-2.224201011195553|mosul |\n",
"|**0** |FAKE |-2.219293536986114|fbi |\n",
"|**0** |FAKE |-2.154063293310608|28 |\n",
"|**0** |FAKE |-2.093787619812607|snip |\n",
"|**0** |FAKE |-2.0371567984631045|oct |\n",
"|**0** |FAKE |-2.0091297136096937|podesta |\n",
"|**0** |FAKE |-1.9792915407008016|uk |\n",
"|**0** |FAKE |-1.9700803192787941|email |\n",
"|**0** |FAKE |-1.94935091960633|donald |\n",
"|**0** |FAKE |-1.8823901751085903|watch |\n",
"|**0** |FAKE |-1.8573641352169936|establishment|\n",
"|**0** |FAKE |-1.85656476829781|kelly |\n",
"|**0** |FAKE |-1.8051549969069207|daesh |\n",
"|**0** |FAKE |-1.7994528991778074|photo |\n",
"|**0** |FAKE |-1.7940036317806896|jewish |\n",
"|**0** |FAKE |-1.769790301111315|com |\n",
"|**0** |FAKE |-1.7628662406412126|just |\n",
"|**0** |FAKE |-1.7588784544047598|ayotte |\n",
"|**0** |FAKE |-1.7586722326248487|obamacare |\n",
"|**0** |FAKE |-1.7183468795344152|wikileaks |\n",
2018-05-13 10:12:19 +02:00
"**Type:** class 'list'\n",
"\n"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"| |0 |1 |2 |\n",
"|-------------------------------------------|\n",
2018-05-13 10:35:15 +02:00
"|**0** |FAKE |-5.288437174814149|2016 |\n",
"|**0** |FAKE |-4.173273952371509|october |\n",
"|**0** |FAKE |-3.1422466009406578|election |\n",
"|**0** |FAKE |-3.0941004148189317|hillary |\n",
"|**0** |FAKE |-3.057544511024138|article |\n",
"|**0** |FAKE |-2.895415419748362|november |\n",
"|**0** |FAKE |-2.71535929690965|advertisement|\n",
"|**0** |FAKE |-2.5250775142313033|share |\n",
"|**0** |FAKE |-2.4821900429193917|print |\n",
"|**0** |FAKE |-2.293722835404013|source |\n",
"|**0** |FAKE |-2.224201011195553|mosul |\n",
"|**0** |FAKE |-2.219293536986114|fbi |\n",
"|**0** |FAKE |-2.154063293310608|28 |\n",
"|**0** |FAKE |-2.093787619812607|snip |\n",
"|**0** |FAKE |-2.0371567984631045|oct |\n",
"|**0** |FAKE |-2.0091297136096937|podesta |\n",
"|**0** |FAKE |-1.9792915407008016|uk |\n",
"|**0** |FAKE |-1.9700803192787941|email |\n",
"|**0** |FAKE |-1.94935091960633|donald |\n",
"|**0** |FAKE |-1.8823901751085903|watch |\n",
"|**0** |FAKE |-1.8573641352169936|establishment|\n",
"|**0** |FAKE |-1.85656476829781|kelly |\n",
"|**0** |FAKE |-1.8051549969069207|daesh |\n",
"|**0** |FAKE |-1.7994528991778074|photo |\n",
"|**0** |FAKE |-1.7940036317806896|jewish |\n",
"|**0** |FAKE |-1.769790301111315|com |\n",
"|**0** |FAKE |-1.7628662406412126|just |\n",
"|**0** |FAKE |-1.7588784544047598|ayotte |\n",
"|**0** |FAKE |-1.7586722326248487|obamacare |\n",
"|**0** |FAKE |-1.7183468795344152|wikileaks |\n",
"|**0** |REAL |5.096605047307298|said |\n",
"|**0** |REAL |3.1397974848991046|says |\n",
"|**0** |REAL |2.818912641958379|gop |\n",
"|**0** |REAL |2.57537492151748|marriage |\n",
"|**0** |REAL |2.5177436576284276|conservative|\n",
"|**0** |REAL |2.354337198374282|friday |\n",
"|**0** |REAL |2.3385922083327895|tuesday |\n",
"|**0** |REAL |2.2569380543369473|debate |\n",
"|**0** |REAL |2.2556129553493736|cruz |\n",
"|**0** |REAL |2.0914301251998175|continue |\n",
"|**0** |REAL |2.0725206776570992|rush |\n",
"|**0** |REAL |2.023007202992292|cnn |\n",
"|**0** |REAL |1.964077452159713|monday |\n",
"|**0** |REAL |1.9351283039486071|attacks |\n",
"|**0** |REAL |1.9103078111895166|jobs |\n",
"|**0** |REAL |1.9070984286838095|2013 |\n",
"|**0** |REAL |1.9033167532673034|say |\n",
"|**0** |REAL |1.888898000457345|convention|\n",
"|**0** |REAL |1.8787265215846798|gay |\n",
"|**0** |REAL |1.855658720921873|fox |\n",
"|**0** |REAL |1.8532683881897454|saturday |\n",
"|**0** |REAL |1.8357678493688905|2012 |\n",
"|**0** |REAL |1.833777119624796|candidates|\n",
"|**0** |REAL |1.8239594739918263|state |\n",
"|**0** |REAL |1.8239506711142206|march |\n",
"|**0** |REAL |1.7444175658575831|isn |\n",
"|**0** |REAL |1.7311078065383732|religious |\n",
"|**0** |REAL |1.7224487358895095|strategy |\n",
"|**0** |REAL |1.6450903626621578|reform |\n",
"|**0** |REAL |1.6171148546777458|paris |\n",
2018-05-13 10:12:19 +02:00
"**Type:** class 'list'\n",
"\n"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
2018-05-11 12:19:53 +02:00
}
],
"source": [
"def most_informative_feature_for_binary_classification(vectorizer, classifier, n=100):\n",
" \"\"\"\n",
" See: https://stackoverflow.com/a/26980472\n",
" \n",
" Identify most important features if given a vectorizer and binary classifier. Set n to the number\n",
" of weighted features you would like to show. (Note: current implementation merely prints and does not \n",
" return top classes.)\n",
" \"\"\"\n",
"\n",
" class_labels = classifier.classes_\n",
" feature_names = vectorizer.get_feature_names()\n",
" topn_class1 = sorted(zip(classifier.coef_[0], feature_names))[:n]\n",
" topn_class2 = sorted(zip(classifier.coef_[0], feature_names))[-n:]\n",
2018-05-13 10:12:19 +02:00
" \n",
" l = []\n",
" \n",
2018-05-11 12:19:53 +02:00
" for coef, feat in topn_class1:\n",
2018-05-13 10:12:19 +02:00
" l.append((class_labels[0], coef, feat))\n",
2018-05-11 12:19:53 +02:00
"\n",
2018-05-13 10:12:19 +02:00
" jupyter_print(l)\n",
2018-05-11 12:19:53 +02:00
"\n",
" for coef, feat in reversed(topn_class2):\n",
2018-05-13 10:12:19 +02:00
" l.append((class_labels[1], coef, feat))\n",
" \n",
" jupyter_print(l)\n",
2018-05-11 12:19:53 +02:00
"\n",
"\n",
2018-05-13 10:12:19 +02:00
"most_informative_feature_for_binary_classification(tfidf_vectorizer_1, linear_clf, n=30)\n",
2018-05-11 12:19:53 +02:00
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-05-13 10:12:19 +02:00
"----\n",
2018-05-13 10:35:15 +02:00
"## configuration 2\n",
2018-05-13 10:12:19 +02:00
"\n",
"* read data"
2018-05-11 12:19:53 +02:00
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 22,
2018-05-11 12:19:53 +02:00
"metadata": {},
"outputs": [
{
"data": {
2018-05-13 10:12:19 +02:00
"text/markdown": [
"----\n",
"#### Train Data:"
],
2018-05-11 12:19:53 +02:00
"text/plain": [
2018-05-13 10:12:19 +02:00
"<IPython.core.display.Markdown object>"
2018-05-11 12:19:53 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
2018-05-13 10:12:19 +02:00
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>label</th>\n",
" <th>statement</th>\n",
" <th>subjects</th>\n",
" <th>speaker</th>\n",
" <th>job</th>\n",
" <th>state</th>\n",
" <th>party</th>\n",
" <th>#barely_true</th>\n",
" <th>#false</th>\n",
" <th>#half_true</th>\n",
" <th>#mostly_true</th>\n",
" <th>#pants_on_fire</th>\n",
" <th>context</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2635.json</td>\n",
" <td>false</td>\n",
" <td>Says the Annies List political group supports ...</td>\n",
" <td>abortion</td>\n",
" <td>dwayne-bohac</td>\n",
" <td>State representative</td>\n",
" <td>Texas</td>\n",
" <td>republican</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>a mailer</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1123.json</td>\n",
" <td>false</td>\n",
" <td>Health care reform legislation is likely to ma...</td>\n",
" <td>health-care</td>\n",
" <td>blog-posting</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>none</td>\n",
" <td>7.0</td>\n",
" <td>19.0</td>\n",
" <td>3.0</td>\n",
" <td>5.0</td>\n",
" <td>44.0</td>\n",
" <td>a news release</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>12465.json</td>\n",
" <td>true</td>\n",
" <td>The Chicago Bears have had more starting quart...</td>\n",
" <td>education</td>\n",
" <td>robin-vos</td>\n",
" <td>Wisconsin Assembly speaker</td>\n",
" <td>Wisconsin</td>\n",
" <td>republican</td>\n",
" <td>0.0</td>\n",
" <td>3.0</td>\n",
" <td>2.0</td>\n",
" <td>5.0</td>\n",
" <td>1.0</td>\n",
" <td>a an online opinion-piece</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>5947.json</td>\n",
" <td>false</td>\n",
" <td>When Mitt Romney was governor of Massachusetts...</td>\n",
" <td>history,state-budget</td>\n",
" <td>mitt-romney</td>\n",
" <td>Former governor</td>\n",
" <td>Massachusetts</td>\n",
" <td>republican</td>\n",
" <td>34.0</td>\n",
" <td>32.0</td>\n",
" <td>58.0</td>\n",
" <td>33.0</td>\n",
" <td>19.0</td>\n",
" <td>an interview with CBN News</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>620.json</td>\n",
" <td>true</td>\n",
" <td>McCain opposed a requirement that the governme...</td>\n",
" <td>federal-budget</td>\n",
" <td>barack-obama</td>\n",
" <td>President</td>\n",
" <td>Illinois</td>\n",
" <td>democrat</td>\n",
" <td>70.0</td>\n",
" <td>71.0</td>\n",
" <td>160.0</td>\n",
" <td>163.0</td>\n",
" <td>9.0</td>\n",
" <td>a radio ad</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" id label statement \\\n",
"0 2635.json false Says the Annies List political group supports ... \n",
"3 1123.json false Health care reform legislation is likely to ma... \n",
"5 12465.json true The Chicago Bears have had more starting quart... \n",
"12 5947.json false When Mitt Romney was governor of Massachusetts... \n",
"16 620.json true McCain opposed a requirement that the governme... \n",
"\n",
" subjects speaker job \\\n",
"0 abortion dwayne-bohac State representative \n",
"3 health-care blog-posting NaN \n",
"5 education robin-vos Wisconsin Assembly speaker \n",
"12 history,state-budget mitt-romney Former governor \n",
"16 federal-budget barack-obama President \n",
"\n",
" state party #barely_true #false #half_true #mostly_true \\\n",
"0 Texas republican 0.0 1.0 0.0 0.0 \n",
"3 NaN none 7.0 19.0 3.0 5.0 \n",
"5 Wisconsin republican 0.0 3.0 2.0 5.0 \n",
"12 Massachusetts republican 34.0 32.0 58.0 33.0 \n",
"16 Illinois democrat 70.0 71.0 160.0 163.0 \n",
"\n",
" #pants_on_fire context \n",
"0 0.0 a mailer \n",
"3 44.0 a news release \n",
"5 1.0 a an online opinion-piece \n",
"12 19.0 an interview with CBN News \n",
"16 9.0 a radio ad "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"----\n",
"#### Test Data:"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>label</th>\n",
" <th>statement</th>\n",
" <th>subjects</th>\n",
" <th>speaker</th>\n",
" <th>job</th>\n",
" <th>state</th>\n",
" <th>party</th>\n",
" <th>#barely_true</th>\n",
" <th>#false</th>\n",
" <th>#half_true</th>\n",
" <th>#mostly_true</th>\n",
" <th>#pants_on_fire</th>\n",
" <th>context</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>11972.json</td>\n",
" <td>true</td>\n",
" <td>Building a wall on the U.S.-Mexico border will...</td>\n",
" <td>immigration</td>\n",
" <td>rick-perry</td>\n",
" <td>Governor</td>\n",
" <td>Texas</td>\n",
" <td>republican</td>\n",
" <td>30</td>\n",
" <td>30</td>\n",
" <td>42</td>\n",
" <td>23</td>\n",
" <td>18</td>\n",
" <td>Radio interview</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>11685.json</td>\n",
" <td>false</td>\n",
" <td>Wisconsin is on pace to double the number of l...</td>\n",
" <td>jobs</td>\n",
" <td>katrina-shankland</td>\n",
" <td>State representative</td>\n",
" <td>Wisconsin</td>\n",
" <td>democrat</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>a news conference</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>11096.json</td>\n",
" <td>false</td>\n",
" <td>Says John McCain has done nothing to help the ...</td>\n",
" <td>military,veterans,voting-record</td>\n",
" <td>donald-trump</td>\n",
" <td>President-Elect</td>\n",
" <td>New York</td>\n",
" <td>republican</td>\n",
" <td>63</td>\n",
" <td>114</td>\n",
" <td>51</td>\n",
" <td>37</td>\n",
" <td>61</td>\n",
" <td>comments on ABC's This Week.</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>5962.json</td>\n",
" <td>true</td>\n",
" <td>Over the past five years the federal governmen...</td>\n",
" <td>federal-budget,pensions,retirement</td>\n",
" <td>brendan-doherty</td>\n",
" <td>NaN</td>\n",
" <td>Rhode Island</td>\n",
" <td>republican</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>a campaign website</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>7070.json</td>\n",
" <td>true</td>\n",
" <td>Says that Tennessee law requires that schools ...</td>\n",
" <td>county-budget,county-government,education,taxes</td>\n",
" <td>stand-children-tennessee</td>\n",
" <td>Child and education advocacy organization.</td>\n",
" <td>Tennessee</td>\n",
" <td>none</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>in a post on Facebook.</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" id label statement \\\n",
"0 11972.json true Building a wall on the U.S.-Mexico border will... \n",
"1 11685.json false Wisconsin is on pace to double the number of l... \n",
"2 11096.json false Says John McCain has done nothing to help the ... \n",
"5 5962.json true Over the past five years the federal governmen... \n",
"6 7070.json true Says that Tennessee law requires that schools ... \n",
"\n",
" subjects speaker \\\n",
"0 immigration rick-perry \n",
"1 jobs katrina-shankland \n",
"2 military,veterans,voting-record donald-trump \n",
"5 federal-budget,pensions,retirement brendan-doherty \n",
"6 county-budget,county-government,education,taxes stand-children-tennessee \n",
"\n",
" job state party \\\n",
"0 Governor Texas republican \n",
"1 State representative Wisconsin democrat \n",
"2 President-Elect New York republican \n",
"5 NaN Rhode Island republican \n",
"6 Child and education advocacy organization. Tennessee none \n",
"\n",
" #barely_true #false #half_true #mostly_true #pants_on_fire \\\n",
"0 30 30 42 23 18 \n",
"1 2 1 0 0 0 \n",
"2 63 114 51 37 61 \n",
"5 1 2 1 1 0 \n",
"6 0 0 0 0 0 \n",
"\n",
" context \n",
"0 Radio interview \n",
"1 a news conference \n",
"2 comments on ABC's This Week. \n",
"5 a campaign website \n",
"6 in a post on Facebook. "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"----"
],
2018-05-11 12:19:53 +02:00
"text/plain": [
2018-05-13 10:12:19 +02:00
"<IPython.core.display.Markdown object>"
2018-05-11 12:19:53 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
2018-05-13 10:12:19 +02:00
"names = [\n",
" \"id\",\n",
" \"label\",\n",
" \"statement\",\n",
" \"subjects\",\n",
" \"speaker\",\n",
" \"job\",\n",
" \"state\",\n",
" \"party\",\n",
" \"#barely_true\",\n",
" \"#false\",\n",
" \"#half_true\",\n",
" \"#mostly_true\",\n",
" \"#pants_on_fire\",\n",
" \"context\"\n",
"]\n",
"\n",
"df_2_train = pd.read_csv(\"data/train.tsv\", delimiter='\\t', names=names)\n",
"df_2_test = pd.read_csv(\"data/test.tsv\", delimiter='\\t', names=names)\n",
"\n",
"# use only 'False' and 'True' statements\n",
"\n",
"df_2_train = df_2_train[df_2_train['label'].isin([\"false\",\"true\"])]\n",
"df_2_test = df_2_test[df_2_test['label'].isin([\"false\",\"true\"])]\n",
"\n",
"jp(\"----\\n#### Train Data:\")\n",
"display(df_2_train.head())\n",
"jp(\"----\\n#### Test Data:\")\n",
"display(df_2_test.head())\n",
"jp(\"----\")"
2018-05-11 12:19:53 +02:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2018-05-13 10:12:19 +02:00
"#### tdidf vectorizer on new dataset\n"
2018-05-11 12:19:53 +02:00
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 23,
2018-05-11 12:19:53 +02:00
"metadata": {},
2018-05-13 10:12:19 +02:00
"outputs": [],
"source": [
2018-05-13 10:35:15 +02:00
"X2 = df_2_train['statement']\n",
"y2 = df_2_train['label']\n",
"Xt2 = df_2_test['statement']\n",
"yt2 = df_2_test['label']\n"
2018-05-13 10:12:19 +02:00
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 24,
2018-05-13 10:12:19 +02:00
"metadata": {},
"outputs": [],
"source": [
"tfidf_vectorizer_2 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
2018-05-13 10:35:15 +02:00
"tfidf_train_2 = tfidf_vectorizer_2.fit_transform(X2)\n",
"tfidf_test_2 = tfidf_vectorizer_2.transform(Xt2)"
2018-05-13 10:12:19 +02:00
]
},
{
2018-05-13 10:35:15 +02:00
"cell_type": "code",
"execution_count": 25,
2018-05-13 10:12:19 +02:00
"metadata": {},
2018-05-13 10:35:15 +02:00
"outputs": [],
2018-05-13 10:12:19 +02:00
"source": [
2018-05-13 10:35:15 +02:00
"def model_b(labels, title, X, Xt, y, yt):\n",
" clf = MultinomialNB()\n",
" clf.fit(X, y)\n",
" pred = clf.predict(Xt)\n",
" score = metrics.accuracy_score(yt, pred)\n",
" pp(\"score: \" + str(score))\n",
" cm = metrics.confusion_matrix(yt, pred, labels=labels)\n",
" plot_confusion_matrix(cm, classes=labels, title=title)"
2018-05-13 10:12:19 +02:00
]
},
{
"cell_type": "code",
2018-05-13 10:35:15 +02:00
"execution_count": 26,
2018-05-13 10:12:19 +02:00
"metadata": {},
2018-05-11 12:19:53 +02:00
"outputs": [
2018-05-13 10:12:19 +02:00
{
"name": "stdout",
"output_type": "stream",
"text": [
"'score: 0.6192560175054704'\n",
"Confusion matrix, without normalization\n"
]
},
2018-05-11 12:19:53 +02:00
{
"data": {
2018-05-13 10:35:15 +02:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEmCAYAAAA0k8gFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcVXX9x/HXG1AUIVBxQRZxQ1NTUtSyVEwz3EIrLbMkJXHLsixz+2VpmrarSUZprpFmWZSomUsqCYqGC664gyjgQsq+fH5/nO/onXGWM5c7c8+deT95nAf3fs+553zuvTOf+S7nfI8iAjMza70u1Q7AzKxWOYGamZXJCdTMrExOoGZmZXICNTMrkxOomVmZnEBbQdKakv4uab6kP63Cfg6X9M9KxlYNkm6WNKrM1/5Q0jxJr1Y6rrbS8Psv2vco6XRJv6t2HJ1KRHS4BfgiMBV4B5gN3Ax8vAL7/TJwP9Ct2u+xifiGAwHc2KB8+1R+V879fB+4pg3jHAQsAtav9mdWq99/+q5nttOx9gfuBd4CXgV+B/Sq9mdQhKXD1UAlfQv4JXAesAHZL+tYYGQFdr8x8HRELK/AvtrKXOCjktYtKRsFPF2pAyizKj87g4DXI2JOGcfutgrHXVXt8v1X4POttN7AD4GNgA8C/YGfVDWioqh2Bq/wX8reZLXOQ5rZpjtZgn0lLb8Euqd1w4GZwMnAHLLa65Fp3Q+ApcCydIzRNKipAYPJanrd0vOvAM8BbwPPA4eXlN9b8rpdgQeA+en/XUvW3QWcA0xK+/kn0LeJ91YX/6XACamsKzAL+B4lNVDgQuBl4H/Ag8BuqXxEg/f5cEkc56Y4FgGbp7KvpvW/Bv5csv8LgNsBNYhx7/T6lWn/V6TyTwPTyWo5dwEfLHnNC8B3gUeAJeSoAQLbALcBbwCvAae3wfff8HvcB3gqfY9jgX+XfD7fp/mflcY+3yOBJ9L3/hxwTNp2rQaf4Ttkya3hMVr6TL+dPtP5wHXAGjl/zz4DPFrt3/ciLFUPoKJvJvvlX97cLxhwNjAZWB9YD/gPcE5aNzy9/mxgNWA/YCGwdlrf8Ae0yV+K9EP+P2DLtK4fsE16/O4vHrAO8CZZ87AbcFh6vm5afxfwLDAEWDM9P7+J9zacLAHsCkxJZfsBtwJfpX4C/RKwbjrmyWRNszUae18lcbxElpi6pc/nLt5LED3IarlfAXYD5gEDmouz5PkQYAHwybTfU4AZwOpp/QvANGAgsGYqGwuMbWL/vciS38nAGun5Lm3w/Zd+j33T9/2Z9Pl8gyzZtiaBNvx89wc2AwTskWLZobHPsOExcn6m95Ml3nXIEvWxOX/Pfgn8sdq/70VYitRMqIR1gXnRfBPrcODsiJgTEXPJahZfLlm/LK1fFhETyf66b1lmPCuBbSWtGRGzI2J6I9vsDzwTEVdHxPKIGA88CRxYss3vI+LpiFgEXA8Mbe6gEfEfYB1JWwJHAFc1ss01EfF6OubPyGpmLb3PKyJienrNsgb7W0j2Of4cuAY4MSJmtrC/Op8HboqI29J+f0r2x2LXkm0uioiX02dARBwfEcc3sb8DgFcj4mcRsTgi3o6IKWldW33/+wHTI+Iv6efvIrI/Sq1R7/ONiJsi4tnI/Jus9bFbzn3l/UxfiYg3gL/Tws8VgKRPknUJfa81b6yj6mgJ9HWgbwv9ZBsBL5Y8fzGVvbuPBgl4IdCztYFExAKyH+JjgdmSbpK0VY546mLqX/K89BcxbzxXA18D9gRubLhS0rclPZFGlN8i6/7o28I+X25uZUpSz5HVmK7PEWOdep9BRKxMxyr9DJo9dgMDyWrtLR6Lyn3/G5XGGFlVLe8fkDr13qOkfSVNlvRG+o72o+XvqDSelj7TVv1cSfoI8AfgcxFRsT71WtbREuh9ZH1kBzWzzStkgwF1BqWyciwga7rW2bB0ZUTcGhGfJGu+Pwn8Nkc8dTHNKjOmOlcDxwMTU+3wXZJ2I2vSHUrWPO1D1g+mutCb2GezU3dJOoGsJvtK2n9e9T4DSSJLgqWfQWumDXsZ2DTPsVi177/UbGBA3ZP0HgaUrG/2ZyV59z1K6g78mazmuEH6jibS8ndUJ89nmpukDwMTgKMi4vZy9tERdagEGhHzyZoWl0g6SFIPSaulv+Q/TpuNB86UtJ6kvmn7a8o85DRgd0mDJPUGTqtbIWkDSSMlrUWW1N8ha9I3NBEYIumLkrpJ+jywNfCPMmMCICKeJ+s3O6OR1b3I+vrmAt0kfQ/4QMn614DBrRkJljSEbKT2S2RN4lMktdgkTK4H9pe0l6TVyPoul5D1T5bjH0A/SSdJ6i6pl6Rd0rpKfv+lbgI+lH7uugEnUD9JNvmz0oTVyf4YzQWWS9qXbJCqzmvAumlfjanYZyppW+AWsm6Zv7f29R1Zh0qgAKk/71vAmWQ/fC+TNWX/mjb5Idk5oo8AjwIPpbJyjnUb2ejlI2Qj2aVJr0uK4xWykeA9gOMa2cfrZH12J5N1QZwCHBAR88qJqcG+742IxmpXt5L9QjxN1sxbTP3mY91FAq9Leqil46SEcQ1wQUQ8HBHPAKcDV6eaVEtxPkWWeC8mG3w6EDgwIpY2c8xLJV3axP7eJhs8OZCsmfoMWVcGVPD7b3DMecAhwI/Jvset03GWpPXN/aw09R6+TpYI3yQ7t3lCyfonyf4YPCfpLUkbNXh9qz/TZpxMNuB2maR30tJYf36no6yrxswqKdXeZ5KdunZnteOxttHhaqBm1SLpU5L6pFr36WT9lZOrHJa1ISdQs8r5KNnof12T+aC6066sY3IT3sysTK6BmpmVqZoTM7SJPuusG/36D6p2GNZKby1Z1vJGVjivPjN9XkSsV6n9df3AxhHL8/V6xKK5t0bEiEoduxwdLoH26z+I39/oQc9a848ZrZ6YyQrg3H23bHgV3SqJ5YvovuWhubZdPO2SvFdltZkOl0DNrJYJCjWTX/OcQM2sOAR06VrtKHJzAjWzYpFa3qYgnEDNrEDchDczK59roGZmZRCugZqZlUeugZqZlc2j8GZm5fAgkplZeYSb8GZmZXMN1MysHG7Cm5mVr4ub8GZmredr4c3MyuUmvJlZ+TwKb2ZWphqqgdZOpGbW8Un5lxZ3pcslzZH0WEnZUEmTJU2TNFXSzqlcki6SNEPSI5J2yBOuE6iZFYu65FtadgXQ8J5JPwZ+EBFDge+l5wD7AlukZQzw6zwHcAI1swJRNgqfZ2lBRNwNvNGwGPhAetwbeCU9HglcFZnJQB9J/Vo6hvtAzaxY8g8i9ZU0teT5uIgY18JrTgJulfRTsgrkrqm8P/ByyXYzU9ns5nbmBGpmxdG6+UDnRcSwVh7hOOCbEfFnSYcClwF7t3If73IT3swKRJXsA23MKOAv6fGfgJ3T41nAwJLtBqSyZjmBmlmxVGgUvgmvAHukx58AnkmPJwBHpNH4jwDzI6LZ5ju4CW9mRVOh80AljQeGk/WVzgTOAo4GLpTUDVhMNuIOMBHYD5gBLASOzHMMJ1AzKw6pYtfCR8RhTazasZFtAzihtcdwAjWzYvGlnGZm5ZETqJlZ62V39HACNTNrPaWlRjiBmlmByDVQM7NydelSO6enO4GaWaG4BmpmVg73gZqZlUfuAzUzK58TqJlZmZxAzczKIVAXJ1Azs7K4BmpmVgYPIpmZrQInUDOzctVO/nQCNbMCkWugZmZl87XwZmZl8CCSmdmqqJ386QRaVON/P5a/X381Emw2ZGvOuOASHn3ofi4+//9YvmwpW247lNPPu5hu3fwVVtPB227AluutxYKlK7h40osAbLNBTz6x+bqs13N1Lr3vJV753xIAugpGbrMBG/VegwiY+OQcnn9jUTXDL54a6wOtnc6GTmTOq6/wp6t+w+U33sG1E+9jxcqV/HPCDZxzynGc88vLuHbifWy40UAm3ji+2qF2ev+d9T+ufHBWvbI57yxl/LRXePHN+slx2MDeAPxq0otcMXUmI7Zcr5YqW+1GUq6lCJxAC2rF8uUsWbyY5cuXs3jRQtbs0YPVVludQZt
2018-05-11 12:19:53 +02:00
"text/plain": [
2018-05-13 10:35:15 +02:00
"<matplotlib.figure.Figure at 0x7f4c9f4c8fd0>"
2018-05-11 12:19:53 +02:00
]
},
"metadata": {},
2018-05-13 10:12:19 +02:00
"output_type": "display_data"
2018-05-11 12:19:53 +02:00
}
],
"source": [
2018-05-13 10:35:15 +02:00
"model_b(labels=[\"true\", \"false\"], title=\"configuration 2\", X=tfidf_train_2, y=y2, Xt=tfidf_test_2, yt=yt2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## configuration 3"
2018-05-09 19:13:08 +02:00
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
2018-05-09 10:47:51 +02:00
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}