1381 lines
168 KiB
Plaintext
1381 lines
168 KiB
Plaintext
|
{
|
|||
|
|
"cells": [
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"import all usefull tool"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 1,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"Populating the interactive namespace from numpy and matplotlib\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"%pylab inline\n",
|
|||
|
|
"\n",
|
|||
|
|
"import pandas as pd\n",
|
|||
|
|
"import numpy as np\n",
|
|||
|
|
"import itertools\n",
|
|||
|
|
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
|
|||
|
|
"from sklearn.model_selection import train_test_split\n",
|
|||
|
|
"from sklearn.linear_model import PassiveAggressiveClassifier\n",
|
|||
|
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
|||
|
|
"from sklearn import metrics\n",
|
|||
|
|
"import matplotlib.pyplot as plt"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Read in Datasets"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Load Dataset 1 - fake_or_real_news.csv\n",
|
|||
|
|
"Read in File fake_or_real_news.csv"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 2,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"df = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/fact checking/fake_or_real_news.csv')"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"ignores first column"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 3,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"df = df.set_index('Unnamed: 0')"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"quick view at the data"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/html": [
|
|||
|
|
"<div>\n",
|
|||
|
|
"<style>\n",
|
|||
|
|
" .dataframe thead tr:only-child th {\n",
|
|||
|
|
" text-align: right;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe thead th {\n",
|
|||
|
|
" text-align: left;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe tbody tr th {\n",
|
|||
|
|
" vertical-align: top;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"</style>\n",
|
|||
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|||
|
|
" <thead>\n",
|
|||
|
|
" <tr style=\"text-align: right;\">\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" <th>title</th>\n",
|
|||
|
|
" <th>text</th>\n",
|
|||
|
|
" <th>label</th>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>Unnamed: 0</th>\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </thead>\n",
|
|||
|
|
" <tbody>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>8476</th>\n",
|
|||
|
|
" <td>You Can Smell Hillary’s Fear</td>\n",
|
|||
|
|
" <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
|
|||
|
|
" <td>FAKE</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>10294</th>\n",
|
|||
|
|
" <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
|
|||
|
|
" <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
|
|||
|
|
" <td>FAKE</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>3608</th>\n",
|
|||
|
|
" <td>Kerry to go to Paris in gesture of sympathy</td>\n",
|
|||
|
|
" <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
|
|||
|
|
" <td>REAL</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>10142</th>\n",
|
|||
|
|
" <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
|
|||
|
|
" <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
|
|||
|
|
" <td>FAKE</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>875</th>\n",
|
|||
|
|
" <td>The Battle of New York: Why This Primary Matters</td>\n",
|
|||
|
|
" <td>It's primary day in New York and front-runners...</td>\n",
|
|||
|
|
" <td>REAL</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </tbody>\n",
|
|||
|
|
"</table>\n",
|
|||
|
|
"</div>"
|
|||
|
|
],
|
|||
|
|
"text/plain": [
|
|||
|
|
" title \\\n",
|
|||
|
|
"Unnamed: 0 \n",
|
|||
|
|
"8476 You Can Smell Hillary’s Fear \n",
|
|||
|
|
"10294 Watch The Exact Moment Paul Ryan Committed Pol... \n",
|
|||
|
|
"3608 Kerry to go to Paris in gesture of sympathy \n",
|
|||
|
|
"10142 Bernie supporters on Twitter erupt in anger ag... \n",
|
|||
|
|
"875 The Battle of New York: Why This Primary Matters \n",
|
|||
|
|
"\n",
|
|||
|
|
" text label \n",
|
|||
|
|
"Unnamed: 0 \n",
|
|||
|
|
"8476 Daniel Greenfield, a Shillman Journalism Fello... FAKE \n",
|
|||
|
|
"10294 Google Pinterest Digg Linkedin Reddit Stumbleu... FAKE \n",
|
|||
|
|
"3608 U.S. Secretary of State John F. Kerry said Mon... REAL \n",
|
|||
|
|
"10142 — Kaydee King (@KaydeeKing) November 9, 2016 T... FAKE \n",
|
|||
|
|
"875 It's primary day in New York and front-runners... REAL "
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"df.head()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"store label from tob row of the table"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 5,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"#store label before dropping it\n",
|
|||
|
|
"bin_y = df.label\n",
|
|||
|
|
"#y.head()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"cut of label column to get an unlabled array"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 6,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"df = df.drop('label', axis=1)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"from skikit learn the function: train_test_split\n",
|
|||
|
|
"* in the dataframe get text column by df['text']\n",
|
|||
|
|
"* use stored y label df\n",
|
|||
|
|
"* use seed 42\n",
|
|||
|
|
"* determine split size: in this case 0.33"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 7,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"bin_X_train, bin_X_test, bin_y_train, bin_y_test = train_test_split(df['text'], bin_y, test_size=0.25, random_state=4222)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Load Dataset 2 - liar_dataset.zip\n",
|
|||
|
|
"Read in File liar_dataset.zip"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 8,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"#training data file\n",
|
|||
|
|
"df = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/fact checking/train.tsv', delimiter=\"\\t\", header=None, usecols=[1,2], names=['y', 'claim'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 9,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/html": [
|
|||
|
|
"<div>\n",
|
|||
|
|
"<style>\n",
|
|||
|
|
" .dataframe thead tr:only-child th {\n",
|
|||
|
|
" text-align: right;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe thead th {\n",
|
|||
|
|
" text-align: left;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe tbody tr th {\n",
|
|||
|
|
" vertical-align: top;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"</style>\n",
|
|||
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|||
|
|
" <thead>\n",
|
|||
|
|
" <tr style=\"text-align: right;\">\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" <th>y</th>\n",
|
|||
|
|
" <th>claim</th>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </thead>\n",
|
|||
|
|
" <tbody>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>0</th>\n",
|
|||
|
|
" <td>false</td>\n",
|
|||
|
|
" <td>Says the Annies List political group supports ...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>1</th>\n",
|
|||
|
|
" <td>half-true</td>\n",
|
|||
|
|
" <td>When did the decline of coal start? It started...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>2</th>\n",
|
|||
|
|
" <td>mostly-true</td>\n",
|
|||
|
|
" <td>Hillary Clinton agrees with John McCain \"by vo...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>3</th>\n",
|
|||
|
|
" <td>false</td>\n",
|
|||
|
|
" <td>Health care reform legislation is likely to ma...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>4</th>\n",
|
|||
|
|
" <td>half-true</td>\n",
|
|||
|
|
" <td>The economic turnaround started at the end of ...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </tbody>\n",
|
|||
|
|
"</table>\n",
|
|||
|
|
"</div>"
|
|||
|
|
],
|
|||
|
|
"text/plain": [
|
|||
|
|
" y claim\n",
|
|||
|
|
"0 false Says the Annies List political group supports ...\n",
|
|||
|
|
"1 half-true When did the decline of coal start? It started...\n",
|
|||
|
|
"2 mostly-true Hillary Clinton agrees with John McCain \"by vo...\n",
|
|||
|
|
"3 false Health care reform legislation is likely to ma...\n",
|
|||
|
|
"4 half-true The economic turnaround started at the end of ..."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 9,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"df.head()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 10,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"mul_X_train = df.claim\n",
|
|||
|
|
"mul_y_train = df.y"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 11,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"#test data file\n",
|
|||
|
|
"df = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/fact checking/test.tsv', delimiter=\"\\t\", header=None, usecols=[1,2], names=['y', 'claim'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 12,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/html": [
|
|||
|
|
"<div>\n",
|
|||
|
|
"<style>\n",
|
|||
|
|
" .dataframe thead tr:only-child th {\n",
|
|||
|
|
" text-align: right;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe thead th {\n",
|
|||
|
|
" text-align: left;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe tbody tr th {\n",
|
|||
|
|
" vertical-align: top;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"</style>\n",
|
|||
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|||
|
|
" <thead>\n",
|
|||
|
|
" <tr style=\"text-align: right;\">\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" <th>y</th>\n",
|
|||
|
|
" <th>claim</th>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </thead>\n",
|
|||
|
|
" <tbody>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>0</th>\n",
|
|||
|
|
" <td>true</td>\n",
|
|||
|
|
" <td>Building a wall on the U.S.-Mexico border will...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>1</th>\n",
|
|||
|
|
" <td>false</td>\n",
|
|||
|
|
" <td>Wisconsin is on pace to double the number of l...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>2</th>\n",
|
|||
|
|
" <td>false</td>\n",
|
|||
|
|
" <td>Says John McCain has done nothing to help the ...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>3</th>\n",
|
|||
|
|
" <td>half-true</td>\n",
|
|||
|
|
" <td>Suzanne Bonamici supports a plan that will cut...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>4</th>\n",
|
|||
|
|
" <td>pants-fire</td>\n",
|
|||
|
|
" <td>When asked by a reporter whether hes at the ce...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </tbody>\n",
|
|||
|
|
"</table>\n",
|
|||
|
|
"</div>"
|
|||
|
|
],
|
|||
|
|
"text/plain": [
|
|||
|
|
" y claim\n",
|
|||
|
|
"0 true Building a wall on the U.S.-Mexico border will...\n",
|
|||
|
|
"1 false Wisconsin is on pace to double the number of l...\n",
|
|||
|
|
"2 false Says John McCain has done nothing to help the ...\n",
|
|||
|
|
"3 half-true Suzanne Bonamici supports a plan that will cut...\n",
|
|||
|
|
"4 pants-fire When asked by a reporter whether hes at the ce..."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 12,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"df.head()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 13,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"mul_X_test = df.claim\n",
|
|||
|
|
"mul_y_test = df.y"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 14,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"#test data file\n",
|
|||
|
|
"df = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/fact checking/valid.tsv', delimiter=\"\\t\", header=None, usecols=[1,2], names=['y', 'claim'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 15,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"mul_X_valid = df.claim\n",
|
|||
|
|
"mul_y_valid = df.y"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 16,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"<class 'pandas.core.series.Series'>\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"print(type(mul_X_valid))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Generate Dataset 3\n",
|
|||
|
|
"* using code from Diego by copy paste with some small modifications\n",
|
|||
|
|
"* thanks for distributing this code @diego ;)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 17,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"-- fake news\n",
|
|||
|
|
"Index(['y', 'claim'], dtype='object')\n",
|
|||
|
|
"3171\n",
|
|||
|
|
"3164\n",
|
|||
|
|
"6335\n",
|
|||
|
|
"-- liar liar\n",
|
|||
|
|
"Index(['y', 'claim'], dtype='object')\n",
|
|||
|
|
"{'mostly-true', 'pants-fire', 'true', 'half-true', 'false', 'barely-true'} 10240\n",
|
|||
|
|
"1676\n",
|
|||
|
|
"1995\n",
|
|||
|
|
"{'true', 'false'} 3671\n",
|
|||
|
|
"false 5159\n",
|
|||
|
|
"true 4847\n",
|
|||
|
|
"Name: y, dtype: int64\n",
|
|||
|
|
"done\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"'''import random\n",
|
|||
|
|
"import sys\n",
|
|||
|
|
"import pandas as pd\n",
|
|||
|
|
"import numpy as np\n",
|
|||
|
|
"from sklearn.cross_validation import train_test_split\n",
|
|||
|
|
"\n",
|
|||
|
|
"ds1 = sys.argv[1]\n",
|
|||
|
|
"ds2 = sys.argv[2]'''\n",
|
|||
|
|
"\n",
|
|||
|
|
"try:\n",
|
|||
|
|
" print('-- fake news')\n",
|
|||
|
|
" df1 = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/fact checking/fake_or_real_news.csv', sep=',', usecols=['title','text','label'])\n",
|
|||
|
|
" df1['claim'] = df1[['title', 'text']].apply(lambda x: '. '.join(x), axis=1)\n",
|
|||
|
|
" del df1['title']\n",
|
|||
|
|
" del df1['text']\n",
|
|||
|
|
" df1.rename(index=str, columns={'label': 'y'}, inplace=True)\n",
|
|||
|
|
" print(df1.keys())\n",
|
|||
|
|
" print(len(df1[df1['y']=='REAL']))\n",
|
|||
|
|
" print(len(df1[df1['y']=='FAKE']))\n",
|
|||
|
|
" df1['y'] = np.where(df1['y'] == 'FAKE', 'false', 'true')\n",
|
|||
|
|
" print(len(df1))\n",
|
|||
|
|
"\n",
|
|||
|
|
" print('-- liar liar')\n",
|
|||
|
|
" df2 = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/fact checking/train.tsv', sep='\\t', header=None, usecols=[1,2], names=['y', 'claim'])\n",
|
|||
|
|
" print(df2.keys())\n",
|
|||
|
|
" print(set(df2.y), len(df2))\n",
|
|||
|
|
" print(len(df2[df2['y'] == 'true']))\n",
|
|||
|
|
" print(len(df2[df2['y'] == 'false']))\n",
|
|||
|
|
" df2=df2[(df2['y'] == 'true') | (df2['y'] == 'false')]\n",
|
|||
|
|
" print(set(df2.y), len(df2))\n",
|
|||
|
|
"\n",
|
|||
|
|
" df3=pd.concat([df1, df2], ignore_index=True)\n",
|
|||
|
|
"\n",
|
|||
|
|
" print(df3['y'].value_counts())\n",
|
|||
|
|
" print('done')\n",
|
|||
|
|
" concat_X_train, concat_X_test, concat_y_train, concat_y_test = train_test_split(df3['claim'], df3['y'], test_size=0.25, random_state=4222)\n",
|
|||
|
|
" \n",
|
|||
|
|
" \n",
|
|||
|
|
"except Exception as e:\n",
|
|||
|
|
" print(e)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Vectorizer Classifiers\n",
|
|||
|
|
"* tfids removes words in pregenerating the vectors by evaluating if this word appears more than 70% often in all articles\n",
|
|||
|
|
"* an immense naive approach would be to store a set of the over all occuring word in all the texts and for each text determining how often this word occurs.\n",
|
|||
|
|
"* also testing some min df thresholds as lower bound for regarded occurences"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"### Generate Vectorizer on Binary Classes"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 18,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"bin_count_vectorizer = CountVectorizer(stop_words='english')\n",
|
|||
|
|
"bin_count_train = bin_count_vectorizer.fit_transform(bin_X_train)\n",
|
|||
|
|
"bin_count_test = bin_count_vectorizer.transform(bin_X_test)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 19,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"bin_tfidf_vectorizer = TfidfVectorizer(stop_words='english', max_df=0.7, min_df=0.0005)\n",
|
|||
|
|
"bin_tfidf_train = bin_tfidf_vectorizer.fit_transform(bin_X_train)\n",
|
|||
|
|
"bin_tfidf_test = bin_tfidf_vectorizer.transform(bin_X_test)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"a short look on the last 10 tokens for the vectors"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 20,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
"['zoomed',\n",
|
|||
|
|
" 'zooming',\n",
|
|||
|
|
" 'zor',\n",
|
|||
|
|
" 'zucker',\n",
|
|||
|
|
" 'zuckerberg',\n",
|
|||
|
|
" 'zuesse',\n",
|
|||
|
|
" 'zurich',\n",
|
|||
|
|
" 'zwick',\n",
|
|||
|
|
" 'état',\n",
|
|||
|
|
" 'œthe']"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 20,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"bin_tfidf_vectorizer.get_feature_names()[-10:]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 21,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
"['تنجح', 'حلب', 'عن', 'لم', 'ما', 'محاولات', 'من', 'هذا', 'والمرضى', 'ยงade']"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 21,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"bin_count_vectorizer.get_feature_names()[-10:]#[:10]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"### Generate Vectorizer on Multilabel Classes"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 22,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"mul_count_vectorizer = CountVectorizer(stop_words='english')\n",
|
|||
|
|
"mul_count_train = mul_count_vectorizer.fit_transform(mul_X_train)\n",
|
|||
|
|
"mul_count_test = mul_count_vectorizer.transform(mul_X_test)\n",
|
|||
|
|
"mul_count_valid = mul_count_vectorizer.transform(mul_X_valid)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 49,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"mul_tfidf_vectorizer = TfidfVectorizer(stop_words='english', max_df=0.7)#, min_df=0.0005)\n",
|
|||
|
|
"mul_tfidf_train = mul_tfidf_vectorizer.fit_transform(mul_X_train)\n",
|
|||
|
|
"mul_tfidf_test = mul_tfidf_vectorizer.transform(mul_X_test)\n",
|
|||
|
|
"mul_tfidf_valid = mul_tfidf_vectorizer.transform(mul_X_valid)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 50,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
"['zip',\n",
|
|||
|
|
" 'zippo',\n",
|
|||
|
|
" 'zombie',\n",
|
|||
|
|
" 'zombies',\n",
|
|||
|
|
" 'zone',\n",
|
|||
|
|
" 'zones',\n",
|
|||
|
|
" 'zoning',\n",
|
|||
|
|
" 'zoo',\n",
|
|||
|
|
" 'zuckerberg',\n",
|
|||
|
|
" 'zuckerbergs']"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 50,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"mul_tfidf_vectorizer.get_feature_names()[-10:]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 51,
|
|||
|
|
"metadata": {
|
|||
|
|
"scrolled": true
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
"['zip',\n",
|
|||
|
|
" 'zippo',\n",
|
|||
|
|
" 'zombie',\n",
|
|||
|
|
" 'zombies',\n",
|
|||
|
|
" 'zone',\n",
|
|||
|
|
" 'zones',\n",
|
|||
|
|
" 'zoning',\n",
|
|||
|
|
" 'zoo',\n",
|
|||
|
|
" 'zuckerberg',\n",
|
|||
|
|
" 'zuckerbergs']"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"execution_count": 51,
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "execute_result"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"mul_count_vectorizer.get_feature_names()[-10:]#[:10]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Confusion Matrix Code\n",
|
|||
|
|
"copy paste by distributed notebook"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 32,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_confusion_matrix(cm, classes,\n",
|
|||
|
|
" normalize=False,\n",
|
|||
|
|
" title='Confusion matrix',\n",
|
|||
|
|
" cmap=plt.cm.Blues):\n",
|
|||
|
|
" \"\"\"\n",
|
|||
|
|
" See full source and example: \n",
|
|||
|
|
" http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
|
|||
|
|
" \n",
|
|||
|
|
" This function prints and plots the confusion matrix.\n",
|
|||
|
|
" Normalization can be applied by setting `normalize=True`.\n",
|
|||
|
|
" \"\"\"\n",
|
|||
|
|
" #added after jonas hint\n",
|
|||
|
|
" fig_1,ax_1 = plt.subplots()\n",
|
|||
|
|
" \n",
|
|||
|
|
" plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
|
|||
|
|
" plt.title(title)\n",
|
|||
|
|
" plt.colorbar()\n",
|
|||
|
|
" tick_marks = np.arange(len(classes))\n",
|
|||
|
|
" plt.xticks(tick_marks, classes, rotation=45)\n",
|
|||
|
|
" plt.yticks(tick_marks, classes)\n",
|
|||
|
|
"\n",
|
|||
|
|
" if normalize:\n",
|
|||
|
|
" cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
|
|||
|
|
" print(\"Normalized confusion matrix\")\n",
|
|||
|
|
" else:\n",
|
|||
|
|
" print('Confusion matrix, without normalization')\n",
|
|||
|
|
"\n",
|
|||
|
|
" thresh = cm.max() / 2.\n",
|
|||
|
|
" for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
|
|||
|
|
" plt.text(j, i, cm[i, j],\n",
|
|||
|
|
" horizontalalignment=\"center\",\n",
|
|||
|
|
" color=\"white\" if cm[i, j] > thresh else \"black\")\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.tight_layout()\n",
|
|||
|
|
" plt.ylabel('True label')\n",
|
|||
|
|
" plt.xlabel('Predicted label')"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Configurations"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Configuration 1\n",
|
|||
|
|
"* model a - train - [performance measures][0:4]\n",
|
|||
|
|
"* model a - test - [performance measures][0:4]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"* model a = MultinomialNB\n",
|
|||
|
|
" * with tfidf vectorizer* dataset 1\n",
|
|||
|
|
" * in contrast to the notebook with count vect.\n",
|
|||
|
|
" * with dataset 1: fake_or_real_news.csv\n",
|
|||
|
|
" * ** Take care with seeds for example split train test data function**\n",
|
|||
|
|
" "
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 33,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf = MultinomialNB()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 34,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"accuracy: 0.914\n",
|
|||
|
|
"Confusion matrix, without normalization\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVYAAAEmCAYAAAA5jbhCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcFNXZ9vHfNYOgCCqCuOCCC2qM\nUdzQaIwaFQEX0LgbtxjQRLM+MSFGozEazeZuNBp3n7g8rzFRXBD3JW6IuEUUVFQUWRURAWG43z+q\nBpuBmW6gerp6+vrmU5+ZPnW66m7G3HPmrlOnFBGYmVl26iodgJlZW+PEamaWMSdWM7OMObGamWXM\nidXMLGNOrGZmGXNitWZJWknS3ZJmSPq/5TjOUZIeyDK2SpG0q6Q3Kh2H5Zs8j7X6SToS+BmwOTAT\nGA2cGxFPLudxjwZ+COwcEfOXO9CckxRAr4gYV+lYrLp5xFrlJP0MuAj4PbAmsD7wV2BgBoffAHiz\nFpJqKSS1q3QMViUiwluVbsCqwGfAIS306UCSeD9Mt4uADum+3YEJwP8Ak4GJwPHpvt8CXwDz0nOc\nAJwF3Fxw7J5AAO3S18cBb5OMmt8Bjipof7LgfTsDzwMz0q87F+x7FPgd8FR6nAeAbs18tsb4f1EQ\n/yBgAPAmMB04raB/H+Bp4JO072VA+3Tf4+lnmZV+3sMKjv9L4CPgpsa29D0bp+fYNn29DjAV2L3S\n/214q+zmEWt1+zqwInBnC31+DewE9Aa2JkkupxfsX4skQfcgSZ6XS+oSEWeSjIJvi4hOEXFNS4FI\nWhm4BOgfEZ1JkufoJfRbHbgn7dsVuAC4R1LXgm5HAscD3YH2wM9bOPVaJP8GPYDfAFcD3wG2A3YF\nfiNpo7RvA/BToBvJv92ewA8AIuKbaZ+t0897W8HxVycZvQ8pPHFEvEWSdP9XUkfgOuD6iHi0hXit\nBjixVreuwNRo+U/1o4CzI2JyREwhGYkeXbB/Xrp/XkTcSzJa22wZ41kAbClppYiYGBGvLaHPvsDY\niLgpIuZHxC3AGGD/gj7XRcSbETEbuJ3kl0Jz5pHUk+cBt5IkzYsjYmZ6/teArQAi4oWIeCY973jg\nb8BuJXymMyNibhrPIiLiamAs8CywNskvMqtxTqzVbRrQrUjtbx3g3YLX76ZtC4/RJDF/DnRa2kAi\nYhbJn88nARMl3SNp8xLiaYypR8Hrj5YinmkR0ZB+35j4JhXsn934fkmbShom6SNJn5KMyLu1cGyA\nKRExp0ifq4EtgUsjYm6RvlYDnFir29PAHJK6YnM+JPkzttH6aduymAV0LHi9VuHOiBgeEXuTjNzG\nkCScYvE0xvTBMsa0NK4giatXRKwCnAaoyHtanDYjqRNJ3foa4Ky01GE1zom1ikXEDJK64uWSBknq\nKGkFSf0l/THtdgtwuqQ1JHVL+9+8jKccDXxT0vqSVgV+1bhD0pqSDkhrrXNJSgoNSzjGvcCmko6U\n1E7SYcAWwLBljGlpdAY+BT5LR9Pfb7J/ErDRYu9q2cXACxHxPZLa8ZXLHaVVPSfWKhcRF5DMYT0d\nmAK8D5wC/Cvtcg4wEngZeAUYlbYty7lGALelx3qBRZNhHcnsgg9JrpTvRnphqMkxpgH7pX2nkVzR\n3y8ipi5LTEvp5yQXxmaSjKZva7L/LOAGSZ9IOrTYwSQNBPqRlD8g+TlsK+mozCK2quQbBMzMMuYR\nq5lZxpxYzcwy5sRqZpYxJ1Yzs4y12UUl1KFz1HUqNvfb8mSL9bpUOgRbSh+8/x7Tp00tNhd4qdSv\nskHE/MVucltMzJ4yPCL6ZXnurLTZxFrXqRud9jm70mHYUvj3xQdXOgRbSgP33iXzY8b82XTYrOhs\nN+aMvjy3I6c2m1jNrFoJVN1VSidWM8sXAXX1lY5iuTixmln+KNOybatzYjWznHEpwMwsex6xmpll\nSHjEamaWLXnEamaWOc8KMDPLki9emZllS7gUYGaWOY9Yzcyy5FKAmVn26lwKMDPLjtcKMDPLmksB\nZmbZ86wAM7OMecRqZpYh+ZZWM7PsecRqZpYleVaAmVnmXAowM8uQ12M1M8ua57GamWXPpQAzs4z5\n4pWZWYbkUoCZWfZcCjAzy5acWM3MspM8maW6E2t1FzLMrO1RiVuxw0jrSXpE0uuSXpP047R9dUkj\nJI1Nv3ZJ2yXpEknjJL0saduCYx2b9h8r6dhi53ZiNbOcEXV1dUW3EswH/icivgLsBJwsaQtgKPBQ\nRPQCHkpfA/QHeqXbEOAKSBIxcCawI9AHOLMxGTfHidXMckdS0a2YiJgYEaPS72cCrwM9gIHADWm3\nG4BB6fcDgRsj8QywmqS1gX2AERExPSI+BkYA/Vo6t2usZpY7JdZYu0kaWfD6qoi4qpnj9QS2AZ4F\n1oyIiZAkX0nd0249gPcL3jYhbWuuvVlOrGaWLyXWUIGpEbF90cNJnYA7gJ9ExKctJO0l7YgW2pvl\nUoCZ5YooXgYoddaApBVIkur/RsQ/0+ZJ6Z/4pF8np+0TgPUK3r4u8GEL7c1yYjWz3MkisSrpdA3w\nekRcULDrLqDxyv6xwL8L2o9JZwfsBMxISwbDgb6SuqQXrfqmbc1yKcDMcqfEq/7F7AIcDbwiaXTa\ndhpwPnC7pBOA94BD0n33AgOAccDnwPEAETFd0u+A59N+Z0fE9JZO7MRqZvlSeo21RRHxZAtH2nMJ\n/QM4uZljXQtcW+q5nVjNLHeq/c4rJ1Yzy5XGi1fVzInVzHLHidXMLGvVnVedWM0sZ5TZrICKcWI1\ns9xxKcDMLEO+eGVmVg7VnVedWPPgsiFfp9826zLl0zl8/Zd3A7Dl+l248IQdWblDO96bOovBlz/J\nzNnzWKG+jou+tyPbbNiVBREMvXEkT74+CYAzDu3N4btuxGort6fHd2+t5EeqKXPnzOHwgXvzxdwv\naGiYT7/9BvGTX57BqT8cwnNPP0HnzqsA8MdLrmKLr2298H0vvziSb/ffnUuuvon++x9YqfDzR9Vf\nCqjuCnEb8Y/H3+Lbf3hokbZLB+/EWbeMYuehwxj2/Hv8aL8tADj2W5sAsPPQYQw67yHO/c52C5+7\ndt+oCXzrjPtaNXaD9h06cPMd93HPo89y98PP8PgjI3hx5HMADD3z9wx75FmGPfLsIkm1oaGBP/zu\nDHbdY69KhZ1rWS3CUilOrDnwnzGT+fizuYu0bbL2Kjw1Jll055FXJnLADusDsHmP1Xjs1Y8AmPrp\nHGbM+oJtNuoKwMhxU5n0yexWjNwgSQIrd+oEwPx585g/b17Rh4ze+Pcr6LfvQLp2695yxxqlOhXd\n8syJNaden/AJA7ZbF4BBO21Aj64rA/Dqex+z7/brUV8nNlijE1tv2JV1V1+5kqEayQh0vz12pM8W\nG7DLbnvSe7s+APzl92cxYLc+nHPGL5g7N/nl+dHED3jg3rs48rjBlQw51zxibYakBkmjC7aeBfsu\nlvSBpLqCtuMkXZZ+XyfpBknXpkt4jZf0SsGxLilX3Hlx8lVPM3jvzXjs3AF0WnEF5s1fAMBNj47j\ng2mf8+g5Azjv6O15buwU5i9YUOForb6+nmGPPMtTL43lpRdH8sbrr3Hq6b9lxH9Gc+cDT/DJxx9z\n1aV/AeCc03/BL844h/r6+gpHnU+lJNW8J9ZyXryaHRG9mzamyfRAkkcdfBN4tMl+AVcCKwDHR0Sk\n/4h7RMTUMsabK2M//JQDz0/qrhuv1Zl9tkmeBNGwIDjt5i+fRvHAWfvw1kczKxKjLW6VVVdjp513\n5fGHRzD45J8A0KFDBw4+4mj+/teLAHjlpVH8+MRjAPh42jQefWg49fX19B1wQMXizpu8J85iKlEK\n2AN4leQJiEcsYf/FQFfgmIio2aFYt1VWBECCUw/8Gtc++CYAK7Wvp2OH5PfhHluuzfyG4I0PZlQs\nToNpU6fw6YxPAJgzezZ
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x10b9b55f8>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(bin_tfidf_train, bin_y_train)\n",
|
|||
|
|
"pred = clf.predict(bin_tfidf_train)\n",
|
|||
|
|
"score = metrics.accuracy_score(bin_y_train, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(bin_y_train, pred, labels=['FAKE', 'REAL'])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['FAKE', 'REAL'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 35,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"accuracy: 0.872\n",
|
|||
|
|
"Confusion matrix, without normalization\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEmCAYAAAA0k8gFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XecXFX9//HXe9MoAUJIgyQQSij+\nQAIJCKhIEwgCQaUjNRJBkC+CBRHbV1DUr3SkiRBQKV9pMSDFACJ8aQmEJiUJBFgSSEIJJZSQfH5/\n3LNh2OzOzE5md+7svp8+7mNnzj1z7md25ZNz7rn3XEUEZmbWdg21DsDMrF45gZqZVcgJ1MysQk6g\nZmYVcgI1M6uQE6iZWYWcQK1VkpaX9HdJ8yX97zK0c5Ck26sZW61I+qKkZ2sdh+WDfB1o/ZN0IHAC\nsCHwDjAVOC0i7l3Gdg8GvgNsExEfL3OgOScpgOERMb3WsVh9cA+0zkk6ATgL+BUwEFgT+AMwpgrN\nrwU81xWSZzkkda91DJYzEeGtTjdgFeBdYJ8idXqRJdhZaTsL6JX2bQc0AicCc4DZwOFp3y+Aj4CF\n6RhjgZ8Dfy5oexgQQPf0/jDgebJe8AvAQQXl9xZ8bhvgYWB++rlNwb67gV8C96V2bgf6tfLdmuL/\nQUH8ewG7Ac8BbwAnF9TfErgfeCvVPQ/omfbdk77Le+n77lfQ/g+BV4Erm8rSZ9ZNx9g8vV8DmAds\nV+v/b3jrmM090Pq2NbAccEOROj8GtgJGAJuSJZFTCvYPIkvEg8mS5PmSVo2In5H1aq+JiN4RcWmx\nQCStCJwDjI6IlciS5NQW6vUFbk51VwPOAG6WtFpBtQOBw4EBQE/ge0UOPYjsdzAY+ClwCfANYCTw\nReCnktZJdRcB3wX6kf3udgS+DRAR26Y6m6bve01B+33JeuPjCg8cETPIkutfJK0AXAZcHhF3F4nX\nOhEn0Pq2GjAvig+xDwL+OyLmRMRcsp7lwQX7F6b9CyPiFrLe1wYVxrMY2FjS8hExOyKeaqHOV4Bp\nEXFlRHwcEVcBzwB7FNS5LCKei4j3gWvJkn9rFpKd710IXE2WHM+OiHfS8Z8CPgsQEVMi4oF03JnA\nRcCXyvhOP4uID1M8nxIRlwDTgAeB1cn+wbIuwgm0vr0O9Ctxbm4N4MWC9y+msiVtNEvAC4DebQ0k\nIt4jG/YeBcyWdLOkDcuIpymmwQXvX21DPK9HxKL0uinBvVaw//2mz0taX9JESa9Kepush92vSNsA\ncyPigxJ1LgE2Bs6NiA9L1LVOxAm0vt0PfEB23q81s8iGn03WTGWVeA9YoeD9oMKdEXFbRHyZrCf2\nDFliKRVPU0yvVBhTW1xAFtfwiFgZOBlQic8UvUxFUm+y88qXAj9Ppyisi3ACrWMRMZ/svN/5kvaS\ntIKkHpJGS/ptqnYVcIqk/pL6pfp/rvCQU4FtJa0paRXgR007JA2UtGc6F/oh2amARS20cQuwvqQD\nJXWXtB/wGWBihTG1xUrA28C7qXd8dLP9rwHrLPWp4s4GpkTEN8nO7V64zFFa3XACrXMRcQbZNaCn\nAHOBl4FjgRtTlVOBycDjwBPAI6mskmPdAVyT2prCp5NeA9ls/iyymekvkSZomrXxOrB7qvs62Qz6\n7hExr5KY2uh7ZBNU75D1jq9ptv/nwHhJb0nat1RjksYAu5KdtoDs77C5pIOqFrHlmi+kNzOrkHug\nZmYVcgI1M6uQE6iZWYWcQM3MKtRpF0dQzxVDy/uSvHqy0bD+tQ7B2mjWyy/x5hvzSl1L2ybdVl4r\n4uOlbvpaSrw/97aI2LWax26rzptAl+9Lr61PqHUY1gbXXnpkrUOwNtp3t21LV2qj+Ph9em1Q8ioy\nPph6fqm7yNpdp02gZlavBKqPs4tOoGaWLwIautU6irLUR5o3s65FKr2VbEIbSJpasL0t6XhJfSXd\nIWla+rlqqi9J50iaLulxSZuXOoYTqJnlTBrCl9pKiIhnI2JERIwgWx92AdnauScBkyJiODApvQcY\nDQxP2ziyxWeKcgI1s/ypQg+0mR2BGRHxItnjbsan8vF8sprZGOCKyDwA9JG0erFGfQ7UzPJFlDuJ\n1E/S5IL3F0fExa3U3Z9sZTKAgRExGyAiZksakMoHky3G06Qxlc1uLQAnUDPLmbJ7mPMiYlTJ1qSe\nwJ4ULL/Y+oGXUnS1JSdQM8uf6s7CjwYeiYimJxW8Jmn11PtcneyBhJD1OIcWfG4IJRYf9zlQM8uZ\n6kwiFTiAT4bvABOAQ9PrQ4GbCsoPSbPxWwHzm4b6rXEP1MzyRVQySdRyU9nTUr8MfKug+HTgWklj\ngZeAfVL5LWSPxJ5ONmN/eKn2nUDNLH+qdCdSRCwge3ptYdnrZLPyzesGcExb2ncCNbOc8a2cZmaV\na6jqAk/txgnUzPKlju6FdwI1s5zxEN7MrHJVmoVvb06gZpY/7oGamVWgssVCasIJ1Mzyxz1QM7NK\nyLPwZmYV8xDezKwC5a8HWnNOoGaWM74O1Mysch7Cm5lVyJNIZmYVkIfwZmaV8xDezKwycgI1M2u7\n7IkeTqBmZm0nWn7AcA45gZpZzoiGBk8imZlVxEN4M7MKOYGamVWijs6B1seJBjPrMoSQSm9ltSX1\nkfQ3Sc9IelrS1pL6SrpD0rT0c9VUV5LOkTRd0uOSNi/VvhOomeVOtRIocDZwa0RsCGwKPA2cBEyK\niOHApPQeYDQwPG3jgAtKNe4Eama509DQUHIrRdLKwLbApQAR8VFEvAWMAcanauOBvdLrMcAVkXkA\n6CNp9aJxVvb1zMzaicrcSlsHmAtcJulRSX+UtCIwMCJmA6SfA1L9wcDLBZ9vTGWtcgI1s9wpcwjf\nT9Lkgm1cs2a6A5sDF0TEZsB7fDJcb/GwLZRFsTg9C29mudI0iVSGeRExqsj+RqAxIh5M7/9GlkBf\nk7R6RMxOQ/Q5BfWHFnx+CDCrWADugZpZ7lRjEikiXgVelrRBKtoR+A8wATg0lR0K3JReTwAOSbPx\nWwHzm4b6rXEP1Mzyp3rXgX4H+IuknsDzwOFkHcdrJY0FXgL2SXVvAXYDpgMLUt2inEDNLF9E1e6F\nj4ipQEvD/B1bqBvAMW1p3wnUzHLHt3KamVWgDZNINecEamb5Ux/50wk0b1ZZsRcXHP9lPjNsNSKC\no868g51GrsURu27C3PkLAPjZ5fdx28Mz6dG9gfOO24nNhw9kcQTfu/Bu/v14Y42/QddzyolHc88/\nb6Vvv/7cOOmhJeV/+dOFXHX5RXTr3p1td9iFE085FYBLzvsfrr/qSrp1a+BH//07Pr/dTrUKPZ/k\nIbxV6H+O2o7bp8zkwNMm0qN7Ayv06sFOI9fi3Bse4azrpnyq7hGjNwFgi6OvpP8qy3PjqV/lC8f9\nlSh66a9V2177HMSBh32Lk4//5Druh+67h7tuv5nr73iAnr168fq8uQDMeO4Z/nHTddx050PMeW02\n3zxgT26+51G6dauPx/h2lHpJoL4ONEdWWqEnX9hkMJff+iQACz9ezPz3Pmy1/oZr9uWuqS8BMHf+\n+8x/90NGDh/YIbHaJ0Zt9QVW6bPqp8quufKPjD3mBHr26gXAav36A3Dn7RMZPebr9OzViyFrDmPN\nYevwxNTJHR5z3qlBJbc8cALNkbUHrcK8+e9z8Yk7c/95B/GH43dihV7ZIOGoPTfloQu+wYXf/TJ9\nemf/UT7x/Dz22HpdujWItQauzGbDBzCk/0q1/AqWzHx+OlMe/D8O2H17Dvv6rjwxNRs9zJk9m0Gr\nD1lSb+CgNZgzu+i12l1SFVdjalftlkAlLZI0tWAbVrDvbEmvSGooKDtM0nnpdYOk8ZL+lO4KmCnp\niYK2zmmvuGupe7cGRqw3gEsmPs7Wx/6FBR98zPf224JLJj7OZw6/jM99+8+8+sZ7nH7ktgCMv+1J\nXpn7LvedeyC/O2o7Hvj
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x114164898>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(bin_tfidf_train, bin_y_train)\n",
|
|||
|
|
"pred = clf.predict(bin_tfidf_test)\n",
|
|||
|
|
"score = metrics.accuracy_score(bin_y_test, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(bin_y_test, pred, labels=['FAKE', 'REAL'])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['FAKE', 'REAL'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Configuration 2\n",
|
|||
|
|
"* model b - train - [performance measures]\n",
|
|||
|
|
"* model b - validation - [performance measures]\n",
|
|||
|
|
"* model b - test - [performance measures]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 53,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf = MultinomialNB()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 58,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"accuracy: 0.602\n",
|
|||
|
|
"Confusion matrix, without normalization\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAV4AAAEmCAYAAAAqWvi2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsnXd8VMXXh5+TBKL0DqG3UKTX0KU3\n4QcISFMsKIJd6UVA6UhRaSqKCiLiKyq99y4JXToCEnqHUJNw3j/uTdyEJBtCdpPVefzMJ/eeO3fm\nu7t4dvbM3DmiqhgMBoPBfXgltQCDwWD4r2Ecr8FgMLgZ43gNBoPBzRjHazAYDG7GOF6DwWBwM8bx\nGgwGg5sxjtfgVkTkSRFZICLXReT/HqOdTiKyPDG1JRUiUlNEDiW1DoP7ELOO1xATItIR+AAoBtwE\ndgHDVXXjY7b7AvA2UE1Vwx5baDJHRBTwV9WjSa3FkHwwI17DQ4jIB8CnwAggO5AXmAK0SITm8wGH\n/wtONz6IiE9SazAkAapqiimRBUgPhABt46jji+WYz9jlU8DXvlYbCAZ6ABeAs8DL9rWPgPtAqN1H\nF2AI8IND2/kBBXzs85eAv7BG3ceBTg72jQ73VQO2A9ftv9Ucrq0FhgKb7HaWA1lieW0R+ns76G8J\nNAUOA1eA/g71KwNbgGt23UlASvvaevu13LJfbzuH9vsA54CZETb7nkJ2H+Xt85zAJaB2Uv/bMCXx\nihnxGqJTFXgC+C2OOgOAKkBZoAyW8xnocD0HlgPPheVcJ4tIRlUdjDWKnqOqaVT1m7iEiEhq4HOg\niaqmxXKuu2KolwlYZNfNDIwHFolIZodqHYGXgWxASqBnHF3nwHoPcgGDgGnA80AFoCYwSEQK2nXD\ngfeBLFjvXT3gDQBVrWXXKWO/3jkO7WfCGv13dexYVY9hOeVZIpIK+Bb4TlXXxqHX4GEYx2uITmbg\nksYdCugEfKyqF1T1ItZI9gWH66H29VBVXYw12iuaQD0PgJIi8qSqnlXVP2Oo8wxwRFVnqmqYqs4G\nDgLNHep8q6qHVfUO8DPWl0ZshGLFs0OBn7Cc6meqetPu/0+gNICqBqnqVrvfE8CXwNPxeE2DVfWe\nrScKqjoNOAJsA/ywvugM/yKM4zVE5zKQxUnsMSdw0uH8pG2LbCOa474NpHlUIap6C+vneTfgrIgs\nEpFi8dAToSmXw/m5R9BzWVXD7eMIx3je4fqdiPtFpIiILBSRcyJyA2tEnyWOtgEuqupdJ3WmASWB\niap6z0ldg4dhHK8hOluAu1hxzdg4g/UzOYK8ti0h3AJSOZzncLyoqstUtQHWyO8glkNypidC0+kE\nanoUpmLp8lfVdEB/QJzcE+dSIhFJgxU3/wYYYodSDP8ijOM1REFVr2PFNSeLSEsRSSUiKUSkiYiM\nsavNBgaKSFYRyWLX/yGBXe4CaolIXhFJD/SLuCAi2UXkf3as9x5WyCI8hjYWA0VEpKOI+IhIO+Ap\nYGECNT0KaYEbQIg9Gu8e7fp5oOBDd8XNZ0CQqr6KFbv+4rFVGpIVxvEaHkJVx2Ot4R0IXAROAW8B\nv9tVhgGBwB5gL7DDtiWkrxXAHLutIKI6Sy+s1RFnsGb6n8aeuIrWxmWgmV33MtaKhGaqeikhmh6R\nnlgTdzexRuNzol0fAnwvItdE5DlnjYlIC6AxVngFrM+hvIh0SjTFhiTHPEBhMBgMbsaMeA0Gg8HN\nGMdrMBgMbsY4XoPBYHAzxvEaDAaDmzEbdLgI7yfTq0/6bEktI0ZK5Eqf1BJi5V7og6SWECc+3s6W\n6CYd3l7JU9vfJ09w6dKlRBPnnS6fathDD/w9hN65uExVGydWv4mJcbwuwid9NnJ3+iypZcTIxpFN\nklpCrBw9H5LUEuIkc5qUSS0hVjIlU23Vq1RK1PY07A6+RZ2uzOPursnOniBMMozjNRgMnoUIeHkn\ntYrHwjheg8HgeYhnT08Zx2swGDwPSZ7x7PhiHK/BYPAwTKjBYDAY3ItgQg0Gg8HgXsTjQw2e/bVh\nMBj+m4iX8+KsCZHpInJBRPY52OaIyC67nBCRXbY9v4jccbj2hcM9FURkr4gcFZHPRZx/K5gRr8Fg\n8DASLcb7HVZy0hkRBlVtF9mLyDis5KkRHFPVmFJGTcXKnbcVa2/oxsCSuDo2I16DweBZCFaowVlx\ngqqux9rn+eEurFHrc1ib/scuRcQPSKeqW9TaY3cGcWdvAYzjNRgMnkj8Qg1ZRCTQoXR11qwDNYHz\nqnrEwVZARHaKyDoRqWnbcgHBDnWCiZrrL0ZMqMFgMHgYAt7xCjVcUtWKCeykA1FHu2eBvKp6WUQq\nAL+LSAlizq/nNLuEGfEmEaPaleKPIfVY0rNmpO39xv4s6lGDBR/U4LuulciWzheANE/48NUrFVjY\nowZLetWkdaXcABTPmZb/e7sqS3rVZFGPGjxT1s+lmu/evUut6gEEVCxLxbIlGfbx4CjXe7z3Ntky\npXWpBkfu3b1Lx+a1aduoGq3qVWbKuOEAqCoTx3xM86fL0bJuRWZNnxp5z/YtG3iucXVa1avMK21d\nt2dFj7e6UqZIHupVKx9p+3Pvbpo3qEXDWpVpWrcaO4O2A3Dt2lW6vPAc9WtU5Jn6NTi4P6YM9q4h\n+NQpGjeoS7lST1GhTEkmT4y6v8in48eSKqUXly65I4tSPIlYTvaYk2uxNm9l2H4WhzROqnrPTjGF\nqgYBx4AiWCPc3A635yYeiV/NiDeJmLs9mJkbTzK2Q5lI27Q1x5mw1Ppl82KNfLzdwJ8P5+7jher5\nOHo+hK7Tg8iUOiUr+tZi/o7T3Ln/gF6zd3Pi0m2ypfNl3vvVWX/wIjfvhsXW7WPh6+vL4mWrSJMm\nDaGhodSvU5OGjZpQOaAKO4ICuXb9uvNGEpGUvr58/dNCUqW29LzUuiE16jTgr6OHOXcmmHlrgvDy\n8uLypYsA3Lh+jREDPmDKzF/xy5Un0u4K2nZ8gZde68573btE2oYP7s/7vQdQt0EjVq1YyvAh/fll\nwQomjh9DiZKl+Wbmzxw9fIgBvd9lzu9LXabNEW8fH0aOGUu5cuW5efMm1QMqUrdeA4o/9RTBp06x\netVK8uTN6xYtj4Rrl5PVBw6qamQIQUSyAldUNVxECgL+wF+qekVEbopIFWAb0BmY6KwDM+JNIrb/\ndZVrt0Oj2ELu/eMwn0zpg9q/WFQhta/1HZnK15vrt0MJe6CcuHSLE5duA3Dhxj0uh9x36e5ZIkKa\nNGkACA0NJTQ0FBEhPDycAf16M2zEaJf1HZueVKktPWFhoYSFhYEIP8/8mtff64OXl/XPO3OWrAAs\nmfd/1GvSHL9ceaLYXUGVajXJkDHjQ3pDbt4A4OaN62TPYf1COXLoADWergNA4SJFCf77JBcvnHeZ\nNkf8/PwoV84aladNm5aixYpz5sxpAHr3/IBhI0YTj9VRbsZe1eCsOGtFZDawBSgqIsEiEvEt2Z6H\nJ9VqAXtEZDfwC9BNVSMm5roDXwNHsUbCca5oADPiTXb0aFKEVhVzcfNOGJ2mbgNg5qYTfPVKRbYM\nrktqXx/embmT6DlKS+dJTwpvL05evu1SfeHh4VSvUpG/jh2la7c3qFQ5gMkTP6PpM83x83NtqCM2\nPR2eqcXfJ/6iXefXKF2uEsEnj7Nswa+sXrqQjJkz0+ejMeQrUJiTfx0lLCyULs815VZICJ1e6Ubz\nNh3dpnXIiLF0atOMoYP68kCVeUvXAPBUyVIsWTCPylWqszNoO8Gn/ubsmdNkzZbdbdoATp44we7d\nO6lUOYCFC+aTM1dOSpcp4/zGpCARnlxT1Q6x2F+KwTYXmBtL/UCg5KP0/a8f8YrIOyJyQERmxXK9\ntogsjOlaUjBuyWFqDF3DvB1neKFGPgBqFs3K/tM3qPrRapqP28iQViVI4/vPd2bWtL6M61iGPj/t\necghJzbe3t5s3b6Tw3+
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x1141259e8>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(mul_tfidf_train, mul_y_train)\n",
|
|||
|
|
"pred = clf.predict(mul_tfidf_train)\n",
|
|||
|
|
"\n",
|
|||
|
|
"#print(pred[:20])\n",
|
|||
|
|
"\n",
|
|||
|
|
"score = metrics.accuracy_score(mul_y_train, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(mul_y_train, pred, labels=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 59,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf = MultinomialNB()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 60,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"accuracy: 0.238\n",
|
|||
|
|
"Confusion matrix, without normalization\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAEmCAYAAAAnRIjxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xd8FFXXwPHfSUJvAULvTUCQIkhX\nioC9dxABUWyIXbG3R0WfVx57b4io2EVEihQFpBcpioAURXovCZBy3j/uBJeQZEPY3dmF8/WzH3Zn\nZmdONvHsnTN37hVVxRhjTOjF+R2AMcYcqyzBGmNMmFiCNcaYMLEEa4wxYWIJ1hhjwsQSrDHGhIkl\nWBNRIlJERL4TkZ0i8vlR7KeniIwLZWx+EZFTReQPv+MwoSfWD9ZkR0R6AHcCDYDdwALgKVWdepT7\n7QXcCrRT1bSjDjTKiYgC9VR1hd+xmMizFqw5jIjcCbwAPA1UAKoDrwEXhGD3NYBlx0NyzQsRSfA7\nBhNGqmoPexx8AKWAPcBluWxTCJeA13mPF4BC3rpOwFrgLmATsB7o6617HDgApHrH6Ac8BnwUsO+a\ngAIJ3us+wEpcK3oV0DNg+dSA97UDZgM7vX/bBaybDDwJTPP2Mw5IyuFny4z/3oD4LwTOBpYB24AH\nArZvBUwHdnjbvgIU9Nb97P0se72f94qA/d8HbACGZS7z3lPHO8bJ3uvKwBagk99/G/Y48oe1YE1W\nbYHCwNe5bPMg0AZoBjTFJZmHAtZXxCXqKrgk+qqIlFbVR3Gt4hGqWlxV380tEBEpBrwEnKWqJXBJ\ndEE225UBvve2LQsMAb4XkbIBm/UA+gLlgYLA3bkcuiLuM6gCPAK8DVwNtABOBR4RkdretunAHUAS\n7rM7HbgZQFVP87Zp6v28IwL2XwbXmu8feGBV/ROXfIeLSFHgfeADVZ2cS7wmSlmCNVmVBbZo7qfw\nPYEnVHWTqm7GtUx7BaxP9danqupoXOutfj7jyQAai0gRVV2vqkuy2eYcYLmqDlPVNFX9BFgKnBew\nzfuqukxVU4DPcF8OOUnF1ZtTgU9xyfNFVd3tHX8J0ARAVeeq6gzvuKuBN4GOefiZHlXV/V48h1DV\nt4HlwEygEu4LzcQgS7Amq61AUpDaYGVgTcDrNd6yg/vIkqCTgeJHGoiq7sWdVt8IrBeR70WkQR7i\nyYypSsDrDUcQz1ZVTfeeZybAjQHrUzLfLyIniMgoEdkgIrtwLfSkXPYNsFlV9wXZ5m2gMfCyqu4P\nsq2JUpZgTVbTgX24umNO1uFObzNV95blx16gaMDrioErVXWsqnbDteSW4hJPsHgyY/onnzEdiddx\ncdVT1ZLAA4AEeU+uXXdEpDiurv0u8JhXAjExyBKsOYSq7sTVHV8VkQtFpKiIFBCRs0TkOW+zT4CH\nRKSciCR523+Uz0MuAE4TkeoiUgq4P3OFiFQQkfO9Wux+XKkhPZt9jAZOEJEeIpIgIlcAJwKj8hnT\nkSgB7AL2eK3rm7Ks3wjUPuxduXsRmKuq1+Fqy28cdZTGF5ZgzWFUdQiuD+xDwGbgb2AA8I23yX+A\nOcBCYBEwz1uWn2ONB0Z4+5rLoUkxDtcbYR3uynpHvAtIWfaxFTjX23YrrgfAuaq6JT8xHaG7cRfQ\nduNa1yOyrH8MGCoiO0Tk8mA7E5ELgDNxZRFwv4eTRaRnyCI2EWM3GhhjTJhYC9YYY8LEEqwxxoSJ\nJVhjjAkTS7DGGBMmNtBEmBQsnqhFylbyO4xs1SpbNPhGPvlt7U6/Q8hVpSj+7BILF/A7hGyt/WsN\nW7duCdY3OM/iS9ZQTTvsBrjDaMrmsap6ZqiOmx+WYMOkSNlKtBv0gd9hZOujXi38DiFHTe79zu8Q\ncvXANc39DiFHF55YJfhGPujesU1I96dpKRSqH7THG/sWvBrsjrqwswRrjIktIhAX73cUeWIJ1hgT\neyQ2Lh9ZgjXGxB4JWUk3rCzBGmNijJUIjDEmPISYKRHERpTGGHOQuBJBsEewvYi8JyKbRGRxwLL/\nishSEVkoIl+LSGLAuvtFZIWI/CEiZ+QlUkuwxpjYI3HBH8F9gBu5LNB4oLGqNsHNwXY/gIicCFwJ\nNPLe85qIBK1TWII1xsQYrwYb7BGEqv6MGwYzcNm4gNk4ZgBVvecXAJ960/ysAlbg5qLLlSVYY0xs\nEfJaIkgSkTkBj/5B9pzVtcAP3vMquHGRM63l0CmJsmUXuYwxsSdvJYAtqtoyX7sXeRBIA4ZnLspm\ns6CDaVuCNcbEGIH48HXTEpHeuBkyTtd/ZyRYC1QL2KwqeZiHzhJslKmSWJj7u9U9+LpSycIMm72W\nBhWKUzWxMADFCyaw50AaAz5fnNNuwmLgTdcxbsxoksqVZ+qsBQBs37aN6/r04K+/1lC9eg3eHfoJ\niaVLRzSuTP0616FH+5oI8PG01bwz6U9OrFKSwVc1p2iheNZuS2bA+3PYsy+3GcnD44GLOlC4aHHi\n4uOIi0/ggfdHMnfC94x690U2rF7BoHe/oUbDJhGPC+D2W65n/JjRJJUrx08z3O/18YcGMX7MKAoU\nLEjNWrV54dV3KJWYGGRPERLGbloiciZwH9BRVZMDVo0EPhaRIbhZjOsBs4Ltz2qwUeafHfsY8Pli\nBny+mIFfLGZfWjq/rNzG4PErDi6funIbv6zcHvHYruzZmxFfHzqP4ItDnuO0jl2YveB3TuvYhReH\nPJfDu8OrfqUS9Ghfk3OenUy3pyfS9aSK1CpXjP9efTJPf7uYrk9N5IcF67mpaz1f4gO489WPeejD\n0Tzw/kgAKtepzw3PvE7dZkGvlYTVFT2u4ZMvD/29dux8OpNnLGDSL/OoXaceLw151qfochCablqf\n4GZRri8ia0WkH/AKbiLL8SKyQETeAFDVJcBnwG/AGOCWgKndc2QJNoo1q1KK9Tv3s2nPgUOWn1a3\nDJNXRGI+v0O163AqpUsfOoP0D99/xxU9ewFwRc9ejB41MuJxAdSrWIJ5q7axLzWd9AxlxvItnNms\nMnXKF2fG8q0ATFm6ibObV/YlvuxUqlmXijXq+B0GbdufethZR6fTu5GQ4E5wW5zSmvXrIjEDel6F\nrBfBVapaSVULqGpVVX1XVeuqajVVbeY9bgzY/ilVraOq9VX1h9z2nckSbBTrWLcMP63YesiyxpVK\nsD05lXU79/sU1aE2b95IxYpu3NuKFSuxZcsmX+JYun43beomUbpYQQoXiKdLo4pULl2EP9bvonsT\nF9+5zatQuXQRX+ITEV687Rqe7nMeU7752JcY8uuTjz6gS7c89auPnND0gw27Y74GKyIDcXPVz1PV\nw6Y+FpFOwN2qem6kY8tNQpzQumZp3p/59yHLO9Ure1jSNbBiw25eHb+MT25tz979afz2z07S05U7\nh83jycubcMfZ9Rm3cAOpaf7MonzPm1+QWK4Cu7Zt4cXbelGxRh3qNW/tSyxH4oX/PkNCQgKXXN7D\n71D+lccSQDQ45hMscDNwltc5OGa0rJ7In1uS2ZHy7wWZOIF2tcow8IvIXtzKTblyFdiwYT0VK1Zi\nw4b1JCWV9y2WT39Zw6e/rAFg0Pknsn5HCn9u3EOPl38BoHb54pzeuIIvsSWWc8ctWSaJZh3PYNVv\nv0Z9gh3x8YeMHzuaz0eORaItocXIYC/R0Y4OE69AXRsYKSL3icgvIjLf+7d+Ntt39ArbC7ztSnjL\n7xGR2d79yY9HIvZOdcsyefmhddbmVUuxdkcKW/YeyOFdkXfm2ecyYvgwAEYMH8ZZ55znWyxlixcE\noHLpIpzVrDLfzF57cJkI3HZWfYZNWR3xuPanJLNv756Dz3+fOYUqtQ/784sqE38cyysv/B9DP/2K\nokWjbZocsRJBNFDVG71uF52BA8DzqpomIl2Bp4FLsrzlbtzVwWkiUhzYJyLdcV0yWuE6iIwUkdO8\n2+wO4d0p0h+gcJmK+Y6
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x1147ef588>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(mul_tfidf_train, mul_y_train)\n",
|
|||
|
|
"pred = clf.predict(mul_tfidf_test)\n",
|
|||
|
|
"score = metrics.accuracy_score(mul_y_test, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(mul_y_test, pred, labels=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf = MultinomialNB()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 76,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"accuracy: 0.231\n",
|
|||
|
|
"Confusion matrix, without normalization\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAEmCAYAAAAnRIjxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsnXd4FFUXh9+ThNAChA5JqKEHBSH0\nIioiSFOpigiCIJ8ooqLyoWJXPjuKimABFWmKCkiVIkWkRIrSQ5OEXg2hhCzn+2MmYQlJdkl2k2y4\nL888mbn3zv2dSZazd85toqoYDAaDwfP4ZbcBBoPBkFsxDtZgMBi8hHGwBoPB4CWMgzUYDAYvYRys\nwWAweAnjYA0Gg8FLGAdryFJEJL+IzBKR0yIyPRP19BKRBZ60LbsQkRYisj277TB4HjHjYA2pISL3\nAU8CNYA4YAPwuqquyGS9vYHHgKaqmphpQ3M4IqJAVVWNzm5bDFmPacEarkJEngQ+AN4ASgPlgU+A\nzh6ovgKw43pwru4gIgHZbYPBi6iqOcyRfABFgDNAt3TK5MVywAfs4wMgr53XCogBngKOAAeBB+28\nl4EE4KKt0R94CfjWqe6KgAIB9nVfYDdWK3oP0MspfYXTfU2BtcBp+2dTp7ylwKvASrueBUCJNJ4t\nyf5nnOy/C7gT2AGcAEY4lW8IrAJO2WXHAIF23jL7WeLt5+3hVP+zwCHgm6Q0+55wW6OefR0CHANa\nZfdnwxzXfpgWrCElTYB8wI/plHkOaAzUBepgOZnnnfLLYDnqUCwn+rGIFFXVF7FaxVNVNUhVv0jP\nEBEpCHwItFPVQlhOdEMq5YoBv9hliwPvAb+ISHGnYvcBDwKlgEBgWDrSZbB+B6HASGA8cD9QH2gB\njBSRynZZB/AEUALrd3cb8AiAqra0y9Sxn3eqU/3FsFrzA52FVXUXlvOdJCIFgK+ACaq6NB17DTkU\n42ANKSkOHNP0X+F7Aa+o6hFVPYrVMu3tlH/Rzr+oqnOwWm/VM2jPJaC2iORX1YOqujmVMu2Bnar6\njaomqupkYBvQ0anMV6q6Q1XPAdOwvhzS4iJWvPkiMAXLeY5W1ThbfzNwI4CqRqnqH7buXuAz4GY3\nnulFVb1g23MFqjoe2AmsBspifaEZfBDjYA0pOQ6UcBEbDAH2OV3vs9OS60jhoM8CQddqiKrGY71W\nDwIOisgvIlLDDXuSbAp1uj50DfYcV1WHfZ7kAA875Z9Lul9EqonIbBE5JCL/YrXQS6RTN8BRVT3v\nosx4oDbwkapecFHWkEMxDtaQklXAeay4Y1ocwHq9TaK8nZYR4oECTtdlnDNVdb6q3o7VktuG5Xhc\n2ZNkU2wGbboWPsWyq6qqFgZGAOLinnSH7ohIEFZc+wvgJTsEYvBBjIM1XIGqnsaKO34sIneJSAER\nySMi7UTkLbvYZOB5ESkpIiXs8t9mUHID0FJEyotIEeC/SRkiUlpEOtmx2AtYoQZHKnXMAaqJyH0i\nEiAiPYBawOwM2nQtFAL+Bc7Yrev/pMg/DFS+6q70GQ1EqepDWLHlsZm20pAtGAdruApVfQ9rDOzz\nwFFgP/Ao8JNd5DVgHbAJ+Av4007LiNZCYKpdVxRXOkU/rNEIB7B61m/G7kBKUcdxoINd9jjWCIAO\nqnosIzZdI8OwOtDisFrXU1PkvwRMFJFTItLdVWUi0hloixUWAevvUE9EennMYkOWYSYaGAwGg5cw\nLViDwWDwEsbBGgwGg5cwDtZgMBi8hHGwBoPB4CXMQhNewr9AEc1TpHSW6YWXvOZx/Jli694jWaoX\nUrZoluoVLxCYpXoBfq6Gzvou+/bt5dixYx57QP/CFVQTr5oAdxV67uh8VW3rKd2MYBysl8hTpDQV\n+3yUZXrfPdI0y7QAGvf/NEv1HhvRJUv1+tQrl6V6wQWz1qFnJc0aRXq0Pk08R97qLke8cX7Dx65m\n1Hkd42ANBoNvIQJ+/tlthVsYB2swGHwP8Y3uI+NgDQaD7yG+EbM2DtZgMPgYJkRgMBgM3kHwmRCB\nb1iZS+nTvAKznmjGzKFNebfnjQQG+NGrSXnmD2vBtlF3EFwgj8e0Dh2I4aEe7bn71kjuad2QSV9+\nAsB7rz/PXbfWp9sdTXhi4H38e/pUhjXGPtWWfdMGs27cg8lpRQvlY/ao7vw1YQCzR3UnOCgvAB2a\nVGHNZ335Y2wfVnz8AE0jQtOq9pq45HAwemBHJowYAED0n7/z4cBOjB7QkU+H9OBY7F6P6KRk/Kcf\n0arJTdzcuC7jPvnQKxrOLJg/jxsjqhNRowpvvzUq1+mlj1ghAldHDsA42GyiVOG89G5anq4fraLT\nB7/j5ye0r1OGP/edpN8Xa4k96Xqc37Xg7x/AU8+/zo+L1/HNT4uY+vV4du3YRuMWt/D9gtVMn7+K\nCpWq8OUn72VY45sFf9N5xPdXpA3r0Yil6/dxQ9/xLF2/j2E9GwOwZP0+Gj48gcaDJjLonbl88qRn\nhiuunDGBUuWrJF//9MFIej73Ho+Pn0Xd2zqy+NtPPKLjzLYtm5n09ZfMWbSSRSvW8ev8OezetdPj\nOkk4HA6GDhnMz7Pmsn7TFqZPmczWLVtyjZ5biJ/rIweQM6y4TvH3E/Ll8cffT8ifx48j/15g64E4\nYk+6Wuz+2ilZugw1b7B2SSkYVIjKVapz5PABmra8jYAAK1J0400NOHww42tUr/wrhhNxV34xdGha\nlW8X/g3Atwv/pmPTqgDEn7+YXKZgvjzpr0DtJqePHmTbH0tpcKfzGEnh/NkzAJyPj6Nw8VIeULqS\nnTu2UT+yEQUKFCAgIIDGzVoyd/bPHtdJYu2aNYSHV6FS5coEBgbSrUdPZs/KPXqusWOwro4cgInB\nZhNH/r3Al8v3snh4Sy5cvMTKncdYufN4lmjH7t/Hts2buKHulQPAf5r2DXd0uMejWqWKFuDQiXgA\nDp2Ip2Tw5c0LOjWryiv9WlIyuAD3PP9DprVmffwa7R5+lgu2QwXoMuwNJvz3IQIC85KvYBCPjPk+\nnRoyRvWatRj16khOnDhOvnz5WbxwHnXq1vO4ThIHDsQSFnZ5IkRoaBhr1qzONXouEXJMCMAVub4F\nKyJDRGSriExKI7+ViGTFyvdXUDh/ALfVKkXrt5bR8o2l5A/0p2Pdsl7XPRt/hmGDevP0yFEEFSqc\nnD7+o7fxDwjgzrt7eN2GJGau3End/l/Q/aUfGdm3eabq2rpqMUHBxQmrVvuK9BXff0XfNz9nxLSV\n1L+jK7M/fSNTOqlRrXpNBj8+jB533cl9XTpSq/YN+Ad4r+2S2hrO4kWHk9V6buEjIYLroQX7CNa2\nz3uy2xBnmlQpTsyJc5yMt16VF24+wk0Vgpm14aDXNC9evMhTg+7nzru6c1u7TsnpM7+fxPJF8/hs\n8iyP/8c5cvIsZYoV5NCJeMoUK8jRU2evKrPyrxgqlw2meOH8HP83Y7HnfX9HseX3RWxb/RuJCRe4\ncPYMX/33IY7+s4vyNa3QSJ1b2vPl8Add1JQx7nvgQe57wKr7jVdeICTEM512qREaGkZMzP7k69jY\nGEJCQtK5w7f0XCPgnzNCAK7IGW7eS4jIWKz9kGaKyLMi8ruIrLd/XrWNtIjcLCIb7GO9iBSy058W\nkbUisklEXvaEbQdPnadO+WDy5bH+BE3Ci7H7aLwnqk4VVeXlZwZTqUp1eg94NDl95dKFTPj0Az74\nYir58xdIp4aM8cuqaO6/3WpV3n97bWb/bnX+VA4JTi5Tt0ppAvP4Z9i5ArQd8DQjpq1k+OTfuPeF\nDwi/qQkPvDaW8/FnOLrf+m7dGbWCkk4dYJ7k2FFr8ZuY/f8wZ9ZP3NXVe28CkQ0aEB29k7179pCQ\nkMD0qVNo36GT6xt9RM8lScO0TAs2e1HVQSLSFrgFSADeVdVEEWmNtb1yyhVEhgGDVXWlvbPneRFp\nA1QFGmL9aWeKSEtVXZZ
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x114858fd0>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(mul_tfidf_train, mul_y_train)\n",
|
|||
|
|
"pred = clf.predict(mul_tfidf_valid)\n",
|
|||
|
|
"score = metrics.accuracy_score(mul_y_valid, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(mul_y_valid, pred, labels=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Configuration 3 (!no train)\n",
|
|||
|
|
"* model a - test - dataset2 - [performance measures]\n",
|
|||
|
|
"* model b - test - dataset1 - [performance measures]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 65,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf = MultinomialNB()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 73,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"0 true\n",
|
|||
|
|
"1 false\n",
|
|||
|
|
"2 false\n",
|
|||
|
|
"3 half-true\n",
|
|||
|
|
"4 pants-fire\n",
|
|||
|
|
"5 true\n",
|
|||
|
|
"6 true\n",
|
|||
|
|
"7 barely-true\n",
|
|||
|
|
"8 true\n",
|
|||
|
|
"9 barely-true\n",
|
|||
|
|
"10 barely-true\n",
|
|||
|
|
"11 barely-true\n",
|
|||
|
|
"12 pants-fire\n",
|
|||
|
|
"13 false\n",
|
|||
|
|
"14 half-true\n",
|
|||
|
|
"15 true\n",
|
|||
|
|
"16 pants-fire\n",
|
|||
|
|
"17 half-true\n",
|
|||
|
|
"18 true\n",
|
|||
|
|
"19 false\n",
|
|||
|
|
"20 mostly-true\n",
|
|||
|
|
"21 half-true\n",
|
|||
|
|
"22 pants-fire\n",
|
|||
|
|
"23 mostly-true\n",
|
|||
|
|
"24 false\n",
|
|||
|
|
"25 true\n",
|
|||
|
|
"26 barely-true\n",
|
|||
|
|
"27 half-true\n",
|
|||
|
|
"28 true\n",
|
|||
|
|
"29 mostly-true\n",
|
|||
|
|
"Name: y, dtype: object\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"#print(type(bin_tfidf_train),type(mul_tfidf_test))\n",
|
|||
|
|
"\n",
|
|||
|
|
"tmp_mul_tfidf_test = bin_tfidf_vectorizer.transform(mul_X_test)\n",
|
|||
|
|
"\n",
|
|||
|
|
"print(mul_y_test[:5])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 96,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(bin_tfidf_train, bin_y_train)\n",
|
|||
|
|
"pred = clf.predict(tmp_mul_tfidf_test)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 97,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"1267\n",
|
|||
|
|
"['true' 'true' 'true' 'true' 'true' 'true' 'true' 'true' 'true' 'true'\n",
|
|||
|
|
" 'true' 'true' 'true' 'true' 'true' 'true' 'false' 'true' 'true' 'true'\n",
|
|||
|
|
" 'true' 'true' 'true' 'false' 'false' 'true' 'false' 'true' 'true' 'false']\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"print(len(pred))\n",
|
|||
|
|
"pred = np.array(pred, dtype=object)\n",
|
|||
|
|
"\n",
|
|||
|
|
"pred[pred == \"FAKE\"] = \"false\"\n",
|
|||
|
|
"pred[pred == \"REAL\"] = \"true\"\n",
|
|||
|
|
"\n",
|
|||
|
|
"print(pred[:30])\n",
|
|||
|
|
"#pred[pred['FAKE']==false]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 99,
|
|||
|
|
"metadata": {
|
|||
|
|
"scrolled": true
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"accuracy: 0.169\n",
|
|||
|
|
"Confusion matrix, without normalization\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAEmCAYAAAAnRIjxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsnXuc1eP2x9+fmZRSJNeaKd0ole6F\nJCEpJdeIUDiuuZ+Qy49w0HE5jlwP5zg4OpXrQShx3EkXIvdKOd2EkG66TOv3x/Od2k0zs/dMe8/e\nu1nvXt/X7P18n/18P3s3s/bzXc961pKZ4TiO4ySfnHQLcBzH2VpxA+s4jpMi3MA6juOkCDewjuM4\nKcINrOM4TopwA+s4jpMi3MA6FYqk6pJelLRU0lNbMM5ASa8mU1u6kHSgpK/TrcNJPvI4WKc4JJ0M\nXAY0B5YB04GbzezdLRz3VOBCoIuZrdtioRmOJAP2NLNZ6dbiVDw+g3U2Q9JlwF+BW4DdgAbA/cBR\nSRh+D+CbymBcE0FSlXRrcFKImfnhx4YD2AFYDvQvpU81ggFeGB1/BapF57oD84E/Aj8Ai4DTo3M3\nAGuAtdE1zgSGA0/EjN0QMKBK9Hww8C1hFj0HGBjT/m7M67oAU4Cl0c8uMefeBG4C3ovGeRXYuYT3\nVqj/ihj9RwNHAN8APwNXx/TvDHwA/Br1vReoGp17O3ovK6L3e2LM+FcC3wP/KmyLXtMkukb76Hk9\n4Cege7p/N/wo++EzWKco+wPbAs+V0ucaYD+gLdCGYGSujTm/O8FQ5xGM6H2SdjSz6wmz4rFmVtPM\n/lGaEEnbASOB3mZWi2BEpxfTrw7wUtR3J+AvwEuSdorpdjJwOrArUBUYWsqldyd8BnnAdcDDwClA\nB+BA4DpJjaO+BcClwM6Ez+5Q4HwAM+sW9WkTvd+xMePXIczmz469sJnNJhjfUZJqAP8EHjWzN0vR\n62QobmCdouwE/GSl38IPBG40sx/M7EfCzPTUmPNro/NrzexlwuytWTn1rAdaSapuZovM7PNi+vQB\nZprZv8xsnZmNBr4Cjozp808z+8bMVgFPEr4cSmItwd+8FhhDMJ53m9my6PqfA60BzGyamU2KrjsX\n+BtwUALv6XozWx3p2QQzexiYCXwI1CV8oTlZiBtYpyhLgJ3j+AbrAd/FPP8uatswRhEDvRKoWVYh\nZraCcFt9LrBI0kuSmiegp1BTXszz78ugZ4mZFUSPCw3g4pjzqwpfL2kvSeMkfS/pN8IMfedSxgb4\n0cx+j9PnYaAVcI+ZrY7T18lQ3MA6RfkA+J3gdyyJhYTb20IaRG3lYQVQI+b57rEnzWyCmR1GmMl9\nRTA88fQUalpQTk1l4QGCrj3NbHvgakBxXlNq6I6kmgS/9j+A4ZELxMlC3MA6m2BmSwl+x/skHS2p\nhqRtJPWWdFvUbTRwraRdJO0c9X+inJecDnST1EDSDsBVhSck7SapX+SLXU1wNRQUM8bLwF6STpZU\nRdKJQAtgXDk1lYVawG/A8mh2fV6R84uBxpu9qnTuBqaZ2R8IvuUHt1ilkxbcwDqbYWZ/IcTAXgv8\nCMwDLgD+E3X5EzAV+BSYAXwUtZXnWhOBsdFY09jUKOYQohEWElbWDyJaQCoyxhKgb9R3CSECoK+Z\n/VQeTWVkKGEBbRlhdj22yPnhwGOSfpV0QrzBJB0F9CK4RSD8P7SXNDBpip0KwzcaOI7jpAifwTqO\n46QIN7CO4zgpwg2s4zhOinAD6ziOkyI80USKqF1nJ6uX3yDdMoql+ja56ZbgpICPZy2O3ykN2Kpf\nsDXL48UGJ0zu9nuYrdtsA1wx1/1xgpn1StZ1y4Mb2BRRL78Bo158K90yimXvvO3TLcFJATse+dd0\nSyiW1e/fmdTxbN0qqjWLG/HG79Pvi7ejLuW4gXUcJ7uQICc77sLcwDqOk30oO5aP3MA6jpN9KGku\n3ZTiBtZxnCzDXQSO4zipQbiLwHEcJzXIXQSO4zgpw2ewjuM4qcB9sI7jOKlBuIvAcRwnZbiLwHEc\nJxUIcrPDRZAdXwOVkIKCAk46oisXnRH2XJ/RvxcDendlQO+u9OzcjMvOOjnNCuHVCeNp3bIZLZs3\n5fbbRqRbzia4tpLJ37km40ccx8d/O41pD57KkKNCBfMda1Zj3M3HMOPvgxh38zHUrlkNgAEHN2Py\n/QOZfP9A3rjzBPZplOYt/oVhWvGODMBnsBnK6H8+QKOmzVi+fBkAjzw1fsO5oeeeQvfD+qRLGhC+\nAC65aAgvvTKRvPx8uu7Xib59+7F3ixZp1eXa4rOuYD3DHn6b6bN/pGb1bXh/5Mm8/vH/OLVHC96c\nPo87nprK0P4dGXpCJ6595F3mfv8bPa94ml+Xr6Znx4bcd1EPul06psL0FkuW+GAzw8w7m7B40QLe\n+e8Ejh5w2mbnVixfxpT336Z7z/Qa2CmTJ9OkSVMaNW5M1apV6X/iAMa9+HxaNRXi2krn+19WMn32\njwAsX7WWr+b9TL2datJ3/8Y88doXADzx2hccuX8ohjvpy0X8unw1AJO/WkTezjUrVO/mRFEE8Y4M\nwA1sBnLHjcO4+KobySnmNueNCePofMBB1KyV3pSDCxcuID+//obneXn5LFiwII2KNuLaEqfBrtvT\ntskuTPn6e3atvR3f/7ISCEZ4lx1qbNZ/8OEtmTB1bgWrLIYscRFkhooUIukiSV9KGlXC+e6SxhV3\nLh28/fp46uy0Cy32aVfs+fEvPE2vfsdXsKrNKa4asTLkts21JcZ2227D6Gv7cPnf3mLZyjVx+3dr\nnc+gnq249pF3K0BdKUiJHXGHUX1Jb0T24XNJF0ftdSRNlDQz+rlj1C5JIyXNkvSppPbxrrHVG1jg\nfOAIM8uKuvKfTJ3EW6+9Qp8D9uGqC89g6vtvc80lZwHw6y8/8/kn0+h68OFpVhlmXvPnz9vwfMGC\n+dSrVy+Nijbi2uJTJTeH0df2ZewbX/H8+7MB+OHXFey+Y5i17r5jDX5cunJD/1YNd+aBS3rQ/8YX\n+HnZ7xWudzOS4yJYB/zRzPYG9gOGSGoBDANeN7M9gdej5wC9gT2j42zggbgyy/7OsgdJDwKNgRck\nXSnpfUkfRz+bFdP/IEnTo+NjSbWi9sslTYm+tW5IpeYLrxzO+Elf8tJ7M7j1nkfo2KUbN//1YQBe\ne+k5DjykF9W23TaVEhKiY6dOzJo1k7lz5rBmzRqeGjuGPn37pVsW4NoS4cFLevD1vJ8Z+dzHG9pe\nmvQtp/QIi22n9GjBuA++BaD+LrUY8399OfP2Ccxa8GuFa90cJcVFYGaLzOyj6PEy4EsgDzgKeCzq\n9hhwdPT4KOBxC0wCakuqW9o1tuooAjM7V1Iv4GBgDXCnma2T1AO4BTiuyEuGAkPM7D1JNYHfJfUk\nfGN1JgSIvCCpm5m9XfR6ks4mfLOxe179oqe3mAkvPsvg8y5N+rjloUqVKtx1970c2edwCgoKGDT4\nDFq0bJluWYBri0eXlvUY2KMFM+b8yKR7w43d9Y+9xx1PTuWJq49g0OEtmffjMgbe/BIAV528L3Vq\nbctfhxwChCiErhePrlDNm5Fkt4qkhkA74ENgNzNbBMEIS9o16pYHzIt52fyobVGJ4xbnE9qakDQX\n6AhUB0YSjKUB25hZc0ndgaFm1lfSMOAYYBTwrJnNl3QHcDxQ+NVdE7jVzP5R2nVbtG5nXpPLqUgy\nuSbX+qXzkmYRc2o3sGpdr4jb7/eXLvwO+Cmm6SEze6hov2gy9RZws5k9K+lXM6sdc/4XM9tR0kuE\nv/13o/bXgSvMbFpJGrbqGWwRbgLeMLNjom+rN4t2MLMR0Yd4BDApmumK8KH+rQK1Oo5TIgkne/nJ\nzDqWOpK0DfAMMMrMno2aF0uqG81e6wI/RO3zgdhb03xgYWnjb9U+2CLsABTGwwwuroOkJmY2w8z+\nDEwFmgMTgDOibzkk5cX
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x1141d4b38>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"score = metrics.accuracy_score(mul_y_test, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(mul_y_test, pred, labels=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true' ])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 119,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf = MultinomialNB()\n",
|
|||
|
|
"from sklearn.neural_network import MLPClassifier\n",
|
|||
|
|
"\n",
|
|||
|
|
"clf = MLPClassifier()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stdout",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"0 true\n",
|
|||
|
|
"1 false\n",
|
|||
|
|
"2 false\n",
|
|||
|
|
"3 half-true\n",
|
|||
|
|
"4 pants-fire\n",
|
|||
|
|
"5 true\n",
|
|||
|
|
"6 true\n",
|
|||
|
|
"7 barely-true\n",
|
|||
|
|
"8 true\n",
|
|||
|
|
"9 barely-true\n",
|
|||
|
|
"Name: y, dtype: object\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"tmp_bin_tfidf_test = mul_tfidf_vectorizer.transform(bin_X_test)\n",
|
|||
|
|
"\n",
|
|||
|
|
"print(mul_y_test[:10])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"clf.fit(mul_tfidf_train, mul_y_train)\n",
|
|||
|
|
"pred = clf.predict(tmp_bin_tfidf_test)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"print(pred[:20])\n",
|
|||
|
|
"\n",
|
|||
|
|
"bin_y_test = np.array(bin_y_test, dtype=object)\n",
|
|||
|
|
"\n",
|
|||
|
|
"bin_y_test[bin_y_test == \"FAKE\"] = \"false\"\n",
|
|||
|
|
"bin_y_test[bin_y_test == \"REAL\"] = \"true\""
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"score = metrics.accuracy_score(bin_y_test, pred)\n",
|
|||
|
|
"print(\"accuracy: %0.3f\" % score)\n",
|
|||
|
|
"cm = metrics.confusion_matrix(bin_y_test, pred, labels=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])\n",
|
|||
|
|
"plot_confusion_matrix(cm, classes=['false', 'barely-true' , 'half-true' , 'mostly-true' , 'true'])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Configuration 4\n",
|
|||
|
|
"* model c - train - [performance measures][0:4]\n",
|
|||
|
|
"* model c - test - [performance measures][0:4]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {
|
|||
|
|
"collapsed": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"kernelspec": {
|
|||
|
|
"display_name": "Python 3",
|
|||
|
|
"language": "python",
|
|||
|
|
"name": "python3"
|
|||
|
|
},
|
|||
|
|
"language_info": {
|
|||
|
|
"codemirror_mode": {
|
|||
|
|
"name": "ipython",
|
|||
|
|
"version": 3
|
|||
|
|
},
|
|||
|
|
"file_extension": ".py",
|
|||
|
|
"mimetype": "text/x-python",
|
|||
|
|
"name": "python",
|
|||
|
|
"nbconvert_exporter": "python",
|
|||
|
|
"pygments_lexer": "ipython3",
|
|||
|
|
"version": "3.6.3"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"nbformat": 4,
|
|||
|
|
"nbformat_minor": 2
|
|||
|
|
}
|