{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP-LAB Exercise 02 by Jonas Weinz (2571421)\n",
    "## links:\n",
    "\n",
    "* Article: https://miguelmalvarez.com/2017/03/23/how-can-machine-learning-and-ai-help-solving-the-fake-news-problem/\n",
    "    * corresponding code: https://github.com/kjam/random_hackery/blob/master/Attempting%20to%20detect%20fake%20news.ipynb\n",
    "\n",
    "* Tutorial on Datacamp: https://www.datacamp.com/community/tutorials/scikit-learn-fake-news\n",
    "\n",
    "* liar dataset paper: https://www.cs.ucsb.edu/~william/papers/acl2017.pdf\n",
    "    * dataset: https://www.cs.ucsb.edu/~william/data/liar_dataset.zip\n",
    "\n",
    "* Mex Vocabulary: http://jens-lehmann.org/files/2015/semantics_mex.pdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependencies for this Notebook:\n",
    "* library [rdflib](https://github.com/RDFLib/rdflib)\n",
    "    * install: `pip3 install rdflib`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools\n",
    "import sklearn.utils as sku\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import PassiveAggressiveClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt\n",
    "from pprint import pprint as pp\n",
    "from IPython.display import display, Markdown, Latex\n",
    "import collections\n",
    "import traceback\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tools used later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          title,\n",
    "                          normalize=False,\n",
    "                          cmap=plt.cm.Blues):\n",
    "    fig_1, ax_1 = plt.subplots()\n",
    "    \"\"\"\n",
    "    See full source and example: \n",
    "    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
    "    \n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title('Confusion Matrix for:\\n' + title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    \n",
    "    pp(cm)\n",
    "    \n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_classifier(labels, title, Xt, yt, clf):\n",
    "    pred = clf.predict(Xt)\n",
    "    score = metrics.accuracy_score(yt, pred)\n",
    "    pp(\"score: \" + str(score))\n",
    "    cm = metrics.confusion_matrix(yt, pred, labels=labels)\n",
    "    plot_confusion_matrix(cm, classes=labels, title=title)\n",
    "    return cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfrom rdflib import Graph, Literal, BNode, RDF, Namespace\\nfrom rdflib.namespace import FOAF, DC, XSD\\n\\nmexcore = Namespace('http://mex.aksw.org/mex-core#')\\nmexperf = Namespace('http://mex.aksw.org/mex-perf#')\\nmexalgo = Namespace('http://mex.aksw.org/mex-algo#')\\nprov = Namespace('http://www.w3.org/ns/prov#')\\n\\ndef create_mex_graph():\\n    graph = Graph()\\n    graph.bind(mexcore)\\n    graph.bind(mexperf)\\n    graph.bind(mexalgo)\\n    graph.bind(prov)\\n    graph.bind(FOAF)\\n    graph.bind(DC)\\n    graph.bind(XSD)\\n    \\n    return graph\\n\\ndef mex_performance(experiment, model, dataset, performance, phase='Train', graph=create_mex_graph()):\\n    \\n    p = BNode()\\n    \\n\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "from rdflib import Graph, Literal, BNode, RDF, Namespace\n",
    "from rdflib.namespace import FOAF, DC, XSD\n",
    "\n",
    "mexcore = Namespace('http://mex.aksw.org/mex-core#')\n",
    "mexperf = Namespace('http://mex.aksw.org/mex-perf#')\n",
    "mexalgo = Namespace('http://mex.aksw.org/mex-algo#')\n",
    "prov = Namespace('http://www.w3.org/ns/prov#')\n",
    "\n",
    "def create_mex_graph():\n",
    "    graph = Graph()\n",
    "    graph.bind(mexcore)\n",
    "    graph.bind(mexperf)\n",
    "    graph.bind(mexalgo)\n",
    "    graph.bind(prov)\n",
    "    graph.bind(FOAF)\n",
    "    graph.bind(DC)\n",
    "    graph.bind(XSD)\n",
    "    \n",
    "    return graph\n",
    "\n",
    "def mex_performance(experiment, model, dataset, performance, phase='Train', graph=create_mex_graph()):\n",
    "    \n",
    "    p = BNode()\n",
    "    \n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate/Download Datasets we are working on\n",
    "\n",
    "* running bash script to download all needed data and store it into the `data` subfolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "checking whether unzip is installed\n",
      "================================================================================\n",
      "UnZip 6.00 of 20 April 2009, by Debian. Original by Info-ZIP.\n",
      "\n",
      "Latest sources and executables are at ftp://ftp.info-zip.org/pub/infozip/ ;\n",
      "see ftp://ftp.info-zip.org/pub/infozip/UnZip.html for other sites.\n",
      "\n",
      "Compiled with gcc 6.3.0 20170415 for Unix (Linux ELF).\n",
      "\n",
      "UnZip special compilation options:\n",
      "        ACORN_FTYPE_NFS\n",
      "        COPYRIGHT_CLEAN (PKZIP 0.9x unreducing method not supported)\n",
      "        SET_DIR_ATTRIB\n",
      "        SYMLINKS (symbolic links supported, if RTL and file system permit)\n",
      "        TIMESTAMP\n",
      "        UNIXBACKUP\n",
      "        USE_EF_UT_TIME\n",
      "        USE_UNSHRINK (PKZIP/Zip 1.x unshrinking method supported)\n",
      "        USE_DEFLATE64 (PKZIP 4.x Deflate64(tm) supported)\n",
      "        UNICODE_SUPPORT [wide-chars, char coding: UTF-8] (handle UTF-8 paths)\n",
      "        LARGE_FILE_SUPPORT (large files over 2 GiB supported)\n",
      "        ZIP64_SUPPORT (archives using Zip64 for large files supported)\n",
      "        USE_BZIP2 (PKZIP 4.6+, using bzip2 lib version 1.0.6, 6-Sept-2010)\n",
      "        VMS_TEXT_CONV\n",
      "        WILD_STOP_AT_DIR\n",
      "        [decryption, version 2.11 of 05 Jan 2007]\n",
      "\n",
      "UnZip and ZipInfo environment options:\n",
      "           UNZIP:  [none]\n",
      "        UNZIPOPT:  [none]\n",
      "         ZIPINFO:  [none]\n",
      "      ZIPINFOOPT:  [none]\n",
      "================================================================================\n",
      "successfully finished action: checking whether unzip is installed\n",
      "================================================================================\n",
      "================================================================================\n",
      "downloading and unpacking https://www.cs.ucsb.edu/~william/data/liar_dataset.zip if not already existing\n",
      "================================================================================\n",
      "================================================================================\n",
      "successfully finished action: downloading and unpacking https://www.cs.ucsb.edu/~william/data/liar_dataset.zip if not already existing\n",
      "================================================================================\n",
      "================================================================================\n",
      "downloading and unpacking https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip if not already existing\n",
      "================================================================================\n",
      "================================================================================\n",
      "successfully finished action: downloading and unpacking https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip if not already existing\n",
      "================================================================================\n",
      "================================================================================\n",
      "downloading Helper script: script_dataset3.py\n",
      "================================================================================\n",
      "================================================================================\n",
      "successfully finished action: downloading Helper script: script_dataset3.py\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "./Task_2_gen_data.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "## configuration 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.read_csv('data/fake_or_real_news.csv').set_index('Unnamed: 0')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* display first 10 entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6335, 3)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8476</th>\n",
       "      <td>You Can Smell Hillary’s Fear</td>\n",
       "      <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10294</th>\n",
       "      <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
       "      <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3608</th>\n",
       "      <td>Kerry to go to Paris in gesture of sympathy</td>\n",
       "      <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10142</th>\n",
       "      <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
       "      <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875</th>\n",
       "      <td>The Battle of New York: Why This Primary Matters</td>\n",
       "      <td>It's primary day in New York and front-runners...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6903</th>\n",
       "      <td>Tehran, USA</td>\n",
       "      <td>\\nI’m not an immigrant, but my grandparents ...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7341</th>\n",
       "      <td>Girl Horrified At What She Watches Boyfriend D...</td>\n",
       "      <td>Share This Baylee Luciani (left), Screenshot o...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>‘Britain’s Schindler’ Dies at 106</td>\n",
       "      <td>A Czech stockbroker who saved more than 650 Je...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4869</th>\n",
       "      <td>Fact check: Trump and Clinton at the 'commande...</td>\n",
       "      <td>Hillary Clinton and Donald Trump made some ina...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2909</th>\n",
       "      <td>Iran reportedly makes new push for uranium con...</td>\n",
       "      <td>Iranian negotiators reportedly have made a las...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        title  \\\n",
       "Unnamed: 0                                                      \n",
       "8476                             You Can Smell Hillary’s Fear   \n",
       "10294       Watch The Exact Moment Paul Ryan Committed Pol...   \n",
       "3608              Kerry to go to Paris in gesture of sympathy   \n",
       "10142       Bernie supporters on Twitter erupt in anger ag...   \n",
       "875          The Battle of New York: Why This Primary Matters   \n",
       "6903                                              Tehran, USA   \n",
       "7341        Girl Horrified At What She Watches Boyfriend D...   \n",
       "95                          ‘Britain’s Schindler’ Dies at 106   \n",
       "4869        Fact check: Trump and Clinton at the 'commande...   \n",
       "2909        Iran reportedly makes new push for uranium con...   \n",
       "\n",
       "                                                         text label  \n",
       "Unnamed: 0                                                           \n",
       "8476        Daniel Greenfield, a Shillman Journalism Fello...  FAKE  \n",
       "10294       Google Pinterest Digg Linkedin Reddit Stumbleu...  FAKE  \n",
       "3608        U.S. Secretary of State John F. Kerry said Mon...  REAL  \n",
       "10142       — Kaydee King (@KaydeeKing) November 9, 2016 T...  FAKE  \n",
       "875         It's primary day in New York and front-runners...  REAL  \n",
       "6903          \\nI’m not an immigrant, but my grandparents ...  FAKE  \n",
       "7341        Share This Baylee Luciani (left), Screenshot o...  FAKE  \n",
       "95          A Czech stockbroker who saved more than 650 Je...  REAL  \n",
       "4869        Hillary Clinton and Donald Trump made some ina...  REAL  \n",
       "2909        Iranian negotiators reportedly have made a las...  REAL  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df_1.shape)\n",
    "display(df_1[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* create test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1, Xt1, y1, yt1 = train_test_split(df_1.drop('label', axis=1)['text'], df_1.label, test_size=0.25, random_state=4222)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_1 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
    "vec_train_1 = vectorizer_1.fit_transform(X1)\n",
    "vec_test_1 = vectorizer_1.transform(Xt1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* trying a Random Forest classifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'score: 0.9960008419280152'\n",
      "Confusion matrix, without normalization\n",
      "array([[2342,    3],\n",
      "       [  16, 2390]])\n",
      "'score: 0.8478535353535354'\n",
      "Confusion matrix, without normalization\n",
      "array([[737,  82],\n",
      "       [159, 606]])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 14604 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5e84a8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 15452 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5e84a048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier as RFC\n",
    "clf_a = RFC(criterion='entropy', random_state=4222)\n",
    "max_size=10000\n",
    "clf_a.fit(vec_train_1[:max_size], y1[:max_size])\n",
    "test_classifier(labels=[\"FAKE\",\"REAL\"], title=\"Configuration 1, model a -- train\", Xt=vec_train_1,yt=y1, clf=clf_a)\n",
    "cm_1 = test_classifier(labels=[\"FAKE\",\"REAL\"], title=\"Configuration 1, model a -- test\", Xt=vec_test_1,yt=yt1, clf=clf_a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "## configuration 2\n",
    "\n",
    "* read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "----\n",
       "#### Train Data:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>statement</th>\n",
       "      <th>subjects</th>\n",
       "      <th>speaker</th>\n",
       "      <th>job</th>\n",
       "      <th>state</th>\n",
       "      <th>party</th>\n",
       "      <th>#barely_true</th>\n",
       "      <th>#false</th>\n",
       "      <th>#half_true</th>\n",
       "      <th>#mostly_true</th>\n",
       "      <th>#pants_on_fire</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2635.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says the Annies List political group supports ...</td>\n",
       "      <td>abortion</td>\n",
       "      <td>dwayne-bohac</td>\n",
       "      <td>State representative</td>\n",
       "      <td>Texas</td>\n",
       "      <td>republican</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>a mailer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Health care reform legislation is likely to ma...</td>\n",
       "      <td>health-care</td>\n",
       "      <td>blog-posting</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>none</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>a news release</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12465.json</td>\n",
       "      <td>true</td>\n",
       "      <td>The Chicago Bears have had more starting quart...</td>\n",
       "      <td>education</td>\n",
       "      <td>robin-vos</td>\n",
       "      <td>Wisconsin Assembly speaker</td>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>republican</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>a an online opinion-piece</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5947.json</td>\n",
       "      <td>false</td>\n",
       "      <td>When Mitt Romney was governor of Massachusetts...</td>\n",
       "      <td>history,state-budget</td>\n",
       "      <td>mitt-romney</td>\n",
       "      <td>Former governor</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>republican</td>\n",
       "      <td>34.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>an interview with CBN News</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>620.json</td>\n",
       "      <td>true</td>\n",
       "      <td>McCain opposed a requirement that the governme...</td>\n",
       "      <td>federal-budget</td>\n",
       "      <td>barack-obama</td>\n",
       "      <td>President</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>democrat</td>\n",
       "      <td>70.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>a radio ad</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id  label                                          statement  \\\n",
       "0    2635.json  false  Says the Annies List political group supports ...   \n",
       "3    1123.json  false  Health care reform legislation is likely to ma...   \n",
       "5   12465.json   true  The Chicago Bears have had more starting quart...   \n",
       "12   5947.json  false  When Mitt Romney was governor of Massachusetts...   \n",
       "16    620.json   true  McCain opposed a requirement that the governme...   \n",
       "\n",
       "                subjects       speaker                         job  \\\n",
       "0               abortion  dwayne-bohac        State representative   \n",
       "3            health-care  blog-posting                         NaN   \n",
       "5              education     robin-vos  Wisconsin Assembly speaker   \n",
       "12  history,state-budget   mitt-romney             Former governor   \n",
       "16        federal-budget  barack-obama                   President   \n",
       "\n",
       "            state       party  #barely_true  #false  #half_true  #mostly_true  \\\n",
       "0           Texas  republican           0.0     1.0         0.0           0.0   \n",
       "3             NaN        none           7.0    19.0         3.0           5.0   \n",
       "5       Wisconsin  republican           0.0     3.0         2.0           5.0   \n",
       "12  Massachusetts  republican          34.0    32.0        58.0          33.0   \n",
       "16       Illinois    democrat          70.0    71.0       160.0         163.0   \n",
       "\n",
       "    #pants_on_fire                     context  \n",
       "0              0.0                    a mailer  \n",
       "3             44.0              a news release  \n",
       "5              1.0   a an online opinion-piece  \n",
       "12            19.0  an interview with CBN News  \n",
       "16             9.0                  a radio ad  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "----\n",
       "#### Test Data:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>statement</th>\n",
       "      <th>subjects</th>\n",
       "      <th>speaker</th>\n",
       "      <th>job</th>\n",
       "      <th>state</th>\n",
       "      <th>party</th>\n",
       "      <th>#barely_true</th>\n",
       "      <th>#false</th>\n",
       "      <th>#half_true</th>\n",
       "      <th>#mostly_true</th>\n",
       "      <th>#pants_on_fire</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11972.json</td>\n",
       "      <td>true</td>\n",
       "      <td>Building a wall on the U.S.-Mexico border will...</td>\n",
       "      <td>immigration</td>\n",
       "      <td>rick-perry</td>\n",
       "      <td>Governor</td>\n",
       "      <td>Texas</td>\n",
       "      <td>republican</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>42</td>\n",
       "      <td>23</td>\n",
       "      <td>18</td>\n",
       "      <td>Radio interview</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11685.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Wisconsin is on pace to double the number of l...</td>\n",
       "      <td>jobs</td>\n",
       "      <td>katrina-shankland</td>\n",
       "      <td>State representative</td>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>democrat</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>a news conference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11096.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says John McCain has done nothing to help the ...</td>\n",
       "      <td>military,veterans,voting-record</td>\n",
       "      <td>donald-trump</td>\n",
       "      <td>President-Elect</td>\n",
       "      <td>New York</td>\n",
       "      <td>republican</td>\n",
       "      <td>63</td>\n",
       "      <td>114</td>\n",
       "      <td>51</td>\n",
       "      <td>37</td>\n",
       "      <td>61</td>\n",
       "      <td>comments on ABC's This Week.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5962.json</td>\n",
       "      <td>true</td>\n",
       "      <td>Over the past five years the federal governmen...</td>\n",
       "      <td>federal-budget,pensions,retirement</td>\n",
       "      <td>brendan-doherty</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Rhode Island</td>\n",
       "      <td>republican</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>a campaign website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7070.json</td>\n",
       "      <td>true</td>\n",
       "      <td>Says that Tennessee law requires that schools ...</td>\n",
       "      <td>county-budget,county-government,education,taxes</td>\n",
       "      <td>stand-children-tennessee</td>\n",
       "      <td>Child and education advocacy organization.</td>\n",
       "      <td>Tennessee</td>\n",
       "      <td>none</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>in a post on Facebook.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id  label                                          statement  \\\n",
       "0  11972.json   true  Building a wall on the U.S.-Mexico border will...   \n",
       "1  11685.json  false  Wisconsin is on pace to double the number of l...   \n",
       "2  11096.json  false  Says John McCain has done nothing to help the ...   \n",
       "5   5962.json   true  Over the past five years the federal governmen...   \n",
       "6   7070.json   true  Says that Tennessee law requires that schools ...   \n",
       "\n",
       "                                          subjects                   speaker  \\\n",
       "0                                      immigration                rick-perry   \n",
       "1                                             jobs         katrina-shankland   \n",
       "2                  military,veterans,voting-record              donald-trump   \n",
       "5               federal-budget,pensions,retirement           brendan-doherty   \n",
       "6  county-budget,county-government,education,taxes  stand-children-tennessee   \n",
       "\n",
       "                                          job         state       party  \\\n",
       "0                                    Governor         Texas  republican   \n",
       "1                        State representative     Wisconsin    democrat   \n",
       "2                             President-Elect      New York  republican   \n",
       "5                                         NaN  Rhode Island  republican   \n",
       "6  Child and education advocacy organization.     Tennessee        none   \n",
       "\n",
       "   #barely_true  #false  #half_true  #mostly_true  #pants_on_fire  \\\n",
       "0            30      30          42            23              18   \n",
       "1             2       1           0             0               0   \n",
       "2            63     114          51            37              61   \n",
       "5             1       2           1             1               0   \n",
       "6             0       0           0             0               0   \n",
       "\n",
       "                        context  \n",
       "0               Radio interview  \n",
       "1             a news conference  \n",
       "2  comments on ABC's This Week.  \n",
       "5            a campaign website  \n",
       "6        in a post on Facebook.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "----\n",
       "#### Valid Data:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>statement</th>\n",
       "      <th>subjects</th>\n",
       "      <th>speaker</th>\n",
       "      <th>job</th>\n",
       "      <th>state</th>\n",
       "      <th>party</th>\n",
       "      <th>#barely_true</th>\n",
       "      <th>#false</th>\n",
       "      <th>#half_true</th>\n",
       "      <th>#mostly_true</th>\n",
       "      <th>#pants_on_fire</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7891.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says Having organizations parading as being so...</td>\n",
       "      <td>campaign-finance,congress,taxes</td>\n",
       "      <td>earl-blumenauer</td>\n",
       "      <td>U.S. representative</td>\n",
       "      <td>Oregon</td>\n",
       "      <td>democrat</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>a U.S. Ways and Means hearing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9416.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says when armed civilians stop mass shootings ...</td>\n",
       "      <td>guns</td>\n",
       "      <td>jim-rubens</td>\n",
       "      <td>Small business owner</td>\n",
       "      <td>New Hampshire</td>\n",
       "      <td>republican</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>in an interview at gun shop in Hudson, N.H.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6861.json</td>\n",
       "      <td>true</td>\n",
       "      <td>Says Tennessee is providing millions of dollar...</td>\n",
       "      <td>education,state-budget</td>\n",
       "      <td>andy-berke</td>\n",
       "      <td>Lawyer and state senator</td>\n",
       "      <td>Tennessee</td>\n",
       "      <td>democrat</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>a letter to state Senate education committee c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1122.json</td>\n",
       "      <td>false</td>\n",
       "      <td>The health care reform plan would set limits s...</td>\n",
       "      <td>health-care</td>\n",
       "      <td>club-growth</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>none</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>a TV ad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13138.json</td>\n",
       "      <td>true</td>\n",
       "      <td>Says Donald Trump started his career back in 1...</td>\n",
       "      <td>candidates-biography,diversity,housing</td>\n",
       "      <td>hillary-clinton</td>\n",
       "      <td>Presidential candidate</td>\n",
       "      <td>New York</td>\n",
       "      <td>democrat</td>\n",
       "      <td>40</td>\n",
       "      <td>29</td>\n",
       "      <td>69</td>\n",
       "      <td>76</td>\n",
       "      <td>7</td>\n",
       "      <td>the first presidential debate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id  label                                          statement  \\\n",
       "2   7891.json  false  Says Having organizations parading as being so...   \n",
       "5   9416.json  false  Says when armed civilians stop mass shootings ...   \n",
       "6   6861.json   true  Says Tennessee is providing millions of dollar...   \n",
       "7   1122.json  false  The health care reform plan would set limits s...   \n",
       "8  13138.json   true  Says Donald Trump started his career back in 1...   \n",
       "\n",
       "                                 subjects          speaker  \\\n",
       "2         campaign-finance,congress,taxes  earl-blumenauer   \n",
       "5                                    guns       jim-rubens   \n",
       "6                  education,state-budget       andy-berke   \n",
       "7                             health-care      club-growth   \n",
       "8  candidates-biography,diversity,housing  hillary-clinton   \n",
       "\n",
       "                        job          state       party  #barely_true  #false  \\\n",
       "2       U.S. representative         Oregon    democrat             0       1   \n",
       "5      Small business owner  New Hampshire  republican             1       1   \n",
       "6  Lawyer and state senator      Tennessee    democrat             0       0   \n",
       "7                       NaN            NaN        none             4       5   \n",
       "8    Presidential candidate       New York    democrat            40      29   \n",
       "\n",
       "   #half_true  #mostly_true  #pants_on_fire  \\\n",
       "2           1             1               0   \n",
       "5           0             1               0   \n",
       "6           0             0               0   \n",
       "7           4             2               0   \n",
       "8          69            76               7   \n",
       "\n",
       "                                             context  \n",
       "2                      a U.S. Ways and Means hearing  \n",
       "5        in an interview at gun shop in Hudson, N.H.  \n",
       "6  a letter to state Senate education committee c...  \n",
       "7                                            a TV ad  \n",
       "8                      the first presidential debate  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "names = [\n",
    "    \"id\",\n",
    "    \"label\",\n",
    "    \"statement\",\n",
    "    \"subjects\",\n",
    "    \"speaker\",\n",
    "    \"job\",\n",
    "    \"state\",\n",
    "    \"party\",\n",
    "    \"#barely_true\",\n",
    "    \"#false\",\n",
    "    \"#half_true\",\n",
    "    \"#mostly_true\",\n",
    "    \"#pants_on_fire\",\n",
    "    \"context\"\n",
    "]\n",
    "\n",
    "df_2_train = pd.read_csv(\"data/train.tsv\", delimiter='\\t', names=names)\n",
    "df_2_test = pd.read_csv(\"data/test.tsv\", delimiter='\\t', names=names)\n",
    "df_2_valid= pd.read_csv(\"data/valid.tsv\", delimiter='\\t', names=names)\n",
    "\n",
    "# use only 'False' and 'True' statements\n",
    "df_2_train = df_2_train[df_2_train['label'].isin([\"false\",\"true\"])]\n",
    "df_2_test = df_2_test[df_2_test['label'].isin([\"false\",\"true\"])]\n",
    "df_2_valid = df_2_valid[df_2_valid['label'].isin([\"false\",\"true\"])]\n",
    "\n",
    "display(Markdown(\"----\\n#### Train Data:\"))\n",
    "display(df_2_train.head())\n",
    "display(Markdown(\"----\\n#### Test Data:\"))\n",
    "display(df_2_test.head())\n",
    "display(Markdown(\"----\\n#### Valid Data:\"))\n",
    "display(df_2_valid.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tdidf vectorizer on new dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X2 = df_2_train['statement']\n",
    "y2 = df_2_train['label']\n",
    "Xt2 = df_2_test['statement']\n",
    "yt2 = df_2_test['label']\n",
    "Xv2 = df_2_valid['statement']\n",
    "yv2 = df_2_valid['label']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_2 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
    "vec_train_2 = vectorizer_2.fit_transform(X2)\n",
    "vec_test_2 = vectorizer_2.transform(Xt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object `MLPClassifier` not found.\n"
     ]
    }
   ],
   "source": [
    "?MLPClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* trying a MLP as classifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(100,), learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=4222,\n",
       "       shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "clf_b = MLPClassifier(hidden_layer_sizes=(100,), random_state=4222)\n",
    "clf_b.fit(vec_train_2, y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'score: 0.9997275946608554'\n",
      "Confusion matrix, without normalization\n",
      "array([[1675,    1],\n",
      "       [   0, 1995]])\n",
      "'score: 0.5776805251641138'\n",
      "Confusion matrix, without normalization\n",
      "array([[109,  99],\n",
      "       [ 94, 155]])\n",
      "'score: 0.6041666666666666'\n",
      "Confusion matrix, without normalization\n",
      "array([[ 92,  77],\n",
      "       [ 94, 169]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 92,  77],\n",
       "       [ 94, 169]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 14852 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5e737f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 13340 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5b205fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 13320 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5b1ebfd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- train\", Xt=vec_train_2, yt=y2, clf=clf_b)\n",
    "cm_2 = test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- test\", Xt=vec_test_2, yt=yt2, clf=clf_b)\n",
    "test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- valid\", Xt=vectorizer_2.transform(Xv2), yt=yv2, clf=clf_b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "## configuration 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'score: 0.5142231947483589'\n",
      "Confusion matrix, without normalization\n",
      "array([[  7, 201],\n",
      "       [ 21, 228]])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 15928 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5e7375c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "yt2_c3 = yt2.copy()\n",
    "yt2_c3[yt2_c3 == \"true\"] = \"REAL\"\n",
    "yt2_c3[yt2_c3 == \"false\"] = \"FAKE\"\n",
    "\n",
    "cm_3b=test_classifier(labels=[\"REAL\", \"FAKE\"], \n",
    "                title=\"configuration 3: model a) → dataset 2\",\n",
    "                Xt=vectorizer_1.transform(Xt2),\n",
    "                yt=yt2_c3, clf=clf_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'score: 0.5391414141414141'\n",
      "Confusion matrix, without normalization\n",
      "array([[274, 491],\n",
      "       [239, 580]])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 16116 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5d5e71c320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "yt1_c3 = yt1.copy()\n",
    "yt1_c3[yt1_c3 == \"REAL\"] = \"true\"\n",
    "yt1_c3[yt1_c3 == \"FAKE\"] = \"false\"\n",
    "\n",
    "cm_3a=test_classifier(labels=[\"true\", \"false\"], \n",
    "                title=\"configuration 3: model b) → dataset 1\",\n",
    "                Xt=vectorizer_2.transform(Xt1),\n",
    "                yt=yt1_c3, clf=clf_b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "## configuration 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset3_split(dataset1_in, dataset2_in):\n",
    "    try:\n",
    "        print('processing datasets')\n",
    "        print('ds1=', dataset1_in)\n",
    "        print('ds2=', dataset2_in)\n",
    "\n",
    "        print('-- fake news')\n",
    "        df1 = pd.read_csv(dataset1_in, sep=',', usecols=['title','text','label'])\n",
    "        df1['claim'] = df1[['title', 'text']].apply(lambda x: '. '.join(x), axis=1)\n",
    "        del df1['title']\n",
    "        del df1['text']\n",
    "        df1.rename(index=str, columns={'label': 'y'}, inplace=True)\n",
    "        print(df1.keys())\n",
    "        print(len(df1[df1['y']=='REAL']))\n",
    "        print(len(df1[df1['y']=='FAKE']))\n",
    "        df1['y'] = np.where(df1['y'] == 'FAKE', 'false', 'true')\n",
    "        print(len(df1))\n",
    "\n",
    "        print('-- liar liar')\n",
    "        df2 = pd.read_csv(dataset2_in, sep='\\t', header=None, usecols=[1,2], names=['y', 'claim'])\n",
    "        print(df2.keys())\n",
    "        print(set(df2.y), len(df2))\n",
    "        print(len(df2[df2['y'] == 'true']))\n",
    "        print(len(df2[df2['y'] == 'false']))\n",
    "        df2=df2[(df2['y'] == 'true') | (df2['y'] == 'false')]\n",
    "        print(set(df2.y), len(df2))\n",
    "\n",
    "        df3=pd.concat([df1, df2], ignore_index=True)\n",
    "\n",
    "        print(df3['y'].value_counts())\n",
    "        print('done')\n",
    "        return train_test_split(df3['claim'], df3['y'], test_size=0.3, random_state=4222)\n",
    "    except Exception as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing datasets\n",
      "ds1= data/fake_or_real_news.csv\n",
      "ds2= data/train.tsv\n",
      "-- fake news\n",
      "Index(['y', 'claim'], dtype='object')\n",
      "3171\n",
      "3164\n",
      "6335\n",
      "-- liar liar\n",
      "Index(['y', 'claim'], dtype='object')\n",
      "{'pants-fire', 'false', 'mostly-true', 'true', 'barely-true', 'half-true'} 10240\n",
      "1676\n",
      "1995\n",
      "{'true', 'false'} 3671\n",
      "false    5159\n",
      "true     4847\n",
      "Name: y, dtype: int64\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "X3, Xt3, y3, yt3 = get_dataset3_split('data/fake_or_real_news.csv', 'data/train.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_3 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
    "vec_train_3 = vectorizer_3.fit_transform(X3)\n",
    "vec_test_3 = vectorizer_3.transform(Xt3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* using MLP again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_3 = MLPClassifier(hidden_layer_sizes=(16,16), random_state=4222)\n",
    "clf_3.fit(vec_train_3, y3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_classifier(labels=[\"true\",\"false\"], title=\"Configuration 4 -- train\", Xt=vec_train_3, yt=y3, clf=clf_3)\n",
    "cm_4=test_classifier(labels=[\"true\",\"false\"], title=\"Configuration 4 -- test\", Xt=vec_test_3, yt=yt3, clf=clf_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### exporting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall(cm):\n",
    "    return cm[0,0] / (cm[0,0] + cm[0,1])\n",
    "def precision(cm):\n",
    "    return cm[0,0] / (cm[0,0] + cm[1,0])\n",
    "\n",
    "def accuracy(cm):\n",
    "    return (cm[0,0] + cm[1,1]) / np.sum(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdflib import Namespace, Graph, Literal\n",
    "from rdflib.namespace import DCTERMS, RDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class mexcore_graph(object):\n",
    "    def __init__(self, name_exp):\n",
    "        self.nsp = {\n",
    "            \"this\": Namespace(\"http://mex.aksw.org/examples/\"),\n",
    "            \"xsd\": Namespace(\"http://www.w3.org/2001/XMLSchema#\"),\n",
    "            \"rdfs\": Namespace(\"http://www.w3.org/2000/01/rdf-schema#\"),\n",
    "            \"mexcore\": Namespace(\"http://mex.aksw.org/mex-core#\"),\n",
    "            \"mexperf\": Namespace(\"http://mex.aksw.org/mex-perf#\"),\n",
    "            \"mexalgo\": Namespace(\"http://mex.aksw.org/mex-algo#\"),\n",
    "            \"prov\": Namespace(\"http://www.w3.org/ns/prov#\"),\n",
    "            \"dct\": DCTERMS,\n",
    "            \"rdf\": RDF\n",
    "        }\n",
    "        self.g = Graph()\n",
    "\n",
    "        for i in nsp.items():\n",
    "            self.g.bind(i[0],i[1])\n",
    "        \n",
    "        self.exp = self.nsp[\"this\"][name_exp]\n",
    "        self.g.add((self.nsp[\"this\"][name_exp], self.nsp[\"rdf\"].type, self.nsp[\"mexcore\"].Experiment))\n",
    "        \n",
    "    def add_dataset(self,dset_name):\n",
    "        self.g.add((self.nsp[\"this\"][dset_name], self.nsp[\"rdf\"].type, self.nsp[\"mexcore\"].dataset))\n",
    "        self.g.add((self.nsp[\"this\"][dset_name], self.nsp[\"rdfs\"].label, Literal(dset_name)))\n",
    "    \n",
    "            \n",
    "    def add_datasets(self, dset_names):\n",
    "        for dset_name in dset_names:\n",
    "            self.add_dataset(dset_name)\n",
    "    \n",
    "    def add_model(self, model_name, literal):\n",
    "        self.g.add((self.nsp[\"this\"][model_name],self.nsp[\"rdf\"].type,self.nsp[\"mexalgo\"].Algorithm))\n",
    "        self.g.add((self.nsp[\"this\"][model_name],self.nsp[\"rdfs\"].label,Literal(model_name)))\n",
    "        self.g.add((self.nsp[\"this\"][model_name],self.nsp[\"dct\"].identifier,Literal(literal)))\n",
    "    \n",
    "    def add_configuration(self, conf_name, used_model, used_dset):\n",
    "        self.g.add((self.nsp[\"this\"][conf_name],self.nsp[\"rdf\"].type,self.nsp[\"mexcore\"].ExperimentConfiguration))\n",
    "        self.g.add((self.nsp[\"this\"][conf_name],self.nsp[\"rdfs\"].label,Literal(conf_name)))\n",
    "        self.g.add((self.nsp[\"this\"][conf_name],self.nsp[\"prov\"].used, self.nsp[\"this\"][used_model]))\n",
    "        self.g.add((self.nsp[\"this\"][conf_name],self.nsp[\"prov\"].used, self.nsp[\"this\"][used_dset]))\n",
    "        self.g.add((self.nsp[\"this\"][conf_name],self.nsp[\"prov\"].wasStartedBy, self.exp))\n",
    "    \n",
    "    def add_measurement(self, meas_name, used_conf, confusion_matrix):\n",
    "        self.g.add((self.nsp[\"this\"][meas_name],self.nsp[\"rdf\"].type,self.nsp[\"mexcore\"].PerformanceMeasure))\n",
    "        self.g.add((self.nsp[\"this\"][meas_name],self.nsp[\"mexperf\"].precision,Literal(str(precision(confusion_matrix)),datatype=self.nsp[\"xsd\"].float)))\n",
    "        self.g.add((self.nsp[\"this\"][meas_name],self.nsp[\"mexperf\"].recall,Literal(str(recall(confusion_matrix)),datatype=self.nsp[\"xsd\"].float)))\n",
    "        self.g.add((self.nsp[\"this\"][meas_name],self.nsp[\"mexperf\"].accuracy,Literal(str(accuracy(confusion_matrix)),datatype=self.nsp[\"xsd\"].float)))\n",
    "        self.g.add((self.nsp[\"this\"][meas_name],self.nsp[\"prov\"].wasGeneratedBy,self.nsp[\"this\"][used_conf]))\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mg = mexcore_graph(\"jonas_weinz_task_2\")\n",
    "mg.add_datasets([\"Dataset01\",\"Dataset02\",\"Dataset03\"])\n",
    "\n",
    "mg.add_model(\"model_a\", \"RandomForestClassifier\")\n",
    "mg.add_model(\"model_b\", \"MLPClassifier\")\n",
    "mg.add_model(\"model_c\", \"MLPClassifier\")\n",
    "\n",
    "mg.add_configuration(\"conf1\",\"model_a\",\"Dataset01\")\n",
    "mg.add_configuration(\"conf2\",\"model_b\", \"Dataset02\")\n",
    "mg.add_configuration(\"conf3a\",\"model_b\", \"Dataset01\")\n",
    "mg.add_configuration(\"conf3b\", \"model_a\", \"Dataset02\")\n",
    "mg.add_configuration(\"conf4\", \"model_c\", \"Dataset03\")\n",
    "\n",
    "mg.add_measurement(\"measure1\", \"conf1\", cm_1)\n",
    "mg.add_measurement(\"measure2\", \"conf2\", cm_2)\n",
    "mg.add_measurement(\"measure3a\", \"conf3a\", cm_3a)\n",
    "mg.add_measurement(\"measure3b\", \"conf3b\", cm_3b)\n",
    "mg.add_measurement(\"measure4\", \"conf4\", cm_4)\n",
    "\n",
    "mg.g.serialize(destination='output.ttl', format='turtle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
