{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# NLP-LAB Exercise 02 by Jonas Weinz (2571421)\n", "## links:\n", "\n", "* Article: https://miguelmalvarez.com/2017/03/23/how-can-machine-learning-and-ai-help-solving-the-fake-news-problem/\n", " * corresponding code: https://github.com/kjam/random_hackery/blob/master/Attempting%20to%20detect%20fake%20news.ipynb\n", "\n", "* Tutorial on Datacamp: https://www.datacamp.com/community/tutorials/scikit-learn-fake-news\n", "\n", "* liar dataset paper: https://www.cs.ucsb.edu/~william/papers/acl2017.pdf\n", " * dataset: https://www.cs.ucsb.edu/~william/data/liar_dataset.zip\n", "\n", "* Mex Vocabulary: http://jens-lehmann.org/files/2015/semantics_mex.pdf" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Dependencies for this Notebook:\n", "* library [rdflib](https://github.com/RDFLib/rdflib)\n", " * install: `pip3 install rdflib`\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import itertools\n", "import sklearn.utils as sku\n", "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.linear_model import PassiveAggressiveClassifier\n", "from sklearn.naive_bayes import MultinomialNB\n", "from sklearn import metrics\n", "import matplotlib.pyplot as plt\n", "from pprint import pprint as pp\n", "from IPython.display import display, Markdown, Latex\n", "import collections\n", "import traceback\n", "import os\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Tools used later" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def plot_confusion_matrix(cm, classes,\n", " title,\n", " normalize=False,\n", " cmap=plt.cm.Blues):\n", " fig_1, ax_1 = plt.subplots()\n", " \"\"\"\n", " See full source and example: \n", " http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n", " \n", " This function prints and plots the confusion matrix.\n", " Normalization can be applied by setting `normalize=True`.\n", " \"\"\"\n", " plt.imshow(cm, interpolation='nearest', cmap=cmap)\n", " plt.title('Confusion Matrix for:\\n' + title)\n", " plt.colorbar()\n", " tick_marks = np.arange(len(classes))\n", " plt.xticks(tick_marks, classes, rotation=45)\n", " plt.yticks(tick_marks, classes)\n", "\n", " if normalize:\n", " cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", " print(\"Normalized confusion matrix\")\n", " else:\n", " print('Confusion matrix, without normalization')\n", "\n", " thresh = cm.max() / 2.\n", " \n", " pp(cm)\n", " \n", " for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n", " plt.text(j, i, cm[i, j],\n", " horizontalalignment=\"center\",\n", " color=\"white\" if cm[i, j] > thresh else \"black\")\n", "\n", " plt.tight_layout()\n", " plt.ylabel('True label')\n", " plt.xlabel('Predicted label')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def test_classifier(labels, title, Xt, yt, clf):\n", " pred = clf.predict(Xt)\n", " score = metrics.accuracy_score(yt, pred)\n", " pp(\"score: \" + str(score))\n", " cm = metrics.confusion_matrix(yt, pred, labels=labels)\n", " plot_confusion_matrix(cm, classes=labels, title=title)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"\\nfrom rdflib import Graph, Literal, BNode, RDF, Namespace\\nfrom rdflib.namespace import FOAF, DC, XSD\\n\\nmexcore = Namespace('http://mex.aksw.org/mex-core#')\\nmexperf = Namespace('http://mex.aksw.org/mex-perf#')\\nmexalgo = Namespace('http://mex.aksw.org/mex-algo#')\\nprov = Namespace('http://www.w3.org/ns/prov#')\\n\\ndef create_mex_graph():\\n graph = Graph()\\n graph.bind(mexcore)\\n graph.bind(mexperf)\\n graph.bind(mexalgo)\\n graph.bind(prov)\\n graph.bind(FOAF)\\n graph.bind(DC)\\n graph.bind(XSD)\\n \\n return graph\\n\\ndef mex_performance(experiment, model, dataset, performance, phase='Train', graph=create_mex_graph()):\\n \\n p = BNode()\\n \\n\"" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "'''\n", "from rdflib import Graph, Literal, BNode, RDF, Namespace\n", "from rdflib.namespace import FOAF, DC, XSD\n", "\n", "mexcore = Namespace('http://mex.aksw.org/mex-core#')\n", "mexperf = Namespace('http://mex.aksw.org/mex-perf#')\n", "mexalgo = Namespace('http://mex.aksw.org/mex-algo#')\n", "prov = Namespace('http://www.w3.org/ns/prov#')\n", "\n", "def create_mex_graph():\n", " graph = Graph()\n", " graph.bind(mexcore)\n", " graph.bind(mexperf)\n", " graph.bind(mexalgo)\n", " graph.bind(prov)\n", " graph.bind(FOAF)\n", " graph.bind(DC)\n", " graph.bind(XSD)\n", " \n", " return graph\n", "\n", "def mex_performance(experiment, model, dataset, performance, phase='Train', graph=create_mex_graph()):\n", " \n", " p = BNode()\n", " \n", "'''" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Namespace('http://xmlns.com/foaf/0.1/')" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "FOAF\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Generate/Download Datasets we are working on\n", "\n", "* running bash script to download all needed data and store it into the `data` subfolder" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "./Task_2_gen_data.sh" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## configuration 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "df_1 = pd.read_csv('data/fake_or_real_news.csv').set_index('Unnamed: 0')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* display first 10 entries" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "display(df_1.shape)\n", "display(df_1[:10])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* create test dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X1, Xt1, y1, yt1 = train_test_split(df_1.drop('label', axis=1)['text'], df_1.label, test_size=0.25, random_state=4222)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "vectorizer_1 = TfidfVectorizer(stop_words='english', max_df=0.7)\n", "vec_train_1 = vectorizer_1.fit_transform(X1)\n", "vec_test_1 = vectorizer_1.transform(Xt1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* trying a Random Forest classifier " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.ensemble import RandomForestClassifier as RFC\n", "clf_a = RFC(criterion='entropy', random_state=4222)\n", "max_size=10000\n", "clf_a.fit(vec_train_1[:max_size], y1[:max_size])\n", "test_classifier(labels=[\"FAKE\",\"REAL\"], title=\"Configuration 1, model a -- train\", Xt=vec_train_1,yt=y1, clf=clf_a)\n", "test_classifier(labels=[\"FAKE\",\"REAL\"], title=\"Configuration 1, model a -- test\", Xt=vec_test_1,yt=yt1, clf=clf_a)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## configuration 2\n", "\n", "* read data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "names = [\n", " \"id\",\n", " \"label\",\n", " \"statement\",\n", " \"subjects\",\n", " \"speaker\",\n", " \"job\",\n", " \"state\",\n", " \"party\",\n", " \"#barely_true\",\n", " \"#false\",\n", " \"#half_true\",\n", " \"#mostly_true\",\n", " \"#pants_on_fire\",\n", " \"context\"\n", "]\n", "\n", "df_2_train = pd.read_csv(\"data/train.tsv\", delimiter='\\t', names=names)\n", "df_2_test = pd.read_csv(\"data/test.tsv\", delimiter='\\t', names=names)\n", "df_2_valid= pd.read_csv(\"data/valid.tsv\", delimiter='\\t', names=names)\n", "\n", "# use only 'False' and 'True' statements\n", "df_2_train = df_2_train[df_2_train['label'].isin([\"false\",\"true\"])]\n", "df_2_test = df_2_test[df_2_test['label'].isin([\"false\",\"true\"])]\n", "df_2_valid = df_2_valid[df_2_valid['label'].isin([\"false\",\"true\"])]\n", "\n", "display(Markdown(\"----\\n#### Train Data:\"))\n", "display(df_2_train.head())\n", "display(Markdown(\"----\\n#### Test Data:\"))\n", "display(df_2_test.head())\n", "display(Markdown(\"----\\n#### Valid Data:\"))\n", "display(df_2_valid.head())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### tdidf vectorizer on new dataset\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X2 = df_2_train['statement']\n", "y2 = df_2_train['label']\n", "Xt2 = df_2_test['statement']\n", "yt2 = df_2_test['label']\n", "Xv2 = df_2_valid['statement']\n", "yv2 = df_2_valid['label']\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "vectorizer_2 = TfidfVectorizer(stop_words='english', max_df=0.7)\n", "vec_train_2 = vectorizer_2.fit_transform(X2)\n", "vec_test_2 = vectorizer_2.transform(Xt2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "?MLPClassifier" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* trying a MLP as classifier " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.neural_network import MLPClassifier\n", "clf_b = MLPClassifier(hidden_layer_sizes=(100,), random_state=4222)\n", "clf_b.fit(vec_train_2, y2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- train\", Xt=vec_train_2, yt=y2, clf=clf_b)\n", "test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- test\", Xt=vec_test_2, yt=yt2, clf=clf_b)\n", "test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- valid\", Xt=vectorizer_2.transform(Xv2), yt=yv2, clf=clf_b)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## configuration 3" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "yt2_c3 = yt2.copy()\n", "yt2_c3[yt2_c3 == \"true\"] = \"REAL\"\n", "yt2_c3[yt2_c3 == \"false\"] = \"FAKE\"\n", "\n", "test_classifier(labels=[\"REAL\", \"FAKE\"], \n", " title=\"configuration 3: model a) → dataset 2\",\n", " Xt=vectorizer_1.transform(Xt2),\n", " yt=yt2_c3, clf=clf_a)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "yt1_c3 = yt1.copy()\n", "yt1_c3[yt1_c3 == \"REAL\"] = \"true\"\n", "yt1_c3[yt1_c3 == \"FAKE\"] = \"false\"\n", "\n", "test_classifier(labels=[\"true\", \"false\"], \n", " title=\"configuration 3: model b) → dataset 1\",\n", " Xt=vectorizer_2.transform(Xt1),\n", " yt=yt1_c3, clf=clf_b)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "## configuration 4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def get_dataset3_split(dataset1_in, dataset2_in):\n", " try:\n", " print('processing datasets')\n", " print('ds1=', dataset1_in)\n", " print('ds2=', dataset2_in)\n", "\n", " print('-- fake news')\n", " df1 = pd.read_csv(dataset1_in, sep=',', usecols=['title','text','label'])\n", " df1['claim'] = df1[['title', 'text']].apply(lambda x: '. '.join(x), axis=1)\n", " del df1['title']\n", " del df1['text']\n", " df1.rename(index=str, columns={'label': 'y'}, inplace=True)\n", " print(df1.keys())\n", " print(len(df1[df1['y']=='REAL']))\n", " print(len(df1[df1['y']=='FAKE']))\n", " df1['y'] = np.where(df1['y'] == 'FAKE', 'false', 'true')\n", " print(len(df1))\n", "\n", " print('-- liar liar')\n", " df2 = pd.read_csv(dataset2_in, sep='\\t', header=None, usecols=[1,2], names=['y', 'claim'])\n", " print(df2.keys())\n", " print(set(df2.y), len(df2))\n", " print(len(df2[df2['y'] == 'true']))\n", " print(len(df2[df2['y'] == 'false']))\n", " df2=df2[(df2['y'] == 'true') | (df2['y'] == 'false')]\n", " print(set(df2.y), len(df2))\n", "\n", " df3=pd.concat([df1, df2], ignore_index=True)\n", "\n", " print(df3['y'].value_counts())\n", " print('done')\n", " return train_test_split(df3['claim'], df3['y'], test_size=0.3, random_state=4222)\n", " except Exception as e:\n", " print(e)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X3, Xt3, y3, yt3 = get_dataset3_split('data/fake_or_real_news.csv', 'data/train.tsv')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "vectorizer_3 = TfidfVectorizer(stop_words='english', max_df=0.7)\n", "vec_train_3 = vectorizer_3.fit_transform(X3)\n", "vec_test_3 = vectorizer_3.transform(Xt3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* using MLP again" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "clf_3 = MLPClassifier(hidden_layer_sizes=(16,16), random_state=4222)\n", "clf_3.fit(vec_train_3, y3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_classifier(labels=[\"true\",\"false\"], title=\"Configuration 4 -- train\", Xt=vec_train_3, yt=y3, clf=clf_3)\n", "test_classifier(labels=[\"true\",\"false\"], title=\"Configuration 4 -- test\", Xt=vec_test_3, yt=yt3, clf=clf_3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }