589 lines
16 KiB
Plaintext
589 lines
16 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# NLP-LAB Exercise 02 by Jonas Weinz (2571421)\n",
|
|
"## links:\n",
|
|
"\n",
|
|
"* Article: https://miguelmalvarez.com/2017/03/23/how-can-machine-learning-and-ai-help-solving-the-fake-news-problem/\n",
|
|
" * corresponding code: https://github.com/kjam/random_hackery/blob/master/Attempting%20to%20detect%20fake%20news.ipynb\n",
|
|
"\n",
|
|
"* Tutorial on Datacamp: https://www.datacamp.com/community/tutorials/scikit-learn-fake-news\n",
|
|
"\n",
|
|
"* liar dataset paper: https://www.cs.ucsb.edu/~william/papers/acl2017.pdf\n",
|
|
" * dataset: https://www.cs.ucsb.edu/~william/data/liar_dataset.zip\n",
|
|
"\n",
|
|
"* Mex Vocabulary: http://jens-lehmann.org/files/2015/semantics_mex.pdf"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Dependencies for this Notebook:\n",
|
|
"* library [rdflib](https://github.com/RDFLib/rdflib)\n",
|
|
" * install: `pip3 install rdflib`\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Populating the interactive namespace from numpy and matplotlib\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%pylab inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import numpy as np\n",
|
|
"import itertools\n",
|
|
"import sklearn.utils as sku\n",
|
|
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from sklearn.linear_model import PassiveAggressiveClassifier\n",
|
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
|
"from sklearn import metrics\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"from pprint import pprint as pp\n",
|
|
"from IPython.display import display, Markdown, Latex\n",
|
|
"import collections\n",
|
|
"import traceback\n",
|
|
"import os\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Tools used later"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plot_confusion_matrix(cm, classes,\n",
|
|
" title,\n",
|
|
" normalize=False,\n",
|
|
" cmap=plt.cm.Blues):\n",
|
|
" fig_1, ax_1 = plt.subplots()\n",
|
|
" \"\"\"\n",
|
|
" See full source and example: \n",
|
|
" http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
|
|
" \n",
|
|
" This function prints and plots the confusion matrix.\n",
|
|
" Normalization can be applied by setting `normalize=True`.\n",
|
|
" \"\"\"\n",
|
|
" plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
|
|
" plt.title('Confusion Matrix for:\\n' + title)\n",
|
|
" plt.colorbar()\n",
|
|
" tick_marks = np.arange(len(classes))\n",
|
|
" plt.xticks(tick_marks, classes, rotation=45)\n",
|
|
" plt.yticks(tick_marks, classes)\n",
|
|
"\n",
|
|
" if normalize:\n",
|
|
" cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
|
|
" print(\"Normalized confusion matrix\")\n",
|
|
" else:\n",
|
|
" print('Confusion matrix, without normalization')\n",
|
|
"\n",
|
|
" thresh = cm.max() / 2.\n",
|
|
" \n",
|
|
" pp(cm)\n",
|
|
" \n",
|
|
" for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
|
|
" plt.text(j, i, cm[i, j],\n",
|
|
" horizontalalignment=\"center\",\n",
|
|
" color=\"white\" if cm[i, j] > thresh else \"black\")\n",
|
|
"\n",
|
|
" plt.tight_layout()\n",
|
|
" plt.ylabel('True label')\n",
|
|
" plt.xlabel('Predicted label')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def test_classifier(labels, title, Xt, yt, clf):\n",
|
|
" pred = clf.predict(Xt)\n",
|
|
" score = metrics.accuracy_score(yt, pred)\n",
|
|
" pp(\"score: \" + str(score))\n",
|
|
" cm = metrics.confusion_matrix(yt, pred, labels=labels)\n",
|
|
" plot_confusion_matrix(cm, classes=labels, title=title)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"\"\\nfrom rdflib import Graph, Literal, BNode, RDF, Namespace\\nfrom rdflib.namespace import FOAF, DC, XSD\\n\\nmexcore = Namespace('http://mex.aksw.org/mex-core#')\\nmexperf = Namespace('http://mex.aksw.org/mex-perf#')\\nmexalgo = Namespace('http://mex.aksw.org/mex-algo#')\\nprov = Namespace('http://www.w3.org/ns/prov#')\\n\\ndef create_mex_graph():\\n graph = Graph()\\n graph.bind(mexcore)\\n graph.bind(mexperf)\\n graph.bind(mexalgo)\\n graph.bind(prov)\\n graph.bind(FOAF)\\n graph.bind(DC)\\n graph.bind(XSD)\\n \\n return graph\\n\\ndef mex_performance(experiment, model, dataset, performance, phase='Train', graph=create_mex_graph()):\\n \\n p = BNode()\\n \\n\""
|
|
]
|
|
},
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"'''\n",
|
|
"from rdflib import Graph, Literal, BNode, RDF, Namespace\n",
|
|
"from rdflib.namespace import FOAF, DC, XSD\n",
|
|
"\n",
|
|
"mexcore = Namespace('http://mex.aksw.org/mex-core#')\n",
|
|
"mexperf = Namespace('http://mex.aksw.org/mex-perf#')\n",
|
|
"mexalgo = Namespace('http://mex.aksw.org/mex-algo#')\n",
|
|
"prov = Namespace('http://www.w3.org/ns/prov#')\n",
|
|
"\n",
|
|
"def create_mex_graph():\n",
|
|
" graph = Graph()\n",
|
|
" graph.bind(mexcore)\n",
|
|
" graph.bind(mexperf)\n",
|
|
" graph.bind(mexalgo)\n",
|
|
" graph.bind(prov)\n",
|
|
" graph.bind(FOAF)\n",
|
|
" graph.bind(DC)\n",
|
|
" graph.bind(XSD)\n",
|
|
" \n",
|
|
" return graph\n",
|
|
"\n",
|
|
"def mex_performance(experiment, model, dataset, performance, phase='Train', graph=create_mex_graph()):\n",
|
|
" \n",
|
|
" p = BNode()\n",
|
|
" \n",
|
|
"'''"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"Namespace('http://xmlns.com/foaf/0.1/')"
|
|
]
|
|
},
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"FOAF\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Generate/Download Datasets we are working on\n",
|
|
"\n",
|
|
"* running bash script to download all needed data and store it into the `data` subfolder"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%bash\n",
|
|
"./Task_2_gen_data.sh"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----\n",
|
|
"## configuration 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_1 = pd.read_csv('data/fake_or_real_news.csv').set_index('Unnamed: 0')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* display first 10 entries"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"display(df_1.shape)\n",
|
|
"display(df_1[:10])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* create test dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X1, Xt1, y1, yt1 = train_test_split(df_1.drop('label', axis=1)['text'], df_1.label, test_size=0.25, random_state=4222)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vectorizer_1 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
|
|
"vec_train_1 = vectorizer_1.fit_transform(X1)\n",
|
|
"vec_test_1 = vectorizer_1.transform(Xt1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* trying a Random Forest classifier "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.ensemble import RandomForestClassifier as RFC\n",
|
|
"clf_a = RFC(criterion='entropy', random_state=4222)\n",
|
|
"max_size=10000\n",
|
|
"clf_a.fit(vec_train_1[:max_size], y1[:max_size])\n",
|
|
"test_classifier(labels=[\"FAKE\",\"REAL\"], title=\"Configuration 1, model a -- train\", Xt=vec_train_1,yt=y1, clf=clf_a)\n",
|
|
"test_classifier(labels=[\"FAKE\",\"REAL\"], title=\"Configuration 1, model a -- test\", Xt=vec_test_1,yt=yt1, clf=clf_a)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----\n",
|
|
"## configuration 2\n",
|
|
"\n",
|
|
"* read data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"names = [\n",
|
|
" \"id\",\n",
|
|
" \"label\",\n",
|
|
" \"statement\",\n",
|
|
" \"subjects\",\n",
|
|
" \"speaker\",\n",
|
|
" \"job\",\n",
|
|
" \"state\",\n",
|
|
" \"party\",\n",
|
|
" \"#barely_true\",\n",
|
|
" \"#false\",\n",
|
|
" \"#half_true\",\n",
|
|
" \"#mostly_true\",\n",
|
|
" \"#pants_on_fire\",\n",
|
|
" \"context\"\n",
|
|
"]\n",
|
|
"\n",
|
|
"df_2_train = pd.read_csv(\"data/train.tsv\", delimiter='\\t', names=names)\n",
|
|
"df_2_test = pd.read_csv(\"data/test.tsv\", delimiter='\\t', names=names)\n",
|
|
"df_2_valid= pd.read_csv(\"data/valid.tsv\", delimiter='\\t', names=names)\n",
|
|
"\n",
|
|
"# use only 'False' and 'True' statements\n",
|
|
"df_2_train = df_2_train[df_2_train['label'].isin([\"false\",\"true\"])]\n",
|
|
"df_2_test = df_2_test[df_2_test['label'].isin([\"false\",\"true\"])]\n",
|
|
"df_2_valid = df_2_valid[df_2_valid['label'].isin([\"false\",\"true\"])]\n",
|
|
"\n",
|
|
"display(Markdown(\"----\\n#### Train Data:\"))\n",
|
|
"display(df_2_train.head())\n",
|
|
"display(Markdown(\"----\\n#### Test Data:\"))\n",
|
|
"display(df_2_test.head())\n",
|
|
"display(Markdown(\"----\\n#### Valid Data:\"))\n",
|
|
"display(df_2_valid.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### tdidf vectorizer on new dataset\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X2 = df_2_train['statement']\n",
|
|
"y2 = df_2_train['label']\n",
|
|
"Xt2 = df_2_test['statement']\n",
|
|
"yt2 = df_2_test['label']\n",
|
|
"Xv2 = df_2_valid['statement']\n",
|
|
"yv2 = df_2_valid['label']\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vectorizer_2 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
|
|
"vec_train_2 = vectorizer_2.fit_transform(X2)\n",
|
|
"vec_test_2 = vectorizer_2.transform(Xt2)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"?MLPClassifier"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* trying a MLP as classifier "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.neural_network import MLPClassifier\n",
|
|
"clf_b = MLPClassifier(hidden_layer_sizes=(100,), random_state=4222)\n",
|
|
"clf_b.fit(vec_train_2, y2)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- train\", Xt=vec_train_2, yt=y2, clf=clf_b)\n",
|
|
"test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- test\", Xt=vec_test_2, yt=yt2, clf=clf_b)\n",
|
|
"test_classifier(labels=[\"true\", \"false\"], title=\"configuration 2 -- valid\", Xt=vectorizer_2.transform(Xv2), yt=yv2, clf=clf_b)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----\n",
|
|
"## configuration 3"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"yt2_c3 = yt2.copy()\n",
|
|
"yt2_c3[yt2_c3 == \"true\"] = \"REAL\"\n",
|
|
"yt2_c3[yt2_c3 == \"false\"] = \"FAKE\"\n",
|
|
"\n",
|
|
"test_classifier(labels=[\"REAL\", \"FAKE\"], \n",
|
|
" title=\"configuration 3: model a) → dataset 2\",\n",
|
|
" Xt=vectorizer_1.transform(Xt2),\n",
|
|
" yt=yt2_c3, clf=clf_a)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"yt1_c3 = yt1.copy()\n",
|
|
"yt1_c3[yt1_c3 == \"REAL\"] = \"true\"\n",
|
|
"yt1_c3[yt1_c3 == \"FAKE\"] = \"false\"\n",
|
|
"\n",
|
|
"test_classifier(labels=[\"true\", \"false\"], \n",
|
|
" title=\"configuration 3: model b) → dataset 1\",\n",
|
|
" Xt=vectorizer_2.transform(Xt1),\n",
|
|
" yt=yt1_c3, clf=clf_b)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"----\n",
|
|
"## configuration 4)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_dataset3_split(dataset1_in, dataset2_in):\n",
|
|
" try:\n",
|
|
" print('processing datasets')\n",
|
|
" print('ds1=', dataset1_in)\n",
|
|
" print('ds2=', dataset2_in)\n",
|
|
"\n",
|
|
" print('-- fake news')\n",
|
|
" df1 = pd.read_csv(dataset1_in, sep=',', usecols=['title','text','label'])\n",
|
|
" df1['claim'] = df1[['title', 'text']].apply(lambda x: '. '.join(x), axis=1)\n",
|
|
" del df1['title']\n",
|
|
" del df1['text']\n",
|
|
" df1.rename(index=str, columns={'label': 'y'}, inplace=True)\n",
|
|
" print(df1.keys())\n",
|
|
" print(len(df1[df1['y']=='REAL']))\n",
|
|
" print(len(df1[df1['y']=='FAKE']))\n",
|
|
" df1['y'] = np.where(df1['y'] == 'FAKE', 'false', 'true')\n",
|
|
" print(len(df1))\n",
|
|
"\n",
|
|
" print('-- liar liar')\n",
|
|
" df2 = pd.read_csv(dataset2_in, sep='\\t', header=None, usecols=[1,2], names=['y', 'claim'])\n",
|
|
" print(df2.keys())\n",
|
|
" print(set(df2.y), len(df2))\n",
|
|
" print(len(df2[df2['y'] == 'true']))\n",
|
|
" print(len(df2[df2['y'] == 'false']))\n",
|
|
" df2=df2[(df2['y'] == 'true') | (df2['y'] == 'false')]\n",
|
|
" print(set(df2.y), len(df2))\n",
|
|
"\n",
|
|
" df3=pd.concat([df1, df2], ignore_index=True)\n",
|
|
"\n",
|
|
" print(df3['y'].value_counts())\n",
|
|
" print('done')\n",
|
|
" return train_test_split(df3['claim'], df3['y'], test_size=0.3, random_state=4222)\n",
|
|
" except Exception as e:\n",
|
|
" print(e)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X3, Xt3, y3, yt3 = get_dataset3_split('data/fake_or_real_news.csv', 'data/train.tsv')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vectorizer_3 = TfidfVectorizer(stop_words='english', max_df=0.7)\n",
|
|
"vec_train_3 = vectorizer_3.fit_transform(X3)\n",
|
|
"vec_test_3 = vectorizer_3.transform(Xt3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* using MLP again"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf_3 = MLPClassifier(hidden_layer_sizes=(16,16), random_state=4222)\n",
|
|
"clf_3.fit(vec_train_3, y3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"test_classifier(labels=[\"true\",\"false\"], title=\"Configuration 4 -- train\", Xt=vec_train_3, yt=y3, clf=clf_3)\n",
|
|
"test_classifier(labels=[\"true\",\"false\"], title=\"Configuration 4 -- test\", Xt=vec_test_3, yt=yt3, clf=clf_3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|