Jonas: first stuff for task 2
This commit is contained in:
Jonas Weinz
parent
12081cb5df
commit
42ac5d8d81
@ -6,13 +6,25 @@
|
||||
"source": [
|
||||
"# NLP-LAB Exercise 01 by jonas weinz\n",
|
||||
"----\n",
|
||||
"## links:\n",
|
||||
"\n",
|
||||
"* Article: https://miguelmalvarez.com/2017/03/23/how-can-machine-learning-and-ai-help-solving-the-fake-news-problem/\n",
|
||||
" * corresponding code: https://github.com/kjam/random_hackery/blob/master/Attempting%20to%20detect%20fake%20news.ipynb\n",
|
||||
"\n",
|
||||
"* Tutorial on Datacamp: https://www.datacamp.com/community/tutorials/scikit-learn-fake-news\n",
|
||||
"\n",
|
||||
"* liar dataset paper: https://www.cs.ucsb.edu/~william/papers/acl2017.pdf\n",
|
||||
" * dataset: https://www.cs.ucsb.edu/~william/data/liar_dataset.zip\n",
|
||||
"\n",
|
||||
"## Dependencies for this Notebook:\n",
|
||||
"* library [rdflib](https://github.com/RDFLib/rdflib)\n",
|
||||
" * install: `pip3 install rdflib`\n",
|
||||
"* "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -24,7 +36,287 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%pylab inline"
|
||||
"%pylab ipympl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import itertools\n",
|
||||
"import sklearn.utils as sku\n",
|
||||
"import os"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Generate/Download Datasets we are working on\n",
|
||||
"\n",
|
||||
"* running bash script to download all needed data and store it into the `data` subfolder"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"================================================================================\n",
|
||||
"downloading and unpacking https://www.cs.ucsb.edu/~william/data/liar_dataset.zip if not already existing\n",
|
||||
"================================================================================\n",
|
||||
"UnZip 6.00 of 20 April 2009, by Debian. Original by Info-ZIP.\n",
|
||||
"\n",
|
||||
"Latest sources and executables are at ftp://ftp.info-zip.org/pub/infozip/ ;\n",
|
||||
"see ftp://ftp.info-zip.org/pub/infozip/UnZip.html for other sites.\n",
|
||||
"\n",
|
||||
"Compiled with gcc 6.3.0 20170415 for Unix (Linux ELF).\n",
|
||||
"\n",
|
||||
"UnZip special compilation options:\n",
|
||||
" ACORN_FTYPE_NFS\n",
|
||||
" COPYRIGHT_CLEAN (PKZIP 0.9x unreducing method not supported)\n",
|
||||
" SET_DIR_ATTRIB\n",
|
||||
" SYMLINKS (symbolic links supported, if RTL and file system permit)\n",
|
||||
" TIMESTAMP\n",
|
||||
" UNIXBACKUP\n",
|
||||
" USE_EF_UT_TIME\n",
|
||||
" USE_UNSHRINK (PKZIP/Zip 1.x unshrinking method supported)\n",
|
||||
" USE_DEFLATE64 (PKZIP 4.x Deflate64(tm) supported)\n",
|
||||
" UNICODE_SUPPORT [wide-chars, char coding: UTF-8] (handle UTF-8 paths)\n",
|
||||
" LARGE_FILE_SUPPORT (large files over 2 GiB supported)\n",
|
||||
" ZIP64_SUPPORT (archives using Zip64 for large files supported)\n",
|
||||
" USE_BZIP2 (PKZIP 4.6+, using bzip2 lib version 1.0.6, 6-Sept-2010)\n",
|
||||
" VMS_TEXT_CONV\n",
|
||||
" WILD_STOP_AT_DIR\n",
|
||||
" [decryption, version 2.11 of 05 Jan 2007]\n",
|
||||
"\n",
|
||||
"UnZip and ZipInfo environment options:\n",
|
||||
" UNZIP: [none]\n",
|
||||
" UNZIPOPT: [none]\n",
|
||||
" ZIPINFO: [none]\n",
|
||||
" ZIPINFOOPT: [none]\n",
|
||||
"Archive: liar_dataset.zip\n",
|
||||
" inflating: README \n",
|
||||
" inflating: test.tsv \n",
|
||||
" inflating: train.tsv \n",
|
||||
" inflating: valid.tsv \n",
|
||||
"================================================================================\n",
|
||||
"successfully finished action: downloading and unpacking https://www.cs.ucsb.edu/~william/data/liar_dataset.zip if not already existing\n",
|
||||
"================================================================================\n",
|
||||
"================================================================================\n",
|
||||
"downloading and unpacking https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip if not already existing\n",
|
||||
"================================================================================\n",
|
||||
"Archive: fake_or_real_news.csv.zip\n",
|
||||
" inflating: fake_or_real_news.csv \n",
|
||||
" creating: __MACOSX/\n",
|
||||
" inflating: __MACOSX/._fake_or_real_news.csv \n",
|
||||
"================================================================================\n",
|
||||
"successfully finished action: downloading and unpacking https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip if not already existing\n",
|
||||
"================================================================================\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
|
||||
" Dload Upload Total Spent Left Speed\n",
|
||||
"100 989k 100 989k 0 0 366k 0 0:00:02 0:00:02 --:--:-- 366k\n",
|
||||
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
|
||||
" Dload Upload Total Spent Left Speed\n",
|
||||
"100 11.3M 100 11.3M 0 0 2590k 0 0:00:04 0:00:04 --:--:-- 2751k\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%bash\n",
|
||||
"./Task_2_gen_data.sh"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Read in fake news table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_1 = pd.read_csv('data/fake_or_real_news.csv').set_index('Unnamed: 0')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* display first 10 entries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(6335, 3)"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>title</th>\n",
|
||||
" <th>text</th>\n",
|
||||
" <th>label</th>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>Unnamed: 0</th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>8476</th>\n",
|
||||
" <td>You Can Smell Hillary’s Fear</td>\n",
|
||||
" <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
|
||||
" <td>FAKE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10294</th>\n",
|
||||
" <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
|
||||
" <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
|
||||
" <td>FAKE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3608</th>\n",
|
||||
" <td>Kerry to go to Paris in gesture of sympathy</td>\n",
|
||||
" <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
|
||||
" <td>REAL</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10142</th>\n",
|
||||
" <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
|
||||
" <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
|
||||
" <td>FAKE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>875</th>\n",
|
||||
" <td>The Battle of New York: Why This Primary Matters</td>\n",
|
||||
" <td>It's primary day in New York and front-runners...</td>\n",
|
||||
" <td>REAL</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6903</th>\n",
|
||||
" <td>Tehran, USA</td>\n",
|
||||
" <td>\\nI’m not an immigrant, but my grandparents ...</td>\n",
|
||||
" <td>FAKE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7341</th>\n",
|
||||
" <td>Girl Horrified At What She Watches Boyfriend D...</td>\n",
|
||||
" <td>Share This Baylee Luciani (left), Screenshot o...</td>\n",
|
||||
" <td>FAKE</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>95</th>\n",
|
||||
" <td>‘Britain’s Schindler’ Dies at 106</td>\n",
|
||||
" <td>A Czech stockbroker who saved more than 650 Je...</td>\n",
|
||||
" <td>REAL</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4869</th>\n",
|
||||
" <td>Fact check: Trump and Clinton at the 'commande...</td>\n",
|
||||
" <td>Hillary Clinton and Donald Trump made some ina...</td>\n",
|
||||
" <td>REAL</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2909</th>\n",
|
||||
" <td>Iran reportedly makes new push for uranium con...</td>\n",
|
||||
" <td>Iranian negotiators reportedly have made a las...</td>\n",
|
||||
" <td>REAL</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" title \\\n",
|
||||
"Unnamed: 0 \n",
|
||||
"8476 You Can Smell Hillary’s Fear \n",
|
||||
"10294 Watch The Exact Moment Paul Ryan Committed Pol... \n",
|
||||
"3608 Kerry to go to Paris in gesture of sympathy \n",
|
||||
"10142 Bernie supporters on Twitter erupt in anger ag... \n",
|
||||
"875 The Battle of New York: Why This Primary Matters \n",
|
||||
"6903 Tehran, USA \n",
|
||||
"7341 Girl Horrified At What She Watches Boyfriend D... \n",
|
||||
"95 ‘Britain’s Schindler’ Dies at 106 \n",
|
||||
"4869 Fact check: Trump and Clinton at the 'commande... \n",
|
||||
"2909 Iran reportedly makes new push for uranium con... \n",
|
||||
"\n",
|
||||
" text label \n",
|
||||
"Unnamed: 0 \n",
|
||||
"8476 Daniel Greenfield, a Shillman Journalism Fello... FAKE \n",
|
||||
"10294 Google Pinterest Digg Linkedin Reddit Stumbleu... FAKE \n",
|
||||
"3608 U.S. Secretary of State John F. Kerry said Mon... REAL \n",
|
||||
"10142 — Kaydee King (@KaydeeKing) November 9, 2016 T... FAKE \n",
|
||||
"875 It's primary day in New York and front-runners... REAL \n",
|
||||
"6903 \\nI’m not an immigrant, but my grandparents ... FAKE \n",
|
||||
"7341 Share This Baylee Luciani (left), Screenshot o... FAKE \n",
|
||||
"95 A Czech stockbroker who saved more than 650 Je... REAL \n",
|
||||
"4869 Hillary Clinton and Donald Trump made some ina... REAL \n",
|
||||
"2909 Iranian negotiators reportedly have made a las... REAL "
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"display(df_1.shape)\n",
|
||||
"display(df_1[:10])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"* create test dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -32,7 +324,25 @@
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"source": [
|
||||
"def create_test_dset(dset, cutoff=0.7):\n",
|
||||
" shuffled = sku.shuffle(dset)\n",
|
||||
" y = shuffled.label\n",
|
||||
" df_1 = shuffled.drop('label', axis=1)\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"cut = 0.7\n",
|
||||
"\n",
|
||||
"y = df_1.label\n",
|
||||
"df_1 = df_1.drop('label', axis=1)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
72
Jonas_Solutions/Task_2_gen_data.sh
Executable file
72
Jonas_Solutions/Task_2_gen_data.sh
Executable file
@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# helper functions:
|
||||
|
||||
function lineprint {
|
||||
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' =
|
||||
}
|
||||
|
||||
function message {
|
||||
lineprint
|
||||
printf "$1\n"
|
||||
lineprint
|
||||
}
|
||||
|
||||
current_action="IDLE"
|
||||
|
||||
function confirm_action {
|
||||
message "successfully finished action: $current_action"
|
||||
}
|
||||
|
||||
function set_action {
|
||||
current_action="$1"
|
||||
message "$1"
|
||||
}
|
||||
|
||||
function perform {
|
||||
"$@"
|
||||
local status=$?
|
||||
if [ $status -ne 0 ]
|
||||
then
|
||||
message "$current_action failed!"
|
||||
fi
|
||||
return $status
|
||||
}
|
||||
|
||||
function perform_and_exit {
|
||||
perform "$@" || exit
|
||||
}
|
||||
|
||||
# Downloading and unzipping dataset
|
||||
|
||||
D1_URL=https://www.cs.ucsb.edu/~william/data/liar_dataset.zip
|
||||
D1_ZIP=${D1_URL##*/}
|
||||
|
||||
D2_URL=https://raw.githubusercontent.com/GeorgeMcIntire/fake_real_news_dataset/master/fake_or_real_news.csv.zip
|
||||
D2_ZIP=${D2_URL##*/}
|
||||
|
||||
set_action "downloading and unpacking $D1_URL if not already existing"
|
||||
|
||||
# testing for unzip:
|
||||
perform_and_exit unzip -v
|
||||
|
||||
perform_and_exit mkdir -p ./data
|
||||
perform_and_exit cd ./data/
|
||||
|
||||
if [ ! -e $D1_ZIP ];
|
||||
then
|
||||
perform_and_exit curl $D1_URL --output ./$D1_ZIP
|
||||
perform_and_exit unzip $D1_ZIP
|
||||
fi
|
||||
|
||||
confirm_action
|
||||
|
||||
set_action "downloading and unpacking $D2_URL if not already existing"
|
||||
|
||||
if [ ! -e $D2_ZIP ];
|
||||
then
|
||||
perform_and_exit curl $D2_URL --output ./$D2_ZIP
|
||||
perform_and_exit unzip $D2_ZIP
|
||||
fi
|
||||
|
||||
confirm_action
|
||||
85
Jonas_Solutions/Untitled.ipynb
Normal file
85
Jonas_Solutions/Untitled.ipynb
Normal file
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user