519 lines
17 KiB
Plaintext
519 lines
17 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 51,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" text sentiment\n",
|
|
"0 RT @NancyLeeGrahn: How did everyone feel about... Neutral\n",
|
|
"1 RT @ScottWalker: Didn't catch the full #GOPdeb... Positive\n",
|
|
"2 RT @TJMShow: No mention of Tamir Rice and the ... Neutral\n",
|
|
"3 RT @RobGeorge: That Carly Fiorina is trending ... Positive\n",
|
|
"4 RT @DanScavino: #GOPDebate w/ @realDonaldTrump... Positive\n",
|
|
"5 RT @GregAbbott_TX: @TedCruz: \"On my first day ... Positive\n",
|
|
"6 RT @warriorwoman91: I liked her and was happy ... Negative\n",
|
|
"7 Going on #MSNBC Live with @ThomasARoberts arou... Neutral\n",
|
|
"8 Deer in the headlights RT @lizzwinstead: Ben C... Negative\n",
|
|
"9 RT @NancyOsborne180: Last night's debate prove... Negative\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import numpy as np \n",
|
|
"import pandas as pd \n",
|
|
"from sklearn.feature_extraction.text import CountVectorizer\n",
|
|
"from keras.preprocessing.text import Tokenizer\n",
|
|
"from keras.preprocessing.sequence import pad_sequences\n",
|
|
"from keras.models import Sequential\n",
|
|
"from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from keras.utils.np_utils import to_categorical\n",
|
|
"import re\n",
|
|
"\n",
|
|
"'''\n",
|
|
"Task 3: playing with NN framwork/keras and basic sentiment analysis\n",
|
|
"- use the following model as a baseline and improve it!\n",
|
|
"- export your metadata (just basic hyperparameters and outcomes for test data!)\n",
|
|
"- test data = 0.3 (not in this example, change it!)\n",
|
|
"- random_state = 4222\n",
|
|
"- no need to cross-validation!\n",
|
|
"'''\n",
|
|
"\n",
|
|
"# parameters\n",
|
|
"max_fatures = 1000\n",
|
|
"embed_dim = 128\n",
|
|
"lstm_out = 196\n",
|
|
"dropout = 0.1\n",
|
|
"dropout_1d = 0.4\n",
|
|
"recurrent_dropout = 0.1\n",
|
|
"random_state = 4222\n",
|
|
"validation_size = 1000\n",
|
|
"batch_size = 16\n",
|
|
"epochs=2\n",
|
|
"verbose= 2\n",
|
|
"\n",
|
|
"df = pd.read_csv('/Users/Carsten/GitRepos/NLP-LAB/Carsten_Solutions/sets/sentiment-analysis/dataset_sentiment.csv')\n",
|
|
"df = df[['text','sentiment']]\n",
|
|
"print(df[0:10])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 52,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = df[df.sentiment != \"Neutral\"]\n",
|
|
"#replace all capital letters with its small character\n",
|
|
"df['text'] = df['text'].apply(lambda x: x.lower())\n",
|
|
"#removes all rt in messages, often occuring in front of twitter raw data\n",
|
|
"df['text'] = df['text'].apply(lambda x: x.replace('rt',' '))\n",
|
|
"#only accepts alphanumerical characters, erease all other characters\n",
|
|
"df['text'] = df['text'].apply((lambda x: re.sub('[^a-zA-Z0-9\\s]','',x)))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 53,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"2236\n",
|
|
"8493\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#evaluate distribution of positive and negative examples\n",
|
|
"print(len(df[df.sentiment == \"Positive\"]))\n",
|
|
"print(len(df[df.sentiment == \"Negative\"]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 54,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" text sentiment\n",
|
|
"1 scottwalker didnt catch the full gopdebate l... Positive\n",
|
|
"3 robgeorge that carly fiorina is trending ho... Positive\n",
|
|
"4 danscavino gopdebate w realdonaldtrump deliv... Positive\n",
|
|
"5 gregabbotttx tedcruz on my first day i will ... Positive\n",
|
|
"6 warriorwoman91 i liked her and was happy whe... Negative\n",
|
|
"8 deer in the headlights lizzwinstead ben cars... Negative\n",
|
|
"9 nancyosborne180 last nights debate proved it... Negative\n",
|
|
"10 jgreendc realdonaldtrump in all fairness billc... Negative\n",
|
|
"11 waynedupreeshow just woke up to tweet this o... Positive\n",
|
|
"12 me reading my familys comments about how great... Negative\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(df[0:10])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 55,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[[ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 363 122 1 722\n",
|
|
" 2 39 58 237 36 210 6 174 12 742]\n",
|
|
" [ 0 0 0 0 0 0 0 0 0 0 0 16 284 252 5 818 102 167\n",
|
|
" 26 135 6 1 172 12 2 233 723 17]\n",
|
|
" [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 309 23\n",
|
|
" 1 216 12 1 702 6 185 207 371 670]\n",
|
|
" [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 17 53\n",
|
|
" 262 410 9 82 303 441 62 194 2 51]\n",
|
|
" [ 0 0 0 0 0 0 0 0 0 9 167 8 21 63 9 612 188 21\n",
|
|
" 189 4 34 1 562 19 819 2 44 743]]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#generates a tokenizer with fixed lenght\n",
|
|
"tok = Tokenizer(num_words=max_fatures, split=' ')\n",
|
|
"#train tokenizer\n",
|
|
"tok.fit_on_texts(df['text'].values)\n",
|
|
"#transforms each sentence to a sequence of intgers\n",
|
|
"X = tok.texts_to_sequences(df['text'].values)\n",
|
|
"\n",
|
|
"#print(X[:20])\n",
|
|
"\n",
|
|
"#for each of these sequences it transforms it to an array of same length by inserting 0 in front by standard configuration of parameters \n",
|
|
"X = pad_sequences(X)\n",
|
|
"print(X[:5])\n",
|
|
"\n",
|
|
"\n",
|
|
"#print(type(X))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 56,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"_________________________________________________________________\n",
|
|
"Layer (type) Output Shape Param # \n",
|
|
"=================================================================\n",
|
|
"embedding_5 (Embedding) (None, 28, 128) 128000 \n",
|
|
"_________________________________________________________________\n",
|
|
"spatial_dropout1d_5 (Spatial (None, 28, 128) 0 \n",
|
|
"_________________________________________________________________\n",
|
|
"lstm_5 (LSTM) (None, 196) 254800 \n",
|
|
"_________________________________________________________________\n",
|
|
"dense_5 (Dense) (None, 2) 394 \n",
|
|
"=================================================================\n",
|
|
"Total params: 383,194\n",
|
|
"Trainable params: 383,194\n",
|
|
"Non-trainable params: 0\n",
|
|
"_________________________________________________________________\n",
|
|
"None\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#configuration of nn\n",
|
|
"nn = Sequential()\n",
|
|
"nn.add(Embedding(max_fatures, embed_dim, input_length = X.shape[1]))\n",
|
|
"nn.add(SpatialDropout1D(dropout_1d))\n",
|
|
"nn.add(LSTM(lstm_out, dropout=dropout, recurrent_dropout=recurrent_dropout))\n",
|
|
"nn.add(Dense(2, activation='softmax'))\n",
|
|
"nn.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])\n",
|
|
"print(nn.summary())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 57,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 1/2\n",
|
|
" - 55s - loss: 0.4244 - acc: 0.8173\n",
|
|
"Epoch 2/2\n",
|
|
" - 53s - loss: 0.3386 - acc: 0.8565\n",
|
|
"score: 0.37\n",
|
|
"acc: 0.86\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#get the teacher values\n",
|
|
"Y = pd.get_dummies(df['sentiment']).values\n",
|
|
"#gnerates X and y train & test data\n",
|
|
"X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.30, random_state = random_state)\n",
|
|
"#trains the machine learning configuration with the declared parameters in the top\n",
|
|
"nn.fit(X_train, Y_train, epochs = epochs, batch_size=batch_size, verbose=verbose)\n",
|
|
"\n",
|
|
"#also generate validation set by cutting the last validation_size elements from test data\n",
|
|
"X_validate = X_test[-validation_size:]\n",
|
|
"Y_validate = Y_test[-validation_size:]\n",
|
|
"X_test = X_test[:-validation_size]\n",
|
|
"Y_test = Y_test[:-validation_size]\n",
|
|
"\n",
|
|
"#evaluates the score and the accuracy\n",
|
|
"score, accuracy = nn.evaluate(X_test, Y_test, verbose = 2, batch_size = batch_size)\n",
|
|
"print(\"score: %.2f\" % (score))\n",
|
|
"print(\"acc: %.2f\" % (accuracy))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 69,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"pos_acc 48.78048780487805 %\n",
|
|
"neg_acc 94.46540880503144 %\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#initialize counter for evaluating prediction\n",
|
|
"pos_cnt, neg_cnt, pos_ok, neg_ok, tp, fp, tn ,fn = 0, 0, 0, 0, 0, 0, 0, 0 \n",
|
|
"for x in range(len(X_validate)):\n",
|
|
" #predict for each element in validation set its true false probability\n",
|
|
" result = nn.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 2)[0]\n",
|
|
" #print(result)\n",
|
|
" #check if highest prob for same class\n",
|
|
" if np.argmax(result) == np.argmax(Y_validate[x]):\n",
|
|
" #if high prob in first array element ---> classification as neg ---> count neg \n",
|
|
" if np.argmax(Y_validate[x]) == 0: neg_ok += 1\n",
|
|
" #else count as pos\n",
|
|
" else: pos_ok += 1\n",
|
|
" #count of teacher labels\n",
|
|
" if np.argmax(Y_validate[x]) == 0: neg_cnt += 1\n",
|
|
" else: pos_cnt += 1\n",
|
|
"\n",
|
|
"#print results\n",
|
|
"print(\"pos_acc\", pos_ok/pos_cnt*100, \"%\")\n",
|
|
"print(\"neg_acc\", neg_ok/neg_cnt*100, \"%\")\n",
|
|
"\n",
|
|
"pos_acc = pos_ok/pos_cnt\n",
|
|
"neg_acc = neg_ok/neg_cnt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 70,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[[ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
|
|
" 5 700 8 8 12 218 8 5 290 14]]\n",
|
|
"[ 0.0937252 0.9062748]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#evaluate on different dataset\n",
|
|
"X2 = ['Jonas is nice and happy and in love and is looking for freedom']\n",
|
|
"X2 = tok.texts_to_sequences(X2)\n",
|
|
"X2 = pad_sequences(X2, maxlen=len(X[0]), dtype='int32', value=0)\n",
|
|
"print(X2)\n",
|
|
"print(nn.predict(X2, batch_size=1, verbose = 2)[0])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 78,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"configuration_1 = {\n",
|
|
" \"name\":\"experiment1\",\n",
|
|
" \n",
|
|
" \"model\":\"generic_model_1_name\",\n",
|
|
" \"model-name\":\"deepNN\",\n",
|
|
"\n",
|
|
"\n",
|
|
" \"dataset\":\"X1\",\n",
|
|
" \"dataset-name\":\"dataset_sentiment.csv\",\n",
|
|
" \"dataset-link\":\"see_github\",\n",
|
|
" \n",
|
|
" \"measurement-name\":\"generic_measurement_hub\",\n",
|
|
" \n",
|
|
" \"execution-name\":\"genereric_execution_name\",\n",
|
|
" \n",
|
|
" \"precision\":-1,\n",
|
|
" \"recall\":-1, \n",
|
|
" \"pos_accn\":pos_acc,\n",
|
|
" \"neg_acc\":neg_acc,\n",
|
|
" \"accuracy\":score\n",
|
|
"}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Export Data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 79,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from rdflib import Namespace, Graph, Literal\n",
|
|
"from rdflib.namespace import FOAF, OWL, XSD, RDFS, DCTERMS, DOAP, DC, RDF"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 80,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"prov = Namespace('http://www.w3.org/ns/prov#')\n",
|
|
"dcat = Namespace('http://www.w3.org/ns/dcat#')\n",
|
|
"mexalgo = Namespace('http://mex.aksw.org/mex-algo#')\n",
|
|
"mexperf = Namespace('http://mex.aksw.org/mex-perf#')\n",
|
|
"mexcore = Namespace('http://mex.aksw.org/mex-core#')\n",
|
|
"this = Namespace('http://mex.aksw.org/examples/')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 81,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def experiment_root_graph(root_node_name):\n",
|
|
" g = Graph()\n",
|
|
" g.add((this[root_node_name],RDF.type, mexcore.Experiment))\n",
|
|
" g.add((this[root_node_name],RDF.type, mexcore.ApplicationContext))\n",
|
|
" g.add((this[root_node_name],RDFS.label, Literal('2719095')))\n",
|
|
" g.add((this[root_node_name],DCTERMS.date, Literal('2018-05-28',datatype=XSD.date)))\n",
|
|
" g.add((this[root_node_name],FOAF.givenName, Literal('Carsten')))\n",
|
|
" g.add((this[root_node_name],FOAF.mbox, Literal('carsten.draschner@gmail.com')))\n",
|
|
" return g ,this[root_node_name]\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 82,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"<class 'rdflib.term.URIRef'>\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"g, r = experiment_root_graph(\"first_test\")\n",
|
|
"print(type(r))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 83,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"#extend a rdf graph with a configuration\n",
|
|
"def experiment_to_rdf(graph, experiment, root_node):\n",
|
|
" \n",
|
|
" g.add((this[experiment[\"name\"]],RDF.type,mexcore.ExperimentConfiguration))\n",
|
|
" g.add((this[experiment[\"name\"]],prov.used, this[experiment[\"model\"]]))\n",
|
|
" g.add((this[experiment[\"name\"]],prov.wasStartedBy, root_node))\n",
|
|
" \n",
|
|
" #dataset information\n",
|
|
" g.add((this[experiment[\"dataset\"]],RDF.type,mexcore.Dataset))\n",
|
|
" g.add((this.dataset2,RDFS.label,Literal(experiment[\"dataset-name\"])))\n",
|
|
" g.add((this.dataset2,DCTERMS.landingPage,Literal(experiment[\"dataset-link\"])))\n",
|
|
" \n",
|
|
" #model description\n",
|
|
" g.add((this[experiment[\"model\"]],RDF.type,mexalgo.Algorithm))\n",
|
|
" g.add((this[experiment[\"model\"]],RDFS.label,Literal(experiment[\"model-name\"])))\n",
|
|
" g.add((this[experiment[\"model\"]],DCTERMS.identifier,Literal(experiment[\"model-name\"])))\n",
|
|
" ###g.add((this.model1,mexalgo.hasHyperParameter,this.hyerparameter1))\n",
|
|
" \n",
|
|
" #execution\n",
|
|
" g.add((this[experiment[\"execution-name\"]],RDF.type,mexcore.ExecutionOverall))\n",
|
|
" g.add((this[experiment[\"execution-name\"]],prov.generated,this[experiment[\"measurement-name\"]]))\n",
|
|
" g.add((this[experiment[\"execution-name\"]],prov.used,this.test))\n",
|
|
" g.add((this[experiment[\"execution-name\"]],prov.used,this[experiment[\"model\"]]))\n",
|
|
" \n",
|
|
" #test\n",
|
|
" g.add((this.test,RDF.type,mexcore.Test))\n",
|
|
" g.add((this.test,RDFS.label,Literal('Test')))\n",
|
|
" \n",
|
|
" #evaluation information\n",
|
|
" g.add((this[experiment[\"measurement-name\"]],RDF.type,mexcore.PerformanceMeasure))\n",
|
|
" g.add((this[experiment[\"measurement-name\"]],mexperf.precision,Literal(experiment[\"precision\"],datatype=XSD.float)))\n",
|
|
" g.add((this[experiment[\"measurement-name\"]],mexperf.recall,Literal(experiment[\"recall\"],datatype=XSD.float)))\n",
|
|
" g.add((this[experiment[\"measurement-name\"]],mexperf.accuracy,Literal(experiment[\"accuracy\"],datatype=XSD.float)))\n",
|
|
" g.add((this[experiment[\"measurement-name\"]],prov.wasGeneratedBy,this[experiment[\"execution-name\"]]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 84,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"experiment_to_rdf(g,configuration_1,r)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 85,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"with open('task3_metadata.ttl','wb') as f:\n",
|
|
" f.write(g.serialize(format='turtle'))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|