master-thesis/Tagging/Conllu_Training.ipynb

291 lines
6.4 KiB
Plaintext

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"\n",
"import sys\n",
"\n",
"from conllu import parse\n",
"\n",
"sys.path.insert(0,'..')\n",
"import settings\n",
"\n",
"from tagging_tools import print_visualized_tags\n",
"\n",
"from train_sample_generator import ConlluReader, ConlluDataProvider\n",
"\n",
"from gensim.test.utils import common_texts, get_tmpfile\n",
"from gensim.models import Word2Vec\n",
"from nltk import PorterStemmer\n",
"import numpy as np\n",
"from sklearn import preprocessing\n",
"porter = PorterStemmer()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"conllu_reader = ConlluReader(\"recipes0.conllu\", iter_documents=False)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[TokenList<Set, oven, to, 350, degrees, F, .>]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"conllu_reader.__iter__().__next__()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"conllu_data_provider = ConlluDataProvider(\"recipes0.conllu\", \n",
" word2vec_model=None,\n",
" batchsize=100,\n",
" window_size=3,\n",
" iter_documents=False,\n",
" food_type=\"ingredient\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"x,y = conllu_data_provider.getNextDataBatch(y_food_type_label=\"ingredient\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1148"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(y)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"sum_tokens = 0\n",
"i = 0\n",
"for x,y in conllu_data_provider:\n",
" sum_tokens += len(x)\n",
" i += 1\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"649423"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sum_tokens"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"576"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## decision tree classifier"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.tree import DecisionTreeClassifier\n",
"from sklearn.ensemble import RandomForestClassifier\n",
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"conllu_data_provider = ConlluDataProvider(\"recipes0.conllu\", \n",
" word2vec_model=None,\n",
" batchsize=100,\n",
" window_size=3,\n",
" iter_documents=False,\n",
" food_type=\"ingredient\")"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"clf = RandomForestClassifier(n_estimators=100 ,random_state=0, warm_start=True)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"for x,y in conllu_data_provider:\n",
" break\n",
" X_train, X_test, y_train, y_test = train_test_split(x,y, random_state=0)\n",
" clf.fit(X_train, y_train)\n",
" pred = tree.predict(X_test)\n",
" print(\"loss: \", np.sum((pred - y_test)**2) / len(x))"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" ...,\n",
" [0., 0., 1., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 1., 0., ..., 0., 0., 0.]])"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"loss: 0.041811846689895474\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.7/site-packages/sklearn/ensemble/forest.py:307: UserWarning: Warm-start fitting without increasing n_estimators does not fit new trees.\n",
" warn(\"Warm-start fitting without increasing n_estimators does not \"\n"
]
}
],
"source": [
"clf.fit(X_train, y_train)\n",
"pred = tree.predict(X_test)\n",
"print(\"loss: \", np.sum((pred - y_test)**2) / len(x))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}