{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "\n", "import sys\n", "\n", "from conllu import parse\n", "\n", "sys.path.insert(0,'..')\n", "import settings\n", "\n", "from tagging_tools import print_visualized_tags\n", "\n", "from train_sample_generator import ConlluReader, ConlluDataProvider\n", "\n", "from gensim.test.utils import common_texts, get_tmpfile\n", "from gensim.models import Word2Vec\n", "from nltk import PorterStemmer\n", "import numpy as np\n", "from sklearn import preprocessing\n", "porter = PorterStemmer()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "conllu_reader = ConlluReader(\"recipes0.conllu\", iter_documents=False)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[TokenList]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "conllu_reader.__iter__().__next__()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "conllu_data_provider = ConlluDataProvider(\"recipes0.conllu\", \n", " word2vec_model=None,\n", " batchsize=100,\n", " window_size=3,\n", " iter_documents=False,\n", " food_type=\"ingredient\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "x,y = conllu_data_provider.getNextDataBatch(y_food_type_label=\"ingredient\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1148" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(y)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "sum_tokens = 0\n", "i = 0\n", "for x,y in conllu_data_provider:\n", " sum_tokens += len(x)\n", " i += 1\n", " " ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "649423" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sum_tokens" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "576" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "i" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## decision tree classifier" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [], "source": [ "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "conllu_data_provider = ConlluDataProvider(\"recipes0.conllu\", \n", " word2vec_model=None,\n", " batchsize=100,\n", " window_size=3,\n", " iter_documents=False,\n", " food_type=\"ingredient\")" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [], "source": [ "clf = RandomForestClassifier(n_estimators=100 ,random_state=0, warm_start=True)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "for x,y in conllu_data_provider:\n", " break\n", " X_train, X_test, y_train, y_test = train_test_split(x,y, random_state=0)\n", " clf.fit(X_train, y_train)\n", " pred = tree.predict(X_test)\n", " print(\"loss: \", np.sum((pred - y_test)**2) / len(x))" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " ...,\n", " [0., 0., 1., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 1., 0., ..., 0., 0., 0.]])" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "loss: 0.041811846689895474\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/jonas/.local/lib/python3.7/site-packages/sklearn/ensemble/forest.py:307: UserWarning: Warm-start fitting without increasing n_estimators does not fit new trees.\n", " warn(\"Warm-start fitting without increasing n_estimators does not \"\n" ] } ], "source": [ "clf.fit(X_train, y_train)\n", "pred = tree.predict(X_test)\n", "print(\"loss: \", np.sum((pred - y_test)**2) / len(x))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }