moved 1 million recipes into database, starting coarse tree generation for recipe analysis
This commit is contained in:
		
							
								
								
									
										223
									
								
								RecipeAnalysis/Recipe Analysis.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										223
									
								
								RecipeAnalysis/Recipe Analysis.ipynb
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,223 @@
 | 
				
			|||||||
 | 
					{
 | 
				
			||||||
 | 
					 "cells": [
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "# Recipe Analysis"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 1,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import sys\n",
 | 
				
			||||||
 | 
					    "sys.path.append(\"../\")\n",
 | 
				
			||||||
 | 
					    "from Recipe import Recipe"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 2,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import settings\n",
 | 
				
			||||||
 | 
					    "import db.db_settings as db_settings\n",
 | 
				
			||||||
 | 
					    "from db.database_connection import DatabaseConnection"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 3,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import random"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 4,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<db.database_connection.DatabaseConnection at 0x7f58b3f41b70>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "execution_count": 4,
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "DatabaseConnection(db_settings.db_host,\n",
 | 
				
			||||||
 | 
					    "                   db_settings.db_port,\n",
 | 
				
			||||||
 | 
					    "                   db_settings.db_user,\n",
 | 
				
			||||||
 | 
					    "                   db_settings.db_pw,\n",
 | 
				
			||||||
 | 
					    "                   db_settings.db_db,\n",
 | 
				
			||||||
 | 
					    "                   db_settings.db_charset)"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* get all recipe id's"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 5,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "ids = DatabaseConnection.global_single_query(\"select id from recipes\")"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* analyse a random recipe"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 6,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "test_rec = Recipe(random.choice(ids)['id'])"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 7,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/markdown": [
 | 
				
			||||||
 | 
					       "## Pat LaFriedas Filet Mignon Steak Sandwich\n",
 | 
				
			||||||
 | 
					       "(eaed08c862)"
 | 
				
			||||||
 | 
					      ],
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<IPython.core.display.Markdown object>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "display_data"
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/markdown": [
 | 
				
			||||||
 | 
					       "### Ingredients"
 | 
				
			||||||
 | 
					      ],
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<IPython.core.display.Markdown object>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "display_data"
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/markdown": [
 | 
				
			||||||
 | 
					       " * '4 tablespoons canola or other neutral-flavored oil , plus more as needed'\n",
 | 
				
			||||||
 | 
					       " * '2 large sweet yellow onions or Spanish onions , thinly sliced \\( about 3 cups \\)'\n",
 | 
				
			||||||
 | 
					       " * '6 ounces thinly sliced Monterey Jack cheese'\n",
 | 
				
			||||||
 | 
					       " * '1 cup beef stock'\n",
 | 
				
			||||||
 | 
					       " * '1 1/2 teaspoons balsamic glaze'\n",
 | 
				
			||||||
 | 
					       " * '12 \\( 1 1/2-inch thick \\) filet medallions \\( about 1 1/2 pounds \\)'\n",
 | 
				
			||||||
 | 
					       " * '1 tablespoon kosher salt'\n",
 | 
				
			||||||
 | 
					       " * '1/2 teaspoon turbinado sugar or light brown sugar'\n",
 | 
				
			||||||
 | 
					       " * '4 demi-baguettes \\( or 6-inch \\) segments of a long baguette'"
 | 
				
			||||||
 | 
					      ],
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<IPython.core.display.Markdown object>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "display_data"
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/markdown": [
 | 
				
			||||||
 | 
					       "### Instructions"
 | 
				
			||||||
 | 
					      ],
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<IPython.core.display.Markdown object>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "display_data"
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/markdown": [
 | 
				
			||||||
 | 
					       " * In a large skillet , heat 2 tablespoons of the oil over medium heat until it slides easily in the pan , 2 to 3 minutes .\n",
 | 
				
			||||||
 | 
					       " * Add the onions and cook , stirring occasionally so they do n't stick to the pan , until they are soft and caramelized , about 20 minutes .\n",
 | 
				
			||||||
 | 
					       " * Spread the onions out over the surface of the pan .\n",
 | 
				
			||||||
 | 
					       " * Remove from the heat and lay the cheese on top of the onions , letting it melt .\n",
 | 
				
			||||||
 | 
					       " * To make a jus , in a small saucepan , bring the stock to a simmer over medium heat .\n",
 | 
				
			||||||
 | 
					       " * Remove from the heat and stir in the balsamic glaze .\n",
 | 
				
			||||||
 | 
					       " * Cover the pan to keep the jus warm .\n",
 | 
				
			||||||
 | 
					       " * Season the meat on both sides with the salt and sugar .\n",
 | 
				
			||||||
 | 
					       " * In another large skillet , heat the remaining 2 tablespoons oil over high heat .\n",
 | 
				
			||||||
 | 
					       " * Add half the medallions , or as many as will fit in a single layer , and sear them until they are caramelized , 1 to 1 1/2 minutes per side .\n",
 | 
				
			||||||
 | 
					       " * Cook the remaining medallions in the same way , adding more oil and letting it get hot before adding the meat to the pan .\n",
 | 
				
			||||||
 | 
					       " * Meanwhile , without opening them , toast the baguettes so that the outsides , top and bottom , are hot and crispy .\n",
 | 
				
			||||||
 | 
					       " * Halve the baguettes horizontally , leaving them hinged on one side .\n",
 | 
				
			||||||
 | 
					       " * To assemble the sandwiches , lay 3 medallions on the bottom of each baguette .\n",
 | 
				
			||||||
 | 
					       " * Top with the onions and cheese , dividing them equally among the sandwiches .\n",
 | 
				
			||||||
 | 
					       " * Drizzle 1/4 cup of the jus on the inside top half of each baguette .\n",
 | 
				
			||||||
 | 
					       " * Close up the sandwiches and you 're good to go ."
 | 
				
			||||||
 | 
					      ],
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<IPython.core.display.Markdown object>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "display_data"
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stdout",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "CPU times: user 1.31 ms, sys: 7.65 ms, total: 8.96 ms\n",
 | 
				
			||||||
 | 
					      "Wall time: 7.88 ms\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%time test_rec.display_recipe()"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 ],
 | 
				
			||||||
 | 
					 "metadata": {
 | 
				
			||||||
 | 
					  "kernelspec": {
 | 
				
			||||||
 | 
					   "display_name": "Python 3",
 | 
				
			||||||
 | 
					   "language": "python",
 | 
				
			||||||
 | 
					   "name": "python3"
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  "language_info": {
 | 
				
			||||||
 | 
					   "codemirror_mode": {
 | 
				
			||||||
 | 
					    "name": "ipython",
 | 
				
			||||||
 | 
					    "version": 3
 | 
				
			||||||
 | 
					   },
 | 
				
			||||||
 | 
					   "file_extension": ".py",
 | 
				
			||||||
 | 
					   "mimetype": "text/x-python",
 | 
				
			||||||
 | 
					   "name": "python",
 | 
				
			||||||
 | 
					   "nbconvert_exporter": "python",
 | 
				
			||||||
 | 
					   "pygments_lexer": "ipython3",
 | 
				
			||||||
 | 
					   "version": "3.7.3"
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 },
 | 
				
			||||||
 | 
					 "nbformat": 4,
 | 
				
			||||||
 | 
					 "nbformat_minor": 4
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										298
									
								
								RecipeAnalysis/Recipe.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										298
									
								
								RecipeAnalysis/Recipe.ipynb
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,298 @@
 | 
				
			|||||||
 | 
					{
 | 
				
			||||||
 | 
					 "cells": [
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "# Recipe class"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 1,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import sys\n",
 | 
				
			||||||
 | 
					    "sys.path.append(\"../\")\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "import settings\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "import pycrfsuite\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "import json\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "import db.db_settings as db_settings\n",
 | 
				
			||||||
 | 
					    "from db.database_connection import DatabaseConnection\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "from Tagging.conllu_generator import ConlluGenerator\n",
 | 
				
			||||||
 | 
					    "from Tagging.crf_data_generator import *\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "from IPython.display import Markdown, HTML, display"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* get vocabulary"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 2,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import importlib.util\n",
 | 
				
			||||||
 | 
					    "# loading ingredients:\n",
 | 
				
			||||||
 | 
					    "spec = importlib.util.spec_from_file_location(\n",
 | 
				
			||||||
 | 
					    "    \"ingredients\", \"../\" + settings.ingredients_file)\n",
 | 
				
			||||||
 | 
					    "ingredients = importlib.util.module_from_spec(spec)\n",
 | 
				
			||||||
 | 
					    "spec.loader.exec_module(ingredients)\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "# loading actions:\n",
 | 
				
			||||||
 | 
					    "spec = importlib.util.spec_from_file_location(\n",
 | 
				
			||||||
 | 
					    "    \"actions\", \"../\" + settings.actions_file)\n",
 | 
				
			||||||
 | 
					    "actions = importlib.util.module_from_spec(spec)\n",
 | 
				
			||||||
 | 
					    "spec.loader.exec_module(actions)\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "# loading containers\n",
 | 
				
			||||||
 | 
					    "spec = importlib.util.spec_from_file_location(\n",
 | 
				
			||||||
 | 
					    "    \"containers\", \"../\" + settings.container_file)\n",
 | 
				
			||||||
 | 
					    "containers = importlib.util.module_from_spec(spec)\n",
 | 
				
			||||||
 | 
					    "spec.loader.exec_module(containers)\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "# loading placeholders\n",
 | 
				
			||||||
 | 
					    "spec = importlib.util.spec_from_file_location(\n",
 | 
				
			||||||
 | 
					    "    \"placeholders\", \"../\" + settings.placeholder_file)\n",
 | 
				
			||||||
 | 
					    "placeholders = importlib.util.module_from_spec(spec)\n",
 | 
				
			||||||
 | 
					    "spec.loader.exec_module(placeholders)\n"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 3,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "<contextlib.closing at 0x7f6743611278>"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "execution_count": 3,
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "tagger = pycrfsuite.Tagger()\n",
 | 
				
			||||||
 | 
					    "tagger.open('../Tagging/test.crfsuite')"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 4,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "id_query = \"select * from recipes where id like %s\""
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 5,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "def escape_md_chars(s):\n",
 | 
				
			||||||
 | 
					    "    s = s.replace(\"*\", \"\\*\")\n",
 | 
				
			||||||
 | 
					    "    s = s.replace(\"(\", \"\\(\")\n",
 | 
				
			||||||
 | 
					    "    s = s.replace(\")\", \"\\)\")\n",
 | 
				
			||||||
 | 
					    "    s = s.replace(\"[\", \"\\[\")\n",
 | 
				
			||||||
 | 
					    "    s = s.replace(\"]\", \"\\]\")\n",
 | 
				
			||||||
 | 
					    "    s = s.replace(\"_\", \"\\_\")\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    return s"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 6,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "class Recipe(object):\n",
 | 
				
			||||||
 | 
					    "    def __init__(self, recipe_db_id = None):\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        self._sentences = None\n",
 | 
				
			||||||
 | 
					    "        self._title = None\n",
 | 
				
			||||||
 | 
					    "        self._part = None\n",
 | 
				
			||||||
 | 
					    "        self._ingredients = None\n",
 | 
				
			||||||
 | 
					    "        self._recipe_id = recipe_db_id\n",
 | 
				
			||||||
 | 
					    "        self._get_from_db()\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        self._extracted_ingredients = None # TODO\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        self.annotate_ingredients()\n",
 | 
				
			||||||
 | 
					    "        self.annotate_sentences()\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def _get_from_db(self):\n",
 | 
				
			||||||
 | 
					    "        result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))\n",
 | 
				
			||||||
 | 
					    "        assert len(result) > 0\n",
 | 
				
			||||||
 | 
					    "        result = result[0]\n",
 | 
				
			||||||
 | 
					    "        self._title = result['title']\n",
 | 
				
			||||||
 | 
					    "        self._part = result['part']\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        raw_sentences = json.loads(result['instructions'])\n",
 | 
				
			||||||
 | 
					    "        raw_ingredients = json.loads(result['ingredients'])\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        # throwing the raw data through our connlu generator to annotate them right\n",
 | 
				
			||||||
 | 
					    "        cg_sents = ConlluGenerator([\"\\n\".join(raw_sentences)])\n",
 | 
				
			||||||
 | 
					    "        cg_ings = ConlluGenerator([\"\\n\".join(raw_ingredients)])\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        cg_sents.tokenize()\n",
 | 
				
			||||||
 | 
					    "        cg_sents.pos_tagging_and_lemmatization()\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        cg_ings.tokenize()\n",
 | 
				
			||||||
 | 
					    "        cg_ings.pos_tagging_and_lemmatization()\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        # TODO\n",
 | 
				
			||||||
 | 
					    "        self._sentences = cg_sents.get_conllu_elements()[0]\n",
 | 
				
			||||||
 | 
					    "        self._ingredients = cg_ings.get_conllu_elements()[0]\n",
 | 
				
			||||||
 | 
					    "        #self._sentences = json.loads(result['instructions'])\n",
 | 
				
			||||||
 | 
					    "        #self._ingredients = json.loads(result['ingredients'])\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def avg_sentence_length(self):\n",
 | 
				
			||||||
 | 
					    "        return sum([len(s) for s in self._sentences])/len(self._sentences)\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def n_instructions(self):\n",
 | 
				
			||||||
 | 
					    "        return len(self._sentences)\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def max_sentence_length(self):\n",
 | 
				
			||||||
 | 
					    "        return max([len(s) for s in self._sentences])\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def keyword_ratio(self):\n",
 | 
				
			||||||
 | 
					    "        sentence_ratios = []\n",
 | 
				
			||||||
 | 
					    "        for sent in self._sentences:\n",
 | 
				
			||||||
 | 
					    "            # FIXME: only works if there are no other misc annotations!\n",
 | 
				
			||||||
 | 
					    "            sentence_ratios.append(sum([token['misc'] is not None for token in sent]))\n",
 | 
				
			||||||
 | 
					    "        return sum(sentence_ratios) / len(sentence_ratios)\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def predict_labels(self):\n",
 | 
				
			||||||
 | 
					    "        features = [sent2features(sent) for sent in self._sentences]\n",
 | 
				
			||||||
 | 
					    "        labels = [tagger.tag(feat) for feat in features]\n",
 | 
				
			||||||
 | 
					    "        return labels\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def predict_ingredient_labels(self):\n",
 | 
				
			||||||
 | 
					    "        features = [sent2features(sent) for sent in self._ingredients]\n",
 | 
				
			||||||
 | 
					    "        labels = [tagger.tag(feat) for feat in features]\n",
 | 
				
			||||||
 | 
					    "        return labels\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def _annotate_sentences(self, sent_token_list, predictions):\n",
 | 
				
			||||||
 | 
					    "        # test whether we predicted an label or found it in our label list\n",
 | 
				
			||||||
 | 
					    "        for i, ing in enumerate(sent_token_list):\n",
 | 
				
			||||||
 | 
					    "            for j, token in enumerate(ing):\n",
 | 
				
			||||||
 | 
					    "                lemma = token['lemma']\n",
 | 
				
			||||||
 | 
					    "                \n",
 | 
				
			||||||
 | 
					    "                # check for ingredient\n",
 | 
				
			||||||
 | 
					    "                if lemma in ingredients.ingredients_stemmed:\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"ingredient\")\n",
 | 
				
			||||||
 | 
					    "                elif predictions[i][j] == 'ingredient':\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"ingredient\")\n",
 | 
				
			||||||
 | 
					    "                    \n",
 | 
				
			||||||
 | 
					    "                # check for action\n",
 | 
				
			||||||
 | 
					    "                if lemma in actions.stemmed_cooking_verbs:\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"action\")\n",
 | 
				
			||||||
 | 
					    "                elif predictions[i][j] == 'action':\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"action\")\n",
 | 
				
			||||||
 | 
					    "                \n",
 | 
				
			||||||
 | 
					    "                # check for container\n",
 | 
				
			||||||
 | 
					    "                if lemma in containers.stemmed_containers:\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"container\")\n",
 | 
				
			||||||
 | 
					    "                elif predictions[i][j] == 'container':\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"container\")\n",
 | 
				
			||||||
 | 
					    "                \n",
 | 
				
			||||||
 | 
					    "                # check for placeholder\n",
 | 
				
			||||||
 | 
					    "                if lemma in placeholders.stemmed_placeholders:\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"placeholder\")\n",
 | 
				
			||||||
 | 
					    "                elif predictions[i][j] == 'placeholder':\n",
 | 
				
			||||||
 | 
					    "                    token.add_misc(\"food_type\", \"placeholder\")\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def annotate_ingredients(self):\n",
 | 
				
			||||||
 | 
					    "        self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def annotate_sentences(self):\n",
 | 
				
			||||||
 | 
					    "        self._annotate_sentences(self._sentences, self.predict_labels())\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def recipe_id(self):\n",
 | 
				
			||||||
 | 
					    "        return self._recipe_id\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def serialize(self):\n",
 | 
				
			||||||
 | 
					    "        result = \"# newdoc\\n\"\n",
 | 
				
			||||||
 | 
					    "        if self._recipe_id is not None:\n",
 | 
				
			||||||
 | 
					    "            result += f\"# id: {self._recipe_id}\\n\"\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        for sent in self._sentences:\n",
 | 
				
			||||||
 | 
					    "            result += f\"{sent.serialize()}\"\n",
 | 
				
			||||||
 | 
					    "        return result + \"\\n\"\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def display_recipe(self):\n",
 | 
				
			||||||
 | 
					    "        display(Markdown(f\"## {self._title}\\n({self._recipe_id})\"))\n",
 | 
				
			||||||
 | 
					    "        display(Markdown(f\"### Ingredients\"))\n",
 | 
				
			||||||
 | 
					    "        display(Markdown(\"\\n\".join([f\" * '{escape_md_chars(self.tokenlist2str(ing))}'\" for ing in self._ingredients])))\n",
 | 
				
			||||||
 | 
					    "        display(Markdown(f\"### Instructions\"))\n",
 | 
				
			||||||
 | 
					    "        display(Markdown(\"\\n\".join([f\" * {escape_md_chars(self.tokenlist2str(ins))}\" for ins in self._sentences])))\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "    def tokenlist2str(self, tokenlist):\n",
 | 
				
			||||||
 | 
					    "        return \" \".join([token['form'] for token in tokenlist])\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def tokenarray2str(self, tokenarray):\n",
 | 
				
			||||||
 | 
					    "        return \"\\n\".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def __repr__(self):\n",
 | 
				
			||||||
 | 
					    "        s = \"recipe: \" + (self._recipe_id if self._recipe_id else \"\") + \"\\n\"\n",
 | 
				
			||||||
 | 
					    "        s += \"instructions: \\n\"\n",
 | 
				
			||||||
 | 
					    "        for sent in self._sentences:\n",
 | 
				
			||||||
 | 
					    "            s += \" \".join([token['form'] for token in sent]) + \"\\n\"\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        s += \"\\nscores:\\n\"\n",
 | 
				
			||||||
 | 
					    "        s += f\"avg_sent_length: {self.avg_sentence_length()}\\n\"\n",
 | 
				
			||||||
 | 
					    "        s += f\"n_instructions: {self.n_instructions()}\\n\"\n",
 | 
				
			||||||
 | 
					    "        s += f\"keyword_ratio: {self.keyword_ratio()}\\n\\n\\n\"\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        return s"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": null,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": []
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 ],
 | 
				
			||||||
 | 
					 "metadata": {
 | 
				
			||||||
 | 
					  "kernelspec": {
 | 
				
			||||||
 | 
					   "display_name": "Python 3",
 | 
				
			||||||
 | 
					   "language": "python",
 | 
				
			||||||
 | 
					   "name": "python3"
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  "language_info": {
 | 
				
			||||||
 | 
					   "codemirror_mode": {
 | 
				
			||||||
 | 
					    "name": "ipython",
 | 
				
			||||||
 | 
					    "version": 3
 | 
				
			||||||
 | 
					   },
 | 
				
			||||||
 | 
					   "file_extension": ".py",
 | 
				
			||||||
 | 
					   "mimetype": "text/x-python",
 | 
				
			||||||
 | 
					   "name": "python",
 | 
				
			||||||
 | 
					   "nbconvert_exporter": "python",
 | 
				
			||||||
 | 
					   "pygments_lexer": "ipython3",
 | 
				
			||||||
 | 
					   "version": "3.7.3"
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 },
 | 
				
			||||||
 | 
					 "nbformat": 4,
 | 
				
			||||||
 | 
					 "nbformat_minor": 4
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										211
									
								
								RecipeAnalysis/Recipe.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										211
									
								
								RecipeAnalysis/Recipe.py
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,211 @@
 | 
				
			|||||||
 | 
					#!/usr/bin/env python3
 | 
				
			||||||
 | 
					# coding: utf-8
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# # Recipe class
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import sys
 | 
				
			||||||
 | 
					sys.path.append("../")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import settings
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import pycrfsuite
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import json
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import db.db_settings as db_settings
 | 
				
			||||||
 | 
					from db.database_connection import DatabaseConnection
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					from Tagging.conllu_generator import ConlluGenerator
 | 
				
			||||||
 | 
					from Tagging.crf_data_generator import *
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					from IPython.display import Markdown, HTML, display
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# * get vocabulary
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import importlib.util
 | 
				
			||||||
 | 
					# loading ingredients:
 | 
				
			||||||
 | 
					spec = importlib.util.spec_from_file_location(
 | 
				
			||||||
 | 
					    "ingredients", "../" + settings.ingredients_file)
 | 
				
			||||||
 | 
					ingredients = importlib.util.module_from_spec(spec)
 | 
				
			||||||
 | 
					spec.loader.exec_module(ingredients)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# loading actions:
 | 
				
			||||||
 | 
					spec = importlib.util.spec_from_file_location(
 | 
				
			||||||
 | 
					    "actions", "../" + settings.actions_file)
 | 
				
			||||||
 | 
					actions = importlib.util.module_from_spec(spec)
 | 
				
			||||||
 | 
					spec.loader.exec_module(actions)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# loading containers
 | 
				
			||||||
 | 
					spec = importlib.util.spec_from_file_location(
 | 
				
			||||||
 | 
					    "containers", "../" + settings.container_file)
 | 
				
			||||||
 | 
					containers = importlib.util.module_from_spec(spec)
 | 
				
			||||||
 | 
					spec.loader.exec_module(containers)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# loading placeholders
 | 
				
			||||||
 | 
					spec = importlib.util.spec_from_file_location(
 | 
				
			||||||
 | 
					    "placeholders", "../" + settings.placeholder_file)
 | 
				
			||||||
 | 
					placeholders = importlib.util.module_from_spec(spec)
 | 
				
			||||||
 | 
					spec.loader.exec_module(placeholders)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					tagger = pycrfsuite.Tagger()
 | 
				
			||||||
 | 
					tagger.open('../Tagging/test.crfsuite')
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					id_query = "select * from recipes where id like %s"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					def escape_md_chars(s):
 | 
				
			||||||
 | 
					    s = s.replace("*", "\*")
 | 
				
			||||||
 | 
					    s = s.replace("(", "\(")
 | 
				
			||||||
 | 
					    s = s.replace(")", "\)")
 | 
				
			||||||
 | 
					    s = s.replace("[", "\[")
 | 
				
			||||||
 | 
					    s = s.replace("]", "\]")
 | 
				
			||||||
 | 
					    s = s.replace("_", "\_")
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    return s
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class Recipe(object):
 | 
				
			||||||
 | 
					    def __init__(self, recipe_db_id = None):
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        self._sentences = None
 | 
				
			||||||
 | 
					        self._title = None
 | 
				
			||||||
 | 
					        self._part = None
 | 
				
			||||||
 | 
					        self._ingredients = None
 | 
				
			||||||
 | 
					        self._recipe_id = recipe_db_id
 | 
				
			||||||
 | 
					        self._get_from_db()
 | 
				
			||||||
 | 
					        self.annotate_ingredients()
 | 
				
			||||||
 | 
					        self.annotate_sentences()
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def _get_from_db(self):
 | 
				
			||||||
 | 
					        result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))
 | 
				
			||||||
 | 
					        assert len(result) > 0
 | 
				
			||||||
 | 
					        result = result[0]
 | 
				
			||||||
 | 
					        self._title = result['title']
 | 
				
			||||||
 | 
					        self._part = result['part']
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        raw_sentences = json.loads(result['instructions'])
 | 
				
			||||||
 | 
					        raw_ingredients = json.loads(result['ingredients'])
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        # throwing the raw data through our connlu generator to annotate them right
 | 
				
			||||||
 | 
					        cg_sents = ConlluGenerator(["\n".join(raw_sentences)])
 | 
				
			||||||
 | 
					        cg_ings = ConlluGenerator(["\n".join(raw_ingredients)])
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        cg_sents.tokenize()
 | 
				
			||||||
 | 
					        cg_sents.pos_tagging_and_lemmatization()
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        cg_ings.tokenize()
 | 
				
			||||||
 | 
					        cg_ings.pos_tagging_and_lemmatization()
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        # TODO
 | 
				
			||||||
 | 
					        self._sentences = cg_sents.get_conllu_elements()[0]
 | 
				
			||||||
 | 
					        self._ingredients = cg_ings.get_conllu_elements()[0]
 | 
				
			||||||
 | 
					        #self._sentences = json.loads(result['instructions'])
 | 
				
			||||||
 | 
					        #self._ingredients = json.loads(result['ingredients'])
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def avg_sentence_length(self):
 | 
				
			||||||
 | 
					        return sum([len(s) for s in self._sentences])/len(self._sentences)
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def n_instructions(self):
 | 
				
			||||||
 | 
					        return len(self._sentences)
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def max_sentence_length(self):
 | 
				
			||||||
 | 
					        return max([len(s) for s in self._sentences])
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def keyword_ratio(self):
 | 
				
			||||||
 | 
					        sentence_ratios = []
 | 
				
			||||||
 | 
					        for sent in self._sentences:
 | 
				
			||||||
 | 
					            # FIXME: only works if there are no other misc annotations!
 | 
				
			||||||
 | 
					            sentence_ratios.append(sum([token['misc'] is not None for token in sent]))
 | 
				
			||||||
 | 
					        return sum(sentence_ratios) / len(sentence_ratios)
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def predict_labels(self):
 | 
				
			||||||
 | 
					        features = [sent2features(sent) for sent in self._sentences]
 | 
				
			||||||
 | 
					        labels = [tagger.tag(feat) for feat in features]
 | 
				
			||||||
 | 
					        return labels
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def predict_ingredient_labels(self):
 | 
				
			||||||
 | 
					        features = [sent2features(sent) for sent in self._ingredients]
 | 
				
			||||||
 | 
					        labels = [tagger.tag(feat) for feat in features]
 | 
				
			||||||
 | 
					        return labels
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def _annotate_sentences(self, sent_token_list, predictions):
 | 
				
			||||||
 | 
					        # test whether we predicted an label or found it in our label list
 | 
				
			||||||
 | 
					        for i, ing in enumerate(sent_token_list):
 | 
				
			||||||
 | 
					            for j, token in enumerate(ing):
 | 
				
			||||||
 | 
					                lemma = token['lemma']
 | 
				
			||||||
 | 
					                
 | 
				
			||||||
 | 
					                # check for ingredient
 | 
				
			||||||
 | 
					                if lemma in ingredients.ingredients_stemmed:
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "ingredient")
 | 
				
			||||||
 | 
					                elif predictions[i][j] == 'ingredient':
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "ingredient")
 | 
				
			||||||
 | 
					                    
 | 
				
			||||||
 | 
					                # check for action
 | 
				
			||||||
 | 
					                if lemma in actions.stemmed_cooking_verbs:
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "action")
 | 
				
			||||||
 | 
					                elif predictions[i][j] == 'action':
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "action")
 | 
				
			||||||
 | 
					                
 | 
				
			||||||
 | 
					                # check for container
 | 
				
			||||||
 | 
					                if lemma in containers.stemmed_containers:
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "container")
 | 
				
			||||||
 | 
					                elif predictions[i][j] == 'container':
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "container")
 | 
				
			||||||
 | 
					                
 | 
				
			||||||
 | 
					                # check for placeholder
 | 
				
			||||||
 | 
					                if lemma in placeholders.stemmed_placeholders:
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "placeholder")
 | 
				
			||||||
 | 
					                elif predictions[i][j] == 'placeholder':
 | 
				
			||||||
 | 
					                    token.add_misc("food_type", "placeholder")
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def annotate_ingredients(self):
 | 
				
			||||||
 | 
					        self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def annotate_sentences(self):
 | 
				
			||||||
 | 
					        self._annotate_sentences(self._sentences, self.predict_labels())
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def recipe_id(self):
 | 
				
			||||||
 | 
					        return self._recipe_id
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def serialize(self):
 | 
				
			||||||
 | 
					        result = "# newdoc\n"
 | 
				
			||||||
 | 
					        if self._recipe_id is not None:
 | 
				
			||||||
 | 
					            result += f"# id: {self._recipe_id}\n"
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        for sent in self._sentences:
 | 
				
			||||||
 | 
					            result += f"{sent.serialize()}"
 | 
				
			||||||
 | 
					        return result + "\n"
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def display_recipe(self):
 | 
				
			||||||
 | 
					        display(Markdown(f"## {self._title}\n({self._recipe_id})"))
 | 
				
			||||||
 | 
					        display(Markdown(f"### Ingredients"))
 | 
				
			||||||
 | 
					        display(Markdown("\n".join([f" * '{escape_md_chars(self.tokenlist2str(ing))}'" for ing in self._ingredients])))
 | 
				
			||||||
 | 
					        display(Markdown(f"### Instructions"))
 | 
				
			||||||
 | 
					        display(Markdown("\n".join([f" * {escape_md_chars(self.tokenlist2str(ins))}" for ins in self._sentences])))
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					    def tokenlist2str(self, tokenlist):
 | 
				
			||||||
 | 
					        return " ".join([token['form'] for token in tokenlist])
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def tokenarray2str(self, tokenarray):
 | 
				
			||||||
 | 
					        return "\n".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    def __repr__(self):
 | 
				
			||||||
 | 
					        s = "recipe: " + (self._recipe_id if self._recipe_id else "") + "\n"
 | 
				
			||||||
 | 
					        s += "instructions: \n"
 | 
				
			||||||
 | 
					        for sent in self._sentences:
 | 
				
			||||||
 | 
					            s += " ".join([token['form'] for token in sent]) + "\n"
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        s += "\nscores:\n"
 | 
				
			||||||
 | 
					        s += f"avg_sent_length: {self.avg_sentence_length()}\n"
 | 
				
			||||||
 | 
					        s += f"n_instructions: {self.n_instructions()}\n"
 | 
				
			||||||
 | 
					        s += f"keyword_ratio: {self.keyword_ratio()}\n\n\n"
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        return s
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							@ -20,7 +20,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 4,
 | 
					   "execution_count": 3,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [
 | 
					   "outputs": [
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
@ -29,7 +29,7 @@
 | 
				
			|||||||
       "TokenList<Dissolve, Jello, in, boiling, water, .>"
 | 
					       "TokenList<Dissolve, Jello, in, boiling, water, .>"
 | 
				
			||||||
      ]
 | 
					      ]
 | 
				
			||||||
     },
 | 
					     },
 | 
				
			||||||
     "execution_count": 4,
 | 
					     "execution_count": 3,
 | 
				
			||||||
     "metadata": {},
 | 
					     "metadata": {},
 | 
				
			||||||
     "output_type": "execute_result"
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -40,7 +40,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 5,
 | 
					   "execution_count": 4,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -109,7 +109,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 6,
 | 
					   "execution_count": 5,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -125,7 +125,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 7,
 | 
					   "execution_count": 6,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -135,7 +135,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 8,
 | 
					   "execution_count": 7,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -145,7 +145,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 9,
 | 
					   "execution_count": 8,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -155,7 +155,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 10,
 | 
					   "execution_count": 9,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -172,7 +172,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 11,
 | 
					   "execution_count": 10,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -181,7 +181,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 14,
 | 
					   "execution_count": 11,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -218,16 +218,16 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 15,
 | 
					   "execution_count": 12,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [
 | 
					   "outputs": [
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
     "data": {
 | 
					     "data": {
 | 
				
			||||||
      "text/plain": [
 | 
					      "text/plain": [
 | 
				
			||||||
       "45442"
 | 
					       "47538"
 | 
				
			||||||
      ]
 | 
					      ]
 | 
				
			||||||
     },
 | 
					     },
 | 
				
			||||||
     "execution_count": 15,
 | 
					     "execution_count": 12,
 | 
				
			||||||
     "metadata": {},
 | 
					     "metadata": {},
 | 
				
			||||||
     "output_type": "execute_result"
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -245,7 +245,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 16,
 | 
					   "execution_count": 13,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -254,7 +254,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 17,
 | 
					   "execution_count": 14,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -266,7 +266,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 18,
 | 
					   "execution_count": 15,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -282,7 +282,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 19,
 | 
					   "execution_count": 16,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [
 | 
					   "outputs": [
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
@ -302,7 +302,7 @@
 | 
				
			|||||||
       " 'max_linesearch']"
 | 
					       " 'max_linesearch']"
 | 
				
			||||||
      ]
 | 
					      ]
 | 
				
			||||||
     },
 | 
					     },
 | 
				
			||||||
     "execution_count": 19,
 | 
					     "execution_count": 16,
 | 
				
			||||||
     "metadata": {},
 | 
					     "metadata": {},
 | 
				
			||||||
     "output_type": "execute_result"
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -313,7 +313,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 20,
 | 
					   "execution_count": 17,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
@ -322,24 +322,24 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 20,
 | 
					   "execution_count": 18,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [
 | 
					   "outputs": [
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
     "data": {
 | 
					     "data": {
 | 
				
			||||||
      "text/plain": [
 | 
					      "text/plain": [
 | 
				
			||||||
       "{'num': 455,\n",
 | 
					       "{'num': 830,\n",
 | 
				
			||||||
       " 'scores': {},\n",
 | 
					       " 'scores': {},\n",
 | 
				
			||||||
       " 'loss': 110.581675,\n",
 | 
					       " 'loss': 41171.669638,\n",
 | 
				
			||||||
       " 'feature_norm': 8.594619,\n",
 | 
					       " 'feature_norm': 126.341894,\n",
 | 
				
			||||||
       " 'error_norm': 0.214403,\n",
 | 
					       " 'error_norm': 85.690855,\n",
 | 
				
			||||||
       " 'active_features': 87,\n",
 | 
					       " 'active_features': 6055,\n",
 | 
				
			||||||
       " 'linesearch_trials': 2,\n",
 | 
					       " 'linesearch_trials': 2,\n",
 | 
				
			||||||
       " 'linesearch_step': 0.5,\n",
 | 
					       " 'linesearch_step': 0.5,\n",
 | 
				
			||||||
       " 'time': 0.001}"
 | 
					       " 'time': 0.724}"
 | 
				
			||||||
      ]
 | 
					      ]
 | 
				
			||||||
     },
 | 
					     },
 | 
				
			||||||
     "execution_count": 20,
 | 
					     "execution_count": 18,
 | 
				
			||||||
     "metadata": {},
 | 
					     "metadata": {},
 | 
				
			||||||
     "output_type": "execute_result"
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -357,16 +357,16 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 21,
 | 
					   "execution_count": 19,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [
 | 
					   "outputs": [
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
     "data": {
 | 
					     "data": {
 | 
				
			||||||
      "text/plain": [
 | 
					      "text/plain": [
 | 
				
			||||||
       "<contextlib.closing at 0x7f056bc6d828>"
 | 
					       "<contextlib.closing at 0x7f26d79813c8>"
 | 
				
			||||||
      ]
 | 
					      ]
 | 
				
			||||||
     },
 | 
					     },
 | 
				
			||||||
     "execution_count": 21,
 | 
					     "execution_count": 19,
 | 
				
			||||||
     "metadata": {},
 | 
					     "metadata": {},
 | 
				
			||||||
     "output_type": "execute_result"
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -378,7 +378,7 @@
 | 
				
			|||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
   "cell_type": "code",
 | 
					   "cell_type": "code",
 | 
				
			||||||
   "execution_count": 22,
 | 
					   "execution_count": 20,
 | 
				
			||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [
 | 
					   "outputs": [
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
@ -388,7 +388,7 @@
 | 
				
			|||||||
     "traceback": [
 | 
					     "traceback": [
 | 
				
			||||||
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
 | 
					      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
 | 
				
			||||||
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
 | 
					      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
 | 
				
			||||||
      "\u001b[0;32m<ipython-input-22-a88100b49642>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m130\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;31m#print(' '.join(feature2tokens(X_test[i])), end='\\n\\n')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Predicted:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Correct:  \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 | 
					      "\u001b[0;32m<ipython-input-20-a88100b49642>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m130\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;31m#print(' '.join(feature2tokens(X_test[i])), end='\\n\\n')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Predicted:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Correct:  \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 | 
				
			||||||
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
 | 
					      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
 | 
				
			||||||
     ]
 | 
					     ]
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
				
			|||||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							@ -16,14 +16,15 @@
 | 
				
			|||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
    "import sys\n",
 | 
					    "import sys\n",
 | 
				
			||||||
 | 
					    "sys.path.append('../')\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "from conllu import parse\n",
 | 
					    "from conllu import parse\n",
 | 
				
			||||||
    "from tagging_tools import print_visualized_tags\n",
 | 
					    "from Tagging.tagging_tools import print_visualized_tags\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "from sklearn import preprocessing\n",
 | 
					    "from sklearn import preprocessing\n",
 | 
				
			||||||
    "import numpy as np\n",
 | 
					    "import numpy as np\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "sys.path.insert(0, '..')\n",
 | 
					    "\n",
 | 
				
			||||||
    "import settings  # noqa\n",
 | 
					    "import settings  # noqa\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "import gzip"
 | 
					    "import gzip"
 | 
				
			||||||
 | 
				
			|||||||
@ -6,14 +6,15 @@
 | 
				
			|||||||
# read conllu documents in batches
 | 
					# read conllu documents in batches
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import sys
 | 
					import sys
 | 
				
			||||||
 | 
					sys.path.append('../')
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from conllu import parse
 | 
					from conllu import parse
 | 
				
			||||||
from tagging_tools import print_visualized_tags
 | 
					from Tagging.tagging_tools import print_visualized_tags
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from sklearn import preprocessing
 | 
					from sklearn import preprocessing
 | 
				
			||||||
import numpy as np
 | 
					import numpy as np
 | 
				
			||||||
 | 
					
 | 
				
			||||||
sys.path.insert(0, '..')
 | 
					
 | 
				
			||||||
import settings  # noqa
 | 
					import settings  # noqa
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import gzip
 | 
					import gzip
 | 
				
			||||||
 | 
				
			|||||||
@ -25,11 +25,14 @@
 | 
				
			|||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import sys\n",
 | 
				
			||||||
 | 
					    "sys.path.append(\"../\")\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
    "import nltk\n",
 | 
					    "import nltk\n",
 | 
				
			||||||
    "from nltk.tag import pos_tag, map_tag\n",
 | 
					    "from nltk.tag import pos_tag, map_tag\n",
 | 
				
			||||||
    "from nltk.stem import PorterStemmer\n",
 | 
					    "from nltk.stem import PorterStemmer\n",
 | 
				
			||||||
    "from nltk.corpus import stopwords as nltk_stopwords\n",
 | 
					    "from nltk.corpus import stopwords as nltk_stopwords\n",
 | 
				
			||||||
    "from stemmed_mwe_tokenizer import StemmedMWETokenizer\n",
 | 
					    "from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer\n",
 | 
				
			||||||
    "from nltk.stem import WordNetLemmatizer"
 | 
					    "from nltk.stem import WordNetLemmatizer"
 | 
				
			||||||
   ]
 | 
					   ]
 | 
				
			||||||
  },
 | 
					  },
 | 
				
			||||||
@ -188,7 +191,22 @@
 | 
				
			|||||||
    "        result = \"\"\n",
 | 
					    "        result = \"\"\n",
 | 
				
			||||||
    "        for attr in CONLLU_ATTRIBUTES:\n",
 | 
					    "        for attr in CONLLU_ATTRIBUTES:\n",
 | 
				
			||||||
    "            result += str(self.__getattribute__(attr)) + \" \\t\"\n",
 | 
					    "            result += str(self.__getattribute__(attr)) + \" \\t\"\n",
 | 
				
			||||||
    "        return replace_tab(result, 16)"
 | 
					    "        return replace_tab(result, 16)\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def __getitem__(self, key):\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        # conllu module compability:\n",
 | 
				
			||||||
 | 
					    "        if key == \"upostag\":\n",
 | 
				
			||||||
 | 
					    "            key = \"upos\"\n",
 | 
				
			||||||
 | 
					    "        if key == \"xpostag\":\n",
 | 
				
			||||||
 | 
					    "            key = \"xpos\"\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        if key not in CONLLU_ATTRIBUTES:\n",
 | 
				
			||||||
 | 
					    "            return None\n",
 | 
				
			||||||
 | 
					    "        attr = self.__getattribute__(key)\n",
 | 
				
			||||||
 | 
					    "        if str(attr) == \"_\":\n",
 | 
				
			||||||
 | 
					    "            return None\n",
 | 
				
			||||||
 | 
					    "        return attr"
 | 
				
			||||||
   ]
 | 
					   ]
 | 
				
			||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
@ -210,6 +228,9 @@
 | 
				
			|||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "    def add(self, conllu_element: ConlluElement):\n",
 | 
					    "    def add(self, conllu_element: ConlluElement):\n",
 | 
				
			||||||
    "        self.conllu_elements.append(conllu_element)\n",
 | 
					    "        self.conllu_elements.append(conllu_element)\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def get_conllu_elements(self):\n",
 | 
				
			||||||
 | 
					    "        return self.conllu_elements\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "    def __repr__(self):\n",
 | 
					    "    def __repr__(self):\n",
 | 
				
			||||||
    "        result = \"\"\n",
 | 
					    "        result = \"\"\n",
 | 
				
			||||||
@ -243,6 +264,9 @@
 | 
				
			|||||||
    "    def add(self, conllu_sentence: ConlluSentence):\n",
 | 
					    "    def add(self, conllu_sentence: ConlluSentence):\n",
 | 
				
			||||||
    "        self.conllu_sentences.append(conllu_sentence)\n",
 | 
					    "        self.conllu_sentences.append(conllu_sentence)\n",
 | 
				
			||||||
    "    \n",
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def get_conllu_elements(self):\n",
 | 
				
			||||||
 | 
					    "        return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences]\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
    "    def __repr__(self):\n",
 | 
					    "    def __repr__(self):\n",
 | 
				
			||||||
    "        result = \"# newdoc\\n\"\n",
 | 
					    "        result = \"# newdoc\\n\"\n",
 | 
				
			||||||
    "        if self.id is not None:\n",
 | 
					    "        if self.id is not None:\n",
 | 
				
			||||||
@ -270,15 +294,18 @@
 | 
				
			|||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
    "class ConlluGenerator(object):\n",
 | 
					    "class ConlluGenerator(object):\n",
 | 
				
			||||||
    "    def __init__(self, documents: list, stemmed_multi_word_tokens, stemmer=PorterStemmer(), ids=None):\n",
 | 
					    "    def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None):\n",
 | 
				
			||||||
    "        self.documents = documents\n",
 | 
					    "        self.documents = documents\n",
 | 
				
			||||||
    "        self.stemmed_multi_word_tokens = stemmed_multi_word_tokens\n",
 | 
					    "        self.stemmed_multi_word_tokens = stemmed_multi_word_tokens\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
 | 
					    "        if self.stemmed_multi_word_tokens is not None:\n",
 | 
				
			||||||
    "            self.mwe_tokenizer = StemmedMWETokenizer(\n",
 | 
					    "            self.mwe_tokenizer = StemmedMWETokenizer(\n",
 | 
				
			||||||
    "                [w.split() for w in stemmed_multi_word_tokens])\n",
 | 
					    "                [w.split() for w in stemmed_multi_word_tokens])\n",
 | 
				
			||||||
 | 
					    "        else:\n",
 | 
				
			||||||
 | 
					    "            self.mwe_tokenizer = None\n",
 | 
				
			||||||
 | 
					    "        \n",
 | 
				
			||||||
    "        self.stemmer = stemmer\n",
 | 
					    "        self.stemmer = stemmer\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "        self.id_counter = 0\n",
 | 
					 | 
				
			||||||
    "\n",
 | 
					 | 
				
			||||||
    "        self.conllu_documents = []\n",
 | 
					    "        self.conllu_documents = []\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "        self.ids = ids\n",
 | 
					    "        self.ids = ids\n",
 | 
				
			||||||
@ -293,6 +320,9 @@
 | 
				
			|||||||
    "            for sent in sentences: \n",
 | 
					    "            for sent in sentences: \n",
 | 
				
			||||||
    "                if (len(sent) > 0):\n",
 | 
					    "                if (len(sent) > 0):\n",
 | 
				
			||||||
    "                    simple_tokenized = nltk.tokenize.word_tokenize(sent)\n",
 | 
					    "                    simple_tokenized = nltk.tokenize.word_tokenize(sent)\n",
 | 
				
			||||||
 | 
					    "                    if self.mwe_tokenizer is None:\n",
 | 
				
			||||||
 | 
					    "                        tokenized_sentences.append(simple_tokenized)\n",
 | 
				
			||||||
 | 
					    "                    else:\n",
 | 
				
			||||||
    "                        tokenized_sentences.append(\n",
 | 
					    "                        tokenized_sentences.append(\n",
 | 
				
			||||||
    "                            self.mwe_tokenizer.tokenize(simple_tokenized))\n",
 | 
					    "                            self.mwe_tokenizer.tokenize(simple_tokenized))\n",
 | 
				
			||||||
    "            tokenized_documents.append(tokenized_sentences)\n",
 | 
					    "            tokenized_documents.append(tokenized_sentences)\n",
 | 
				
			||||||
@ -304,13 +334,14 @@
 | 
				
			|||||||
    "            else:\n",
 | 
					    "            else:\n",
 | 
				
			||||||
    "                conllu_doc = ConlluDocument()\n",
 | 
					    "                conllu_doc = ConlluDocument()\n",
 | 
				
			||||||
    "            for sent in doc:\n",
 | 
					    "            for sent in doc:\n",
 | 
				
			||||||
 | 
					    "                token_id = 0\n",
 | 
				
			||||||
    "                conllu_sent = ConlluSentence()\n",
 | 
					    "                conllu_sent = ConlluSentence()\n",
 | 
				
			||||||
    "                for token in sent:\n",
 | 
					    "                for token in sent:\n",
 | 
				
			||||||
 | 
					    "                    token_id += 1\n",
 | 
				
			||||||
    "                    conllu_sent.add(ConlluElement(\n",
 | 
					    "                    conllu_sent.add(ConlluElement(\n",
 | 
				
			||||||
    "                        id=self.id_counter + 1,\n",
 | 
					    "                        id=token_id,\n",
 | 
				
			||||||
    "                        form=token,\n",
 | 
					    "                        form=token,\n",
 | 
				
			||||||
    "                    ))\n",
 | 
					    "                    ))\n",
 | 
				
			||||||
    "                    self.id_counter += 1\n",
 | 
					 | 
				
			||||||
    "                conllu_doc.add(conllu_sent)\n",
 | 
					    "                conllu_doc.add(conllu_sent)\n",
 | 
				
			||||||
    "            self.conllu_documents.append(conllu_doc)\n",
 | 
					    "            self.conllu_documents.append(conllu_doc)\n",
 | 
				
			||||||
    "            i += 1\n",
 | 
					    "            i += 1\n",
 | 
				
			||||||
@ -340,6 +371,9 @@
 | 
				
			|||||||
    "                for elem in conllu_sent.conllu_elements:\n",
 | 
					    "                for elem in conllu_sent.conllu_elements:\n",
 | 
				
			||||||
    "                    if elem.lemma in stemmed_keyword_list:\n",
 | 
					    "                    if elem.lemma in stemmed_keyword_list:\n",
 | 
				
			||||||
    "                        elem.add_misc(key, value)\n",
 | 
					    "                        elem.add_misc(key, value)\n",
 | 
				
			||||||
 | 
					    "    \n",
 | 
				
			||||||
 | 
					    "    def get_conllu_elements(self):\n",
 | 
				
			||||||
 | 
					    "        return [doc.get_conllu_elements() for doc in self.conllu_documents]\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "    def __repr__(self):\n",
 | 
					    "    def __repr__(self):\n",
 | 
				
			||||||
    "        result = \"\"\n",
 | 
					    "        result = \"\"\n",
 | 
				
			||||||
 | 
				
			|||||||
@ -10,11 +10,14 @@
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
# ## imports and settings
 | 
					# ## imports and settings
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import sys
 | 
				
			||||||
 | 
					sys.path.append("../")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import nltk
 | 
					import nltk
 | 
				
			||||||
from nltk.tag import pos_tag, map_tag
 | 
					from nltk.tag import pos_tag, map_tag
 | 
				
			||||||
from nltk.stem import PorterStemmer
 | 
					from nltk.stem import PorterStemmer
 | 
				
			||||||
from nltk.corpus import stopwords as nltk_stopwords
 | 
					from nltk.corpus import stopwords as nltk_stopwords
 | 
				
			||||||
from stemmed_mwe_tokenizer import StemmedMWETokenizer
 | 
					from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer
 | 
				
			||||||
from nltk.stem import WordNetLemmatizer
 | 
					from nltk.stem import WordNetLemmatizer
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -129,6 +132,21 @@ class ConlluElement(object):
 | 
				
			|||||||
            result += str(self.__getattribute__(attr)) + " \t"
 | 
					            result += str(self.__getattribute__(attr)) + " \t"
 | 
				
			||||||
        return replace_tab(result, 16)
 | 
					        return replace_tab(result, 16)
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
 | 
					    def __getitem__(self, key):
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        # conllu module compability:
 | 
				
			||||||
 | 
					        if key == "upostag":
 | 
				
			||||||
 | 
					            key = "upos"
 | 
				
			||||||
 | 
					        if key == "xpostag":
 | 
				
			||||||
 | 
					            key = "xpos"
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        if key not in CONLLU_ATTRIBUTES:
 | 
				
			||||||
 | 
					            return None
 | 
				
			||||||
 | 
					        attr = self.__getattribute__(key)
 | 
				
			||||||
 | 
					        if str(attr) == "_":
 | 
				
			||||||
 | 
					            return None
 | 
				
			||||||
 | 
					        return attr
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
# ## Conllu Sentence Class
 | 
					# ## Conllu Sentence Class
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -139,6 +157,9 @@ class ConlluSentence(object):
 | 
				
			|||||||
    def add(self, conllu_element: ConlluElement):
 | 
					    def add(self, conllu_element: ConlluElement):
 | 
				
			||||||
        self.conllu_elements.append(conllu_element)
 | 
					        self.conllu_elements.append(conllu_element)
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
 | 
					    def get_conllu_elements(self):
 | 
				
			||||||
 | 
					        return self.conllu_elements
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def __repr__(self):
 | 
					    def __repr__(self):
 | 
				
			||||||
        result = ""
 | 
					        result = ""
 | 
				
			||||||
        for elem in self.conllu_elements:
 | 
					        for elem in self.conllu_elements:
 | 
				
			||||||
@ -160,6 +181,9 @@ class ConlluDocument(object):
 | 
				
			|||||||
    def add(self, conllu_sentence: ConlluSentence):
 | 
					    def add(self, conllu_sentence: ConlluSentence):
 | 
				
			||||||
        self.conllu_sentences.append(conllu_sentence)
 | 
					        self.conllu_sentences.append(conllu_sentence)
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
 | 
					    def get_conllu_elements(self):
 | 
				
			||||||
 | 
					        return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences]
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
    def __repr__(self):
 | 
					    def __repr__(self):
 | 
				
			||||||
        result = "# newdoc\n"
 | 
					        result = "# newdoc\n"
 | 
				
			||||||
        if self.id is not None:
 | 
					        if self.id is not None:
 | 
				
			||||||
@ -176,14 +200,17 @@ class ConlluDocument(object):
 | 
				
			|||||||
# ## Conllu Generator Class
 | 
					# ## Conllu Generator Class
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class ConlluGenerator(object):
 | 
					class ConlluGenerator(object):
 | 
				
			||||||
    def __init__(self, documents: list, stemmed_multi_word_tokens, stemmer=PorterStemmer(), ids=None):
 | 
					    def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None):
 | 
				
			||||||
        self.documents = documents
 | 
					        self.documents = documents
 | 
				
			||||||
        self.stemmed_multi_word_tokens = stemmed_multi_word_tokens
 | 
					        self.stemmed_multi_word_tokens = stemmed_multi_word_tokens
 | 
				
			||||||
 | 
					        
 | 
				
			||||||
 | 
					        if self.stemmed_multi_word_tokens is not None:
 | 
				
			||||||
            self.mwe_tokenizer = StemmedMWETokenizer(
 | 
					            self.mwe_tokenizer = StemmedMWETokenizer(
 | 
				
			||||||
                [w.split() for w in stemmed_multi_word_tokens])
 | 
					                [w.split() for w in stemmed_multi_word_tokens])
 | 
				
			||||||
        self.stemmer = stemmer
 | 
					        else:
 | 
				
			||||||
 | 
					            self.mwe_tokenizer = None
 | 
				
			||||||
        
 | 
					        
 | 
				
			||||||
        self.id_counter = 0
 | 
					        self.stemmer = stemmer
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        self.conllu_documents = []
 | 
					        self.conllu_documents = []
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -199,6 +226,9 @@ class ConlluGenerator(object):
 | 
				
			|||||||
            for sent in sentences: 
 | 
					            for sent in sentences: 
 | 
				
			||||||
                if (len(sent) > 0):
 | 
					                if (len(sent) > 0):
 | 
				
			||||||
                    simple_tokenized = nltk.tokenize.word_tokenize(sent)
 | 
					                    simple_tokenized = nltk.tokenize.word_tokenize(sent)
 | 
				
			||||||
 | 
					                    if self.mwe_tokenizer is None:
 | 
				
			||||||
 | 
					                        tokenized_sentences.append(simple_tokenized)
 | 
				
			||||||
 | 
					                    else:
 | 
				
			||||||
                        tokenized_sentences.append(
 | 
					                        tokenized_sentences.append(
 | 
				
			||||||
                            self.mwe_tokenizer.tokenize(simple_tokenized))
 | 
					                            self.mwe_tokenizer.tokenize(simple_tokenized))
 | 
				
			||||||
            tokenized_documents.append(tokenized_sentences)
 | 
					            tokenized_documents.append(tokenized_sentences)
 | 
				
			||||||
@ -210,13 +240,14 @@ class ConlluGenerator(object):
 | 
				
			|||||||
            else:
 | 
					            else:
 | 
				
			||||||
                conllu_doc = ConlluDocument()
 | 
					                conllu_doc = ConlluDocument()
 | 
				
			||||||
            for sent in doc:
 | 
					            for sent in doc:
 | 
				
			||||||
 | 
					                token_id = 0
 | 
				
			||||||
                conllu_sent = ConlluSentence()
 | 
					                conllu_sent = ConlluSentence()
 | 
				
			||||||
                for token in sent:
 | 
					                for token in sent:
 | 
				
			||||||
 | 
					                    token_id += 1
 | 
				
			||||||
                    conllu_sent.add(ConlluElement(
 | 
					                    conllu_sent.add(ConlluElement(
 | 
				
			||||||
                        id=self.id_counter + 1,
 | 
					                        id=token_id,
 | 
				
			||||||
                        form=token,
 | 
					                        form=token,
 | 
				
			||||||
                    ))
 | 
					                    ))
 | 
				
			||||||
                    self.id_counter += 1
 | 
					 | 
				
			||||||
                conllu_doc.add(conllu_sent)
 | 
					                conllu_doc.add(conllu_sent)
 | 
				
			||||||
            self.conllu_documents.append(conllu_doc)
 | 
					            self.conllu_documents.append(conllu_doc)
 | 
				
			||||||
            i += 1
 | 
					            i += 1
 | 
				
			||||||
@ -247,6 +278,9 @@ class ConlluGenerator(object):
 | 
				
			|||||||
                    if elem.lemma in stemmed_keyword_list:
 | 
					                    if elem.lemma in stemmed_keyword_list:
 | 
				
			||||||
                        elem.add_misc(key, value)
 | 
					                        elem.add_misc(key, value)
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
 | 
					    def get_conllu_elements(self):
 | 
				
			||||||
 | 
					        return [doc.get_conllu_elements() for doc in self.conllu_documents]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def __repr__(self):
 | 
					    def __repr__(self):
 | 
				
			||||||
        result = ""
 | 
					        result = ""
 | 
				
			||||||
        for document in self.conllu_documents:
 | 
					        for document in self.conllu_documents:
 | 
				
			||||||
 | 
				
			|||||||
@ -13,7 +13,17 @@
 | 
				
			|||||||
   "metadata": {},
 | 
					   "metadata": {},
 | 
				
			||||||
   "outputs": [],
 | 
					   "outputs": [],
 | 
				
			||||||
   "source": [
 | 
					   "source": [
 | 
				
			||||||
    "import conllu_batch_generator as cbg"
 | 
					    "import sys\n",
 | 
				
			||||||
 | 
					    "sys.path.append(\"../\")"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 2,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import Tagging.conllu_batch_generator as cbg"
 | 
				
			||||||
   ]
 | 
					   ]
 | 
				
			||||||
  },
 | 
					  },
 | 
				
			||||||
  {
 | 
					  {
 | 
				
			||||||
 | 
				
			|||||||
@ -3,7 +3,11 @@
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
# # crf data Generator
 | 
					# # crf data Generator
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import conllu_batch_generator as cbg
 | 
					import sys
 | 
				
			||||||
 | 
					sys.path.append("../")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import Tagging.conllu_batch_generator as cbg
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def word2features(sent, i):
 | 
					def word2features(sent, i):
 | 
				
			||||||
 | 
				
			|||||||
@ -46,6 +46,12 @@
 | 
				
			|||||||
    "containers = importlib.util.module_from_spec(spec)\n",
 | 
					    "containers = importlib.util.module_from_spec(spec)\n",
 | 
				
			||||||
    "spec.loader.exec_module(containers)\n",
 | 
					    "spec.loader.exec_module(containers)\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "# loading placeholders\n",
 | 
				
			||||||
 | 
					    "spec = importlib.util.spec_from_file_location(\n",
 | 
				
			||||||
 | 
					    "    \"placeholders\", \"../\" + settings.placeholder_file)\n",
 | 
				
			||||||
 | 
					    "placeholders = importlib.util.module_from_spec(spec)\n",
 | 
				
			||||||
 | 
					    "spec.loader.exec_module(placeholders)\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
    "# skipping recipes:\n",
 | 
					    "# skipping recipes:\n",
 | 
				
			||||||
    "n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0\n",
 | 
					    "n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0\n",
 | 
				
			||||||
    "print(\"start reading at recipe \" + str(n_skipped_recipes))\n",
 | 
					    "print(\"start reading at recipe \" + str(n_skipped_recipes))\n",
 | 
				
			||||||
@ -80,7 +86,8 @@
 | 
				
			|||||||
    "    \n",
 | 
					    "    \n",
 | 
				
			||||||
    "    cg.add_misc_value_by_list(\"food_type\", \"ingredient\", [w.replace(\" \",\"_\") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)\n",
 | 
					    "    cg.add_misc_value_by_list(\"food_type\", \"ingredient\", [w.replace(\" \",\"_\") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)\n",
 | 
				
			||||||
    "    cg.add_misc_value_by_list(\"food_type\", \"action\", actions.stemmed_cooking_verbs)\n",
 | 
					    "    cg.add_misc_value_by_list(\"food_type\", \"action\", actions.stemmed_cooking_verbs)\n",
 | 
				
			||||||
    "    cg.add_misc_value_by_list(\"food_type\", \"containers\", containers.containers)\n",
 | 
					    "    cg.add_misc_value_by_list(\"food_type\", \"containers\", containers.stemmed_containers)\n",
 | 
				
			||||||
 | 
					    "    cg.add_misc_value_by_list(\"food_type\", \"placeholders\", placeholders.stemmed_placeholders)\n",
 | 
				
			||||||
    "\n",
 | 
					    "\n",
 | 
				
			||||||
    "    savefile.write(str(cg))"
 | 
					    "    savefile.write(str(cg))"
 | 
				
			||||||
   ]
 | 
					   ]
 | 
				
			||||||
 | 
				
			|||||||
@ -30,6 +30,12 @@ spec = importlib.util.spec_from_file_location(
 | 
				
			|||||||
containers = importlib.util.module_from_spec(spec)
 | 
					containers = importlib.util.module_from_spec(spec)
 | 
				
			||||||
spec.loader.exec_module(containers)
 | 
					spec.loader.exec_module(containers)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					# loading placeholders
 | 
				
			||||||
 | 
					spec = importlib.util.spec_from_file_location(
 | 
				
			||||||
 | 
					    "placeholders", "../" + settings.placeholder_file)
 | 
				
			||||||
 | 
					placeholders = importlib.util.module_from_spec(spec)
 | 
				
			||||||
 | 
					spec.loader.exec_module(placeholders)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
# skipping recipes:
 | 
					# skipping recipes:
 | 
				
			||||||
n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0
 | 
					n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0
 | 
				
			||||||
print("start reading at recipe " + str(n_skipped_recipes))
 | 
					print("start reading at recipe " + str(n_skipped_recipes))
 | 
				
			||||||
@ -58,7 +64,8 @@ def process_instructions(instructions: list, document_ids=None):
 | 
				
			|||||||
    
 | 
					    
 | 
				
			||||||
    cg.add_misc_value_by_list("food_type", "ingredient", [w.replace(" ","_") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)
 | 
					    cg.add_misc_value_by_list("food_type", "ingredient", [w.replace(" ","_") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)
 | 
				
			||||||
    cg.add_misc_value_by_list("food_type", "action", actions.stemmed_cooking_verbs)
 | 
					    cg.add_misc_value_by_list("food_type", "action", actions.stemmed_cooking_verbs)
 | 
				
			||||||
    cg.add_misc_value_by_list("food_type", "containers", containers.containers)
 | 
					    cg.add_misc_value_by_list("food_type", "containers", containers.stemmed_containers)
 | 
				
			||||||
 | 
					    cg.add_misc_value_by_list("food_type", "placeholders", placeholders.stemmed_placeholders)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    savefile.write(str(cg))
 | 
					    savefile.write(str(cg))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
				
			|||||||
										
											Binary file not shown.
										
									
								
							
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							
							
								
								
									
										1298
									
								
								db/create_database.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1298
									
								
								db/create_database.ipynb
									
									
									
									
									
										Normal file
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							
							
								
								
									
										304
									
								
								db/create_database_docker.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										304
									
								
								db/create_database_docker.ipynb
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,304 @@
 | 
				
			|||||||
 | 
					{
 | 
				
			||||||
 | 
					 "cells": [
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "# Create Database with Docker"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* download mariadb image"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 2,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stdout",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "Using default tag: latest\n",
 | 
				
			||||||
 | 
					      "latest: Pulling from mariadb/server\n",
 | 
				
			||||||
 | 
					      "Digest: sha256:b5762c478d38ae54c464e3ab63e10e0c3f951633ed7619d52fa3c22bcf36218a\n",
 | 
				
			||||||
 | 
					      "Status: Image is up to date for mariadb/server:latest\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%%bash \n",
 | 
				
			||||||
 | 
					    "docker pull mariadb/server"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* create and run container"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 3,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stdout",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "09342f7c540b2b6224bb96b2e3a542a5ff144e6bbc1cf243ae33b0dab9262c47\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%%bash\n",
 | 
				
			||||||
 | 
					    "docker run --name recipe_db -e MYSQL_ROOT_PASSWORD=\"g00d_r3c1p3s\" -d mariadb/server"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* checking docker logs"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 5,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stdout",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "Initializing database\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "PLEASE REMEMBER TO SET A PASSWORD FOR THE MariaDB root USER !\n",
 | 
				
			||||||
 | 
					      "To do so, start the server, then issue the following commands:\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "'/usr/bin/mysqladmin' -u root password 'new-password'\n",
 | 
				
			||||||
 | 
					      "'/usr/bin/mysqladmin' -u root -h  password 'new-password'\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "Alternatively you can run:\n",
 | 
				
			||||||
 | 
					      "'/usr/bin/mysql_secure_installation'\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "which will also give you the option of removing the test\n",
 | 
				
			||||||
 | 
					      "databases and anonymous user created by default.  This is\n",
 | 
				
			||||||
 | 
					      "strongly recommended for production servers.\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "See the MariaDB Knowledgebase at http://mariadb.com/kb or the\n",
 | 
				
			||||||
 | 
					      "MySQL manual for more instructions.\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "Please report any problems at http://mariadb.org/jira\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "The latest information about MariaDB is available at http://mariadb.org/.\n",
 | 
				
			||||||
 | 
					      "You can find additional information about the MySQL part at:\n",
 | 
				
			||||||
 | 
					      "http://dev.mysql.com\n",
 | 
				
			||||||
 | 
					      "Consider joining MariaDB's strong and vibrant community:\n",
 | 
				
			||||||
 | 
					      "https://mariadb.org/get-involved/\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "Database initialized\n",
 | 
				
			||||||
 | 
					      "MySQL init process in progress...\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "MySQL init process done. Ready for start up.\n",
 | 
				
			||||||
 | 
					      "\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stderr",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] mysqld (mysqld 10.3.13-MariaDB-1:10.3.13+maria~bionic) starting as process 104 ...\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Using Linux native AIO\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Uses event mutexes\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Compressed tables use zlib 1.2.11\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Number of pools: 1\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Using SSE2 crc32 instructions\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Initializing buffer pool, total size = 256M, instances = 1, chunk size = 128M\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Completed initialization of buffer pool\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: 128 out of 128 rollback segments are active.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Creating shared tablespace for temporary tables\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:43 0 [Note] InnoDB: Waiting for purge to start\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] InnoDB: 10.3.13 started; log sequence number 1630815; transaction id 21\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] Plugin 'FEEDBACK' is disabled.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] InnoDB: Buffer pool(s) load completed at 190808 14:46:44\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Warning] 'user' entry 'root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Warning] 'user' entry '@09342f7c540b' ignored in --skip-name-resolve mode.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] Reading of all Master_info entries succeded\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] Added new Master_info '' to hash table\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:44 0 [Note] mysqld: ready for connections.\n",
 | 
				
			||||||
 | 
					      "Version: '10.3.13-MariaDB-1:10.3.13+maria~bionic'  socket: '/var/run/mysqld/mysqld.sock'  port: 0  mariadb.org binary distribution\n",
 | 
				
			||||||
 | 
					      "Warning: Unable to load '/usr/share/zoneinfo/leap-seconds.list' as time zone. Skipping it.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 10 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 0 [Note] mysqld (initiated by: unknown): Normal shutdown\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 0 [Note] Event Scheduler: Purging the queue. 0 events\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 0 [Note] InnoDB: FTS optimize thread exiting.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 0 [Note] InnoDB: Starting shutdown...\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 0 [Note] InnoDB: Dumping buffer pool(s) to /var/lib/mysql/ib_buffer_pool\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:46 0 [Note] InnoDB: Buffer pool(s) dump completed at 190808 14:46:46\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Shutdown completed; log sequence number 1630824; transaction id 24\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Removed temporary tablespace data file: \"ibtmp1\"\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] mysqld: Shutdown complete\n",
 | 
				
			||||||
 | 
					      "\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] mysqld (mysqld 10.3.13-MariaDB-1:10.3.13+maria~bionic) starting as process 1 ...\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Using Linux native AIO\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Uses event mutexes\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Compressed tables use zlib 1.2.11\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Number of pools: 1\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Using SSE2 crc32 instructions\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Initializing buffer pool, total size = 256M, instances = 1, chunk size = 128M\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Completed initialization of buffer pool\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: 128 out of 128 rollback segments are active.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Creating shared tablespace for temporary tables\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:47 0 [Note] InnoDB: Waiting for purge to start\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] InnoDB: 10.3.13 started; log sequence number 1630824; transaction id 21\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] Plugin 'FEEDBACK' is disabled.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] Server socket created on IP: '::'.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] InnoDB: Buffer pool(s) load completed at 190808 14:46:48\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] Reading of all Master_info entries succeded\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] Added new Master_info '' to hash table\n",
 | 
				
			||||||
 | 
					      "2019-08-08 14:46:48 0 [Note] mysqld: ready for connections.\n",
 | 
				
			||||||
 | 
					      "Version: '10.3.13-MariaDB-1:10.3.13+maria~bionic'  socket: '/var/run/mysqld/mysqld.sock'  port: 3306  mariadb.org binary distribution\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%%bash\n",
 | 
				
			||||||
 | 
					    "docker logs recipe_db"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* get local db address. **NOTE**: dont forget to set this address in settings.py!"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 9,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stdout",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "172.17.0.2\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%%bash\n",
 | 
				
			||||||
 | 
					    "docker inspect --format '{{ .NetworkSettings.IPAddress }}' recipe_db"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* and create databse"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 10,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stderr",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "mysql: [Warning] Using a password on the command line interface can be insecure.\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%%bash\n",
 | 
				
			||||||
 | 
					    "mysql -h 172.17.0.2 -u root --password=\"g00d_r3c1p3s\" -e \"CREATE DATABASE recipe_db\""
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "markdown",
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "* check whether database is created:"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 11,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stdout",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "Database\n",
 | 
				
			||||||
 | 
					      "information_schema\n",
 | 
				
			||||||
 | 
					      "mysql\n",
 | 
				
			||||||
 | 
					      "performance_schema\n",
 | 
				
			||||||
 | 
					      "recipe_db\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    },
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "name": "stderr",
 | 
				
			||||||
 | 
					     "output_type": "stream",
 | 
				
			||||||
 | 
					     "text": [
 | 
				
			||||||
 | 
					      "mysql: [Warning] Using a password on the command line interface can be insecure.\n"
 | 
				
			||||||
 | 
					     ]
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "%%bash\n",
 | 
				
			||||||
 | 
					    "mysql -h 172.17.0.2 -u root --password=\"g00d_r3c1p3s\" -e \"show databases\""
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": null,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": []
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 ],
 | 
				
			||||||
 | 
					 "metadata": {
 | 
				
			||||||
 | 
					  "kernelspec": {
 | 
				
			||||||
 | 
					   "display_name": "Python 3",
 | 
				
			||||||
 | 
					   "language": "python",
 | 
				
			||||||
 | 
					   "name": "python3"
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  "language_info": {
 | 
				
			||||||
 | 
					   "codemirror_mode": {
 | 
				
			||||||
 | 
					    "name": "ipython",
 | 
				
			||||||
 | 
					    "version": 3
 | 
				
			||||||
 | 
					   },
 | 
				
			||||||
 | 
					   "file_extension": ".py",
 | 
				
			||||||
 | 
					   "mimetype": "text/x-python",
 | 
				
			||||||
 | 
					   "name": "python",
 | 
				
			||||||
 | 
					   "nbconvert_exporter": "python",
 | 
				
			||||||
 | 
					   "pygments_lexer": "ipython3",
 | 
				
			||||||
 | 
					   "version": "3.7.3"
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 },
 | 
				
			||||||
 | 
					 "nbformat": 4,
 | 
				
			||||||
 | 
					 "nbformat_minor": 4
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										190
									
								
								db/database_connection.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										190
									
								
								db/database_connection.ipynb
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,190 @@
 | 
				
			|||||||
 | 
					{
 | 
				
			||||||
 | 
					 "cells": [
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 1,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "import pymysql.cursors\n",
 | 
				
			||||||
 | 
					    "import sys\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "def get_sql_time(datetime_object):\n",
 | 
				
			||||||
 | 
					    "    return datetime_object.strftime('%Y-%m-%d %H:%M:%S')\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "class SQLInjectionError(Exception):\n",
 | 
				
			||||||
 | 
					    "    def __init__(self):\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "        # Call the base class constructor with the parameters it needs\n",
 | 
				
			||||||
 | 
					    "        super().__init__(\"Detected possible SQL injection attack!\")\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "class DatabaseConnection(object):\n",
 | 
				
			||||||
 | 
					    "    \"\"\"\n",
 | 
				
			||||||
 | 
					    "    a singleton class for a global database connection\n",
 | 
				
			||||||
 | 
					    "    \"\"\"\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    instance = None\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    @staticmethod\n",
 | 
				
			||||||
 | 
					    "    def global_cursor():\n",
 | 
				
			||||||
 | 
					    "        assert DatabaseConnection.instance is not None\n",
 | 
				
			||||||
 | 
					    "        return DatabaseConnection.instance.get_cursor()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    @staticmethod\n",
 | 
				
			||||||
 | 
					    "    def global_close():\n",
 | 
				
			||||||
 | 
					    "        assert DatabaseConnection.instance is not None\n",
 | 
				
			||||||
 | 
					    "        DatabaseConnection.instance.close()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    @staticmethod\n",
 | 
				
			||||||
 | 
					    "    def global_commit():\n",
 | 
				
			||||||
 | 
					    "        assert DatabaseConnection.instance is not None\n",
 | 
				
			||||||
 | 
					    "        DatabaseConnection.instance.commit()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    @staticmethod\n",
 | 
				
			||||||
 | 
					    "    def global_ping():\n",
 | 
				
			||||||
 | 
					    "        assert DatabaseConnection.instance is not None\n",
 | 
				
			||||||
 | 
					    "        DatabaseConnection.instance.connection.ping()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    @staticmethod\n",
 | 
				
			||||||
 | 
					    "    def global_single_query(query, params=None):\n",
 | 
				
			||||||
 | 
					    "        DatabaseConnection.global_ping()\n",
 | 
				
			||||||
 | 
					    "        if ';' in query:\n",
 | 
				
			||||||
 | 
					    "            # Possible injection!\n",
 | 
				
			||||||
 | 
					    "            raise SQLInjectionError()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "        with DatabaseConnection.global_cursor() as c:\n",
 | 
				
			||||||
 | 
					    "            if params is None:\n",
 | 
				
			||||||
 | 
					    "                c.execute(query)\n",
 | 
				
			||||||
 | 
					    "            else:\n",
 | 
				
			||||||
 | 
					    "                c.execute(query, params)\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "            return c.fetchall()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    @staticmethod\n",
 | 
				
			||||||
 | 
					    "    def global_single_execution(sql_statement, params=None):\n",
 | 
				
			||||||
 | 
					    "        DatabaseConnection.global_ping()\n",
 | 
				
			||||||
 | 
					    "        if ';' in sql_statement:\n",
 | 
				
			||||||
 | 
					    "            # Possible injection detected!\n",
 | 
				
			||||||
 | 
					    "            raise SQLInjectionError()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "        with DatabaseConnection.global_cursor() as c:\n",
 | 
				
			||||||
 | 
					    "            if params is None:\n",
 | 
				
			||||||
 | 
					    "                c.execute(sql_statement)\n",
 | 
				
			||||||
 | 
					    "            else:\n",
 | 
				
			||||||
 | 
					    "                c.execute(sql_statement, params)\n",
 | 
				
			||||||
 | 
					    "            DatabaseConnection.global_commit()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    def __init__(self,\n",
 | 
				
			||||||
 | 
					    "                 host: str,\n",
 | 
				
			||||||
 | 
					    "                 port: int,\n",
 | 
				
			||||||
 | 
					    "                 user: str,\n",
 | 
				
			||||||
 | 
					    "                 password: str,\n",
 | 
				
			||||||
 | 
					    "                 db: str,\n",
 | 
				
			||||||
 | 
					    "                 charset: str):\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "        assert DatabaseConnection.instance is None\n",
 | 
				
			||||||
 | 
					    "        try:\n",
 | 
				
			||||||
 | 
					    "            self.connection = pymysql.connect(\n",
 | 
				
			||||||
 | 
					    "                host=host,\n",
 | 
				
			||||||
 | 
					    "                port=port,\n",
 | 
				
			||||||
 | 
					    "                user=user,\n",
 | 
				
			||||||
 | 
					    "                password=password,\n",
 | 
				
			||||||
 | 
					    "                db=db,\n",
 | 
				
			||||||
 | 
					    "                charset=charset,\n",
 | 
				
			||||||
 | 
					    "                cursorclass=pymysql.cursors.DictCursor)\n",
 | 
				
			||||||
 | 
					    "            DatabaseConnection.instance = self\n",
 | 
				
			||||||
 | 
					    "        except Exception as e:\n",
 | 
				
			||||||
 | 
					    "            sys.stderr.write(\"could not connect to database '\" +\n",
 | 
				
			||||||
 | 
					    "                             str(db) +\n",
 | 
				
			||||||
 | 
					    "                             \"' at \" +\n",
 | 
				
			||||||
 | 
					    "                             user +\n",
 | 
				
			||||||
 | 
					    "                             \"@\" +\n",
 | 
				
			||||||
 | 
					    "                             host +\n",
 | 
				
			||||||
 | 
					    "                             \":\" +\n",
 | 
				
			||||||
 | 
					    "                             str(port) +\n",
 | 
				
			||||||
 | 
					    "                             \"\\nCheck the configuration in settings.py!\\n\")\n",
 | 
				
			||||||
 | 
					    "            raise Exception('could not connect to database')\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    def get_cursor(self):\n",
 | 
				
			||||||
 | 
					    "        return self.connection.cursor()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    def close(self):\n",
 | 
				
			||||||
 | 
					    "        self.connection.close()\n",
 | 
				
			||||||
 | 
					    "        DatabaseConnection.instance = None\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "    def commit(self):\n",
 | 
				
			||||||
 | 
					    "        self.connection.commit()\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "def test_connection():\n",
 | 
				
			||||||
 | 
					    "    import db_settings as settings\n",
 | 
				
			||||||
 | 
					    "    DatabaseConnection(settings.db_host,\n",
 | 
				
			||||||
 | 
					    "                       settings.db_port,\n",
 | 
				
			||||||
 | 
					    "                       settings.db_user,\n",
 | 
				
			||||||
 | 
					    "                       settings.db_pw,\n",
 | 
				
			||||||
 | 
					    "                       settings.db_db,\n",
 | 
				
			||||||
 | 
					    "                       settings.db_charset)"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 2,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "test_connection()"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 3,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [
 | 
				
			||||||
 | 
					    {
 | 
				
			||||||
 | 
					     "data": {
 | 
				
			||||||
 | 
					      "text/plain": [
 | 
				
			||||||
 | 
					       "()"
 | 
				
			||||||
 | 
					      ]
 | 
				
			||||||
 | 
					     },
 | 
				
			||||||
 | 
					     "execution_count": 3,
 | 
				
			||||||
 | 
					     "metadata": {},
 | 
				
			||||||
 | 
					     "output_type": "execute_result"
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					   ],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "DatabaseConnection.global_single_query(\"show tables\")"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": null,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": []
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 ],
 | 
				
			||||||
 | 
					 "metadata": {
 | 
				
			||||||
 | 
					  "kernelspec": {
 | 
				
			||||||
 | 
					   "display_name": "Python 3",
 | 
				
			||||||
 | 
					   "language": "python",
 | 
				
			||||||
 | 
					   "name": "python3"
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  "language_info": {
 | 
				
			||||||
 | 
					   "codemirror_mode": {
 | 
				
			||||||
 | 
					    "name": "ipython",
 | 
				
			||||||
 | 
					    "version": 3
 | 
				
			||||||
 | 
					   },
 | 
				
			||||||
 | 
					   "file_extension": ".py",
 | 
				
			||||||
 | 
					   "mimetype": "text/x-python",
 | 
				
			||||||
 | 
					   "name": "python",
 | 
				
			||||||
 | 
					   "nbconvert_exporter": "python",
 | 
				
			||||||
 | 
					   "pygments_lexer": "ipython3",
 | 
				
			||||||
 | 
					   "version": "3.7.3"
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 },
 | 
				
			||||||
 | 
					 "nbformat": 4,
 | 
				
			||||||
 | 
					 "nbformat_minor": 4
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										125
									
								
								db/database_connection.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										125
									
								
								db/database_connection.py
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,125 @@
 | 
				
			|||||||
 | 
					#!/usr/bin/env python3
 | 
				
			||||||
 | 
					# coding: utf-8
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import pymysql.cursors
 | 
				
			||||||
 | 
					import sys
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					def get_sql_time(datetime_object):
 | 
				
			||||||
 | 
					    return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class SQLInjectionError(Exception):
 | 
				
			||||||
 | 
					    def __init__(self):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        # Call the base class constructor with the parameters it needs
 | 
				
			||||||
 | 
					        super().__init__("Detected possible SQL injection attack!")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class DatabaseConnection(object):
 | 
				
			||||||
 | 
					    """
 | 
				
			||||||
 | 
					    a singleton class for a global database connection
 | 
				
			||||||
 | 
					    """
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    instance = None
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @staticmethod
 | 
				
			||||||
 | 
					    def global_cursor():
 | 
				
			||||||
 | 
					        assert DatabaseConnection.instance is not None
 | 
				
			||||||
 | 
					        return DatabaseConnection.instance.get_cursor()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @staticmethod
 | 
				
			||||||
 | 
					    def global_close():
 | 
				
			||||||
 | 
					        assert DatabaseConnection.instance is not None
 | 
				
			||||||
 | 
					        DatabaseConnection.instance.close()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @staticmethod
 | 
				
			||||||
 | 
					    def global_commit():
 | 
				
			||||||
 | 
					        assert DatabaseConnection.instance is not None
 | 
				
			||||||
 | 
					        DatabaseConnection.instance.commit()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @staticmethod
 | 
				
			||||||
 | 
					    def global_ping():
 | 
				
			||||||
 | 
					        assert DatabaseConnection.instance is not None
 | 
				
			||||||
 | 
					        DatabaseConnection.instance.connection.ping()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @staticmethod
 | 
				
			||||||
 | 
					    def global_single_query(query, params=None):
 | 
				
			||||||
 | 
					        DatabaseConnection.global_ping()
 | 
				
			||||||
 | 
					        if ';' in query:
 | 
				
			||||||
 | 
					            # Possible injection!
 | 
				
			||||||
 | 
					            raise SQLInjectionError()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        with DatabaseConnection.global_cursor() as c:
 | 
				
			||||||
 | 
					            if params is None:
 | 
				
			||||||
 | 
					                c.execute(query)
 | 
				
			||||||
 | 
					            else:
 | 
				
			||||||
 | 
					                c.execute(query, params)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            return c.fetchall()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @staticmethod
 | 
				
			||||||
 | 
					    def global_single_execution(sql_statement, params=None):
 | 
				
			||||||
 | 
					        DatabaseConnection.global_ping()
 | 
				
			||||||
 | 
					        if ';' in sql_statement:
 | 
				
			||||||
 | 
					            # Possible injection detected!
 | 
				
			||||||
 | 
					            raise SQLInjectionError()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        with DatabaseConnection.global_cursor() as c:
 | 
				
			||||||
 | 
					            if params is None:
 | 
				
			||||||
 | 
					                c.execute(sql_statement)
 | 
				
			||||||
 | 
					            else:
 | 
				
			||||||
 | 
					                c.execute(sql_statement, params)
 | 
				
			||||||
 | 
					            DatabaseConnection.global_commit()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    def __init__(self,
 | 
				
			||||||
 | 
					                 host: str,
 | 
				
			||||||
 | 
					                 port: int,
 | 
				
			||||||
 | 
					                 user: str,
 | 
				
			||||||
 | 
					                 password: str,
 | 
				
			||||||
 | 
					                 db: str,
 | 
				
			||||||
 | 
					                 charset: str):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        assert DatabaseConnection.instance is None
 | 
				
			||||||
 | 
					        try:
 | 
				
			||||||
 | 
					            self.connection = pymysql.connect(
 | 
				
			||||||
 | 
					                host=host,
 | 
				
			||||||
 | 
					                port=port,
 | 
				
			||||||
 | 
					                user=user,
 | 
				
			||||||
 | 
					                password=password,
 | 
				
			||||||
 | 
					                db=db,
 | 
				
			||||||
 | 
					                charset=charset,
 | 
				
			||||||
 | 
					                cursorclass=pymysql.cursors.DictCursor)
 | 
				
			||||||
 | 
					            DatabaseConnection.instance = self
 | 
				
			||||||
 | 
					        except Exception as e:
 | 
				
			||||||
 | 
					            sys.stderr.write("could not connect to database '" +
 | 
				
			||||||
 | 
					                             str(db) +
 | 
				
			||||||
 | 
					                             "' at " +
 | 
				
			||||||
 | 
					                             user +
 | 
				
			||||||
 | 
					                             "@" +
 | 
				
			||||||
 | 
					                             host +
 | 
				
			||||||
 | 
					                             ":" +
 | 
				
			||||||
 | 
					                             str(port) +
 | 
				
			||||||
 | 
					                             "\nCheck the configuration in settings.py!\n")
 | 
				
			||||||
 | 
					            raise Exception('could not connect to database')
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    def get_cursor(self):
 | 
				
			||||||
 | 
					        return self.connection.cursor()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    def close(self):
 | 
				
			||||||
 | 
					        self.connection.close()
 | 
				
			||||||
 | 
					        DatabaseConnection.instance = None
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    def commit(self):
 | 
				
			||||||
 | 
					        self.connection.commit()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					def test_connection():
 | 
				
			||||||
 | 
					    import settings
 | 
				
			||||||
 | 
					    DatabaseConnection(settings.db_host,
 | 
				
			||||||
 | 
					                       settings.db_port,
 | 
				
			||||||
 | 
					                       settings.db_user,
 | 
				
			||||||
 | 
					                       settings.db_pw,
 | 
				
			||||||
 | 
					                       settings.db_db,
 | 
				
			||||||
 | 
					                       settings.db_charset)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
							
								
								
									
										48
									
								
								db/db_settings.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										48
									
								
								db/db_settings.ipynb
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,48 @@
 | 
				
			|||||||
 | 
					{
 | 
				
			||||||
 | 
					 "cells": [
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": 5,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": [
 | 
				
			||||||
 | 
					    "db_host = \"172.17.0.2\"\n",
 | 
				
			||||||
 | 
					    "db_port = 3306\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "db_user = \"root\"\n",
 | 
				
			||||||
 | 
					    "db_pw = \"g00d_r3c1p3s\"\n",
 | 
				
			||||||
 | 
					    "db_db = \"recipe_db\"\n",
 | 
				
			||||||
 | 
					    "\n",
 | 
				
			||||||
 | 
					    "db_charset = 'utf8mb4'"
 | 
				
			||||||
 | 
					   ]
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  {
 | 
				
			||||||
 | 
					   "cell_type": "code",
 | 
				
			||||||
 | 
					   "execution_count": null,
 | 
				
			||||||
 | 
					   "metadata": {},
 | 
				
			||||||
 | 
					   "outputs": [],
 | 
				
			||||||
 | 
					   "source": []
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 ],
 | 
				
			||||||
 | 
					 "metadata": {
 | 
				
			||||||
 | 
					  "kernelspec": {
 | 
				
			||||||
 | 
					   "display_name": "Python 3",
 | 
				
			||||||
 | 
					   "language": "python",
 | 
				
			||||||
 | 
					   "name": "python3"
 | 
				
			||||||
 | 
					  },
 | 
				
			||||||
 | 
					  "language_info": {
 | 
				
			||||||
 | 
					   "codemirror_mode": {
 | 
				
			||||||
 | 
					    "name": "ipython",
 | 
				
			||||||
 | 
					    "version": 3
 | 
				
			||||||
 | 
					   },
 | 
				
			||||||
 | 
					   "file_extension": ".py",
 | 
				
			||||||
 | 
					   "mimetype": "text/x-python",
 | 
				
			||||||
 | 
					   "name": "python",
 | 
				
			||||||
 | 
					   "nbconvert_exporter": "python",
 | 
				
			||||||
 | 
					   "pygments_lexer": "ipython3",
 | 
				
			||||||
 | 
					   "version": "3.7.3"
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					 },
 | 
				
			||||||
 | 
					 "nbformat": 4,
 | 
				
			||||||
 | 
					 "nbformat_minor": 4
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										13
									
								
								db/db_settings.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										13
									
								
								db/db_settings.py
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,13 @@
 | 
				
			|||||||
 | 
					#!/usr/bin/env python3
 | 
				
			||||||
 | 
					# coding: utf-8
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					db_host = "172.17.0.2"
 | 
				
			||||||
 | 
					db_port = 3306
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					db_user = "root"
 | 
				
			||||||
 | 
					db_pw = "g00d_r3c1p3s"
 | 
				
			||||||
 | 
					db_db = "recipe_db"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					db_charset = 'utf8mb4'
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
							
								
								
									
										2247
									
								
								reports/demos.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										2247
									
								
								reports/demos.ipynb
									
									
									
									
									
										Normal file
									
								
							
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							@ -9,5 +9,6 @@ fooddb_folder = data_root + "foodb_2017_06_29_csv/"
 | 
				
			|||||||
ingredients_file = data_root + "ingredients.py"
 | 
					ingredients_file = data_root + "ingredients.py"
 | 
				
			||||||
actions_file = data_root + "actions.py"
 | 
					actions_file = data_root + "actions.py"
 | 
				
			||||||
container_file = data_root + "containers.py"
 | 
					container_file = data_root + "containers.py"
 | 
				
			||||||
 | 
					placeholder_file = data_root + "placeholders.py"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
gzipped_conllu_data_root  = data_root + "1M_recipes_conllu/"
 | 
					gzipped_conllu_data_root  = data_root + "1M_recipes_conllu/"
 | 
				
			||||||
 | 
				
			|||||||
		Reference in New Issue
	
	Block a user