moved 1 million recipes into database, starting coarse tree generation for recipe analysis
This commit is contained in:
		
							
								
								
									
										1298
									
								
								db/create_database.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1298
									
								
								db/create_database.ipynb
									
									
									
									
									
										Normal file
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							
							
								
								
									
										304
									
								
								db/create_database_docker.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										304
									
								
								db/create_database_docker.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,304 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "# Create Database with Docker" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* download mariadb image" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Using default tag: latest\n", | ||||
|       "latest: Pulling from mariadb/server\n", | ||||
|       "Digest: sha256:b5762c478d38ae54c464e3ab63e10e0c3f951633ed7619d52fa3c22bcf36218a\n", | ||||
|       "Status: Image is up to date for mariadb/server:latest\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash \n", | ||||
|     "docker pull mariadb/server" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* create and run container" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "09342f7c540b2b6224bb96b2e3a542a5ff144e6bbc1cf243ae33b0dab9262c47\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "docker run --name recipe_db -e MYSQL_ROOT_PASSWORD=\"g00d_r3c1p3s\" -d mariadb/server" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* checking docker logs" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Initializing database\n", | ||||
|       "\n", | ||||
|       "\n", | ||||
|       "PLEASE REMEMBER TO SET A PASSWORD FOR THE MariaDB root USER !\n", | ||||
|       "To do so, start the server, then issue the following commands:\n", | ||||
|       "\n", | ||||
|       "'/usr/bin/mysqladmin' -u root password 'new-password'\n", | ||||
|       "'/usr/bin/mysqladmin' -u root -h  password 'new-password'\n", | ||||
|       "\n", | ||||
|       "Alternatively you can run:\n", | ||||
|       "'/usr/bin/mysql_secure_installation'\n", | ||||
|       "\n", | ||||
|       "which will also give you the option of removing the test\n", | ||||
|       "databases and anonymous user created by default.  This is\n", | ||||
|       "strongly recommended for production servers.\n", | ||||
|       "\n", | ||||
|       "See the MariaDB Knowledgebase at http://mariadb.com/kb or the\n", | ||||
|       "MySQL manual for more instructions.\n", | ||||
|       "\n", | ||||
|       "Please report any problems at http://mariadb.org/jira\n", | ||||
|       "\n", | ||||
|       "The latest information about MariaDB is available at http://mariadb.org/.\n", | ||||
|       "You can find additional information about the MySQL part at:\n", | ||||
|       "http://dev.mysql.com\n", | ||||
|       "Consider joining MariaDB's strong and vibrant community:\n", | ||||
|       "https://mariadb.org/get-involved/\n", | ||||
|       "\n", | ||||
|       "Database initialized\n", | ||||
|       "MySQL init process in progress...\n", | ||||
|       "\n", | ||||
|       "\n", | ||||
|       "MySQL init process done. Ready for start up.\n", | ||||
|       "\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "2019-08-08 14:46:43 0 [Note] mysqld (mysqld 10.3.13-MariaDB-1:10.3.13+maria~bionic) starting as process 104 ...\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Using Linux native AIO\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Uses event mutexes\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Compressed tables use zlib 1.2.11\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Number of pools: 1\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Using SSE2 crc32 instructions\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Initializing buffer pool, total size = 256M, instances = 1, chunk size = 128M\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Completed initialization of buffer pool\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: 128 out of 128 rollback segments are active.\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Creating shared tablespace for temporary tables\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.\n", | ||||
|       "2019-08-08 14:46:43 0 [Note] InnoDB: Waiting for purge to start\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] InnoDB: 10.3.13 started; log sequence number 1630815; transaction id 21\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] Plugin 'FEEDBACK' is disabled.\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] InnoDB: Buffer pool(s) load completed at 190808 14:46:44\n", | ||||
|       "2019-08-08 14:46:44 0 [Warning] 'user' entry 'root@09342f7c540b' ignored in --skip-name-resolve mode.\n", | ||||
|       "2019-08-08 14:46:44 0 [Warning] 'user' entry '@09342f7c540b' ignored in --skip-name-resolve mode.\n", | ||||
|       "2019-08-08 14:46:44 0 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] Reading of all Master_info entries succeded\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] Added new Master_info '' to hash table\n", | ||||
|       "2019-08-08 14:46:44 0 [Note] mysqld: ready for connections.\n", | ||||
|       "Version: '10.3.13-MariaDB-1:10.3.13+maria~bionic'  socket: '/var/run/mysqld/mysqld.sock'  port: 0  mariadb.org binary distribution\n", | ||||
|       "Warning: Unable to load '/usr/share/zoneinfo/leap-seconds.list' as time zone. Skipping it.\n", | ||||
|       "2019-08-08 14:46:46 10 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n", | ||||
|       "2019-08-08 14:46:46 0 [Note] mysqld (initiated by: unknown): Normal shutdown\n", | ||||
|       "2019-08-08 14:46:46 0 [Note] Event Scheduler: Purging the queue. 0 events\n", | ||||
|       "2019-08-08 14:46:46 0 [Note] InnoDB: FTS optimize thread exiting.\n", | ||||
|       "2019-08-08 14:46:46 0 [Note] InnoDB: Starting shutdown...\n", | ||||
|       "2019-08-08 14:46:46 0 [Note] InnoDB: Dumping buffer pool(s) to /var/lib/mysql/ib_buffer_pool\n", | ||||
|       "2019-08-08 14:46:46 0 [Note] InnoDB: Buffer pool(s) dump completed at 190808 14:46:46\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Shutdown completed; log sequence number 1630824; transaction id 24\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Removed temporary tablespace data file: \"ibtmp1\"\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] mysqld: Shutdown complete\n", | ||||
|       "\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] mysqld (mysqld 10.3.13-MariaDB-1:10.3.13+maria~bionic) starting as process 1 ...\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Using Linux native AIO\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Uses event mutexes\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Compressed tables use zlib 1.2.11\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Number of pools: 1\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Using SSE2 crc32 instructions\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Initializing buffer pool, total size = 256M, instances = 1, chunk size = 128M\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Completed initialization of buffer pool\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: 128 out of 128 rollback segments are active.\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Creating shared tablespace for temporary tables\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.\n", | ||||
|       "2019-08-08 14:46:47 0 [Note] InnoDB: Waiting for purge to start\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] InnoDB: 10.3.13 started; log sequence number 1630824; transaction id 21\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] Plugin 'FEEDBACK' is disabled.\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] Server socket created on IP: '::'.\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] InnoDB: Buffer pool(s) load completed at 190808 14:46:48\n", | ||||
|       "2019-08-08 14:46:48 0 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] Reading of all Master_info entries succeded\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] Added new Master_info '' to hash table\n", | ||||
|       "2019-08-08 14:46:48 0 [Note] mysqld: ready for connections.\n", | ||||
|       "Version: '10.3.13-MariaDB-1:10.3.13+maria~bionic'  socket: '/var/run/mysqld/mysqld.sock'  port: 3306  mariadb.org binary distribution\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "docker logs recipe_db" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* get local db address. **NOTE**: dont forget to set this address in settings.py!" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "172.17.0.2\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "docker inspect --format '{{ .NetworkSettings.IPAddress }}' recipe_db" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* and create databse" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "mysql: [Warning] Using a password on the command line interface can be insecure.\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "mysql -h 172.17.0.2 -u root --password=\"g00d_r3c1p3s\" -e \"CREATE DATABASE recipe_db\"" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* check whether database is created:" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Database\n", | ||||
|       "information_schema\n", | ||||
|       "mysql\n", | ||||
|       "performance_schema\n", | ||||
|       "recipe_db\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "mysql: [Warning] Using a password on the command line interface can be insecure.\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "mysql -h 172.17.0.2 -u root --password=\"g00d_r3c1p3s\" -e \"show databases\"" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.7.3" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 4 | ||||
| } | ||||
							
								
								
									
										190
									
								
								db/database_connection.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										190
									
								
								db/database_connection.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,190 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import pymysql.cursors\n", | ||||
|     "import sys\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "def get_sql_time(datetime_object):\n", | ||||
|     "    return datetime_object.strftime('%Y-%m-%d %H:%M:%S')\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "class SQLInjectionError(Exception):\n", | ||||
|     "    def __init__(self):\n", | ||||
|     "\n", | ||||
|     "        # Call the base class constructor with the parameters it needs\n", | ||||
|     "        super().__init__(\"Detected possible SQL injection attack!\")\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "class DatabaseConnection(object):\n", | ||||
|     "    \"\"\"\n", | ||||
|     "    a singleton class for a global database connection\n", | ||||
|     "    \"\"\"\n", | ||||
|     "\n", | ||||
|     "    instance = None\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def global_cursor():\n", | ||||
|     "        assert DatabaseConnection.instance is not None\n", | ||||
|     "        return DatabaseConnection.instance.get_cursor()\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def global_close():\n", | ||||
|     "        assert DatabaseConnection.instance is not None\n", | ||||
|     "        DatabaseConnection.instance.close()\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def global_commit():\n", | ||||
|     "        assert DatabaseConnection.instance is not None\n", | ||||
|     "        DatabaseConnection.instance.commit()\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def global_ping():\n", | ||||
|     "        assert DatabaseConnection.instance is not None\n", | ||||
|     "        DatabaseConnection.instance.connection.ping()\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def global_single_query(query, params=None):\n", | ||||
|     "        DatabaseConnection.global_ping()\n", | ||||
|     "        if ';' in query:\n", | ||||
|     "            # Possible injection!\n", | ||||
|     "            raise SQLInjectionError()\n", | ||||
|     "\n", | ||||
|     "        with DatabaseConnection.global_cursor() as c:\n", | ||||
|     "            if params is None:\n", | ||||
|     "                c.execute(query)\n", | ||||
|     "            else:\n", | ||||
|     "                c.execute(query, params)\n", | ||||
|     "\n", | ||||
|     "            return c.fetchall()\n", | ||||
|     "\n", | ||||
|     "    @staticmethod\n", | ||||
|     "    def global_single_execution(sql_statement, params=None):\n", | ||||
|     "        DatabaseConnection.global_ping()\n", | ||||
|     "        if ';' in sql_statement:\n", | ||||
|     "            # Possible injection detected!\n", | ||||
|     "            raise SQLInjectionError()\n", | ||||
|     "\n", | ||||
|     "        with DatabaseConnection.global_cursor() as c:\n", | ||||
|     "            if params is None:\n", | ||||
|     "                c.execute(sql_statement)\n", | ||||
|     "            else:\n", | ||||
|     "                c.execute(sql_statement, params)\n", | ||||
|     "            DatabaseConnection.global_commit()\n", | ||||
|     "\n", | ||||
|     "    def __init__(self,\n", | ||||
|     "                 host: str,\n", | ||||
|     "                 port: int,\n", | ||||
|     "                 user: str,\n", | ||||
|     "                 password: str,\n", | ||||
|     "                 db: str,\n", | ||||
|     "                 charset: str):\n", | ||||
|     "\n", | ||||
|     "        assert DatabaseConnection.instance is None\n", | ||||
|     "        try:\n", | ||||
|     "            self.connection = pymysql.connect(\n", | ||||
|     "                host=host,\n", | ||||
|     "                port=port,\n", | ||||
|     "                user=user,\n", | ||||
|     "                password=password,\n", | ||||
|     "                db=db,\n", | ||||
|     "                charset=charset,\n", | ||||
|     "                cursorclass=pymysql.cursors.DictCursor)\n", | ||||
|     "            DatabaseConnection.instance = self\n", | ||||
|     "        except Exception as e:\n", | ||||
|     "            sys.stderr.write(\"could not connect to database '\" +\n", | ||||
|     "                             str(db) +\n", | ||||
|     "                             \"' at \" +\n", | ||||
|     "                             user +\n", | ||||
|     "                             \"@\" +\n", | ||||
|     "                             host +\n", | ||||
|     "                             \":\" +\n", | ||||
|     "                             str(port) +\n", | ||||
|     "                             \"\\nCheck the configuration in settings.py!\\n\")\n", | ||||
|     "            raise Exception('could not connect to database')\n", | ||||
|     "\n", | ||||
|     "    def get_cursor(self):\n", | ||||
|     "        return self.connection.cursor()\n", | ||||
|     "\n", | ||||
|     "    def close(self):\n", | ||||
|     "        self.connection.close()\n", | ||||
|     "        DatabaseConnection.instance = None\n", | ||||
|     "\n", | ||||
|     "    def commit(self):\n", | ||||
|     "        self.connection.commit()\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "def test_connection():\n", | ||||
|     "    import db_settings as settings\n", | ||||
|     "    DatabaseConnection(settings.db_host,\n", | ||||
|     "                       settings.db_port,\n", | ||||
|     "                       settings.db_user,\n", | ||||
|     "                       settings.db_pw,\n", | ||||
|     "                       settings.db_db,\n", | ||||
|     "                       settings.db_charset)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "test_connection()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "()" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 3, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "DatabaseConnection.global_single_query(\"show tables\")" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.7.3" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 4 | ||||
| } | ||||
							
								
								
									
										125
									
								
								db/database_connection.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										125
									
								
								db/database_connection.py
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,125 @@ | ||||
| #!/usr/bin/env python3 | ||||
| # coding: utf-8 | ||||
|  | ||||
| import pymysql.cursors | ||||
| import sys | ||||
|  | ||||
|  | ||||
| def get_sql_time(datetime_object): | ||||
|     return datetime_object.strftime('%Y-%m-%d %H:%M:%S') | ||||
|  | ||||
|  | ||||
| class SQLInjectionError(Exception): | ||||
|     def __init__(self): | ||||
|  | ||||
|         # Call the base class constructor with the parameters it needs | ||||
|         super().__init__("Detected possible SQL injection attack!") | ||||
|  | ||||
|  | ||||
| class DatabaseConnection(object): | ||||
|     """ | ||||
|     a singleton class for a global database connection | ||||
|     """ | ||||
|  | ||||
|     instance = None | ||||
|  | ||||
|     @staticmethod | ||||
|     def global_cursor(): | ||||
|         assert DatabaseConnection.instance is not None | ||||
|         return DatabaseConnection.instance.get_cursor() | ||||
|  | ||||
|     @staticmethod | ||||
|     def global_close(): | ||||
|         assert DatabaseConnection.instance is not None | ||||
|         DatabaseConnection.instance.close() | ||||
|  | ||||
|     @staticmethod | ||||
|     def global_commit(): | ||||
|         assert DatabaseConnection.instance is not None | ||||
|         DatabaseConnection.instance.commit() | ||||
|  | ||||
|     @staticmethod | ||||
|     def global_ping(): | ||||
|         assert DatabaseConnection.instance is not None | ||||
|         DatabaseConnection.instance.connection.ping() | ||||
|  | ||||
|     @staticmethod | ||||
|     def global_single_query(query, params=None): | ||||
|         DatabaseConnection.global_ping() | ||||
|         if ';' in query: | ||||
|             # Possible injection! | ||||
|             raise SQLInjectionError() | ||||
|  | ||||
|         with DatabaseConnection.global_cursor() as c: | ||||
|             if params is None: | ||||
|                 c.execute(query) | ||||
|             else: | ||||
|                 c.execute(query, params) | ||||
|  | ||||
|             return c.fetchall() | ||||
|  | ||||
|     @staticmethod | ||||
|     def global_single_execution(sql_statement, params=None): | ||||
|         DatabaseConnection.global_ping() | ||||
|         if ';' in sql_statement: | ||||
|             # Possible injection detected! | ||||
|             raise SQLInjectionError() | ||||
|  | ||||
|         with DatabaseConnection.global_cursor() as c: | ||||
|             if params is None: | ||||
|                 c.execute(sql_statement) | ||||
|             else: | ||||
|                 c.execute(sql_statement, params) | ||||
|             DatabaseConnection.global_commit() | ||||
|  | ||||
|     def __init__(self, | ||||
|                  host: str, | ||||
|                  port: int, | ||||
|                  user: str, | ||||
|                  password: str, | ||||
|                  db: str, | ||||
|                  charset: str): | ||||
|  | ||||
|         assert DatabaseConnection.instance is None | ||||
|         try: | ||||
|             self.connection = pymysql.connect( | ||||
|                 host=host, | ||||
|                 port=port, | ||||
|                 user=user, | ||||
|                 password=password, | ||||
|                 db=db, | ||||
|                 charset=charset, | ||||
|                 cursorclass=pymysql.cursors.DictCursor) | ||||
|             DatabaseConnection.instance = self | ||||
|         except Exception as e: | ||||
|             sys.stderr.write("could not connect to database '" + | ||||
|                              str(db) + | ||||
|                              "' at " + | ||||
|                              user + | ||||
|                              "@" + | ||||
|                              host + | ||||
|                              ":" + | ||||
|                              str(port) + | ||||
|                              "\nCheck the configuration in settings.py!\n") | ||||
|             raise Exception('could not connect to database') | ||||
|  | ||||
|     def get_cursor(self): | ||||
|         return self.connection.cursor() | ||||
|  | ||||
|     def close(self): | ||||
|         self.connection.close() | ||||
|         DatabaseConnection.instance = None | ||||
|  | ||||
|     def commit(self): | ||||
|         self.connection.commit() | ||||
|  | ||||
|  | ||||
| def test_connection(): | ||||
|     import settings | ||||
|     DatabaseConnection(settings.db_host, | ||||
|                        settings.db_port, | ||||
|                        settings.db_user, | ||||
|                        settings.db_pw, | ||||
|                        settings.db_db, | ||||
|                        settings.db_charset) | ||||
|  | ||||
							
								
								
									
										48
									
								
								db/db_settings.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										48
									
								
								db/db_settings.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,48 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "db_host = \"172.17.0.2\"\n", | ||||
|     "db_port = 3306\n", | ||||
|     "\n", | ||||
|     "db_user = \"root\"\n", | ||||
|     "db_pw = \"g00d_r3c1p3s\"\n", | ||||
|     "db_db = \"recipe_db\"\n", | ||||
|     "\n", | ||||
|     "db_charset = 'utf8mb4'" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.7.3" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 4 | ||||
| } | ||||
							
								
								
									
										13
									
								
								db/db_settings.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										13
									
								
								db/db_settings.py
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,13 @@ | ||||
| #!/usr/bin/env python3 | ||||
| # coding: utf-8 | ||||
|  | ||||
| db_host = "172.17.0.2" | ||||
| db_port = 3306 | ||||
|  | ||||
| db_user = "root" | ||||
| db_pw = "g00d_r3c1p3s" | ||||
| db_db = "recipe_db" | ||||
|  | ||||
| db_charset = 'utf8mb4' | ||||
|  | ||||
|  | ||||
		Reference in New Issue
	
	Block a user