moved 1 million recipes into database, starting coarse tree generation for recipe analysis

This commit is contained in:
Jonas Weinz
2019-08-10 23:10:30 +02:00
parent cbccf169bb
commit 09eb58e703
24 changed files with 7571 additions and 3663 deletions

1298
db/create_database.ipynb Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,304 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Create Database with Docker"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* download mariadb image"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using default tag: latest\n",
"latest: Pulling from mariadb/server\n",
"Digest: sha256:b5762c478d38ae54c464e3ab63e10e0c3f951633ed7619d52fa3c22bcf36218a\n",
"Status: Image is up to date for mariadb/server:latest\n"
]
}
],
"source": [
"%%bash \n",
"docker pull mariadb/server"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create and run container"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"09342f7c540b2b6224bb96b2e3a542a5ff144e6bbc1cf243ae33b0dab9262c47\n"
]
}
],
"source": [
"%%bash\n",
"docker run --name recipe_db -e MYSQL_ROOT_PASSWORD=\"g00d_r3c1p3s\" -d mariadb/server"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* checking docker logs"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Initializing database\n",
"\n",
"\n",
"PLEASE REMEMBER TO SET A PASSWORD FOR THE MariaDB root USER !\n",
"To do so, start the server, then issue the following commands:\n",
"\n",
"'/usr/bin/mysqladmin' -u root password 'new-password'\n",
"'/usr/bin/mysqladmin' -u root -h password 'new-password'\n",
"\n",
"Alternatively you can run:\n",
"'/usr/bin/mysql_secure_installation'\n",
"\n",
"which will also give you the option of removing the test\n",
"databases and anonymous user created by default. This is\n",
"strongly recommended for production servers.\n",
"\n",
"See the MariaDB Knowledgebase at http://mariadb.com/kb or the\n",
"MySQL manual for more instructions.\n",
"\n",
"Please report any problems at http://mariadb.org/jira\n",
"\n",
"The latest information about MariaDB is available at http://mariadb.org/.\n",
"You can find additional information about the MySQL part at:\n",
"http://dev.mysql.com\n",
"Consider joining MariaDB's strong and vibrant community:\n",
"https://mariadb.org/get-involved/\n",
"\n",
"Database initialized\n",
"MySQL init process in progress...\n",
"\n",
"\n",
"MySQL init process done. Ready for start up.\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2019-08-08 14:46:43 0 [Note] mysqld (mysqld 10.3.13-MariaDB-1:10.3.13+maria~bionic) starting as process 104 ...\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Using Linux native AIO\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Uses event mutexes\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Compressed tables use zlib 1.2.11\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Number of pools: 1\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Using SSE2 crc32 instructions\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Initializing buffer pool, total size = 256M, instances = 1, chunk size = 128M\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Completed initialization of buffer pool\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: 128 out of 128 rollback segments are active.\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Creating shared tablespace for temporary tables\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.\n",
"2019-08-08 14:46:43 0 [Note] InnoDB: Waiting for purge to start\n",
"2019-08-08 14:46:44 0 [Note] InnoDB: 10.3.13 started; log sequence number 1630815; transaction id 21\n",
"2019-08-08 14:46:44 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool\n",
"2019-08-08 14:46:44 0 [Note] Plugin 'FEEDBACK' is disabled.\n",
"2019-08-08 14:46:44 0 [Note] InnoDB: Buffer pool(s) load completed at 190808 14:46:44\n",
"2019-08-08 14:46:44 0 [Warning] 'user' entry 'root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
"2019-08-08 14:46:44 0 [Warning] 'user' entry '@09342f7c540b' ignored in --skip-name-resolve mode.\n",
"2019-08-08 14:46:44 0 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
"2019-08-08 14:46:44 0 [Note] Reading of all Master_info entries succeded\n",
"2019-08-08 14:46:44 0 [Note] Added new Master_info '' to hash table\n",
"2019-08-08 14:46:44 0 [Note] mysqld: ready for connections.\n",
"Version: '10.3.13-MariaDB-1:10.3.13+maria~bionic' socket: '/var/run/mysqld/mysqld.sock' port: 0 mariadb.org binary distribution\n",
"Warning: Unable to load '/usr/share/zoneinfo/leap-seconds.list' as time zone. Skipping it.\n",
"2019-08-08 14:46:46 10 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
"2019-08-08 14:46:46 0 [Note] mysqld (initiated by: unknown): Normal shutdown\n",
"2019-08-08 14:46:46 0 [Note] Event Scheduler: Purging the queue. 0 events\n",
"2019-08-08 14:46:46 0 [Note] InnoDB: FTS optimize thread exiting.\n",
"2019-08-08 14:46:46 0 [Note] InnoDB: Starting shutdown...\n",
"2019-08-08 14:46:46 0 [Note] InnoDB: Dumping buffer pool(s) to /var/lib/mysql/ib_buffer_pool\n",
"2019-08-08 14:46:46 0 [Note] InnoDB: Buffer pool(s) dump completed at 190808 14:46:46\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Shutdown completed; log sequence number 1630824; transaction id 24\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Removed temporary tablespace data file: \"ibtmp1\"\n",
"2019-08-08 14:46:47 0 [Note] mysqld: Shutdown complete\n",
"\n",
"2019-08-08 14:46:47 0 [Note] mysqld (mysqld 10.3.13-MariaDB-1:10.3.13+maria~bionic) starting as process 1 ...\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Using Linux native AIO\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Uses event mutexes\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Compressed tables use zlib 1.2.11\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Number of pools: 1\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Using SSE2 crc32 instructions\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Initializing buffer pool, total size = 256M, instances = 1, chunk size = 128M\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Completed initialization of buffer pool\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: 128 out of 128 rollback segments are active.\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Creating shared tablespace for temporary tables\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.\n",
"2019-08-08 14:46:47 0 [Note] InnoDB: Waiting for purge to start\n",
"2019-08-08 14:46:48 0 [Note] InnoDB: 10.3.13 started; log sequence number 1630824; transaction id 21\n",
"2019-08-08 14:46:48 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool\n",
"2019-08-08 14:46:48 0 [Note] Plugin 'FEEDBACK' is disabled.\n",
"2019-08-08 14:46:48 0 [Note] Server socket created on IP: '::'.\n",
"2019-08-08 14:46:48 0 [Note] InnoDB: Buffer pool(s) load completed at 190808 14:46:48\n",
"2019-08-08 14:46:48 0 [Warning] 'proxies_priv' entry '@% root@09342f7c540b' ignored in --skip-name-resolve mode.\n",
"2019-08-08 14:46:48 0 [Note] Reading of all Master_info entries succeded\n",
"2019-08-08 14:46:48 0 [Note] Added new Master_info '' to hash table\n",
"2019-08-08 14:46:48 0 [Note] mysqld: ready for connections.\n",
"Version: '10.3.13-MariaDB-1:10.3.13+maria~bionic' socket: '/var/run/mysqld/mysqld.sock' port: 3306 mariadb.org binary distribution\n"
]
}
],
"source": [
"%%bash\n",
"docker logs recipe_db"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get local db address. **NOTE**: dont forget to set this address in settings.py!"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"172.17.0.2\n"
]
}
],
"source": [
"%%bash\n",
"docker inspect --format '{{ .NetworkSettings.IPAddress }}' recipe_db"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* and create databse"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"mysql: [Warning] Using a password on the command line interface can be insecure.\n"
]
}
],
"source": [
"%%bash\n",
"mysql -h 172.17.0.2 -u root --password=\"g00d_r3c1p3s\" -e \"CREATE DATABASE recipe_db\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* check whether database is created:"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Database\n",
"information_schema\n",
"mysql\n",
"performance_schema\n",
"recipe_db\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"mysql: [Warning] Using a password on the command line interface can be insecure.\n"
]
}
],
"source": [
"%%bash\n",
"mysql -h 172.17.0.2 -u root --password=\"g00d_r3c1p3s\" -e \"show databases\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -0,0 +1,190 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pymysql.cursors\n",
"import sys\n",
"\n",
"\n",
"def get_sql_time(datetime_object):\n",
" return datetime_object.strftime('%Y-%m-%d %H:%M:%S')\n",
"\n",
"\n",
"class SQLInjectionError(Exception):\n",
" def __init__(self):\n",
"\n",
" # Call the base class constructor with the parameters it needs\n",
" super().__init__(\"Detected possible SQL injection attack!\")\n",
"\n",
"\n",
"class DatabaseConnection(object):\n",
" \"\"\"\n",
" a singleton class for a global database connection\n",
" \"\"\"\n",
"\n",
" instance = None\n",
"\n",
" @staticmethod\n",
" def global_cursor():\n",
" assert DatabaseConnection.instance is not None\n",
" return DatabaseConnection.instance.get_cursor()\n",
"\n",
" @staticmethod\n",
" def global_close():\n",
" assert DatabaseConnection.instance is not None\n",
" DatabaseConnection.instance.close()\n",
"\n",
" @staticmethod\n",
" def global_commit():\n",
" assert DatabaseConnection.instance is not None\n",
" DatabaseConnection.instance.commit()\n",
"\n",
" @staticmethod\n",
" def global_ping():\n",
" assert DatabaseConnection.instance is not None\n",
" DatabaseConnection.instance.connection.ping()\n",
"\n",
" @staticmethod\n",
" def global_single_query(query, params=None):\n",
" DatabaseConnection.global_ping()\n",
" if ';' in query:\n",
" # Possible injection!\n",
" raise SQLInjectionError()\n",
"\n",
" with DatabaseConnection.global_cursor() as c:\n",
" if params is None:\n",
" c.execute(query)\n",
" else:\n",
" c.execute(query, params)\n",
"\n",
" return c.fetchall()\n",
"\n",
" @staticmethod\n",
" def global_single_execution(sql_statement, params=None):\n",
" DatabaseConnection.global_ping()\n",
" if ';' in sql_statement:\n",
" # Possible injection detected!\n",
" raise SQLInjectionError()\n",
"\n",
" with DatabaseConnection.global_cursor() as c:\n",
" if params is None:\n",
" c.execute(sql_statement)\n",
" else:\n",
" c.execute(sql_statement, params)\n",
" DatabaseConnection.global_commit()\n",
"\n",
" def __init__(self,\n",
" host: str,\n",
" port: int,\n",
" user: str,\n",
" password: str,\n",
" db: str,\n",
" charset: str):\n",
"\n",
" assert DatabaseConnection.instance is None\n",
" try:\n",
" self.connection = pymysql.connect(\n",
" host=host,\n",
" port=port,\n",
" user=user,\n",
" password=password,\n",
" db=db,\n",
" charset=charset,\n",
" cursorclass=pymysql.cursors.DictCursor)\n",
" DatabaseConnection.instance = self\n",
" except Exception as e:\n",
" sys.stderr.write(\"could not connect to database '\" +\n",
" str(db) +\n",
" \"' at \" +\n",
" user +\n",
" \"@\" +\n",
" host +\n",
" \":\" +\n",
" str(port) +\n",
" \"\\nCheck the configuration in settings.py!\\n\")\n",
" raise Exception('could not connect to database')\n",
"\n",
" def get_cursor(self):\n",
" return self.connection.cursor()\n",
"\n",
" def close(self):\n",
" self.connection.close()\n",
" DatabaseConnection.instance = None\n",
"\n",
" def commit(self):\n",
" self.connection.commit()\n",
"\n",
"\n",
"def test_connection():\n",
" import db_settings as settings\n",
" DatabaseConnection(settings.db_host,\n",
" settings.db_port,\n",
" settings.db_user,\n",
" settings.db_pw,\n",
" settings.db_db,\n",
" settings.db_charset)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"test_connection()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"()"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"DatabaseConnection.global_single_query(\"show tables\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

125
db/database_connection.py Normal file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env python3
# coding: utf-8
import pymysql.cursors
import sys
def get_sql_time(datetime_object):
return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
class SQLInjectionError(Exception):
def __init__(self):
# Call the base class constructor with the parameters it needs
super().__init__("Detected possible SQL injection attack!")
class DatabaseConnection(object):
"""
a singleton class for a global database connection
"""
instance = None
@staticmethod
def global_cursor():
assert DatabaseConnection.instance is not None
return DatabaseConnection.instance.get_cursor()
@staticmethod
def global_close():
assert DatabaseConnection.instance is not None
DatabaseConnection.instance.close()
@staticmethod
def global_commit():
assert DatabaseConnection.instance is not None
DatabaseConnection.instance.commit()
@staticmethod
def global_ping():
assert DatabaseConnection.instance is not None
DatabaseConnection.instance.connection.ping()
@staticmethod
def global_single_query(query, params=None):
DatabaseConnection.global_ping()
if ';' in query:
# Possible injection!
raise SQLInjectionError()
with DatabaseConnection.global_cursor() as c:
if params is None:
c.execute(query)
else:
c.execute(query, params)
return c.fetchall()
@staticmethod
def global_single_execution(sql_statement, params=None):
DatabaseConnection.global_ping()
if ';' in sql_statement:
# Possible injection detected!
raise SQLInjectionError()
with DatabaseConnection.global_cursor() as c:
if params is None:
c.execute(sql_statement)
else:
c.execute(sql_statement, params)
DatabaseConnection.global_commit()
def __init__(self,
host: str,
port: int,
user: str,
password: str,
db: str,
charset: str):
assert DatabaseConnection.instance is None
try:
self.connection = pymysql.connect(
host=host,
port=port,
user=user,
password=password,
db=db,
charset=charset,
cursorclass=pymysql.cursors.DictCursor)
DatabaseConnection.instance = self
except Exception as e:
sys.stderr.write("could not connect to database '" +
str(db) +
"' at " +
user +
"@" +
host +
":" +
str(port) +
"\nCheck the configuration in settings.py!\n")
raise Exception('could not connect to database')
def get_cursor(self):
return self.connection.cursor()
def close(self):
self.connection.close()
DatabaseConnection.instance = None
def commit(self):
self.connection.commit()
def test_connection():
import settings
DatabaseConnection(settings.db_host,
settings.db_port,
settings.db_user,
settings.db_pw,
settings.db_db,
settings.db_charset)

48
db/db_settings.ipynb Normal file
View File

@ -0,0 +1,48 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"db_host = \"172.17.0.2\"\n",
"db_port = 3306\n",
"\n",
"db_user = \"root\"\n",
"db_pw = \"g00d_r3c1p3s\"\n",
"db_db = \"recipe_db\"\n",
"\n",
"db_charset = 'utf8mb4'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

13
db/db_settings.py Normal file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env python3
# coding: utf-8
db_host = "172.17.0.2"
db_port = 3306
db_user = "root"
db_pw = "g00d_r3c1p3s"
db_db = "recipe_db"
db_charset = 'utf8mb4'