first evolutionary algorithm

This commit is contained in:
Jonas Weinz 2019-09-05 12:03:01 +02:00
parent 09eb58e703
commit 5c25e04143
12 changed files with 59303 additions and 97 deletions

1
.gitignore vendored
View File

@ -4,3 +4,4 @@ __pycache__
*.conllu
*.gz
.vscode
*.pickle

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,136 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Adjacency Matrix"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"from scipy.sparse import csr_matrix, lil_matrix, coo_matrix"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"class adj_matrix(object):\n",
" def __init__(self, symmetric_indices=False):\n",
" \n",
" self._sym = symmetric_indices\n",
" if not symmetric_indices:\n",
" self._x_labels = []\n",
" self._y_labels = []\n",
"\n",
" self._x_label_index={}\n",
" self._y_label_index={}\n",
" \n",
" else:\n",
" self._labels = []\n",
" self._label_index={}\n",
" \n",
" self._x = []\n",
" self._y = []\n",
" self._data = []\n",
" \n",
" self._mat = None\n",
" \n",
" def _get_ix(self, label):\n",
" i = self._x_label_index.get(label)\n",
" if i is None:\n",
" i = len(self._x_labels)\n",
" self._x_labels.append(label)\n",
" self._x_label_index[label] = i\n",
" return i\n",
" \n",
" def _get_iy(self, label):\n",
" i = self._y_label_index.get(label)\n",
" if i is None:\n",
" i = len(self._y_labels)\n",
" self._y_labels.append(label)\n",
" self._y_label_index[label] = i\n",
" return i\n",
" \n",
" def _get_i(self, label):\n",
" i = self._label_index.get(label)\n",
" if i is None:\n",
" i = len(self._labels)\n",
" self._labels.append(label)\n",
" self._label_index[label] = i\n",
" return i\n",
" \n",
" def add_entry(self, x, y, data):\n",
" \n",
" if self._sym:\n",
" ix = self._get_i(x)\n",
" iy = self._get_i(y)\n",
" \n",
" else:\n",
" ix = self._get_ix(x)\n",
" iy = self._get_iy(y)\n",
" \n",
" self._x.append(ix)\n",
" self._y.append(iy)\n",
" self._data.append(data)\n",
" \n",
" \n",
" def compile_to_mat(self):\n",
" if self._sym:\n",
" sx = len(self._labels)\n",
" sy = len(self._labels)\n",
" else:\n",
" sx = len(self._x_labels)\n",
" sy = len(self._y_labels)\n",
" \n",
" self._mat = coo_matrix((self._data, (self._x, self._y)), shape=(sx,sy))\n",
" return self._mat\n",
" \n",
" def get_csr(self):\n",
" return self.compile_to_mat().tocsr()\n",
" \n",
" def get_labels(self):\n",
" if self._sym:\n",
" return self._labels\n",
" return self._x_labels, self._y_labels"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -0,0 +1,91 @@
#!/usr/bin/env python3
# coding: utf-8
# # Adjacency Matrix
import numpy as np
from scipy.sparse import csr_matrix, lil_matrix, coo_matrix
class adj_matrix(object):
def __init__(self, symmetric_indices=False):
self._sym = symmetric_indices
if not symmetric_indices:
self._x_labels = []
self._y_labels = []
self._x_label_index={}
self._y_label_index={}
else:
self._labels = []
self._label_index={}
self._x = []
self._y = []
self._data = []
self._mat = None
def _get_ix(self, label):
i = self._x_label_index.get(label)
if i is None:
i = len(self._x_labels)
self._x_labels.append(label)
self._x_label_index[label] = i
return i
def _get_iy(self, label):
i = self._y_label_index.get(label)
if i is None:
i = len(self._y_labels)
self._y_labels.append(label)
self._y_label_index[label] = i
return i
def _get_i(self, label):
i = self._label_index.get(label)
if i is None:
i = len(self._labels)
self._labels.append(label)
self._label_index[label] = i
return i
def add_entry(self, x, y, data):
if self._sym:
ix = self._get_i(x)
iy = self._get_i(y)
else:
ix = self._get_ix(x)
iy = self._get_iy(y)
self._x.append(ix)
self._y.append(iy)
self._data.append(data)
def compile_to_mat(self):
if self._sym:
sx = len(self._labels)
sy = len(self._labels)
else:
sx = len(self._x_labels)
sy = len(self._y_labels)
self._mat = coo_matrix((self._data, (self._x, self._y)), shape=(sx,sy))
return self._mat
def get_csr(self):
return self.compile_to_mat().tocsr()
def get_labels(self):
if self._sym:
return self._labels
return self._x_labels, self._y_labels

View File

@ -0,0 +1,72 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Evaluate Adjacency Matrices"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pickle"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"m_act = pickle.load(open(\"m_act.pickle\", \"rb\"))\n",
"m_mix = pickle.load(open(\"m_mix.pickle\", \"rb\"))\n",
"m_base_act = pickle.load(open(\"m_base_act.pickle\", \"rb\"))\n",
"m_base_mix = pickle.load(open(\"m_base_mix.pickle\", \"rb\"))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"c_act = m_act.get_csr()\n",
"c_mix = m_mix.get_csr()\n",
"c_base_act = m_base_act.get_csr()\n",
"c_base_mix = m_base_mix.get_csr()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,383 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Matrix Generation"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
" </script>\n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import sys\n",
"sys.path.append(\"../\")\n",
"from Recipe import Recipe, Ingredient, RecipeGraph\n",
"\n",
"import settings\n",
"import db.db_settings as db_settings\n",
"from db.database_connection import DatabaseConnection\n",
"\n",
"import random\n",
"\n",
"import itertools\n",
"\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<db.database_connection.DatabaseConnection at 0x7fc3a1bedac8>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"DatabaseConnection(db_settings.db_host,\n",
" db_settings.db_port,\n",
" db_settings.db_user,\n",
" db_settings.db_pw,\n",
" db_settings.db_db,\n",
" db_settings.db_charset)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 8.71 s, sys: 942 ms, total: 9.66 s\n",
"Wall time: 9.77 s\n"
]
}
],
"source": [
"%time ids = DatabaseConnection.global_single_query(\"select id from recipes\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import AdjacencyMatrix"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* create Adjacency Matrix"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def add_entries_from_rec_state(rec_state, m_act, m_mix, m_base_act, m_base_mix):\n",
" mix_m, mix_label = rec_state.get_mixing_matrix()\n",
" act_m, act_a, act_i = rec_state.get_action_matrix()\n",
"\n",
" # create list of tuples: [action, ingredient]\n",
" seen_actions = np.array(list(itertools.product(act_a,act_i))).reshape((len(act_a), len(act_i), 2))\n",
"\n",
" # create list of tuples [ingredient, ingredient]\n",
" seen_mixes = np.array(list(itertools.product(mix_label,mix_label))).reshape((len(mix_label), len(mix_label), 2))\n",
"\n",
" seen_actions = seen_actions[act_m == 1]\n",
" seen_mixes = seen_mixes[mix_m == 1]\n",
"\n",
" seen_actions = set([tuple(x) for x in seen_actions.tolist()])\n",
" seen_mixes = set([tuple(x) for x in seen_mixes.tolist()])\n",
" \n",
" seen_base_actions = set()\n",
" seen_base_mixes = set()\n",
" \n",
" for act, ing in seen_actions:\n",
" m_act.add_entry(act, ing.to_json(), 1)\n",
" if (act, ing._base_ingredient) not in seen_base_actions:\n",
" seen_base_actions.add((act, ing._base_ingredient))\n",
" m_base_act.add_entry(act, ing._base_ingredient, 1)\n",
" \n",
" for x,y in seen_mixes:\n",
" xj = x.to_json()\n",
" yj = y.to_json()\n",
" if xj < yj:\n",
" m_mix.add_entry(xj,yj,1)\n",
" if (x._base_ingredient, y._base_ingredient) not in seen_base_mixes:\n",
" seen_base_mixes.add((x._base_ingredient, y._base_ingredient))\n",
" m_base_mix.add_entry(x._base_ingredient, y._base_ingredient, 1)\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"m_act = AdjacencyMatrix.adj_matrix()\n",
"m_mix = AdjacencyMatrix.adj_matrix(True)\n",
"m_base_act = AdjacencyMatrix.adj_matrix()\n",
"m_base_mix = AdjacencyMatrix.adj_matrix(True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"warning: recipe a9dc137b48 has no ingredient! skipping it\n",
"CPU times: user 13min 35s, sys: 3.52 s, total: 13min 39s\n",
"Wall time: 13min 50s\n"
]
}
],
"source": [
"%%time\n",
"for i in range(10000):\n",
" id = random.choice(ids)['id']\n",
" rec = Recipe(id)\n",
" #rec.display_recipe()\n",
" ing = rec.extract_ingredients()\n",
" if len(ing) == 0:\n",
" print(f\"warning: recipe {id} has no ingredient! skipping it\")\n",
" continue\n",
" rec.apply_instructions(debug=False)\n",
" add_entries_from_rec_state(rec._recipe_state, m_act, m_mix, m_base_act, m_base_mix)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"import pickle"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"pickle.dump(m_act, file=open(\"m_act.pickle\", 'wb'))\n",
"pickle.dump(m_mix, file=open(\"m_mix.pickle\", 'wb'))\n",
"pickle.dump(m_base_act, file=open(\"m_base_act.pickle\", 'wb'))\n",
"pickle.dump(m_base_mix, file=open(\"m_base_mix.pickle\", 'wb'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"c_mix = m_mix.get_csr()\n",
"c_act = m_act.get_csr()\n",
"c_base_mix = m_base_mix.get_csr()\n",
"c_base_act = m_base_act.get_csr()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(65, 64699) (71548, 71548)\n",
"113994 537369\n",
"(65, 4738) (5850, 5850)\n",
"30820 122390\n"
]
}
],
"source": [
"print(c_act.shape, c_mix.shape)\n",
"print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))\n",
"print(c_base_act.shape, c_base_mix.shape)\n",
"print(len(c_base_act.nonzero()[0]),len(c_base_mix.nonzero()[0]))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(64, 63787) (70933, 70933)\n",
"112841 524285\n"
]
}
],
"source": [
"print(c_act.shape, c_mix.shape)\n",
"print(len(c_act.nonzero()[0]),len(c_mix.nonzero()[0]))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"17560"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sum(c_act.toarray() > 1)"
]
},
{
"cell_type": "code",
"execution_count": 99,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[1, 1, 0, ..., 0, 0, 0],\n",
" [0, 0, 1, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
]
},
"execution_count": 99,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* values after 100:\n",
"```\n",
"(53, 1498) (1620, 1620)\n",
"1982 6489\n",
"```\n",
"\n",
"* after 1000:\n",
"```\n",
"(60, 9855) (10946, 10946)\n",
"15446 59943\n",
"```\n",
"\n",
"* after 10000:\n",
"```\n",
"(65, 65235) (72448, 72448)\n",
"114808 546217\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -0,0 +1,113 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Playground"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from graphviz import Digraph"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"dot = Digraph(comment=\"testgraph\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"dot.node(\"B\", \"test2\", shape=\"diamond\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
" -->\n",
"<!-- Title: %3 Pages: 1 -->\n",
"<svg width=\"158pt\" height=\"44pt\"\n",
" viewBox=\"0.00 0.00 157.60 44.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 40)\">\n",
"<title>%3</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-40 153.6046,-40 153.6046,4 -4,4\"/>\n",
"<!-- A -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>A</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"27\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"27\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">test</text>\n",
"</g>\n",
"<!-- B -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>B</title>\n",
"<polygon fill=\"none\" stroke=\"#000000\" points=\"111,-36 72.2905,-18 111,0 149.7095,-18 111,-36\"/>\n",
"<text text-anchor=\"middle\" x=\"111\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">test2</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.dot.Digraph at 0x7f1d4e10e978>"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dot"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long