switched from pickle to dill to for creating adjaency matrices

This commit is contained in:
Jonas Weinz 2019-11-02 10:57:11 +01:00
parent e546bad48f
commit b7b4a0a83d
4 changed files with 245 additions and 548 deletions

File diff suppressed because one or more lines are too long

View File

@ -28,6 +28,18 @@ class adj_matrix(object):
self._data = []
self._mat = None
self._csr = None
# for a TF-IDF like approach we need also a counter how frequently ingredients
# and actions appear in documents.
self._current_document_labels = set()
self._label_document_count = {}
self._document_count = 0
# building type dependend functions:
self._build_funcs()
def _get_ix(self, label):
i = self._x_label_index.get(label)
@ -52,6 +64,39 @@ class adj_matrix(object):
self._labels.append(label)
self._label_index[label] = i
return i
def _end_document(self):
self._document_count += 1
# adding all seen labels to our counter:
for label in self._current_document_labels:
self._label_document_count[label] += 1
else:
self._label_document_count[label] = 1
self._current_document_labels = set()
def apply_threshold(self, min_count=5):
csr = self.get_csr()
new_x = []
new_y = []
new_data = []
for i in range(len(self._data)):
if csr[self._x[i],self._y[i]] >= min_count:
new_x.append(self._x[i])
new_y.append(self._y[i])
new_data.append(self._data[i])
self._x = new_x
self._y = new_y
self._data = new_data
def next_document(self):
self._end_document()
def add_entry(self, x, y, data):
@ -66,6 +111,18 @@ class adj_matrix(object):
self._x.append(ix)
self._y.append(iy)
self._data.append(data)
self._current_document_labels.add(x)
self._current_document_labels.add(y)
def compile(self):
self._csr = self.get_csr()
if self._sym:
self._np_labels = np.array(self._labels)
else:
self._np_x_labels = np.array(self._x_labels)
self._np_y_labels = np.array(self._y_labels)
def compile_to_mat(self):
if self._sym:
@ -85,6 +142,163 @@ class adj_matrix(object):
if self._sym:
return self._labels
return self._x_labels, self._y_labels
def _build_funcs(self):
def get_sym_adjacent(key):
assert self._csr is not None
c = self._csr
index = self._label_index[key]
i1 = c[index,:].nonzero()[1]
i2 = c[:,index].nonzero()[0]
i = np.concatenate((i1,i2))
names = self._np_labels[i]
counts = np.concatenate((c[index, i1].toarray().flatten(), c[i2, index].toarray().flatten()))
s = np.argsort(-counts)
return names[s], counts[s]
def get_forward_adjacent(key):
assert self._csr is not None
c = self._csr
index = self._x_label_index[key]
i = c[index,:].nonzero()[1]
names = self._np_y_labels[i]
counts = c[index, i].toarray().flatten()
s = np.argsort(-counts)
return names[s], counts[s]
def get_backward_adjacent(key):
assert self._csr is not None
c = self._csr
index = self._y_label_index[key]
i = c[:,index].nonzero()[0]
names = self._np_x_labels[i]
counts = c[i, index].toarray().flatten()
s = np.argsort(-counts)
return names[s], counts[s]
# sum functions:
def sym_sum(key):
return np.sum(self.get_adjacent(key)[1])
def fw_sum(key):
return np.sum(self.get_forward_adjacent(key)[1])
def bw_sum(key):
return np.sum(self.get_backward_adjacent(key)[1])
# normalization stuff:
def fw_normalization_factor(key, quotient_func):
assert self._csr is not None
c = self._csr
ia = self._x_label_index[key]
occurances = c[ia,:].nonzero()[1]
return 1. / quotient_func(c[ia,occurances].toarray())
def bw_normalization_factor(key, quotient_func):
assert self._csr is not None
c = self._csr
ib = m._y_label_index[key]
occurances = c[:,ib].nonzero()[0]
return 1. / quotient_func(c[occurances,ib].toarray())
def sym_normalization_factor(key, quotient_func):
assert self._csr is not None
c = self._csr
ii = m._label_index[key]
fw_occurances = c[ii,:].nonzero()[1]
bw_occurances = c[:,ii].nonzero()[0]
return 1. / quotient_func(np.concatenate(
[c[ii,fw_occurances].toarray().flatten(),
c[bw_occurances,ii].toarray().flatten()]
))
def sym_p_a_given_b(key_a, key_b, quot_func = np.max):
assert self._csr is not None
c = self._csr
ia = m._label_index[key_a]
ib = m._label_index[key_b]
v = c[ia,ib] + c[ib,ia]
return v * self.sym_normalization_factor(key_b, quot_func)
def fw_p_a_given_b(key_a, key_b, quot_func = np.max):
assert self._csr is not None
c = self._csr
ia = m._x_label_index[key_a]
ib = m._y_label_index[key_b]
v = c[ia,ib]
return v * self.bw_normalization_factor(key_b, quot_func)
def bw_p_a_given_b(key_a, key_b, quot_func = np.max):
assert self._csr is not None
c = self._csr
ia = m._y_label_index[key_a]
ib = m._x_label_index[key_b]
v = c[ib,ia]
return v * self.fw_normalization_factor(key_b, quot_func)
if self._sym:
self.get_adjacent = get_sym_adjacent
self.get_sum = sym_sum
self.get_sym_normalization_factor = sym_normalization_factor
self.p_a_given_b = sym_p_a_given_b
else:
self.get_forward_adjacent = get_forward_adjacent
self.get_backward_adjacent = get_backward_adjacent
self.get_fw_sum = fw_sum
self.get_bw_sum = bw_sum
self.get_fw_normalization_factor = fw_normalization_factor
self.get_bw_normalization_factor = bw_normalization_factor
self.fw_p_a_given_b = fw_p_a_given_b
self.bw_p_a_given_b = bw_p_a_given_b

File diff suppressed because one or more lines are too long

View File

@ -9,7 +9,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 1,
"metadata": {},
"outputs": [
{
@ -45,7 +45,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@ -56,7 +56,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@ -65,16 +65,16 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<db.database_connection.DatabaseConnection at 0x7f5f9d0b5160>"
"<db.database_connection.DatabaseConnection at 0x7fa5f2e0d9e8>"
]
},
"execution_count": 6,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@ -97,15 +97,15 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 8.22 s, sys: 1.25 s, total: 9.46 s\n",
"Wall time: 9.56 s\n"
"CPU times: user 9 s, sys: 850 ms, total: 9.85 s\n",
"Wall time: 9.92 s\n"
]
}
],
@ -122,7 +122,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@ -131,7 +131,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 7,
"metadata": {},
"outputs": [
{
@ -217,8 +217,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 9.53 ms, sys: 0 ns, total: 9.53 ms\n",
"Wall time: 8.74 ms\n"
"CPU times: user 10.8 ms, sys: 274 µs, total: 11 ms\n",
"Wall time: 9.98 ms\n"
]
}
],
@ -3142,8 +3142,8 @@
1
],
"range": [
-0.7394662921348321,
30.739466292134832
-0.7173913043478262,
30.717391304347828
],
"type": "category"
},
@ -4197,8 +4197,8 @@
1
],
"range": [
-11.044917257683217,
15.044917257683217
-11.06323877068558,
15.06323877068558
],
"scaleanchor": "x",
"scaleratio": 1,
@ -5336,8 +5336,8 @@
1
],
"range": [
-0.9618863049095596,
12.961886304909559
-0.9702842377260978,
12.970284237726098
],
"scaleanchor": "x",
"scaleratio": 1,
@ -9113,8 +9113,8 @@
1
],
"range": [
-0.8529850746268686,
42.85298507462687
-0.88507462686567,
42.88507462686567
],
"type": "category"
},
@ -13399,8 +13399,8 @@
1
],
"range": [
-0.9393241167434745,
43.93932411674348
-0.9049079754601266,
43.90490797546013
],
"type": "category"
},
@ -14673,8 +14673,8 @@
1
],
"range": [
-13.429078014184398,
20.429078014184398
-13.455082742316787,
20.455082742316787
],
"scaleanchor": "x",
"scaleratio": 1,
@ -16670,8 +16670,8 @@
1
],
"range": [
-0.7020860495436789,
30.70208604954368
-0.681640625,
30.681640625
],
"type": "category"
},
@ -17759,8 +17759,8 @@
1
],
"range": [
-11.55260047281324,
16.55260047281324
-11.570921985815605,
16.570921985815605
],
"scaleanchor": "x",
"scaleratio": 1,