switched from pickle to dill to for creating adjaency matrices
This commit is contained in:
Jonas Weinz
parent
e546bad48f
commit
b7b4a0a83d
File diff suppressed because one or more lines are too long
@ -28,6 +28,18 @@ class adj_matrix(object):
|
||||
self._data = []
|
||||
|
||||
self._mat = None
|
||||
self._csr = None
|
||||
|
||||
# for a TF-IDF like approach we need also a counter how frequently ingredients
|
||||
# and actions appear in documents.
|
||||
|
||||
self._current_document_labels = set()
|
||||
self._label_document_count = {}
|
||||
|
||||
self._document_count = 0
|
||||
|
||||
# building type dependend functions:
|
||||
self._build_funcs()
|
||||
|
||||
def _get_ix(self, label):
|
||||
i = self._x_label_index.get(label)
|
||||
@ -52,6 +64,39 @@ class adj_matrix(object):
|
||||
self._labels.append(label)
|
||||
self._label_index[label] = i
|
||||
return i
|
||||
|
||||
def _end_document(self):
|
||||
self._document_count += 1
|
||||
|
||||
# adding all seen labels to our counter:
|
||||
for label in self._current_document_labels:
|
||||
self._label_document_count[label] += 1
|
||||
else:
|
||||
self._label_document_count[label] = 1
|
||||
|
||||
self._current_document_labels = set()
|
||||
|
||||
def apply_threshold(self, min_count=5):
|
||||
csr = self.get_csr()
|
||||
|
||||
new_x = []
|
||||
new_y = []
|
||||
new_data = []
|
||||
|
||||
for i in range(len(self._data)):
|
||||
if csr[self._x[i],self._y[i]] >= min_count:
|
||||
new_x.append(self._x[i])
|
||||
new_y.append(self._y[i])
|
||||
new_data.append(self._data[i])
|
||||
|
||||
self._x = new_x
|
||||
self._y = new_y
|
||||
self._data = new_data
|
||||
|
||||
|
||||
def next_document(self):
|
||||
self._end_document()
|
||||
|
||||
|
||||
def add_entry(self, x, y, data):
|
||||
|
||||
@ -66,6 +111,18 @@ class adj_matrix(object):
|
||||
self._x.append(ix)
|
||||
self._y.append(iy)
|
||||
self._data.append(data)
|
||||
|
||||
self._current_document_labels.add(x)
|
||||
self._current_document_labels.add(y)
|
||||
|
||||
def compile(self):
|
||||
self._csr = self.get_csr()
|
||||
if self._sym:
|
||||
self._np_labels = np.array(self._labels)
|
||||
else:
|
||||
self._np_x_labels = np.array(self._x_labels)
|
||||
self._np_y_labels = np.array(self._y_labels)
|
||||
|
||||
|
||||
def compile_to_mat(self):
|
||||
if self._sym:
|
||||
@ -85,6 +142,163 @@ class adj_matrix(object):
|
||||
if self._sym:
|
||||
return self._labels
|
||||
return self._x_labels, self._y_labels
|
||||
|
||||
def _build_funcs(self):
|
||||
|
||||
def get_sym_adjacent(key):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
index = self._label_index[key]
|
||||
i1 = c[index,:].nonzero()[1]
|
||||
i2 = c[:,index].nonzero()[0]
|
||||
|
||||
i = np.concatenate((i1,i2))
|
||||
|
||||
names = self._np_labels[i]
|
||||
|
||||
counts = np.concatenate((c[index, i1].toarray().flatten(), c[i2, index].toarray().flatten()))
|
||||
|
||||
s = np.argsort(-counts)
|
||||
|
||||
return names[s], counts[s]
|
||||
|
||||
def get_forward_adjacent(key):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
index = self._x_label_index[key]
|
||||
i = c[index,:].nonzero()[1]
|
||||
|
||||
names = self._np_y_labels[i]
|
||||
|
||||
counts = c[index, i].toarray().flatten()
|
||||
|
||||
s = np.argsort(-counts)
|
||||
|
||||
return names[s], counts[s]
|
||||
|
||||
def get_backward_adjacent(key):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
index = self._y_label_index[key]
|
||||
i = c[:,index].nonzero()[0]
|
||||
|
||||
|
||||
names = self._np_x_labels[i]
|
||||
|
||||
counts = c[i, index].toarray().flatten()
|
||||
|
||||
s = np.argsort(-counts)
|
||||
|
||||
return names[s], counts[s]
|
||||
|
||||
# sum functions:
|
||||
def sym_sum(key):
|
||||
return np.sum(self.get_adjacent(key)[1])
|
||||
|
||||
def fw_sum(key):
|
||||
return np.sum(self.get_forward_adjacent(key)[1])
|
||||
|
||||
def bw_sum(key):
|
||||
return np.sum(self.get_backward_adjacent(key)[1])
|
||||
|
||||
# normalization stuff:
|
||||
def fw_normalization_factor(key, quotient_func):
|
||||
assert self._csr is not None
|
||||
c = self._csr
|
||||
|
||||
ia = self._x_label_index[key]
|
||||
|
||||
occurances = c[ia,:].nonzero()[1]
|
||||
|
||||
return 1. / quotient_func(c[ia,occurances].toarray())
|
||||
|
||||
def bw_normalization_factor(key, quotient_func):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
ib = m._y_label_index[key]
|
||||
|
||||
occurances = c[:,ib].nonzero()[0]
|
||||
|
||||
return 1. / quotient_func(c[occurances,ib].toarray())
|
||||
|
||||
def sym_normalization_factor(key, quotient_func):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
ii = m._label_index[key]
|
||||
|
||||
fw_occurances = c[ii,:].nonzero()[1]
|
||||
bw_occurances = c[:,ii].nonzero()[0]
|
||||
|
||||
return 1. / quotient_func(np.concatenate(
|
||||
[c[ii,fw_occurances].toarray().flatten(),
|
||||
c[bw_occurances,ii].toarray().flatten()]
|
||||
))
|
||||
|
||||
def sym_p_a_given_b(key_a, key_b, quot_func = np.max):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
ia = m._label_index[key_a]
|
||||
ib = m._label_index[key_b]
|
||||
|
||||
v = c[ia,ib] + c[ib,ia]
|
||||
|
||||
return v * self.sym_normalization_factor(key_b, quot_func)
|
||||
|
||||
def fw_p_a_given_b(key_a, key_b, quot_func = np.max):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
ia = m._x_label_index[key_a]
|
||||
ib = m._y_label_index[key_b]
|
||||
|
||||
v = c[ia,ib]
|
||||
|
||||
return v * self.bw_normalization_factor(key_b, quot_func)
|
||||
|
||||
def bw_p_a_given_b(key_a, key_b, quot_func = np.max):
|
||||
assert self._csr is not None
|
||||
|
||||
c = self._csr
|
||||
|
||||
ia = m._y_label_index[key_a]
|
||||
ib = m._x_label_index[key_b]
|
||||
|
||||
v = c[ib,ia]
|
||||
|
||||
return v * self.fw_normalization_factor(key_b, quot_func)
|
||||
|
||||
|
||||
if self._sym:
|
||||
self.get_adjacent = get_sym_adjacent
|
||||
self.get_sum = sym_sum
|
||||
self.get_sym_normalization_factor = sym_normalization_factor
|
||||
self.p_a_given_b = sym_p_a_given_b
|
||||
|
||||
else:
|
||||
self.get_forward_adjacent = get_forward_adjacent
|
||||
self.get_backward_adjacent = get_backward_adjacent
|
||||
|
||||
self.get_fw_sum = fw_sum
|
||||
self.get_bw_sum = bw_sum
|
||||
|
||||
self.get_fw_normalization_factor = fw_normalization_factor
|
||||
self.get_bw_normalization_factor = bw_normalization_factor
|
||||
|
||||
self.fw_p_a_given_b = fw_p_a_given_b
|
||||
self.bw_p_a_given_b = bw_p_a_given_b
|
||||
|
||||
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -9,7 +9,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -45,7 +45,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -56,7 +56,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -65,16 +65,16 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<db.database_connection.DatabaseConnection at 0x7f5f9d0b5160>"
|
||||
"<db.database_connection.DatabaseConnection at 0x7fa5f2e0d9e8>"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -97,15 +97,15 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"CPU times: user 8.22 s, sys: 1.25 s, total: 9.46 s\n",
|
||||
"Wall time: 9.56 s\n"
|
||||
"CPU times: user 9 s, sys: 850 ms, total: 9.85 s\n",
|
||||
"Wall time: 9.92 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -122,7 +122,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -131,7 +131,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -217,8 +217,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"CPU times: user 9.53 ms, sys: 0 ns, total: 9.53 ms\n",
|
||||
"Wall time: 8.74 ms\n"
|
||||
"CPU times: user 10.8 ms, sys: 274 µs, total: 11 ms\n",
|
||||
"Wall time: 9.98 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -3142,8 +3142,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-0.7394662921348321,
|
||||
30.739466292134832
|
||||
-0.7173913043478262,
|
||||
30.717391304347828
|
||||
],
|
||||
"type": "category"
|
||||
},
|
||||
@ -4197,8 +4197,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-11.044917257683217,
|
||||
15.044917257683217
|
||||
-11.06323877068558,
|
||||
15.06323877068558
|
||||
],
|
||||
"scaleanchor": "x",
|
||||
"scaleratio": 1,
|
||||
@ -5336,8 +5336,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-0.9618863049095596,
|
||||
12.961886304909559
|
||||
-0.9702842377260978,
|
||||
12.970284237726098
|
||||
],
|
||||
"scaleanchor": "x",
|
||||
"scaleratio": 1,
|
||||
@ -9113,8 +9113,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-0.8529850746268686,
|
||||
42.85298507462687
|
||||
-0.88507462686567,
|
||||
42.88507462686567
|
||||
],
|
||||
"type": "category"
|
||||
},
|
||||
@ -13399,8 +13399,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-0.9393241167434745,
|
||||
43.93932411674348
|
||||
-0.9049079754601266,
|
||||
43.90490797546013
|
||||
],
|
||||
"type": "category"
|
||||
},
|
||||
@ -14673,8 +14673,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-13.429078014184398,
|
||||
20.429078014184398
|
||||
-13.455082742316787,
|
||||
20.455082742316787
|
||||
],
|
||||
"scaleanchor": "x",
|
||||
"scaleratio": 1,
|
||||
@ -16670,8 +16670,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-0.7020860495436789,
|
||||
30.70208604954368
|
||||
-0.681640625,
|
||||
30.681640625
|
||||
],
|
||||
"type": "category"
|
||||
},
|
||||
@ -17759,8 +17759,8 @@
|
||||
1
|
||||
],
|
||||
"range": [
|
||||
-11.55260047281324,
|
||||
16.55260047281324
|
||||
-11.570921985815605,
|
||||
16.570921985815605
|
||||
],
|
||||
"scaleanchor": "x",
|
||||
"scaleratio": 1,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user