switched from pickle to dill to for creating adjaency matrices
This commit is contained in:
		
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							| @ -28,6 +28,18 @@ class adj_matrix(object): | ||||
|         self._data = [] | ||||
|          | ||||
|         self._mat = None | ||||
|         self._csr = None | ||||
|  | ||||
|         # for a TF-IDF like approach we need also a counter how frequently ingredients | ||||
|         # and actions appear in documents.  | ||||
|  | ||||
|         self._current_document_labels = set() | ||||
|         self._label_document_count = {} | ||||
|          | ||||
|         self._document_count = 0 | ||||
|          | ||||
|         # building type dependend functions: | ||||
|         self._build_funcs() | ||||
|      | ||||
|     def _get_ix(self, label): | ||||
|         i = self._x_label_index.get(label) | ||||
| @ -53,6 +65,39 @@ class adj_matrix(object): | ||||
|             self._label_index[label] = i | ||||
|         return i | ||||
|  | ||||
|     def _end_document(self): | ||||
|         self._document_count += 1 | ||||
|  | ||||
|         # adding all seen labels to our counter: | ||||
|         for label in self._current_document_labels: | ||||
|             self._label_document_count[label] += 1 | ||||
|         else: | ||||
|             self._label_document_count[label] = 1 | ||||
|          | ||||
|         self._current_document_labels = set() | ||||
|      | ||||
|     def apply_threshold(self, min_count=5): | ||||
|         csr = self.get_csr() | ||||
|  | ||||
|         new_x = [] | ||||
|         new_y = [] | ||||
|         new_data = [] | ||||
|  | ||||
|         for i in range(len(self._data)): | ||||
|             if csr[self._x[i],self._y[i]] >= min_count: | ||||
|                 new_x.append(self._x[i]) | ||||
|                 new_y.append(self._y[i]) | ||||
|                 new_data.append(self._data[i]) | ||||
|          | ||||
|         self._x = new_x | ||||
|         self._y = new_y | ||||
|         self._data = new_data | ||||
|  | ||||
|      | ||||
|     def next_document(self): | ||||
|         self._end_document() | ||||
|  | ||||
|      | ||||
|     def add_entry(self, x, y, data): | ||||
|          | ||||
|         if self._sym: | ||||
| @ -67,6 +112,18 @@ class adj_matrix(object): | ||||
|         self._y.append(iy) | ||||
|         self._data.append(data) | ||||
|  | ||||
|         self._current_document_labels.add(x) | ||||
|         self._current_document_labels.add(y) | ||||
|      | ||||
|     def compile(self): | ||||
|         self._csr = self.get_csr() | ||||
|         if self._sym: | ||||
|             self._np_labels = np.array(self._labels) | ||||
|         else: | ||||
|             self._np_x_labels = np.array(self._x_labels) | ||||
|             self._np_y_labels = np.array(self._y_labels) | ||||
|          | ||||
|      | ||||
|     def compile_to_mat(self): | ||||
|         if self._sym: | ||||
|             sx = len(self._labels) | ||||
| @ -86,6 +143,163 @@ class adj_matrix(object): | ||||
|             return self._labels | ||||
|         return self._x_labels, self._y_labels | ||||
|      | ||||
|     def _build_funcs(self): | ||||
|          | ||||
|         def get_sym_adjacent(key): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             index = self._label_index[key] | ||||
|             i1 = c[index,:].nonzero()[1] | ||||
|             i2 = c[:,index].nonzero()[0] | ||||
|  | ||||
|             i = np.concatenate((i1,i2)) | ||||
|  | ||||
|             names = self._np_labels[i] | ||||
|  | ||||
|             counts = np.concatenate((c[index, i1].toarray().flatten(), c[i2, index].toarray().flatten())) | ||||
|  | ||||
|             s = np.argsort(-counts) | ||||
|  | ||||
|             return names[s], counts[s] | ||||
|          | ||||
|         def get_forward_adjacent(key): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             index = self._x_label_index[key] | ||||
|             i = c[index,:].nonzero()[1] | ||||
|  | ||||
|             names = self._np_y_labels[i] | ||||
|  | ||||
|             counts = c[index, i].toarray().flatten() | ||||
|  | ||||
|             s = np.argsort(-counts) | ||||
|  | ||||
|             return names[s], counts[s] | ||||
|          | ||||
|         def get_backward_adjacent(key): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             index = self._y_label_index[key] | ||||
|             i = c[:,index].nonzero()[0] | ||||
|  | ||||
|              | ||||
|             names = self._np_x_labels[i] | ||||
|  | ||||
|             counts = c[i, index].toarray().flatten() | ||||
|  | ||||
|             s = np.argsort(-counts) | ||||
|  | ||||
|             return names[s], counts[s] | ||||
|          | ||||
|         # sum functions: | ||||
|         def sym_sum(key): | ||||
|             return np.sum(self.get_adjacent(key)[1]) | ||||
|  | ||||
|         def fw_sum(key): | ||||
|             return np.sum(self.get_forward_adjacent(key)[1]) | ||||
|  | ||||
|         def bw_sum(key): | ||||
|             return np.sum(self.get_backward_adjacent(key)[1]) | ||||
|          | ||||
|         # normalization stuff: | ||||
|         def fw_normalization_factor(key, quotient_func): | ||||
|             assert self._csr is not None | ||||
|             c = self._csr | ||||
|              | ||||
|             ia = self._x_label_index[key] | ||||
|  | ||||
|             occurances = c[ia,:].nonzero()[1] | ||||
|  | ||||
|             return 1. / quotient_func(c[ia,occurances].toarray()) | ||||
|  | ||||
|         def bw_normalization_factor(key, quotient_func): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             ib = m._y_label_index[key] | ||||
|  | ||||
|             occurances = c[:,ib].nonzero()[0] | ||||
|  | ||||
|             return 1. / quotient_func(c[occurances,ib].toarray()) | ||||
|  | ||||
|         def sym_normalization_factor(key, quotient_func): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             ii = m._label_index[key] | ||||
|  | ||||
|             fw_occurances = c[ii,:].nonzero()[1] | ||||
|             bw_occurances = c[:,ii].nonzero()[0] | ||||
|  | ||||
|             return 1. / quotient_func(np.concatenate( | ||||
|                 [c[ii,fw_occurances].toarray().flatten(), | ||||
|                  c[bw_occurances,ii].toarray().flatten()] | ||||
|             )) | ||||
|          | ||||
|         def sym_p_a_given_b(key_a, key_b, quot_func = np.max): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             ia = m._label_index[key_a] | ||||
|             ib = m._label_index[key_b] | ||||
|  | ||||
|             v = c[ia,ib] + c[ib,ia] | ||||
|  | ||||
|             return v * self.sym_normalization_factor(key_b, quot_func) | ||||
|  | ||||
|         def fw_p_a_given_b(key_a, key_b, quot_func = np.max): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             ia = m._x_label_index[key_a] | ||||
|             ib = m._y_label_index[key_b] | ||||
|  | ||||
|             v = c[ia,ib] | ||||
|  | ||||
|             return v * self.bw_normalization_factor(key_b, quot_func) | ||||
|  | ||||
|         def bw_p_a_given_b(key_a, key_b, quot_func = np.max): | ||||
|             assert self._csr is not None | ||||
|              | ||||
|             c = self._csr | ||||
|              | ||||
|             ia = m._y_label_index[key_a] | ||||
|             ib = m._x_label_index[key_b] | ||||
|  | ||||
|             v = c[ib,ia] | ||||
|  | ||||
|             return v * self.fw_normalization_factor(key_b, quot_func) | ||||
|  | ||||
|          | ||||
|         if self._sym: | ||||
|             self.get_adjacent = get_sym_adjacent | ||||
|             self.get_sum = sym_sum | ||||
|             self.get_sym_normalization_factor = sym_normalization_factor | ||||
|             self.p_a_given_b = sym_p_a_given_b | ||||
|          | ||||
|         else: | ||||
|             self.get_forward_adjacent = get_forward_adjacent | ||||
|             self.get_backward_adjacent = get_backward_adjacent | ||||
|              | ||||
|             self.get_fw_sum = fw_sum | ||||
|             self.get_bw_sum = bw_sum | ||||
|              | ||||
|             self.get_fw_normalization_factor = fw_normalization_factor | ||||
|             self.get_bw_normalization_factor = bw_normalization_factor | ||||
|  | ||||
|             self.fw_p_a_given_b = fw_p_a_given_b | ||||
|             self.bw_p_a_given_b = bw_p_a_given_b | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							| @ -9,7 +9,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -45,7 +45,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -56,7 +56,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -65,16 +65,16 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "execution_count": 4, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "<db.database_connection.DatabaseConnection at 0x7f5f9d0b5160>" | ||||
|        "<db.database_connection.DatabaseConnection at 0x7fa5f2e0d9e8>" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 6, | ||||
|      "execution_count": 4, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -97,15 +97,15 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "execution_count": 5, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "CPU times: user 8.22 s, sys: 1.25 s, total: 9.46 s\n", | ||||
|       "Wall time: 9.56 s\n" | ||||
|       "CPU times: user 9 s, sys: 850 ms, total: 9.85 s\n", | ||||
|       "Wall time: 9.92 s\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
| @ -122,7 +122,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "execution_count": 6, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -131,7 +131,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "execution_count": 7, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -217,8 +217,8 @@ | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "CPU times: user 9.53 ms, sys: 0 ns, total: 9.53 ms\n", | ||||
|       "Wall time: 8.74 ms\n" | ||||
|       "CPU times: user 10.8 ms, sys: 274 µs, total: 11 ms\n", | ||||
|       "Wall time: 9.98 ms\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
| @ -3142,8 +3142,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -0.7394662921348321, | ||||
|           30.739466292134832 | ||||
|           -0.7173913043478262, | ||||
|           30.717391304347828 | ||||
|          ], | ||||
|          "type": "category" | ||||
|         }, | ||||
| @ -4197,8 +4197,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -11.044917257683217, | ||||
|           15.044917257683217 | ||||
|           -11.06323877068558, | ||||
|           15.06323877068558 | ||||
|          ], | ||||
|          "scaleanchor": "x", | ||||
|          "scaleratio": 1, | ||||
| @ -5336,8 +5336,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -0.9618863049095596, | ||||
|           12.961886304909559 | ||||
|           -0.9702842377260978, | ||||
|           12.970284237726098 | ||||
|          ], | ||||
|          "scaleanchor": "x", | ||||
|          "scaleratio": 1, | ||||
| @ -9113,8 +9113,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -0.8529850746268686, | ||||
|           42.85298507462687 | ||||
|           -0.88507462686567, | ||||
|           42.88507462686567 | ||||
|          ], | ||||
|          "type": "category" | ||||
|         }, | ||||
| @ -13399,8 +13399,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -0.9393241167434745, | ||||
|           43.93932411674348 | ||||
|           -0.9049079754601266, | ||||
|           43.90490797546013 | ||||
|          ], | ||||
|          "type": "category" | ||||
|         }, | ||||
| @ -14673,8 +14673,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -13.429078014184398, | ||||
|           20.429078014184398 | ||||
|           -13.455082742316787, | ||||
|           20.455082742316787 | ||||
|          ], | ||||
|          "scaleanchor": "x", | ||||
|          "scaleratio": 1, | ||||
| @ -16670,8 +16670,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -0.7020860495436789, | ||||
|           30.70208604954368 | ||||
|           -0.681640625, | ||||
|           30.681640625 | ||||
|          ], | ||||
|          "type": "category" | ||||
|         }, | ||||
| @ -17759,8 +17759,8 @@ | ||||
|           1 | ||||
|          ], | ||||
|          "range": [ | ||||
|           -11.55260047281324, | ||||
|           16.55260047281324 | ||||
|           -11.570921985815605, | ||||
|           16.570921985815605 | ||||
|          ], | ||||
|          "scaleanchor": "x", | ||||
|          "scaleratio": 1, | ||||
|  | ||||
		Reference in New Issue
	
	Block a user