switched from pickle to dill to for creating adjaency matrices

RecipeAnalysis/AdjacencyMatrix.py

@@ -28,6 +28,18 @@ class adj_matrix(object):
         self._data = []
         
         self._mat = None
+        self._csr = None
+
+        # for a TF-IDF like approach we need also a counter how frequently ingredients
+        # and actions appear in documents. 
+
+        self._current_document_labels = set()
+        self._label_document_count = {}
+        
+        self._document_count = 0
+        
+        # building type dependend functions:
+        self._build_funcs()
     
     def _get_ix(self, label):
         i = self._x_label_index.get(label)
@@ -52,6 +64,39 @@ class adj_matrix(object):
             self._labels.append(label)
             self._label_index[label] = i
         return i
+
+    def _end_document(self):
+        self._document_count += 1
+
+        # adding all seen labels to our counter:
+        for label in self._current_document_labels:
+            self._label_document_count[label] += 1
+        else:
+            self._label_document_count[label] = 1
+        
+        self._current_document_labels = set()
+    
+    def apply_threshold(self, min_count=5):
+        csr = self.get_csr()
+
+        new_x = []
+        new_y = []
+        new_data = []
+
+        for i in range(len(self._data)):
+            if csr[self._x[i],self._y[i]] >= min_count:
+                new_x.append(self._x[i])
+                new_y.append(self._y[i])
+                new_data.append(self._data[i])
+        
+        self._x = new_x
+        self._y = new_y
+        self._data = new_data
+
+    
+    def next_document(self):
+        self._end_document()
+
     
     def add_entry(self, x, y, data):
         
@@ -66,6 +111,18 @@ class adj_matrix(object):
         self._x.append(ix)
         self._y.append(iy)
         self._data.append(data)
+
+        self._current_document_labels.add(x)
+        self._current_document_labels.add(y)
+    
+    def compile(self):
+        self._csr = self.get_csr()
+        if self._sym:
+            self._np_labels = np.array(self._labels)
+        else:
+            self._np_x_labels = np.array(self._x_labels)
+            self._np_y_labels = np.array(self._y_labels)
+        
     
     def compile_to_mat(self):
         if self._sym:
@@ -85,6 +142,163 @@ class adj_matrix(object):
         if self._sym:
             return self._labels
         return self._x_labels, self._y_labels
+    
+    def _build_funcs(self):
+        
+        def get_sym_adjacent(key):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            index = self._label_index[key]
+            i1 = c[index,:].nonzero()[1]
+            i2 = c[:,index].nonzero()[0]
+
+            i = np.concatenate((i1,i2))
+
+            names = self._np_labels[i]
+
+            counts = np.concatenate((c[index, i1].toarray().flatten(), c[i2, index].toarray().flatten()))
+
+            s = np.argsort(-counts)
+
+            return names[s], counts[s]
+        
+        def get_forward_adjacent(key):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            index = self._x_label_index[key]
+            i = c[index,:].nonzero()[1]
+
+            names = self._np_y_labels[i]
+
+            counts = c[index, i].toarray().flatten()
+
+            s = np.argsort(-counts)
+
+            return names[s], counts[s]
+        
+        def get_backward_adjacent(key):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            index = self._y_label_index[key]
+            i = c[:,index].nonzero()[0]
+
+            
+            names = self._np_x_labels[i]
+
+            counts = c[i, index].toarray().flatten()
+
+            s = np.argsort(-counts)
+
+            return names[s], counts[s]
+        
+        # sum functions:
+        def sym_sum(key):
+            return np.sum(self.get_adjacent(key)[1])
+
+        def fw_sum(key):
+            return np.sum(self.get_forward_adjacent(key)[1])
+
+        def bw_sum(key):
+            return np.sum(self.get_backward_adjacent(key)[1])
+        
+        # normalization stuff:
+        def fw_normalization_factor(key, quotient_func):
+            assert self._csr is not None
+            c = self._csr
+            
+            ia = self._x_label_index[key]
+
+            occurances = c[ia,:].nonzero()[1]
+
+            return 1. / quotient_func(c[ia,occurances].toarray())
+
+        def bw_normalization_factor(key, quotient_func):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            ib = m._y_label_index[key]
+
+            occurances = c[:,ib].nonzero()[0]
+
+            return 1. / quotient_func(c[occurances,ib].toarray())
+
+        def sym_normalization_factor(key, quotient_func):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            ii = m._label_index[key]
+
+            fw_occurances = c[ii,:].nonzero()[1]
+            bw_occurances = c[:,ii].nonzero()[0]
+
+            return 1. / quotient_func(np.concatenate(
+                [c[ii,fw_occurances].toarray().flatten(),
+                 c[bw_occurances,ii].toarray().flatten()]
+            ))
+        
+        def sym_p_a_given_b(key_a, key_b, quot_func = np.max):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            ia = m._label_index[key_a]
+            ib = m._label_index[key_b]
+
+            v = c[ia,ib] + c[ib,ia]
+
+            return v * self.sym_normalization_factor(key_b, quot_func)
+
+        def fw_p_a_given_b(key_a, key_b, quot_func = np.max):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            ia = m._x_label_index[key_a]
+            ib = m._y_label_index[key_b]
+
+            v = c[ia,ib]
+
+            return v * self.bw_normalization_factor(key_b, quot_func)
+
+        def bw_p_a_given_b(key_a, key_b, quot_func = np.max):
+            assert self._csr is not None
+            
+            c = self._csr
+            
+            ia = m._y_label_index[key_a]
+            ib = m._x_label_index[key_b]
+
+            v = c[ib,ia]
+
+            return v * self.fw_normalization_factor(key_b, quot_func)
+
+        
+        if self._sym:
+            self.get_adjacent = get_sym_adjacent
+            self.get_sum = sym_sum
+            self.get_sym_normalization_factor = sym_normalization_factor
+            self.p_a_given_b = sym_p_a_given_b
+        
+        else:
+            self.get_forward_adjacent = get_forward_adjacent
+            self.get_backward_adjacent = get_backward_adjacent
+            
+            self.get_fw_sum = fw_sum
+            self.get_bw_sum = bw_sum
+            
+            self.get_fw_normalization_factor = fw_normalization_factor
+            self.get_bw_normalization_factor = bw_normalization_factor
+
+            self.fw_p_a_given_b = fw_p_a_given_b
+            self.bw_p_a_given_b = bw_p_a_given_b
 
 
 

RecipeAnalysis/Recipe Analysis.ipynb

@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -45,7 +45,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -65,16 +65,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<db.database_connection.DatabaseConnection at 0x7f5f9d0b5160>"
+       "<db.database_connection.DatabaseConnection at 0x7fa5f2e0d9e8>"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -97,15 +97,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 8.22 s, sys: 1.25 s, total: 9.46 s\n",
+      "CPU times: user 9 s, sys: 850 ms, total: 9.85 s\n",
-      "Wall time: 9.56 s\n"
+      "Wall time: 9.92 s\n"
      ]
     }
    ],
@@ -122,7 +122,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -131,7 +131,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -217,8 +217,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 9.53 ms, sys: 0 ns, total: 9.53 ms\n",
+      "CPU times: user 10.8 ms, sys: 274 µs, total: 11 ms\n",
-      "Wall time: 8.74 ms\n"
+      "Wall time: 9.98 ms\n"
      ]
     }
    ],
@@ -3142,8 +3142,8 @@
           1
          ],
          "range": [
-          -0.7394662921348321,
+          -0.7173913043478262,
-          30.739466292134832
+          30.717391304347828
          ],
          "type": "category"
         },
@@ -4197,8 +4197,8 @@
           1
          ],
          "range": [
-          -11.044917257683217,
+          -11.06323877068558,
-          15.044917257683217
+          15.06323877068558
          ],
          "scaleanchor": "x",
          "scaleratio": 1,
@@ -5336,8 +5336,8 @@
           1
          ],
          "range": [
-          -0.9618863049095596,
+          -0.9702842377260978,
-          12.961886304909559
+          12.970284237726098
          ],
          "scaleanchor": "x",
          "scaleratio": 1,
@@ -9113,8 +9113,8 @@
           1
          ],
          "range": [
-          -0.8529850746268686,
+          -0.88507462686567,
-          42.85298507462687
+          42.88507462686567
          ],
          "type": "category"
         },
@@ -13399,8 +13399,8 @@
           1
          ],
          "range": [
-          -0.9393241167434745,
+          -0.9049079754601266,
-          43.93932411674348
+          43.90490797546013
          ],
          "type": "category"
         },
@@ -14673,8 +14673,8 @@
           1
          ],
          "range": [
-          -13.429078014184398,
+          -13.455082742316787,
-          20.429078014184398
+          20.455082742316787
          ],
          "scaleanchor": "x",
          "scaleratio": 1,
@@ -16670,8 +16670,8 @@
           1
          ],
          "range": [
-          -0.7020860495436789,
+          -0.681640625,
-          30.70208604954368
+          30.681640625
          ],
          "type": "category"
         },
@@ -17759,8 +17759,8 @@
           1
          ],
          "range": [
-          -11.55260047281324,
+          -11.570921985815605,
-          16.55260047281324
+          16.570921985815605
          ],
          "scaleanchor": "x",
          "scaleratio": 1,