CMR-10139: Re-writing the cache contents to cover when the same subscription is updated.

Author: eereiter

metadata-db-app/src/cmr/metadata_db/services/concept_service.clj

@@ -852,7 +852,7 @@
                       (= concept-type :tool-association))
               (ingest-events/publish-event
                context (ingest-events/concept-delete-event revisioned-tombstone)))
-            (subscriptions/delete-subscription context concept-type revisioned-tombstone)
+            (subscriptions/change-subscription context concept-type revisioned-tombstone)
             revisioned-tombstone)))
       (if revision-id
         (cmsg/data-error :not-found
@@ -940,8 +940,7 @@
       (ingest-events/publish-event
        context
        (ingest-events/concept-update-event concept))
-
-      (subscriptions/add-subscription context concept-type concept)
+      (subscriptions/change-subscription context concept-type concept)
       concept)))
 
 (defn- delete-associated-tag-associations

metadata-db-app/src/cmr/metadata_db/services/subscription_cache.clj

@@ -1,14 +1,11 @@
 (ns cmr.metadata-db.services.subscription-cache
   "Defines common functions and defs for the subscription cache.
 	Structure of the hash-cache is as follows:
-	<collection-concept-id> --> <ingest subscription map>
+	<collection-concept-id> --> <ingest subscription vector>
 
 	Example:
-  {Collection concept id 1: {\"New\" 1
-                             \"Update\" 1}
-   Collection concept id 2: {\"New\" 2
-                             \"Update\" 1
-                             \"Delete\" 3}"
+  {Collection concept id 1: [\"New\" \"Update\"]
+   Collection concept id 2: [\"New\" \"Update\" \"Delete\"]"
   (:require
    [cmr.common.hash-cache :as hash-cache]
    [cmr.common.redis-log-util :as rl-util]

metadata-db-app/src/cmr/metadata_db/services/subscriptions.clj

@@ -24,82 +24,64 @@
   (and subscriptions-enabled?
        (= :granule concept-type)))
 
-(defn remove-from-existing-mode
-  "Depending on the passed in new-mode ['New' 'Update'] remove from the structure
-  {Collection concept id: {\"New\" 1
-                           \"Update\" 1}}
-   either the count or the mode if count is 0."
-  [existing-mode new-mode]
-  (loop [ms new-mode
-         result existing-mode]
-    (let [mode (first ms)
-          mode-count (if (and result
-                              (result mode))
-                       (result mode) 0)
-          compare-to-one (compare mode-count 1)]
-      (if mode
-        (when (seq result)
-          (if (or (= 0 compare-to-one)
-                  (= -1 compare-to-one))
-            (recur (rest ms) (dissoc result mode))
-            (recur (rest ms) (assoc result mode (dec mode-count)))))
-        result))))
-
-(defn delete-subscription 
-  "When a subscription is deleted, the collection-concept-id must be removed 
-  from the subscription cache. Decrement the count if more than 1 subscription
-  uses the collection-concept-id."
-  [context concept-type revisioned-tombstone]
-  (when (subscription-concept? concept-type revisioned-tombstone)
-    (let [coll-concept-id (:collection-concept-id (:extra-fields revisioned-tombstone))
-          existing-value (subscription-cache/get-value context coll-concept-id)
-          mode (:mode (:extra-fields revisioned-tombstone))
-          new-mode (remove-from-existing-mode existing-value mode)]
-      (if (seq new-mode)
-        (subscription-cache/set-value context coll-concept-id new-mode)
-        (subscription-cache/remove-value context coll-concept-id)))))
+(defn ^:dynamic get-subscriptions-from-db
+  "Get the subscriptions from the database. This function primarily exists so that
+  it can be stubbed out for unit tests."
+  ([context]
+   (mdb-search/find-concepts context {:latest true
+                                      :concept-type :subscription
+                                      :subscription-type "granule"}))
+  ([context coll-concept-id]
+   (mdb-search/find-concepts context {:latest true
+                                      :concept-type :subscription
+                                      :collection-concept-id coll-concept-id})))
 
 (defn add-to-existing-mode
-  "Depending on the passed in new-mode ['New' 'Update'] create a structure that looks like
-  {Collection concept id: {\"New\" 1
-                           \"Update\" 1}}"
-  [existing-mode new-mode]
-  (loop [ms new-mode
-         result existing-mode]
-    (let [mode (first ms)
-          mode-count (if (and result
-                              (result mode))
-                       (result mode)
-                       0)]
-      (if mode
+  "Depending on the passed in new-mode [\"New\" \"Update\"] create a structure that merges
+  the new mode to the existing mode. The result looks like [\"New\" \"Update\"]"
+  [existing-modes new-modes]
+  (loop [ms new-modes
+         result existing-modes]
+    (let [mode (first ms)]
+      (if (nil? mode)
+        result
         (if result
-          (recur (rest ms) (assoc result mode (inc mode-count)))
-          (recur (rest ms) (assoc {} mode (inc mode-count))))
-        result))))
+          (if (some #(= mode %) result)
+            (recur (rest ms) result)
+            (recur (rest ms) (merge result mode)))
+          (recur (rest ms) [mode]))))))
+
+(defn merge-modes
+  "Go through the list of subscriptions to see if any exist that match the
+  passed in collection-concept-id. Return true if any subscription exists
+  otherwise return false."
+  [subscriptions]
+  (loop [subs subscriptions
+         result []]
+    (let [sub (first subs)]
+      (if (nil? sub)
+        result
+        (recur (rest subs) (add-to-existing-mode result (get-in sub [:extra-fields :mode])))))))
 
-(defn add-subscription
-  "When a subscription is added, the collection-concept-id must be put into  
-  the subscription cache. If the collection-concept-id already exists then another
-  subscription is also using it so increment the count. The count is used to determine
-  whether or not the entry can be deleted. The modes are concatenated so that the user
-  of the cache knows which modes to key off of."
+(defn change-subscription
+  "When a subscription is added or deleted, the collection-concept-id must be put into
+  or deleted from the subscription cache. Get the subscriptions that match the collection-concept-id
+  from the database and rebuild the modes list."
   [context concept-type concept]
   (when (subscription-concept? concept-type concept)
     (let [coll-concept-id (:collection-concept-id (:extra-fields concept))
-          mode (:mode (:extra-fields concept))
-          existing-mode (subscription-cache/get-value context coll-concept-id)
-          new-value (add-to-existing-mode existing-mode mode)]
-      (subscription-cache/set-value context coll-concept-id new-value))))
+          subs (filter #(subscription-concept? (get % :concept-type) %)
+                       (get-subscriptions-from-db context coll-concept-id))]
+      (if (seq subs)
+        (subscription-cache/set-value context coll-concept-id (merge-modes subs))
+        (subscription-cache/remove-value context coll-concept-id)))))
 
 (defn create-subscription-cache-contents-for-refresh
   "Go through all of the subscriptions and find the ones that are 
   ingest subscriptions. Create the mode values for each collection-concept-id
   and put those into a map. The end result looks like:
-  {Collection concept id 1: {\"New\" 1
-                           \"Update\" 1}
-   Collection concept id 2: {\"New\" 2
-                           \"Update\" 1
-                           \"Delete\" 3}
+  {Collection concept id 1: [\"New\" \"Update\"]
+   Collection concept id 2: [\"New\" \"Update\" \"Delete\"]
    ...}"
   [result sub]
   (let [metadata (:metadata sub)
@@ -111,16 +93,9 @@
             mode (:Mode metadata-edn)]
         (if concept-map
           (update result coll-concept-id #(add-to-existing-mode % mode))
-          (assoc concept-map coll-concept-id (add-to-existing-mode nil mode))))
+          (assoc result coll-concept-id (add-to-existing-mode nil mode))))
       result)))
 
-(defn ^:dynamic get-subscriptions-from-db
-  "Get the subscriptions from the database. This function primarily exists so that
-  it can be stubbed out for unit tests."
-  [context]
-  (mdb-search/find-concepts context {:latest true
-                                     :concept-type :subscription}))
-
 (defn refresh-subscription-cache
   "Go through all of the subscriptions and create a map of collection concept ids and
   their mode values. Get the old keys from the cache and if the keys exist in the new structure