CMR-10139: Updated code to remove subscription information from redis cache upon subscription delete. Updated schema to allow multiple modes. (#2179)

* CMR-10139: Updating Ingest Subscriptions to add/delete needed information to redis.

* CMR-10139: Adding in code to correctly delete the subscription from the redis subscription cache.

* CMR-10139: Adding data to the ingest subscription cache.

* CMR-10139: Fixing subscription api.

* CMR-10139: Adding redis library.

* CMR-10139: Removing debug statement.

* CMR-10139: adding helper function to create tombstone.

Author: eereiter

indexer-app/src/cmr/indexer/services/index_service.clj

@@ -326,6 +326,7 @@
             (Long/parseLong (str milliseconds))
             all-value)))
 
+#_{:clj-kondo/ignore [:unresolved-var]}
 (defn- send-time-to-visibility-log
   "Send either a JSON message as a report or the original log entry to info."
   [concept-id revision-id ms-durration all-revisions-index?]
@@ -337,15 +338,16 @@
   "Add a log message indicating the time it took to go from ingest to completed indexing."
   [{:keys [concept-id revision-id revision-date]} options]
   (let [now (tk/now)
-        {:keys [all-revisions-index?]} options
-        rev-datetime (f/parse (f/formatters :date-time) revision-date)
-        ms-duration (t/in-millis (t/interval rev-datetime now))]
-
+        rev-datetime (f/parse (f/formatters :date-time) revision-date)]
     ;; Guard against revision-date that is set to the future by a provider or a test.
+    ;; if the interval is set before with rev-datetime later an exception will be thrown.
     (if (t/before? rev-datetime now)
-      ;; WARNING: Splunk is dependent on this log message. DO NOT change this without updating
-      ;; Splunk searches used by ops.
-      (send-time-to-visibility-log concept-id revision-id ms-duration all-revisions-index?)
+      (let [{:keys [all-revisions-index?]} options
+            interval (t/interval rev-datetime now)
+            ms-duration (t/in-millis interval)]
+        ;; WARNING: Splunk is dependent on this log message. DO NOT change this without updating
+        ;; Splunk searches used by ops.
+        (send-time-to-visibility-log concept-id revision-id ms-duration all-revisions-index?))
       (warn (format
              "Cannot compute time from ingest to search visibility for [%s] with revision date [%s]."
              concept-id
@@ -833,6 +835,7 @@
          (concept-mapping-types concept-type)
          {:term {(query-field->elastic-field :provider-id concept-type) provider-id}})))))
 
+#_{:clj-kondo/ignore [:unresolved-var]}
 (defn publish-provider-event
   "Put a provider event on the message queue."
   [context msg]
@@ -881,6 +884,7 @@
   (es/update-indexes context params)
   (reset-index-set-mappings-cache context))
 
+#_{:clj-kondo/ignore [:unresolved-var]}
 (def health-check-fns
   "A map of keywords to functions to be called for health checks"
   {:elastic_search #(es-util/health % :db)

ingest-app/src/cmr/ingest/services/ingest_service/subscription.clj

@@ -15,7 +15,8 @@
                   :subscription-type (or (:Type subscription) "granule")
                   :normalized-query (:normalized-query concept)
                   :endpoint (:EndPoint subscription)
-                  :mode (:Mode subscription)}))
+                  :mode (:Mode subscription)
+                  :method (:Method subscription)}))
 
 (declare save-subscription)
 #_{:clj-kondo/ignore [:unresolved-symbol]}

metadata-db-app/project.clj

@@ -13,6 +13,7 @@
                  [nasa-cmr/cmr-common-lib "0.1.1-SNAPSHOT"]
                  [nasa-cmr/cmr-message-queue-lib "0.1.0-SNAPSHOT"]
                  [nasa-cmr/cmr-oracle-lib "0.1.0-SNAPSHOT"]
+                 [nasa-cmr/cmr-redis-utils-lib "0.1.0-SNAPSHOT"]
                  [nasa-cmr/cmr-schemas "0.0.1-SNAPSHOT"]
                  [nasa-cmr/cmr-umm-spec-lib "0.1.0-SNAPSHOT"]
                  [org.clojure/clojure "1.11.2"]

metadata-db-app/src/cmr/metadata_db/api/routes.clj

@@ -10,14 +10,14 @@
    [cmr.common.api.context :as context]
    [cmr.common.api.errors :as errors]
    [cmr.common.cache :as cache]
-   [cmr.common.log :refer (debug info warn error)]
+   [cmr.common.log :refer (info)]
    [cmr.metadata-db.api.concepts :as concepts-api]
    [cmr.metadata-db.api.provider :as provider-api]
    [cmr.metadata-db.api.subscriptions :as subscription-api]
    [cmr.metadata-db.services.concept-service :as concept-service]
    [cmr.metadata-db.services.health-service :as hs]
    [cmr.metadata-db.services.jobs :as mdb-jobs]
-   [compojure.core :refer :all]
+   [compojure.core :refer [GET POST context routes]]
    [compojure.route :as route]
    [drift.core]
    [drift.execute]
@@ -54,12 +54,12 @@
         ;; Dev dockerfile manually creates /app/cmr-files to store the unzipped cmr jar so that drift
         ;; can find the migration files correctly
         ;; we had to force method change in drift to set the correct path
-        (if (not (instance? cmr.common.memory_db.connection.MemoryStore db))
+        (when (not (instance? cmr.common.memory_db.connection.MemoryStore db))
           (try
             ;; trying non-local path to find drift migration files for external oracle db"
             (with-redefs [drift.core/user-directory (fn [] (new File (str (.getProperty (System/getProperties) "user.dir") "/drift-migration-files")))]
               (drift.execute/run migrate-args))
-            (catch Exception e
+            (catch Exception _e
               (println "caught exception trying to find migration files. We are probably in local env w/ external db. Trying local route to migration files...")
               (with-redefs [drift.core/user-directory (fn [] (new File (str (.getProperty (System/getProperties) "user.dir") "/checkouts/metadata-db-app/src")))]
                 (drift.execute/run migrate-args))))))
@@ -69,12 +69,12 @@
   (common-routes/job-api-routes
     (routes
       ;; Trigger the old revision concept cleanup
-      (POST "/old-revision-concept-cleanup" {:keys [request-context params headers]}
+      (POST "/old-revision-concept-cleanup" {:keys [request-context]}
         (acl/verify-ingest-management-permission request-context :update)
         (mdb-jobs/old-revision-concept-cleanup request-context)
         {:status 204})
 
-      (POST "/expired-concept-cleanup" {:keys [request-context params headers]}
+      (POST "/expired-concept-cleanup" {:keys [request-context]}
         (acl/verify-ingest-management-permission request-context :update)
         (mdb-jobs/expired-concept-cleanup request-context)
         {:status 204}))))

metadata-db-app/src/cmr/metadata_db/api/subscriptions.clj

@@ -2,21 +2,23 @@
   "Defines the HTTP URL routes for the application as related to subscriptions."
   (:require
    [cmr.metadata-db.services.sub-notifications :as sub-note]
-   [compojure.core :refer :all]))
+   [cmr.metadata-db.services.subscriptions :as subscriptions]
+   [compojure.core :refer [PUT POST context]]))
 
 (defn- update-subscription-notification-time
   "Update a subscription notification time"
   [context params]
   (let [sub-id (:subscription-concept-id params)
-        result (sub-note/update-subscription-notification context sub-id)]
+        _ (sub-note/update-subscription-notification context sub-id)]
     {:status 204}))
 
-; TODO add a get interface to help with testing. CMR-6621
-
 (def subscription-api-routes
   (context "/subscription" []
     ;; receive notification to update subscription time
     (PUT "/:subscription-concept-id/notification-time"
-      {{:keys [subscription-concept-id] :as params} :params
+      {params :params
        request-context :request-context}
-      (update-subscription-notification-time request-context params))))
+      (update-subscription-notification-time request-context params))
+    (POST "/refresh-subscription-cache"
+      {request-context :request-context}
+      (subscriptions/refresh-subscription-cache request-context))))

metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/subscription.clj

@@ -13,7 +13,7 @@
     subscription))
 
 (defmethod concepts/db-result->concept-map :subscription
-  [concept-type db provider-id result]
+  [_concept-type db provider-id result]
   (some-> (concepts/db-result->concept-map :default db provider-id result)
           (assoc :concept-type :subscription)
           (assoc :provider-id (:provider_id result))
@@ -52,7 +52,7 @@
   (subscription-concept->insert-args concept))
 
 (defmethod concepts/after-save :subscription
-  [db provider sub]
+  [db _provider sub]
   (when (:deleted sub)
     ;; Cascade deletion to real deletes of subscription notifications. The intended functionality
     ;; since the subscription_notification row is purged is that, if this subscription were

metadata-db-app/src/cmr/metadata_db/services/concept_service.clj

@@ -1,6 +1,7 @@
 (ns cmr.metadata-db.services.concept-service
   "Services to support the business logic of the metadata db."
   (:require
+   [cheshire.core :as json]
    [clj-time.core :as t]
    [clojure.set :as set]
    [cmr.common.api.context :as cmn-context]
@@ -757,6 +758,32 @@
 
     :else nil))
 
+(defn create-tombstone-concept
+  "Helper function to create a tombstone concept by merging in parts of the
+  previous concept to the tombstone."
+  [metadata concept previous-revision]
+  (let [{:keys [concept-id revision-id revision-date user-id]} concept
+        {:keys [concept-type _]} (cu/parse-concept-id concept-id)]
+    (if (= :subscription concept-type)
+      (let [metadata-edn (json/decode metadata true)
+            extra-fields (:extra-fields previous-revision)]
+        (merge previous-revision {:concept-id concept-id
+                                  :revision-id revision-id
+                                  :revision-date revision-date
+                                  :user-id user-id
+                                  :metadata ""
+                                  :deleted true
+                                  :extra-fields (merge extra-fields
+                                                       {:endpoint (:EndPoint metadata-edn)
+                                                        :mode (:Mode metadata-edn)
+                                                        :method (:Method metadata-edn)})}))
+      (merge previous-revision {:concept-id concept-id
+                                :revision-id revision-id
+                                :revision-date revision-date
+                                :user-id user-id
+                                :metadata metadata
+                                :deleted true}))))
+
 ;; true implies creation of tombstone for the revision
 (defmethod save-concept-revision true
   [context concept]
@@ -776,15 +803,11 @@
       ;; to send concept updates and deletions out of order.
       (if (and (util/is-tombstone? previous-revision) (nil? revision-id))
         previous-revision
-        (let [metadata (if (cu/generic-concept? concept-type)
+        (let [metadata (if (or (cu/generic-concept? concept-type)
+                               (= :subscription concept-type))
                          (:metadata previous-revision)
                          "")
-              tombstone (merge previous-revision {:concept-id concept-id
-                                                  :revision-id revision-id
-                                                  :revision-date revision-date
-                                                  :user-id user-id
-                                                  :metadata metadata
-                                                  :deleted true})]
+              tombstone (create-tombstone-concept metadata concept previous-revision)]
           (cv/validate-concept tombstone)
           (validate-concept-revision-id db provider tombstone previous-revision)
           (let [revisioned-tombstone (->> (set-or-generate-revision-id db provider tombstone previous-revision)

metadata-db-app/src/cmr/metadata_db/services/concept_validations.clj

@@ -96,7 +96,7 @@
   can sometimes be nil."
   [concept]
   (nil-fields-validation (apply dissoc (:extra-fields concept)
-                                [:delete-time :version-id :source-revision-id :associated-revision-id :target-provider-id :collection-concept-id :endpoint :mode])))
+                                [:delete-time :version-id :source-revision-id :associated-revision-id :target-provider-id :collection-concept-id :endpoint :mode :method])))
 
 (defn concept-id-validation
   [concept]

metadata-db-app/src/cmr/metadata_db/services/subscription_cache.clj

@@ -4,8 +4,11 @@
 	<collection-concept-id> --> <ingest subscription map>
 
 	Example:
-	'C12344554-PROV1' --> { 'enabled' true|false
-                          'mode'    New|Update|Delete|All} "
+  {Collection concept id 1: {\"New\" 1
+                             \"Update\" 1}
+   Collection concept id 2: {\"New\" 2
+                             \"Update\" 1
+                             \"Delete\" 3}"
   (:require
    [cmr.common.hash-cache :as hash-cache]
    [cmr.common.redis-log-util :as rl-util]
@@ -47,3 +50,12 @@
                     (hash-cache/remove-value cache-client subscription-cache-key collection-concept-id))]
     (rl-util/log-redis-write-complete "ingest-subscription-cache remove-value" subscription-cache-key tm)
     value))
+
+(defn get-keys
+  "Gets the collection-concept-ids from the cache."
+  [context]
+  (let [cache-client (hash-cache/context->cache context subscription-cache-key)
+        [tm value] (util/time-execution
+                    (hash-cache/get-keys cache-client subscription-cache-key))]
+    (rl-util/log-redis-read-complete "ingest-subscription-cache get-keys" subscription-cache-key tm)
+    value))