combine elastic server setup funcs, add log toggle

Author: jmaeng72

dev-system/src/cmr/dev_system/system.clj

@@ -12,7 +12,7 @@
    [cmr.common.util :as u]
    [cmr.dev-system.config :as dev-config]
    [cmr.dev-system.control :as control]
-   [cmr.elastic-utils.config :as elastic-config]
+   [cmr.elastic-utils.config :as es-config]
    [cmr.elastic-utils.embedded-elastic-server :as elastic-server]
    [cmr.indexer.config :as indexer-config]
    [cmr.indexer.system :as indexer-system]
@@ -100,46 +100,24 @@
 (def in-memory-elastic-log-level-atom
   (atom :info))
 
-(defmulti create-gran-elastic-server
-  "Sets elastic configuration values and returns an instance of an Elasticsearch component representing the granule cluster to run
-  in memory if applicable. A granule elasticsearch cluster will hold only the granule and deleted granule indexes."
-  (fn [type]
-    type))
-
-(defmethod create-gran-elastic-server :in-memory
-  [_]
-  (let [http-port (elastic-config/gran-elastic-port)]
-    (elastic-server/create-server http-port
-                                  {:log-level (name @in-memory-elastic-log-level-atom)
-                                   :kibana-port (dev-config/embedded-kibana-gran-port)
-                                   :image-cfg {"Dockerfile" "elasticsearch/Dockerfile.elasticsearch"
-                                               "es_libs" "elasticsearch/es_libs"
-                                               "embedded-security.policy" "elasticsearch/embedded-security.policy"
-                                               "plugins" "elasticsearch/plugins"}})))
-
-(defmethod create-gran-elastic-server :external
-  [_]
-  nil)
-
 (defmulti create-elastic-server
-          "Sets elastic configuration values and returns an instance of an Elasticsearch component representing the non-granule cluster to run
-          in memory if applicable. This non-granule cluster will hold all CMR concept indexes that are NOT the granule or deleted granule indexes."
-          (fn [type]
-            type))
+  "Sets elastic configuration values and returns an instance of an Elasticsearch component representing the non-granule cluster to run
+  in memory if applicable. This non-granule cluster will hold all CMR concept indexes that are NOT the granule or deleted granule indexes."
+  (fn [type elastic-port kibana-port]
+    type))
 
 (defmethod create-elastic-server :in-memory
-  [_]
-  (let [http-port (elastic-config/elastic-port)]
-    (elastic-server/create-server http-port
-                                  {:log-level (name @in-memory-elastic-log-level-atom)
-                                   :kibana-port (dev-config/embedded-kibana-port)
-                                   :image-cfg {"Dockerfile" "elasticsearch/Dockerfile.elasticsearch"
-                                               "es_libs" "elasticsearch/es_libs"
-                                               "embedded-security.policy" "elasticsearch/embedded-security.policy"
-                                               "plugins" "elasticsearch/plugins"}})))
+  [type elastic-port kibana-port]
+  (elastic-server/create-server elastic-port
+                                {:log-level (name @in-memory-elastic-log-level-atom)
+                                 :kibana-port kibana-port
+                                 :image-cfg {"Dockerfile" "elasticsearch/Dockerfile.elasticsearch"
+                                             "es_libs" "elasticsearch/es_libs"
+                                             "embedded-security.policy" "elasticsearch/embedded-security.policy"
+                                             "plugins" "elasticsearch/plugins"}}))
 
 (defmethod create-elastic-server :external
-  [_]
+  [type elastic-port kibana-port]
   nil)
 
 (defmulti create-redis
@@ -307,8 +285,8 @@
         db-component (create-db db)
         echo-component (create-echo echo)
         queue-broker (create-queue-broker message-queue)
-        gran-elastic-server (create-gran-elastic-server elastic)
-        elastic-server (create-elastic-server elastic)
+        gran-elastic-server (create-elastic-server elastic (es-config/gran-elastic-port) (dev-config/embedded-kibana-gran-port))
+        elastic-server (create-elastic-server elastic (es-config/elastic-port) (dev-config/embedded-kibana-port))
         redis-server (create-redis redis)
         sqs-server (create-sqs-server sqs-server)
         control-server (control/create-server)]

elastic-utils-lib/src/cmr/elastic_utils/config.clj

@@ -5,6 +5,12 @@
    [cmr.common.config :refer [defconfig]]
    [cmr.common.services.errors :as errors]))
 
+(declare split-cluster-log-toggle)
+(defconfig split-cluster-log-toggle
+  "This is the temporary feature toggle that will turn on and off the split cluster logs. True means on."
+  {:default false
+   :type Boolean})
+
 (declare elastic-name)
 (def elastic-name
   "Defining the elastic cluster name for non-gran elastic cluster.

elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj

@@ -16,7 +16,8 @@
   "Performs a search query across one or more indexes and one or more mapping types"
   [conn index _mapping-type opts]
   ;; Temporarily putting in this log to check which indexes are going to which cluster during ES cluster split
-  (info "CMR-10600 ES search for index: " index " connected to " conn)
+  (when (es-config/split-cluster-log-toggle)
+    (info "CMR-10600 ES search for index: " index " connected to " conn))
   (let [qk [:search_type :scroll :routing :preference :ignore_unavailable]
         qp (merge {:track_total_hits true}
                   (select-keys opts qk))

indexer-app/src/cmr/indexer/data/elasticsearch.clj

@@ -442,7 +442,8 @@
   (doseq [es-index es-indexes]
     (let [conn (get-es-cluster-conn context es-index)
           ;; Temporarily putting in this log to check which indexes are going to which cluster during ES cluster split
-          _ (info "CMR-10600 Saving doc for concept " concept-id " under index " es-index " in es cluster " conn)
+          _ (when (es-config/split-cluster-log-toggle)
+              (info "CMR-10600 Saving doc for concept " concept-id " under index " es-index " in es cluster " conn))
           {:keys [ignore-conflict? all-revisions-index?]} options
           elastic-id (get-elastic-id concept-id revision-id all-revisions-index?)
           result (try-elastic-operation
@@ -462,7 +463,8 @@
   [context es-index es-type elastic-id]
   (let [es-cluster-name (es-index/get-es-cluster-name-from-concept-id elastic-id)]
     ;; Temporarily putting in this log to check which indexes are going to which cluster during ES cluster split
-    (info "CMR-10600 Get document of index " es-index " from ES cluster " es-cluster-name)
+    (when (es-config/split-cluster-log-toggle)
+      (info "CMR-10600 Get document of index " es-index " from ES cluster " es-cluster-name))
     (es-helper/doc-get (indexer-util/context->conn context es-cluster-name) es-index es-type elastic-id)))
 
 (defn delete-document
@@ -474,7 +476,8 @@
      ;; Cannot use elasticsearch for deletion as we require special headers on delete
      (let [es-cluster-name (es-index/get-es-cluster-name-from-concept-id concept-id)
            ;; Temporarily putting in this log to check which indexes are going to which cluster during ES cluster split
-           _ (info "CMR-10600 Deleting concept " concept-id " with index " es-index " in ES cluster " es-cluster-name)
+           _ (when (es-config/split-cluster-log-toggle)
+               (info "CMR-10600 Deleting concept " concept-id " with index " es-index " in ES cluster " es-cluster-name))
            {:keys [admin-token]} (context->es-config context es-cluster-name)
            {:keys [uri http-opts]} (indexer-util/context->conn context es-cluster-name)
            {:keys [ignore-conflict? all-revisions-index?]} options

indexer-app/src/cmr/indexer/data/index_set_elasticsearch.clj

@@ -120,7 +120,8 @@
   This func should only be called once during the first time we transition to the split cluster.
   It may be deleted after the transition is determined to be successful and permanent with the ticket CMR-10949."
   [context es-cluster-name index-set-id]
-  (info "CMR 10600 Getting old index set to copy over to new split cluster index set indexes.")
+  (when (es-config/split-cluster-log-toggle)
+    (info "CMR-10600 Getting old index set to copy over to new split cluster index set indexes."))
   (let [es-cluster-name-keyword (es-config/elastic-name-str->keyword es-cluster-name)
         {:keys [mapping]} (config/idx-cfg-for-index-sets es-cluster-name)
         idx-mapping-type (first (keys mapping))]