use DB_TYPE=postgres by default (DB_TYPE=none for no db at all)

guymguym · guymguym · commit a1974b8a13f6 · 2023-05-18T19:03:51.000+03:00
Signed-off-by: Guy Margalit &lt;guymguym@gmail.com&gt;
diff --git a/.dockerignore b/.dockerignore
@@ -5,7 +5,6 @@ coverage/
 logs/
 *.log
 storage/
-noobaa_storage/
 heapdump-*
 .vscode/
 *.IGNORE
diff --git a/.gitignore b/.gitignore
@@ -16,9 +16,6 @@ config-local.js
 
 ## STORAGE
 /storage/
-/agent_storage*
-/noobaa_storage*
-/agent_conf*.json*
 
 ## DEVELOPER
 .vscode
diff --git a/Makefile b/Makefile
@@ -111,7 +111,7 @@ test: tester
 	@$(call run_mongo)
 	@$(call run_blob_mock)
 	@echo "\033[1;34mRunning tests\033[0m"
-	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG)
+	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG)
 	@$(call stop_noobaa)
 	@$(call stop_blob_mock)
 	@$(call stop_mongo)
@@ -123,7 +123,7 @@ root-perm-test: tester
 	@$(call create_docker_network)
 	@$(call run_mongo)
 	@echo "\033[1;34mRunning root permission tests\033[0m"
-	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s sudo_index.js
+	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s sudo_index.js
 	@$(call stop_noobaa)
 	@$(call stop_mongo)
 	@$(call remove_docker_network)
@@ -135,7 +135,7 @@ run-single-test: tester
 	@$(call run_mongo)
 	@$(call run_blob_mock)
 	@echo "\033[1;34mRunning tests\033[0m"
-	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname)
+	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname)
 	@$(call stop_noobaa)
 	@$(call stop_blob_mock)
 	@$(call stop_mongo)
diff --git a/config.js b/config.js
@@ -4,7 +4,7 @@
 // we populate the exports object but prefer to keep name referencing
 // with config.NAME so that it will be consistent with the code that imports it
 // and will make searching easier.
-var config = exports;
+const config = exports;
 
 const os = require('os');
 const fs = require('fs');
@@ -401,7 +401,7 @@ config.LONG_DB_QUERY_THRESHOLD = parseInt(process.env.LONG_DB_QUERY_THRESHOLD, 1
   Central Stats Collection & Diagnostics
 */
 
-var is_windows = (process.platform === "win32");
+const is_windows = (process.platform === 'win32');
 
 if (!is_windows) {
     process.env.ProgramData = '/tmp';
@@ -535,7 +535,7 @@ config.WORM_ENABLED = false;
 // Should we allow the creation of buckets on internal storage
 config.ALLOW_BUCKET_CREATE_ON_INTERNAL = true;
 
-config.DB_TYPE = process.env.DB_TYPE || 'mongodb';
+config.DB_TYPE = /** @type {nb.DBType} */ (process.env.DB_TYPE || 'postgres');
 
 ////////////////////////////////
 //      NAMESPACE MONITOR     //
diff --git a/docs/standalone.md b/docs/standalone.md
@@ -27,7 +27,8 @@ git clone https://github.com/noobaa/noobaa-core
 cd noobaa-core
 npm install
 npm run build
-npm run pkg # optional create a single-executable build/noobaa-core
+# optional package everything into a single-executable at build/noobaa-core
+npm run pkg 
 ```
 
 ### 3. Quick test
@@ -107,6 +108,7 @@ config.IO_CALC_SHA256_ENABLED = false;
 
 config.MAX_OBJECT_PART_SIZE = 1024 * 1024 * 1024;
 config.IO_CHUNK_READ_CACHE_SIZE = 4 * 1024 * 1024 * 1024;
+
 config.CHUNK_SPLIT_AVG_CHUNK = 256 * 1024 * 1024;
 config.CHUNK_SPLIT_DELTA_CHUNK = 0;
 
@@ -118,7 +120,7 @@ config.CHUNK_CODER_CIPHER_TYPE = 'none';
 config.CHUNK_CODER_REPLICAS = 1;
 config.CHUNK_CODER_EC_DATA_FRAGS = 2;
 config.CHUNK_CODER_EC_PARITY_FRAGS = 2;
-config.CHUNK_CODER_EC_PARITY_TYPE = 'isa-c1';
+config.CHUNK_CODER_EC_PARITY_TYPE = 'cm256';
 config.CHUNK_CODER_EC_TOLERANCE_THRESHOLD = 2;
 config.CHUNK_CODER_EC_IS_DEFAULT = true;
 
@@ -173,6 +175,12 @@ POSTGRES_HOST=ip \
 
 ## STORAGE
 
+### Create a pool
+
+```sh
+npm run api -- pool_api create_hosts_pool '{ "name":"backingstores", "is_managed": false, "host_count": 9999 }'
+```
+
 ### Start backingstores
 
 ```sh
@@ -193,7 +201,7 @@ npm run api -- node aggregate_nodes '{}'
 ### Check local storage
 
 ```sh
-du -sh storage/backingstores
+du -sh storage/backingstores/*
 find storage/backingstores -name '*.data' -type f -ls
 ```
 
@@ -204,51 +212,51 @@ find storage/backingstores -name '*.data' -type f -ls
 ### Get access and secret keys
 
 ```sh
-export AWS_ACCESS_KEY_ID=$(npm run api -- account read_account '{}' --json | jq -r '.access_keys[0].access_key')
-export AWS_SECRET_ACCESS_KEY=$(npm run api -- account read_account '{}' --json | jq -r '.access_keys[0].secret_key')
+export AWS_ACCESS_KEY_ID=$(npm run api -- account read_account '{}' --json | tail -1 | jq -r '.access_keys[0].access_key')
+export AWS_SECRET_ACCESS_KEY=$(npm run api -- account read_account '{}' --json | tail -1 | jq -r '.access_keys[0].secret_key')
 ```
 
-### Listing
+### Create bucket
 
 ```sh
-node src/tools/s3cat --endpoint http://localhost:6001
-node src/tools/s3cat --endpoint http://localhost:6001 --bucket first.bucket --ls
-aws --endpoint http://localhost:6001 s3 ls
-aws --endpoint http://localhost:6001 s3 ls s3://first.bucket
+aws --endpoint http://localhost:6001 s3 mb s3://testbucket
 ```
 
-### Create bucket
+### Listing
 
 ```sh
-aws --endpoint http://localhost:6001 s3 mb s3://lala
+node src/tools/s3cat --endpoint http://localhost:6001
+node src/tools/s3cat --endpoint http://localhost:6001 --bucket testbucket --ls
+aws --endpoint http://localhost:6001 s3 ls
+aws --endpoint http://localhost:6001 s3 ls s3://testbucket
 ```
 
 ### Read/Write
 
 ```sh
-node src/tools/s3cat --endpoint http://localhost:6001 --sig s3 --bucket first.bucket --put ggg --size 4096
-node src/tools/s3cat --endpoint http://localhost:6001 --sig s3 --bucket first.bucket --get ggg
-dd if=/dev/zero bs=1M count=1024 | aws --endpoint http://localhost:6001 s3 cp - s3://first.bucket/ggg
-aws --endpoint http://localhost:6001 s3 cp s3://first.bucket/ggg - | xxd -a
-aws --endpoint http://localhost:6001 s3 rm s3://first.bucket/ggg
+node src/tools/s3cat --endpoint http://localhost:6001 --sig s3 --bucket testbucket --put testobject --size 4096
+node src/tools/s3cat --endpoint http://localhost:6001 --sig s3 --bucket testbucket --get testobject
+dd if=/dev/zero bs=1M count=1024 | aws --endpoint http://localhost:6001 s3 cp - s3://testbucket/testobject
+aws --endpoint http://localhost:6001 s3 cp s3://testbucket/testobject - | xxd -a
+aws --endpoint http://localhost:6001 s3 rm s3://testbucket/testobject
 ```
 
 ## Multipart uploads
 
 ```sh
-node src/tools/s3cat --endpoint http://localhost:6001 --sig s3 --bucket first.bucket --upload ggg --size 4096 --part_size 1024 --concur 4 
+node src/tools/s3cat --endpoint http://localhost:6001 --sig s3 --bucket testbucket --upload testobject --size 4096 --part_size 1024 --concur 4 
 ```
 
 ### Perf tools
 
 ```sh
-node src/tools/s3perf --endpoint http://localhost:6001 --sig s3 --bucket first.bucket --put s3perf/ --concur 4 --size 128 --size_units MB --time 5
-node src/tools/s3perf --endpoint http://localhost:6001 --sig s3 --bucket first.bucket --get s3perf/ --concur 4 --size 128 --size_units MB --time 5
+node src/tools/s3perf --endpoint http://localhost:6001 --sig s3 --bucket testbucket --put s3perf/ --concur 4 --size 128 --size_units MB --time 5
+node src/tools/s3perf --endpoint http://localhost:6001 --sig s3 --bucket testbucket --get s3perf/ --concur 4 --size 128 --size_units MB --time 5
 ```
 
 ### Using sigv4 for streaming requires https endpoint 6443 (selfsigned)
 
 ```sh
-node src/tools/s3cat --endpoint https://localhost:6443 --selfsigned --bucket first.bucket --put ggg --size 4096
-node src/tools/s3cat --endpoint https://localhost:6443 --selfsigned --bucket first.bucket --upload ggg --size 4096 --part_size 1024 --concur 4
+node src/tools/s3cat --endpoint https://localhost:6443 --selfsigned --bucket testbucket --put testobject --size 4096
+node src/tools/s3cat --endpoint https://localhost:6443 --selfsigned --bucket testbucket --upload testobject --size 4096 --part_size 1024 --concur 4
 ```
diff --git a/package.json b/package.json
@@ -30,7 +30,6 @@
     "cmd": "node --unhandled-rejections=warn src/cmd",
     "api": "node --unhandled-rejections=warn src/cmd api",
     "nsfs": "node --unhandled-rejections=warn src/cmd nsfs",
-    "nscache": "node --unhandled-rejections=warn src/cmd nscache",
     "backingstore": "node --unhandled-rejections=warn src/cmd/backingstore",
     "-----": "----------------------------------------------------------------",
     "web": "node --unhandled-rejections=warn src/server/web_server",
@@ -41,8 +40,8 @@
     "agent": "node --unhandled-rejections=warn src/agent/agent_cli",
     "agents": "node --unhandled-rejections=warn src/agent/agent_cli --scale 10",
     "------": "---------------------------------------------------------------",
-    "db": "postgres -D storage/postgres",
-    "db:init": "initdb -D storage/postgres -E UTF-8 --locale=C",
+    "db": "postgres -D ./storage/postgres",
+    "db:init": "initdb -D ./storage/postgres -E UTF-8 --locale=C",
     "db:create": "npm run db:createuser && npm run db:createdb",
     "db:createuser": "createuser postgres && psql postgres -c 'ALTER USER postgres CREATEDB'",
     "db:createdb": "createdb -U postgres nbcore",
@@ -54,7 +53,7 @@
     "clean:build": "rm -rf ./build",
     "clean:test": "rm -rf ./coverage ./logs",
     "clean:garbage": "find . -name .DS_Store -print -delete",
-    "clean:storage": "rm -rf ./storage/* ./noobaa_storage* ./agent_conf*.json"
+    "clean:storage": "rm -rf ./storage"
   },
   "config": {
     "covdir": "coverage"
diff --git a/src/cmd/api.js b/src/cmd/api.js
@@ -2,6 +2,7 @@
 'use strict';
 
 require('../util/dotenv').load();
+require('aws-sdk/lib/maintenance_mode_message').suppress = true;
 
 const dbg = require('../util/debug_module')(__filename);
 if (!dbg.get_process_name()) dbg.set_process_name('nsfs');
@@ -51,6 +52,7 @@ function print_usage() {
 
 async function main(argv = minimist(process.argv.slice(2))) {
     try {
+        dbg.original_console();
         if (argv.help || argv.h) return print_usage();
         if (argv.debug) {
             const debug_level = Number(argv.debug) || 5;
diff --git a/src/cmd/index.js b/src/cmd/index.js
@@ -3,6 +3,7 @@
 
 // load envs first
 require('../util/dotenv').load();
+require('aws-sdk/lib/maintenance_mode_message').suppress = true;
 
 // set the console output to raw
 const dbg = require('../util/debug_module')(__filename);
diff --git a/src/cmd/nsfs.js b/src/cmd/nsfs.js
@@ -2,18 +2,20 @@
 'use strict';
 
 require('../util/dotenv').load();
+require('aws-sdk/lib/maintenance_mode_message').suppress = true;
 
 const dbg = require('../util/debug_module')(__filename);
 if (!dbg.get_process_name()) dbg.set_process_name('nsfs');
 dbg.original_console();
 
 const config = require('../../config');
-config.NSFS_VERSIONING_ENABLED = true;
 
 const fs = require('fs');
 const util = require('util');
 const minimist = require('minimist');
 
+require('../server/system_services/system_store').get_instance({ standalone: true });
+
 const nb_native = require('../util/nb_native');
 const ObjectSDK = require('../sdk/object_sdk');
 const NamespaceFS = require('../sdk/namespace_fs');
@@ -75,6 +77,9 @@ function print_usage() {
 
 async function main(argv = minimist(process.argv.slice(2))) {
     try {
+        config.DB_TYPE = 'none';
+        config.NSFS_VERSIONING_ENABLED = true;
+
         if (argv.help || argv.h) return print_usage();
         if (argv.debug) {
             const debug_level = Number(argv.debug) || 5;
diff --git a/src/deploy/NVA_build/noobaa_core.yaml b/src/deploy/NVA_build/noobaa_core.yaml
@@ -142,6 +142,8 @@ spec:
             - mountPath: /log
               name: logdir
           env:
+            - name: DB_TYPE
+              value: mongodb
             - name: CONTAINER_PLATFORM
               value: KUBERNETES
             - name: ENDPOINT_FORKS_NUMBER
@@ -247,6 +249,8 @@ spec:
             - containerPort: 6001
             - containerPort: 6443
           env:
+            - name: DB_TYPE
+              value: mongodb
             - name: JWT_SECRET
               valueFrom:
                 secretKeyRef:
diff --git a/src/hosted_agents/hosted_agents.js b/src/hosted_agents/hosted_agents.js
@@ -103,7 +103,7 @@ class HostedAgents {
     async _start_pool_agent(pool) {
         if (!this._started) return;
         if (!pool) throw new Error(`Internal error: received pool ${pool}`);
-        if (config.DB_TYPE === 'postgres' && pool.resource_type === 'INTERNAL') return;
+        if (config.DB_TYPE !== 'mongodb' && pool.resource_type === 'INTERNAL') return;
         dbg.log0(`_start_pool_agent for pool ${pool.name}`);
         const pool_id = String(pool._id);
         const node_name = 'noobaa-internal-agent-' + pool_id;
diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts
@@ -662,6 +662,7 @@ interface APIClient {
  *
  **********************************************************/
 
+type DBType = 'postgres' | 'mongodb' | 'none';
 
 interface DBClient {
     operators: Set<string>;
diff --git a/src/server/object_services/map_builder.js b/src/server/object_services/map_builder.js
@@ -256,7 +256,7 @@ class MapBuilder {
         // check if bucket master_key is enabled and chunk master keys disabled
         } else if (!chunk_master_key_id && bucket_m_key && bucket_m_key.disabled === false) {
             // TODO: Or we can check if chunk.cipher_key instanceof mongodb.Binary
-            const cipher_key_buffer = config.DB_TYPE === 'postgres' ? chunk.cipher_key : chunk.cipher_key.buffer;
+            const cipher_key_buffer = config.DB_TYPE === 'mongodb' ? chunk.cipher_key.buffer : chunk.cipher_key;
             const encrypted_cipher = mkm.encrypt_buffer_with_master_key_id(cipher_key_buffer, bucket_m_key._id);
             await MDStore.instance().update_chunk_by_id(chunk._id,
                 { cipher_key: encrypted_cipher, master_key_id: bucket_m_key._id });
diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js
@@ -546,7 +546,7 @@ class MDStore {
         const sort = { bucket: 1, key: 1 };
 
         // for mongodb add version_past to the sort
-        if (process.env.DB_TYPE === 'mongodb') {
+        if (config.DB_TYPE === 'mongodb') {
             sort.version_past = 1;
         }
 
diff --git a/src/server/object_services/schemas/object_md_indexes.js b/src/server/object_services/schemas/object_md_indexes.js
@@ -1,6 +1,8 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
+const config = require('../../../../config');
+
 module.exports = [
 
     // TODO index ??? find_objects() not indexed for the create_time
@@ -13,7 +15,7 @@ module.exports = [
             key: 1,
             // For MONGO deployments - we include version_past as extra index field to separate from null_version_index.
             // note that version_past is always null here by partialFilterExpression.
-            ...(process.env.DB_TYPE === 'postgres' ? {} : { version_past: 1 })
+            ...(config.DB_TYPE === 'mongodb' ? { version_past: 1 } : {}),
         },
         options: {
             name: 'latest_version_index',
@@ -38,7 +40,7 @@ module.exports = [
             key: 1,
             // For MONGO deployments - we include version_enabled as extra index field to separate from latest_version_index.
             // note that version_enabled is always null here by partialFilterExpression.
-            ...(process.env.DB_TYPE === 'postgres' ? {} : { version_enabled: 1 })
+            ...(config.DB_TYPE === 'mongodb' ? { version_enabled: 1 } : {}),
         },
         options: {
             name: 'null_version_index',
diff --git a/src/server/system_services/cluster_server.js b/src/server/system_services/cluster_server.js
@@ -41,7 +41,7 @@ const VERIFY_RESPONSE = [
 ];
 
 async function _init() {
-    if (config.DB_TYPE === 'postgres') return;
+    if (config.DB_TYPE !== 'mongodb') return;
     return MongoCtrl.init();
 }
 
diff --git a/src/server/system_services/pool_server.js b/src/server/system_services/pool_server.js
@@ -468,9 +468,9 @@ function create_mongo_pool(req) {
     const name = req.rpc_params.name;
     const mongo_info = {};
 
-    if (config.DB_TYPE === 'postgres') {
-        dbg.error('Cannot create mongo pool on PostgreSQL');
-        throw new Error('Cannot create mongo pool on PostgreSQL');
+    if (config.DB_TYPE !== 'mongodb') {
+        dbg.error(`Cannot create mongo pool with DB_TYPE=${config.DB_TYPE}`);
+        throw new Error(`Cannot create mongo pool with DB_TYPE=${config.DB_TYPE}`);
     }
 
     if (get_internal_mongo_pool(req.system)) {
diff --git a/src/server/system_services/system_server.js b/src/server/system_services/system_server.js
@@ -934,7 +934,7 @@ async function _ensure_internal_structure(system_id) {
     const support_account = _.find(system_store.data.accounts, account => account.is_support);
     if (!support_account) throw new Error('SUPPORT ACCOUNT DOES NOT EXIST');
     // Skip creation of agent on PostgreSQL
-    if (config.DB_TYPE === 'postgres') return;
+    if (config.DB_TYPE !== 'mongodb') return;
     try {
         server_rpc.client.hosted_agents.create_pool_agent({
             pool_name: `${config.INTERNAL_STORAGE_POOL_NAME}-${system_id}`
diff --git a/src/test/unit_tests/test_lifecycle.js b/src/test/unit_tests/test_lifecycle.js
diff --git a/src/test/unit_tests/test_mdsequence.js b/src/test/unit_tests/test_mdsequence.js
diff --git a/src/test/unit_tests/test_system_servers.js b/src/test/unit_tests/test_system_servers.js
diff --git a/src/tools/s3cat.js b/src/tools/s3cat.js
diff --git a/src/tools/s3perf.js b/src/tools/s3perf.js
diff --git a/src/util/db_client.js b/src/util/db_client.js
diff --git a/src/util/mongo_client.js b/src/util/mongo_client.js
diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js
