Merge pull request #7216 from liranmauda/liran-move-md5-functions-into-utils

Adding crypto_utils

Author: liranmauda

Diff for: src/test/utils/s3ops.js

@@ -7,9 +7,10 @@ const util = require('util');
 const https = require('https');
 const stream = require('stream');
 const crypto = require('crypto');
+const querystring = require('querystring');
 const P = require('../../util/promise');
 const RandStream = require('../../util/rand_stream');
-const querystring = require('querystring');
+const crypto_utils = require('../../util/crypto_utils');
 
 require('../../util/dotenv').load();
 
@@ -78,7 +79,7 @@ class S3OPS {
             data_size = data_size || 50;
 
             const actual_size = data_size * multiplier;
-            const md5_stream = new_md5_stream();
+            const md5_stream = crypto_utils.new_md5_stream();
             const input_stream = new RandStream(actual_size, {
                 highWaterMark: 1024 * 1024,
             });
@@ -175,7 +176,7 @@ class S3OPS {
                 VersionId: versionid ? versionid : undefined,
             });
             const read_stream = req.createReadStream();
-            const md5_stream = new_md5_stream();
+            const md5_stream = crypto_utils.new_md5_stream();
             read_stream.pipe(md5_stream).pipe(target_stream);
             await new Promise((resolve, reject) => {
                 read_stream.once('error', reject);
@@ -774,7 +775,7 @@ class S3OPS {
     async _multipart_upload_internal(bucket, file_name, data, part_size, overlook_error) {
         try {
             bucket = bucket || 'first.bucket';
-            const md5_stream = new_md5_stream();
+            const md5_stream = crypto_utils.new_md5_stream();
             const start_ts = Date.now();
 
             let body;
@@ -822,18 +823,4 @@ class S3OPS {
 
 }
 
-function new_md5_stream() {
-    const md5_stream = new stream.Transform({
-        transform(buf, encoding, next) {
-            this.md5.update(buf);
-            this.size += buf.length;
-            this.push(buf);
-            next();
-        }
-    });
-    md5_stream.md5 = crypto.createHash('md5');
-    md5_stream.size = 0;
-    return md5_stream;
-}
-
 exports.S3OPS = S3OPS;

Diff for: src/util/crypto_utils.js

@@ -0,0 +1,40 @@
+/* Copyright (C) 2023 NooBaa */
+'use strict';
+
+const crypto = require('crypto');
+const stream = require('stream');
+
+function new_md5_stream() {
+    const md5_stream = new stream.Transform({
+        transform(buf, encoding, next) {
+            this.md5.update(buf);
+            this.size += buf.length;
+            next(null, buf);
+        }
+    });
+
+    md5_stream.md5 = crypto.createHash('md5');
+    md5_stream.size = 0;
+    md5_stream.md5_buf = null;
+
+    return md5_stream;
+}
+
+function calc_body_md5(stream_file) {
+    const md5_stream = new_md5_stream();
+    const new_stream = stream_file.pipe(md5_stream);
+    return {
+        new_stream,
+        md5_buf: () => {
+            if (md5_stream.md5_buf) return md5_stream.md5_buf;
+
+            const final_md5 = md5_stream.md5.digest('hex');
+            md5_stream.md5_buf = Buffer.from(final_md5, 'hex');
+
+            return md5_stream.md5_buf;
+        }
+    };
+}
+
+exports.new_md5_stream = new_md5_stream;
+exports.calc_body_md5 = calc_body_md5;

Diff for: src/util/google_storage_wrap.js

@@ -4,6 +4,7 @@
 const { Util } = require('@google-cloud/storage/build/src/nodejs-common/util');
 const pkg = require('../../package.json');
 const DEV_MODE = (process.env.DEV_MODE === 'true');
+const crypto_utils = require('./crypto_utils');
 
 let stage_or_prod = 'production';
 if (DEV_MODE) {
@@ -12,4 +13,7 @@ if (DEV_MODE) {
 Util.prototype.getUserAgentFromPackageJson = () => `NooBaa/${pkg.version} (GPN:noobaa.com; ${stage_or_prod}) NooBaa/${pkg.version}`;
 
 const { Storage } = require('@google-cloud/storage');
+
+Storage.calc_body_md5 = stream_file => crypto_utils.calc_body_md5(stream_file);
+
 module.exports = Storage;

Diff for: src/util/new_azure_storage_wrap.js

@@ -1,46 +1,15 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
-const crypto = require('crypto');
-const stream = require('stream');
+const crypto_utils = require('./crypto_utils');
 const azure_storage = require('@azure/storage-blob');
 // needed only for enerateBlockIdPrefix() and get_block_id() functions
 const old_azure_storage = require('azure-storage');
 
-function new_md5_stream() {
-    const md5_stream = new stream.Transform({
-        transform(buf, encoding, next) {
-            this.md5.update(buf);
-            this.size += buf.length;
-            next(null, buf);
-        }
-    });
-
-    md5_stream.md5 = crypto.createHash('md5');
-    md5_stream.size = 0;
-    md5_stream.md5_buf = null;
-
-    return md5_stream;
-}
-
 azure_storage.get_container_client = (blob_service, container) => blob_service.getContainerClient(container);
 
 azure_storage.get_blob_client = (container_client, blob) => container_client.getBlobClient(blob).getBlockBlobClient();
 
-azure_storage.calc_body_md5 = stream_file => {
-    const md5_stream = new_md5_stream();
-    const new_stream = stream_file.pipe(md5_stream);
-    return {
-            new_stream,
-            md5_buf: () => {
-                if (md5_stream.md5_buf) return md5_stream.md5_buf;
-
-                const final_md5 = md5_stream.md5.digest('hex');
-                md5_stream.md5_buf = Buffer.from(final_md5, 'hex');
-
-                return md5_stream.md5_buf;
-            }
-    };
-};
+azure_storage.calc_body_md5 = stream_file => crypto_utils.calc_body_md5(stream_file);
 
 // create old lib blob service - needed for functions that do not exist in the new lib
 // needed only for generateBlockIdPrefix() and get_block_id() functions