lifecycle | unit test

Signed-off-by: naveenpaul1 <[email protected]>

Author: naveenpaul1

src/server/object_services/md_store.js

@@ -431,11 +431,12 @@ class MDStore {
         WHERE ctid in (
             SELECT ctid
             FROM ${table_name} t1
-            WHERE t1.data->>'bucket' = '${bucket_id}'
-                AND t1.data->>'deleted' IS NULL
-                AND t1.data->>'delete_marker' IS NOT NULL
-                AND t1.data->>'version_past' IS NULL
-                AND NOT EXISTS (
+            WHERE ${sql_and_conditions(
+                `t1.data->>'bucket' = '${bucket_id}'`,
+                `t1.data->>'deleted' IS NULL`,
+                `t1.data->>'delete_marker' IS NOT NULL`,
+                `t1.data->>'version_past' IS NULL`,
+                `NOT EXISTS (
                     SELECT 1
                     FROM ${table_name} t2
                     WHERE t2.data->>'key' = t1.data->>'key'
@@ -444,8 +445,9 @@ class MDStore {
                             t2.data->>'deleted' IS NULL -- is not deleted
                             OR t2.data->>'delete_marker' IS NOT NULL -- is a delete marker
                         )
-                )
-                ${sql_and_conditions(sql_condition0, sql_condition1, sql_condition2, sql_condition3)}
+                )`,
+                sql_condition0, sql_condition1, sql_condition2, sql_condition3
+            )} 
             ${sql_limit}
         );`;
 

src/test/lifecycle/common.js

@@ -27,6 +27,7 @@ function marker_lifecycle_configuration(Bucket, Key) {
         },
     };
 }
+exports.marker_lifecycle_configuration = marker_lifecycle_configuration;
 
 function empty_filter_marker_lifecycle_configuration(Bucket) {
     return {
@@ -390,7 +391,7 @@ function duplicate_id_lifecycle_configuration(Bucket, Key) {
     };
 }
 
-function version_lifecycle_configuration(Bucket, Key, Days, ExpiredDeleteMarker, NewnonCurrentVersion, NonCurrentDays) {
+function version_lifecycle_configuration(Bucket, Key, Days, NewnonCurrentVersion, NonCurrentDays) {
     const ID = 'rule_id';
     return {
         Bucket,
@@ -402,7 +403,6 @@ function version_lifecycle_configuration(Bucket, Key, Days, ExpiredDeleteMarker,
                 },
                 Expiration: {
                     Days: Days,
-                    ExpiredObjectDeleteMarker: ExpiredDeleteMarker,
                 },
                 NoncurrentVersionExpiration: {
                     NewerNoncurrentVersions: NewnonCurrentVersion,
@@ -467,7 +467,7 @@ exports.test_multipart = async function(Bucket, Key, s3) {
 };
 
 exports.test_version = async function(Bucket, Key, s3) {
-    const putLifecycleParams = version_lifecycle_configuration(Bucket, Key, 10, true, 5, 10);
+    const putLifecycleParams = version_lifecycle_configuration(Bucket, Key, 10, 5, 10);
     const getLifecycleResult = await put_get_lifecycle_configuration(Bucket, putLifecycleParams, s3);
 
     const expirationDays = getLifecycleResult.Rules[0].Expiration.Days;

src/test/unit_tests/test_lifecycle.js

@@ -37,13 +37,15 @@ const TagValue2 = 'tagvalue2';
 const object_io = new ObjectIO();
 object_io.set_verification_mode();
 
-
+// eslint-disable-next-line max-lines-per-function
 mocha.describe('lifecycle', () => {
 
     let s3;
     mocha.before(async function() {
         this.timeout(60000);
 
+        config.LIFECYCLE_SCHEDULE_MIN = 0;
+        config.LIFECYCLE_INTERVAL = 0;
         const account_info = await rpc_client.account.read_account({ email: EMAIL, });
         s3 = new S3({
             endpoint: coretest.get_http_address(),
@@ -347,9 +349,6 @@ mocha.describe('lifecycle', () => {
             const size = num_parts * part_size;
             const data = generator.update(Buffer.alloc(size));
             const { obj_id } = await rpc_client.object.create_object_upload({ bucket, key, content_type });
-            const mp_list_before = await rpc_client.object.list_multiparts({ obj_id, bucket, key });
-            coretest.log('list_multiparts before', mp_list_before);
-            assert.strictEqual(mp_list_before.multiparts.length, 0);
             const multiparts_ids = [];
 
             const get_part_slice = i => data.slice(i * part_size, (i + 1) * part_size);
@@ -381,15 +380,65 @@ mocha.describe('lifecycle', () => {
             await MDStore.instance().update_object_by_id(obj_id, update);
 
             const mp_list_after = await rpc_client.object.list_multiparts({ obj_id, bucket, key });
-            coretest.log('mp_list_after after', mp_list_after);
-            assert.strictEqual(mp_list_after.multiparts.length, num_parts);
+            assert.strictEqual(mp_list_after.multiparts.length, num_parts,
+                        `list_multiparts actual ${mp_list_after.multiparts.length} !== ${num_parts}`);
 
             return obj_id;
         }
 
-        mocha.it('lifecyle - listMultiPart verify', async () => {
-            await create_mock_multipart_upload('test-lifecycle-multipart', multipart_bucket, 3, 45, 7);
+        mocha.it('lifecycle - delete multipart after 30 days', async () => {
+            const days = 30;
+            const multi_bucket_key = 'test-lifecycle-multipart1';
+            const obj_id = await create_mock_multipart_upload(multi_bucket_key, multipart_bucket, days, 45, 7);
+            const putLifecycleParams = commonTests.multipart_lifecycle_configuration(multipart_bucket, multi_bucket_key, days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            await verify_multipart_deleted(obj_id, multi_bucket_key, 0);
+        });
+
+        mocha.it('lifecycle - should not delete multipart after 30 days, before expiration', async () => {
+            const days = 30;
+            const multi_bucket_key = 'test-lifecycle-multipart2';
+            // create_time updated to 29 days and expire is 30 days, do not delete any multipart
+            const obj_id = await create_mock_multipart_upload(multi_bucket_key, multipart_bucket, days - 1, 45, 7);
+            const putLifecycleParams = commonTests.multipart_lifecycle_configuration(multipart_bucket, multi_bucket_key, days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            await verify_multipart_deleted(obj_id, multi_bucket_key, 7);
         });
+
+        mocha.it('lifecycle - delete multipart after 30 days, with prfix', async () => {
+            const days = 30;
+            const multi_bucket_key_prefix = 'prefix-test-lifecycle-multipart3';
+            const multi_bucket_key = 'test-lifecycle-multipart3';
+            // create_time updated to days + 1(31 days) and expire is 30 days,
+            const obj_id1 = await create_mock_multipart_upload(multi_bucket_key_prefix, multipart_bucket, days + 1, 45, 7);
+            const obj_id2 = await create_mock_multipart_upload(multi_bucket_key, multipart_bucket, days + 1, 45, 7);
+            const putLifecycleParams = commonTests.multipart_lifecycle_configuration(multipart_bucket, multi_bucket_key_prefix, days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            // delete only those multipart with prefix `prefix-test-lifecycle-multipart3` not others
+            await verify_multipart_deleted(obj_id1, multi_bucket_key_prefix, 0);
+            await verify_multipart_deleted(obj_id2, multi_bucket_key, 7);
+        });
+
+        async function verify_multipart_deleted(obj_id, key, expected_length) {
+            try {
+                const mp_list = await rpc_client.object.list_multiparts({ obj_id, bucket: multipart_bucket, key });
+                console.log('verify_multipart_deleted : multipart upload objects :', mp_list);
+                const actual_length = mp_list.objects.length;
+                console.log('list_objects_admin objects: ', util.inspect(mp_list.objects));
+                assert.strictEqual(actual_length, expected_length, `listObjectResult actual ${actual_length} !== ${expected_length}`);
+            } catch (err) {
+                console.log('verify_multipart_deleted error is :', err);
+                if (err.code === 'NO_SUCH_UPLOAD' && expected_length === 0) {
+                    assert.ok(true);
+                }
+            }
+        }
     });
 
     mocha.describe('bucket-lifecycle-version', function() {
@@ -553,16 +602,16 @@ mocha.describe('lifecycle', () => {
                 const update = {
                     create_time: moment().subtract(age, 'days').toDate(),
                 };
-                console.log('blow_version_objects: bucket', bucket, 'multiparts_ids', obj_upload_ids, " obj_upload_ids length: ", obj_upload_ids.length, "update :", update);
+                console.log('create_mock_version: bucket', bucket, 'obj_upload_ids:', obj_upload_ids, " obj_upload_ids length: ", obj_upload_ids.length, "update :", update);
                 const update_result = await MDStore.instance().update_objects_by_ids(obj_upload_ids, update);
-                console.log('blow_version_objects: update_objects_by_ids', update_result);
+                console.log('create_mock_version: update_objects_by_ids:', update_result);
             }
         }
 
         mocha.it('lifecyle - version object expired', async () => {
             const age = 30;
             const version_count = 10;
-            const version_bucket_key = 'test-lifecycle-version1';
+            const version_bucket_key = 'test-lifecycle-version1-1';
             await create_mock_version(version_bucket_key, version_bucket, age, version_count, true);
             const putLifecycleParams = commonTests.days_lifecycle_configuration(version_bucket, version_bucket_key);
             await s3.putBucketLifecycleConfiguration(putLifecycleParams);
@@ -574,7 +623,7 @@ mocha.describe('lifecycle', () => {
         mocha.it('lifecyle - version object not expired', async () => {
             const age = 5;
             const version_count = 10;
-            const version_bucket_key = 'test-lifecycle-version2';
+            const version_bucket_key = 'test-lifecycle-version2-0';
             await create_mock_version(version_bucket_key, version_bucket, age, version_count, true);
             const putLifecycleParams = commonTests.days_lifecycle_configuration(version_bucket, version_bucket_key);
             await s3.putBucketLifecycleConfiguration(putLifecycleParams);
@@ -583,7 +632,141 @@ mocha.describe('lifecycle', () => {
             await verify_version_deleted(version_count, version_bucket_key);
         });
 
-        async function verify_version_deleted(expected_length, key) {
+        mocha.it('lifecyle - version not expiration', async () => {
+            const days = 30;
+            const version_count = 10;
+            const newnon_current_version = 1;
+            const noncurrent_days = 15;
+            const version_bucket_key = 'test-lifecycle-version2-1';
+            // create_time updated to 29 days and expire is 30 days and noncurrent expire in 15
+            await create_mock_version(version_bucket_key, version_bucket, days - 1, version_count);
+            const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket,
+                                            version_bucket_key, days, newnon_current_version, noncurrent_days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            // remove all the noncurrent versions expect one( newnon_current_version = 1) once 15 days passed 
+            // current version has not expired. In that case, list will return 2 objects
+            await verify_version_deleted(2, version_bucket_key);
+        });
+
+        mocha.it('lifecyle - version expiration - only NewerNoncurrentVersions exceeded', async () => {
+            const days = 35;
+            const version_count = 10;
+            const newnon_current_version = 5;
+            const noncurrent_days = 30;
+            const version_bucket_key = 'test-lifecycle-version1-1';
+            // create_time updated to 36 days and expire is 35 days
+            await create_mock_version(version_bucket_key, version_bucket, days + 1, version_count, true);
+            const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket,
+                                            version_bucket_key, days, newnon_current_version, noncurrent_days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            // object will expire, noncurrent version count(5) + deleted current version(1) + delete marker(1)
+            await verify_version_deleted(7, version_bucket_key);
+        });
+
+        mocha.it('lifecyle - version expiration - only NoncurrentDays exceeded', async () => {
+            const days = 45;
+            const version_count = 10;
+            const newnon_current_version = 100;
+            const noncurrent_days = 30;
+            const version_bucket_key = 'test-lifecycle-version3';
+            // create_time updated to 45+30+1= 76 days and expire is 45 days
+            await create_mock_version(version_bucket_key, version_bucket, days + noncurrent_days + 1, version_count, true);
+            const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket,
+                                            version_bucket_key, days, newnon_current_version, noncurrent_days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            // dekete only current version, do not delete noncurrent version because newnon_current_version = 100
+            // noncurrent version count + delete marker(1)
+            await verify_version_deleted(11, version_bucket_key);
+        });
+
+        mocha.it('lifecyle - version expiration - both NoncurrentDays and NewerNoncurrentVersions exceeded', async () => {
+            const days = 30;
+            const version_count = 10;
+            const newnon_current_version = 1;
+            const noncurrent_days = 15;
+            const version_bucket_key = 'test-lifecycle-version4';
+            // create_time updated to 46 days and expire is 30 days
+            await create_mock_version(version_bucket_key, version_bucket, days + noncurrent_days + 1, version_count);
+            const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket,
+                                            version_bucket_key, days, newnon_current_version, noncurrent_days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            // newnon_current_version count + current version
+            await verify_version_deleted(2, version_bucket_key);
+        });
+
+        mocha.it('lifecyle - version not expiration - delete marker true', async () => {
+            const days = 30;
+            const version_count = 10;
+            const newnon_current_version = 0;
+            const noncurrent_days = 15;
+            const version_bucket_key = 'test-lifecycle-version5';
+            // create_time updated to 29 days and expire is 30 days,
+            await create_mock_version(version_bucket_key, version_bucket, days - 1, version_count, true);
+            const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket,
+                                            version_bucket_key, days, newnon_current_version, noncurrent_days);
+
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            // remove all the noncurrent version because the noncurrent_days is 15(< 30) and 
+            // newnon_current_version is 0. Only current version exists
+            await verify_version_deleted(1, version_bucket_key);
+        });
+
+        mocha.it('lifecyle - version expiration all - delete marker true', async () => {
+            const days = 30;
+            const version_count = 10;
+            const newnon_current_version = 0;
+            const noncurrent_days = 15;
+            const version_bucket_key = 'test-lifecycle-version6';
+            await create_mock_version(version_bucket_key, version_bucket, days + noncurrent_days + 1, version_count, true);
+            const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket,
+                                            version_bucket_key, days, newnon_current_version, noncurrent_days);
+            await s3.putBucketLifecycleConfiguration(putLifecycleParams);
+            await lifecycle.background_worker();
+            await verify_version_deleted(2, version_bucket_key);
+            // add a delay between two worker run 
+            await P.delay(100); // 0.1sec
+            // Complete deletion will happen in two lifecycle run,
+            // First delete all the noncurrent version and  current version to noncurrent version
+            // last noncurrent version( current in first step ) and delete marker
+            await age_object(days + noncurrent_days + 1, version_bucket_key);
+            await lifecycle.background_worker();
+            await verify_version_deleted(1, version_bucket_key);
+
+            const putDeleteMarkerLifecycleParams = commonTests.marker_lifecycle_configuration(version_bucket, version_bucket_key);
+            await s3.putBucketLifecycleConfiguration(putDeleteMarkerLifecycleParams);
+            await lifecycle.background_worker();
+            await verify_version_deleted(0, version_bucket_key);
+        });
+
+        async function age_object(age, key = '') {
+            const obj_params = {
+                bucket: version_bucket,
+                prefix: key,
+            };
+            const obj_ids = [];
+            const {objects} = await rpc_client.object.list_object_versions(obj_params);
+            for (const object of objects) {
+                obj_ids.push(object.obj_id);
+            }
+            if (age > 0) {
+                const update = {
+                    create_time: moment().subtract(age, 'days').toDate(),
+                };
+                console.log('age_object: bucket', version_bucket, 'obj_ids', obj_ids, " obj_upload_ids length: ", obj_ids.length, "update :", update);
+                await MDStore.instance().update_objects_by_ids(obj_ids, update);
+            }
+        }
+
+        async function verify_version_deleted(expected_length, key = '') {
             const obj_params = {
                 bucket: version_bucket,
                 prefix: key,