feat: TOOLS-3078 Added 8.1 Server Schema based Test cases (#71)

* feat: TOOLS-3078 Add test cases for Aerospike 8.1 config conversion

Introduces new test data and configuration files for Aerospike server version 8.1.0.0, including .conf, .yaml, and associated test definitions. Updates the schema submodule to the latest commit to support these tests.

* fix: correct apply-ttl-reduction key to apply-ttl-reductions in server81.yaml

* feat: implement exponential backoff for Docker image pulling to handle rate limiting

* fix: remove exponential backoff logic for Docker container creation in integration tests

* feat: TOOLS-3078 Update schema submodule to 8.1.0 commit

Advanced the schema/schemas submodule to commit f99d3c1, pulling in the latest changes from the schema repository.

* Update dependencies in go.mod and go.sum

Upgraded multiple direct and indirect dependencies to newer versions, including aerospike-client-go, aerospike-management-lib, tools-common-go, docker, go-logr, fsnotify, viper, cast, and OpenTelemetry packages. Removed unused and outdated dependencies to keep the project up to date and improve compatibility and security.

Author: a-spiker

go.mod

@@ -6,8 +6,8 @@ toolchain go1.24.5
 
 require (
 	github.com/aerospike/aerospike-client-go/v8 v8.2.2
-	github.com/aerospike/aerospike-management-lib v1.7.1-0.20250709120612-377d0ff87581
-	github.com/aerospike/tools-common-go v0.2.1-0.20250130070321-acda09110e14
+	github.com/aerospike/aerospike-management-lib v1.8.0
+	github.com/aerospike/tools-common-go v0.3.0
 	github.com/bombsimon/logrusr/v4 v4.1.0
 	github.com/docker/docker v28.3.2+incompatible
 	github.com/go-logr/logr v1.4.3
@@ -31,45 +31,40 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
-	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/viper v1.19.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/viper v1.20.1 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel v1.37.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect
 	go.opentelemetry.io/otel/metric v1.37.0 // indirect
 	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
 	golang.org/x/sync v0.12.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 	golang.org/x/tools v0.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
 	k8s.io/apimachinery v0.29.0 // indirect
 )

go.sum

@@ -452,34 +452,9 @@ func runServer(version string, serverVersion string, confPath string, auth testu
 
 	// TODO if possible, don't create new containers each time
 	var id string
-	maxRetries := 5
-	retryCount := 0
-	backoffTime := 5 * time.Second
-
-	for {
-		id, err = testutils.CreateAerospikeContainer(containerName, containerConf, containerHostConf, imagePullOptions, dockerClient)
-		if err == nil {
-			break
-		}
-
-		// Check if this is a rate limit error
-		if strings.Contains(err.Error(), "toomanyrequests") {
-			retryCount++
-			if retryCount > maxRetries {
-				t.Errorf("Failed after %d retries: %v", maxRetries, err)
-				return "", ""
-			}
-
-			// Log the retry
-			t.Logf("Docker pull rate limit reached, retrying in %v (attempt %d/%d)", backoffTime, retryCount, maxRetries)
 
-			// Sleep with exponential backoff
-			time.Sleep(backoffTime)
-			backoffTime *= 2 // exponential backoff
-			continue
-		}
-
-		// If it's not a rate limit error, just fail
+	id, err = testutils.CreateAerospikeContainer(containerName, containerConf, containerHostConf, imagePullOptions, dockerClient)
+	if err != nil {
 		t.Error(err)
 		return "", ""
 	}

integration_test.go

@@ -0,0 +1,11 @@
+[
+    {
+        "Source":"testdata/cases/server81/server81.conf",
+        "Destination":"testdata/cases/server81/server81-res-.yaml",
+        "Expected":"testdata/cases/server81/server81.yaml",
+        "Arguments":["convert","--aerospike-version","8.1.0.0","--format","asconfig","--output","testdata/cases/server81/server81-res-.yaml"],
+        "SkipServerTest":false,
+        "ServerErrorAllowList":null,
+        "ServerImage":"aerospike/aerospike-server-enterprise:8.1.0.0-rc3"
+    }
+]

schema/schemas

@@ -0,0 +1,111 @@
+
+service {
+    cluster-name cl1
+
+	user root
+	group root
+	pidfile /dummy/file/path1
+
+	batch-max-requests 1000
+	tls-refresh-period 300S
+}
+
+logging {
+	console {
+		context namespace info
+	}
+}
+
+network {
+	service {
+		address any
+		port 3000
+	}
+
+	heartbeat {
+		mode multicast
+		multicast-group 127.0.0.1
+		port 9918
+
+
+
+
+		interval 150
+		timeout 10
+	}
+
+	fabric {
+		port 3001
+	}
+
+	admin {
+		address any
+		port 3003
+	}
+}
+
+namespace ns1 {
+	replication-factor 2
+
+	index-type shmem
+	sindex-type shmem
+
+	disable-mrt-writes false
+	mrt-duration 60
+	strong-consistency true
+	apply-ttl-reductions false
+
+	migrate-skip-unreadable false
+
+	default-read-touch-ttl-pct 50
+
+	storage-engine memory {
+		data-size 32G
+    	}
+
+	evict-indexes-memory-pct 50
+
+	indexes-memory-budget 2147483648
+
+	nsup-period 2048
+
+	set s1 {
+		default-read-touch-ttl-pct -1
+	}
+
+	active-rack 2
+}
+
+namespace ns2 {
+	replication-factor 2
+	index-type shmem
+	sindex-type shmem
+	apply-ttl-reductions true
+	default-read-touch-ttl-pct 0
+	storage-engine device {
+        	device /dummy/mount/point1
+        	evict-used-pct 70
+        	stop-writes-used-pct 70
+        	stop-writes-avail-pct 20
+		flush-size 2M
+    	}
+
+	set s2 {
+		default-read-touch-ttl-pct 50
+	}
+}
+
+security {
+	default-password-file /dummy/file/path2
+}
+
+xdr {
+	dc dataCenter1 {
+		node-address-port 127.0.0.1 3000
+		namespace ns1 {
+			remote-namespace ns2
+			ship-versions-interval 2M
+			ship-versions-policy interval
+		}
+	}
+}

testdata/cases/server81/conf-tests.json

@@ -0,0 +1,82 @@
+logging:
+    - namespace: info
+      name: console
+namespaces:
+    - default-read-touch-ttl-pct: 0
+      index-type:
+        type: shmem
+      name: ns2
+      replication-factor: 2
+      apply-ttl-reductions: true
+      sets:
+        - default-read-touch-ttl-pct: 50
+          name: s2
+      sindex-type:
+        type: shmem
+      storage-engine:
+        devices:
+            - /dummy/mount/point1
+        evict-used-pct: 70
+        flush-size: 2097152
+        stop-writes-avail-pct: 20
+        stop-writes-used-pct: 70
+        type: device
+    - active-rack: 2
+      default-read-touch-ttl-pct: 50
+      evict-indexes-memory-pct: 50
+      index-type:
+        type: shmem
+      indexes-memory-budget: 2147483648
+      name: ns1
+      nsup-period: 2048
+      replication-factor: 2
+      disable-mrt-writes: false
+      mrt-duration: 60
+      strong-consistency: true
+      apply-ttl-reductions: false
+      migrate-skip-unreadable: false
+      sets:
+        - default-read-touch-ttl-pct: -1
+          name: s1
+      sindex-type:
+        type: shmem
+      storage-engine:
+        data-size: 34359738368
+        type: memory
+network:
+    fabric:
+        port: 3001
+    heartbeat:
+        interval: 150
+        mode: multicast
+        multicast-groups:
+            - 127.0.0.1
+        port: 9918
+        timeout: 10
+    admin:
+        addresses:
+            - any
+        port: 3003
+    service:
+        addresses:
+            - any
+        port: 3000
+security:
+    default-password-file: /dummy/file/path2
+service:
+    batch-max-requests: 1000
+    cluster-name: cl1
+    group: root
+    pidfile: /dummy/file/path1
+    tls-refresh-period: 300
+    user: root
+xdr:
+    dcs:
+        - name: dataCenter1
+          namespaces:
+            - name: ns1
+              remote-namespace: ns2
+              ship-versions-interval: 120
+              ship-versions-policy: interval
+          node-address-ports:
+            - 127.0.0.1:3000

testdata/cases/server81/server81.conf

@@ -0,0 +1 @@
+{"TestedVersion":"8.1.0.0-rc3","OriginallyUsedVersion":"8.1.0.0-rc3"}

testdata/cases/server81/server81.yaml

@@ -0,0 +1,11 @@
+[
+    {
+        "Source":"testdata/cases/server81/server81.yaml",
+        "Destination":"testdata/cases/server81/server81-res-.conf",
+        "Expected":"testdata/cases/server81/server81.conf",
+        "Arguments":["convert","--aerospike-version","8.1.0.0","--format","yaml","--output","testdata/cases/server81/server81-res-.conf"],
+        "SkipServerTest":false,
+        "ServerErrorAllowList":null,
+        "ServerImage":"aerospike/aerospike-server-enterprise:8.1.0.0-rc3"
+    }
+]

testdata/cases/server81/versions.json

@@ -8,6 +8,8 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"strings"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
@@ -70,8 +72,41 @@ func RemoveAerospikeContainer(id string, cli *client.Client) error {
 
 func CreateAerospikeContainer(name string, c *container.Config, ch *container.HostConfig, imagePullOpts image.PullOptions, cli *client.Client) (string, error) {
 	ctx := context.Background()
-	reader, err := cli.ImagePull(ctx, name, imagePullOpts)
-	if err != nil {
+
+	// Retry configuration for Docker Hub rate limiting
+	maxRetries := 10
+	baseBackoff := 10 * time.Second // Base backoff time
+	maxBackoff := 5 * time.Minute   // Maximum backoff time
+
+	var reader io.ReadCloser
+	var err error
+
+	// Retry loop for image pulling with exponential backoff
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		reader, err = cli.ImagePull(ctx, name, imagePullOpts)
+		if err == nil {
+			break
+		}
+
+		// Check if this is a rate limit error
+		if strings.Contains(err.Error(), "toomanyrequests") || strings.Contains(err.Error(), "rate limit") {
+			if attempt == maxRetries {
+				log.Printf("Failed to pull image %s after %d attempts due to rate limiting: %s", name, maxRetries, err)
+				return "", err
+			}
+
+			// Exponential backoff: 2^(attempt-1) * baseBackoff, capped at maxBackoff
+			backoffTime := time.Duration(1<<(attempt-1)) * baseBackoff
+			if backoffTime > maxBackoff {
+				backoffTime = maxBackoff
+			}
+
+			log.Printf("Docker pull rate limit reached for %s, retrying in %v (attempt %d/%d)", name, backoffTime, attempt, maxRetries)
+			time.Sleep(backoffTime)
+			continue
+		}
+
+		// If it's not a rate limit error, don't retry
 		log.Printf("Unable to pull image %s: %s", name, err)
 		return "", err
 	}