CLIENT-3852 tests against multi node cluster

.github/actions/deploy-aerolab/action.yml

@@ -0,0 +1,76 @@
+name: "Deploy Aerospike Cluster using Aerolab"
+description: "Creates an Aerospike cluster using Aerolab on the current runner"
+
+inputs:
+  aerospike_version:
+    description: "Aerospike version for cluster creation"
+    required: false
+    default: "8.1.0.1c"
+  nodes:
+    description: "Number of cluster nodes"
+    required: false
+    default: "3"
+  cluster_name:
+    description: "Cluster name"
+    required: false
+    default: "ce"
+  aerolab_version:
+    # https://github.com/aerospike/aerolab/releases/
+    description: "Aerolab version to run."
+    required: false
+    default: "7.9.0"
+
+runs:
+  using: composite
+  steps:
+    - name: Install dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y curl
+
+    - name: Download Aerolab .deb
+      shell: bash
+      run: |
+        echo "Downloading Aerolab ${{ inputs.aerolab_version }} (amd64)"
+        curl -L -o aerolab.deb https://github.com/aerospike/aerolab/releases/download/${{ inputs.aerolab_version }}/aerolab-linux-amd64-${{ inputs.aerolab_version }}.deb
+
+    - name: Install Aerolab
+      shell: bash
+      run: |
+        sudo dpkg -i aerolab.deb
+        sudo apt-get install -f -y
+
+    - name: Prepare Aerolab home
+      shell: bash
+      run: |
+        mkdir -p /tmp/aerolab-home
+        echo "AEROLAB_HOME=/tmp/aerolab-home" >> $GITHUB_ENV
+
+    - name: Configure Aerolab backend
+      shell: bash
+      run: |
+        aerolab config backend -t docker
+
+    - name: Create Aerospike cluster
+      shell: bash
+      run: |
+        aerolab cluster create \
+          -v "${{ inputs.aerospike_version }}" \
+          -c "${{ inputs.nodes }}" \
+          -n "${{ inputs.cluster_name }}"
+
+    - name: Validate cluster list
+      shell: bash
+      run: |
+        echo "Listing clusters..."
+        aerolab cluster list | tee cluster-list.txt
+
+    - name: Show cluster list output as job summary
+      shell: bash
+      run: |
+        echo "## Aerolab Cluster Created" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo '````' >> $GITHUB_STEP_SUMMARY
+        cat cluster-list.txt >> $GITHUB_STEP_SUMMARY
+        echo '````' >> $GITHUB_STEP_SUMMARY

.github/actions/make-matrix/action.yaml

@@ -37,13 +37,12 @@ runs:
         echo "Validating JSON..."
         echo "${JSON}" | jq . >/dev/null
 
-        MATRIX="$(echo "${JSON}" | jq -c --arg go "${GO_VERSION}" '[to_entries[] | { server: .key, go: $go } + .value]')"
+        MATRIX="$(echo "${JSON}" | jq -c --arg go "${GO_VERSION}" '[to_entries[] | {server: .key, version: .value.version, "go-version": $go}]')"
+
+        echo "matrix<<EOF" >> "$GITHUB_OUTPUT"
+        echo "${MATRIX}" >> "$GITHUB_OUTPUT"
+        echo "EOF" >> "$GITHUB_OUTPUT"
 
-        {
-          echo "matrix<<__JSON__"
-          echo "${MATRIX}"
-          echo "__JSON__"
-        } >> "$GITHUB_OUTPUT"
 
 outputs:
   matrix:

.github/workflows/build-multi-node.yml

@@ -0,0 +1,113 @@
+name: Build and test against multi-node cluster
+
+on:
+  workflow_dispatch:
+    inputs:
+      aerospike_version:
+        description: "Aerolab/Aerospike version"
+        required: false
+        default: "8.1.0.1c"
+      nodes:
+        description: "Number of cluster nodes"
+        required: false
+        default: "3"
+      cluster_name:
+        description: "Cluster name"
+        required: false
+        default: "ce"
+      runner_version:
+        description: "Ubuntu runner version"
+        required: false
+        default: "ubuntu-24.04" # https://github.com/actions/runner-images/blob/main/README.md
+  schedule:
+    - cron: "0 2 * * *" # Nightly at 02:00 UTC
+
+env:
+  # defaults to use against workflow triggered through cron
+  AEROSPIKE_VERSION: 8.1.0.1c
+  NODES: 3
+  CLUSTER_NAME: ce
+  RUNNER_VERSION: ubuntu-24.04
+
+jobs:
+  make-matrix:
+    runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner_version || 'ubuntu-24.04' }}
+    outputs:
+      input-matrix: ${{ steps.create-server-matrix.outputs.matrix }}
+      go-version: ${{ steps.get-go-version.outputs.go-version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Go version
+        id: get-go-version
+        run: |
+          echo "go-version=$(grep '^go [0-9]' go.mod | awk '{print $2}')" >> $GITHUB_OUTPUT
+
+      - id: create-server-matrix
+        uses: ./.github/actions/make-matrix
+        with:
+          default_json: |
+            {
+              "stable": { "version": "${{ github.event_name == 'workflow_dispatch' && inputs.aerospike_version || env.AEROSPIKE_VERSION }}" }
+            }
+          go_version: ${{ steps.get-go-version.outputs.go-version }}
+
+      - name: debug - print input variables
+        run: |
+          echo "Aerospike version: ${{ github.event_name == 'workflow_dispatch' && inputs.aerospike_version || env.AEROSPIKE_VERSION }}"
+          echo "Nodes: ${{ github.event_name == 'workflow_dispatch' && inputs.nodes || env.NODES }}"
+          echo "Cluster name: ${{ github.event_name == 'workflow_dispatch' && inputs.cluster_name || env.CLUSTER_NAME }}"
+          echo "Go version: ${{ steps.get-go-version.outputs.go-version }}"
+          echo 'Matrix output:'
+          echo '${{ steps.create-server-matrix.outputs.matrix }}' | jq .
+
+  build:
+    timeout-minutes: 30
+    needs: make-matrix
+    runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner_version || 'ubuntu-24.04' }}
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.make-matrix.outputs.input-matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Deploy Aerolab Cluster
+        uses: ./.github/actions/deploy-aerolab
+        with:
+          aerospike_version: ${{ github.event_name == 'workflow_dispatch' && inputs.aerospike_version || env.AEROSPIKE_VERSION }}
+          nodes: ${{ github.event_name == 'workflow_dispatch' && inputs.nodes || env.NODES }}
+          cluster_name: ${{ github.event_name == 'workflow_dispatch' && inputs.cluster_name || env.CLUSTER_NAME }}
+
+      - name: "Setup Go ${{ needs.make-matrix.outputs.go-version }}"
+        uses: actions/setup-go@v4
+        with:
+          go-version: "${{ needs.make-matrix.outputs.go-version }}"
+          cache: true
+      - name: Fetch dependencies
+        env:
+          GODEBUG: fips140=on
+        run: |
+          # Install all dependencies
+          go mod download
+
+          # Install ginkgo CLI and gocovmerge for testing
+          go install github.com/onsi/ginkgo/v2/[email protected]
+          go mod download github.com/wadey/gocovmerge
+
+          # Vendor the dependencies
+          go mod vendor
+
+      - name: Display Go version
+        run: |
+          echo "Go version: ${{ needs.make-matrix.outputs.go-version }}"
+      - name: Run the tests
+        env: {GOPROXY: off, GOSUMDB: off, GOFLAGS: -mod=vendor}
+        run: go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo -output-dir=./ -coverprofile=cover_native.out -covermode=atomic -coverpkg=./... -race -keep-going -succinct -randomize-suites -skip="HyperLogLog" -- -hosts=127.0.0.1:3100,127.0.0.1:3101,127.0.0.1:3102 -use-services-alternate=true
+      - name: Combine Cover Profiles
+        env: {GOPROXY: off, GOSUMDB: off, GOFLAGS: -mod=vendor}
+        run: go run -mod=vendor github.com/wadey/gocovmerge cover_*.out > cover_all.out
+      - name: Check Code Coverage
+        uses: vladopajic/go-test-coverage@v2
+        with:
+          # Configure action using config file (option 1)
+          config: ./.testcoverage.yml

.github/workflows/build.yml

@@ -33,8 +33,10 @@ jobs:
 
       - name: debug - print input variables
         run: |
-          echo ${{ steps.get-go-version.outputs.go-version }}
-          echo ${{ steps.create-server-matrix.outputs.input-matrix }}
+          echo "Go version: ${{ steps.get-go-version.outputs.go-version }}"
+          echo 'Matrix output:'
+          echo '${{ steps.create-server-matrix.outputs.matrix }}' | jq .
+
   build:
     timeout-minutes: 30
     needs: make-matrix
@@ -100,7 +102,7 @@ jobs:
         run: go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo build -tags="app_engine" .
       - name: Run the tests
         env: {GOPROXY: off, GOSUMDB: off, GOFLAGS: -mod=vendor}
-        run: go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo -coverprofile=./cover_native.out -covermode=atomic -coverpkg=./... -race -keep-going -succinct -randomize-suites -skip="HyperLogLog"
+        run: go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo -output-dir=./ -coverprofile=cover_native.out -covermode=atomic -coverpkg=./... -race -keep-going -succinct -randomize-suites -skip="HyperLogLog"
       - name: Combine Cover Profiles
         env: {GOPROXY: off, GOSUMDB: off, GOFLAGS: -mod=vendor}
         run: go run -mod=vendor github.com/wadey/gocovmerge cover_*.out > cover_all.out

aerospike_suite_test.go

@@ -115,6 +115,8 @@ func initTestVars() {
 		log.Fatal(err.Error())
 	}
 
+	printConnectedNodes(client)
+
 	defaultBatchPolicy := as.NewBatchPolicy()
 	defaultBatchPolicy.TotalTimeout = 15 * time.Second
 	defaultBatchPolicy.SocketTimeout = 5 * time.Second
@@ -154,6 +156,13 @@ func initTestVars() {
 	client.EnableMetrics(nil)
 }
 
+func printConnectedNodes(client *as.Client) {
+	for _, node := range client.GetNodes() {
+		h := node.GetHost()
+		fmt.Printf("Node: %s, Host: %s, Port: %d\n", node.GetName(), h.Name, h.Port)
+	}
+}
+
 func TestMain(m *testing.M) {
 	rand.Seed(time.Now().UnixNano())
 	flag.Parse()

client_test.go

@@ -108,11 +108,8 @@ var _ = gg.Describe("Aerospike", func() {
 
 	gg.Describe("Client Management", func() {
 
-		dbHost := as.NewHost(*host, *port)
-		dbHost.TLSName = *nodeTLSName
-
 		gg.It("must open and close the client without a problem", func() {
-			client, err := as.NewClientWithPolicyAndHost(clientPolicy, dbHost)
+			client, err := as.NewClientWithPolicyAndHost(clientPolicy, dbHosts...)
 			gm.Expect(err).ToNot(gm.HaveOccurred())
 			gm.Expect(client.IsConnected()).To(gm.BeTrue())
 
@@ -150,7 +147,7 @@ var _ = gg.Describe("Aerospike", func() {
 			cpolicy := *clientPolicy
 			cpolicy.ClusterName = "haha"
 			cpolicy.Timeout = 10 * time.Second
-			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHost)
+			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHosts...)
 			gm.Expect(err).To(gm.HaveOccurred())
 			gm.Expect(err.Matches(ast.CLUSTER_NAME_MISMATCH_ERROR)).To(gm.BeTrue())
 			gm.Expect(nclient).To(gm.BeNil())
@@ -161,7 +158,7 @@ var _ = gg.Describe("Aerospike", func() {
 			cpolicy.ClusterName = "haha"
 			cpolicy.Timeout = 10 * time.Second
 			cpolicy.FailIfNotConnected = false
-			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHost)
+			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHosts...)
 			gm.Expect(err).To(gm.HaveOccurred())
 			gm.Expect(err.Matches(ast.CLUSTER_NAME_MISMATCH_ERROR)).To(gm.BeTrue())
 			gm.Expect(nclient).NotTo(gm.BeNil())
@@ -174,7 +171,7 @@ var _ = gg.Describe("Aerospike", func() {
 			cpolicy := *clientPolicy
 			cpolicy.ClusterName = actualClusterName
 			cpolicy.Timeout = 10 * time.Second
-			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHost)
+			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHosts...)
 			gm.Expect(err).NotTo(gm.HaveOccurred())
 			gm.Expect(len(nclient.GetNodes())).To(gm.Equal(nodeCount))
 		})
@@ -191,7 +188,7 @@ var _ = gg.Describe("Aerospike", func() {
 			cpolicy.AuthMode = as.AuthModeExternal
 			cpolicy.User = "badwan"
 			cpolicy.Password = "blastoff"
-			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHost)
+			nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHosts...)
 			gm.Expect(err).NotTo(gm.HaveOccurred())
 			gm.Expect(len(nclient.GetNodes())).To(gm.Equal(nodeCount))
 		})
@@ -202,7 +199,7 @@ var _ = gg.Describe("Aerospike", func() {
 				cpolicy := *clientPolicy
 				cpolicy.User = *user
 				cpolicy.Password = *password
-				c, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHost)
+				c, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHosts...)
 				gm.Expect(err).NotTo(gm.HaveOccurred())
 
 				info := info(c, "rack-ids")
@@ -215,7 +212,7 @@ var _ = gg.Describe("Aerospike", func() {
 				cpolicy.RackAware = true
 
 				for rid := 1; rid <= 20; rid++ {
-					nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHost)
+					nclient, err := as.NewClientWithPolicyAndHost(&cpolicy, dbHosts...)
 					gm.Expect(err).NotTo(gm.HaveOccurred())
 
 					wpolicy := as.NewWritePolicy(0, 0)

connection_test.go

@@ -42,7 +42,7 @@ var _ = gg.Describe("Connection Test", func() {
 		gm.Expect(conn).ToNot(gm.BeNil())
 	})
 
-	gg.It("Dealines should be calculated correctly", func() {
+	gg.It("Deadlines should be calculated correctly", func() {
 		deadline := func(timeout time.Duration) (res time.Time) {
 			if timeout > 0 {
 				res = time.Now().Add(timeout)
@@ -67,9 +67,9 @@ var _ = gg.Describe("Connection Test", func() {
 
 			gg.By(fmt.Sprintf("expTotalDeadline: %v, expSocketDeadline: %v, expSocketTimeout: %v", matrix.expTotalDeadline, matrix.expSocketDeadline, matrix.expSocketTimeout))
 
-			gm.Expect(expTotalDeadline).To(gm.BeTemporally("~", matrix.expTotalDeadline, time.Millisecond))
-			gm.Expect(expSocketDeadline).To(gm.BeTemporally("~", matrix.expSocketDeadline, time.Millisecond))
-			gm.Expect(expSocketTimeout).To(gm.BeNumerically("~", matrix.expSocketTimeout, time.Millisecond))
+			gm.Expect(expTotalDeadline).To(gm.BeTemporally("~", matrix.expTotalDeadline, 2*time.Millisecond))
+			gm.Expect(expSocketDeadline).To(gm.BeTemporally("~", matrix.expSocketDeadline, 2*time.Millisecond))
+			gm.Expect(expSocketTimeout).To(gm.BeNumerically("~", matrix.expSocketTimeout, 2*time.Millisecond))
 		}
 	})
 

go.mod

@@ -3,24 +3,28 @@ module github.com/aerospike/aerospike-client-go/v8
 go 1.23.0
 
 require (
-	github.com/onsi/ginkgo/v2 v2.22.2
-	github.com/onsi/gomega v1.36.2
+	github.com/onsi/ginkgo/v2 v2.27.2
+	github.com/onsi/gomega v1.38.2
 	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/sync v0.12.0
+	golang.org/x/sync v0.16.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
-	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/stretchr/testify v1.10.0 // indirect
-	golang.org/x/net v0.37.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
-	golang.org/x/tools v0.31.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/mod v0.27.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
+	google.golang.org/protobuf v1.36.7 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )