CLIENT-681: Create dynamically allocate byte pool. #7492
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR tests | |
| permissions: | |
| contents: read | |
| env: | |
| LOWEST_SUPPORTED_PY_VERSION: '3.9' | |
| # pull_request event doesn't support inputs | |
| REGISTRY_NAME: ${{ github.event_name == 'workflow_dispatch' && inputs.registry-name || 'docker.io' }} | |
| SERVER_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.server-tag || 'latest' }} | |
| # Trigger test workflow whenever: | |
| # 1. A pull request is updated (e.g with new commits) | |
| # 2. Commits are pushed directly to the dev, stage or master branch | |
| on: | |
| push: | |
| branches: ["dev*", "stage*", "master*", "v*-backport*"] | |
| pull_request: | |
| branches: ["dev*", "stage*", "v*-backport*"] | |
| types: [ | |
| # Default triggers | |
| opened, | |
| synchronize, | |
| reopened, | |
| # Additional triggers | |
| labeled, | |
| unlabeled | |
| ] | |
| workflow_dispatch: | |
| inputs: | |
| # Used to test server RCs | |
| registry-name: | |
| description: Registry name | |
| type: string | |
| default: docker.io | |
| required: true | |
| server-tag: | |
| description: Server tag | |
| type: string | |
| default: latest | |
| required: true | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| architecture: 'x64' | |
| - uses: pre-commit/[email protected] | |
| build: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| py-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] | |
| # Make sure we can build and run tests on an instrumented build that uses libasan | |
| # We aren't necessarily checking for memory errors / leaks in this test | |
| sanitizer: [false] | |
| include: | |
| - py-version: 3.9 | |
| sanitizer: true | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.py-version }} | |
| architecture: 'x64' | |
| allow-prereleases: true | |
| - run: sudo apt update | |
| - name: Install build dependencies (apt packages) | |
| run: sudo apt install python3-dev libssl-dev -y | |
| - name: Install build dependencies (pip packages) | |
| run: python3 -m pip install -r requirements.txt | |
| - if: ${{ matrix.sanitizer }} | |
| run: echo SANITIZER=1 >> $GITHUB_ENV | |
| - name: Build client | |
| run: python3 -m build | |
| env: | |
| CFLAGS: '-Werror' | |
| - run: echo WHEEL_GH_ARTIFACT_NAME=wheel-${{ matrix.py-version }} >> $GITHUB_ENV | |
| - if: ${{ matrix.sanitizer }} | |
| run: echo WHEEL_GH_ARTIFACT_NAME=${{ env.WHEEL_GH_ARTIFACT_NAME }}-sanitizer >> $GITHUB_ENV | |
| - name: Send wheel to test jobs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.WHEEL_GH_ARTIFACT_NAME }} | |
| path: ./dist/*.whl | |
| generate-coverage-report: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| architecture: 'x64' | |
| - name: Build client | |
| # Use --wheel command to generate object .o files in build/temp*/src/main directory | |
| # They will also contain .gcno files there | |
| # and .gcda files will be generated there after running the tests | |
| run: | | |
| python3 -m pip install build -c requirements.txt | |
| COVERAGE=1 UNOPTIMIZED=1 python3 -m build --wheel | |
| - name: Install client | |
| # Install in user directory to prevent permission errors | |
| run: python3 -m pip install --user . | |
| - run: pip install -r requirements.txt | |
| working-directory: test | |
| - name: Run Aerospike server | |
| uses: ./.github/actions/run-ee-server | |
| with: | |
| registry-name: ${{ env.REGISTRY_NAME }} | |
| server-tag: ${{ env.SERVER_TAG }} | |
| registry-username: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }} | |
| registry-password: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }} | |
| - run: python3 -m pytest --cov=aerospike_helpers --cov-report xml:coverage.xml ./new_tests | |
| working-directory: test | |
| - name: Copy over source files to build dir | |
| if: ${{ !cancelled() }} | |
| # The build/temp*/src/main directory will contain a hierarchy of object .o files | |
| # But the source files must be stored in the same folder hierarchy at build/temp*/src/main/src/main | |
| run: | | |
| cd build/temp*/src/main | |
| mkdir -p src/main/ | |
| mkdir -p cdt_types/src/main/cdt_types | |
| mkdir -p client/src/main/client | |
| mkdir -p geospatial/src/main/geospatial | |
| mkdir -p global_hosts/src/main/global_hosts | |
| mkdir -p key_ordered_dict/src/main/key_ordered_dict | |
| mkdir -p nullobject/src/main/nullobject | |
| mkdir -p query/src/main/query | |
| mkdir -p scan/src/main/scan | |
| mkdir -p transaction/src/main/transaction | |
| mkdir -p config_provider/src/main/config_provider | |
| cd ../../../../ | |
| cp src/main/*.c build/temp*/src/main/src/main | |
| cp src/main/cdt_types/*.c build/temp*/src/main/cdt_types/src/main/cdt_types/ | |
| cp src/main/client/*.c build/temp*/src/main/client/src/main/client/ | |
| cp src/main/geospatial/*.c build/temp*/src/main/geospatial/src/main/geospatial/ | |
| cp src/main/global_hosts/*.c build/temp*/src/main/global_hosts/src/main/global_hosts/ | |
| cp src/main/key_ordered_dict/*.c build/temp*/src/main/key_ordered_dict/src/main/key_ordered_dict/ | |
| cp src/main/nullobject/*.c build/temp*/src/main/nullobject/src/main/nullobject/ | |
| cp src/main/query/*.c build/temp*/src/main/query/src/main/query/ | |
| cp src/main/scan/*.c build/temp*/src/main/scan/src/main/scan/ | |
| cp src/main/transaction/*.c build/temp*/src/main/transaction/src/main/transaction/ | |
| cp src/main/config_provider/*.c build/temp*/src/main/config_provider/src/main/config_provider/ | |
| - name: Generate coverage report for all object files | |
| if: ${{ !cancelled() }} | |
| run: | | |
| cd build/temp*/src/main | |
| find . -type f -name "*.o" -execdir gcov {} \; | |
| - name: Move aerospike_helpers coverage report to this directory | |
| if: ${{ !cancelled() }} | |
| run: mv test/coverage.xml build/temp*/src/main | |
| - name: Upload coverage report folder to Github | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: build/temp*/src/main/ | |
| coverage-upload: | |
| needs: generate-coverage-report | |
| if: ${{ !cancelled() }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: ./coverage-report | |
| - uses: codecov/codecov-action@v5 | |
| with: | |
| directory: coverage-report | |
| verbose: true # optional (default = false) | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| stubtest: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| architecture: 'x64' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| - name: Install client | |
| run: pip install *.whl | |
| - run: pip install mypy -c requirements.txt | |
| working-directory: .github/workflows | |
| - run: stubtest aerospike --allowlist stubtest-allowlist | |
| test-full-suite: | |
| strategy: | |
| matrix: | |
| type: | |
| - sanitizer | |
| - dont_validate_keys | |
| fail-fast: false | |
| runs-on: ubuntu-22.04 | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| architecture: 'x64' | |
| - if: ${{ matrix.type == 'sanitizer' }} | |
| run: echo WHEEL_GH_ARTIFACT_NAME=wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}-sanitizer >> $GITHUB_ENV | |
| - if: ${{ matrix.type == 'dont_validate_keys' }} | |
| run: echo WHEEL_GH_ARTIFACT_NAME=wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }} >> $GITHUB_ENV | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.WHEEL_GH_ARTIFACT_NAME }} | |
| - name: Install client | |
| run: pip install *.whl | |
| - name: Install test dependencies | |
| run: pip install -r test/requirements.txt | |
| # We can already detect leaks using valgrind | |
| # halt_on_error must be enabled, or else this job won't fail if a memory error occurs | |
| - if: ${{ matrix.type == 'sanitizer' }} | |
| run: | | |
| echo ASAN_OPTIONS='detect_stack_use_after_return=1:detect_leaks=0' >> $GITHUB_ENV | |
| echo LD_PRELOAD=$(gcc --print-file-name=libasan.so) >> $GITHUB_ENV | |
| - uses: ./.github/actions/run-ee-server | |
| with: | |
| registry-name: ${{ env.REGISTRY_NAME }} | |
| server-tag: ${{ env.SERVER_TAG }} | |
| registry-username: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }} | |
| registry-password: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }} | |
| - if: ${{ matrix.type == 'dont_validate_keys' }} | |
| run: crudini --existing=param --set config.conf input-validation validate_keys false | |
| working-directory: test | |
| - name: Run tests | |
| # We need to disable capturing output or else we cannot see memory errors reported by | |
| # libasan during the test run | |
| run: python -m pytest ./new_tests -vvs | |
| working-directory: test | |
| test-ce: | |
| runs-on: ubuntu-22.04 | |
| needs: build | |
| strategy: | |
| matrix: | |
| py-version: [ | |
| "3.9", | |
| "3.10", | |
| "3.11", | |
| "3.12", | |
| "3.13", | |
| "3.14" | |
| ] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.py-version }} | |
| architecture: 'x64' | |
| allow-prereleases: true | |
| - run: echo WHEEL_GH_ARTIFACT_NAME=wheel-${{ matrix.py-version }} >> $GITHUB_ENV | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.WHEEL_GH_ARTIFACT_NAME }} | |
| - name: Install client | |
| run: pip install *.whl | |
| - name: Install test dependencies | |
| run: pip install -r test/requirements.txt | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY_NAME }} | |
| username: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }} | |
| password: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }} | |
| - name: Run Aerospike server | |
| run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 ${{ env.REGISTRY_NAME }}/aerospike/aerospike-server:${{ env.SERVER_TAG }} | |
| - name: Create config.conf | |
| run: cp config.conf.template config.conf | |
| working-directory: test | |
| - uses: ./.github/actions/wait-for-ce-server-to-start | |
| with: | |
| container-name: aerospike | |
| - name: Run tests | |
| run: python -m pytest ./new_tests -vv -W error::pytest.PytestUnraisableExceptionWarning | |
| working-directory: test | |
| test-ee: | |
| runs-on: ubuntu-22.04 | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.13" | |
| architecture: 'x64' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel-3.13 | |
| - name: Install client | |
| run: pip install *.whl | |
| - name: Install test dependencies | |
| run: pip install -r test/requirements.txt | |
| - uses: ./.github/actions/run-ee-server | |
| with: | |
| registry-name: ${{ env.REGISTRY_NAME }} | |
| server-tag: ${{ env.SERVER_TAG }} | |
| registry-username: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }} | |
| registry-password: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }} | |
| - name: Run tests | |
| # -s: we want to check that the test_create_pki_user test case passes or raises an exception as expected | |
| # There's no way to tell unless we see the logs | |
| run: python -m pytest -s ./new_tests/test_{mrt_functionality,admin_*,compress}.py -W error::pytest.PytestUnraisableExceptionWarning | |
| working-directory: test | |
| - name: Show logs if failed | |
| if: ${{ failure() }} | |
| run: | | |
| docker container logs aerospike | |
| cat ./configs/aerospike.conf | |
| sphinx: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| builder-command: | |
| - 'spelling . spelling -W --keep-going' | |
| # -vv is too verbose and makes it hard to read logs | |
| - 'linkcheck -v . links' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.11 | |
| architecture: 'x64' | |
| - name: Install dependencies | |
| # TODO: find way to split up dependencies | |
| run: python -m pip install -r doc/requirements.txt | |
| - name: Run builder | |
| run: sphinx-build -b ${{ matrix.builder-command }} | |
| working-directory: doc | |
| test-metrics: | |
| needs: build | |
| strategy: | |
| matrix: | |
| suffix: | |
| - ext_metrics_node_close_listener | |
| - ext_metrics_cluster_name | |
| fail-fast: false | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| architecture: 'x64' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| - run: python3 -m pip install *.whl | |
| - run: python3 -m pip install -r requirements.txt | |
| working-directory: test/standalone | |
| - run: python3 test_${{ matrix.suffix }}.py | |
| working-directory: test/standalone | |
| test-misc: | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-case: | |
| - "lowest_supported_server_version" | |
| - "user_agent_with_ce" | |
| - "user_agent_with_ee" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| architecture: 'x64' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }} | |
| - run: python3 -m pip install *.whl | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY_NAME }} | |
| username: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }} | |
| password: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }} | |
| - name: Run Aerospike server | |
| if: ${{ matrix.test-case != 'user_agent_with_ee' }} | |
| run: docker run -d --name aerospike -p 3000:3000 -e DEFAULT_TTL=2592000 ${{ env.REGISTRY_NAME }}/aerospike/aerospike-server:${{ env.SERVER_TAG }} | |
| - uses: ./.github/actions/wait-for-ce-server-to-start | |
| if: ${{ matrix.test-case != 'user_agent_with_ee' }} | |
| with: | |
| container-name: aerospike | |
| - uses: ./.github/actions/run-ee-server | |
| if: ${{ matrix.test-case == 'user_agent_with_ee' }} | |
| with: | |
| registry-name: ${{ env.REGISTRY_NAME }} | |
| server-tag: ${{ env.SERVER_TAG }} | |
| registry-username: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }} | |
| registry-password: ${{ env.REGISTRY_NAME == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }} | |
| # The user agent can also send the client's username if app-id is not set by the client | |
| docker-run-extra-flags: '-e NO_SC=1 -e NO_TLS=1' | |
| - name: Install test dependencies | |
| if: ${{ matrix.test-case == 'lowest_supported_server_version' }} | |
| run: pip install -r test/requirements.txt | |
| - name: Create config.conf | |
| if: ${{ matrix.test-case == 'lowest_supported_server_version' }} | |
| run: cp config.conf.template config.conf | |
| working-directory: test | |
| - name: Run tests | |
| if: ${{ matrix.test-case == 'lowest_supported_server_version' }} | |
| run: python -m pytest ./new_tests | |
| working-directory: test | |
| - if: ${{ startsWith(matrix.test-case, 'user_agent') }} | |
| # Even for server versions < 8.1 that don't support user agent, this client version should still work on older servers | |
| name: Run client in background | |
| run: | | |
| cat <<-EOF >> run-client-in-bg.py | |
| import aerospike | |
| import time | |
| import sys | |
| config = { | |
| "hosts": [ | |
| ("127.0.0.1", 3000) | |
| ] | |
| } | |
| if "${{ endsWith(matrix.test-case, 'with_ee') }}" == "true": | |
| config["user"] = "superuser" | |
| config["password"] = "superuser" | |
| client = aerospike.client(config) | |
| while True: | |
| time.sleep(1) | |
| EOF | |
| # This shell will be closed once this step completes | |
| nohup python3 run-client-in-bg.py & | |
| # TODO: We want to check if the python script raises a syntax error or not. (should fail immediately) | |
| # When background processes fail, this step won't fail. | |
| - if: ${{ startsWith(matrix.test-case, 'user_agent') }} | |
| name: Confirm that the user agent shows the correct client language and version (only for server 8.1) | |
| run: | | |
| server_version=$(docker exec ${{ env.CONTAINER_NAME }} asinfo -v build) | |
| major_num=$(echo $server_version | cut -d '.' -f 1) | |
| minor_num=$(echo $server_version | cut -d '.' -f 2) | |
| if [[ $major_num -lt 8 || ( $major_num -eq 8 && $minor_num -lt 1 ) ]]; then | |
| exit 0 | |
| fi | |
| info_cmd_response=$(docker exec ${{ env.CONTAINER_NAME }} asinfo -v "user-agents" | head -n 1) | |
| echo "Info command response: $info_cmd_response" | |
| # Response format: user-agent=<base64 encoded string>:... | |
| user_agent_base64_encoded=$(echo $info_cmd_response | perl -n -E 'say $1 if m/user-agent= ([a-zA-Z0-9+\/=]+) :/x') | |
| echo $user_agent_base64_encoded | |
| user_agent=$(echo $user_agent_base64_encoded | base64 -d) | |
| echo $user_agent | |
| # TODO: combine into one regex | |
| # User agent format: <format-version>,<client language>-<version>,... | |
| client_language=$(echo $user_agent | perl -n -E 'say $1 if m/[0-9]+, ([a-z]+) -/x') | |
| client_version=$(echo $user_agent | perl -n -E 'say $1 if m/[0-9]+,[a-z]+- ([0-9.a-zA-Z+]+),/x') | |
| echo $client_language | |
| echo $client_version | |
| test "$client_language" = "python" | |
| # Client version from user agent | |
| expected_client_version=$(python3 -m pip show aerospike | grep -i version | cut -d " " -f 2-) | |
| test "$client_version" = "$expected_client_version" | |
| shell: bash | |
| env: | |
| CONTAINER_NAME: aerospike |