Cache charmcraft pack container, skip unstable tests except on schedule (#47)

Ported from canonical/mysql-k8s-operator#146

Author: carlcsaposs-canonical

.github/workflows/ci.yaml

@@ -1,6 +1,12 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
 name: Tests
+
 on:
   pull_request:
+  schedule:
+    - cron: '53 0 * * *' # Daily at 00:53 UTC
+  # Triggered on push to branch "main" by .github/workflows/release.yaml
   workflow_call:
 
 jobs:
@@ -10,103 +16,70 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
         run: python3 -m pip install tox
       - name: Run linters
-        run: tox -e lint
+        run: tox run -e lint
 
   unit-test:
     name: Unit tests
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run tests
-        run: tox -e unit
-
-  integration-standalone:
-    name: Integration tests for standalone charm
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-          lxd-channel: 5.7/stable
-      - name: Run integration tests
-        run: tox -e standalone-integration
+        run: tox run -e unit
 
-  integration-backend:
-    name: Integration tests for backend relation
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-          lxd-channel: 5.7/stable
-      - name: Run integration tests
-        run: tox -e backend-integration
+  build:
+      name: Build charms
+      uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1
 
-  integration-client-relations:
-    name: Integration tests for client relations
+  integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-environments:
+          - standalone-integration
+          - backend-integration
+          - client-relation-integration
+          - legacy-client-relation-integration
+          - legacy-client-relation-integration-admin
+          - scaling-integration
+    name: ${{ matrix.tox-environments }}
     needs:
       - lint
       - unit-test
+      - build
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup operator environment
+        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: lxd
           lxd-channel: 5.7/stable
-      - name: Run integration tests
-        run: tox -e client-relation-integration
-
-  integration-legacy-client-relations:
-    name: Integration tests for legacy client relations
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
+      - name: Download packed charm(s)
+        uses: actions/download-artifact@v3
         with:
-          provider: lxd
-          lxd-channel: 5.7/stable
+          name: ${{ needs.build.outputs.artifact-name }}
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+          fi
       - name: Run integration tests
-        run: tox -e legacy-client-relation-integration
-
-  integration-scaling:
-    name: Integration tests for scaling pgbouncer (microk8s)
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-         provider: lxd
-         lxd-channel: 5.7/stable
-      - name: Run scaling integration tests
-        run: tox -e scaling-integration
+        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
+        env:
+          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}

CONTRIBUTING.md

@@ -21,21 +21,21 @@ this operator.
 
 ## Developing
 
-You can use the environments created by `tox` for development:
+You can create an environment for development with `tox`:
 
 ```shell
-tox --notest -e unit
-source .tox/unit/bin/activate
+tox devenv -e integration
+source venv/bin/activate
 ```
 
 ### Testing
 
 ```shell
-tox -e fmt           # update your code according to linting rules
-tox -e lint          # code style
-tox -e unit          # unit tests
-tox -e integration   # integration tests
-tox                  # runs 'lint' and 'unit' environments
+tox run -e format        # update your code according to linting rules
+tox run -e lint          # code style
+tox run -e unit          # unit tests
+tox run -e integration   # integration tests
+tox                      # runs 'lint' and 'unit' environments
 ```
 
 ## Build charm

pyproject.toml

@@ -12,15 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = [
-    "smoke: quick integration tests that can be run to quickly verify that the charm builds and deploys ok.",
-    "dev: a quick development marker to run one test",
-    "standalone: integration tests that relate to standalone charm function",
-    "backend: integration tests that test backend relation to postgresql",
-    "legacy_relation: integration tests that test legacy client relations",
-    "client_relation: integration tests that test modern client relations",
-    "scaling: integration tests for scaling pgbouncer",
-]
+markers = ["unstable"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/conftest.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from pytest_operator.plugin import OpsTest
+
+
+@pytest.fixture(scope="module")
+def ops_test(ops_test: OpsTest) -> OpsTest:
+    if os.environ.get("CI") == "true":
+        # Running in GitHub Actions; skip build step
+        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
+        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
+
+        async def build_charm(charm_path, bases_index: int = None) -> Path:
+            for charm in packed_charms:
+                if Path(charm_path) == Path(charm["directory_path"]):
+                    if bases_index is None or bases_index == charm["bases_index"]:
+                        return charm["file_path"]
+            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
+
+        ops_test.build_charm = build_charm
+    return ops_test

tests/integration/relations/pgbouncer_provider/test_pgbouncer_provider.py

@@ -44,7 +44,6 @@
 
 
 @pytest.mark.abort_on_fail
-@pytest.mark.client_relation
 async def test_database_relation_with_charm_libraries(
     ops_test: OpsTest, application_charm, pgb_charm
 ):
@@ -81,7 +80,6 @@ async def test_database_relation_with_charm_libraries(
     await ops_test.model.wait_for_idle(status="active", raise_on_blocked=True)
 
 
-@pytest.mark.client_relation
 async def test_database_usage(ops_test: OpsTest):
     """Check we can update and delete things."""
     update_query = (
@@ -100,7 +98,6 @@ async def test_database_usage(ops_test: OpsTest):
     assert "some data" in json.loads(run_update_query["results"])[0]
 
 
-@pytest.mark.client_relation
 async def test_database_version(ops_test: OpsTest):
     """Check version is accurate."""
     version_query = "SELECT version();"
@@ -120,7 +117,6 @@ async def test_database_version(ops_test: OpsTest):
     assert version in json.loads(run_version_query["results"])[0][0]
 
 
-@pytest.mark.client_relation
 async def test_readonly_reads(ops_test: OpsTest):
     """Check we can read things in readonly."""
     select_query = "SELECT data FROM test;"
@@ -136,7 +132,6 @@ async def test_readonly_reads(ops_test: OpsTest):
     assert "some data" in json.loads(run_select_query_readonly["results"])[0]
 
 
-@pytest.mark.client_relation
 async def test_cant_write_in_readonly(ops_test: OpsTest):
     """Check we can't write in readonly."""
     drop_query = "DROP TABLE test;"
@@ -151,7 +146,6 @@ async def test_cant_write_in_readonly(ops_test: OpsTest):
     assert run_drop_query_readonly["Code"] == "1"
 
 
-@pytest.mark.client_relation
 async def test_database_admin_permissions(ops_test: OpsTest):
     """Test admin permissions."""
     create_database_query = "CREATE DATABASE another_database;"
@@ -175,7 +169,6 @@ async def test_database_admin_permissions(ops_test: OpsTest):
     assert "no results to fetch" in json.loads(run_create_user_query["results"])
 
 
-@pytest.mark.client_relation
 async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest):
     """Test that there is no read-only endpoint in a standalone cluster."""
     await scale_application(ops_test, PGB, 1)
@@ -193,7 +186,6 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest):
     ), f"read-only-endpoints in pgb databag: {databag}"
 
 
-@pytest.mark.client_relation
 async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest):
     """Test that there is read-only endpoint in a scaled up cluster."""
     await scale_application(ops_test, PGB, 2)
@@ -210,7 +202,6 @@ async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest):
     assert read_only_endpoints, f"read-only-endpoints not in pgb databag: {databag}"
 
 
-@pytest.mark.client_relation
 async def test_two_applications_dont_share_the_same_relation_data(
     ops_test: OpsTest, application_charm
 ):
@@ -244,7 +235,6 @@ async def test_two_applications_dont_share_the_same_relation_data(
     assert application_connection_string != another_application_connection_string
 
 
-@pytest.mark.client_relation
 async def test_an_application_can_connect_to_multiple_database_clusters(
     ops_test: OpsTest, pgb_charm
 ):
@@ -293,7 +283,6 @@ async def test_an_application_can_connect_to_multiple_database_clusters(
     assert application_connection_string != another_application_connection_string
 
 
-@pytest.mark.client_relation
 async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, application_charm):
     """Test that an application can request additional databases using the same interface.
 
@@ -316,7 +305,6 @@ async def test_an_application_can_request_multiple_databases(ops_test: OpsTest,
     assert first_database_connection_string != second_database_connection_string
 
 
-@pytest.mark.client_relation
 async def test_with_legacy_relation(ops_test: OpsTest):
     """Test that this relation and the legacy relation can be used simultaneously."""
     psql = "psql"
@@ -373,7 +361,6 @@ async def test_with_legacy_relation(ops_test: OpsTest):
     )
 
 
-@pytest.mark.client_relation
 async def test_scaling(ops_test: OpsTest):
     """Check these relations all work when scaling pgbouncer."""
     await scale_application(ops_test, PGB, 1)
@@ -395,7 +382,6 @@ async def test_scaling(ops_test: OpsTest):
     )
 
 
-@pytest.mark.client_relation
 async def test_relation_broken(ops_test: OpsTest):
     """Test that the user is removed when the relation is broken."""
     client_unit_name = ops_test.model.applications[CLIENT_APP_NAME].units[0].name
@@ -427,7 +413,6 @@ async def test_relation_broken(ops_test: OpsTest):
     assert "first-database_readonly" not in cfg["databases"].keys()
 
 
-@pytest.mark.client_relation
 async def test_relation_with_data_integrator(ops_test: OpsTest):
     """Test that the charm can be related to the data integrator without extra user roles."""
     config = {"database-name": "test-database"}

tests/integration/relations/test_backend_database.py

@@ -5,7 +5,6 @@
 import logging
 from pathlib import Path
 
-import pytest
 import yaml
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
@@ -35,7 +34,6 @@
 RELATION = "backend-database"
 
 
-@pytest.mark.backend
 async def test_relate_pgbouncer_to_postgres(ops_test: OpsTest):
     """Test that the pgbouncer and postgres charms can relate to one another."""
     # Build, deploy, and relate charms.
@@ -80,7 +78,6 @@ async def test_relate_pgbouncer_to_postgres(ops_test: OpsTest):
     logger.info(cfg.render())
 
 
-@pytest.mark.backend
 async def test_tls_encrypted_connection_to_postgres(ops_test: OpsTest):
     async with ops_test.fast_forward():
         # Relate PgBouncer to PostgreSQL.