✨ [#4788] ConfigurationStep for Objects API registration config

.github/workflows/ci.yml

@@ -442,7 +442,8 @@ jobs:
     needs:
       - docker_build_setup
       - docker_build
-
+    env:
+      RUN_SETUP_CONFIG: "False"  # Disable running the setup_configuration
     name: Simulate upgrading instances
     runs-on: ubuntu-latest
     strategy:

Dockerfile

@@ -16,6 +16,8 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
         build-essential \
         python3-dev \
         libpq-dev \
+        # TODO should be removed
+        git \
         shared-mime-info \
         # lxml/xmlsec deps
         zlib1g-dev \
@@ -86,6 +88,8 @@ COPY ./bin/celery_worker.sh /celery_worker.sh
 COPY ./bin/celery_beat.sh /celery_beat.sh
 COPY ./bin/celery_flower.sh /celery_flower.sh
 COPY ./bin/dump_configuration.sh /dump_configuration.sh
+COPY ./bin/wait_for_db.sh /wait_for_db.sh
+COPY ./bin/setup_configuration.sh /setup_configuration.sh
 RUN mkdir /app/bin /app/log /app/media /app/private_media /app/certifi_ca_bundle /app/tmp
 COPY \
     ./bin/check_celery_worker_liveness.py \

bin/docker_start.sh

@@ -2,11 +2,6 @@
 
 set -ex
 
-# Wait for the database container
-# See: https://docs.docker.com/compose/startup-order/
-export PGHOST=${DB_HOST:-db}
-export PGPORT=${DB_PORT:-5432}
-
 fixtures_dir=${FIXTURES_DIR:-/app/fixtures}
 
 uwsgi_port=${UWSGI_PORT:-8000}
@@ -15,12 +10,8 @@ uwsgi_threads=${UWSGI_THREADS:-1}
 
 mountpoint=${SUBPATH:-/}
 
-until pg_isready; do
-  >&2 echo "Waiting for database connection..."
-  sleep 1
-done
-
->&2 echo "Database is up."
+# wait for required services
+${SCRIPTPATH}/wait_for_db.sh
 
 # Apply database migrations
 >&2 echo "Apply database migrations"

bin/setup_configuration.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# setup initial configuration using environment variables
+# Run this script from the root of the repository
+
+set -e
+
+if [[ "${RUN_SETUP_CONFIG,,}" =~ ^(true|1|yes)$ ]]; then
+    # wait for required services
+    ${SCRIPTPATH}/wait_for_db.sh
+
+    if [[ "${SKIP_SELFTEST,,}" =~ ^(true|1|yes)$ ]]; then
+        NO_SELFTEST_FLAG="--no-selftest"
+    else
+        NO_SELFTEST_FLAG=""
+    fi
+
+    src/manage.py migrate
+    src/manage.py setup_configuration $NO_SELFTEST_FLAG \
+                    --yaml-file data/services.yaml \
+                    --yaml-file data/objects_api.yaml
+fi

bin/wait_for_db.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Wait for the database container
+# See: https://docs.docker.com/compose/startup-order/
+export PGHOST=${DB_HOST:-db}
+export PGPORT=${DB_PORT:-5432}
+
+until pg_isready; do
+  >&2 echo "Waiting for database connection..."
+  sleep 1
+done
+
+>&2 echo "Database is up."

docker-compose.yml

@@ -85,18 +85,39 @@ services:
       - ENVIRONMENT_LABEL=docker-compose
       - ENVIRONMENT_BACKGROUND_COLOR=#1d63ed
       - ENVIRONMENT_FOREGROUND_COLOR=white
+      # Django-setup-configuration
+      - RUN_SETUP_CONFIG=${RUN_SETUP_CONFIG:-True}
+      - SKIP_SELFTEST=${SKIP_SELFTEST:-False}
     volumes: &web_volumes
       - media:/app/media
       - private_media:/app/private_media
+      - ./docker/setup_configuration:/app/data
       - log:/app/log
       - certifi_ca_bundle:/app/certifi_ca_bundle
     ports:
       - 8000:8000
+    depends_on:
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
+      smtp:
+        condition: service_started
+      clamav:
+        condition: service_started
+      web-init:
+        condition: service_completed_successfully
+    networks:
+      - open-forms-dev
+
+  web-init:
+    build: .
+    environment: *web_env
+    command: /setup_configuration.sh
+    volumes: *web_volumes
     depends_on:
       - db
       - redis
-      - smtp
-      - clamav
     networks:
       - open-forms-dev
 

docker/setup_configuration/objects_api.yaml

@@ -0,0 +1,22 @@
+OBJECTS_API_CONFIG_ENABLE: True
+OBJECTS_API:
+  groups:
+  - name: Config 1
+    objects_service_slug: objecten-test
+    objecttypes_service_slug: objecttypen-test
+    drc_service_slug: documenten-test
+    catalogi_service_slug: catalogi-test
+    catalogue_domain: TEST
+    catalogue_rsin: "000000000"
+    organisatie_rsin: "000000000"
+    iot_submission_report: PDF Informatieobjecttype
+    iot_submission_csv: CSV Informatieobjecttype
+    iot_attachment: Attachment Informatieobjecttype
+  - name: Config 2
+    objects_service_slug: objecten-test
+    objecttypes_service_slug: objecttypen-test
+    drc_service_slug: documenten-test
+    catalogi_service_slug: catalogi-test
+    catalogue_domain: OTHER
+    catalogue_rsin: "000000000"
+    organisatie_rsin: "000000000"

docker/setup_configuration/services.yaml

@@ -0,0 +1,31 @@
+ZGW_CONSUMERS_CONFIG_ENABLE: True
+ZGW_CONSUMERS:
+  services:
+  - slug: objecttypen-test
+    label: Objecttypen API test
+    api_root: http://objecttypes-web:8000/api/v2/
+    api_type: orc
+    auth_type: api_key
+    header_key: Authorization
+    header_value: Token 171be5abaf41e7856b423ad513df1ef8f867ff48
+  - slug: objecten-test
+    label: Objecten API test
+    api_root: http://objects-web:8000/api/v2/
+    api_type: orc
+    auth_type: api_key
+    header_key: Authorization
+    header_value: Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9
+  - slug: documenten-test
+    label: Documenten API test
+    api_root: http://openzaak-web.local:8000/documenten/api/v1/
+    api_type: drc
+    auth_type: zgw
+    client_id: test_client_id
+    secret: test_secret_key
+  - slug: catalogi-test
+    label: Catalogi API test
+    api_root: http://openzaak-web.local:8000/catalogi/api/v1/
+    api_type: ztc
+    auth_type: zgw
+    client_id: test_client_id
+    secret: test_secret_key

docs/installation/config.rst

@@ -358,10 +358,10 @@ Other settings
 * ``SENDFILE_BACKEND``: which backend to use to serve the content of non-public files. The value depends on the
   reverse proxy solution used with Open Forms. For available backends, see the `django-sendfile documentation`_.
   Defaults to ``sendfile.backends.nginx``.
-  
+
   .. note:: Open Forms only considers nginx to be in scope. You can deviate from using nginx, but we cannot offer any
     support on other backends.
-  
+
 .. _django-sendfile documentation: https://django-sendfile2.readthedocs.io/en/stable/backends.html
 
 .. _`Django DATABASE settings`: https://docs.djangoproject.com/en/4.2/ref/settings/#engine
@@ -393,6 +393,13 @@ variables, linking to the description of their behaviour in their respective mod
      installations likely do not provide all this information yet.
 
 
+Initial configuration
+---------------------
+
+Open Forms supports the ``setup_configuration`` management command, which allows configuration via
+environment variables.
+All these environment variables are described at :ref:`installation_configuration_cli`.
+
 Specifying the environment variables
 =====================================
 

docs/installation/index.rst

@@ -24,6 +24,7 @@ and expertise.
    ansible
    security
    config
+   setup_configuration
    file_uploads
    self_signed
    form_hosting

docs/installation/setup_configuration.rst

@@ -0,0 +1,74 @@
+.. _installation_configuration_cli:
+
+==============================
+Open Forms configuration (CLI)
+==============================
+
+After deploying Open Forms, it needs to be configured to be fully functional. The
+command line tool ``setup_configuration`` assist with this configuration:
+
+* It uses environment variables for all configuration choices, therefore you can integrate this with your
+  infrastructure tooling such as init containers and/or Kubernetes Jobs.
+* The command can self-test the configuration to detect problems early on
+
+You can get the full command documentation with:
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --help
+
+.. warning:: This command is declarative - if configuration is manually changed after
+   running the command and you then run the exact same command again, the manual
+   changes will be reverted.
+
+Preparation
+===========
+
+The command executes the list of pluggable configuration steps, and each step
+has required specific environment variables, that should be prepared.
+Here is the description of all available configuration steps and the environment variables,
+used by each step.
+
+
+Services configuration
+----------------------
+
+TODO: add generated documentation for ``zgw_consumers.ServiceConfigurationStep``
+
+Objects API registration configuration
+--------------------------------------
+
+TODO: add generated documentation for ``ObjectsAPIConfigurationStep``
+
+Execution
+=========
+
+Open Forms configuration
+------------------------
+
+With the full command invocation, everything is configured at once and immediately
+tested. For all the self-tests to succeed, it's important that services that are to be configured
+can be reached by Open Forms.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration
+
+
+Alternatively, you can skip the self-tests by using the ``--no-selftest`` flag.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --no-self-test
+
+
+``setup_configuration`` command checks if the configuration already exists before changing it.
+If you want to change some of the values of the existing configuration you can use ``--overwrite`` flag.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --overwrite
+
+
+.. note:: Due to a cache-bug in the underlying framework, you need to restart all
+   replicas for part of this change to take effect everywhere.

requirements/base.in

@@ -86,7 +86,7 @@ elastic-apm  # Elastic APM integration
 flower # task monitoring
 
 # Common Ground integration
-zgw-consumers
+zgw-consumers[setup-configuration] @ git+https://github.com/maykinmedia/zgw-consumers@feature/django-setup-configuration#egg=zgw-consumers
 
 # Anti virus scan
 clamd

requirements/base.txt

@@ -2,6 +2,8 @@
 #    ./bin/compile_dependencies.sh
 amqp==5.2.0
     # via kombu
+annotated-types==0.7.0
+    # via pydantic
 ape-pie==0.1.0
     # via
     #   -r requirements/base.in
@@ -110,6 +112,7 @@ django==4.2.16
     #   django-relativedelta
     #   django-sendfile2
     #   django-sessionprofile
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-timeline-logger
@@ -194,6 +197,8 @@ django-sendfile2==0.7.1
     # via django-privates
 django-sessionprofile==2.0.0
     # via django-digid-eherkenning
+django-setup-configuration @ git+https://github.com/maykinmedia/django-setup-configuration@b3600c3a75d1bb49fce8730a39bdc1acbc94edcf#egg=django-setup-configuration
+    # via zgw-consumers
 django-simple-certmanager==2.4.1
     # via
     #   -r requirements/base.in
@@ -369,6 +374,14 @@ pycountry==23.12.11
     # via schwifty
 pycparser==2.20
     # via cffi
+pydantic==2.9.2
+    # via
+    #   django-setup-configuration
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via pydantic
+pydantic-settings==2.6.1
+    # via django-setup-configuration
 pydyf==0.8.0
     # via weasyprint
 pyjwt==2.6.0
@@ -389,8 +402,10 @@ python-dateutil==2.9.0.post0
     #   o365
 python-decouple==3.8
     # via -r requirements/base.in
-python-dotenv==0.14.0
-    # via -r requirements/base.in
+python-dotenv==1.0.1
+    # via
+    #   -r requirements/base.in
+    #   pydantic-settings
 python-magic==0.4.27
     # via -r requirements/base.in
 pytz==2024.1
@@ -404,6 +419,7 @@ pyyaml==6.0.1
     # via
     #   drf-spectacular
     #   jsonschema-spec
+    #   pydantic-settings
     #   tablib
 qrcode==7.4.2
     # via django-two-factor-auth
@@ -499,6 +515,8 @@ typing-extensions==4.11.0
     #   -r requirements/base.in
     #   mozilla-django-oidc-db
     #   psycopg
+    #   pydantic
+    #   pydantic-core
     #   qrcode
     #   zgw-consumers
 tzdata==2023.3
@@ -545,7 +563,7 @@ xmltodict==0.12.0
     # via -r requirements/base.in
 zeep==4.2.1
     # via -r requirements/base.in
-zgw-consumers==0.32.0
+zgw-consumers @ git+https://github.com/maykinmedia/zgw-consumers@7f0c147a670e513328c864021a4baba5219f034e#egg=zgw-consumers
     # via -r requirements/base.in
 zopfli==0.2.3
     # via fonttools