Merge pull request #524 from maykinmedia/feature/458-add-django-confi…

…g-zgw

[#458] Add django setup configuration and ZGW steps

Dockerfile

@@ -71,6 +71,7 @@ COPY ./backend/bin/celery_worker.sh /celery_worker.sh
 COPY ./backend/bin/celery_beat.sh /celery_beat.sh
 COPY ./backend/bin/celery_flower.sh /celery_flower.sh
 COPY ./backend/bin/check_celery_worker_liveness.py /check_celery_worker_liveness.py
+COPY ./backend/bin/setup_configuration.sh /setup_configuration.sh
 COPY ./frontend/scripts/replace-envvars.sh /replace-envvars.sh
 
 RUN mkdir -p /app/log /app/media /app/src/openarchiefbeheer/static/

backend/bin/setup_configuration.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Waiting for database to be up
+until pg_isready; do
+  >&2 echo "Waiting for database connection..."
+  sleep 1
+done
+
+# Waiting for migrations to be done
+attempt_counter=0
+max_attempts=${CHECK_MIGRATIONS_MAX_ATTEMPT:-10}
+
+while ! python src/manage.py migrate --check; do
+  attempt_counter=$((attempt_counter + 1))
+
+  if [ $attempt_counter -ge $max_attempts ]; then
+    >&2 echo "Timed out while waiting for django migrations."
+    exit 1
+  fi
+
+  >&2 echo "Attempt $attempt_counter/$max_attempts: Waiting for migrations to be done..."
+  sleep 10
+done
+
+# Run setup configuration
+python src/manage.py setup_configuration --yaml-file setup_configuration/data.yaml

backend/docker-services/setup-configuration/data.yaml

@@ -0,0 +1,40 @@
+zgw_consumers_config_enable: True
+zgw_consumers:
+  services:
+  - identifier: zaken-test
+    label: Open Zaak - Zaken API
+    api_root: http://localhost:8003/zaken/api/v1/
+    api_type: zrc
+    auth_type: zgw
+    client_id: test-vcr
+    secret: test-vcr
+  - identifier: documenten-test
+    label: Open Zaak - Documenten API
+    api_root: http://localhost:8003/documenten/api/v1/
+    api_type: drc
+    auth_type: zgw
+    client_id: test-vcr
+    secret: test-vcr
+  - identifier: catalogi-test
+    label: Open Zaak - Catalogi API
+    api_root: http://localhost:8003/catalogi/api/v1/
+    api_type: ztc
+    auth_type: zgw
+    client_id: test-vcr
+    secret: test-vcr
+  - identifier: besluiten-test
+    label: Open Zaak - Besluiten API
+    api_root: http://localhost:8003/besluiten/api/v1/
+    api_type: brc
+    auth_type: zgw
+    client_id: test-vcr
+    secret: test-vcr
+  - identifier: selectielijst
+    label: Open Zaak (public) - Selectielijst API
+    api_root: https://selectielijst.openzaak.nl/api/v1/
+    api_type: orc
+    auth_type: no_auth
+
+api_configuration_enabled: True
+api_configuration:
+  selectielijst_service_identifier: selectielijst

backend/requirements/base.in

@@ -16,6 +16,7 @@ django-timeline-logger
 django-solo
 mozilla-django-oidc-db
 django-privates
+django-setup-configuration
 
 # API libraries
 djangorestframework
@@ -35,7 +36,7 @@ elastic-apm  # Elastic APM integration
 celery
 
 # Additional libraries
-zgw-consumers
+zgw-consumers[setup-configuration]
 furl
 python-slugify
 XlsxWriter

backend/requirements/base.txt

@@ -2,6 +2,8 @@
 #    ./bin/compile_dependencies.sh
 amqp==5.2.0
     # via kombu
+annotated-types==0.7.0
+    # via pydantic
 ape-pie==0.1.0
     # via zgw-consumers
 asgiref==3.8.1
@@ -71,6 +73,7 @@ django==4.2.16
     #   django-relativedelta
     #   django-rosetta
     #   django-sendfile2
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-timeline-logger
@@ -116,6 +119,10 @@ django-rosetta==0.10.0
     # via -r requirements/base.in
 django-sendfile2==0.7.1
     # via django-privates
+django-setup-configuration==0.4.0
+    # via
+    #   -r requirements/base.in
+    #   zgw-consumers
 django-simple-certmanager==2.0.0
     # via zgw-consumers
 django-solo==2.2.0
@@ -185,6 +192,14 @@ psycopg2==2.9.9
     # via -r requirements/base.in
 pycparser==2.22
     # via cffi
+pydantic==2.9.2
+    # via
+    #   django-setup-configuration
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via pydantic
+pydantic-settings==2.6.1
+    # via django-setup-configuration
 pyjwt==2.8.0
     # via zgw-consumers
 pyopenssl==24.2.1
@@ -200,11 +215,15 @@ python-dateutil==2.9.0.post0
 python-decouple==3.8
     # via -r requirements/base.in
 python-dotenv==1.0.1
-    # via -r requirements/base.in
+    # via
+    #   -r requirements/base.in
+    #   pydantic-settings
 python-slugify==8.0.4
     # via -r requirements/base.in
 pyyaml==6.0.1
-    # via drf-spectacular
+    # via
+    #   drf-spectacular
+    #   pydantic-settings
 qrcode==7.4.2
     # via django-two-factor-auth
 redis==5.0.4
@@ -237,6 +256,8 @@ text-unidecode==1.3
 typing-extensions==4.11.0
     # via
     #   mozilla-django-oidc-db
+    #   pydantic
+    #   pydantic-core
     #   qrcode
     #   zgw-consumers
 tzdata==2024.1

backend/requirements/ci.txt

@@ -7,6 +7,11 @@ amqp==5.2.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   pydantic
 ape-pie==0.1.0
     # via
     #   -c requirements/base.txt
@@ -134,6 +139,7 @@ django==4.2.16
     #   django-relativedelta
     #   django-rosetta
     #   django-sendfile2
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-timeline-logger
@@ -220,6 +226,10 @@ django-sendfile2==0.7.1
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   django-privates
+django-setup-configuration==0.4.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-simple-certmanager==2.0.0
     # via
     #   -c requirements/base.txt
@@ -423,6 +433,22 @@ pycparser==2.22
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   cffi
+pydantic==2.9.2
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   django-setup-configuration
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   pydantic
+pydantic-settings==2.6.1
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   django-setup-configuration
 pyee==11.1.0
     # via playwright
 pyflakes==3.2.0
@@ -474,6 +500,7 @@ python-dotenv==1.0.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+    #   pydantic-settings
 python-slugify==8.0.4
     # via
     #   -c requirements/base.txt
@@ -484,6 +511,7 @@ pyyaml==6.0.1
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   drf-spectacular
+    #   pydantic-settings
     #   vcrpy
 qrcode==7.4.2
     # via
@@ -578,6 +606,8 @@ typing-extensions==4.11.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   mozilla-django-oidc-db
+    #   pydantic
+    #   pydantic-core
     #   pyee
     #   qrcode
     #   zgw-consumers

backend/requirements/dev.txt

@@ -10,6 +10,11 @@ amqp==5.2.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+    #   pydantic
 ape-pie==0.1.0
     # via
     #   -c requirements/ci.txt
@@ -166,6 +171,7 @@ django==4.2.16
     #   django-relativedelta
     #   django-rosetta
     #   django-sendfile2
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-timeline-logger
@@ -258,6 +264,10 @@ django-sendfile2==0.7.1
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   django-privates
+django-setup-configuration==0.4.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 django-simple-certmanager==2.0.0
     # via
     #   -c requirements/ci.txt
@@ -536,6 +546,22 @@ pycparser==2.22
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   cffi
+pydantic==2.9.2
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+    #   django-setup-configuration
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+    #   pydantic
+pydantic-settings==2.6.1
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+    #   django-setup-configuration
 pyee==11.1.0
     # via
     #   -c requirements/ci.txt
@@ -610,6 +636,7 @@ python-dotenv==1.0.1
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   pydantic-settings
 python-slugify==8.0.4
     # via
     #   -c requirements/ci.txt
@@ -620,6 +647,7 @@ pyyaml==6.0.1
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   drf-spectacular
+    #   pydantic-settings
     #   vcrpy
 qrcode==7.4.2
     # via
@@ -760,6 +788,8 @@ typing-extensions==4.11.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   mozilla-django-oidc-db
+    #   pydantic
+    #   pydantic-core
     #   pyee
     #   qrcode
     #   zgw-consumers

backend/src/openarchiefbeheer/conf/base.py

@@ -132,6 +132,7 @@
     "mozilla_django_oidc",
     "mozilla_django_oidc_db",
     "privates",
+    "django_setup_configuration",
     # Project applications.
     "openarchiefbeheer.accounts",
     "openarchiefbeheer.destruction",
@@ -645,8 +646,17 @@
     "OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS", default=60 * 15
 )
 
+#
 # Django privates
 #
 PRIVATE_MEDIA_ROOT = os.path.join(BASE_DIR, "private_media")
 
 PRIVATE_MEDIA_URL = "/private-media/"
+
+#
+# Django setup configuration
+#
+SETUP_CONFIGURATION_STEPS = [
+    "zgw_consumers.contrib.setup_configuration.steps.ServiceConfigurationStep",
+    "openarchiefbeheer.config.setup_configuration.steps.APIConfigConfigurationStep",
+]

backend/src/openarchiefbeheer/config/setup_configuration/models.py

@@ -0,0 +1,10 @@
+from django_setup_configuration import ConfigurationModel
+from django_setup_configuration.fields import DjangoModelRef
+
+from ..models import APIConfig
+
+
+class APIConfigConfigurationModel(ConfigurationModel):
+    selectielijst_service_identifier: str = DjangoModelRef(
+        APIConfig, "selectielijst_api_service"
+    )

backend/src/openarchiefbeheer/config/setup_configuration/steps.py

@@ -0,0 +1,30 @@
+from django_setup_configuration import BaseConfigurationStep
+from django_setup_configuration.exceptions import ConfigurationRunFailed
+from zgw_consumers.models import Service
+
+from ..models import APIConfig
+from .models import APIConfigConfigurationModel
+
+
+class APIConfigConfigurationStep(BaseConfigurationStep[APIConfigConfigurationModel]):
+    """Configure API settings"""
+
+    config_model = APIConfigConfigurationModel
+    enable_setting = "api_configuration_enabled"
+    namespace = "api_configuration"
+    verbose_name = "API Configuration"
+
+    def execute(self, model: APIConfigConfigurationModel) -> None:
+        config = APIConfig.get_solo()
+
+        try:
+            config.selectielijst_api_service = Service.objects.get(
+                slug=model.selectielijst_service_identifier
+            )
+        except Service.DoesNotExist:
+            raise ConfigurationRunFailed(
+                f"Could not find an existing `selectielijst` service with identifier `{model.selectielijst_service_identifier}`."
+                " Make sure it is already configured, manually or by first running the configuration step of `zgw_consumers`."
+            )
+
+        config.save(update_fields=["selectielijst_api_service"])

backend/src/openarchiefbeheer/config/setup_configuration/tests/files/setup_config_api.yaml

@@ -0,0 +1,3 @@
+api_configuration_enabled: True
+api_configuration:
+  selectielijst_service_identifier: selectielijst