diff --git a/Dockerfile b/Dockerfile index 161c9484..a73f3943 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,7 @@ COPY ./backend/bin/celery_worker.sh /celery_worker.sh COPY ./backend/bin/celery_beat.sh /celery_beat.sh COPY ./backend/bin/celery_flower.sh /celery_flower.sh COPY ./backend/bin/check_celery_worker_liveness.py /check_celery_worker_liveness.py +COPY ./backend/bin/setup_configuration.sh /setup_configuration.sh COPY ./frontend/scripts/replace-envvars.sh /replace-envvars.sh RUN mkdir -p /app/log /app/media /app/src/openarchiefbeheer/static/ diff --git a/backend/bin/setup_configuration.sh b/backend/bin/setup_configuration.sh new file mode 100755 index 00000000..9905c150 --- /dev/null +++ b/backend/bin/setup_configuration.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Waiting for database to be up +until pg_isready; do + >&2 echo "Waiting for database connection..." + sleep 1 +done + +# Waiting for migrations to be done +attempt_counter=0 +max_attempts=${CHECK_MIGRATIONS_MAX_ATTEMPT:-10} + +while ! python src/manage.py migrate --check; do + attempt_counter=$((attempt_counter + 1)) + + if [ $attempt_counter -ge $max_attempts ]; then + >&2 echo "Timed out while waiting for django migrations." + exit 1 + fi + + >&2 echo "Attempt $attempt_counter/$max_attempts: Waiting for migrations to be done..." + sleep 10 +done + +# Run setup configuration +python src/manage.py setup_configuration --yaml-file setup_configuration/data.yaml \ No newline at end of file diff --git a/backend/docker-services/setup-configuration/data.yaml b/backend/docker-services/setup-configuration/data.yaml new file mode 100644 index 00000000..bf67e0fc --- /dev/null +++ b/backend/docker-services/setup-configuration/data.yaml @@ -0,0 +1,40 @@ +zgw_consumers_config_enable: True +zgw_consumers: + services: + - identifier: zaken-test + label: Open Zaak - Zaken API + api_root: http://localhost:8003/zaken/api/v1/ + api_type: zrc + auth_type: zgw + client_id: test-vcr + secret: test-vcr + - identifier: documenten-test + label: Open Zaak - Documenten API + api_root: http://localhost:8003/documenten/api/v1/ + api_type: drc + auth_type: zgw + client_id: test-vcr + secret: test-vcr + - identifier: catalogi-test + label: Open Zaak - Catalogi API + api_root: http://localhost:8003/catalogi/api/v1/ + api_type: ztc + auth_type: zgw + client_id: test-vcr + secret: test-vcr + - identifier: besluiten-test + label: Open Zaak - Besluiten API + api_root: http://localhost:8003/besluiten/api/v1/ + api_type: brc + auth_type: zgw + client_id: test-vcr + secret: test-vcr + - identifier: selectielijst + label: Open Zaak (public) - Selectielijst API + api_root: https://selectielijst.openzaak.nl/api/v1/ + api_type: orc + auth_type: no_auth + +api_configuration_enabled: True +api_configuration: + selectielijst_service_identifier: selectielijst diff --git a/backend/requirements/base.in b/backend/requirements/base.in index 0310fcde..cdbd1140 100644 --- a/backend/requirements/base.in +++ b/backend/requirements/base.in @@ -16,6 +16,7 @@ django-timeline-logger django-solo mozilla-django-oidc-db django-privates +django-setup-configuration # API libraries djangorestframework @@ -35,7 +36,7 @@ elastic-apm # Elastic APM integration celery # Additional libraries -zgw-consumers +zgw-consumers[setup-configuration] furl python-slugify XlsxWriter \ No newline at end of file diff --git a/backend/requirements/base.txt b/backend/requirements/base.txt index 21fc7b12..86a6fd04 100644 --- a/backend/requirements/base.txt +++ b/backend/requirements/base.txt @@ -2,6 +2,8 @@ # ./bin/compile_dependencies.sh amqp==5.2.0 # via kombu +annotated-types==0.7.0 + # via pydantic ape-pie==0.1.0 # via zgw-consumers asgiref==3.8.1 @@ -71,6 +73,7 @@ django==4.2.16 # django-relativedelta # django-rosetta # django-sendfile2 + # django-setup-configuration # django-simple-certmanager # django-solo # django-timeline-logger @@ -116,6 +119,10 @@ django-rosetta==0.10.0 # via -r requirements/base.in django-sendfile2==0.7.1 # via django-privates +django-setup-configuration==0.4.0 + # via + # -r requirements/base.in + # zgw-consumers django-simple-certmanager==2.0.0 # via zgw-consumers django-solo==2.2.0 @@ -185,6 +192,14 @@ psycopg2==2.9.9 # via -r requirements/base.in pycparser==2.22 # via cffi +pydantic==2.9.2 + # via + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-settings==2.6.1 + # via django-setup-configuration pyjwt==2.8.0 # via zgw-consumers pyopenssl==24.2.1 @@ -200,11 +215,15 @@ python-dateutil==2.9.0.post0 python-decouple==3.8 # via -r requirements/base.in python-dotenv==1.0.1 - # via -r requirements/base.in + # via + # -r requirements/base.in + # pydantic-settings python-slugify==8.0.4 # via -r requirements/base.in pyyaml==6.0.1 - # via drf-spectacular + # via + # drf-spectacular + # pydantic-settings qrcode==7.4.2 # via django-two-factor-auth redis==5.0.4 @@ -237,6 +256,8 @@ text-unidecode==1.3 typing-extensions==4.11.0 # via # mozilla-django-oidc-db + # pydantic + # pydantic-core # qrcode # zgw-consumers tzdata==2024.1 diff --git a/backend/requirements/ci.txt b/backend/requirements/ci.txt index 4745e0ea..4f13c780 100644 --- a/backend/requirements/ci.txt +++ b/backend/requirements/ci.txt @@ -7,6 +7,11 @@ amqp==5.2.0 # -c requirements/base.txt # -r requirements/base.txt # kombu +annotated-types==0.7.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # pydantic ape-pie==0.1.0 # via # -c requirements/base.txt @@ -134,6 +139,7 @@ django==4.2.16 # django-relativedelta # django-rosetta # django-sendfile2 + # django-setup-configuration # django-simple-certmanager # django-solo # django-timeline-logger @@ -220,6 +226,10 @@ django-sendfile2==0.7.1 # -c requirements/base.txt # -r requirements/base.txt # django-privates +django-setup-configuration==0.4.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt django-simple-certmanager==2.0.0 # via # -c requirements/base.txt @@ -423,6 +433,22 @@ pycparser==2.22 # -c requirements/base.txt # -r requirements/base.txt # cffi +pydantic==2.9.2 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # pydantic +pydantic-settings==2.6.1 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # django-setup-configuration pyee==11.1.0 # via playwright pyflakes==3.2.0 @@ -474,6 +500,7 @@ python-dotenv==1.0.1 # via # -c requirements/base.txt # -r requirements/base.txt + # pydantic-settings python-slugify==8.0.4 # via # -c requirements/base.txt @@ -484,6 +511,7 @@ pyyaml==6.0.1 # -c requirements/base.txt # -r requirements/base.txt # drf-spectacular + # pydantic-settings # vcrpy qrcode==7.4.2 # via @@ -578,6 +606,8 @@ typing-extensions==4.11.0 # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc-db + # pydantic + # pydantic-core # pyee # qrcode # zgw-consumers diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt index ef49ced7..2e52d99e 100644 --- a/backend/requirements/dev.txt +++ b/backend/requirements/dev.txt @@ -10,6 +10,11 @@ amqp==5.2.0 # -c requirements/ci.txt # -r requirements/ci.txt # kombu +annotated-types==0.7.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pydantic ape-pie==0.1.0 # via # -c requirements/ci.txt @@ -166,6 +171,7 @@ django==4.2.16 # django-relativedelta # django-rosetta # django-sendfile2 + # django-setup-configuration # django-simple-certmanager # django-solo # django-timeline-logger @@ -258,6 +264,10 @@ django-sendfile2==0.7.1 # -c requirements/ci.txt # -r requirements/ci.txt # django-privates +django-setup-configuration==0.4.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt django-simple-certmanager==2.0.0 # via # -c requirements/ci.txt @@ -536,6 +546,22 @@ pycparser==2.22 # -c requirements/ci.txt # -r requirements/ci.txt # cffi +pydantic==2.9.2 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pydantic +pydantic-settings==2.6.1 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # django-setup-configuration pyee==11.1.0 # via # -c requirements/ci.txt @@ -610,6 +636,7 @@ python-dotenv==1.0.1 # via # -c requirements/ci.txt # -r requirements/ci.txt + # pydantic-settings python-slugify==8.0.4 # via # -c requirements/ci.txt @@ -620,6 +647,7 @@ pyyaml==6.0.1 # -c requirements/ci.txt # -r requirements/ci.txt # drf-spectacular + # pydantic-settings # vcrpy qrcode==7.4.2 # via @@ -760,6 +788,8 @@ typing-extensions==4.11.0 # -c requirements/ci.txt # -r requirements/ci.txt # mozilla-django-oidc-db + # pydantic + # pydantic-core # pyee # qrcode # zgw-consumers diff --git a/backend/src/openarchiefbeheer/conf/base.py b/backend/src/openarchiefbeheer/conf/base.py index 83cdbdc8..93f78eb9 100644 --- a/backend/src/openarchiefbeheer/conf/base.py +++ b/backend/src/openarchiefbeheer/conf/base.py @@ -132,6 +132,7 @@ "mozilla_django_oidc", "mozilla_django_oidc_db", "privates", + "django_setup_configuration", # Project applications. "openarchiefbeheer.accounts", "openarchiefbeheer.destruction", @@ -645,8 +646,17 @@ "OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS", default=60 * 15 ) +# # Django privates # PRIVATE_MEDIA_ROOT = os.path.join(BASE_DIR, "private_media") PRIVATE_MEDIA_URL = "/private-media/" + +# +# Django setup configuration +# +SETUP_CONFIGURATION_STEPS = [ + "zgw_consumers.contrib.setup_configuration.steps.ServiceConfigurationStep", + "openarchiefbeheer.config.setup_configuration.steps.APIConfigConfigurationStep", +] diff --git a/backend/src/openarchiefbeheer/config/setup_configuration/__init__.py b/backend/src/openarchiefbeheer/config/setup_configuration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/openarchiefbeheer/config/setup_configuration/models.py b/backend/src/openarchiefbeheer/config/setup_configuration/models.py new file mode 100644 index 00000000..3bb015e0 --- /dev/null +++ b/backend/src/openarchiefbeheer/config/setup_configuration/models.py @@ -0,0 +1,10 @@ +from django_setup_configuration import ConfigurationModel +from django_setup_configuration.fields import DjangoModelRef + +from ..models import APIConfig + + +class APIConfigConfigurationModel(ConfigurationModel): + selectielijst_service_identifier: str = DjangoModelRef( + APIConfig, "selectielijst_api_service" + ) diff --git a/backend/src/openarchiefbeheer/config/setup_configuration/steps.py b/backend/src/openarchiefbeheer/config/setup_configuration/steps.py new file mode 100644 index 00000000..7f6188b3 --- /dev/null +++ b/backend/src/openarchiefbeheer/config/setup_configuration/steps.py @@ -0,0 +1,30 @@ +from django_setup_configuration import BaseConfigurationStep +from django_setup_configuration.exceptions import ConfigurationRunFailed +from zgw_consumers.models import Service + +from ..models import APIConfig +from .models import APIConfigConfigurationModel + + +class APIConfigConfigurationStep(BaseConfigurationStep[APIConfigConfigurationModel]): + """Configure API settings""" + + config_model = APIConfigConfigurationModel + enable_setting = "api_configuration_enabled" + namespace = "api_configuration" + verbose_name = "API Configuration" + + def execute(self, model: APIConfigConfigurationModel) -> None: + config = APIConfig.get_solo() + + try: + config.selectielijst_api_service = Service.objects.get( + slug=model.selectielijst_service_identifier + ) + except Service.DoesNotExist: + raise ConfigurationRunFailed( + f"Could not find an existing `selectielijst` service with identifier `{model.selectielijst_service_identifier}`." + " Make sure it is already configured, manually or by first running the configuration step of `zgw_consumers`." + ) + + config.save(update_fields=["selectielijst_api_service"]) diff --git a/backend/src/openarchiefbeheer/config/setup_configuration/tests/__init__.py b/backend/src/openarchiefbeheer/config/setup_configuration/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/openarchiefbeheer/config/setup_configuration/tests/files/setup_config_api.yaml b/backend/src/openarchiefbeheer/config/setup_configuration/tests/files/setup_config_api.yaml new file mode 100644 index 00000000..aba93364 --- /dev/null +++ b/backend/src/openarchiefbeheer/config/setup_configuration/tests/files/setup_config_api.yaml @@ -0,0 +1,3 @@ +api_configuration_enabled: True +api_configuration: + selectielijst_service_identifier: selectielijst \ No newline at end of file diff --git a/backend/src/openarchiefbeheer/config/setup_configuration/tests/test_setup_configuration.py b/backend/src/openarchiefbeheer/config/setup_configuration/tests/test_setup_configuration.py new file mode 100644 index 00000000..3b772b47 --- /dev/null +++ b/backend/src/openarchiefbeheer/config/setup_configuration/tests/test_setup_configuration.py @@ -0,0 +1,70 @@ +from pathlib import Path + +from django.core.cache import cache +from django.test import TestCase + +from django_setup_configuration.exceptions import ConfigurationRunFailed +from django_setup_configuration.test_utils import execute_single_step +from zgw_consumers.constants import APITypes +from zgw_consumers.test.factories import ServiceFactory + +from ...models import APIConfig +from ..steps import APIConfigConfigurationStep + +TEST_FILES = (Path(__file__).parent / "files").resolve() +CONFIG_FILE_PATH = str(TEST_FILES / "setup_config_api.yaml") + + +class APIConfigConfigurationStepTests(TestCase): + def setUp(self): + super().setUp() + + self.addCleanup(cache.clear) + + def test_configure_api_config_create_new(self): + service = ServiceFactory( + slug="selectielijst", + api_type=APITypes.orc, + api_root="https://selectielijst.openzaak.nl/api/v1/", + ) + config = APIConfig.get_solo() + + self.assertIsNone(config.selectielijst_api_service) + + execute_single_step(APIConfigConfigurationStep, yaml_source=CONFIG_FILE_PATH) + + config.refresh_from_db() + + self.assertEqual(service.pk, config.selectielijst_api_service.pk) + + def test_configure_api_config_update_existing(self): + service1 = ServiceFactory( + slug="selectielijst", + ) + service2 = ServiceFactory( + slug="selectielijst-new", + ) + + config = APIConfig.get_solo() + config.selectielijst_api_service = service1 + config.save() + + execute_single_step( + APIConfigConfigurationStep, + object_source={ + "api_configuration_enabled": True, + "api_configuration": { + "selectielijst_service_identifier": "selectielijst-new" + }, + }, + ) + + config.refresh_from_db() + + self.assertEqual(service2.pk, config.selectielijst_api_service.pk) + + def test_configure_api_config_missing_service(self): + with self.assertRaises(ConfigurationRunFailed): + execute_single_step( + APIConfigConfigurationStep, yaml_source=CONFIG_FILE_PATH + ) diff --git a/docker-compose.yml b/docker-compose.yml index 35408154..296700ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,18 @@ services: networks: - open-archiefbeheer-dev + web-init: + build: . + environment: *web_env + command: /setup_configuration.sh + volumes: + - ./backend/docker-services/setup-configuration:/app/setup_configuration + depends_on: + - db + - redis + networks: + - open-archiefbeheer-dev + celery: build: . command: /celery_worker.sh