chore: update CI conf to run Docker tests in parallel (#547)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

.github/workflows/check-code-and-docs-validation.yml

@@ -1,4 +1,4 @@
-name: Run unit tests
+name: Check code and docs validation
 
 on:
   workflow_dispatch:

.github/workflows/run-unit-tests-docker.yml

@@ -22,5 +22,10 @@ jobs:
       - name: Generate the .env file and the SECRET_KEY
         run: make envfile
 
+      - name: Build Docker image
+        run: docker compose -f compose.yml -f compose.build.yml build
+
       - name: Run tests
-        run: docker compose run web python ./manage.py test --verbosity=2 --noinput
+        run: |
+          docker compose -f compose.yml -f compose.build.yml run web \
+            python ./manage.py test --verbosity=2 --noinput --parallel auto

Makefile

@@ -47,7 +47,7 @@ superuser:
 	${MANAGE} createsuperuser
 
 ########################################################################################
-# Utilities
+# Linters / docs
 ########################################################################################
 
 DOCS_LOCATION=./docs
@@ -76,36 +76,12 @@ docs:
 	uvx --from sphinx==9.1.0 --with furo==2025.12.19 sphinx-build -b html ${DOCS_LOCATION} ${DOCS_LOCATION}/_build/html/
 
 ########################################################################################
-
-VENV_LOCATION=.venv
-ACTIVATE?=. ${VENV_LOCATION}/bin/activate;
-#MANAGE=${VENV_LOCATION}/bin/python manage.py
-# Do not depend on Python to generate the SECRET_KEY
-GET_SECRET_KEY=`head -c50 /dev/urandom | base64 | head -c50`
-# Customize with `$ make envfile ENV_FILE=/etc/dejacode/.env`
-ENV_FILE=.env
-DOCKER_COMPOSE=docker compose -f docker-compose.yml
-DOCKER_EXEC=${DOCKER_COMPOSE} exec
-DB_NAME=dejacode_db
-DB_USERNAME=dejacode
-DB_PASSWORD=dejacode
-DB_CONTAINER_NAME=db
-DB_INIT_FILE=./data/postgresql/initdb.sql.gz
-POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8
-TIMESTAMP=$(shell date +"%Y-%m-%d_%H%M")
-
-conf: virtualenv
-	@echo "-> Install dependencies"
-	uv sync --frozen
-	@echo "-> Create the var/ directory"
-	@mkdir -p var
-
-dev: virtualenv
-	@echo "-> Configure and install development dependencies"
-	uv sync --frozen --extra dev
+# Utilities
+########################################################################################
 
 outdated:
 	@echo "-> Check for outdated packages (with 7 days cooldown)"
+	uv sync --frozen --quiet
 	uv pip list --outdated \
 		--no-config \
 		--index-url https://pypi.org/simple \
@@ -119,25 +95,61 @@ upgrade:
 		exit 1; \
 	fi
 	@echo "-> Download $(PACKAGE) wheels for Linux x86_64"
-	pip download $(PACKAGE) \
+	uvx pip download $(PACKAGE) \
 		--only-binary=:all: \
 		--platform manylinux_2_28_x86_64 \
 		--platform manylinux_2_17_x86_64 \
 		--python-version 3.14 \
 		--dest ./thirdparty/dist/
 	@echo "-> Download $(PACKAGE) wheels for macOS ARM64"
-	pip download $(PACKAGE) \
+	uvx pip download $(PACKAGE) \
 		--only-binary=:all: \
 		--platform macosx_11_0_arm64 \
 		--python-version 3.14 \
 		--dest ./thirdparty/dist/
 	@echo "-> Update pyproject.toml and uv.lock"
-	uv add $(PACKAGE)
+	uvx uv add $(PACKAGE)
 
 lock:
 	@echo "-> Regenerate uv.lock from local wheels"
 	uv lock
 
+clean:
+	@echo "-> Clean the Python env"
+	rm -rf .venv/ .*_cache/ *.egg-info/ build/ dist/
+	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+
+########################################################################################
+# Local venv commands (legacy)
+########################################################################################
+
+VENV_LOCATION=.venv
+ACTIVATE?=. ${VENV_LOCATION}/bin/activate;
+#MANAGE=${VENV_LOCATION}/bin/python manage.py
+# Do not depend on Python to generate the SECRET_KEY
+GET_SECRET_KEY=`head -c50 /dev/urandom | base64 | head -c50`
+# Customize with `$ make envfile ENV_FILE=/etc/dejacode/.env`
+ENV_FILE=.env
+DOCKER_COMPOSE=docker compose -f docker-compose.yml
+DOCKER_EXEC=${DOCKER_COMPOSE} exec
+DB_NAME=dejacode_db
+DB_USERNAME=dejacode
+DB_PASSWORD=dejacode
+DB_CONTAINER_NAME=db
+DB_INIT_FILE=./data/postgresql/initdb.sql.gz
+POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8
+TIMESTAMP=$(shell date +"%Y-%m-%d_%H%M")
+
+conf: virtualenv
+	@echo "-> Install dependencies"
+	uv sync --frozen
+	@echo "-> Create the var/ directory"
+	@mkdir -p var
+
+dev: virtualenv
+	@echo "-> Configure and install development dependencies"
+	uv sync --frozen --extra dev
+
 envfile:
 	@echo "-> Create the .env file and generate a secret key"
 	@if test -f ${ENV_FILE}; then echo "${ENV_FILE} file exists already"; exit 1; fi
@@ -148,15 +160,6 @@ envfile_dev: envfile
 	@echo "-> Update the .env file for development"
 	@echo DATABASE_PASSWORD=\"dejacode\" >> ${ENV_FILE}
 
-check-deploy:
-	@echo "-> Check Django deployment settings"
-	${MANAGE} check --deploy
-
-clean:
-	@echo "-> Clean the Python env"
-	rm -rf .venv/ .*_cache/ *.egg-info/ build/ dist/
-	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
-
 initdb:
 	@echo "-> Stop Docker services that access the database"
 	${DOCKER_COMPOSE} stop web worker
@@ -181,4 +184,4 @@ psql:
 log:
 	${DOCKER_COMPOSE} logs --tail="100" ${SERVICE}
 
-.PHONY: virtualenv conf dev lock upgrade envfile envfile_dev check outdated doc8 valid check-deploy clean initdb postgresdb postgresdb_clean migrate run test docs build psql bash shell log superuser
+.PHONY: virtualenv conf dev lock upgrade envfile envfile_dev check outdated doc8 valid clean initdb postgresdb postgresdb_clean migrate run test docs build psql bash shell log superuser

component_catalog/tests/test_importers.py

@@ -601,10 +601,10 @@ def test_importers_view_num_queries_view(self):
         with self.assertMaxQueries(9):
             self.client.get(reverse("admin:component_catalog_package_import"))
 
-        with self.assertNumQueries(4):
+        with self.assertMaxQueries(5):
             self.client.get(reverse("admin:organization_owner_import"))
 
-        with self.assertMaxQueries(10):
+        with self.assertMaxQueries(11):
             self.client.get(reverse("admin:component_catalog_component_import"))
 
     def test_component_import_keywords(self):

component_catalog/tests/test_views.py

@@ -52,6 +52,7 @@
 from dje.models import ExternalReference
 from dje.models import ExternalSource
 from dje.models import History
+from dje.tests import MaxQueryMixin
 from dje.tests import add_perm
 from dje.tests import add_perms
 from dje.tests import create_superuser
@@ -77,7 +78,7 @@
 User = get_user_model()
 
 
-class ComponentUserViewsTestCase(TestCase):
+class ComponentUserViewsTestCase(MaxQueryMixin, TestCase):
     def setUp(self):
         self.nexb_dataspace = Dataspace.objects.create(name="nexB")
         self.nexb_user = User.objects.create_superuser(
@@ -980,7 +981,7 @@ def test_component_catalog_details_view_num_queries(self):
         History.log_change(self.basic_user, self.component1, "Changed version.")
         History.log_change(self.nexb_user, self.component1, "Changed notes.")
 
-        with self.assertNumQueries(32):
+        with self.assertMaxQueries(33):
             self.client.get(url)
 
     def test_component_catalog_details_view_package_tab_fields_visibility(self):
@@ -1095,7 +1096,7 @@ def test_component_catalog_component_create_ajax_view(self):
         self.assertContains(response, expected, html=True)
 
 
-class PackageUserViewsTestCase(TestCase):
+class PackageUserViewsTestCase(MaxQueryMixin, TestCase):
     testfiles_location = join(dirname(__file__), "testfiles")
 
     def setUp(self):
@@ -1133,7 +1134,7 @@ def setUp(self):
 
     def test_package_list_view_num_queries(self):
         self.client.login(username=self.super_user.username, password="secret")
-        with self.assertNumQueries(16):
+        with self.assertMaxQueries(17):
             self.client.get(reverse("component_catalog:package_list"))
 
     def test_package_list_view_pagination(self):
@@ -1271,7 +1272,7 @@ def test_package_details_view_num_queries(self):
         )
 
         self.client.login(username=self.super_user.username, password="secret")
-        with self.assertNumQueries(30):
+        with self.assertMaxQueries(31):
             self.client.get(self.package1.get_absolute_url())
 
     def test_package_details_view_content(self):
@@ -3797,7 +3798,7 @@ def test_component_catalog_package_update_view_save_as_with_collect_data(
         self.assertEqual(1, len(mock_collect_data.mock_calls))
 
 
-class ComponentListViewTestCase(TestCase):
+class ComponentListViewTestCase(MaxQueryMixin, TestCase):
     def setUp(self):
         self.dataspace = Dataspace.objects.create(
             name="nexB",
@@ -3887,7 +3888,7 @@ def setUp(self):
 
     def test_component_catalog_list_view_num_queries(self):
         self.client.login(username="nexb_user", password="t3st")
-        with self.assertNumQueries(17):
+        with self.assertMaxQueries(18):
             self.client.get(reverse("component_catalog:component_list"))
 
     def test_component_catalog_list_view_default(self):

license_library/tests/test_views.py

@@ -31,7 +31,7 @@
 from organization.models import Subowner
 
 
-class LicenseListViewTestCase(TestCase):
+class LicenseListViewTestCase(MaxQueryMixin, TestCase):
     def setUp(self):
         self.nexb_dataspace = Dataspace.objects.create(
             name="nexB",
@@ -286,7 +286,7 @@ def test_license_library_list_previous_next_license_link(self):
     def test_license_library_list_view_num_queries(self):
         self.client.login(username="nexb_user", password="t3st")
 
-        with self.assertNumQueries(16):
+        with self.assertMaxQueries(17):
             self.client.get(reverse("license_library:license_list"))
 
     def test_license_profile_column_availability_in_license_list_view(self):

organization/tests/test_views.py

@@ -13,6 +13,7 @@
 from django.urls import reverse
 
 from dje.models import Dataspace
+from dje.tests import MaxQueryMixin
 from dje.tests import add_perm
 from dje.tests import create_superuser
 from dje.tests import create_user
@@ -24,7 +25,7 @@
 Component = apps.get_model("component_catalog", "Component")
 
 
-class OwnerUserViewsTestCase(TestCase):
+class OwnerUserViewsTestCase(MaxQueryMixin, TestCase):
     def setUp(self):
         self.dataspace = Dataspace.objects.create(name="Dataspace")
         self.super_user = create_superuser("super_user", self.dataspace)
@@ -94,12 +95,12 @@ def test_object_details_view_tab_owner(self):
 
     def test_owner_list_view_num_queries(self):
         self.client.login(username=self.super_user.username, password="secret")
-        with self.assertNumQueries(13):
+        with self.assertMaxQueries(14):
             self.client.get(reverse("organization:owner_list"))
 
     def test_owner_details_view_num_queries(self):
         self.client.login(username=self.super_user.username, password="secret")
-        with self.assertNumQueries(18):
+        with self.assertMaxQueries(19):
             self.client.get(self.owner1.get_absolute_url())
 
     def test_owner_list_view_search_unicode_utf8_name_support(self):

product_portfolio/tests/test_views.py

@@ -136,7 +136,7 @@ def test_product_portfolio_detail_view_tab_inventory_and_hierarchy_availability(
         ProductComponent.objects.create(
             product=self.product1, component=self.component1, dataspace=self.dataspace
         )
-        with self.assertNumQueries(27):
+        with self.assertMaxQueries(28):
             response = self.client.get(url)
         self.assertContains(response, expected1)
         self.assertContains(response, expected2)
@@ -162,7 +162,7 @@ def test_product_portfolio_detail_view_tab_inventory_availability(self):
         ProductPackage.objects.create(
             product=self.product1, package=self.package1, dataspace=self.dataspace
         )
-        with self.assertNumQueries(25):
+        with self.assertMaxQueries(26):
             response = self.client.get(url)
         self.assertContains(response, expected)
 
@@ -267,7 +267,7 @@ def test_product_portfolio_detail_view_tab_dependency_view(self):
             resolved_to_package=package2,
         )
 
-        with self.assertMaxQueries(9):
+        with self.assertMaxQueries(10):
             response = self.client.get(url)
         self.assertContains(response, "4 results")
 
@@ -289,7 +289,7 @@ def test_product_portfolio_detail_view_tab_vulnerability_queryset(self):
         self.assertEqual(4, product1.packages.vulnerable().count())
 
         url = product1.get_url("tab_vulnerabilities")
-        with self.assertMaxQueries(11):
+        with self.assertMaxQueries(12):
             response = self.client.get(url)
         self.assertContains(response, "4 results")
 
@@ -357,7 +357,7 @@ def test_product_portfolio_tab_vulnerability_view_queries(self):
         make_vulnerability_analysis(product_package2, vulnerability2)
 
         url = product1.get_url("tab_vulnerabilities")
-        with self.assertNumQueries(11):
+        with self.assertMaxQueries(12):
             self.client.get(url)
 
     def test_product_portfolio_tab_vulnerability_risk_threshold(self):

pyproject.toml

@@ -50,6 +50,8 @@ dependencies = [
     "django-guardian==3.3.1",
     "django-environ==0.13.0",
     "django-debug-toolbar==6.3.0",
+    # Parallel testing
+    "tblib==3.2.2",
     # CAPTCHA
     "altcha==1.0.0",
     "django_altcha==0.10.0",
@@ -96,7 +98,7 @@ dependencies = [
     "XlsxWriter==3.2.9",
     # Markdown
     "markdown==3.10.2",
-    "bleach==6.3.0",
+    "bleach==6.4.0",
     "bleach_allowlist==1.0.3",
     "webencodings==0.5.1",
     # Authentication
@@ -160,8 +162,6 @@ dependencies = [
 dev = [
     # Linter and Validation
     "ruff==0.15.14",
-    # Parallel testing
-    "tblib==3.2.2"
 ]
 
 [project.urls]

reporting/tests/test_views.py

@@ -19,6 +19,7 @@
 from component_catalog.models import Component
 from dje.copier import copy_object
 from dje.models import Dataspace
+from dje.tests import MaxQueryMixin
 from license_library.models import License
 from license_library.models import LicenseCategory
 from organization.models import Owner
@@ -32,7 +33,7 @@
 from reporting.models import Report
 
 
-class ReportDetailsViewTestCase(TestCase):
+class ReportDetailsViewTestCase(MaxQueryMixin, TestCase):
     def setUp(self):
         self.dataspace = Dataspace.objects.create(name="nexB")
         self.owner = Owner.objects.create(dataspace=self.dataspace, name="My Fancy Owner Name")
@@ -1166,7 +1167,7 @@ def test_report_list_view_num_queries(self):
         # Needed to clear the queries from the License batch creation in setUp
         self.client.get(url)
 
-        with self.assertNumQueries(9):
+        with self.assertMaxQueries(10):
             self.client.get(url)
 
     def test_run_report_view_query_using_related_fields(self):