feat: move scanpipe settings to local settings.py with lazy resolution (#2187)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

scancodeio/settings.py

@@ -63,8 +63,6 @@
     "SCANCODEIO_REQUIRE_AUTHENTICATION", default=False
 )
 
-SCANCODEIO_ENABLE_ADMIN_SITE = env.bool("SCANCODEIO_ENABLE_ADMIN_SITE", default=False)
-
 SECURE_CONTENT_TYPE_NOSNIFF = env.bool("SECURE_CONTENT_TYPE_NOSNIFF", default=True)
 
 X_FRAME_OPTIONS = env.str("X_FRAME_OPTIONS", default="DENY")
@@ -79,102 +77,8 @@
 
 # ScanCode.io
 
-SCANCODEIO_WORKSPACE_LOCATION = env.str("SCANCODEIO_WORKSPACE_LOCATION", default="var")
-
-SCANCODEIO_CONFIG_DIR = env.str("SCANCODEIO_CONFIG_DIR", default=".scancode")
-
-SCANCODEIO_CONFIG_FILE = env.str(
-    "SCANCODEIO_CONFIG_FILE", default="scancode-config.yml"
-)
-
 SCANCODEIO_LOG_LEVEL = env.str("SCANCODEIO_LOG_LEVEL", "INFO")
 
-# Set the number of parallel processes to use for ScanCode related scan execution.
-# If the SCANCODEIO_PROCESSES argument is not set, defaults to an optimal number of CPUs
-# available on the machine.
-SCANCODEIO_PROCESSES = env.int("SCANCODEIO_PROCESSES", default=None)
-
-SCANCODEIO_POLICIES_FILE = env.str("SCANCODEIO_POLICIES_FILE", default="policies.yml")
-
-# This setting defines the additional locations ScanCode.io will search for pipelines.
-# This should be set to a list of strings that contain full paths to your additional
-# pipelines directories.
-SCANCODEIO_PIPELINES_DIRS = env.list("SCANCODEIO_PIPELINES_DIRS", default=[])
-
-# Maximum time allowed for a pipeline to complete.
-SCANCODEIO_TASK_TIMEOUT = env.str("SCANCODEIO_TASK_TIMEOUT", default="24h")
-
-# Default to 2 minutes.
-SCANCODEIO_SCAN_FILE_TIMEOUT = env.int("SCANCODEIO_SCAN_FILE_TIMEOUT", default=120)
-
-# Default to None which scans all files
-SCANCODEIO_SCAN_MAX_FILE_SIZE = env.int("SCANCODEIO_SCAN_MAX_FILE_SIZE", default=None)
-
-# List views pagination, controls the number of items displayed per page.
-# Syntax in .env: SCANCODEIO_PAGINATE_BY=project=10,project_error=10
-SCANCODEIO_PAGINATE_BY = env.dict(
-    "SCANCODEIO_PAGINATE_BY",
-    default={
-        "project": 20,
-        "error": 50,
-        "resource": 100,
-        "package": 100,
-        "dependency": 100,
-        "license": 100,
-        "relation": 100,
-    },
-)
-
-# Default limit for "most common" entries in QuerySets.
-SCANCODEIO_MOST_COMMON_LIMIT = env.int("SCANCODEIO_MOST_COMMON_LIMIT", default=7)
-
-# The base URL (e.g., https://hostname/) of this application instance.
-# Required for generating URLs to reference objects within the app,
-# such as in webhook notifications.
-SCANCODEIO_SITE_URL = env.str("SCANCODEIO_SITE_URL", default="")
-
-# Fetch authentication credentials
-
-# SCANCODEIO_FETCH_BASIC_AUTH="host=user,password;"
-SCANCODEIO_FETCH_BASIC_AUTH = env.dict(
-    "SCANCODEIO_FETCH_BASIC_AUTH",
-    cast={"value": tuple},
-    default={},
-)
-
-# SCANCODEIO_FETCH_DIGEST_AUTH="host=user,password;"
-SCANCODEIO_FETCH_DIGEST_AUTH = env.dict(
-    "SCANCODEIO_FETCH_DIGEST_AUTH",
-    cast={"value": tuple},
-    default={},
-)
-
-# SCANCODEIO_FETCH_HEADERS="host=Header1=value,Header2=value;"
-SCANCODEIO_FETCH_HEADERS = {}
-FETCH_HEADERS_STR = env.str("SCANCODEIO_FETCH_HEADERS", default="")
-for entry in FETCH_HEADERS_STR.split(";"):
-    if entry.strip():
-        host, headers = entry.split("=", 1)
-        SCANCODEIO_FETCH_HEADERS[host] = env.parse_value(headers, cast=dict)
-
-# SCANCODEIO_NETRC_LOCATION="~/.netrc"
-SCANCODEIO_NETRC_LOCATION = env.str("SCANCODEIO_NETRC_LOCATION", default="")
-if SCANCODEIO_NETRC_LOCATION:
-    # Propagate the location to the environ for `requests.utils.get_netrc_auth`
-    env.ENVIRON["NETRC"] = SCANCODEIO_NETRC_LOCATION
-
-# SCANCODEIO_SKOPEO_CREDENTIALS="host1=user:password,host2=user:password"
-SCANCODEIO_SKOPEO_CREDENTIALS = env.dict("SCANCODEIO_SKOPEO_CREDENTIALS", default={})
-
-# SCANCODEIO_SKOPEO_AUTHFILE_LOCATION="/path/to/auth.json"
-SCANCODEIO_SKOPEO_AUTHFILE_LOCATION = env.str(
-    "SCANCODEIO_SKOPEO_AUTHFILE_LOCATION", default=""
-)
-
-# This webhook will be added as WebhookSubscription for each new project.
-# SCANCODEIO_GLOBAL_WEBHOOK=target_url=https://webhook.url,trigger_on_each_run=False,include_summary=True,include_results=False
-SCANCODEIO_GLOBAL_WEBHOOK = env.dict("SCANCODEIO_GLOBAL_WEBHOOK", default={})
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -288,11 +192,13 @@
     from django.core.management.utils import get_random_secret_key
 
     SECRET_KEY = get_random_secret_key()
-    # Do not pollute the workspace while running the tests.
-    SCANCODEIO_WORKSPACE_LOCATION = tempfile.mkdtemp()
+    SCANPIPE = {
+        # Do not pollute the workspace while running the tests.
+        "WORKSPACE_LOCATION": tempfile.mkdtemp(),
+        "SCAN_FILE_TIMEOUT": 120,
+        "POLICIES_FILE": None,
+    }
     SCANCODEIO_REQUIRE_AUTHENTICATION = True
-    SCANCODEIO_SCAN_FILE_TIMEOUT = 120
-    SCANCODEIO_POLICIES_FILE = None
     # The default password hasher is rather slow by design.
     # Using a faster hashing algorithm in the testing context to speed up the run.
     PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
@@ -389,8 +295,8 @@
     },
 }
 
-SCANCODEIO_ASYNC = env.bool("SCANCODEIO_ASYNC", default=False)
-if not SCANCODEIO_ASYNC:
+# Runtime async mode is accessed via scanpipe_settings.ASYNC
+if not env.bool("SCANCODEIO_ASYNC", default=False):
     for queue_config in RQ_QUEUES.values():
         queue_config["ASYNC"] = False
 

scancodeio/urls.py

@@ -31,6 +31,7 @@
 from scanpipe.admin import admin_site
 from scanpipe.api.views import ProjectViewSet
 from scanpipe.api.views import RunViewSet
+from scanpipe.settings import scanpipe_settings
 from scanpipe.views import AccountProfileView
 from scanpipe.views import GenerateAPIKeyView
 from scanpipe.views import RevokeAPIKeyView
@@ -67,7 +68,7 @@
 ]
 
 
-if settings.SCANCODEIO_ENABLE_ADMIN_SITE:
+if scanpipe_settings.ENABLE_ADMIN_SITE:
     urlpatterns.append(path("admin/", admin_site.urls))
 
 

scanpipe/apps.py

@@ -23,13 +23,13 @@
 import importlib.util
 import inspect
 import logging
+import os
 import sys
 import warnings
 from importlib.machinery import SourceFileLoader
 from pathlib import Path
 
 from django.apps import AppConfig
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management.color import color_style
 from django.db.models import BLANK_CHOICE_DASH
@@ -39,6 +39,7 @@
 from licensedcode.models import load_licenses
 
 from scanpipe.policies import load_policies_file
+from scanpipe.settings import scanpipe_settings
 
 try:
     from importlib import metadata as importlib_metadata
@@ -62,21 +63,24 @@ def __init__(self, app_name, app_module):
         self._pipelines = {}
         self.policies = {}
 
-        workspace_location = settings.SCANCODEIO_WORKSPACE_LOCATION
+        workspace_location = scanpipe_settings.WORKSPACE_LOCATION
         self.workspace_path = Path(workspace_location).expanduser().resolve()
 
     def ready(self):
-        logger.debug(f"Read environment variables from {settings.ENV_FILE}")
         self.load_pipelines()
         self.set_policies()
 
+        if netrc_location := scanpipe_settings.NETRC_LOCATION:
+            # Propagate the location to the environ for `requests.utils.get_netrc_auth`
+            os.environ["NETRC"] = netrc_location
+
         # In SYNC mode, the Run instances cleanup is triggered on app.ready()
         # only when the app is started through "runserver".
         # This cleanup is required if a running pipeline process gets killed and
         # since KeyboardInterrupt cannot be captured to properly update the Run instance
         # before its running process death.
         # In ASYNC mode, the cleanup is handled by the "ScanCodeIOWorker" worker.
-        if not settings.SCANCODEIO_ASYNC and "runserver" in sys.argv:
+        if not scanpipe_settings.ASYNC and "runserver" in sys.argv:
             warnings.filterwarnings(
                 "ignore",
                 message="Accessing the database during app initialization",
@@ -96,7 +100,7 @@ def load_pipelines(self):
         for entry_point in sorted(pipeline_entry_points):
             self.register_pipeline(name=entry_point.name, cls=entry_point.load())
 
-        pipelines_dirs = getattr(settings, "SCANCODEIO_PIPELINES_DIRS", [])
+        pipelines_dirs = scanpipe_settings.PIPELINES_DIRS
         logger.debug(f"Load user provided pipelines from {pipelines_dirs}")
 
         for pipelines_dir in pipelines_dirs:
@@ -231,7 +235,7 @@ def set_policies(self):
         include the proper content, we want to raise an exception while the app
         is loading to warn system admins about the issue.
         """
-        policies_file_setting = getattr(settings, "SCANCODEIO_POLICIES_FILE", None)
+        policies_file_setting = scanpipe_settings.POLICIES_FILE
         if not policies_file_setting:
             return
 
@@ -259,5 +263,5 @@ def sync_runs_and_jobs(self):
 
     @property
     def site_url(self):
-        if site_url := settings.SCANCODEIO_SITE_URL:
+        if site_url := scanpipe_settings.SITE_URL:
             return site_url.rstrip("/")

scanpipe/management/commands/__init__.py

@@ -25,7 +25,6 @@
 from pathlib import Path
 
 from django.apps import apps
-from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ValidationError
 from django.core.management.base import BaseCommand
@@ -39,6 +38,7 @@
 from scanpipe.models import ProjectMessage
 from scanpipe.pipes import count_group_by
 from scanpipe.pipes.fetch import set_project_purl_from_input_url
+from scanpipe.settings import scanpipe_settings
 
 scanpipe_app = apps.get_app_config("scanpipe")
 
@@ -405,7 +405,7 @@ def execute_project(project, run_async=False, command=None):  # noqa: C901
         raise CommandError(f"No pipelines to run on project {project}")
 
     if run_async:
-        if not settings.SCANCODEIO_ASYNC:
+        if not scanpipe_settings.ASYNC:
             msg = "SCANCODEIO_ASYNC=False is not compatible with --async option."
             raise CommandError(msg)
 

scanpipe/migrations/0031_scancode_toolkit_v32_data_updates.py

@@ -16,7 +16,7 @@ def compute_package_declared_license_expression_spdx(apps, schema_editor):
     """
     from licensedcode.cache import build_spdx_license_expression
 
-    if settings.IS_TESTS:
+    if getattr(settings, "IS_TESTS", False):
         return
 
     DiscoveredPackage = apps.get_model("scanpipe", "DiscoveredPackage")
@@ -55,7 +55,7 @@ def compute_resource_detected_license_expression(apps, schema_editor):
     from license_expression import combine_expressions
     from licensedcode.cache import build_spdx_license_expression
 
-    if settings.IS_TESTS:
+    if getattr(settings, "IS_TESTS", False):
         return
 
     CodebaseResource = apps.get_model("scanpipe", "CodebaseResource")
@@ -161,7 +161,7 @@ def _convert_matches_to_detections(license_matches):
 
 def compute_resource_license_detections(apps, schema_editor):
     """Compute CodebaseResource `license_detections` from old `licenses` field."""
-    if settings.IS_TESTS:
+    if getattr(settings, "IS_TESTS", False):
         return
 
     CodebaseResource = apps.get_model("scanpipe", "CodebaseResource")