fix(checker): Add submission terms if lots==master is used

The logging in ca1b09d is unnecessary. If a user wants to know about malformed extensions metadata, they run the checker.

Closes #436 (we can re-add the logic if the new CCE API has the same issue)

process/management/commands/checker.py

@@ -19,7 +19,7 @@
     using_libcoveocds = False
 
 from process.models import CollectionFile, ProcessingStep, Record, RecordCheck, Release, ReleaseCheck
-from process.util import consume, decorator, delete_step
+from process.util import consume, decorator, delete_step, get_extensions
 from process.util import wrap as w
 
 consume_routing_keys = ["file_worker", "addchecks"]
@@ -114,13 +114,8 @@ def _check_collection_file(collection_file):
         package = item.package_data.data
         package[items_key] = [item.data.data]
 
-        extensions = package.get("extensions")
-        if isinstance(extensions, list):
-            extensions = frozenset(extension for extension in extensions if isinstance(extension, str))
-        else:
-            extensions = frozenset()
         # Security: Potential SSRF via extension URLs (within OCDS publication).
-        schema = _get_schema(items_key, extensions)
+        schema = _get_schema(items_key, get_extensions(package))
 
         logger.debug("Checking %s of %s", items_key, item)
         cove_output = context_api_transform(

process/management/commands/release_compiler.py

@@ -8,7 +8,7 @@
 from process.exceptions import AlreadyExists
 from process.models import Collection, CollectionNote, CompiledRelease, ProcessingStep, Release
 from process.processors.compiler import compile_releases_by_ocdskit, save_compiled_release
-from process.util import consume, create_note, decorator, delete_step
+from process.util import consume, create_note, decorator, delete_step, get_extensions
 from process.util import wrap as w
 
 consume_routing_keys = ["compiler_release"]
@@ -69,13 +69,8 @@ def compile_release(collection, ocid):
     extensions = set()
     for release in releases.iterator():
         data.append(release.data.data)
-
         if release.package_data:
-            package_extensions = release.package_data.data.get("extensions", [])
-            if isinstance(package_extensions, list):
-                extensions.update(package_extensions)
-            elif package_extensions is not None:
-                logger.error("Ignored malformed extensions for release %s: %s", release, package_extensions)
+            extensions.update(get_extensions(release.package_data.data))
 
     # estonia_digiwhist publishes release packages containing a single release with a "compiled" tag, and it sometimes
     # publishes the same OCID with identical data in different packages with a different `publishedDate`. The releases

process/processors/compiler.py

@@ -11,7 +11,6 @@
 from process.util import create_note, create_warnings_note, get_or_create
 
 logger = logging.getLogger(__name__)
-format_string = "https://raw.githubusercontent.com/open-contracting-extensions/ocds_{}_extension/master/extension.json"
 
 
 def save_compiled_release(merged, collection, ocid):
@@ -33,15 +32,6 @@ def save_compiled_release(merged, collection, ocid):
 
 
 def compile_releases_by_ocdskit(collection, ocid, releases, extensions):
-    # It otherwise takes a very long time for requests and retries to time out.
-    # https://github.com/open-contracting/kingfisher-process/issues/436
-    if collection.source_id == "colombia_api":
-        extensions = {extension.replace(":8443", "") for extension in extensions}
-
-    # The master version of the lots extension depends on OCDS 1.2 or the submission terms extension.
-    if format_string.format("lots") in extensions:
-        extensions.add(format_string.format("submissionTerms"))
-
     with create_warnings_note(collection, ExtensionWarning):
         merger = _get_merger(frozenset(extensions))
 

process/util.py

@@ -19,6 +19,7 @@
 logger = logging.getLogger(__name__)
 
 YAPW_KWARGS = {"url": settings.RABBIT_URL, "exchange": settings.RABBIT_EXCHANGE_NAME, "prefetch_count": 20}
+EXTENSION_URL = "https://raw.githubusercontent.com/open-contracting-extensions/ocds_{}_extension/master/extension.json"
 
 
 def wrap(string):
@@ -178,3 +179,17 @@ def create_logger_note(collection, name):
 
     if note := stream.getvalue():
         create_note(collection, CollectionNote.Level.WARNING, note)
+
+
+def get_extensions(package):
+    extensions = set()
+
+    package_extensions = package.get("extensions")
+    if isinstance(package_extensions, list):
+        extensions = {extension for extension in package_extensions if isinstance(extension, str)}
+
+    # The master version of the lots extension depends on OCDS 1.2 or the submission terms extension.
+    if EXTENSION_URL.format("lots") in extensions:
+        extensions.add(EXTENSION_URL.format("submissionTerms"))
+
+    return frozenset(extensions)