Check if a Scan is already available before submitting #387

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

component_catalog/tests/test_scancodeio.py

@@ -17,9 +17,9 @@
 from component_catalog.models import Package
 from component_catalog.tests import make_package
 from dejacode_toolkit.scancodeio import ScanCodeIO
-from dejacode_toolkit.scancodeio import check_for_existing_scan_workaround
 from dejacode_toolkit.scancodeio import get_hash_uid
 from dejacode_toolkit.scancodeio import get_notice_text_from_key_files
+from dejacode_toolkit.scancodeio import update_package_from_existing_scan_data
 from dje.models import Dataspace
 from dje.models import History
 from dje.tasks import scancodeio_submit_scan
@@ -67,9 +67,7 @@ def test_scancodeio_submit_scan_task(self, mock_submit_scan, mock_request_head):
 
         expected = [
             mock.call("http://okurl.com", user_uuid, dataspace_uuid),
-            mock.call().__bool__(),
             mock.call("https://okurl2.com", user_uuid, dataspace_uuid),
-            mock.call().__bool__(),
         ]
         self.assertEqual(expected, mock_submit_scan.mock_calls)
 
@@ -334,19 +332,13 @@ def test_scancodeio_get_notice_text_from_key_files(self):
         self.assertEqual("", notice_text)
 
     @mock.patch("component_catalog.models.Package.update_from_scan")
-    def test_scancodeio_check_for_existing_scan_workaround(self, mock_update_from_scan):
+    def test_scancodeio_update_package_from_existing_scan_data(self, mock_update_from_scan):
         mock_update_from_scan.return_value = ["updated_field"]
         download_url = self.package1.download_url
         user = self.basic_user
 
-        response_json = None
-        results = check_for_existing_scan_workaround(response_json, download_url, user)
+        results = update_package_from_existing_scan_data("unknown_url", user)
         self.assertIsNone(results)
 
-        response_json = {"success": True}
-        results = check_for_existing_scan_workaround(response_json, download_url, user)
-        self.assertIsNone(results)
-
-        response_json = {"name": "project with this name already exists."}
-        results = check_for_existing_scan_workaround(response_json, download_url, user)
+        results = update_package_from_existing_scan_data(download_url, user)
         self.assertEqual(["updated_field"], results)

dejacode_toolkit/scancodeio.py

@@ -556,22 +556,15 @@ def get_notice_text_from_key_files(scan_summary, separator="\n\n---\n\n"):
     return notice_text
 
 
-def check_for_existing_scan_workaround(response_json, uri, user):
+def update_package_from_existing_scan_data(uri, user):
     """
-    Workaroud the case where the Scan already exisit on the ScanCode.io side before
+    Workaroud the case where the Scan already exisits on the ScanCode.io side before
     the package is created on the DejaCode side.
     This can happen if the package is deleted then re-created from the same user
     providing the same download URL.
     """
-    if not response_json or not isinstance(response_json, dict):
-        return
-
-    already_exists_message = "project with this name already exists."
-    already_exists = already_exists_message in response_json.get("name", [])
-
-    if already_exists:
-        Package = apps.get_model("component_catalog", "package")
-        package = Package.objects.get_or_none(download_url=uri, dataspace=user.dataspace)
-        if package:
-            updated_fields = package.update_from_scan(user)
-            return updated_fields
+    Package = apps.get_model("component_catalog", "package")
+    package = Package.objects.get_or_none(download_url=uri, dataspace=user.dataspace)
+    if package:
+        updated_fields = package.update_from_scan(user)
+        return updated_fields

dje/tasks.py

@@ -20,7 +20,7 @@
 from django_rq import job
 
 from dejacode_toolkit.scancodeio import ScanCodeIO
-from dejacode_toolkit.scancodeio import check_for_existing_scan_workaround
+from dejacode_toolkit.scancodeio import update_package_from_existing_scan_data
 from dje.utils import is_available
 
 logger = logging.getLogger(__name__)
@@ -109,11 +109,17 @@ def scancodeio_submit_scan(uris, user_uuid, dataspace_uuid):
 
     scancodeio = ScanCodeIO(user.dataspace)
     for uri in uris:
-        if is_available(uri):
-            response_json = scancodeio.submit_scan(uri, user_uuid, dataspace_uuid)
-            check_for_existing_scan_workaround(response_json, uri, user)
-        else:
+        if not is_available(uri):
             logger.info(f'uri="{uri}" is not reachable.')
+            continue
+
+        # Check if a Scan is already available in ScanCode.io for this URI.
+        existing_project = scancodeio.get_project_info(download_url=uri)
+        if existing_project:
+            logger.info(f'Update the local uri="{uri}" package from available Scan data.')
+            update_package_from_existing_scan_data(uri, user)
+        else:
+            scancodeio.submit_scan(uri, user_uuid, dataspace_uuid)
 
 
 @job("default", timeout="3h")