Clear to_codebase status before D2D #1938

* Many D2D steps look for resources without a status. However, the earlier
  scanning step flags all resources as "scanned". To prevent D2D steps from
  skipping these resources, we clear the status of all `to` resources

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

scanpipe/pipelines/scan_nix_package.py

@@ -57,11 +57,11 @@ def steps(cls):
             cls.collect_and_create_codebase_resources,
             cls.scan_for_application_packages,
             cls.scan_for_files,
+            cls.clear_to_codebase_status,
             cls.collect_and_create_license_detections,
             cls.add_from_to_tag,
             cls.d2d_steps,
             cls.validate_package_license_integrity,
-            cls.flag_mapped_status,
             cls.cleanup_docker_volumes,
         )
 
@@ -120,6 +120,13 @@ def extract_input_to_codebase_directory(self):
 
             self.env = self.project.get_env()
 
+    def clear_to_codebase_status(self):
+        """
+        Clear the status of the to codebase resources in the project as
+        having status will prevent D2D from running.
+        """
+        flag.clear_status(self.project.codebaseresources.to_codebase())
+
     def add_from_to_tag(self):
         """Update 'from' and 'to' tag to resources based on their path."""
         if self.d2d_enable:
@@ -212,9 +219,6 @@ def d2d_process(self):
         self.perform_house_keeping_tasks()
         self.match_purldb_resources_post_process()
         self.remove_packages_without_resources()
-        self.scan_ignored_to_files()
-        self.scan_unmapped_to_files()
-        self.scan_mapped_from_for_files()
         self.flag_deployed_from_resources_with_missing_license()
         self.create_local_files_packages()
 

scanpipe/pipes/d2d.py

@@ -141,11 +141,8 @@ def _map_checksum_resource(to_resource, from_resources, checksum_field):
 
 def map_checksum(project, checksum_field, logger=None):
     """Map using checksum."""
-    project_files = project.codebaseresources.files().no_status()
-    from_resources = project_files.from_codebase().has_value(checksum_field)
-    to_resources = (
-        project_files.to_codebase().has_value(checksum_field).has_no_relation()
-    )
+    from_resources = project.codebaseresources.files().from_codebase().has_value(checksum_field)
+    to_resources = project.codebaseresources.files().to_codebase().has_value(checksum_field).has_no_relation()
     resource_count = to_resources.count()
 
     if logger:

scanpipe/pipes/flag.py

@@ -139,3 +139,8 @@ def flag_mapped_resources(project):
     """Flag all codebase resources that were mapped during the d2d pipeline."""
     resources = project.codebaseresources.has_relation().no_status()
     return resources.update(status=MAPPED)
+
+
+def clear_status(resource_qs):
+    """Clear the status of given codebase resources."""
+    return resource_qs.update(status="")