Enhance D2D pipeline to identify generated code #1900

Makefile

@@ -36,6 +36,10 @@ start:
 	@echo "-> Start the Docker compose services in background"
 	${COMPOSE} up -d
 
+stop:
+	@echo "-> Stop the Docker compose services"
+	${COMPOSE} stop
+
 bash:
 	# Open a bash session in the running web container
 	${COMPOSE} exec web bash
@@ -89,11 +93,13 @@ fix:
 outdated:
 	@echo "-> Check for outdated packages (with 7 days cooldown)"
 	uv tree --outdated --exclude-newer "7 days"
+	# Run `make start` first following any pyproject.toml changes
+	${COMPOSE} exec web uv pip list --outdated --exclude-newer "7 days"
 	@echo "-> Audit the project's dependencies for known vulnerabilities"
 	uv audit
 
 lock:
-	@echo "-> Regenerate uv.lock from local wheels"
+	@echo "-> Regenerate uv.lock from pyproject.toml"
 	uv lock
 
 ########################################################################################

docker-compose.yml

@@ -4,7 +4,7 @@ x-description: "Full ScanCode.io stack: all services, local image build"
 
 services:
   db:
-    image: docker.io/library/postgres:17.10
+    image: docker.io/library/postgres:17.11
     env_file:
       - docker.env
     volumes:
@@ -18,7 +18,7 @@ services:
       retries: 5
 
   redis:
-    image: docker.io/library/redis:8.8.0-alpine
+    image: docker.io/library/redis:8.10.1-alpine
     # Enable redis data persistence using the "Append Only File" with the
     # default policy of fsync every second. See https://redis.io/topics/persistence
     command: redis-server --appendonly yes --loglevel warning
@@ -77,7 +77,7 @@ services:
         condition: service_started
 
   nginx:
-    image: docker.io/library/nginx:1.31.2-alpine
+    image: docker.io/library/nginx:1.31.4-alpine
     ports:
       - "${NGINX_PUBLISHED_HTTP_PORT:-80}:80"
       - "${NGINX_PUBLISHED_HTTPS_PORT:-443}:443"
@@ -91,7 +91,7 @@ services:
     restart: always
 
   clamav:
-    image: docker.io/clamav/clamav:1.5.2_base
+    image: docker.io/clamav/clamav:1.5.4_base
     volumes:
       - clamav_data:/var/lib/clamav
       - workspace:/var/scancodeio/workspace/

pyproject.toml

@@ -33,28 +33,28 @@ classifiers = [
 ]
 
 dependencies = [
-  "uv==0.11.29",
+  "uv==0.12.5",
   "importlib-metadata==9.0.0",
-  "setuptools==83.0.0",
+  "setuptools==84.0.0",
   # Django related
-  "django==6.0.7",
+  "django==6.0.8",
   "django-environ==0.14.0",
-  "django-crispy-forms==2.6",
+  "django-crispy-forms==2.7",
   "crispy-bootstrap3==2024.1",
-  "django-filter==25.2",
-  "djangorestframework==3.17.1",
-  "django-htmx==1.27.0",
-  "django-debug-toolbar==7.0.0",
+  "django-filter==26.1",
+  "djangorestframework==3.18.0",
+  "django-htmx==1.29.0",
+  "django-debug-toolbar==7.1.1",
   # Database
   "psycopg[binary]==3.3.4",
   # wait_for_database Django management command
   "django-probes==1.8.0",
   # Task queue
   "rq==2.10.0",
   "django-rq==4.1.1",
-  "redis==8.0.1",
+  "redis==8.1.0",
   # WSGI server
-  "gunicorn==26.0.0",
+  "gunicorn==26.1.0",
   "packaging==26.2",
   # Docker
   "container-inspector==33.1.0",
@@ -63,6 +63,7 @@ dependencies = [
   "extractcode[full]==31.1.0",
   "commoncode==32.5.2",
   "beautifulsoup4[chardet]==4.15.0",
+  "click==8.4.2",
   "packageurl-python==0.17.6",
   # FetchCode
   "fetchcode==0.8.2",
@@ -79,17 +80,17 @@ dependencies = [
   "XlsxWriter==3.2.9",
   "openpyxl==3.1.5",
   "requests==2.34.2",
-  "charset-normalizer==3.4.9",
-  "chardet==7.4.3",
+  "charset-normalizer==3.5.1",
+  "chardet==7.6.0",
   "urllib3==2.7.0",
-  "idna==3.18",
-  "GitPython==3.1.50",
-  "lxml==6.1.1",
-  "certifi==2026.6.17",
+  "idna==3.19",
+  "GitPython==3.1.59",
+  "lxml==6.1.2",
+  "certifi==2026.7.22",
   # Profiling
-  "pyinstrument==5.1.2",
+  "pyinstrument==5.1.3",
   # CycloneDX
-  "cyclonedx-python-lib==11.11.0",
+  "cyclonedx-python-lib==11.12.0",
   "jsonschema==4.26.0",
   # MatchCode-toolkit
   "matchcode-toolkit==7.2.2",
@@ -108,14 +109,16 @@ dependencies = [
   # ScoreCode
   "scorecode==0.0.4",
   # Transitive dependencies
-  "cryptography==49.0.0",
-  "aiohttp==3.14.1",
+  "cryptography==50.0.0",
+  "aiohttp==3.14.3",
   "pydantic-settings==2.14.2",
-  "croniter==6.2.3",
+  "croniter==6.2.4",
   "lief==0.17.6",
   "symbolic==13.8.0",
-  "cffi==2.1.0",
+  "Pygments==2.20.0",
+  "cffi==2.1.1",
   "stevedore==5.9.0",
+  "sqlparse==0.6.0",
 ]
 
 [project.optional-dependencies]
@@ -151,6 +154,7 @@ run = "scancodeio:combined_run"
 analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker"
 analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS"
 analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows"
+analyze_symbols_reachability = "scanpipe.pipelines.analyze_symbols_reachability:SymbolReachability"
 benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls"
 collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext"
 collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags"

scanpipe/pipelines/analyze_symbols_reachability.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import reachability
+from scanpipe.pipes.symbols import TS_QUERIES
+
+
+class SymbolReachability(Pipeline):
+    """
+    Determine the reachability of vulnerabilities identified in the project.
+
+    Note: You must run `find_vulnerabilities` pipeline before running this pipeline.
+
+    For every patch the git repository is cloned and extract the vulnerable and fixed
+    symbols from the patch commit. These symbols are then matched against
+    the project's codebase resources to determine if the vulnerable code
+    is actually present and reachable.
+
+    The analysis checks if vulnerable symbols are defined, imported, called,
+    or exactly match a code within the project files. The results, including
+    tool_details and a reachability status (yes, unknown, or no), are stored
+    in the `extra_data` of the matching resources under the `symbols_reachability` key.
+
+    Finally, a summary report is generated for each vulnerability
+    advisory and saved as a JSON output file.
+    """
+
+    download_inputs = False
+    is_addon = True
+    results_url = "/project/{slug}/resources/?extra_data=symbol_reachability"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.get_vulnerabilities_patches,
+            cls.collect_resource_index,
+            cls.collect_patch_symbols,
+            cls.collect_and_match_resources,
+            cls.generate_advisory_reachability_report,
+        )
+
+    def get_vulnerabilities_patches(self):
+        """Get unique patch for all vulnerabilities."""
+        self.patches = reachability.get_vulnerabilities_patches(
+            package_vulnerabilities=self.project.package_vulnerabilities,
+            dependency_vulnerabilities=self.project.dependency_vulnerabilities,
+        )
+
+    def collect_resource_index(self):
+        """Collect resources symbols for each resource"""
+        self.candidate_resources = self.project.codebaseresources.files().filter(
+            is_binary=False,
+            is_archive=False,
+            is_media=False,
+            programming_language__in=TS_QUERIES.keys(),
+        )
+        self.resource_indexes = reachability.collect_resource_index(
+            candidate_resources=self.candidate_resources, logger=self.log
+        )
+
+    def collect_patch_symbols(self):
+        """Collect patch symbols for all related commits."""
+        self.patch_symbols = reachability.collect_patch_symbols(
+            patches=self.patches, logger=self.log
+        )
+
+    def collect_and_match_resources(self):
+        """Match resource symbols against patch symbols."""
+        reachability.match_patches_to_resources(
+            patches=self.patches,
+            patch_symbols=self.patch_symbols,
+            resource_indexes=self.resource_indexes,
+            candidate_resources=self.candidate_resources,
+            logger=self.log,
+        )
+
+    def generate_advisory_reachability_report(self):
+        """Generate a reachability report summarizing status by advisory."""
+        reachability.generate_advisory_reachability_report(
+            project=self.project,
+            patches=self.patches,
+            candidate_resources=self.candidate_resources,
+        )

scanpipe/pipelines/deploy_to_develop.py

@@ -111,6 +111,7 @@ def steps(cls):
             cls.map_javascript_colocation,
             cls.map_thirdparty_npm_packages,
             cls.map_path,
+            cls.flag_generated_code,
             cls.flag_mapped_resources_archives_and_ignored_directories,
             cls.perform_house_keeping_tasks,
             cls.match_purldb_resources_post_process,
@@ -458,6 +459,13 @@ def map_path(self):
         """Map using path similarities."""
         d2d.map_path(project=self.project, logger=self.log)
 
+    def flag_generated_code(self):
+        """
+        Flag deployed files that have corresponding sources that are
+        potential generated.
+        """
+        d2d.flag_generated_file(self.project)
+
     def flag_mapped_resources_archives_and_ignored_directories(self):
         """Flag all codebase resources that were mapped during the pipeline."""
         flag.flag_mapped_resources(self.project)

scanpipe/pipes/d2d.py

@@ -1297,6 +1297,36 @@ def _map_javascript_colocation_resource(
     )
 
 
+def flag_generated_file(project):
+    """Flag generated files based on path patterns or bytecode markers."""
+    to_resources = (
+        project.codebaseresources.all()
+        .to_codebase()
+        .no_status()
+        .has_no_relation()
+        .path_pattern("*.class")
+    )
+    for resource in to_resources:
+        reason = None
+        path_segments = resource.location.split("/")
+        if any(
+            seg.lower() == "generated"
+            or seg.lower().startswith("generated-")
+            or seg.lower().startswith("generated_")
+            for seg in path_segments
+        ):
+            reason = "Path pattern matches generated directory convention"
+        else:
+            try:
+                data = Path(resource.location).read_bytes()
+                reason = is_generated_code(data)
+            except Exception as e:
+                print(f"Could not process {resource}: {e}")
+        if reason:
+            resource.update(status=flag.GENERATED)
+            resource.update_extra_data({"Generated code": reason})
+
+
 def flag_processed_archives(project):
     """
     Flag package archives as processed if they meet the following criteria:
@@ -1824,6 +1854,71 @@ def is_invalid_match(match, matched_path_length):
     return matched_path_length == 1 and len(match.resource_ids) != 1
 
 
+def is_generated_code(class_bytes):
+    """
+    Return a reason indicating why the file was identified as generated.
+    None otherwise.
+    """
+    generated_code_markers = {
+        #  @Generated annotations
+        # https://docs.oracle.com/javase/8/docs/api/javax/annotation/Generated.html
+        # https://docs.oracle.com/en/java/javase/11/docs/api/java.compiler/javax/annotation/processing/Generated.html
+        # https://jakarta.ee/specifications/annotations/2.1/apidocs/jakarta.annotation/jakarta/annotation/generated
+        b"Ljavax/annotation/Generated;": "@Generated annotation detected",
+        b"Ljavax/annotation/processing/Generated;": "@Generated annotation detected",
+        b"Ljakarta/annotation/Generated;": "@Generated annotation detected",
+        b"Ljakarta/annotation/processing/Generated;": "@Generated annotation detected",
+        # Protobuf
+        # https://protobuf.dev/reference/java/api-docs/com/google/protobuf/GeneratedMessage.html
+        b"Lcom/google/protobuf/GeneratedMessageV3;": "Google Protocol Buffers",
+        b"Lcom/google/protobuf/GeneratedMessage;": "Google Protocol Buffers",
+        # Apache Thrift
+        # https://javadoc.io/doc/org.apache.thrift/libthrift/latest/org/apache/thrift/TBase.html
+        b"Lorg/apache/thrift/TBase;": "Apache Thrift IDL compiler",
+        # Apache Avro
+        # https://avro.apache.org/docs/current/api/java/org/apache/avro/specific/SpecificRecord.html
+        # https://avro.apache.org/docs/current/api/java/org/apache/avro/specific/SpecificRecordBase.html
+        b"Lorg/apache/avro/specific/SpecificRecordBase;": "Apache Avro schema compiler",
+        # JAXB (ObjectFactory)
+        # https://docs.oracle.com/javase/8/docs/api/javax/xml/bind/annotation/XmlRegistry.html
+        b"Ljavax/xml/bind/annotation/XmlRegistry;": "JAXB XmlRegistry",
+        b"Ljakarta/xml/bind/annotation/XmlRegistry;": "JAXB XmlRegistry",
+        # JAX-WS Stubs
+        # https://docs.oracle.com/javase/8/docs/api/javax/xml/ws/WebServiceClient.html
+        b"Ljavax/xml/ws/WebServiceClient;": "JAX-WS client stub",
+        b"Ljakarta/xml/ws/WebServiceClient;": "JAX-WS client stub",
+        # gRPC Stubs
+        # https://grpc.github.io/grpc-java/javadoc/io/grpc/stub/annotations/GrpcGenerated.html
+        b"Lio/grpc/stub/annotations/GrpcGenerated;": "gRPC compiler stub",
+        # Others
+        # Immutables - https://github.com/immutables/immutables/issues/756
+        b"Lorg/immutables/value/Generated;": "Immutables Generated",
+    }
+
+    for marker, reason in generated_code_markers.items():
+        if marker in class_bytes:
+            return reason
+
+    # The JAXB compiler (xjc) mechanically applies both @XmlType and
+    # @XmlAccessorType together on every generated class
+    javax_jaxb_cluster = [
+        b"Ljavax/xml/bind/annotation/XmlAccessorType;",
+        b"Ljavax/xml/bind/annotation/XmlType;",
+    ]
+    jakarta_jaxb_cluster = [
+        b"Ljakarta/xml/bind/annotation/XmlAccessorType;",
+        b"Ljakarta/xml/bind/annotation/XmlType;",
+    ]
+
+    if all(marker in class_bytes for marker in javax_jaxb_cluster):
+        return "JAXB schema compiler cluster (@XmlAccessorType + @XmlType)"
+
+    if all(marker in class_bytes for marker in jakarta_jaxb_cluster):
+        return "Jakarta JAXB schema compiler cluster (@XmlAccessorType + @XmlType)"
+
+    return None
+
+
 def map_elfs_with_dwarf_paths(project, logger=None):
     """Map ELF binaries to their sources in ``project``."""
     from_resources = project.codebaseresources.files().from_codebase()