Merge pull request #533 from DependencyTrack/issue-1024

Populate VULNERABILITY_POLICY_BUNDLE during vuln policy bundle sync

src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityPolicyDao.java

@@ -3,6 +3,7 @@
 import org.dependencytrack.persistence.jdbi.mapping.VulnPolicyAnalysisArgumentFactory;
 import org.dependencytrack.persistence.jdbi.mapping.VulnPolicyRatingsArgumentFactory;
 import org.dependencytrack.persistence.jdbi.mapping.VulnerabilityPolicyRowMapper;
+import org.dependencytrack.policy.vulnerability.VulnerabilityPolicy;
 import org.jdbi.v3.sqlobject.config.RegisterArgumentFactories;
 import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory;
 import org.jdbi.v3.sqlobject.config.RegisterRowMapper;
@@ -21,35 +22,41 @@
 public interface VulnerabilityPolicyDao {
 
     @SqlQuery("""  
-                    SELECT "ID", "ANALYSIS", "AUTHOR", "CONDITIONS", "CREATED", "DESCRIPTION", "NAME", "RATINGS", "UPDATED", "VALID_FROM", "VALID_UNTIL"
-                	FROM "VULNERABILITY_POLICY";
+            SELECT * FROM "VULNERABILITY_POLICY";
             """)
-    @RegisterRowMapper(VulnerabilityPolicyRowMapper.class)
-    List<org.dependencytrack.policy.vulnerability.VulnerabilityPolicy> getAllVulnerabilityPolicies();
+    List<VulnerabilityPolicy> getAllVulnerabilityPolicies();
 
     @SqlQuery("""  
-                SELECT "ID", "ANALYSIS", "AUTHOR", "CONDITIONS", "CREATED", "DESCRIPTION", "NAME", "RATINGS", "UPDATED", "VALID_FROM", "VALID_UNTIL"
-                	FROM "VULNERABILITY_POLICY" WHERE "NAME" = ?;
+            SELECT * FROM "VULNERABILITY_POLICY" WHERE "NAME" = ?;
             """)
-    @RegisterRowMapper(VulnerabilityPolicyRowMapper.class)
-    org.dependencytrack.policy.vulnerability.VulnerabilityPolicy getVulnerabilityPolicyByName(@Bind String name);
+    VulnerabilityPolicy getVulnerabilityPolicyByName(@Bind String name);
 
     @SqlUpdate("""
-        INSERT INTO "VULNERABILITY_POLICY"(
-                	"ANALYSIS", "AUTHOR", "CONDITIONS", "CREATED", "DESCRIPTION", "NAME", "RATINGS", "UPDATED", "VALID_FROM", "VALID_UNTIL")
-             VALUES (:vulnerabilityPolicy.analysis::JSON, :vulnerabilityPolicy.author, :vulnerabilityPolicy.conditions, :vulnerabilityPolicy.created, :vulnerabilityPolicy.description, :vulnerabilityPolicy.name, :vulnerabilityPolicy.ratings::JSON, :vulnerabilityPolicy.updated, :vulnerabilityPolicy.validFrom, :vulnerabilityPolicy.validUntil)
-       
-    """)
-    int createVulnerabilityPolicy(@BindBean("vulnerabilityPolicy") org.dependencytrack.policy.vulnerability.VulnerabilityPolicy vulnerabilityPolicy);
+            INSERT INTO "VULNERABILITY_POLICY"
+              ("ANALYSIS", "AUTHOR", "CONDITIONS", "CREATED", "DESCRIPTION", "NAME", "RATINGS", "UPDATED", "VALID_FROM", "VALID_UNTIL")
+            VALUES
+              ((:analysis)::JSON, :author, :conditions, :created, :description, :name, (:ratings)::JSON, :updated, :validFrom, :validUntil)
+            """)
+    int createVulnerabilityPolicy(@BindBean VulnerabilityPolicy vulnerabilityPolicy);
 
     @SqlUpdate("""
-        DELETE FROM "VULNERABILITY_POLICY" WHERE "NAME" = ?     
-        """)
+            DELETE FROM "VULNERABILITY_POLICY" WHERE "NAME" = ?
+            """)
     int deleteVulnerabilityPolicyByName(@Bind String name);
+
     @SqlUpdate("""
-                UPDATE "VULNERABILITY_POLICY"
-                	SET "ANALYSIS"=:vulnerabilityPolicy.analysis::JSON, "AUTHOR"=:vulnerabilityPolicy.author, "CONDITIONS"=:vulnerabilityPolicy.conditions, "DESCRIPTION"=:vulnerabilityPolicy.description, "RATINGS"=:vulnerabilityPolicy.ratings::JSON, "UPDATED"=:vulnerabilityPolicy.updated, "VALID_FROM"=:vulnerabilityPolicy.validFrom, "VALID_UNTIL"=:vulnerabilityPolicy.validUntil
-                	WHERE "NAME" = :name
-                """)
-    int updateVulnerabilityPolicyByName(@BindBean("vulnerabilityPolicy") org.dependencytrack.policy.vulnerability.VulnerabilityPolicy vulnerabilityPolicy, @Bind String name);
+            UPDATE "VULNERABILITY_POLICY"
+            SET
+              "ANALYSIS" = :analysis::JSON,
+              "AUTHOR" = :author,
+              "CONDITIONS" = :conditions,
+              "DESCRIPTION" = :description,
+              "RATINGS" = :ratings::JSON,
+              "UPDATED" = :updated,
+              "VALID_FROM" = :validFrom,
+              "VALID_UNTIL" = :validUntil
+            WHERE "NAME" = :name
+            """)
+    int updateVulnerabilityPolicyByName(@BindBean VulnerabilityPolicy vulnerabilityPolicy);
+
 }

src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/VulnerabilityPolicyFetchTask.java

@@ -7,13 +7,16 @@
 import net.javacrumbs.shedlock.core.LockingTaskExecutor.Task;
 import org.dependencytrack.common.ConfigKey;
 import org.dependencytrack.event.VulnerabilityPolicyFetchEvent;
+import org.dependencytrack.model.VulnerabilityPolicyBundle;
 import org.dependencytrack.model.WorkflowState;
 import org.dependencytrack.persistence.QueryManager;
 import org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage.BlobStorageAccessFactory;
 import org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage.BlobStorageAccessHandler;
+import org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage.VulnerabilityPolicyBundleFile;
 import org.dependencytrack.util.VulnerabilityPolicyUtil;
 
 import javax.naming.OperationNotSupportedException;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.Date;
 import java.util.zip.ZipInputStream;
@@ -68,20 +71,40 @@ private void syncPolicyBundle(final QueryManager qm) throws IOException {
         }
 
         try {
-            // perform head request on file server to see if the hash of file has changed from previous fetch
-            if (handler.verifyDownloadNeeded()) {
-                LOGGER.info("It has been verified that file download would be needed from bundle source");
-                //if hash has changed, get the new zip file and unzip it to get the policy file
-                try (ZipInputStream inputStream = handler.downloadZippedContent()) {
-                    LOGGER.info("Parsing downloaded policies for saving/ updating");
-                    VulnerabilityPolicyUtil.parseAndSavePolicies(inputStream);
-                    inputStream.close();
-                    LOGGER.info("Policies saved to database successfully");
-                }
-            } else {
+            if (!handler.verifyDownloadNeeded()) {
                 LOGGER.info("The zipped file content has not changed since last check. Will check in the next iteration");
+                qm.updateWorkflowStateToComplete(workflowState);
+                return;
             }
 
+            // perform head request on file server to see if the hash of file has changed from previous fetch
+            LOGGER.info("It has been verified that file download would be needed from bundle source");
+            final VulnerabilityPolicyBundleFile bundleFile = handler.download();
+            //if hash has changed, get the new zip file and unzip it to get the policy file
+            try (ZipInputStream inputStream = new ZipInputStream(new ByteArrayInputStream(bundleFile.content()))) {
+                LOGGER.info("Parsing downloaded policies for saving/ updating");
+                VulnerabilityPolicyUtil.parseAndSavePolicies(inputStream);
+                LOGGER.info("Policies saved to database successfully");
+            }
+
+            qm.runInTransaction(() -> {
+                final var now = new Date();
+                VulnerabilityPolicyBundle policyBundle = qm.getVulnerabilityPolicyBundle();
+                if (policyBundle == null) {
+                    policyBundle = new VulnerabilityPolicyBundle();
+                    policyBundle.setUrl(bundleFile.url());
+                    policyBundle.setHash(bundleFile.hash());
+                    policyBundle.setCreated(now);
+                    policyBundle.setLastSuccessfulSync(now);
+                    qm.getPersistenceManager().makePersistent(policyBundle);
+                } else {
+                    policyBundle.setUrl(bundleFile.url());
+                    policyBundle.setHash(bundleFile.hash());
+                    policyBundle.setUpdated(now);
+                    policyBundle.setLastSuccessfulSync(now);
+                }
+            });
+
             qm.updateWorkflowStateToComplete(workflowState);
         } catch (RuntimeException e) {
             qm.updateWorkflowStateToFailed(workflowState, e.getMessage());

src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/blobstorage/BlobStorageAccessHandler.java

@@ -2,16 +2,11 @@
 
 
 import java.io.IOException;
-import java.util.zip.ZipInputStream;
 
 public interface BlobStorageAccessHandler {
 
-    default boolean verifyDownloadNeeded() throws IOException {
-        return false;
-    }
+    boolean verifyDownloadNeeded() throws IOException;
 
-    default ZipInputStream downloadZippedContent() throws IOException {
-        return null;
-    }
+    VulnerabilityPolicyBundleFile download() throws IOException;
 
 }

src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/blobstorage/NginxStorageHandler.java

@@ -1,14 +1,15 @@
 package org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage;
 
 import alpine.Config;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.Header;
-import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
 import org.dependencytrack.common.ConfigKey;
 import org.dependencytrack.common.HttpClientPool;
 import org.dependencytrack.util.HttpUtil;
@@ -17,7 +18,6 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Optional;
-import java.util.zip.ZipInputStream;
 
 public class NginxStorageHandler implements BlobStorageAccessHandler {
 
@@ -90,9 +90,17 @@ public boolean verifyDownloadNeeded() throws IOException {
     }
 
     @Override
-    public ZipInputStream downloadZippedContent() throws IOException {
-        CloseableHttpResponse response = performGetRequest();
-        HttpEntity entity = response.getEntity();
-        return new ZipInputStream(entity.getContent());
+    public VulnerabilityPolicyBundleFile download() throws IOException {
+        final byte[] bundleFileContent;
+        try (final CloseableHttpResponse response = performGetRequest()) {
+            bundleFileContent = EntityUtils.toByteArray(response.getEntity());
+        }
+
+        return new VulnerabilityPolicyBundleFile(
+                Config.getInstance().getProperty(ConfigKey.VULNERABILITY_POLICY_BUNDLE_URL),
+                DigestUtils.sha256Hex(bundleFileContent),
+                bundleFileContent
+        );
     }
+
 }

src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/blobstorage/S3StorageHandler.java

@@ -1,16 +1,16 @@
 package org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage;
 
 import alpine.Config;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.dependencytrack.common.ConfigKey;
 import org.dependencytrack.tasks.vulnerabilitypolicy.S3Client;
 import org.dependencytrack.util.VulnerabilityPolicyUtil;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.NoSuchElementException;
-import java.util.zip.ZipInputStream;
 
 public class S3StorageHandler implements BlobStorageAccessHandler {
+
     private final S3Client s3Client;
 
     public S3StorageHandler() {
@@ -34,8 +34,11 @@ public boolean verifyDownloadNeeded() throws IOException {
     }
 
     @Override
-    public ZipInputStream downloadZippedContent() throws IOException {
-        return new ZipInputStream(new ByteArrayInputStream(s3Client.getObject()));
+    public VulnerabilityPolicyBundleFile download() throws IOException {
+        final var bundleUrl = "%s/%s/%s".formatted(Config.getInstance().getProperty(ConfigKey.VULNERABILITY_POLICY_BUNDLE_URL), s3Client.getS3BucketName(), s3Client.getBundleToFetch());
+        final byte[] bundleContent = s3Client.getObject();
+        final String bundleHash = DigestUtils.sha256Hex(bundleContent);
+        return new VulnerabilityPolicyBundleFile(bundleUrl, bundleHash, bundleContent);
     }
 
 }

src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/blobstorage/VulnerabilityPolicyBundleFile.java

@@ -0,0 +1,11 @@
+package org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage;
+
+/**
+ * A vulnerability policy bundle file downloaded via {@link BlobStorageAccessHandler}.
+ *
+ * @param url     URL the file was downloaded from
+ * @param hash    SHA256 digest of the file content
+ * @param content The file content
+ */
+public record VulnerabilityPolicyBundleFile(String url, String hash, byte[] content) {
+}

src/main/java/org/dependencytrack/util/VulnerabilityPolicyUtil.java

@@ -2,39 +2,43 @@
 
 import alpine.common.logging.Logger;
 import alpine.model.ConfigProperty;
-import alpine.server.util.DbUtil;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.networknt.schema.JsonSchema;
 import com.networknt.schema.JsonSchemaFactory;
 import com.networknt.schema.SpecVersion;
 import com.networknt.schema.ValidationMessage;
 import org.dependencytrack.model.ConfigPropertyConstants;
-import org.dependencytrack.model.VulnerabilityPolicy;
 import org.dependencytrack.persistence.QueryManager;
+import org.dependencytrack.persistence.jdbi.VulnerabilityPolicyDao;
 import org.dependencytrack.policy.cel.CelPolicyScriptHost;
 import org.dependencytrack.policy.cel.CelPolicyType;
+import org.dependencytrack.policy.vulnerability.VulnerabilityPolicy;
 import org.projectnessie.cel.tools.ScriptCreateException;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
-import java.sql.Connection;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 import static org.apache.commons.io.IOUtils.resourceToString;
+import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi;
 
 public class VulnerabilityPolicyUtil {
+
     private static final Logger LOGGER = Logger.getLogger(VulnerabilityPolicyUtil.class);
-    private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory()).registerModule(new JsonOrgModule());
+    private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory())
+            .registerModule(new JavaTimeModule())
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
     private static JsonSchema schema = null;
 
@@ -89,8 +93,8 @@ public static void parseAndSavePolicies(ZipInputStream zipInputStream) throws IO
         ZipEntry zipEntry = zipInputStream.getNextEntry();
         List<VulnerabilityPolicy> createVulnerabilityPolicyList = new ArrayList<>();
         List<VulnerabilityPolicy> updateVulnerabilityPolicyList = new ArrayList<>();
-        try (QueryManager queryManager = new QueryManager()) {
-            List<VulnerabilityPolicy> existingVulnerabilityPolicies = queryManager.getAllVulnerabilityPolicies();
+        try (final var qm = new QueryManager()) {
+            List<VulnerabilityPolicy> existingVulnerabilityPolicies = jdbi(qm).withExtension(VulnerabilityPolicyDao.class, VulnerabilityPolicyDao::getAllVulnerabilityPolicies);
             List<String> policyNames = existingVulnerabilityPolicies.stream().map(VulnerabilityPolicy::getName).toList();
             int numTotalPolicies = 0, numParsingFailures = 0;
             while (zipEntry != null) {
@@ -116,7 +120,7 @@ public static void parseAndSavePolicies(ZipInputStream zipInputStream) throws IO
                         Policy bundles can only be applied when all policy definitions within them are valid.\
                         """.formatted(numParsingFailures, numTotalPolicies));
             }
-            saveParsedVulnerabilities(queryManager, createVulnerabilityPolicyList, updateVulnerabilityPolicyList, policyNames);
+            saveParsedVulnerabilities(qm, createVulnerabilityPolicyList, updateVulnerabilityPolicyList, policyNames);
         }
     }
 
@@ -146,7 +150,7 @@ public static boolean matchWithHashConfigProperty(String etag) {
         }
     }
 
-    public static void saveParsedVulnerabilities(QueryManager queryManager, List<VulnerabilityPolicy> createVulnerabilityPolicyList,
+    public static void saveParsedVulnerabilities(QueryManager qm, List<VulnerabilityPolicy> createVulnerabilityPolicyList,
                                                  List<VulnerabilityPolicy> updateVulnerabilityPolicyList,
                                                  List<String> policyNames) {
         List<String> receivedPolicyNames = new ArrayList<>();
@@ -158,26 +162,21 @@ public static void saveParsedVulnerabilities(QueryManager queryManager, List<Vul
         }
         List<String> vulnerabilityPoliciesToBeDeleted = policyNames.stream()
                 .filter(policyName -> !receivedPolicyNames.contains(policyName)).toList();
-        queryManager.runInTransaction(() -> {
-            Connection connection = null;
-            try {
-                connection = (Connection) queryManager.getPersistenceManager().getDataStoreConnection();
-                for (var vulnerabilityPolicy : createVulnerabilityPolicyList) {
-                    LOGGER.info("Creating vulnerability policy: %s".formatted(vulnerabilityPolicy.getName()));
-                    queryManager.createVulnerabilityPolicy(vulnerabilityPolicy, connection);
-                }
-                for (var vulnerabilityPolicy : updateVulnerabilityPolicyList) {
-                    LOGGER.info("Updating vulnerability policy: %s".formatted(vulnerabilityPolicy.getName()));
-                    queryManager.updateVulnerablePolicyByName(vulnerabilityPolicy, connection);
-                }
-                for (var name : vulnerabilityPoliciesToBeDeleted) {
-                    LOGGER.info("Deleting vulnerability policy: %s".formatted(name));
-                    queryManager.deleteVulnerabilityPolicyByName(name, connection);
-                }
-            } finally {
-                DbUtil.close(connection);
-            }
 
+        jdbi(qm).useTransaction(jdbiHandle -> {
+            final var vulnPolicyDao = jdbiHandle.attach(VulnerabilityPolicyDao.class);
+            for (var vulnerabilityPolicy : createVulnerabilityPolicyList) {
+                LOGGER.info("Creating vulnerability policy: %s".formatted(vulnerabilityPolicy.getName()));
+                vulnPolicyDao.createVulnerabilityPolicy(vulnerabilityPolicy);
+            }
+            for (var vulnerabilityPolicy : updateVulnerabilityPolicyList) {
+                LOGGER.info("Updating vulnerability policy: %s".formatted(vulnerabilityPolicy.getName()));
+                vulnPolicyDao.updateVulnerabilityPolicyByName(vulnerabilityPolicy);
+            }
+            for (var name : vulnerabilityPoliciesToBeDeleted) {
+                LOGGER.info("Deleting vulnerability policy: %s".formatted(name));
+                vulnPolicyDao.deleteVulnerabilityPolicyByName(name);
+            }
         });
     }
 

src/test/java/org/dependencytrack/persistence/jdbi/VulnerabilityPolicyDaoTest.java

@@ -159,7 +159,7 @@ public void testUpdateVulnerabilityPolicyByName() throws Exception {
         vulnPolicy.setAuthor("Jon Doe");
 
         int updatedCount = JdbiFactory.jdbi(qm).withExtension(VulnerabilityPolicyDao.class,
-                dao -> dao.updateVulnerabilityPolicyByName(vulnPolicy, "name"));
+                dao -> dao.updateVulnerabilityPolicyByName(vulnPolicy));
 
         assertThat(updatedCount).isEqualTo(1);