Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use lock for VulnerabilityPolicyFetchTask #504

Merged
merged 1 commit into from
Dec 21, 2023
Merged

Use lock for VulnerabilityPolicyFetchTask #504

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/main/java/org/dependencytrack/common/ConfigKey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ public enum ConfigKey implements Config.Key {
TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_LEAST_FOR("task.portfolio.repoMetaAnalysis.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())),
TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR("task.portfolio.vulnAnalysis.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())),
TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR("task.portfolio.vulnAnalysis.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())),
TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR("task.vulnerability.policy.bundle.fetch.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())),
TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_LEAST_FOR("task.vulnerability.policy.bundle.fetch.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(1).toMillis())),
BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD("bom.upload.processing.trx.flush.threshold", "10000"),
WORKFLOW_RETENTION_DURATION("workflow.retention.duration", "P3D"),
WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"),
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/dependencytrack/tasks/LockName.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ public enum LockName {
WORKFLOW_STEP_CLEANUP_TASK_LOCK,
PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK,
PORTFOLIO_VULN_ANALYSIS_TASK_LOCK,
INTEGRITY_META_INITIALIZER_LOCK
INTEGRITY_META_INITIALIZER_LOCK,
VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK
}
50 changes: 28 additions & 22 deletions ...main/java/org/dependencytrack/tasks/vulnerabilitypolicy/VulnerabilityPolicyFetchTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import net.javacrumbs.shedlock.core.LockingTaskExecutor.Task;
import org.dependencytrack.common.ConfigKey;
import org.dependencytrack.event.VulnerabilityPolicyFetchEvent;
import org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage.BlobStorageAccessFactory;
import org.dependencytrack.tasks.vulnerabilitypolicy.blobstorage.BlobStorageAccessHandler;
import org.dependencytrack.util.VulnerabilityPolicyUtil;
import org.projectnessie.cel.tools.ScriptException;

import javax.naming.OperationNotSupportedException;
import java.io.IOException;
import java.util.zip.ZipInputStream;

import static org.dependencytrack.tasks.LockName.VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK;
import static org.dependencytrack.util.LockProvider.executeWithLock;

public class VulnerabilityPolicyFetchTask implements Subscriber {
private static final Logger LOGGER = Logger.getLogger(VulnerabilityPolicyFetchTask.class);
private final BlobStorageAccessHandler handler;
Expand All @@ -29,27 +31,31 @@ public VulnerabilityPolicyFetchTask() throws OperationNotSupportedException {

@Override
public void inform(Event event) {
if (event instanceof VulnerabilityPolicyFetchEvent) {
if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.VULNERABILITY_POLICY_ANALYSIS_ENABLED)) {
LOGGER.debug("Currently vulnerability policy analysis is not enabled");
} else {
try {
// perform head request on file server to see if the hash of file has changed from previous fetch
if (handler.verifyDownloadNeeded()) {
LOGGER.info("It has been verified that file download would be needed from bundle source");
//if hash has changed, get the new zip file and unzip it to get the policy file
ZipInputStream inputStream = handler.downloadZippedContent();
LOGGER.info("Parsing downloaded policies for saving/ updating");
VulnerabilityPolicyUtil.parseAndSavePolicies(inputStream);
inputStream.close();
LOGGER.info("Policies saved to database successfully");
} else {
LOGGER.info("The zipped file content has not changed since last check. Will check in the next iteration");
}
} catch (IOException | RuntimeException ex) {
LOGGER.error("An error occurred while verifying changes in policy file or downloading policy file itself", ex);
if (!(event instanceof VulnerabilityPolicyFetchEvent)) {
return;
}
if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.VULNERABILITY_POLICY_ANALYSIS_ENABLED)) {
LOGGER.debug("Currently vulnerability policy analysis is not enabled");
return;
}

try {
executeWithLock(VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK, (Task) () -> {
// perform head request on file server to see if the hash of file has changed from previous fetch
if (handler.verifyDownloadNeeded()) {
LOGGER.info("It has been verified that file download would be needed from bundle source");
//if hash has changed, get the new zip file and unzip it to get the policy file
ZipInputStream inputStream = handler.downloadZippedContent();
LOGGER.info("Parsing downloaded policies for saving/ updating");
VulnerabilityPolicyUtil.parseAndSavePolicies(inputStream);
inputStream.close();
LOGGER.info("Policies saved to database successfully");
} else {
LOGGER.info("The zipped file content has not changed since last check. Will check in the next iteration");
}
}
});
} catch (Throwable ex) {
LOGGER.error("An error occurred while verifying changes in policy file or downloading policy file itself", ex);
}
}
}
7 changes: 7 additions & 0 deletions src/main/java/org/dependencytrack/util/LockProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_MOST_FOR;
import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR;
import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR;
import static org.dependencytrack.common.ConfigKey.TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_LEAST_FOR;
import static org.dependencytrack.common.ConfigKey.TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR;
import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR;
import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR;
import static org.dependencytrack.tasks.LockName.EPSS_MIRROR_TASK_LOCK;
Expand All @@ -44,6 +46,7 @@
import static org.dependencytrack.tasks.LockName.PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK;
import static org.dependencytrack.tasks.LockName.PORTFOLIO_VULN_ANALYSIS_TASK_LOCK;
import static org.dependencytrack.tasks.LockName.VULNERABILITY_METRICS_TASK_LOCK;
import static org.dependencytrack.tasks.LockName.VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK;
import static org.dependencytrack.tasks.LockName.WORKFLOW_STEP_CLEANUP_TASK_LOCK;

public class LockProvider {
Expand Down Expand Up @@ -141,6 +144,10 @@ public static LockConfiguration getLockConfigurationByLockName(LockName lockName
INTEGRITY_META_INITIALIZER_LOCK.name(),
Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR)),
Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR)));
case VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK -> new LockConfiguration(Instant.now(),
VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK.name(),
Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR)),
Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_LEAST_FOR)));
};

}
Expand Down