diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index fd54e0f20..89985970e 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -20,8 +20,6 @@ import alpine.Config; -import java.time.Duration; - public enum ConfigKey implements Config.Key { ALPINE_WORKER_POOL_DRAIN_TIMEOUT_DURATION("alpine.worker.pool.drain.timeout.duration", "PT30S"), @@ -38,49 +36,9 @@ public enum ConfigKey implements Config.Key { KAFKA_TRUST_STORE_PASSWORD("kafka.truststore.password", ""), KAFKA_TRUST_STORE_PATH("kafka.truststore.path", ""), - CRON_EXPRESSION_FOR_PORTFOLIO_METRICS_TASK("task.cron.metrics.portfolio", "10 * * * *"), - CRON_EXPRESSION_FOR_VULNERABILITY_METRICS_TASK("task.cron.metrics.vulnerability", "40 * * * *"), - CRON_EXPRESSION_FOR_COMPONENT_IDENTIFICATION_TASK("task.cron.componentIdentification", "25 */6 * * *"), - CRON_EXPRESSION_FOR_EPSS_MIRRORING_TASK("task.cron.mirror.epss", "0 1 * * *"), - CRON_EXPRESSION_FOR_GITHUB_MIRRORING_TASK("task.cron.mirror.github", "0 2 * * *"), - CRON_EXPRESSION_FOR_OSV_MIRRORING_TASK("task.cron.mirror.osv", "0 3 * * *"), - CRON_EXPRESSION_FOR_NIST_MIRRORING_TASK("task.cron.mirror.nist", "0 4 * * *"), - CRON_EXPRESSION_FOR_VULNERABILITY_POLICY_BUNDLE_FETCH_TASK("task.cron.vulnerability.policy.bundle.fetch", "*/5 * * * *"), - CRON_EXPRESSION_FOR_LDAP_SYNC_TASK("task.cron.ldapSync", "0 */6 * * *"), - CRON_EXPRESSION_FOR_REPO_META_ANALYSIS_TASK("task.cron.repoMetaAnalysis", "0 1 * * *"), - CRON_EXPRESSION_FOR_VULN_ANALYSIS_TASK("task.cron.vulnAnalysis", "0 6 * * *"), - CRON_EXPRESSION_FOR_VULN_SCAN_CLEANUP_TASK("task.cron.vulnScanCleanUp", "5 8 * * 4"), - CRON_EXPRESSION_FOR_FORTIFY_SSC_SYNC("task.cron.fortify.ssc.sync", "0 2 * * *"), - CRON_EXPRESSION_FOR_DEFECT_DOJO_SYNC("task.cron.defectdojo.sync", "0 2 * * *"), - CRON_EXPRESSION_FOR_KENNA_SYNC("task.cron.kenna.sync", "0 2 * * *"), - CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK("task.cron.workflow.state.cleanup", "*/15 * * * *"), - CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK("task.cron.integrityInitializer", "0 */12 * * *"), TASK_SCHEDULER_INITIAL_DELAY("task.scheduler.initial.delay", "180000"), TASK_SCHEDULER_POLLING_INTERVAL("task.scheduler.polling.interval", "60000"), - TASK_PORTFOLIO_LOCK_AT_MOST_FOR("task.metrics.portfolio.lockAtMostForInMillis", "900000"), - TASK_PORTFOLIO_LOCK_AT_LEAST_FOR("task.metrics.portfolio.lockAtLeastForInMillis", "90000"), - TASK_METRICS_VULNERABILITY_LOCK_AT_MOST_FOR("task.metrics.vulnerability.lockAtMostForInMillis", "900000"), - TASK_METRICS_VULNERABILITY_LOCK_AT_LEAST_FOR("task.metrics.vulnerability.lockAtLeastForInMillis", "90000"), - TASK_MIRROR_EPSS_LOCK_AT_MOST_FOR("task.mirror.epss.lockAtMostForInMillis", "900000"), - TASK_MIRROR_EPSS_LOCK_AT_LEAST_FOR("task.mirror.epss.lockAtLeastForInMillis", "90000"), - TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR("task.componentIdentification.lockAtMostForInMillis", "900000"), - TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR("task.componentIdentification.lockAtLeastForInMillis", "90000"), - TASK_LDAP_SYNC_LOCK_AT_MOST_FOR("task.ldapSync.lockAtMostForInMillis", "900000"), - TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR("task.ldapSync.lockAtLeastForInMillis", "90000"), - TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR("task.workflow.state.cleanup.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR("task.workflow.state.cleanup.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_MOST_FOR("task.portfolio.repoMetaAnalysis.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_LEAST_FOR("task.portfolio.repoMetaAnalysis.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), - TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR("task.portfolio.vulnAnalysis.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR("task.portfolio.vulnAnalysis.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), - TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR("task.vulnerability.policy.bundle.fetch.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), - TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_LEAST_FOR("task.vulnerability.policy.bundle.fetch.lockAtLeastForInMillis", String.valueOf(Duration.ofSeconds(5).toMillis())), - BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD("bom.upload.processing.trx.flush.threshold", "10000"), - WORKFLOW_RETENTION_DURATION("workflow.retention.duration", "P3D"), - WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"), TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"), - INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR("integrityMetaInitializer.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR("integrityMetaInitializer.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), INTEGRITY_INITIALIZER_ENABLED("integrity.initializer.enabled", "false"), INTEGRITY_CHECK_ENABLED("integrity.check.enabled", "false"), VULNERABILITY_POLICY_ANALYSIS_ENABLED("vulnerability.policy.analysis.enabled", false), diff --git a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java index 61d48bf5a..6ca19da0d 100644 --- a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java +++ b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java @@ -22,7 +22,16 @@ import alpine.common.logging.Logger; import alpine.event.LdapSyncEvent; import alpine.event.framework.EventService; +import alpine.event.framework.SingleThreadedEventService; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.maintenance.ComponentMetadataMaintenanceEvent; +import org.dependencytrack.event.maintenance.MetricsMaintenanceEvent; +import org.dependencytrack.event.maintenance.TagMaintenanceEvent; +import org.dependencytrack.event.maintenance.VulnerabilityDatabaseMaintenanceEvent; +import org.dependencytrack.event.maintenance.VulnerabilityScanMaintenanceEvent; +import org.dependencytrack.event.maintenance.WorkflowMaintenanceEvent; import org.dependencytrack.tasks.BomUploadProcessingTask; import org.dependencytrack.tasks.CallbackTask; import org.dependencytrack.tasks.CloneProjectTask; @@ -36,21 +45,23 @@ import org.dependencytrack.tasks.KennaSecurityUploadTask; import org.dependencytrack.tasks.LdapSyncTaskWrapper; import org.dependencytrack.tasks.NistMirrorTask; -import org.dependencytrack.tasks.OsvDownloadTask; +import org.dependencytrack.tasks.OsvMirrorTask; import org.dependencytrack.tasks.PolicyEvaluationTask; -import org.dependencytrack.tasks.RepositoryMetaAnalyzerTask; +import org.dependencytrack.tasks.RepositoryMetaAnalysisTask; import org.dependencytrack.tasks.TaskScheduler; import org.dependencytrack.tasks.VexUploadProcessingTask; import org.dependencytrack.tasks.VulnerabilityAnalysisTask; -import org.dependencytrack.tasks.vulnerabilitypolicy.VulnerabilityPolicyFetchTask; -import org.dependencytrack.tasks.VulnerabilityScanCleanupTask; -import org.dependencytrack.tasks.WorkflowStateCleanupTask; +import org.dependencytrack.tasks.maintenance.ComponentMetadataMaintenanceTask; +import org.dependencytrack.tasks.maintenance.MetricsMaintenanceTask; +import org.dependencytrack.tasks.maintenance.TagMaintenanceTask; +import org.dependencytrack.tasks.maintenance.VulnerabilityDatabaseMaintenanceTask; +import org.dependencytrack.tasks.maintenance.VulnerabilityScanMaintenanceTask; +import org.dependencytrack.tasks.maintenance.WorkflowMaintenanceTask; import org.dependencytrack.tasks.metrics.PortfolioMetricsUpdateTask; import org.dependencytrack.tasks.metrics.ProjectMetricsUpdateTask; import org.dependencytrack.tasks.metrics.VulnerabilityMetricsUpdateTask; +import org.dependencytrack.tasks.vulnerabilitypolicy.VulnerabilityPolicyFetchTask; -import jakarta.servlet.ServletContextEvent; -import jakarta.servlet.ServletContextListener; import java.time.Duration; /** @@ -65,6 +76,10 @@ public class EventSubsystemInitializer implements ServletContextListener { // Starts the EventService private static final EventService EVENT_SERVICE = EventService.getInstance(); + + // Starts the SingleThreadedEventService + private static final SingleThreadedEventService EVENT_SERVICE_ST = SingleThreadedEventService.getInstance(); + private static final Duration DRAIN_TIMEOUT_DURATION = Duration.parse(Config.getInstance().getProperty(ConfigKey.ALPINE_WORKER_POOL_DRAIN_TIMEOUT_DURATION)); @@ -79,11 +94,11 @@ public void contextInitialized(final ServletContextEvent event) { EVENT_SERVICE.subscribe(VexUploadEvent.class, VexUploadProcessingTask.class); EVENT_SERVICE.subscribe(LdapSyncEvent.class, LdapSyncTaskWrapper.class); EVENT_SERVICE.subscribe(GitHubAdvisoryMirrorEvent.class, GitHubAdvisoryMirrorTask.class); - EVENT_SERVICE.subscribe(OsvMirrorEvent.class, OsvDownloadTask.class); + EVENT_SERVICE.subscribe(OsvMirrorEvent.class, OsvMirrorTask.class); EVENT_SERVICE.subscribe(ProjectVulnerabilityAnalysisEvent.class, VulnerabilityAnalysisTask.class); EVENT_SERVICE.subscribe(PortfolioVulnerabilityAnalysisEvent.class, VulnerabilityAnalysisTask.class); - EVENT_SERVICE.subscribe(ProjectRepositoryMetaAnalysisEvent.class, RepositoryMetaAnalyzerTask.class); - EVENT_SERVICE.subscribe(PortfolioRepositoryMetaAnalysisEvent.class, RepositoryMetaAnalyzerTask.class); + EVENT_SERVICE.subscribe(ProjectRepositoryMetaAnalysisEvent.class, RepositoryMetaAnalysisTask.class); + EVENT_SERVICE.subscribe(PortfolioRepositoryMetaAnalysisEvent.class, RepositoryMetaAnalysisTask.class); EVENT_SERVICE.subscribe(ProjectMetricsUpdateEvent.class, ProjectMetricsUpdateTask.class); EVENT_SERVICE.subscribe(PortfolioMetricsUpdateEvent.class, PortfolioMetricsUpdateTask.class); EVENT_SERVICE.subscribe(VulnerabilityMetricsUpdateEvent.class, VulnerabilityMetricsUpdateTask.class); @@ -93,16 +108,23 @@ public void contextInitialized(final ServletContextEvent event) { EVENT_SERVICE.subscribe(KennaSecurityUploadEventAbstract.class, KennaSecurityUploadTask.class); EVENT_SERVICE.subscribe(InternalComponentIdentificationEvent.class, InternalComponentIdentificationTask.class); EVENT_SERVICE.subscribe(CallbackEvent.class, CallbackTask.class); - EVENT_SERVICE.subscribe(VulnerabilityScanCleanupEvent.class, VulnerabilityScanCleanupTask.class); EVENT_SERVICE.subscribe(NistMirrorEvent.class, NistMirrorTask.class); EVENT_SERVICE.subscribe(VulnerabilityPolicyFetchEvent.class, VulnerabilityPolicyFetchTask.class); EVENT_SERVICE.subscribe(EpssMirrorEvent.class, EpssMirrorTask.class); EVENT_SERVICE.subscribe(ComponentPolicyEvaluationEvent.class, PolicyEvaluationTask.class); EVENT_SERVICE.subscribe(ProjectPolicyEvaluationEvent.class, PolicyEvaluationTask.class); - EVENT_SERVICE.subscribe(WorkflowStateCleanupEvent.class, WorkflowStateCleanupTask.class); EVENT_SERVICE.subscribe(IntegrityMetaInitializerEvent.class, IntegrityMetaInitializerTask.class); EVENT_SERVICE.subscribe(IntegrityAnalysisEvent.class, IntegrityAnalysisTask.class); + // Execute maintenance tasks on the single-threaded event service. + // This way, they are not blocked by, and don't block, actual processing tasks on the main event service. + EVENT_SERVICE_ST.subscribe(ComponentMetadataMaintenanceEvent.class, ComponentMetadataMaintenanceTask.class); + EVENT_SERVICE_ST.subscribe(MetricsMaintenanceEvent.class, MetricsMaintenanceTask.class); + EVENT_SERVICE_ST.subscribe(TagMaintenanceEvent.class, TagMaintenanceTask.class); + EVENT_SERVICE_ST.subscribe(VulnerabilityDatabaseMaintenanceEvent.class, VulnerabilityDatabaseMaintenanceTask.class); + EVENT_SERVICE_ST.subscribe(VulnerabilityScanMaintenanceEvent.class, VulnerabilityScanMaintenanceTask.class); + EVENT_SERVICE_ST.subscribe(WorkflowMaintenanceEvent.class, WorkflowMaintenanceTask.class); + TaskScheduler.getInstance(); } @@ -118,9 +140,9 @@ public void contextDestroyed(final ServletContextEvent event) { EVENT_SERVICE.unsubscribe(VexUploadProcessingTask.class); EVENT_SERVICE.unsubscribe(LdapSyncTaskWrapper.class); EVENT_SERVICE.unsubscribe(GitHubAdvisoryMirrorTask.class); - EVENT_SERVICE.unsubscribe(OsvDownloadTask.class); + EVENT_SERVICE.unsubscribe(OsvMirrorTask.class); EVENT_SERVICE.unsubscribe(VulnerabilityAnalysisTask.class); - EVENT_SERVICE.unsubscribe(RepositoryMetaAnalyzerTask.class); + EVENT_SERVICE.unsubscribe(RepositoryMetaAnalysisTask.class); EVENT_SERVICE.unsubscribe(ProjectMetricsUpdateTask.class); EVENT_SERVICE.unsubscribe(PortfolioMetricsUpdateTask.class); EVENT_SERVICE.unsubscribe(VulnerabilityMetricsUpdateTask.class); @@ -130,14 +152,20 @@ public void contextDestroyed(final ServletContextEvent event) { EVENT_SERVICE.unsubscribe(KennaSecurityUploadTask.class); EVENT_SERVICE.unsubscribe(InternalComponentIdentificationTask.class); EVENT_SERVICE.unsubscribe(CallbackTask.class); - EVENT_SERVICE.unsubscribe(VulnerabilityScanCleanupTask.class); EVENT_SERVICE.unsubscribe(NistMirrorTask.class); EVENT_SERVICE.unsubscribe(EpssMirrorTask.class); EVENT_SERVICE.unsubscribe(PolicyEvaluationTask.class); - EVENT_SERVICE.unsubscribe(WorkflowStateCleanupTask.class); EVENT_SERVICE.unsubscribe(IntegrityMetaInitializerTask.class); EVENT_SERVICE.unsubscribe(IntegrityAnalysisTask.class); EVENT_SERVICE.unsubscribe(VulnerabilityPolicyFetchTask.class); EVENT_SERVICE.shutdown(DRAIN_TIMEOUT_DURATION); + + EVENT_SERVICE_ST.unsubscribe(ComponentMetadataMaintenanceTask.class); + EVENT_SERVICE_ST.unsubscribe(MetricsMaintenanceTask.class); + EVENT_SERVICE_ST.unsubscribe(TagMaintenanceTask.class); + EVENT_SERVICE_ST.unsubscribe(VulnerabilityDatabaseMaintenanceTask.class); + EVENT_SERVICE_ST.unsubscribe(VulnerabilityScanMaintenanceTask.class); + EVENT_SERVICE_ST.unsubscribe(WorkflowMaintenanceTask.class); + EVENT_SERVICE_ST.shutdown(DRAIN_TIMEOUT_DURATION); } } diff --git a/src/main/java/org/dependencytrack/event/PurlMigrator.java b/src/main/java/org/dependencytrack/event/PurlMigrator.java index de609cd11..ab60fe8b1 100644 --- a/src/main/java/org/dependencytrack/event/PurlMigrator.java +++ b/src/main/java/org/dependencytrack/event/PurlMigrator.java @@ -20,15 +20,15 @@ import alpine.Config; import alpine.common.logging.Logger; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import net.javacrumbs.shedlock.core.LockingTaskExecutor; import org.dependencytrack.common.ConfigKey; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.util.LockProvider; - -import jakarta.servlet.ServletContextEvent; -import jakarta.servlet.ServletContextListener; +import org.dependencytrack.tasks.IntegrityMetaInitializerTask; -import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_LOCK; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; public class PurlMigrator implements ServletContextListener { @@ -48,7 +48,9 @@ public PurlMigrator() { public void contextInitialized(final ServletContextEvent event) { if (integrityInitializerEnabled) { try { - LockProvider.executeWithLock(INTEGRITY_META_INITIALIZER_LOCK, (LockingTaskExecutor.Task) () -> process()); + executeWithLock( + getLockConfigForTask(IntegrityMetaInitializerTask.class), + (LockingTaskExecutor.Task) this::process); } catch (Throwable e) { throw new RuntimeException("An unexpected error occurred while running Initializer for integrity meta", e); } diff --git a/src/main/java/org/dependencytrack/event/VulnerabilityScanCleanupEvent.java b/src/main/java/org/dependencytrack/event/maintenance/ComponentMetadataMaintenanceEvent.java similarity index 70% rename from src/main/java/org/dependencytrack/event/VulnerabilityScanCleanupEvent.java rename to src/main/java/org/dependencytrack/event/maintenance/ComponentMetadataMaintenanceEvent.java index cb73da811..86223af5f 100644 --- a/src/main/java/org/dependencytrack/event/VulnerabilityScanCleanupEvent.java +++ b/src/main/java/org/dependencytrack/event/maintenance/ComponentMetadataMaintenanceEvent.java @@ -16,16 +16,12 @@ * SPDX-License-Identifier: Apache-2.0 * Copyright (c) OWASP Foundation. All Rights Reserved. */ -package org.dependencytrack.event; +package org.dependencytrack.event.maintenance; import alpine.event.framework.Event; -import org.dependencytrack.model.VulnerabilityScan; /** - * Defines an {@link Event} that is triggered to clean up {@link VulnerabilityScan}s - * that have not been updated for a certain amount of time. - * - * @since 5.0.0 + * @since 5.6.0 */ -public record VulnerabilityScanCleanupEvent() implements Event { +public class ComponentMetadataMaintenanceEvent implements Event { } diff --git a/src/main/java/org/dependencytrack/tasks/LockName.java b/src/main/java/org/dependencytrack/event/maintenance/MetricsMaintenanceEvent.java similarity index 61% rename from src/main/java/org/dependencytrack/tasks/LockName.java rename to src/main/java/org/dependencytrack/event/maintenance/MetricsMaintenanceEvent.java index e94a215b1..fc7f18e5f 100644 --- a/src/main/java/org/dependencytrack/tasks/LockName.java +++ b/src/main/java/org/dependencytrack/event/maintenance/MetricsMaintenanceEvent.java @@ -16,17 +16,12 @@ * SPDX-License-Identifier: Apache-2.0 * Copyright (c) OWASP Foundation. All Rights Reserved. */ -package org.dependencytrack.tasks; +package org.dependencytrack.event.maintenance; -public enum LockName { - PORTFOLIO_METRICS_TASK_LOCK, - LDAP_SYNC_TASK_LOCK, - EPSS_MIRROR_TASK_LOCK, - VULNERABILITY_METRICS_TASK_LOCK, - INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK, - WORKFLOW_STEP_CLEANUP_TASK_LOCK, - PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, - PORTFOLIO_VULN_ANALYSIS_TASK_LOCK, - INTEGRITY_META_INITIALIZER_LOCK, - VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK +import alpine.event.framework.Event; + +/** + * @since 5.6.0 + */ +public class MetricsMaintenanceEvent implements Event { } diff --git a/src/main/java/org/dependencytrack/event/WorkflowStateCleanupEvent.java b/src/main/java/org/dependencytrack/event/maintenance/TagMaintenanceEvent.java similarity index 85% rename from src/main/java/org/dependencytrack/event/WorkflowStateCleanupEvent.java rename to src/main/java/org/dependencytrack/event/maintenance/TagMaintenanceEvent.java index 365d47085..5398e0626 100644 --- a/src/main/java/org/dependencytrack/event/WorkflowStateCleanupEvent.java +++ b/src/main/java/org/dependencytrack/event/maintenance/TagMaintenanceEvent.java @@ -16,9 +16,12 @@ * SPDX-License-Identifier: Apache-2.0 * Copyright (c) OWASP Foundation. All Rights Reserved. */ -package org.dependencytrack.event; +package org.dependencytrack.event.maintenance; import alpine.event.framework.Event; -public class WorkflowStateCleanupEvent implements Event { +/** + * @since 5.6.0 + */ +public class TagMaintenanceEvent implements Event { } diff --git a/src/main/java/org/dependencytrack/event/maintenance/VulnerabilityDatabaseMaintenanceEvent.java b/src/main/java/org/dependencytrack/event/maintenance/VulnerabilityDatabaseMaintenanceEvent.java new file mode 100644 index 000000000..784314874 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/maintenance/VulnerabilityDatabaseMaintenanceEvent.java @@ -0,0 +1,27 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.event.maintenance; + +import alpine.event.framework.Event; + +/** + * @since 5.6.0 + */ +public class VulnerabilityDatabaseMaintenanceEvent implements Event { +} diff --git a/src/main/java/org/dependencytrack/event/maintenance/VulnerabilityScanMaintenanceEvent.java b/src/main/java/org/dependencytrack/event/maintenance/VulnerabilityScanMaintenanceEvent.java new file mode 100644 index 000000000..2bdb4ac79 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/maintenance/VulnerabilityScanMaintenanceEvent.java @@ -0,0 +1,27 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.event.maintenance; + +import alpine.event.framework.Event; + +/** + * @since 5.6.0 + */ +public class VulnerabilityScanMaintenanceEvent implements Event { +} diff --git a/src/main/java/org/dependencytrack/event/maintenance/WorkflowMaintenanceEvent.java b/src/main/java/org/dependencytrack/event/maintenance/WorkflowMaintenanceEvent.java new file mode 100644 index 000000000..5c4fa9185 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/maintenance/WorkflowMaintenanceEvent.java @@ -0,0 +1,27 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.event.maintenance; + +import alpine.event.framework.Event; + +/** + * @since 5.6.0 + */ +public class WorkflowMaintenanceEvent implements Event { +} diff --git a/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java b/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java index d6aee916c..9fbd52fb0 100644 --- a/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java +++ b/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java @@ -44,6 +44,11 @@ public enum ConfigPropertyConstants { JIRA_URL("integrations", "jira.url", null, PropertyType.URL, "The base URL of the JIRA instance", ConfigPropertyAccessMode.READ_WRITE), JIRA_USERNAME("integrations", "jira.username", null, PropertyType.STRING, "The optional username to authenticate with when creating an Jira issue", ConfigPropertyAccessMode.READ_WRITE), JIRA_PASSWORD("integrations", "jira.password", null, PropertyType.ENCRYPTEDSTRING, "The optional password for the username used for authentication", ConfigPropertyAccessMode.READ_WRITE), + MAINTENANCE_METRICS_RETENTION_DAYS("maintenance", "metrics.retention.days", "90", PropertyType.INTEGER, "Number of days to retain metrics data for", ConfigPropertyAccessMode.READ_WRITE), + MAINTENANCE_TAGS_DELETE_UNUSED("maintenance", "tags.delete.unused", "true", PropertyType.BOOLEAN, "Whether unused tags shall be deleted", ConfigPropertyAccessMode.READ_WRITE), + MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS("maintenance", "vuln.scan.retention.hours", "24", PropertyType.INTEGER, "Number of hours to retain vulnerability scan records for", ConfigPropertyAccessMode.READ_WRITE), + MAINTENANCE_WORKFLOW_RETENTION_HOURS("maintenance", "workflow.retention.hours", "72", PropertyType.INTEGER, "Number of hours to retain workflow records for", ConfigPropertyAccessMode.READ_WRITE), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES("maintenance", "workflow.step.timeout.minutes", "60", PropertyType.INTEGER, "Number of minutes after which workflow steps are timed out", ConfigPropertyAccessMode.READ_WRITE), SCANNER_INTERNAL_ENABLED("scanner", "internal.enabled", "true", PropertyType.BOOLEAN, "Flag to enable/disable the internal analyzer", ConfigPropertyAccessMode.READ_WRITE), SCANNER_INTERNAL_FUZZY_ENABLED("scanner", "internal.fuzzy.enabled", "false", PropertyType.BOOLEAN, "Flag to enable/disable non-exact fuzzy matching using the internal analyzer", ConfigPropertyAccessMode.READ_WRITE), SCANNER_INTERNAL_FUZZY_EXCLUDE_PURL("scanner", "internal.fuzzy.exclude.purl", "true", PropertyType.BOOLEAN, "Flag to enable/disable fuzzy matching on components that have a Package URL (PURL) defined", ConfigPropertyAccessMode.READ_WRITE), diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 6bcfa32b6..2751efc86 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -1899,10 +1899,6 @@ public void updateWorkflowStateToFailed(WorkflowState workflowState, String fail getWorkflowStateQueryManager().updateWorkflowStateToFailed(workflowState, failureReason); } - public boolean hasWorkflowStepWithStatus(final UUID token, final WorkflowStep step, final WorkflowStatus status) { - return getWorkflowStateQueryManager().hasWorkflowStepWithStatus(token, step, status); - } - public IntegrityMetaComponent getIntegrityMetaComponent(String purl) { return getIntegrityMetaQueryManager().getIntegrityMetaComponent(purl); } diff --git a/src/main/java/org/dependencytrack/persistence/WorkflowStateQueryManager.java b/src/main/java/org/dependencytrack/persistence/WorkflowStateQueryManager.java index 6ef5c7bba..468f03f5f 100644 --- a/src/main/java/org/dependencytrack/persistence/WorkflowStateQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/WorkflowStateQueryManager.java @@ -35,7 +35,6 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Map; import java.util.UUID; public class WorkflowStateQueryManager extends QueryManager implements IQueryManager { @@ -329,20 +328,4 @@ public void createWorkflowSteps(UUID token) { } } - public boolean hasWorkflowStepWithStatus(final UUID token, final WorkflowStep step, final WorkflowStatus status) { - final Query stateQuery = pm.newQuery(WorkflowState.class); - stateQuery.setFilter("token == :token && step == :step && status == :status"); - stateQuery.setNamedParameters(Map.of( - "token", token, - "step", step, - "status", status - )); - stateQuery.setResult("count(this)"); - try { - return stateQuery.executeResultUnique(Long.class) > 0; - } finally { - stateQuery.closeAll(); - } - } - } diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/ComponentMetaDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/ComponentMetaDao.java new file mode 100644 index 000000000..582f01984 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/ComponentMetaDao.java @@ -0,0 +1,47 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.persistence.jdbi; + +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +/** + * @since 5.6.0 + */ +public interface ComponentMetaDao { + + @SqlUpdate(""" + DELETE + FROM "INTEGRITY_META_COMPONENT" + WHERE NOT EXISTS( + SELECT 1 + FROM "COMPONENT" + WHERE "COMPONENT"."PURL" = "INTEGRITY_META_COMPONENT"."PURL") + """) + int deleteOrphanIntegrityMetaComponents(); + + // TODO: Do a NOT EXISTS query against the COMPONENT table instead. + // Requires https://github.com/DependencyTrack/hyades/issues/1465. + @SqlUpdate(""" + DELETE + FROM "REPOSITORY_META_COMPONENT" + WHERE NOW() - "LAST_CHECK" > INTERVAL '30' DAY + """) + int deleteOrphanRepositoryMetaComponents(); + +} diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/ConfigPropertyDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/ConfigPropertyDao.java new file mode 100644 index 000000000..2041af106 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/ConfigPropertyDao.java @@ -0,0 +1,83 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.persistence.jdbi; + +import alpine.model.IConfigProperty; +import alpine.security.crypto.DataEncryption; +import org.dependencytrack.model.ConfigPropertyConstants; +import org.jdbi.v3.sqlobject.customizer.BindBean; +import org.jdbi.v3.sqlobject.statement.SqlQuery; + +import java.util.NoSuchElementException; +import java.util.Optional; + +/** + * @since 5.6.0 + */ +public interface ConfigPropertyDao { + + @SqlQuery(""" + SELECT "PROPERTYVALUE" + FROM "CONFIGPROPERTY" + WHERE "GROUPNAME" = :groupName + AND "PROPERTYNAME" = :propertyName + """) + Optional getOptionalRawValue(@BindBean ConfigPropertyConstants property); + + default Optional getOptionalValue(final ConfigPropertyConstants property) { + final Optional optionalRawValue = getOptionalRawValue(property); + if (optionalRawValue.isEmpty() || property.getPropertyType() != IConfigProperty.PropertyType.ENCRYPTEDSTRING) { + return optionalRawValue; + } + + try { + final String decryptedValue = DataEncryption.decryptAsString(optionalRawValue.get()); + return Optional.of(decryptedValue); + } catch (Exception e) { + throw new IllegalStateException("Failed to decrypt value", e); + } + } + + default Optional getOptionalValue(final ConfigPropertyConstants property, final Class clazz) { + final Optional optionalStringValue = getOptionalValue(property); + if (optionalStringValue.isEmpty()) { + return Optional.empty(); + } + + final T convertedValue; + + // Add more conversions as needed. + if (clazz.isAssignableFrom(CharSequence.class)) { + convertedValue = clazz.cast(optionalStringValue.get()); + } else if (clazz.isAssignableFrom(Boolean.class)) { + convertedValue = clazz.cast(Boolean.parseBoolean(optionalStringValue.get())); + } else if (clazz.isAssignableFrom(Integer.class)) { + convertedValue = clazz.cast(Integer.parseInt(optionalStringValue.get())); + } else { + throw new IllegalArgumentException("Cannot convert to %s".formatted(clazz.getName())); + } + + return Optional.of(convertedValue); + } + + default T getValue(final ConfigPropertyConstants property, final Class clazz) { + return getOptionalValue(property, clazz).orElseThrow(NoSuchElementException::new); + } + +} diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/MetricsDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/MetricsDao.java new file mode 100644 index 000000000..d1ad50a72 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/MetricsDao.java @@ -0,0 +1,58 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.persistence.jdbi; + +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +import java.time.Duration; + +/** + * @since 5.6.0 + */ +public interface MetricsDao { + + @SqlUpdate(""" + DELETE + FROM "DEPENDENCYMETRICS" + USING "PROJECT" + WHERE "PROJECT"."ID" = "DEPENDENCYMETRICS"."PROJECT_ID" + AND "PROJECT"."ACTIVE" + AND NOW() - "DEPENDENCYMETRICS"."LAST_OCCURRENCE" > :duration + """) + int deleteComponentMetricsForRetentionDuration(@Bind Duration duration); + + @SqlUpdate(""" + DELETE + FROM "PROJECTMETRICS" + USING "PROJECT" + WHERE "PROJECT"."ID" = "PROJECTMETRICS"."PROJECT_ID" + AND "PROJECT"."ACTIVE" + AND NOW() - "PROJECTMETRICS"."LAST_OCCURRENCE" > :duration + """) + int deleteProjectMetricsForRetentionDuration(@Bind Duration duration); + + @SqlUpdate(""" + DELETE + FROM "PORTFOLIOMETRICS" + WHERE NOW() - "LAST_OCCURRENCE" > :duration + """) + int deletePortfolioMetricsForRetentionDuration(@Bind Duration duration); + +} diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/TagDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/TagDao.java new file mode 100644 index 000000000..eed52bc41 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/TagDao.java @@ -0,0 +1,46 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.persistence.jdbi; + +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +/** + * @since 5.6.0 + */ +public interface TagDao { + + @SqlUpdate(""" + DELETE + FROM "TAG" + WHERE NOT EXISTS( + SELECT 1 + FROM "PROJECTS_TAGS" + WHERE "PROJECTS_TAGS"."TAG_ID" = "TAG"."ID") + AND NOT EXISTS( + SELECT 1 + FROM "POLICY_TAGS" + WHERE "POLICY_TAGS"."TAG_ID" = "TAG"."ID") + AND NOT EXISTS( + SELECT 1 + FROM "VULNERABILITIES_TAGS" + WHERE "VULNERABILITIES_TAGS"."TAG_ID" = "TAG"."ID") + """) + int deleteUnused(); + +} diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityDao.java index b9785ca6b..8ce880fc5 100644 --- a/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityDao.java +++ b/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityDao.java @@ -27,6 +27,7 @@ import org.jdbi.v3.sqlobject.customizer.Bind; import org.jdbi.v3.sqlobject.customizer.DefineNamedBindings; import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; import java.util.List; import java.util.UUID; @@ -231,4 +232,15 @@ record AffectedProjectCountRow( """) @RegisterBeanMapper(Component.class) List getVulnerableComponents(@Bind long projectId, @Bind List vulnerabilityIds); + + @SqlUpdate(""" + DELETE + FROM "VULNERABLESOFTWARE" + WHERE NOT EXISTS( + SELECT 1 + FROM "VULNERABLESOFTWARE_VULNERABILITIES" + WHERE "VULNERABLESOFTWARE_ID" = "VULNERABLESOFTWARE"."ID") + """) + int deleteOrphanVulnerableSoftware(); + } diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityScanDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityScanDao.java index c1f0048a1..77ee3a820 100644 --- a/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityScanDao.java +++ b/src/main/java/org/dependencytrack/persistence/jdbi/VulnerabilityScanDao.java @@ -24,7 +24,9 @@ import org.jdbi.v3.sqlobject.customizer.Bind; import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; import org.jdbi.v3.sqlobject.statement.SqlBatch; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import java.time.Duration; import java.util.List; import java.util.UUID; @@ -55,9 +57,17 @@ THEN CASE WHEN (CAST("SCAN_FAILED" + :scannerResultsFailed AS DOUBLE PRECISION) """) @RegisterBeanMapper(VulnerabilityScan.class) @GetGeneratedKeys({"TOKEN", "STATUS", "TARGET_TYPE", "TARGET_IDENTIFIER", "FAILURE_REASON"}) - List updateAll(@Bind("token") List tokens, - @Bind List resultsTotal, - @Bind List scannerResultsTotal, - @Bind List scannerResultsFailed); + List updateAll( + @Bind("token") List tokens, + @Bind List resultsTotal, + @Bind List scannerResultsTotal, + @Bind List scannerResultsFailed); + + @SqlUpdate(""" + DELETE + FROM "VULNERABILITYSCAN" + WHERE NOW() - "UPDATED_AT" > :duration + """) + int deleteAllForRetentionDuration(@Bind Duration duration); } diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/WorkflowDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/WorkflowDao.java index 7f92ae834..eb3f75572 100644 --- a/src/main/java/org/dependencytrack/persistence/jdbi/WorkflowDao.java +++ b/src/main/java/org/dependencytrack/persistence/jdbi/WorkflowDao.java @@ -21,12 +21,15 @@ import org.dependencytrack.model.WorkflowState; import org.dependencytrack.model.WorkflowStatus; import org.dependencytrack.model.WorkflowStep; +import org.jdbi.v3.sqlobject.SqlObject; import org.jdbi.v3.sqlobject.config.RegisterBeanMapper; import org.jdbi.v3.sqlobject.customizer.Bind; import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; import org.jdbi.v3.sqlobject.statement.SqlBatch; import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import java.time.Duration; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -34,7 +37,7 @@ import java.util.Set; import java.util.UUID; -public interface WorkflowDao { +public interface WorkflowDao extends SqlObject { @SqlBatch(""" UPDATE "WORKFLOW_STATE" @@ -47,16 +50,22 @@ public interface WorkflowDao { """) @GetGeneratedKeys("*") @RegisterBeanMapper(WorkflowState.class) - List updateAllStates(@Bind WorkflowStep step, - @Bind("token") List tokens, - @Bind("status") List statuses, - @Bind("failureReason") List failureReasons); + List updateAllStates( + @Bind WorkflowStep step, + @Bind("token") List tokens, + @Bind("status") List statuses, + @Bind("failureReason") List failureReasons); - default Optional updateState(final WorkflowStep step, - final UUID token, - final WorkflowStatus status, - final String failureReason) { - final List updatedStates = updateAllStates(step, List.of(token), List.of(status), Collections.singletonList(failureReason)); + default Optional updateState( + final WorkflowStep step, + final UUID token, + final WorkflowStatus status, + final String failureReason) { + final List updatedStates = updateAllStates( + step, + List.of(token), + List.of(status), + Collections.singletonList(failureReason)); if (updatedStates.isEmpty()) { return Optional.empty(); } @@ -71,9 +80,10 @@ default Optional updateState(final WorkflowStep step, AND "STATUS" = :status AND "TOKEN" = ANY(:tokens) """) - Set getTokensByStepAndStateAndTokenAnyOf(@Bind WorkflowStep step, - @Bind WorkflowStatus status, - @Bind Collection tokens); + Set getTokensByStepAndStateAndTokenAnyOf( + @Bind WorkflowStep step, + @Bind WorkflowStatus status, + @Bind Collection tokens); @SqlBatch(""" WITH RECURSIVE @@ -96,8 +106,83 @@ Set getTokensByStepAndStateAndTokenAnyOf(@Bind WorkflowStep step, UPDATE "WORKFLOW_STATE" SET "STATUS" = 'CANCELLED' , "UPDATED_AT" = NOW() - WHERE "ID" IN (SELECT "ID" FROM "CTE_CHILDREN") + WHERE "ID" = ANY(SELECT "ID" FROM "CTE_CHILDREN") """) void cancelAllChildren(@Bind WorkflowStep step, @Bind("token") List tokens); + /** + * @since 5.6.0 + */ + @SqlBatch(""" + WITH RECURSIVE + "CTE_CHILDREN" ("ID") AS ( + SELECT "ID" + FROM "WORKFLOW_STATE" + WHERE "PARENT_STEP_ID" = :parentId + UNION ALL + SELECT "CHILD"."ID" + FROM "WORKFLOW_STATE" AS "CHILD" + INNER JOIN "CTE_CHILDREN" AS "PARENT" + ON "PARENT"."ID" = "CHILD"."PARENT_STEP_ID" + ) + UPDATE "WORKFLOW_STATE" + SET "STATUS" = 'CANCELLED' + , "UPDATED_AT" = NOW() + WHERE "ID" = ANY(SELECT "ID" FROM "CTE_CHILDREN") + """) + int[] cancelAllChildrenByParentStepIdAnyOf(@Bind("parentId") List parentIds); + + /** + * @since 5.6.0 + */ + @SqlUpdate(""" + UPDATE "WORKFLOW_STATE" + SET "STATUS" = 'TIMED_OUT' + , "UPDATED_AT" = NOW() + WHERE "STATUS" = 'PENDING' + AND NOW() - "UPDATED_AT" > :timeoutDuration + """) + int transitionAllPendingStepsToTimedOutForTimeout(@Bind Duration timeoutDuration); + + /** + * @since 5.6.0 + */ + default List transitionAllTimedOutStepsToFailedForTimeout(final Duration timeoutDuration) { + // NB: Can't use interface method here due to https://github.com/jdbi/jdbi/issues/1807. + return getHandle().createUpdate(""" + UPDATE "WORKFLOW_STATE" + SET "STATUS" = 'FAILED' + , "FAILURE_REASON" = 'Timed out' + , "UPDATED_AT" = NOW() + WHERE "STATUS" = 'TIMED_OUT' + AND NOW() - "UPDATED_AT" > :timeoutDuration + RETURNING "ID" + """) + .bind("timeoutDuration", timeoutDuration) + .executeAndReturnGeneratedKeys() + .mapTo(Long.class) + .list(); + } + + /** + * @since 5.6.0 + */ + @SqlUpdate(""" + WITH "CTE_ELIGIBLE_TOKENS" AS ( + SELECT "TOKEN" + FROM "WORKFLOW_STATE" AS "WFS_PARENT" + WHERE NOT EXISTS( + SELECT 1 + FROM "WORKFLOW_STATE" AS "WFS" + WHERE "WFS"."TOKEN" = "WFS_PARENT"."TOKEN" + AND "WFS"."STATUS" IN ('PENDING', 'TIMED_OUT')) + GROUP BY "TOKEN" + HAVING NOW() - MAX("UPDATED_AT") > :retentionDuration + ) + DELETE + FROM "WORKFLOW_STATE" + WHERE "TOKEN" = ANY(SELECT "TOKEN" FROM "CTE_ELIGIBLE_TOKENS") + """) + int deleteAllForRetention(@Bind Duration retentionDuration); + } diff --git a/src/main/java/org/dependencytrack/resources/v1/IntegrationResource.java b/src/main/java/org/dependencytrack/resources/v1/IntegrationResource.java index cf6aa8a7b..f378b1609 100644 --- a/src/main/java/org/dependencytrack/resources/v1/IntegrationResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/IntegrationResource.java @@ -35,7 +35,7 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import org.dependencytrack.auth.Permissions; -import org.dependencytrack.tasks.OsvDownloadTask; +import org.dependencytrack.tasks.OsvMirrorTask; import java.util.List; import java.util.stream.Collectors; @@ -65,8 +65,8 @@ public class IntegrationResource extends AlpineResource { }) @PermissionRequired({Permissions.Constants.SYSTEM_CONFIGURATION, Permissions.Constants.SYSTEM_CONFIGURATION_READ}) public Response getAllEcosystems() { - OsvDownloadTask osvDownloadTask = new OsvDownloadTask(); - final List ecosystems = osvDownloadTask.getEcosystems(); + OsvMirrorTask osvMirrorTask = new OsvMirrorTask(); + final List ecosystems = osvMirrorTask.getEcosystems(); return Response.ok(ecosystems).build(); } @@ -87,9 +87,9 @@ public Response getAllEcosystems() { }) @PermissionRequired({Permissions.Constants.SYSTEM_CONFIGURATION, Permissions.Constants.SYSTEM_CONFIGURATION_READ}) public Response getInactiveEcosystems() { - OsvDownloadTask osvDownloadTask = new OsvDownloadTask(); - var selectedEcosystems = osvDownloadTask.getEnabledEcosystems(); - final List ecosystems = osvDownloadTask.getEcosystems().stream() + OsvMirrorTask osvMirrorTask = new OsvMirrorTask(); + var selectedEcosystems = osvMirrorTask.getEnabledEcosystems(); + final List ecosystems = osvMirrorTask.getEcosystems().stream() .filter(element -> !selectedEcosystems.contains(element)) .collect(Collectors.toList()); return Response.ok(ecosystems).build(); diff --git a/src/main/java/org/dependencytrack/tasks/BaseTaskScheduler.java b/src/main/java/org/dependencytrack/tasks/BaseTaskScheduler.java index aab5caa6f..99cffdb10 100644 --- a/src/main/java/org/dependencytrack/tasks/BaseTaskScheduler.java +++ b/src/main/java/org/dependencytrack/tasks/BaseTaskScheduler.java @@ -66,7 +66,7 @@ public void run() { long timeToExecuteTask = schedule.nextDuration(TimeUnit.MILLISECONDS); LOGGER.debug("Time in milliseconds to execute " + event + "is: " + timeToExecuteTask); if (timeToExecuteTask <= pollingIntervalInMillis) { - EventService.getInstance().publish(event); + Event.dispatch(event); } }); } diff --git a/src/main/java/org/dependencytrack/tasks/EpssMirrorTask.java b/src/main/java/org/dependencytrack/tasks/EpssMirrorTask.java index 36d83b686..1a176feac 100644 --- a/src/main/java/org/dependencytrack/tasks/EpssMirrorTask.java +++ b/src/main/java/org/dependencytrack/tasks/EpssMirrorTask.java @@ -21,7 +21,6 @@ import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.LoggableSubscriber; -import alpine.model.ConfigProperty; import org.dependencytrack.event.EpssMirrorEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; import org.dependencytrack.persistence.QueryManager; @@ -29,13 +28,13 @@ import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_EPSS_ENABLED; public class EpssMirrorTask implements LoggableSubscriber { + private static final Logger LOGGER = Logger.getLogger(EpssMirrorTask.class); private final boolean isEnabled; public EpssMirrorTask() { try (final QueryManager qm = new QueryManager()) { - final ConfigProperty enabled = qm.getConfigProperty(VULNERABILITY_SOURCE_EPSS_ENABLED.getGroupName(), VULNERABILITY_SOURCE_EPSS_ENABLED.getPropertyName()); - this.isEnabled = enabled != null && Boolean.valueOf(enabled.getPropertyValue()); + this.isEnabled = qm.isEnabled(VULNERABILITY_SOURCE_EPSS_ENABLED); } } diff --git a/src/main/java/org/dependencytrack/tasks/InternalComponentIdentificationTask.java b/src/main/java/org/dependencytrack/tasks/InternalComponentIdentificationTask.java index 9fab02850..618bdb68d 100644 --- a/src/main/java/org/dependencytrack/tasks/InternalComponentIdentificationTask.java +++ b/src/main/java/org/dependencytrack/tasks/InternalComponentIdentificationTask.java @@ -29,7 +29,6 @@ import org.dependencytrack.event.InternalComponentIdentificationEvent; import org.dependencytrack.model.Component; import org.dependencytrack.util.InternalComponentIdentifier; -import org.dependencytrack.util.LockProvider; import org.jdbi.v3.core.statement.PreparedBatch; import org.jdbi.v3.core.statement.SqlStatements; @@ -41,8 +40,9 @@ import static org.dependencytrack.persistence.jdbi.JdbiFactory.useJdbiTransaction; import static org.dependencytrack.persistence.jdbi.JdbiFactory.withJdbiHandle; -import static org.dependencytrack.tasks.LockName.INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK; -import static org.dependencytrack.util.LockProvider.isLockToBeExtended; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; /** * Subscriber task that identifies internal components throughout the entire portfolio. @@ -58,7 +58,9 @@ public class InternalComponentIdentificationTask implements Subscriber { public void inform(final Event e) { if (e instanceof InternalComponentIdentificationEvent) { try { - LockProvider.executeWithLock(INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK, (LockingTaskExecutor.Task) this::analyze); + executeWithLock( + getLockConfigForTask(InternalComponentIdentificationTask.class), + (LockingTaskExecutor.Task) this::analyze); } catch (Throwable ex) { LOGGER.error("Error in acquiring lock and executing internal component identification task", ex); } @@ -68,7 +70,7 @@ public void inform(final Event e) { private void analyze() { final Instant startTime = Instant.now(); LOGGER.info("Starting internal component identification"); - LockConfiguration lockConfiguration = LockProvider.getLockConfigurationByLockName(INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK); + LockConfiguration lockConfiguration = getLockConfigForTask(InternalComponentIdentificationTask.class); final var internalComponentIdentifier = new InternalComponentIdentifier(); if (!internalComponentIdentifier.hasPatterns() && !internalComponentsExist()) { @@ -87,7 +89,7 @@ private void analyze() { //It might finish execution before lock could be extended resulting in error LOGGER.debug("extending lock of internal component identification by 5 min"); long cumulativeProcessingDuration = System.currentTimeMillis() - startTime.toEpochMilli(); - if (isLockToBeExtended(cumulativeProcessingDuration, INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK)) { + if (isTaskLockToBeExtended(cumulativeProcessingDuration, InternalComponentIdentificationTask.class)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } diff --git a/src/main/java/org/dependencytrack/tasks/LdapSyncTaskWrapper.java b/src/main/java/org/dependencytrack/tasks/LdapSyncTaskWrapper.java index 7fbe09488..5da7bdfb7 100644 --- a/src/main/java/org/dependencytrack/tasks/LdapSyncTaskWrapper.java +++ b/src/main/java/org/dependencytrack/tasks/LdapSyncTaskWrapper.java @@ -22,9 +22,9 @@ import alpine.event.framework.Event; import alpine.event.framework.Subscriber; import alpine.server.tasks.LdapSyncTask; -import org.dependencytrack.util.LockProvider; -import static org.dependencytrack.tasks.LockName.LDAP_SYNC_TASK_LOCK; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; public class LdapSyncTaskWrapper implements Subscriber { @@ -41,7 +41,10 @@ public LdapSyncTaskWrapper() { @Override public void inform(Event e) { if (e instanceof LdapSyncEvent) { - LockProvider.executeWithLock(LDAP_SYNC_TASK_LOCK, (Runnable) () -> this.ldapSyncTask.inform(new LdapSyncEvent())); + executeWithLock( + getLockConfigForTask(LdapSyncTask.class), + (Runnable) () -> this.ldapSyncTask.inform(new LdapSyncEvent()) + ); } } } diff --git a/src/main/java/org/dependencytrack/tasks/OsvDownloadTask.java b/src/main/java/org/dependencytrack/tasks/OsvMirrorTask.java similarity index 96% rename from src/main/java/org/dependencytrack/tasks/OsvDownloadTask.java rename to src/main/java/org/dependencytrack/tasks/OsvMirrorTask.java index 8dec6964d..b52f74a0d 100644 --- a/src/main/java/org/dependencytrack/tasks/OsvDownloadTask.java +++ b/src/main/java/org/dependencytrack/tasks/OsvMirrorTask.java @@ -46,13 +46,13 @@ import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GOOGLE_OSV_BASE_URL; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED; -public class OsvDownloadTask implements LoggableSubscriber { +public class OsvMirrorTask implements LoggableSubscriber { - private static final Logger LOGGER = Logger.getLogger(OsvDownloadTask.class); + private static final Logger LOGGER = Logger.getLogger(OsvMirrorTask.class); private String osvBaseUrl; private Set ecosystems; - public OsvDownloadTask() { + public OsvMirrorTask() { try (final QueryManager qm = new QueryManager()) { final ConfigProperty enabled = qm.getConfigProperty(VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED.getGroupName(), VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED.getPropertyName()); if (enabled != null) { diff --git a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalysisTask.java similarity index 90% rename from src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java rename to src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalysisTask.java index 68e6f0399..dc0b07070 100644 --- a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java +++ b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalysisTask.java @@ -32,7 +32,6 @@ import org.dependencytrack.model.Project; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; -import org.dependencytrack.util.LockProvider; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -41,8 +40,9 @@ import java.util.List; import java.util.UUID; -import static org.dependencytrack.tasks.LockName.PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK; -import static org.dependencytrack.util.LockProvider.isLockToBeExtended; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; /** * A {@link Subscriber} to {@link ProjectRepositoryMetaAnalysisEvent} and {@link PortfolioRepositoryMetaAnalysisEvent} @@ -53,9 +53,9 @@ * components are submitted by distinct PURL coordinates. As such, there is no 1:1 correlation between total number * of components in the portfolio or project, and records submitted for analysis. */ -public class RepositoryMetaAnalyzerTask implements Subscriber { +public class RepositoryMetaAnalysisTask implements Subscriber { - private static final Logger LOGGER = Logger.getLogger(RepositoryMetaAnalyzerTask.class); + private static final Logger LOGGER = Logger.getLogger(RepositoryMetaAnalysisTask.class); private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); @@ -74,7 +74,9 @@ public void inform(final Event e) { } } else if (e instanceof PortfolioRepositoryMetaAnalysisEvent) { try { - LockProvider.executeWithLock(PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, (LockingTaskExecutor.Task) () -> processPortfolio()); + executeWithLock( + getLockConfigForTask(RepositoryMetaAnalysisTask.class), + (LockingTaskExecutor.Task) this::processPortfolio); } catch (Throwable ex) { LOGGER.error("An unexpected error occurred while submitting components for repository meta analysis", ex); } @@ -110,7 +112,7 @@ private void processProject(final UUID projectUuid) throws Exception { private void processPortfolio() throws Exception { LOGGER.info("Submitting all components in portfolio for repository meta analysis"); - LockConfiguration lockConfiguration = LockProvider.getLockConfigurationByLockName(PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK); + LockConfiguration lockConfiguration = getLockConfigForTask(RepositoryMetaAnalysisTask.class); try (final QueryManager qm = new QueryManager()) { final PersistenceManager pm = qm.getPersistenceManager(); @@ -120,7 +122,7 @@ private void processPortfolio() throws Exception { List components = fetchNextComponentsPage(pm, null, offset); while (!components.isEmpty()) { long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if (isLockToBeExtended(cumulativeProcessingTime, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK)) { + if (isTaskLockToBeExtended(cumulativeProcessingTime, RepositoryMetaAnalysisTask.class)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } //latest version information does not need to be fetched for project as triggered for portfolio means it is a scheduled event happening diff --git a/src/main/java/org/dependencytrack/tasks/TaskScheduler.java b/src/main/java/org/dependencytrack/tasks/TaskScheduler.java index 8a86d789c..91c7bf7a4 100644 --- a/src/main/java/org/dependencytrack/tasks/TaskScheduler.java +++ b/src/main/java/org/dependencytrack/tasks/TaskScheduler.java @@ -18,14 +18,10 @@ */ package org.dependencytrack.tasks; -import alpine.Config; -import alpine.common.util.BooleanUtil; import alpine.event.LdapSyncEvent; import alpine.event.framework.Event; -import alpine.model.ConfigProperty; -import com.asahaf.javacron.InvalidExpressionException; +import alpine.server.tasks.LdapSyncTask; import com.asahaf.javacron.Schedule; -import org.dependencytrack.common.ConfigKey; import org.dependencytrack.event.DefectDojoUploadEventAbstract; import org.dependencytrack.event.EpssMirrorEvent; import org.dependencytrack.event.FortifySscUploadEventAbstract; @@ -40,33 +36,33 @@ import org.dependencytrack.event.PortfolioVulnerabilityAnalysisEvent; import org.dependencytrack.event.VulnerabilityMetricsUpdateEvent; import org.dependencytrack.event.VulnerabilityPolicyFetchEvent; -import org.dependencytrack.event.VulnerabilityScanCleanupEvent; -import org.dependencytrack.event.WorkflowStateCleanupEvent; +import org.dependencytrack.event.maintenance.ComponentMetadataMaintenanceEvent; +import org.dependencytrack.event.maintenance.MetricsMaintenanceEvent; +import org.dependencytrack.event.maintenance.TagMaintenanceEvent; +import org.dependencytrack.event.maintenance.VulnerabilityDatabaseMaintenanceEvent; +import org.dependencytrack.event.maintenance.VulnerabilityScanMaintenanceEvent; +import org.dependencytrack.event.maintenance.WorkflowMaintenanceEvent; import org.dependencytrack.model.ConfigPropertyConstants; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.tasks.maintenance.ComponentMetadataMaintenanceTask; +import org.dependencytrack.tasks.maintenance.MetricsMaintenanceTask; +import org.dependencytrack.tasks.maintenance.TagMaintenanceTask; +import org.dependencytrack.tasks.maintenance.VulnerabilityDatabaseMaintenanceTask; +import org.dependencytrack.tasks.maintenance.VulnerabilityScanMaintenanceTask; +import org.dependencytrack.tasks.maintenance.WorkflowMaintenanceTask; +import org.dependencytrack.tasks.metrics.PortfolioMetricsUpdateTask; +import org.dependencytrack.tasks.metrics.VulnerabilityMetricsUpdateTask; +import org.dependencytrack.tasks.vulnerabilitypolicy.VulnerabilityPolicyFetchTask; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_COMPONENT_IDENTIFICATION_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_EPSS_MIRRORING_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_GITHUB_MIRRORING_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_LDAP_SYNC_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_NIST_MIRRORING_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_OSV_MIRRORING_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_PORTFOLIO_METRICS_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_REPO_META_ANALYSIS_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_VULNERABILITY_METRICS_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_VULNERABILITY_POLICY_BUNDLE_FETCH_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_VULN_ANALYSIS_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_VULN_SCAN_CLEANUP_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK; import static org.dependencytrack.model.ConfigPropertyConstants.DEFECTDOJO_ENABLED; import static org.dependencytrack.model.ConfigPropertyConstants.FORTIFY_SSC_ENABLED; import static org.dependencytrack.model.ConfigPropertyConstants.KENNA_ENABLED; +import static org.dependencytrack.util.TaskUtil.getCronScheduleForTask; /** * @author Steve Springett @@ -81,46 +77,45 @@ public final class TaskScheduler extends BaseTaskScheduler { * Private constructor. */ private TaskScheduler() { - final Config configInstance = Config.getInstance(); - try { - Map configurableTasksMap = new HashMap<>(); - Map eventScheduleMap = Map.ofEntries( - Map.entry(new VulnerabilityPolicyFetchEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULNERABILITY_POLICY_BUNDLE_FETCH_TASK))), - Map.entry(new LdapSyncEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_LDAP_SYNC_TASK))), - Map.entry(new NistMirrorEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_NIST_MIRRORING_TASK))), - Map.entry(new OsvMirrorEvent(null), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_OSV_MIRRORING_TASK))), - Map.entry(new GitHubAdvisoryMirrorEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_GITHUB_MIRRORING_TASK))), - Map.entry(new EpssMirrorEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_EPSS_MIRRORING_TASK))), - Map.entry(new PortfolioMetricsUpdateEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_PORTFOLIO_METRICS_TASK))), - Map.entry(new VulnerabilityMetricsUpdateEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULNERABILITY_METRICS_TASK))), - Map.entry(new InternalComponentIdentificationEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_COMPONENT_IDENTIFICATION_TASK))), - Map.entry(new PortfolioVulnerabilityAnalysisEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULN_ANALYSIS_TASK))), - Map.entry(new VulnerabilityScanCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULN_SCAN_CLEANUP_TASK))), - Map.entry(new PortfolioRepositoryMetaAnalysisEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_REPO_META_ANALYSIS_TASK))), - Map.entry(new WorkflowStateCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK))), - Map.entry(new IntegrityMetaInitializerEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK))) - ); + final Map eventScheduleMap = Map.ofEntries( + Map.entry(new VulnerabilityPolicyFetchEvent(), getCronScheduleForTask(VulnerabilityPolicyFetchTask.class)), + Map.entry(new LdapSyncEvent(), getCronScheduleForTask(LdapSyncTask.class)), + Map.entry(new NistMirrorEvent(), getCronScheduleForTask(NistMirrorTask.class)), + Map.entry(new OsvMirrorEvent(null), getCronScheduleForTask(OsvMirrorTask.class)), + Map.entry(new GitHubAdvisoryMirrorEvent(), getCronScheduleForTask(GitHubAdvisoryMirrorTask.class)), + Map.entry(new EpssMirrorEvent(), getCronScheduleForTask(EpssMirrorTask.class)), + Map.entry(new PortfolioMetricsUpdateEvent(), getCronScheduleForTask(PortfolioMetricsUpdateTask.class)), + Map.entry(new VulnerabilityMetricsUpdateEvent(), getCronScheduleForTask(VulnerabilityMetricsUpdateTask.class)), + Map.entry(new InternalComponentIdentificationEvent(), getCronScheduleForTask(InternalComponentIdentificationTask.class)), + Map.entry(new PortfolioVulnerabilityAnalysisEvent(), getCronScheduleForTask(VulnerabilityAnalysisTask.class)), + Map.entry(new PortfolioRepositoryMetaAnalysisEvent(), getCronScheduleForTask(RepositoryMetaAnalysisTask.class)), + Map.entry(new IntegrityMetaInitializerEvent(), getCronScheduleForTask(IntegrityMetaInitializerTask.class)), + Map.entry(new ComponentMetadataMaintenanceEvent(), getCronScheduleForTask(ComponentMetadataMaintenanceTask.class)), + Map.entry(new MetricsMaintenanceEvent(), getCronScheduleForTask(MetricsMaintenanceTask.class)), + Map.entry(new TagMaintenanceEvent(), getCronScheduleForTask(TagMaintenanceTask.class)), + Map.entry(new VulnerabilityDatabaseMaintenanceEvent(), getCronScheduleForTask(VulnerabilityDatabaseMaintenanceTask.class)), + Map.entry(new VulnerabilityScanMaintenanceEvent(), getCronScheduleForTask(VulnerabilityScanMaintenanceTask.class)), + Map.entry(new WorkflowMaintenanceEvent(), getCronScheduleForTask(WorkflowMaintenanceTask.class))); - if (isTaskEnabled(FORTIFY_SSC_ENABLED)) { - configurableTasksMap.put(new FortifySscUploadEventAbstract(), Schedule.create(configInstance.getProperty(ConfigKey.CRON_EXPRESSION_FOR_FORTIFY_SSC_SYNC))); - } - if (isTaskEnabled(DEFECTDOJO_ENABLED)) { - configurableTasksMap.put(new DefectDojoUploadEventAbstract(), Schedule.create(configInstance.getProperty(ConfigKey.CRON_EXPRESSION_FOR_DEFECT_DOJO_SYNC))); - } - if (isTaskEnabled(KENNA_ENABLED)) { - configurableTasksMap.put(new KennaSecurityUploadEventAbstract(), Schedule.create(configInstance.getProperty(ConfigKey.CRON_EXPRESSION_FOR_KENNA_SYNC))); - } - - Map mergedEventScheduleMap = Stream.concat(eventScheduleMap.entrySet().stream(), configurableTasksMap.entrySet().stream()) - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue)); + Map configurableTasksMap = new HashMap<>(); + if (isTaskEnabled(FORTIFY_SSC_ENABLED)) { + configurableTasksMap.put(new FortifySscUploadEventAbstract(), getCronScheduleForTask(FortifySscUploadTask.class)); + } + if (isTaskEnabled(DEFECTDOJO_ENABLED)) { + configurableTasksMap.put(new DefectDojoUploadEventAbstract(), getCronScheduleForTask(DefectDojoUploadTask.class)); + } + if (isTaskEnabled(KENNA_ENABLED)) { + configurableTasksMap.put(new KennaSecurityUploadEventAbstract(), getCronScheduleForTask(KennaSecurityUploadTask.class)); + } - scheduleTask(mergedEventScheduleMap); + final Map mergedEventScheduleMap = Stream.concat( + eventScheduleMap.entrySet().stream(), + configurableTasksMap.entrySet().stream()) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue)); - } catch (InvalidExpressionException invalidExpressionException) { - throw new RuntimeException("Cron expression cannot be parsed to schedule tasks", invalidExpressionException); - } + scheduleTask(mergedEventScheduleMap); } /** @@ -133,18 +128,8 @@ public static TaskScheduler getInstance() { } private boolean isTaskEnabled(final ConfigPropertyConstants enabledConstraint) { - try (QueryManager qm = new QueryManager()) { - final ConfigProperty enabledProperty = qm.getConfigProperty( - enabledConstraint.getGroupName(), enabledConstraint.getPropertyName()); - if (enabledProperty != null && enabledProperty.getPropertyValue() != null) { - final boolean isEnabled = BooleanUtil.valueOf(enabledProperty.getPropertyValue()); - if (!isEnabled) { - return false; - } - } else { - return false; - } - return true; + try (final var qm = new QueryManager()) { + return qm.isEnabled(enabledConstraint); } } } diff --git a/src/main/java/org/dependencytrack/tasks/VulnerabilityAnalysisTask.java b/src/main/java/org/dependencytrack/tasks/VulnerabilityAnalysisTask.java index e4505a150..c132eccbb 100644 --- a/src/main/java/org/dependencytrack/tasks/VulnerabilityAnalysisTask.java +++ b/src/main/java/org/dependencytrack/tasks/VulnerabilityAnalysisTask.java @@ -33,7 +33,6 @@ import org.dependencytrack.model.VulnerabilityAnalysisLevel; import org.dependencytrack.model.VulnerabilityScan; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.util.LockProvider; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -43,8 +42,9 @@ import java.util.UUID; import static java.lang.Math.toIntExact; -import static org.dependencytrack.tasks.LockName.PORTFOLIO_VULN_ANALYSIS_TASK_LOCK; -import static org.dependencytrack.util.LockProvider.isLockToBeExtended; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; /** * A {@link Subscriber} to {@link ProjectVulnerabilityAnalysisEvent} and {@link PortfolioVulnerabilityAnalysisEvent} @@ -72,10 +72,9 @@ public void inform(final Event e) { } } else if (e instanceof final PortfolioVulnerabilityAnalysisEvent event) { try { - LockProvider.executeWithLock( - PORTFOLIO_VULN_ANALYSIS_TASK_LOCK, - (LockingTaskExecutor.Task) () -> processPortfolio(event.getChainIdentifier()) - ); + executeWithLock( + getLockConfigForTask(VulnerabilityAnalysisTask.class), + (LockingTaskExecutor.Task) () -> processPortfolio(event.getChainIdentifier())); } catch (Throwable ex) { LOGGER.error("An unexpected error occurred while submitting components for vulnerability analysis", ex); } @@ -117,7 +116,7 @@ private void processProject(final UUID projectUuid, final UUID scanToken) throws private void processPortfolio(final UUID scanToken) throws Exception { LOGGER.info("Submitting all components in portfolio for vulnerability analysis"); - LockConfiguration lockConfiguration = LockProvider.getLockConfigurationByLockName(PORTFOLIO_VULN_ANALYSIS_TASK_LOCK); + LockConfiguration lockConfiguration = getLockConfigForTask(VulnerabilityAnalysisTask.class); long submittedComponents = 0; try (final QueryManager qm = new QueryManager()) { @@ -126,7 +125,7 @@ private void processPortfolio(final UUID scanToken) throws Exception { List components = fetchNextComponentsPage(pm, null, null); while (!components.isEmpty()) { long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if (isLockToBeExtended(cumulativeProcessingTime, PORTFOLIO_VULN_ANALYSIS_TASK_LOCK)) { + if (isTaskLockToBeExtended(cumulativeProcessingTime, VulnerabilityAnalysisTask.class)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } dispatchComponents(scanToken, components); diff --git a/src/main/java/org/dependencytrack/tasks/VulnerabilityScanCleanupTask.java b/src/main/java/org/dependencytrack/tasks/VulnerabilityScanCleanupTask.java deleted file mode 100644 index 8a5aa2adf..000000000 --- a/src/main/java/org/dependencytrack/tasks/VulnerabilityScanCleanupTask.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import alpine.common.logging.Logger; -import alpine.event.framework.Event; -import alpine.event.framework.Subscriber; -import org.dependencytrack.event.VulnerabilityScanCleanupEvent; -import org.dependencytrack.model.VulnerabilityScan; -import org.dependencytrack.persistence.QueryManager; - -import javax.jdo.Query; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Date; - -/** - * A {@link Subscriber} task that deletes {@link VulnerabilityScan}s from the database - * that have not been updated for over a day. - *

- * The purpose of this task is to prevent {@link VulnerabilityScan}s to increase in numbers indefinitely. - * - * @since 5.0.0 - */ -public class VulnerabilityScanCleanupTask implements Subscriber { - - private static final Logger LOGGER = Logger.getLogger(VulnerabilityScanCleanupTask.class); - - /** - * {@inheritDoc} - */ - @Override - public void inform(final Event event) { - if (event instanceof VulnerabilityScanCleanupEvent) { - final Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS); - LOGGER.info("Cleaning up vulnerability scans that received no update since %s".formatted(threshold)); - - try (final var qm = new QueryManager(); - final Query query = qm.getPersistenceManager().newQuery(""" - DELETE FROM org.dependencytrack.model.VulnerabilityScan - WHERE this.updatedAt < :threshold - """)) { - query.setParameters(Date.from(threshold)); - final long numDeleted = query.executeResultUnique(Long.class); - LOGGER.info("Deleted %d vulnerability scans".formatted(numDeleted)); - } catch (Exception e) { - LOGGER.error("Cleaning up vulnerability scans failed", e); - } - } - } - -} diff --git a/src/main/java/org/dependencytrack/tasks/WorkflowStateCleanupTask.java b/src/main/java/org/dependencytrack/tasks/WorkflowStateCleanupTask.java deleted file mode 100644 index 87eff5a87..000000000 --- a/src/main/java/org/dependencytrack/tasks/WorkflowStateCleanupTask.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import alpine.Config; -import alpine.common.logging.Logger; -import alpine.event.framework.Event; -import alpine.event.framework.Subscriber; -import org.apache.commons.collections4.ListUtils; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.WorkflowStateCleanupEvent; -import org.dependencytrack.model.WorkflowState; -import org.dependencytrack.model.WorkflowStatus; -import org.dependencytrack.persistence.QueryManager; - -import javax.jdo.Query; -import java.time.Duration; -import java.time.Instant; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import static org.apache.commons.lang3.time.DateFormatUtils.ISO_8601_EXTENDED_DATETIME_FORMAT; -import static org.dependencytrack.tasks.LockName.WORKFLOW_STEP_CLEANUP_TASK_LOCK; -import static org.dependencytrack.util.LockProvider.executeWithLock; - -public class WorkflowStateCleanupTask implements Subscriber { - - private static final Logger LOGGER = Logger.getLogger(WorkflowStateCleanupTask.class); - - private final Duration stepTimeoutDuration; - private final Duration retentionDuration; - - - @SuppressWarnings("unused") // Called by Alpine's event system - public WorkflowStateCleanupTask() { - this( - Duration.parse(Config.getInstance().getProperty(ConfigKey.WORKFLOW_STEP_TIMEOUT_DURATION)), - Duration.parse(Config.getInstance().getProperty(ConfigKey.WORKFLOW_RETENTION_DURATION)) - ); - } - - WorkflowStateCleanupTask(final Duration stepTimeoutDuration, final Duration retentionDuration) { - this.stepTimeoutDuration = stepTimeoutDuration; - this.retentionDuration = retentionDuration; - } - - /** - * {@inheritDoc} - */ - @Override - public void inform(final Event e) { - if (e instanceof WorkflowStateCleanupEvent) { - final Instant now = Instant.now(); - final Date timeoutCutoff = Date.from(now.minus(stepTimeoutDuration)); - final Date retentionCutoff = Date.from(now.minus(retentionDuration)); - - executeWithLock(WORKFLOW_STEP_CLEANUP_TASK_LOCK, (Runnable) () -> { - try (final var qm = new QueryManager()) { - transitionPendingStepsToTimedOut(qm, timeoutCutoff); - transitionTimedOutStepsToFailed(qm, timeoutCutoff); - deleteExpiredWorkflows(qm, retentionCutoff); - } - }); - } - } - - /** - * Transition steps to {@link WorkflowStatus#TIMED_OUT}, if: - * - their state is non-terminal, and - * - they have not been updated for the threshold time frame - *

- * Because {@link WorkflowStatus#TIMED_OUT} states can still eventually become {@link WorkflowStatus#COMPLETED} - * or {@link WorkflowStatus#FAILED}, child steps do not have to be cancelled. - *

- * TODO: Change this to bulk update query once https://github.com/datanucleus/datanucleus-rdbms/issues/474 - * is resolved. Fetching all records and updating them individually is SUPER inefficient. - * - * @param qm The {@link QueryManager} to use - * @param timeoutCutoff The timeout cutoff - */ - private static void transitionPendingStepsToTimedOut(final QueryManager qm, final Date timeoutCutoff) { - final Query timeoutQuery = qm.getPersistenceManager().newQuery(WorkflowState.class); - timeoutQuery.setFilter("status == :status && updatedAt < :cutoff"); - timeoutQuery.setNamedParameters(Map.of( - "status", WorkflowStatus.PENDING, - "cutoff", timeoutCutoff - )); - int stepsTimedOut = 0; - try { - for (final WorkflowState state : timeoutQuery.executeList()) { - qm.runInTransaction(() -> { - state.setStatus(WorkflowStatus.TIMED_OUT); - state.setUpdatedAt(new Date()); - }); - stepsTimedOut++; - } - } finally { - timeoutQuery.closeAll(); - } - - if (stepsTimedOut > 0) { - LOGGER.warn("Transitioned %d workflow steps to %s state".formatted(stepsTimedOut, WorkflowStatus.TIMED_OUT)); - } else { - LOGGER.info("No workflow steps to transition to %s state".formatted(WorkflowStatus.TIMED_OUT)); - } - } - - /** - * Transition states to {@link WorkflowStatus#FAILED}, if their current status is {@link WorkflowStatus#TIMED_OUT}, - * and they have not been updated for the threshold time frame. - * - * @param qm The {@link QueryManager} to use - * @param timeoutCutoff The timeout cutoff - */ - private static void transitionTimedOutStepsToFailed(final QueryManager qm, final Date timeoutCutoff) { - final Query failedQuery = qm.getPersistenceManager().newQuery(WorkflowState.class); - failedQuery.setFilter("status == :status && updatedAt < :cutoff"); - failedQuery.setNamedParameters(Map.of( - "status", WorkflowStatus.TIMED_OUT, - "cutoff", timeoutCutoff - )); - int stepsFailed = 0; - int stepsCancelled = 0; - try { - for (final WorkflowState state : failedQuery.executeList()) { - stepsCancelled += qm.callInTransaction(() -> { - final Date now = new Date(); - state.setStatus(WorkflowStatus.FAILED); - state.setFailureReason("Timed out"); - state.setUpdatedAt(now); - return qm.updateAllDescendantStatesOfParent(state, WorkflowStatus.CANCELLED, now); - }); - stepsFailed++; - } - } finally { - failedQuery.closeAll(); - } - - if (stepsFailed > 0) { - LOGGER.warn("Transitioned %d %s workflow steps to %s status, and cancelled %d follow-up steps" - .formatted(stepsFailed, WorkflowStatus.TIMED_OUT, WorkflowStatus.FAILED, stepsCancelled)); - } else { - LOGGER.info("No %s workflow steps to transition to %s status" - .formatted(WorkflowStatus.TIMED_OUT, WorkflowStatus.FAILED)); - } - } - - /** - * Delete all {@link WorkflowState}s grouped by the same {@code token}, given all of their - * steps are in a terminal state, and their last update timestamp falls below {@code retentionCutoff}. - * - * @param qm The {@link QueryManager} to use - * @param retentionCutoff The retention cutoff time - */ - private static void deleteExpiredWorkflows(final QueryManager qm, final Date retentionCutoff) { - // Find all workflow tokens for which the most recently updated step falls below the - // retention cutoff time. - // TODO: There's likely a better way to do this. What we really want is all tokens where - // all steps' statuses are terminal, and "max(updatedAt) < :cutoff" is true. In JDO, - // the HAVING clause can only contain aggregates, but not individual fields, and we - // cannot aggregate the status field. - final Query deletionCandidatesQuery = qm.getPersistenceManager().newQuery(Query.JDOQL, """ - SELECT token FROM org.dependencytrack.model.WorkflowState - GROUP BY token - HAVING max(updatedAt) < :cutoff - """); - deletionCandidatesQuery.setNamedParameters(Map.of( - "cutoff", retentionCutoff - )); - final List deletableWorkflows; - try { - deletableWorkflows = List.copyOf(deletionCandidatesQuery.executeResultList(UUID.class)); - } finally { - deletionCandidatesQuery.closeAll(); - } - - // Bulk delete workflows based on the previously collected tokens. - // Do so in batches of up to 100 tokens in order to keep the query size manageable. - // Workflows are only deleted when all of their steps are in a terminal status. - long stepsDeleted = 0; - final List> tokenBatches = ListUtils.partition(deletableWorkflows, 100); - for (final List tokenBatch : tokenBatches) { - final Query workflowDeleteQuery = qm.getPersistenceManager().newQuery(Query.JDOQL, """ - DELETE FROM org.dependencytrack.model.WorkflowState - WHERE :tokens.contains(token) && ( - SELECT FROM org.dependencytrack.model.WorkflowState w - WHERE w.token == this.token && :nonTerminalStatuses.contains(w.status) - ).isEmpty() - """); - try { - stepsDeleted += qm.callInTransaction(() -> (long) workflowDeleteQuery.executeWithMap(Map.of( - "tokens", tokenBatch, - "nonTerminalStatuses", Set.of(WorkflowStatus.PENDING, WorkflowStatus.TIMED_OUT) - ))); - } finally { - workflowDeleteQuery.closeAll(); - } - } - - if (stepsDeleted > 0) { - LOGGER.info("Deleted %d workflow steps falling below retention cutoff %s" - .formatted(stepsDeleted, ISO_8601_EXTENDED_DATETIME_FORMAT.format(retentionCutoff))); - } else { - LOGGER.info("No workflows to delete for retention cutoff %s" - .formatted(ISO_8601_EXTENDED_DATETIME_FORMAT.format(retentionCutoff))); - } - } - -} diff --git a/src/main/java/org/dependencytrack/tasks/maintenance/ComponentMetadataMaintenanceTask.java b/src/main/java/org/dependencytrack/tasks/maintenance/ComponentMetadataMaintenanceTask.java new file mode 100644 index 000000000..591e9ac9e --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/maintenance/ComponentMetadataMaintenanceTask.java @@ -0,0 +1,83 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.event.maintenance.ComponentMetadataMaintenanceEvent; +import org.dependencytrack.persistence.jdbi.ComponentMetaDao; +import org.jdbi.v3.core.Handle; + +import java.time.Duration; + +import static net.javacrumbs.shedlock.core.LockAssert.assertLocked; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.openJdbiHandle; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; + +/** + * @since 5.6.0 + */ +public class ComponentMetadataMaintenanceTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(ComponentMetadataMaintenanceTask.class); + + @Override + public void inform(final Event event) { + if (!(event instanceof ComponentMetadataMaintenanceEvent)) { + return; + } + + final long startTimeNs = System.nanoTime(); + try (final Handle jdbiHandle = openJdbiHandle()) { + LOGGER.info("Starting component metadata maintenance"); + final Statistics statistics = executeWithLock( + getLockConfigForTask(ComponentMetadataMaintenanceTask.class), + () -> informLocked(jdbiHandle)); + if (statistics == null) { + LOGGER.info("Task is locked by another instance; Skipping"); + return; + } + + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.info("Completed in %s: %s".formatted(taskDuration, statistics)); + } catch (Throwable e) { + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.error("Failed to complete after %s".formatted(taskDuration), e); + } + } + + private record Statistics( + int deletedIntegrityMetadata, + int deletedRepositoryMetadata) { + } + + private Statistics informLocked(final Handle jdbiHandle) { + assertLocked(); + + final var dao = jdbiHandle.attach(ComponentMetaDao.class); + + final int numDeletedIntegrityMeta = dao.deleteOrphanIntegrityMetaComponents(); + final int numDeletedRepoMeta = dao.deleteOrphanRepositoryMetaComponents(); + + return new Statistics(numDeletedIntegrityMeta, numDeletedRepoMeta); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/maintenance/MetricsMaintenanceTask.java b/src/main/java/org/dependencytrack/tasks/maintenance/MetricsMaintenanceTask.java new file mode 100644 index 000000000..6c1edce1c --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/maintenance/MetricsMaintenanceTask.java @@ -0,0 +1,92 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.event.maintenance.MetricsMaintenanceEvent; +import org.dependencytrack.persistence.jdbi.ConfigPropertyDao; +import org.dependencytrack.persistence.jdbi.MetricsDao; +import org.jdbi.v3.core.Handle; + +import java.time.Duration; + +import static net.javacrumbs.shedlock.core.LockAssert.assertLocked; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_METRICS_RETENTION_DAYS; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.openJdbiHandle; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; + +/** + * @since 5.6.0 + */ +public class MetricsMaintenanceTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(MetricsMaintenanceTask.class); + + @Override + public void inform(final Event event) { + if (!(event instanceof MetricsMaintenanceEvent)) { + return; + } + + final long startTimeNs = System.nanoTime(); + try (final Handle jdbiHandle = openJdbiHandle()) { + LOGGER.info("Starting metrics maintenance"); + final Statistics statistics = executeWithLock( + getLockConfigForTask(MetricsMaintenanceTask.class), + () -> informLocked(jdbiHandle)); + if (statistics == null) { + LOGGER.info("Task is locked by another instance; Skipping"); + return; + } + + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.info("Completed in %s: %s".formatted(taskDuration, statistics)); + } catch (Throwable e) { + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.error("Failed to complete after %s".formatted(taskDuration), e); + } + } + + private record Statistics( + Duration retentionDuration, + int deletedComponentMetrics, + int deletedProjectMetrics, + int deletedPortfolioMetrics) { + } + + private Statistics informLocked(final Handle jdbiHandle) { + assertLocked(); + + final var configPropertyDao = jdbiHandle.attach(ConfigPropertyDao.class); + final var metricsDao = jdbiHandle.attach(MetricsDao.class); + + final Integer retentionDays = configPropertyDao.getValue(MAINTENANCE_METRICS_RETENTION_DAYS, Integer.class); + final Duration retentionDuration = Duration.ofDays(retentionDays); + + final int numDeletedComponent = metricsDao.deleteComponentMetricsForRetentionDuration(retentionDuration); + final int numDeletedProject = metricsDao.deleteProjectMetricsForRetentionDuration(retentionDuration); + final int numDeletedPortfolio = metricsDao.deletePortfolioMetricsForRetentionDuration(retentionDuration); + + return new Statistics(retentionDuration, numDeletedComponent, numDeletedProject, numDeletedPortfolio); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/maintenance/TagMaintenanceTask.java b/src/main/java/org/dependencytrack/tasks/maintenance/TagMaintenanceTask.java new file mode 100644 index 000000000..096906eab --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/maintenance/TagMaintenanceTask.java @@ -0,0 +1,90 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.event.maintenance.TagMaintenanceEvent; +import org.dependencytrack.persistence.jdbi.ConfigPropertyDao; +import org.dependencytrack.persistence.jdbi.TagDao; +import org.jdbi.v3.core.Handle; + +import java.time.Duration; + +import static net.javacrumbs.shedlock.core.LockAssert.assertLocked; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_TAGS_DELETE_UNUSED; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.openJdbiHandle; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; + +/** + * @since 5.6.0 + */ +public class TagMaintenanceTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(TagMaintenanceTask.class); + + @Override + public void inform(final Event event) { + if (!(event instanceof TagMaintenanceEvent)) { + return; + } + + final long startTimeNs = System.nanoTime(); + try (final Handle jdbiHandle = openJdbiHandle()) { + LOGGER.info("Starting tag maintenance"); + final Statistics statistics = executeWithLock( + getLockConfigForTask(TagMaintenanceTask.class), + () -> informLocked(jdbiHandle)); + if (statistics == null) { + LOGGER.info("Task is locked by another instance; Skipping"); + return; + } + + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.info("Completed in %s: %s".formatted(taskDuration, statistics)); + } catch (Throwable e) { + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.error("Failed to complete after %s".formatted(taskDuration), e); + } + } + + private record Statistics(int deletedUnused) { + } + + private Statistics informLocked(final Handle jdbiHandle) { + assertLocked(); + + final var configPropertyDao = jdbiHandle.attach(ConfigPropertyDao.class); + final var tagDao = jdbiHandle.attach(TagDao.class); + + int numDeleted = 0; + if (configPropertyDao.getValue(MAINTENANCE_TAGS_DELETE_UNUSED, Boolean.class)) { + numDeleted = tagDao.deleteUnused(); + } else { + LOGGER.info("Not deleting unused tags because %s:%s is disabled".formatted( + MAINTENANCE_TAGS_DELETE_UNUSED.getGroupName(), + MAINTENANCE_TAGS_DELETE_UNUSED.getPropertyName())); + } + + return new Statistics(numDeleted); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/maintenance/VulnerabilityDatabaseMaintenanceTask.java b/src/main/java/org/dependencytrack/tasks/maintenance/VulnerabilityDatabaseMaintenanceTask.java new file mode 100644 index 000000000..6dcb6aa1a --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/maintenance/VulnerabilityDatabaseMaintenanceTask.java @@ -0,0 +1,79 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.event.maintenance.VulnerabilityDatabaseMaintenanceEvent; +import org.dependencytrack.persistence.jdbi.VulnerabilityDao; +import org.jdbi.v3.core.Handle; + +import java.time.Duration; + +import static net.javacrumbs.shedlock.core.LockAssert.assertLocked; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.openJdbiHandle; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; + +/** + * @since 5.6.0 + */ +public class VulnerabilityDatabaseMaintenanceTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(VulnerabilityDatabaseMaintenanceTask.class); + + @Override + public void inform(final Event event) { + if (!(event instanceof VulnerabilityDatabaseMaintenanceEvent)) { + return; + } + + final long startTimeNs = System.nanoTime(); + try (final Handle jdbiHandle = openJdbiHandle()) { + LOGGER.info("Starting vulnerability database maintenance"); + final Statistics statistics = executeWithLock( + getLockConfigForTask(VulnerabilityDatabaseMaintenanceTask.class), + () -> informLocked(jdbiHandle)); + if (statistics == null) { + LOGGER.info("Task is locked by another instance; Skipping"); + return; + } + + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.info("Completed in %s: %s".formatted(taskDuration, statistics)); + } catch (Throwable e) { + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.error("Failed to complete after %s".formatted(taskDuration), e); + } + } + + private record Statistics(int deletedVulnerableSoftwareOrphans) { + } + + private Statistics informLocked(final Handle jdbiHandle) { + assertLocked(); + + final var dao = jdbiHandle.attach(VulnerabilityDao.class); + final int numDeletedVs = dao.deleteOrphanVulnerableSoftware(); + + return new Statistics(numDeletedVs); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/maintenance/VulnerabilityScanMaintenanceTask.java b/src/main/java/org/dependencytrack/tasks/maintenance/VulnerabilityScanMaintenanceTask.java new file mode 100644 index 000000000..db1e91c2d --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/maintenance/VulnerabilityScanMaintenanceTask.java @@ -0,0 +1,86 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.event.maintenance.VulnerabilityScanMaintenanceEvent; +import org.dependencytrack.persistence.jdbi.ConfigPropertyDao; +import org.dependencytrack.persistence.jdbi.VulnerabilityScanDao; +import org.jdbi.v3.core.Handle; + +import java.time.Duration; + +import static net.javacrumbs.shedlock.core.LockAssert.assertLocked; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.openJdbiHandle; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; + +/** + * @since 5.6.0 + */ +public class VulnerabilityScanMaintenanceTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(VulnerabilityScanMaintenanceTask.class); + + @Override + public void inform(final Event event) { + if (!(event instanceof VulnerabilityScanMaintenanceEvent)) { + return; + } + + final long startTimeNs = System.nanoTime(); + try (final Handle jdbiHandle = openJdbiHandle()) { + LOGGER.info("Starting vulnerability scan maintenance"); + final Statistics statistics = executeWithLock( + getLockConfigForTask(VulnerabilityScanMaintenanceTask.class), + () -> informLocked(jdbiHandle)); + if (statistics == null) { + LOGGER.info("Task is locked by another instance; Skipping"); + return; + } + + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.info("Completed in %s: %s".formatted(taskDuration, statistics)); + } catch (Throwable e) { + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.error("Failed to complete after %s".formatted(taskDuration), e); + } + } + + private record Statistics(int deletedScans) { + } + + private Statistics informLocked(final Handle handle) { + assertLocked(); + + final var configPropertyDao = handle.attach(ConfigPropertyDao.class); + final var vulnScanDao = handle.attach(VulnerabilityScanDao.class); + + final Integer retentionHours = configPropertyDao.getValue(MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS, Integer.class); + final Duration retentionDuration = Duration.ofHours(retentionHours); + + final int scansDeleted = vulnScanDao.deleteAllForRetentionDuration(retentionDuration); + + return new Statistics(scansDeleted); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/maintenance/WorkflowMaintenanceTask.java b/src/main/java/org/dependencytrack/tasks/maintenance/WorkflowMaintenanceTask.java new file mode 100644 index 000000000..beba9336c --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/maintenance/WorkflowMaintenanceTask.java @@ -0,0 +1,132 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.event.maintenance.WorkflowMaintenanceEvent; +import org.dependencytrack.model.WorkflowStatus; +import org.dependencytrack.persistence.jdbi.ConfigPropertyDao; +import org.dependencytrack.persistence.jdbi.WorkflowDao; +import org.jdbi.v3.core.Handle; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static net.javacrumbs.shedlock.core.LockAssert.assertLocked; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_WORKFLOW_RETENTION_HOURS; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.openJdbiHandle; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; + +/** + * @since 5.6.0 + */ +public class WorkflowMaintenanceTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(WorkflowMaintenanceTask.class); + + @Override + public void inform(final Event event) { + if (!(event instanceof WorkflowMaintenanceEvent)) { + return; + } + + final long startTimeNs = System.nanoTime(); + try (final Handle jdbiHandle = openJdbiHandle()) { + LOGGER.info("Starting workflow maintenance"); + final Statistics statistics = executeWithLock( + getLockConfigForTask(WorkflowMaintenanceTask.class), + () -> informLocked(jdbiHandle)); + if (statistics == null) { + LOGGER.info("Task is locked by another instance; Skipping"); + return; + } + + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.info("Completed in %s: %s".formatted(taskDuration, statistics)); + } catch (Throwable e) { + final var taskDuration = Duration.ofNanos(System.nanoTime() - startTimeNs); + LOGGER.error("Failed to complete after %s".formatted(taskDuration), e); + } + } + + private record Statistics( + Duration retentionDuration, + Duration stepTimeoutDuration, + int stepsTimedOut, + int stepsFailed, + int stepsCancelled, + int workflowsDeleted) { + } + + private Statistics informLocked(final Handle jdbiHandle) { + assertLocked(); + + final var configPropertyDao = jdbiHandle.attach(ConfigPropertyDao.class); + final var workflowDao = jdbiHandle.attach(WorkflowDao.class); + + final Integer retentionHours = configPropertyDao.getValue(MAINTENANCE_WORKFLOW_RETENTION_HOURS, Integer.class); + final Duration retentionDuration = Duration.ofHours(retentionHours); + + final Integer stepTimeoutMinutes = configPropertyDao.getValue(MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES, Integer.class); + final Duration stepTimeoutDuration = Duration.ofMinutes(stepTimeoutMinutes); + + final int numStepsTimedOut = workflowDao.transitionAllPendingStepsToTimedOutForTimeout(stepTimeoutDuration); + if (numStepsTimedOut > 0) { + LOGGER.warn("Transitioned %d workflow step(s) from %s to %s for timeout %s" + .formatted(numStepsTimedOut, WorkflowStatus.PENDING, WorkflowStatus.TIMED_OUT, stepTimeoutDuration)); + } + + final var failedStepsResult = new Object() { + int numStepsFailed = 0; + int numStepsCancelled = 0; + }; + jdbiHandle.useTransaction(ignored -> { + final List failedStepIds = workflowDao.transitionAllTimedOutStepsToFailedForTimeout(stepTimeoutDuration); + if (failedStepIds.isEmpty()) { + return; + } + + failedStepsResult.numStepsFailed = failedStepIds.size(); + LOGGER.warn("Transitioned %d workflow step(s) from %s to %s for timeout %s" + .formatted(failedStepsResult.numStepsFailed, WorkflowStatus.TIMED_OUT, WorkflowStatus.FAILED, stepTimeoutDuration)); + + failedStepsResult.numStepsCancelled = Arrays.stream(workflowDao.cancelAllChildrenByParentStepIdAnyOf(failedStepIds)).sum(); + if (failedStepsResult.numStepsCancelled > 0) { + LOGGER.warn("Transitioned %d workflow step(s) to %s because their parent steps transitioned to %s" + .formatted(failedStepsResult.numStepsCancelled, WorkflowStatus.CANCELLED, WorkflowStatus.FAILED)); + } + }); + + final int numWorkflowsDeleted = workflowDao.deleteAllForRetention(retentionDuration); + + return new Statistics( + retentionDuration, + stepTimeoutDuration, + numStepsTimedOut, + failedStepsResult.numStepsFailed, + failedStepsResult.numStepsCancelled, + numWorkflowsDeleted); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java index 14cc54db1..7f8c364e0 100644 --- a/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java +++ b/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java @@ -32,7 +32,6 @@ import org.dependencytrack.metrics.Metrics; import org.dependencytrack.model.Project; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.util.LockProvider; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -42,8 +41,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.dependencytrack.tasks.LockName.PORTFOLIO_METRICS_TASK_LOCK; -import static org.dependencytrack.util.LockProvider.isLockToBeExtended; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; /** @@ -61,7 +61,9 @@ public class PortfolioMetricsUpdateTask implements Subscriber { public void inform(final Event e) { if (e instanceof final PortfolioMetricsUpdateEvent event) { try { - LockProvider.executeWithLock(PORTFOLIO_METRICS_TASK_LOCK, (LockingTaskExecutor.Task)() -> updateMetrics(event.isForceRefresh())); + executeWithLock( + getLockConfigForTask(PortfolioMetricsUpdateTask.class), + (LockingTaskExecutor.Task)() -> updateMetrics(event.isForceRefresh())); } catch (Throwable ex) { LOGGER.error("Error in acquiring lock and executing portfolio metrics task", ex); } @@ -89,7 +91,7 @@ private static void refreshProjectMetrics() throws Exception { final PersistenceManager pm = qm.getPersistenceManager(); LOGGER.debug("Fetching first " + BATCH_SIZE + " projects"); - LockConfiguration portfolioMetricsTaskConfig = LockProvider.getLockConfigurationByLockName(PORTFOLIO_METRICS_TASK_LOCK); + LockConfiguration portfolioMetricsTaskConfig = getLockConfigForTask(PortfolioMetricsUpdateTask.class); List activeProjects = fetchNextActiveProjectsPage(pm, null); long processStartTime = System.currentTimeMillis(); while (!activeProjects.isEmpty()) { @@ -133,7 +135,7 @@ private static void refreshProjectMetrics() throws Exception { //initial duration of portfolio metrics can be set to 20min. //No thread calculating metrics would be executing for more than 15min. //lock can only be extended if lock until is held for time after current db time - if(isLockToBeExtended(cumulativeDurationInMillis, PORTFOLIO_METRICS_TASK_LOCK)) { + if(isTaskLockToBeExtended(cumulativeDurationInMillis, PortfolioMetricsUpdateTask.class)) { Duration extendLockByDuration = Duration.ofMillis(processDurationInMillis).plus(portfolioMetricsTaskConfig.getLockAtLeastFor()); LOGGER.debug("Extending lock duration by ms: " + extendLockByDuration); LockExtender.extendActiveLock(extendLockByDuration, portfolioMetricsTaskConfig.getLockAtLeastFor()); diff --git a/src/main/java/org/dependencytrack/tasks/metrics/VulnerabilityMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/VulnerabilityMetricsUpdateTask.java index 462d464d6..7e1031b48 100644 --- a/src/main/java/org/dependencytrack/tasks/metrics/VulnerabilityMetricsUpdateTask.java +++ b/src/main/java/org/dependencytrack/tasks/metrics/VulnerabilityMetricsUpdateTask.java @@ -25,7 +25,6 @@ import org.dependencytrack.event.VulnerabilityMetricsUpdateEvent; import org.dependencytrack.model.VulnerabilityMetrics; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.util.LockProvider; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -38,8 +37,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.dependencytrack.tasks.LockName.VULNERABILITY_METRICS_TASK_LOCK; import static org.dependencytrack.tasks.metrics.VulnerabilityMetricsUpdateTask.VulnerabilityDateCounters.queryForMetrics; +import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; /** * A {@link Subscriber} task that updates vulnerability metrics. @@ -54,7 +54,9 @@ public class VulnerabilityMetricsUpdateTask implements Subscriber { public void inform(final Event e) { if (e instanceof VulnerabilityMetricsUpdateEvent) { try { - LockProvider.executeWithLock(VULNERABILITY_METRICS_TASK_LOCK, (LockingTaskExecutor.Task) () -> updateMetrics()); + executeWithLock( + getLockConfigForTask(VulnerabilityMetricsUpdateTask.class), + (LockingTaskExecutor.Task) this::updateMetrics); } catch (Throwable ex) { LOGGER.error("Error in acquiring lock and executing vulnerability metrics update task", ex); } diff --git a/src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/VulnerabilityPolicyFetchTask.java b/src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/VulnerabilityPolicyFetchTask.java index 97a3ebc30..5dce768b8 100644 --- a/src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/VulnerabilityPolicyFetchTask.java +++ b/src/main/java/org/dependencytrack/tasks/vulnerabilitypolicy/VulnerabilityPolicyFetchTask.java @@ -41,8 +41,8 @@ import static org.dependencytrack.event.VulnerabilityPolicyFetchEvent.CHAIN_IDENTIFIER; import static org.dependencytrack.model.WorkflowStatus.PENDING; import static org.dependencytrack.model.WorkflowStep.POLICY_BUNDLE_SYNC; -import static org.dependencytrack.tasks.LockName.VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK; import static org.dependencytrack.util.LockProvider.executeWithLock; +import static org.dependencytrack.util.TaskUtil.getLockConfigForTask; public class VulnerabilityPolicyFetchTask implements Subscriber { @@ -68,7 +68,9 @@ public void inform(Event event) { } try (final var qm = new QueryManager()) { - executeWithLock(VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK, (Task) () -> syncPolicyBundle(qm)); + executeWithLock( + getLockConfigForTask(VulnerabilityPolicyFetchTask.class), + (Task) () -> syncPolicyBundle(qm)); } catch (Throwable ex) { LOGGER.error("An error occurred while verifying changes in policy file or downloading policy file itself", ex); } diff --git a/src/main/java/org/dependencytrack/util/LockProvider.java b/src/main/java/org/dependencytrack/util/LockProvider.java index 9b401ec05..f50e5d3c4 100644 --- a/src/main/java/org/dependencytrack/util/LockProvider.java +++ b/src/main/java/org/dependencytrack/util/LockProvider.java @@ -19,9 +19,12 @@ package org.dependencytrack.util; import alpine.Config; +import alpine.event.framework.Subscriber; import net.javacrumbs.shedlock.core.DefaultLockingTaskExecutor; import net.javacrumbs.shedlock.core.LockConfiguration; import net.javacrumbs.shedlock.core.LockingTaskExecutor; +import net.javacrumbs.shedlock.core.LockingTaskExecutor.Task; +import net.javacrumbs.shedlock.core.LockingTaskExecutor.TaskWithResult; import net.javacrumbs.shedlock.provider.jdbc.JdbcLockProvider; import org.apache.commons.lang3.reflect.FieldUtils; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; @@ -29,70 +32,46 @@ import org.datanucleus.store.rdbms.ConnectionFactoryImpl; import org.datanucleus.store.rdbms.RDBMSStoreManager; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.tasks.LockName; import javax.jdo.PersistenceManager; import javax.sql.DataSource; -import java.time.Duration; -import java.time.Instant; - -import static org.dependencytrack.common.ConfigKey.INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_METRICS_VULNERABILITY_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_METRICS_VULNERABILITY_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_MIRROR_EPSS_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_MIRROR_EPSS_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR; -import static org.dependencytrack.tasks.LockName.EPSS_MIRROR_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_LOCK; -import static org.dependencytrack.tasks.LockName.INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.LDAP_SYNC_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.PORTFOLIO_METRICS_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.PORTFOLIO_VULN_ANALYSIS_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.VULNERABILITY_METRICS_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.WORKFLOW_STEP_CLEANUP_TASK_LOCK; public class LockProvider { private static JdbcLockProvider instance; - public static void executeWithLock(LockName lockName, Runnable task) { - LockConfiguration lockConfiguration = getLockConfigurationByLockName(lockName); - LockingTaskExecutor executor = getLockingTaskExecutorInstance(); - executor.executeWithLock(task, lockConfiguration); + /** + * @since 5.6.0 + */ + public static void executeWithLock(final LockConfiguration lockConfiguration, final Runnable runnable) { + final LockingTaskExecutor executor = getLockingTaskExecutorInstance(); + executor.executeWithLock(runnable, lockConfiguration); } - public static void executeWithLock(LockName lockName, LockingTaskExecutor.Task task) throws Throwable { - LockConfiguration lockConfiguration = getLockConfigurationByLockName(lockName); - LockingTaskExecutor executor = getLockingTaskExecutorInstance(); + /** + * @since 5.6.0 + */ + public static void executeWithLock(final LockConfiguration lockConfiguration, final Task task) throws Throwable { + final LockingTaskExecutor executor = getLockingTaskExecutorInstance(); executor.executeWithLock(task, lockConfiguration); } - public static void executeWithLockWaiting(final WaitingLockConfiguration lockConfiguration, - final LockingTaskExecutor.Task task) throws Throwable { + /** + * @since 5.6.0 + */ + public static T executeWithLock(final LockConfiguration lockConfiguration, final TaskWithResult task) throws Throwable { + final LockingTaskExecutor executor = getLockingTaskExecutorInstance(); + return executor.executeWithLock(task, lockConfiguration).getResult(); + } + + public static void executeWithLockWaiting(final WaitingLockConfiguration lockConfiguration, final Task task) throws Throwable { executeWithLockWaiting(lockConfiguration, () -> { task.call(); return null; }); } - public static T executeWithLockWaiting(final WaitingLockConfiguration lockConfiguration, - final LockingTaskExecutor.TaskWithResult task) throws Throwable { + public static T executeWithLockWaiting(final WaitingLockConfiguration lockConfiguration, final TaskWithResult task) throws Throwable { final JdbcLockProvider jdbcLockProvider = getJdbcLockProviderInstance(); final var waitingLockProvider = new WaitingLockProvider(jdbcLockProvider, lockConfiguration.getPollInterval(), lockConfiguration.getWaitTimeout()); @@ -100,6 +79,14 @@ public static T executeWithLockWaiting(final WaitingLockConfiguration lockCo return executor.executeWithLock(task, lockConfiguration).getResult(); } + /** + * @since 5.6.0 + */ + public static boolean isTaskLockToBeExtended(long cumulativeDurationInMillis, final Class taskClass) { + final LockConfiguration lockConfiguration = TaskUtil.getLockConfigForTask(taskClass); + return cumulativeDurationInMillis >= (lockConfiguration.getLockAtMostFor().minus(lockConfiguration.getLockAtLeastFor())).toMillis(); + } + private static JdbcLockProvider getJdbcLockProviderInstance() { if (instance == null || Config.isUnitTestsEnabled()) { try (final QueryManager qm = new QueryManager()) { @@ -122,7 +109,7 @@ private static DataSource getDataSource(final JDOPersistenceManagerFactory pmf) // DataNucleus doesn't provide access to the underlying DataSource // after the PMF has been created. We use reflection to still get access if (pmf.getNucleusContext().getStoreManager() instanceof final RDBMSStoreManager storeManager - && storeManager.getConnectionManager() instanceof final ConnectionManagerImpl connectionManager) { + && storeManager.getConnectionManager() instanceof final ConnectionManagerImpl connectionManager) { return getDataSourceUsingReflection(FieldUtils.readField(connectionManager, "primaryConnectionFactory", true)); } return null; @@ -136,54 +123,4 @@ private static DataSource getDataSourceUsingReflection(final Object connectionFa return null; } - public static LockConfiguration getLockConfigurationByLockName(LockName lockName) { - return switch (lockName) { - case PORTFOLIO_METRICS_TASK_LOCK -> new LockConfiguration(Instant.now(), - PORTFOLIO_METRICS_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_LOCK_AT_LEAST_FOR))); - case LDAP_SYNC_TASK_LOCK -> new LockConfiguration(Instant.now(), - LDAP_SYNC_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_LDAP_SYNC_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR))); - case EPSS_MIRROR_TASK_LOCK -> new LockConfiguration(Instant.now(), - EPSS_MIRROR_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_MIRROR_EPSS_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_MIRROR_EPSS_LOCK_AT_LEAST_FOR))); - case VULNERABILITY_METRICS_TASK_LOCK -> new LockConfiguration(Instant.now(), - VULNERABILITY_METRICS_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_METRICS_VULNERABILITY_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_METRICS_VULNERABILITY_LOCK_AT_LEAST_FOR))); - case INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK -> new LockConfiguration(Instant.now(), - INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR))); - case WORKFLOW_STEP_CLEANUP_TASK_LOCK -> new LockConfiguration(Instant.now(), - WORKFLOW_STEP_CLEANUP_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR))); - case PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK -> new LockConfiguration(Instant.now(), - PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_REPO_META_ANALYSIS_LOCK_AT_LEAST_FOR))); - case PORTFOLIO_VULN_ANALYSIS_TASK_LOCK -> new LockConfiguration(Instant.now(), - PORTFOLIO_VULN_ANALYSIS_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR))); - case INTEGRITY_META_INITIALIZER_LOCK -> new LockConfiguration(Instant.now(), - INTEGRITY_META_INITIALIZER_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR))); - case VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK -> new LockConfiguration(Instant.now(), - VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_LEAST_FOR))); - }; - - } - - public static boolean isLockToBeExtended(long cumulativeDurationInMillis, LockName lockName) { - LockConfiguration lockConfiguration = LockProvider.getLockConfigurationByLockName(lockName); - return cumulativeDurationInMillis >= (lockConfiguration.getLockAtMostFor().minus(lockConfiguration.getLockAtLeastFor())).toMillis() ? true : false; - } } diff --git a/src/main/java/org/dependencytrack/util/TaskUtil.java b/src/main/java/org/dependencytrack/util/TaskUtil.java new file mode 100644 index 000000000..a15da8d54 --- /dev/null +++ b/src/main/java/org/dependencytrack/util/TaskUtil.java @@ -0,0 +1,104 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.util; + +import alpine.Config; +import alpine.event.framework.Subscriber; +import com.asahaf.javacron.InvalidExpressionException; +import com.asahaf.javacron.Schedule; +import com.google.common.base.CaseFormat; +import net.javacrumbs.shedlock.core.LockConfiguration; + +import java.time.Duration; +import java.time.Instant; +import java.util.NoSuchElementException; + +import static java.util.Objects.requireNonNull; + +/** + * @since 5.6.0 + */ +public final class TaskUtil { + + private static final String PROPERTY_CRON = "cron"; + private static final String PROPERTY_LOCK_MAX_DURATION = "lock.max.duration"; + private static final String PROPERTY_LOCK_MIN_DURATION = "lock.min.duration"; + + private TaskUtil() { + } + + public static LockConfiguration getLockConfigForTask(final Class taskClass) { + final String taskName = getTaskConfigName(taskClass); + + final String maxLockDurationString = Config.getInstance().getProperty(new TaskConfigKey(taskName, PROPERTY_LOCK_MAX_DURATION)); + if (maxLockDurationString == null) { + throw new NoSuchElementException("No max lock duration configured for task %s".formatted(taskName)); + } + + final String minLockDurationString = Config.getInstance().getProperty(new TaskConfigKey(taskName, PROPERTY_LOCK_MIN_DURATION)); + if (minLockDurationString == null) { + throw new NoSuchElementException("No min lock duration configured for task %s".formatted(taskName)); + } + + return new LockConfiguration( + Instant.now(), + taskName, + Duration.parse(maxLockDurationString), + Duration.parse(minLockDurationString)); + } + + public static Schedule getCronScheduleForTask(final Class taskClass) { + final String taskName = getTaskConfigName(taskClass); + + final String cronExpression = Config.getInstance().getProperty(new TaskConfigKey(taskName, PROPERTY_CRON)); + if (cronExpression == null) { + throw new NoSuchElementException("No cron expression configured for task %s".formatted(taskName)); + } + + try { + return Schedule.create(cronExpression); + } catch (InvalidExpressionException e) { + throw new IllegalStateException("Failed to create schedule for task %s from cron expression %s".formatted(taskName, cronExpression), e); + } + } + + private static String getTaskConfigName(final Class taskClass) { + requireNonNull(taskClass); + + // e.g. SomeFancyTask -> some.fancy + return CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, taskClass.getSimpleName()) + .replaceAll("_", ".") + .replaceAll("\\.task$", ""); + } + + private record TaskConfigKey(String taskName, String property) implements Config.Key { + + @Override + public String getPropertyName() { + return "task.%s.%s".formatted(taskName, property); + } + + @Override + public Object getDefaultValue() { + return null; + } + + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f78092471..5d8943f91 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -939,240 +939,254 @@ task.scheduler.initial.delay=180000 # @required task.scheduler.polling.interval=60000 -# Specifies how long the lock should be kept in case the executing node dies. -# This is just a fallback, under normal circumstances the lock is released as soon the tasks finishes. -# Set lockAtMostFor to a value which is much longer than normal execution time. Default value is 15min -# Lock will be extended dynamically till task execution is finished +# Maximum duration in ISO 8601 format for which the portfolio metrics update task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. # # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.metrics.portfolio.lockAtMostForInMillis=900000 +task.portfolio.metrics.update.lock.max.duration=PT15M -# Specifies minimum amount of time for which the lock should be kept. -# Its main purpose is to prevent execution from multiple nodes in case of really short tasks and clock difference between the nodes. +# Minimum duration in ISO 8601 format for which the portfolio metrics update task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. # # @category: Task Scheduling -# @type: integer -# @required -task.metrics.portfolio.lockAtLeastForInMillis=90000 - -# @category: Task Scheduling -# @type: integer -# @required -task.metrics.vulnerability.lockAtMostForInMillis=900000 - -# @category: Task Scheduling -# @type: integer -# @required -task.metrics.vulnerability.lockAtLeastForInMillis=90000 - -# @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.mirror.epss.lockAtMostForInMillis=900000 +task.portfolio.metrics.update.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the vulnerability metrics update task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.mirror.epss.lockAtLeastForInMillis=90000 +task.vulnerability.metrics.update.lock.max.duration=PT15M +# Minimum duration in ISO 8601 format for which the vulnerability metrics update task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.componentIdentification.lockAtMostForInMillis=900000 +task.vulnerability.metrics.update.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the internal component identification task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.componentIdentification.lockAtLeastForInMillis=90000 +task.internal.component.identification.lock.max.duration=PT15M +# Minimum duration in ISO 8601 format for which the internal component identification task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.ldapSync.lockAtMostForInMillis=900000 +task.internal.component.identification.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the LDAP synchronization task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.ldapSync.lockAtLeastForInMillis=90000 +task.ldap.sync.lock.max.duration=PT15M +# Minimum duration in ISO 8601 format for which the LDAP synchronization task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.workflow.state.cleanup.lockAtMostForInMillis=900000 +task.ldap.sync.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the portfolio repository metadata analysis task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.workflow.state.cleanup.lockAtLeastForInMillis=900000 +task.repository.meta.analysis.lock.max.duration=PT15M +# Minimum duration in ISO 8601 format for which the portfolio repository metadata analysis task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.portfolio.repoMetaAnalysis.lockAtMostForInMillis=900000 +task.repository.meta.analysis.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the portfolio vulnerability analysis task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.portfolio.repoMetaAnalysis.lockAtLeastForInMillis=90000 +task.vulnerability.analysis.lock.max.duration=PT15M +# Minimum duration in ISO 8601 format for which the portfolio vulnerability analysis task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.portfolio.vulnAnalysis.lockAtMostForInMillis=900000 +task.vulnerability.analysis.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the integrity metadata initializer task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -task.portfolio.vulnAnalysis.lockAtLeastForInMillis=90000 +task.integrity.meta.initializer.lock.max.duration=PT15M +# Minimum duration in ISO 8601 format for which the integrity metadata initializer task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -integrityMetaInitializer.lockAtMostForInMillis=900000 +task.integrity.meta.initializer.lock.min.duration=PT90S +# Maximum duration in ISO 8601 format for which the vulnerability policy bundle fetch task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# # @category: Task Scheduling -# @type: integer +# @type: duration # @required -integrityMetaInitializer.lockAtLeastForInMillis=90000 +task.vulnerability.policy.fetch.lock.max.duration=PT5M -# Schedule task for 10th minute of every hour +# Minimum duration in ISO 8601 format for which the vulnerability policy bundle fetch task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. # # @category: Task Scheduling -# @type: cron +# @type: duration # @required -task.cron.metrics.portfolio=10 * * * * +task.vulnerability.policy.fetch.lock.min.duration=PT5S -# Schedule task for 40th minute of every hour +# Cron expression of the portfolio metrics update task. # # @category: Task Scheduling # @type: cron # @required -task.cron.metrics.vulnerability=40 * * * * +task.portfolio.metrics.update.cron=10 * * * * -# Schedule task every 24 hrs at 02:00 UTC +# Cron expression of the vulnerability metrics update task. # # @category: Task Scheduling # @type: cron # @required -task.cron.mirror.github=0 2 * * * +task.vulnerability.metrics.update.cron=40 * * * * -# Schedule task every 24 hrs at 03:00 UTC +# Cron expression of the vulnerability GitHub Advisories mirroring task. # # @category: Task Scheduling # @type: cron # @required -task.cron.mirror.osv=0 3 * * * +task.git.hub.advisory.mirror.cron=0 2 * * * -# Schedule task every 24 hrs at 04:00 UTC +# Cron expression of the OSV mirroring task. # # @category: Task Scheduling # @type: cron # @required -task.cron.mirror.nist=0 4 * * * +task.osv.mirror.cron=0 3 * * * -# Schedule task every 6 hrs at 25th min +# Cron expression of the NIST / NVD mirroring task. # # @category: Task Scheduling # @type: cron # @required -task.cron.componentIdentification=25 */6 * * * +task.nist.mirror.cron=0 4 * * * -# Schedule task every 6 hrs at 0th min +# Cron expression of the EPSS mirroring task. # # @category: Task Scheduling # @type: cron # @required -task.cron.ldapSync=0 */6 * * * +task.epss.mirror.cron=0 1 * * * -# Schedule task every 24 hrs at 01:00 UTC +# Cron expression of the internal component identification task. # # @category: Task Scheduling # @type: cron # @required -task.cron.repoMetaAnalysis=0 1 * * * +task.internal.component.identification.cron=25 */6 * * * -# Schedule task every 24hrs at 06:00 UTC +# Cron expression of the LDAP synchronization task. # # @category: Task Scheduling # @type: cron # @required -task.cron.vulnAnalysis=0 6 * * * +task.ldap.sync.cron=0 */6 * * * -# Schedule task at 8:05 UTC on Wednesday every week +# Cron expression of the portfolio repository metadata analysis task. # # @category: Task Scheduling # @type: cron # @required -task.cron.vulnScanCleanUp=5 8 * * 4 +task.repository.meta.analysis.cron=0 1 * * * -# Schedule task every 5 minutes +# Cron expression of the portfolio vulnerability analysis task. # # @category: Task Scheduling # @type: cron # @required -task.cron.vulnerability.policy.bundle.fetch=*/5 * * * * +task.vulnerability.analysis.cron=0 6 * * * -# Schedule task every 24 hrs at 02:00 UTC +# Cron expression of the vulnerability policy bundle fetch task. # # @category: Task Scheduling # @type: cron # @required -task.cron.fortify.ssc.sync=0 2 * * * +task.vulnerability.policy.fetch.cron=*/5 * * * * -# Schedule task every 24 hrs at 02:00 UTC +# Cron expression of the Fortify SSC upload task. # # @category: Task Scheduling # @type: cron # @required -task.cron.defectdojo.sync=0 2 * * * +task.fortify.ssc.upload.cron=0 2 * * * -# Schedule task every 24 hrs at 02:00 UTC +# Cron expression of the DefectDojo upload task. # # @category: Task Scheduling # @type: cron # @required -task.cron.kenna.sync=0 2 * * * +task.defect.dojo.upload.cron=0 2 * * * -# Schedule task every 15 minutes +# Cron expression of the Kenna Security upload task. # # @category: Task Scheduling # @type: cron # @required -task.cron.workflow.state.cleanup=*/15 * * * * +task.kenna.security.upload.cron=0 2 * * * -# Schedule task at 0 min past every 12th hr +# Cron expression of the integrity metadata initializer task. # # @category: Task Scheduling # @type: cron # @required -task.cron.integrityInitializer=0 */12 * * * - -# Defines the number of write operations to perform during BOM processing before changes are flushed to the database. -# Smaller values may lower memory usage of the API server, whereas higher values will improve performance as fewer -# network round-trips to the database are necessary. -# -# @category: General -# @type: integer -bom.upload.processing.trx.flush.threshold=10000 - -# Defines the duration for how long a workflow step is allowed to remain in PENDING state -# after being started. If this duration is exceeded, workflow steps will transition into the TIMED_OUT state. -# If they remain in TIMED_OUT for the same duration, they will transition to the FAILED state. -# The duration must be specified in ISO8601 notation (https://en.wikipedia.org/wiki/ISO_8601#Durations). -# -# @category: General -# @type: duration -workflow.step.timeout.duration=PT1H - -# Defines the duration for how long workflow data is being retained, after all steps transitioned into a non-terminal -# state (CANCELLED, COMPLETED, FAILED, NOT_APPLICABLE). -# The duration must be specified in ISO8601 notation (https://en.wikipedia.org/wiki/ISO_8601#Durations). -# -# @category: General -# @type: duration -workflow.retention.duration=P3D +task.integrity.meta.initializer.cron=0 */12 * * * # Delays the BOM_PROCESSED notification until the vulnerability analysis associated with a given BOM upload # is completed. The intention being that it is then "safe" to query the API for any identified vulnerabilities. @@ -1314,3 +1328,177 @@ dev.services.image.kafka=docker.redpanda.com/vectorized/redpanda:v24.2.4 # @category: Development # @type: string dev.services.image.postgres=postgres:16 + +# Cron expression of the component metadata maintenance task. +#

+# The task deletes orphaned records from the `INTEGRITY_META_COMPONENT` and +# `REPOSITORY_META_COMPONENT` tables. +# +# @category: Task Scheduling +# @type: cron +# @required +task.component.metadata.maintenance.cron=0 */12 * * * + +# Maximum duration in ISO 8601 format for which the component metadata maintenance task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# +# @category: Task Scheduling +# @type: duration +# @required +task.component.metadata.maintenance.lock.max.duration=PT15M + +# Minimum duration in ISO 8601 format for which the component metadata maintenance task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# +# @category: Task Scheduling +# @type: duration +# @required +task.component.metadata.maintenance.lock.min.duration=PT1M + +# Cron expression of the metrics maintenance task. +#

+# The task deletes records older than the configured metrics retention duration from the following tables: +#

    +#
  • DEPENDENCYMETRICS
    • +#
  • PROJECTMETRICS
    • +#
  • PORTFOLIOMETRICS
    • +#
+# +# @category: Task Scheduling +# @type: cron +# @required +task.metrics.maintenance.cron=0 */3 * * * + +# Maximum duration in ISO 8601 format for which the metrics maintenance task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# +# @category: Task Scheduling +# @type: duration +# @required +task.metrics.maintenance.lock.max.duration=PT15M + +# Minimum duration in ISO 8601 format for which the metrics maintenance task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# +# @category: Task Scheduling +# @type: duration +# @required +task.metrics.maintenance.lock.min.duration=PT1M + +# Cron expression of the tag maintenance task. +#

+# The task deletes orphaned tags that are not used anymore. +# +# @category: Task Scheduling +# @type: cron +# @required +task.tag.maintenance.cron=0 */12 * * * + +# Maximum duration in ISO 8601 format for which the tag maintenance task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# +# @category: Task Scheduling +# @type: duration +# @required +task.tag.maintenance.lock.max.duration=PT15M + +# Minimum duration in ISO 8601 format for which the tag maintenance task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# +# @category: Task Scheduling +# @type: duration +# @required +task.tag.maintenance.lock.min.duration=PT1M + +# Cron expression of the vulnerability database maintenance task. +#

+# The task deletes orphaned records from the `VULNERABLESOFTWARE` table. +# +# @category: Task Scheduling +# @type: cron +# @required +task.vulnerability.database.maintenance.cron=0 0 * * * + +# Maximum duration in ISO 8601 format for which the vulnerability database maintenance task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# +# @category: Task Scheduling +# @type: duration +# @required +task.vulnerability.database.maintenance.lock.max.duration=PT15M + +# Minimum duration in ISO 8601 format for which the vulnerability database maintenance task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# +# @category: Task Scheduling +# @type: duration +# @required +task.vulnerability.database.maintenance.lock.min.duration=PT1M + +# Cron expression of the vulnerability scan maintenance task. +#

+# The task deletes records older than the configured retention duration from the `VULNERABILITYSCAN` table. +# +# @category: Task Scheduling +# @type: cron +# @required +task.vulnerability.scan.maintenance.cron=0 * * * * + +# Maximum duration in ISO 8601 format for which the vulnerability database maintenance task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# +# @category: Task Scheduling +# @type: duration +# @required +task.vulnerability.scan.maintenance.lock.max.duration=PT15M + +# Minimum duration in ISO 8601 format for which the vulnerability database maintenance task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# +# @category: Task Scheduling +# @type: duration +# @required +task.vulnerability.scan.maintenance.lock.min.duration=PT1M + +# Cron expression of the workflow maintenance task. +#

+# The task: +#
    +#
  • Transitions workflow steps from PENDING to TIMED_OUT state
    • +#
  • Transitions workflow steps from TIMED_OUT to FAILED state
    • +#
  • Transitions children of FAILED steps to CANCELLED state
    • +#
  • Deletes finished workflows according to the configured retention duration
    • +#
+# +# @category: Task Scheduling +# @type: cron +# @required +task.workflow.maintenance.cron=*/15 * * * * + +# Maximum duration in ISO 8601 format for which the workflow maintenance task will hold a lock. +#

+# The duration should be long enough to cover the task's execution duration. +# +# @category: Task Scheduling +# @type: duration +# @required +task.workflow.maintenance.lock.max.duration=PT5M + +# Minimum duration in ISO 8601 format for which the workflow maintenance task will hold a lock. +#

+# The duration should be long enough to cover eventual clock skew across API server instances. +# +# @category: Task Scheduling +# @type: duration +# @required +task.workflow.maintenance.lock.min.duration=PT1M diff --git a/src/test/java/org/dependencytrack/PersistenceCapableTest.java b/src/test/java/org/dependencytrack/PersistenceCapableTest.java index e2e69aed2..c785aaecb 100644 --- a/src/test/java/org/dependencytrack/PersistenceCapableTest.java +++ b/src/test/java/org/dependencytrack/PersistenceCapableTest.java @@ -29,8 +29,6 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.testcontainers.containers.PostgreSQLContainer; import javax.jdo.JDOHelper; @@ -40,9 +38,6 @@ public abstract class PersistenceCapableTest { - @Rule - public EnvironmentVariables environmentVariables = new EnvironmentVariables(); - protected static PostgresTestContainer postgresContainer; protected MockProducer kafkaMockProducer; protected QueryManager qm; @@ -62,7 +57,6 @@ public void before() throws Exception { qm = new QueryManager(); - environmentVariables.set("TASK_PORTFOLIO_REPOMETAANALYSIS_LOCKATLEASTFORINMILLIS", "2000"); this.kafkaMockProducer = (MockProducer) KafkaProducerInitializer.getProducer(); } diff --git a/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java b/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalysisTaskTest.java similarity index 97% rename from src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java rename to src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalysisTaskTest.java index 6a5e2b64c..cd52caaac 100644 --- a/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java +++ b/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalysisTaskTest.java @@ -31,7 +31,7 @@ import static org.assertj.core.api.Assertions.assertThatNoException; import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; -public class RepositoryMetaAnalyzerTaskTest extends PersistenceCapableTest { +public class RepositoryMetaAnalysisTaskTest extends PersistenceCapableTest { @Test public void testPortfolioRepositoryMetaAnalysis() { @@ -85,7 +85,7 @@ public void testPortfolioRepositoryMetaAnalysis() { componentProjectE.setInternal(true); qm.persist(componentProjectE); - new RepositoryMetaAnalyzerTask().inform(new PortfolioRepositoryMetaAnalysisEvent()); + new RepositoryMetaAnalysisTask().inform(new PortfolioRepositoryMetaAnalysisEvent()); assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), // projectA @@ -156,7 +156,7 @@ public void testProjectRepositoryMetaAnalysis() { componentE.setInternal(true); qm.persist(componentE); - new RepositoryMetaAnalyzerTask().inform(new ProjectRepositoryMetaAnalysisEvent(project.getUuid())); + new RepositoryMetaAnalysisTask().inform(new ProjectRepositoryMetaAnalysisEvent(project.getUuid())); assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), @@ -192,7 +192,7 @@ public void testProjectRepositoryMetaAnalysisWithInactiveProject() { componentA.setVersion("1.0.1"); qm.persist(componentA); - new RepositoryMetaAnalyzerTask().inform(new ProjectRepositoryMetaAnalysisEvent(project.getUuid())); + new RepositoryMetaAnalysisTask().inform(new ProjectRepositoryMetaAnalysisEvent(project.getUuid())); assertThat(kafkaMockProducer.history()).satisfiesExactly( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()) @@ -203,7 +203,7 @@ record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_ @Test public void testProjectRepositoryMetaAnalysisWithNonExistentProject() { assertThatNoException() - .isThrownBy(() -> new RepositoryMetaAnalyzerTask().inform(new ProjectRepositoryMetaAnalysisEvent(UUID.randomUUID()))); + .isThrownBy(() -> new RepositoryMetaAnalysisTask().inform(new ProjectRepositoryMetaAnalysisEvent(UUID.randomUUID()))); } } \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/TaskSchedulerTest.java b/src/test/java/org/dependencytrack/tasks/TaskSchedulerTest.java index b568b98a6..bb4a5082f 100644 --- a/src/test/java/org/dependencytrack/tasks/TaskSchedulerTest.java +++ b/src/test/java/org/dependencytrack/tasks/TaskSchedulerTest.java @@ -19,10 +19,11 @@ package org.dependencytrack.tasks; import alpine.Config; +import alpine.event.framework.Event; import alpine.event.framework.EventService; +import alpine.event.framework.Subscriber; import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.VulnerabilityScanCleanupEvent; -import org.dependencytrack.model.VulnerabilityScan; +import org.dependencytrack.event.PortfolioMetricsUpdateEvent; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -31,61 +32,66 @@ import org.junit.Test; import org.junit.contrib.java.lang.system.EnvironmentVariables; -import java.sql.Date; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.UUID; +import java.time.Duration; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; public class TaskSchedulerTest extends PersistenceCapableTest { - TaskScheduler taskScheduler; @Rule public EnvironmentVariables environmentVariables = new EnvironmentVariables(); + private static final Queue EVENTS = new ConcurrentLinkedQueue<>(); + + public static class TestSubscriber implements Subscriber { + + @Override + public void inform(final Event event) { + EVENTS.add(event); + } + + } + @BeforeClass public static void setUpClass() { Config.enableUnitTests(); - EventService.getInstance().subscribe(VulnerabilityScanCleanupEvent.class, VulnerabilityScanCleanupTask.class); + + EventService.getInstance().subscribe(PortfolioMetricsUpdateEvent.class, TestSubscriber.class); } @AfterClass public static void tearDownClass() { - EventService.getInstance().unsubscribe(VulnerabilityScanCleanupTask.class); + EventService.getInstance().unsubscribe(TestSubscriber.class); } @Before public void before() throws Exception { - environmentVariables.set("TASK_CRON_VULNSCANCLEANUP", "* * * * * *"); + environmentVariables.set("TASK_PORTFOLIO_METRICS_UPDATE_CRON", "* * * * * *"); environmentVariables.set("TASK_SCHEDULER_INITIAL_DELAY", "5"); environmentVariables.set("TASK_SCHEDULER_POLLING_INTERVAL", "1000"); + super.before(); + + // Force initialization of TaskScheduler. + final var ignored = TaskScheduler.getInstance(); } @After public void after() { - environmentVariables.clear("TASK_CRON_VULNSCANCLEANUP", - "TASK_SCHEDULER_INITIAL_DELAY", - "TASK_SCHEDULER_POLLING_INTERVAL"); - taskScheduler.shutdown(); + TaskScheduler.getInstance().shutdown(); + EVENTS.clear(); + super.after(); } @Test public void test() throws Exception { - - taskScheduler = TaskScheduler.getInstance(); - final var scanA = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), UUID.randomUUID(), 5); - final var scanB = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), UUID.randomUUID(), 1); - qm.runInTransaction(() -> scanB.setUpdatedAt(Date.from(Instant.now().minus(25, ChronoUnit.HOURS)))); - final var scanC = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), UUID.randomUUID(), 3); - qm.runInTransaction(() -> scanC.setUpdatedAt(Date.from(Instant.now().minus(13, ChronoUnit.HOURS)))); - //Sleeping for 500ms after initial delay so event would be sent - Thread.sleep(1000); - - assertThat(qm.getVulnerabilityScan(scanA.getToken())).isNotNull(); - assertThat(qm.getVulnerabilityScan(scanB.getToken())).isNull(); - assertThat(qm.getVulnerabilityScan(scanC.getToken())).isNotNull(); + await("Event Dispatch") + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertThat(EVENTS).hasSize(1)); } -} + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/VulnerabilityScanCleanupTaskTest.java b/src/test/java/org/dependencytrack/tasks/VulnerabilityScanCleanupTaskTest.java deleted file mode 100644 index 6381e6e69..000000000 --- a/src/test/java/org/dependencytrack/tasks/VulnerabilityScanCleanupTaskTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.VulnerabilityScanCleanupEvent; -import org.dependencytrack.model.VulnerabilityScan; -import org.junit.Test; - -import java.sql.Date; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; - -public class VulnerabilityScanCleanupTaskTest extends PersistenceCapableTest { - - @Test - public void testInform() { - final var scanA = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), UUID.randomUUID(), 5); - final var scanB = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), UUID.randomUUID(), 1); - qm.runInTransaction(() -> scanB.setUpdatedAt(Date.from(Instant.now().minus(25, ChronoUnit.HOURS)))); - final var scanC = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), UUID.randomUUID(), 3); - qm.runInTransaction(() -> scanC.setUpdatedAt(Date.from(Instant.now().minus(13, ChronoUnit.HOURS)))); - - new VulnerabilityScanCleanupTask().inform(new VulnerabilityScanCleanupEvent()); - - assertThat(qm.getVulnerabilityScan(scanA.getToken())).isNotNull(); - assertThat(qm.getVulnerabilityScan(scanB.getToken())).isNull(); - assertThat(qm.getVulnerabilityScan(scanC.getToken())).isNotNull(); - } - -} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/maintenance/ComponentMetadataMaintenanceTaskTest.java b/src/test/java/org/dependencytrack/tasks/maintenance/ComponentMetadataMaintenanceTaskTest.java new file mode 100644 index 000000000..fd93a0cce --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/maintenance/ComponentMetadataMaintenanceTaskTest.java @@ -0,0 +1,89 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.maintenance.ComponentMetadataMaintenanceEvent; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.RepositoryMetaComponent; +import org.dependencytrack.model.RepositoryType; +import org.junit.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; + +public class ComponentMetadataMaintenanceTaskTest extends PersistenceCapableTest { + + @Test + public void test() { + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setVersion("1.0.0"); + component.setPurl("pkg:maven/com.acme/acme-lib@1.0.0"); + qm.persist(component); + + final Instant now = Instant.now(); + + final var integrityMetadata = new IntegrityMetaComponent(); + integrityMetadata.setPurl("pkg:maven/com.acme/acme-lib@1.0.0"); + integrityMetadata.setLastFetch(Date.from(now)); + qm.persist(integrityMetadata); + + final var orphanedIntegrityMetadata = new IntegrityMetaComponent(); + orphanedIntegrityMetadata.setPurl("pkg:maven/foo/bar@1.2.3"); + orphanedIntegrityMetadata.setLastFetch(Date.from(now)); + qm.persist(orphanedIntegrityMetadata); + + final var repoMetadata = new RepositoryMetaComponent(); + repoMetadata.setRepositoryType(RepositoryType.MAVEN); + repoMetadata.setNamespace("com.acme"); + repoMetadata.setName("acme-lib"); + repoMetadata.setLatestVersion("2.0.0"); + repoMetadata.setLastCheck(Date.from(now.minus(29, ChronoUnit.DAYS))); + qm.persist(repoMetadata); + + final var orphanedRepoMetadata = new RepositoryMetaComponent(); + orphanedRepoMetadata.setRepositoryType(RepositoryType.MAVEN); + orphanedRepoMetadata.setNamespace("foo"); + orphanedRepoMetadata.setName("bar"); + orphanedRepoMetadata.setLatestVersion("3.2.1"); + orphanedRepoMetadata.setLastCheck(Date.from(now.minus(31, ChronoUnit.DAYS))); + qm.persist(orphanedRepoMetadata); + + final var task = new ComponentMetadataMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new ComponentMetadataMaintenanceEvent())); + + assertThat(qm.getIntegrityMetaComponent("pkg:maven/com.acme/acme-lib@1.0.0")).isNotNull(); + assertThat(qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3")).isNull(); + assertThat(qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "com.acme", "acme-lib")).isNotNull(); + assertThat(qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar")).isNull(); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/maintenance/MetricsMaintenanceTaskTest.java b/src/test/java/org/dependencytrack/tasks/maintenance/MetricsMaintenanceTaskTest.java new file mode 100644 index 000000000..1d1aaca07 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/maintenance/MetricsMaintenanceTaskTest.java @@ -0,0 +1,192 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.maintenance.MetricsMaintenanceEvent; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.DependencyMetrics; +import org.dependencytrack.model.PortfolioMetrics; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.ProjectMetrics; +import org.junit.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.function.BiConsumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_METRICS_RETENTION_DAYS; + +public class MetricsMaintenanceTaskTest extends PersistenceCapableTest { + + @Test + public void test() { + qm.createConfigProperty( + MAINTENANCE_METRICS_RETENTION_DAYS.getGroupName(), + MAINTENANCE_METRICS_RETENTION_DAYS.getPropertyName(), + MAINTENANCE_METRICS_RETENTION_DAYS.getDefaultPropertyValue(), + MAINTENANCE_METRICS_RETENTION_DAYS.getPropertyType(), + MAINTENANCE_METRICS_RETENTION_DAYS.getDescription() + ); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + final BiConsumer createComponentMetricsForLastOccurrence = (lastOccurrence, vulns) -> { + final var metrics = new DependencyMetrics(); + metrics.setProject(project); + metrics.setComponent(component); + metrics.setVulnerabilities(vulns); + metrics.setFirstOccurrence(Date.from(lastOccurrence)); + metrics.setLastOccurrence(Date.from(lastOccurrence)); + qm.persist(metrics); + }; + + final BiConsumer createProjectMetricsForLastOccurrence = (lastOccurrence, vulns) -> { + final var metrics = new ProjectMetrics(); + metrics.setProject(project); + metrics.setVulnerabilities(vulns); + metrics.setFirstOccurrence(Date.from(lastOccurrence)); + metrics.setLastOccurrence(Date.from(lastOccurrence)); + qm.persist(metrics); + }; + + final BiConsumer createPortfolioMetricsForLastOccurrence = (lastOccurrence, vulns) -> { + final var metrics = new PortfolioMetrics(); + metrics.setVulnerabilities(vulns); + metrics.setFirstOccurrence(Date.from(lastOccurrence)); + metrics.setLastOccurrence(Date.from(lastOccurrence)); + qm.persist(metrics); + }; + + final Instant now = Instant.now(); + + createComponentMetricsForLastOccurrence.accept(now.minus(91, ChronoUnit.DAYS), 91); + createComponentMetricsForLastOccurrence.accept(now.minus(90, ChronoUnit.DAYS), 90); + createComponentMetricsForLastOccurrence.accept(now.minus(89, ChronoUnit.DAYS), 89); + + createProjectMetricsForLastOccurrence.accept(now.minus(91, ChronoUnit.DAYS), 91); + createProjectMetricsForLastOccurrence.accept(now.minus(90, ChronoUnit.DAYS), 90); + createProjectMetricsForLastOccurrence.accept(now.minus(89, ChronoUnit.DAYS), 89); + + createPortfolioMetricsForLastOccurrence.accept(now.minus(91, ChronoUnit.DAYS), 91); + createPortfolioMetricsForLastOccurrence.accept(now.minus(90, ChronoUnit.DAYS), 90); + createPortfolioMetricsForLastOccurrence.accept(now.minus(89, ChronoUnit.DAYS), 89); + + final var task = new MetricsMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new MetricsMaintenanceEvent())); + + assertThat(qm.getDependencyMetrics(component).getList(DependencyMetrics.class)).satisfiesExactly( + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(89)); + + assertThat(qm.getProjectMetrics(project).getList(ProjectMetrics.class)).satisfiesExactly( + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(89)); + + assertThat(qm.getPortfolioMetrics().getList(PortfolioMetrics.class)).satisfiesExactly( + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(89)); + } + + @Test + public void testWithInactiveProject() { + qm.createConfigProperty( + MAINTENANCE_METRICS_RETENTION_DAYS.getGroupName(), + MAINTENANCE_METRICS_RETENTION_DAYS.getPropertyName(), + MAINTENANCE_METRICS_RETENTION_DAYS.getDefaultPropertyValue(), + MAINTENANCE_METRICS_RETENTION_DAYS.getPropertyType(), + MAINTENANCE_METRICS_RETENTION_DAYS.getDescription() + ); + + final var project = new Project(); + project.setName("acme-app"); + project.setActive(false); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + final BiConsumer createComponentMetricsForLastOccurrence = (lastOccurrence, vulns) -> { + final var metrics = new DependencyMetrics(); + metrics.setProject(project); + metrics.setComponent(component); + metrics.setVulnerabilities(vulns); + metrics.setFirstOccurrence(Date.from(lastOccurrence)); + metrics.setLastOccurrence(Date.from(lastOccurrence)); + qm.persist(metrics); + }; + + final BiConsumer createProjectMetricsForLastOccurrence = (lastOccurrence, vulns) -> { + final var metrics = new ProjectMetrics(); + metrics.setProject(project); + metrics.setVulnerabilities(vulns); + metrics.setFirstOccurrence(Date.from(lastOccurrence)); + metrics.setLastOccurrence(Date.from(lastOccurrence)); + qm.persist(metrics); + }; + + final BiConsumer createPortfolioMetricsForLastOccurrence = (lastOccurrence, vulns) -> { + final var metrics = new PortfolioMetrics(); + metrics.setVulnerabilities(vulns); + metrics.setFirstOccurrence(Date.from(lastOccurrence)); + metrics.setLastOccurrence(Date.from(lastOccurrence)); + qm.persist(metrics); + }; + + final Instant now = Instant.now(); + + createComponentMetricsForLastOccurrence.accept(now.minus(91, ChronoUnit.DAYS), 91); + createComponentMetricsForLastOccurrence.accept(now.minus(90, ChronoUnit.DAYS), 90); + createComponentMetricsForLastOccurrence.accept(now.minus(89, ChronoUnit.DAYS), 89); + + createProjectMetricsForLastOccurrence.accept(now.minus(91, ChronoUnit.DAYS), 91); + createProjectMetricsForLastOccurrence.accept(now.minus(90, ChronoUnit.DAYS), 90); + createProjectMetricsForLastOccurrence.accept(now.minus(89, ChronoUnit.DAYS), 89); + + createPortfolioMetricsForLastOccurrence.accept(now.minus(91, ChronoUnit.DAYS), 91); + createPortfolioMetricsForLastOccurrence.accept(now.minus(90, ChronoUnit.DAYS), 90); + createPortfolioMetricsForLastOccurrence.accept(now.minus(89, ChronoUnit.DAYS), 89); + + final var task = new MetricsMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new MetricsMaintenanceEvent())); + + assertThat(qm.getDependencyMetrics(component).getList(DependencyMetrics.class)).satisfiesExactlyInAnyOrder( + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(89), + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(90), // Retained b/c project is inactive. + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(91)); // Retained b/c project is inactive. + + assertThat(qm.getProjectMetrics(project).getList(ProjectMetrics.class)).satisfiesExactlyInAnyOrder( + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(89), + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(90), // Retained b/c project is inactive. + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(91)); // Retained b/c project is inactive. + + assertThat(qm.getPortfolioMetrics().getList(PortfolioMetrics.class)).satisfiesExactlyInAnyOrder( + metrics -> assertThat(metrics.getVulnerabilities()).isEqualTo(89)); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/maintenance/TagMaintenanceTaskTest.java b/src/test/java/org/dependencytrack/tasks/maintenance/TagMaintenanceTaskTest.java new file mode 100644 index 000000000..708729be4 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/maintenance/TagMaintenanceTaskTest.java @@ -0,0 +1,94 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.maintenance.TagMaintenanceEvent; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Vulnerability; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_TAGS_DELETE_UNUSED; + +public class TagMaintenanceTaskTest extends PersistenceCapableTest { + + @Test + public void test() { + qm.createConfigProperty( + MAINTENANCE_TAGS_DELETE_UNUSED.getGroupName(), + MAINTENANCE_TAGS_DELETE_UNUSED.getPropertyName(), + "true", + MAINTENANCE_TAGS_DELETE_UNUSED.getPropertyType(), + MAINTENANCE_TAGS_DELETE_UNUSED.getDescription()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var policy = new Policy(); + policy.setName("foo-policy"); + policy.setOperator(Policy.Operator.ALL); + policy.setViolationState(Policy.ViolationState.INFO); + qm.persist(policy); + + final var vuln = new Vulnerability(); + vuln.setVulnId("CVE-123"); + vuln.setSource(Vulnerability.Source.NVD); + qm.persist(vuln); + + qm.bind(project, List.of(qm.createTag("tag-project"))); + qm.bind(policy, List.of(qm.createTag("tag-policy"))); + qm.bind(vuln, List.of(qm.createTag("tag-vuln"))); + qm.createTag("tag-orphaned"); + + final var task = new TagMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new TagMaintenanceEvent())); + + assertThat(qm.getTags()).satisfiesExactlyInAnyOrder( + tag -> assertThat(tag.name()).isEqualTo("tag-project"), + tag -> assertThat(tag.name()).isEqualTo("tag-policy"), + tag -> assertThat(tag.name()).isEqualTo("tag-vuln") + ); + } + + @Test + public void testWithDeleteUnusedDisabled() { + qm.createConfigProperty( + MAINTENANCE_TAGS_DELETE_UNUSED.getGroupName(), + MAINTENANCE_TAGS_DELETE_UNUSED.getPropertyName(), + "false", + MAINTENANCE_TAGS_DELETE_UNUSED.getPropertyType(), + MAINTENANCE_TAGS_DELETE_UNUSED.getDescription()); + + qm.createTag("tag-orphaned"); + + final var task = new TagMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new TagMaintenanceEvent())); + + assertThat(qm.getTags()).satisfiesExactly( + tag -> assertThat(tag.name()).isEqualTo("tag-orphaned") + ); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/maintenance/VulnerabilityDatabaseMaintenanceTaskTest.java b/src/test/java/org/dependencytrack/tasks/maintenance/VulnerabilityDatabaseMaintenanceTaskTest.java new file mode 100644 index 000000000..3b6955138 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/maintenance/VulnerabilityDatabaseMaintenanceTaskTest.java @@ -0,0 +1,58 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.maintenance.VulnerabilityDatabaseMaintenanceEvent; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerableSoftware; +import org.junit.Test; + +import javax.jdo.JDOObjectNotFoundException; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatNoException; + +public class VulnerabilityDatabaseMaintenanceTaskTest extends PersistenceCapableTest { + + @Test + public void test() { + final var vuln = new Vulnerability(); + vuln.setVulnId("CVE-123"); + vuln.setSource(Vulnerability.Source.NVD); + qm.persist(vuln); + + final var vs = new VulnerableSoftware(); + vs.setVulnerabilities(List.of(vuln)); + qm.persist(vs); + + final var vsOrphaned = new VulnerableSoftware(); + qm.persist(vsOrphaned); + + final var task = new VulnerabilityDatabaseMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new VulnerabilityDatabaseMaintenanceEvent())); + + assertThatNoException() + .isThrownBy(() -> qm.getObjectById(VulnerableSoftware.class, vs.getId())); + assertThatExceptionOfType(JDOObjectNotFoundException.class) + .isThrownBy(() -> qm.getObjectById(VulnerableSoftware.class, vsOrphaned.getId())); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/maintenance/VulnerabilityScanMaintenanceTaskTest.java b/src/test/java/org/dependencytrack/tasks/maintenance/VulnerabilityScanMaintenanceTaskTest.java new file mode 100644 index 000000000..15e079133 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/maintenance/VulnerabilityScanMaintenanceTaskTest.java @@ -0,0 +1,64 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks.maintenance; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.maintenance.VulnerabilityScanMaintenanceEvent; +import org.dependencytrack.model.VulnerabilityScan; +import org.junit.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS; + +public class VulnerabilityScanMaintenanceTaskTest extends PersistenceCapableTest { + + @Test + public void test() { + qm.createConfigProperty( + MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS.getGroupName(), + MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS.getPropertyName(), + MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS.getDefaultPropertyValue(), + MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS.getPropertyType(), + MAINTENANCE_VULNERABILITY_SCAN_RETENTION_HOURS.getDescription()); + + final var tokenA = UUID.randomUUID(); + final var tokenB = UUID.randomUUID(); + final var tokenC = UUID.randomUUID(); + + qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), tokenA, 5); + final var scanB = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), tokenB, 1); + qm.runInTransaction(() -> scanB.setUpdatedAt(Date.from(Instant.now().minus(25, ChronoUnit.HOURS)))); + final var scanC = qm.createVulnerabilityScan(VulnerabilityScan.TargetType.PROJECT, UUID.randomUUID(), tokenC, 3); + qm.runInTransaction(() -> scanC.setUpdatedAt(Date.from(Instant.now().minus(13, ChronoUnit.HOURS)))); + + final var task = new VulnerabilityScanMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new VulnerabilityScanMaintenanceEvent())); + + assertThat(qm.getVulnerabilityScan(tokenA)).isNotNull(); + assertThat(qm.getVulnerabilityScan(tokenB)).isNull(); + assertThat(qm.getVulnerabilityScan(tokenC)).isNotNull(); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/tasks/WorkflowStateCleanupTaskTest.java b/src/test/java/org/dependencytrack/tasks/maintenance/WorkflowMaintenanceTaskTest.java similarity index 69% rename from src/test/java/org/dependencytrack/tasks/WorkflowStateCleanupTaskTest.java rename to src/test/java/org/dependencytrack/tasks/maintenance/WorkflowMaintenanceTaskTest.java index c7b19d679..f004ed97d 100644 --- a/src/test/java/org/dependencytrack/tasks/WorkflowStateCleanupTaskTest.java +++ b/src/test/java/org/dependencytrack/tasks/maintenance/WorkflowMaintenanceTaskTest.java @@ -16,34 +16,49 @@ * SPDX-License-Identifier: Apache-2.0 * Copyright (c) OWASP Foundation. All Rights Reserved. */ -package org.dependencytrack.tasks; +package org.dependencytrack.tasks.maintenance; import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.WorkflowStateCleanupEvent; +import org.dependencytrack.event.maintenance.WorkflowMaintenanceEvent; import org.dependencytrack.model.WorkflowState; import org.dependencytrack.model.WorkflowStatus; import org.dependencytrack.model.WorkflowStep; import org.junit.Test; import javax.jdo.JDOObjectNotFoundException; -import java.sql.Date; -import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Date; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_WORKFLOW_RETENTION_HOURS; +import static org.dependencytrack.model.ConfigPropertyConstants.MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES; -public class WorkflowStateCleanupTaskTest extends PersistenceCapableTest { +public class WorkflowMaintenanceTaskTest extends PersistenceCapableTest { @Test - public void testTransitionToTimedOut() { - final Duration timeoutDuration = Duration.ofHours(6); - final Duration retentionDuration = Duration.ofHours(666); // Not relevant for this test. + public void testWithTransitionToTimedOut() { + qm.createConfigProperty( + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getGroupName(), + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getPropertyName(), + "666", // Not relevant for this test. + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getPropertyType(), + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getDescription() + ); + + qm.createConfigProperty( + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getGroupName(), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getPropertyName(), + "360", // 6 hours + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getPropertyType(), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getDescription() + ); + final Instant now = Instant.now(); - final Instant timeoutCutoff = now.minus(timeoutDuration); + final Instant timeoutCutoff = now.minus(6, ChronoUnit.HOURS); final var token = UUID.randomUUID(); final var parentState = new WorkflowState(); @@ -61,7 +76,8 @@ public void testTransitionToTimedOut() { childState.setUpdatedAt(Date.from(timeoutCutoff.plus(1, ChronoUnit.HOURS))); qm.persist(childState); - new WorkflowStateCleanupTask(timeoutDuration, retentionDuration).inform(new WorkflowStateCleanupEvent()); + final var task = new WorkflowMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new WorkflowMaintenanceEvent())); qm.getPersistenceManager().refreshAll(parentState, childState); assertThat(parentState.getStatus()).isEqualTo(WorkflowStatus.TIMED_OUT); @@ -71,11 +87,25 @@ public void testTransitionToTimedOut() { } @Test - public void testTransitionTimedOutToFailed() { - final Duration timeoutDuration = Duration.ofHours(6); - final Duration retentionDuration = Duration.ofHours(666); // Not relevant for this test. + public void testWithTransitionTimedOutToFailed() { + qm.createConfigProperty( + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getGroupName(), + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getPropertyName(), + "666", // Not relevant for this test. + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getPropertyType(), + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getDescription() + ); + + qm.createConfigProperty( + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getGroupName(), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getPropertyName(), + "360", // 6 hours + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getPropertyType(), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getDescription() + ); + final Instant now = Instant.now(); - final Instant timeoutCutoff = now.minus(timeoutDuration); + final Instant timeoutCutoff = now.minus(6, ChronoUnit.HOURS); final var token = UUID.randomUUID(); final var parentState = new WorkflowState(); @@ -93,7 +123,8 @@ public void testTransitionTimedOutToFailed() { childState.setUpdatedAt(Date.from(timeoutCutoff.plus(1, ChronoUnit.HOURS))); qm.persist(childState); - new WorkflowStateCleanupTask(timeoutDuration, retentionDuration).inform(new WorkflowStateCleanupEvent()); + final var task = new WorkflowMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new WorkflowMaintenanceEvent())); qm.getPersistenceManager().refreshAll(parentState, childState); assertThat(parentState.getStatus()).isEqualTo(WorkflowStatus.FAILED); @@ -105,11 +136,25 @@ public void testTransitionTimedOutToFailed() { } @Test - public void testDeleteExpiredWorkflows() { - final Duration timeoutDuration = Duration.ofHours(666); // Not relevant for this test. - final Duration retentionDuration = Duration.ofHours(6); + public void testWithDeleteExpired() { + qm.createConfigProperty( + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getGroupName(), + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getPropertyName(), + "6", + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getPropertyType(), + MAINTENANCE_WORKFLOW_RETENTION_HOURS.getDescription() + ); + + qm.createConfigProperty( + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getGroupName(), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getPropertyName(), + "39960", // 666 hours; Not relevant for this test. + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getPropertyType(), + MAINTENANCE_WORKFLOW_STEP_TIMEOUT_MINUTES.getDescription() + ); + final Instant now = Instant.now(); - final Instant retentionCutoff = now.minus(retentionDuration); + final Instant retentionCutoff = now.minus(6, ChronoUnit.HOURS); // Create a workflow where all steps are in a terminal state, // and they all fall below the retention cutoff time. @@ -166,7 +211,8 @@ public void testDeleteExpiredWorkflows() { childStateC.setUpdatedAt(Date.from(retentionCutoff.minus(1, ChronoUnit.HOURS))); qm.persist(childStateC); - new WorkflowStateCleanupTask(timeoutDuration, retentionDuration).inform(new WorkflowStateCleanupEvent()); + final var task = new WorkflowMaintenanceTask(); + assertThatNoException().isThrownBy(() -> task.inform(new WorkflowMaintenanceEvent())); // Workflow A must've been deleted, because all steps are in terminal status, and fall below the retention cutoff. assertThatExceptionOfType(JDOObjectNotFoundException.class).isThrownBy(() -> qm.getObjectById(WorkflowState.class, childStateA.getId())); diff --git a/src/test/java/org/dependencytrack/tasks/metrics/AbstractMetricsUpdateTaskTest.java b/src/test/java/org/dependencytrack/tasks/metrics/AbstractMetricsUpdateTaskTest.java index 8d81aa661..e71cd5395 100644 --- a/src/test/java/org/dependencytrack/tasks/metrics/AbstractMetricsUpdateTaskTest.java +++ b/src/test/java/org/dependencytrack/tasks/metrics/AbstractMetricsUpdateTaskTest.java @@ -23,8 +23,6 @@ import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyViolation; -import org.junit.After; -import org.junit.Before; import org.junit.Rule; import org.junit.contrib.java.lang.system.EnvironmentVariables; @@ -33,28 +31,15 @@ import static org.dependencytrack.model.ConfigPropertyConstants.CUSTOM_RISK_SCORE_CRITICAL; import static org.dependencytrack.model.ConfigPropertyConstants.CUSTOM_RISK_SCORE_HIGH; -import static org.dependencytrack.model.ConfigPropertyConstants.CUSTOM_RISK_SCORE_MEDIUM; import static org.dependencytrack.model.ConfigPropertyConstants.CUSTOM_RISK_SCORE_LOW; +import static org.dependencytrack.model.ConfigPropertyConstants.CUSTOM_RISK_SCORE_MEDIUM; import static org.dependencytrack.model.ConfigPropertyConstants.CUSTOM_RISK_SCORE_UNASSIGNED; abstract class AbstractMetricsUpdateTaskTest extends PersistenceCapableTest { @Rule - public EnvironmentVariables environmentVariables = new EnvironmentVariables(); - - @Before - public void before() throws Exception { - super.before(); - - environmentVariables.set("TASK_METRICS_PORTFOLIO_LOCKATLEASTFORINMILLIS", "2000"); - } - - @After - public void after() { - environmentVariables.clear("TASK_METRICS_PORTFOLIO_LOCKATLEASTFORINMILLIS"); - - super.after(); - } + public EnvironmentVariables environmentVariables = new EnvironmentVariables() + .set("TASK_PORTFOLIO_METRICS_UPDATE_LOCK_MIN_DURATION", "PT2S"); protected PolicyViolation createPolicyViolation(final Component component, final Policy.ViolationState violationState, final PolicyViolation.Type type) { final var policy = qm.createPolicy(UUID.randomUUID().toString(), Policy.Operator.ALL, violationState);