From b3629756871aca188572c77ef60156270c6bcf6d Mon Sep 17 00:00:00 2001 From: VithikaS <122881935+VithikaS@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:57:29 +0100 Subject: [PATCH] fix integrity meta initializer (#391) Signed-off-by: vithikashukla Co-authored-by: vithikashukla --- .../org/dependencytrack/common/ConfigKey.java | 5 +- .../event/EventSubsystemInitializer.java | 3 + .../event/IntegrityMetaInitializer.java | 99 -------------- .../event/IntegrityMetaInitializerEvent.java | 6 + .../dependencytrack/event/PurlMigrator.java | 48 +++++++ .../RepositoryMetaResultProcessor.java | 2 +- .../persistence/ComponentQueryManager.java | 6 +- .../IntegrityMetaQueryManager.java | 7 +- .../persistence/QueryManager.java | 6 +- .../tasks/IntegrityMetaInitializerTask.java | 58 +++++++++ .../org/dependencytrack/tasks/LockName.java | 2 +- .../dependencytrack/tasks/TaskScheduler.java | 5 +- .../dependencytrack/util/LockProvider.java | 14 +- src/main/resources/application.properties | 6 +- src/main/webapp/WEB-INF/web.xml | 2 +- .../event/IntegrityMetaInitializerTest.java | 121 ------------------ .../event/PurlMigratorTest.java | 71 ++++++++++ ...IntegrityMetaQueryManagerPostgresTest.java | 35 +++++ .../IntegrityMetaQueryManagerTest.java | 33 ----- .../IntegrityMetaInitializerTaskTest.java | 77 +++++++++++ 20 files changed, 329 insertions(+), 277 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java create mode 100644 src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java create mode 100644 src/main/java/org/dependencytrack/event/PurlMigrator.java create mode 100644 src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java delete mode 100644 src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java create mode 100644 src/test/java/org/dependencytrack/event/PurlMigratorTest.java create mode 100644 src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index 64bae1d9a..22f47c50f 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -49,6 +49,7 @@ public enum ConfigKey implements Config.Key { CRON_EXPRESSION_FOR_DEFECT_DOJO_SYNC("task.cron.defectdojo.sync", "0 2 * * *"), CRON_EXPRESSION_FOR_KENNA_SYNC("task.cron.kenna.sync", "0 2 * * *"), CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK("task.cron.workflow.state.cleanup", "*/15 * * * *"), + CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK("task.cron.integrityInitializer", "0 */12 * * *"), TASK_SCHEDULER_INITIAL_DELAY("task.scheduler.initial.delay", "180000"), TASK_SCHEDULER_POLLING_INTERVAL("task.scheduler.polling.interval", "60000"), TASK_PORTFOLIO_LOCK_AT_MOST_FOR("task.metrics.portfolio.lockAtMostForInMillis", "900000"), @@ -72,8 +73,8 @@ public enum ConfigKey implements Config.Key { WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"), TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"), CEL_POLICY_ENGINE_ENABLED("cel.policy.engine.enabled", "false"), - TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR("task.integrity.integrityMetaInitializer.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR("task.integrity.integrityMetaInitializer.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), + INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR("integrityMetaInitializer.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), + INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR("integrityMetaInitializer.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), INTEGRITY_INITIALIZER_ENABLED("integrity.initializer.enabled", "false"), INTEGRITY_CHECK_ENABLED("integrity.check.enabled", "false"); diff --git a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java index f853112a2..32d3200ea 100644 --- a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java +++ b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java @@ -30,6 +30,7 @@ import org.dependencytrack.tasks.EpssMirrorTask; import org.dependencytrack.tasks.FortifySscUploadTask; import org.dependencytrack.tasks.GitHubAdvisoryMirrorTask; +import org.dependencytrack.tasks.IntegrityMetaInitializerTask; import org.dependencytrack.tasks.InternalComponentIdentificationTask; import org.dependencytrack.tasks.KennaSecurityUploadTask; import org.dependencytrack.tasks.LdapSyncTaskWrapper; @@ -98,6 +99,7 @@ public void contextInitialized(final ServletContextEvent event) { EVENT_SERVICE.subscribe(ComponentPolicyEvaluationEvent.class, PolicyEvaluationTask.class); EVENT_SERVICE.subscribe(ProjectPolicyEvaluationEvent.class, PolicyEvaluationTask.class); EVENT_SERVICE.subscribe(WorkflowStateCleanupEvent.class, WorkflowStateCleanupTask.class); + EVENT_SERVICE.subscribe(IntegrityMetaInitializerEvent.class, IntegrityMetaInitializerTask.class); TaskScheduler.getInstance(); } @@ -132,6 +134,7 @@ public void contextDestroyed(final ServletContextEvent event) { EVENT_SERVICE.unsubscribe(EpssMirrorTask.class); EVENT_SERVICE.unsubscribe(PolicyEvaluationTask.class); EVENT_SERVICE.unsubscribe(WorkflowStateCleanupTask.class); + EVENT_SERVICE.unsubscribe(IntegrityMetaInitializerTask.class); EVENT_SERVICE.shutdown(DRAIN_TIMEOUT_DURATION); } } diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java deleted file mode 100644 index 73868c7f1..000000000 --- a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.dependencytrack.event; - -import alpine.Config; -import alpine.common.logging.Logger; -import net.javacrumbs.shedlock.core.LockConfiguration; -import net.javacrumbs.shedlock.core.LockExtender; -import net.javacrumbs.shedlock.core.LockingTaskExecutor; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.util.LockProvider; -import org.hyades.proto.repometaanalysis.v1.FetchMeta; - -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_TASK_LOCK; -import static org.dependencytrack.util.LockProvider.isLockToBeExtended; - -public class IntegrityMetaInitializer implements ServletContextListener { - - private static final Logger LOGGER = Logger.getLogger(IntegrityMetaInitializer.class); - private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - private final boolean integrityInitializerEnabled; - - public IntegrityMetaInitializer() { - this(Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_INITIALIZER_ENABLED)); - } - - IntegrityMetaInitializer(final boolean integrityInitializerEnabled) { - this.integrityInitializerEnabled = integrityInitializerEnabled; - } - - - @Override - public void contextInitialized(final ServletContextEvent event) { - if (integrityInitializerEnabled) { - try { - LockProvider.executeWithLock(INTEGRITY_META_INITIALIZER_TASK_LOCK, (LockingTaskExecutor.Task) () -> process()); - } catch (Throwable e) { - throw new RuntimeException("An unexpected error occurred while running Initializer for integrity meta", e); - } - } else { - LOGGER.info("Component integrity initializer is disabled."); - } - } - - private void process() { - LOGGER.info("Initializing integrity meta component sync"); - try (final var qm = new QueryManager()) { - if (qm.getIntegrityMetaComponentCount() == 0) { - // Sync purls from Component only if IntegrityMetaComponent is empty - qm.synchronizeIntegrityMetaComponent(); - } - // dispatch purls not processed yet - batchProcessPurls(qm); - } - } - - private void batchProcessPurls(QueryManager qm) { - LockConfiguration lockConfiguration = LockProvider.getLockConfigurationByLockName(INTEGRITY_META_INITIALIZER_TASK_LOCK); - long offset = 0; - long startTime = System.currentTimeMillis(); - List purls = qm.fetchNextPurlsPage(offset); - while (!purls.isEmpty()) { - long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if (isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) { - LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); - } - dispatchPurls(qm, purls); - updateIntegrityMetaForPurls(qm, purls); - offset += purls.size(); - purls = qm.fetchNextPurlsPage(offset); - } - } - - private void updateIntegrityMetaForPurls(QueryManager qm, List purls) { - List purlRecords = new ArrayList<>(); - for (var purl : purls) { - purlRecords.add(qm.getIntegrityMetaComponent(purl)); - } - qm.batchUpdateIntegrityMetaComponent(purlRecords); - } - - private void dispatchPurls(QueryManager qm, List purls) { - for (final var purl : purls) { - ComponentProjection componentProjection = qm.getComponentByPurl(purl); - //Initializer will not trigger Integrity Check on component so component uuid is not required - kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(null, purl, componentProjection.internal, FetchMeta.FETCH_META_INTEGRITY_DATA)); - } - } - - public record ComponentProjection(String purlCoordinates, Boolean internal) { - } -} diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java new file mode 100644 index 000000000..c2208c6cd --- /dev/null +++ b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java @@ -0,0 +1,6 @@ +package org.dependencytrack.event; + +import alpine.event.framework.Event; + +public class IntegrityMetaInitializerEvent implements Event { +} diff --git a/src/main/java/org/dependencytrack/event/PurlMigrator.java b/src/main/java/org/dependencytrack/event/PurlMigrator.java new file mode 100644 index 000000000..3d4213f82 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/PurlMigrator.java @@ -0,0 +1,48 @@ +package org.dependencytrack.event; + +import alpine.Config; +import alpine.common.logging.Logger; +import net.javacrumbs.shedlock.core.LockingTaskExecutor; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.util.LockProvider; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_LOCK; + +public class PurlMigrator implements ServletContextListener { + + private static final Logger LOGGER = Logger.getLogger(PurlMigrator.class); + private final boolean integrityInitializerEnabled; + + public PurlMigrator() { + this(Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_INITIALIZER_ENABLED)); + } + + PurlMigrator(final boolean integrityInitializerEnabled) { + this.integrityInitializerEnabled = integrityInitializerEnabled; + } + + + @Override + public void contextInitialized(final ServletContextEvent event) { + if (integrityInitializerEnabled) { + try { + LockProvider.executeWithLock(INTEGRITY_META_INITIALIZER_LOCK, (LockingTaskExecutor.Task) () -> process()); + } catch (Throwable e) { + throw new RuntimeException("An unexpected error occurred while running Initializer for integrity meta", e); + } + } else { + LOGGER.info("Component integrity initializer is disabled."); + } + } + + private void process() { + LOGGER.info("Initializing integrity meta component sync"); + try (final var qm = new QueryManager()) { + qm.synchronizeIntegrityMetaComponent(); + } + } +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index cf5947f61..df43e556b 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -146,7 +146,7 @@ private RepositoryMetaComponent createRepositoryMetaResult(Record incomingAnalysisResultRecord, QueryManager queryManager, PackageURL purl) { final AnalysisResult result = incomingAnalysisResultRecord.value(); IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(purl.toString()); - if (persistentIntegrityMetaComponent != null && persistentIntegrityMetaComponent.getStatus().equals(FetchStatus.PROCESSED)) { + if (persistentIntegrityMetaComponent != null && persistentIntegrityMetaComponent.getStatus() != null && persistentIntegrityMetaComponent.getStatus().equals(FetchStatus.PROCESSED)) { LOGGER.warn(""" Received hash information for %s that has already been processed; Discarding """.formatted(purl)); diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index cfb95aadf..e4e13382d 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -25,7 +25,6 @@ import alpine.resources.AlpineRequest; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import org.dependencytrack.event.IntegrityMetaInitializer; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; import org.dependencytrack.model.ConfigPropertyConstants; @@ -33,6 +32,7 @@ import org.dependencytrack.model.RepositoryMetaComponent; import org.dependencytrack.model.RepositoryType; import org.dependencytrack.resources.v1.vo.DependencyGraphResponse; +import org.dependencytrack.tasks.IntegrityMetaInitializerTask; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -240,14 +240,14 @@ public PaginatedResult getComponentByHash(String hash) { * @param purl the purl of the component to retrieve * @return associated ComponentProjection */ - public IntegrityMetaInitializer.ComponentProjection getComponentByPurl(String purl) { + public IntegrityMetaInitializerTask.ComponentProjection getComponentByPurl(String purl) { if (purl == null) { return null; } final Query query = pm.newQuery(Component.class, "purl == :purl"); query.setParameters(purl); query.setResult("DISTINCT purlCoordinates, internal"); - return query.executeResultUnique(IntegrityMetaInitializer.ComponentProjection.class); + return query.executeResultUnique(IntegrityMetaInitializerTask.ComponentProjection.class); } /** diff --git a/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java b/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java index 0555ee112..5d5be64dc 100644 --- a/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java @@ -110,6 +110,7 @@ public synchronized void synchronizeIntegrityMetaComponent() { SELECT DISTINCT "PURL" FROM "COMPONENT" WHERE "PURL" IS NOT NULL + ON CONFLICT DO NOTHING """; Connection connection = null; PreparedStatement preparedStatement = null; @@ -147,13 +148,13 @@ public long getIntegrityMetaComponentCount() { * * @return the list of purls */ - public List fetchNextPurlsPage(long offset) { + public List fetchNextPurlsPage(long offset) { try (final Query query = pm.newQuery(IntegrityMetaComponent.class, "status == null || (status == :inProgress && lastFetch < :latest)")) { query.setParameters(FetchStatus.IN_PROGRESS, Date.from(Instant.now().minus(1, ChronoUnit.HOURS))); query.setRange(offset, offset + 5000); - query.setResult("purl"); - return List.copyOf(query.executeResultList(String.class)); + query.setResult("id, purl"); + return List.copyOf(query.executeResultList(IntegrityMetaComponent.class)); } catch (Exception e) { LOGGER.error("Error in getting purls from integrity meta.", e); throw new RuntimeException(e); diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 3c1d22a3e..98448fa9f 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -34,7 +34,6 @@ import org.apache.commons.lang3.ClassUtils; import org.datanucleus.PropertyNames; import org.datanucleus.api.jdo.JDOQuery; -import org.dependencytrack.event.IntegrityMetaInitializer; import org.dependencytrack.model.AffectedVersionAttribution; import org.dependencytrack.model.Analysis; import org.dependencytrack.model.AnalysisComment; @@ -87,6 +86,7 @@ import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.notification.publisher.PublisherClass; import org.dependencytrack.resources.v1.vo.DependencyGraphResponse; +import org.dependencytrack.tasks.IntegrityMetaInitializerTask; import org.hyades.proto.vulnanalysis.v1.ScanResult; import org.hyades.proto.vulnanalysis.v1.ScanStatus; import org.hyades.proto.vulnanalysis.v1.ScannerResult; @@ -597,7 +597,7 @@ public PaginatedResult getComponentByHash(String hash) { return getComponentQueryManager().getComponentByHash(hash); } - public IntegrityMetaInitializer.ComponentProjection getComponentByPurl(String purl) { + public IntegrityMetaInitializerTask.ComponentProjection getComponentByPurl(String purl) { return getComponentQueryManager().getComponentByPurl(purl); } @@ -1823,7 +1823,7 @@ public long getIntegrityMetaComponentCount() { return getIntegrityMetaQueryManager().getIntegrityMetaComponentCount(); } - public List fetchNextPurlsPage(long offset) { + public List fetchNextPurlsPage(long offset) { return getIntegrityMetaQueryManager().fetchNextPurlsPage(offset); } diff --git a/src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java b/src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java new file mode 100644 index 000000000..faee278ac --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java @@ -0,0 +1,58 @@ +package org.dependencytrack.tasks; + +import alpine.Config; +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.Subscriber; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; +import org.dependencytrack.event.IntegrityMetaInitializerEvent; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; + +import java.util.List; + +import static org.hyades.proto.repometaanalysis.v1.FetchMeta.FETCH_META_INTEGRITY_DATA; + +public class IntegrityMetaInitializerTask implements Subscriber { + + private static final Logger LOGGER = Logger.getLogger(IntegrityMetaInitializerTask.class); + + private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + + public void inform(final Event e) { + if (e instanceof IntegrityMetaInitializerEvent) { + if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_INITIALIZER_ENABLED)) { + LOGGER.debug("Integrity initializer is disabled"); + return; + } + try (final var qm = new QueryManager()) { + batchProcessPurls(qm); + } + } + } + + private void batchProcessPurls(QueryManager qm) { + long offset = 0; + List integrityMetaPurls = qm.fetchNextPurlsPage(offset); + while (!integrityMetaPurls.isEmpty()) { + dispatchPurls(qm, integrityMetaPurls); + qm.batchUpdateIntegrityMetaComponent(integrityMetaPurls); + offset += integrityMetaPurls.size(); + integrityMetaPurls = qm.fetchNextPurlsPage(offset); + } + } + + private void dispatchPurls(QueryManager qm, List integrityMetaPurls) { + for (final var integrityMetaPurl : integrityMetaPurls) { + IntegrityMetaInitializerTask.ComponentProjection componentProjection = qm.getComponentByPurl(integrityMetaPurl.getPurl()); + LOGGER.debug("Dispatching purl for integrity metadata: " + integrityMetaPurl.getPurl()); + //Initializer will not trigger Integrity Check on component so component uuid is not required + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(null, integrityMetaPurl.getPurl(), componentProjection.internal(), FETCH_META_INTEGRITY_DATA)); + } + } + + public record ComponentProjection(String purlCoordinates, Boolean internal) { + } +} diff --git a/src/main/java/org/dependencytrack/tasks/LockName.java b/src/main/java/org/dependencytrack/tasks/LockName.java index 9b25cee2c..5d1d1a169 100644 --- a/src/main/java/org/dependencytrack/tasks/LockName.java +++ b/src/main/java/org/dependencytrack/tasks/LockName.java @@ -9,5 +9,5 @@ public enum LockName { WORKFLOW_STEP_CLEANUP_TASK_LOCK, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, PORTFOLIO_VULN_ANALYSIS_TASK_LOCK, - INTEGRITY_META_INITIALIZER_TASK_LOCK + INTEGRITY_META_INITIALIZER_LOCK } diff --git a/src/main/java/org/dependencytrack/tasks/TaskScheduler.java b/src/main/java/org/dependencytrack/tasks/TaskScheduler.java index 0c8aa16cb..9b0a7a7e1 100644 --- a/src/main/java/org/dependencytrack/tasks/TaskScheduler.java +++ b/src/main/java/org/dependencytrack/tasks/TaskScheduler.java @@ -30,6 +30,7 @@ import org.dependencytrack.event.DefectDojoUploadEventAbstract; import org.dependencytrack.event.FortifySscUploadEventAbstract; import org.dependencytrack.event.GitHubAdvisoryMirrorEvent; +import org.dependencytrack.event.IntegrityMetaInitializerEvent; import org.dependencytrack.event.InternalComponentIdentificationEvent; import org.dependencytrack.event.KennaSecurityUploadEventAbstract; import org.dependencytrack.event.NistMirrorEvent; @@ -51,6 +52,7 @@ import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_COMPONENT_IDENTIFICATION_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_GITHUB_MIRRORING_TASK; +import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_LDAP_SYNC_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_NIST_MIRRORING_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_OSV_MIRRORING_TASK; @@ -94,7 +96,8 @@ private TaskScheduler() { Map.entry(new PortfolioVulnerabilityAnalysisEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULN_ANALYSIS_TASK))), Map.entry(new VulnerabilityScanCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULN_SCAN_CLEANUP_TASK))), Map.entry(new PortfolioRepositoryMetaAnalysisEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_REPO_META_ANALYSIS_TASK))), - Map.entry(new WorkflowStateCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK))) + Map.entry(new WorkflowStateCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK))), + Map.entry(new IntegrityMetaInitializerEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK))) ); if (isTaskEnabled(FORTIFY_SSC_ENABLED)) { diff --git a/src/main/java/org/dependencytrack/util/LockProvider.java b/src/main/java/org/dependencytrack/util/LockProvider.java index 86cbd2d4e..8a5a01c13 100644 --- a/src/main/java/org/dependencytrack/util/LockProvider.java +++ b/src/main/java/org/dependencytrack/util/LockProvider.java @@ -20,8 +20,8 @@ import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR; +import static org.dependencytrack.common.ConfigKey.INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR; +import static org.dependencytrack.common.ConfigKey.INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_MOST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_METRICS_VULNERABILITY_LOCK_AT_LEAST_FOR; @@ -37,7 +37,7 @@ import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR; import static org.dependencytrack.tasks.LockName.EPSS_MIRROR_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_TASK_LOCK; +import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_LOCK; import static org.dependencytrack.tasks.LockName.INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK; import static org.dependencytrack.tasks.LockName.LDAP_SYNC_TASK_LOCK; import static org.dependencytrack.tasks.LockName.PORTFOLIO_METRICS_TASK_LOCK; @@ -137,10 +137,10 @@ public static LockConfiguration getLockConfigurationByLockName(LockName lockName PORTFOLIO_VULN_ANALYSIS_TASK_LOCK.name(), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR)), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR))); - case INTEGRITY_META_INITIALIZER_TASK_LOCK -> new LockConfiguration(Instant.now(), - INTEGRITY_META_INITIALIZER_TASK_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR))); + case INTEGRITY_META_INITIALIZER_LOCK -> new LockConfiguration(Instant.now(), + INTEGRITY_META_INITIALIZER_LOCK.name(), + Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR)), + Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR))); }; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1d8469eb8..2cccb1b1e 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -456,8 +456,8 @@ task.portfolio.repoMetaAnalysis.lockAtMostForInMillis=900000 task.portfolio.repoMetaAnalysis.lockAtLeastForInMillis=90000 task.portfolio.vulnAnalysis.lockAtMostForInMillis=900000 task.portfolio.vulnAnalysis.lockAtLeastForInMillis=90000 -task.integrity.integrityMetaInitializer.lockAtMostForInMillis=900000 -task.integrity.integrityMetaInitializer.lockAtLeastForInMillis=90000 +integrityMetaInitializer.lockAtMostForInMillis=900000 +integrityMetaInitializer.lockAtLeastForInMillis=90000 #schedule task for 10th minute of every hour task.cron.metrics.portfolio=10 * * * * @@ -489,6 +489,8 @@ task.cron.defectdojo.sync=0 2 * * * task.cron.kenna.sync=0 2 * * * #schedule task every 15 minutes task.cron.workflow.state.cleanup=*/15 * * * * +#schedule task at 0 min past every 12th hr +task.cron.integrityInitializer=0 */12 * * * # Optional # Defines the number of write operations to perform during BOM processing before changes are flushed to the database. diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index ff32ad71f..9650f8fb3 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -57,7 +57,7 @@ org.dependencytrack.event.kafka.KafkaStreamsInitializer - org.dependencytrack.event.IntegrityMetaInitializer + org.dependencytrack.event.PurlMigrator diff --git a/src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java b/src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java deleted file mode 100644 index 30b93f631..000000000 --- a/src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java +++ /dev/null @@ -1,121 +0,0 @@ -package org.dependencytrack.event; - -import org.dependencytrack.AbstractPostgresEnabledTest; -import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.junit.Before; -import org.junit.Test; - -import java.sql.Date; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.dependencytrack.model.FetchStatus.IN_PROGRESS; -import static org.dependencytrack.model.FetchStatus.PROCESSED; - -public class IntegrityMetaInitializerTest extends AbstractPostgresEnabledTest { - - final Component componentPersisted = new Component(); - - @Before - public void persistComponentData() { - final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); - componentPersisted.setProject(projectA); - componentPersisted.setName("acme-lib-a"); - componentPersisted.setInternal(false); - componentPersisted.setPurlCoordinates("pkg:maven/acme/acme-lib-a@1.0.1"); - componentPersisted.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); - qm.persist(componentPersisted); - kafkaMockProducer.clear(); - } - - @Test - public void testIntegrityMetaInitializerWhenDisabledByDefault() { - IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(); - initializer.contextInitialized(null); - assertThat(qm.getIntegrityMetaComponentCount()).isZero(); - assertThat(kafkaMockProducer.history().size()).isZero(); - } - - @Test - public void testIntegrityMetaInitializer() { - IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); - initializer.contextInitialized(null); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - assertThat(kafkaMockProducer.history()).satisfiesExactly( - record -> assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()) - ); - assertThat(qm.getIntegrityMetaComponent(componentPersisted.getPurl().toString())).satisfies( - meta -> { - assertThat(meta.getStatus()).isEqualTo(IN_PROGRESS); - assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); - assertThat(meta.getId()).isEqualTo(1L); - assertThat(meta.getMd5()).isNull(); - assertThat(meta.getSha1()).isNull(); - assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isNotNull(); - assertThat(meta.getPublishedAt()).isNull(); - } - ); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataProcessed() { - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - integrityMetaExisting.setStatus(PROCESSED); - qm.persist(integrityMetaExisting); - // data exists in IntegrityMetaComponent so sync will be skipped - IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); - initializer.contextInitialized(null); - // kafka event is not dispatched - assertThat(kafkaMockProducer.history().size()).isZero(); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataNotProcessed() { - // data exists in IntegrityMetaComponent but not processed yet - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - qm.persist(integrityMetaExisting); - IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); - initializer.contextInitialized(null); - // kafka event is dispatched - assertThat(kafkaMockProducer.history().size()).isEqualTo(1); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataFetchedNotRecently() { - // data exists in IntegrityMetaComponent but last fetched 3 hours ago > 1 hour wait time - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - integrityMetaExisting.setStatus(IN_PROGRESS); - integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(3, ChronoUnit.HOURS))); - qm.persist(integrityMetaExisting); - IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); - initializer.contextInitialized(null); - // kafka event is dispatched - assertThat(kafkaMockProducer.history().size()).isEqualTo(1); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataFetchedRecently() { - // data exists in IntegrityMetaComponent but last fetched 30 min ago < 1 hour wait time - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - integrityMetaExisting.setStatus(IN_PROGRESS); - integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(30, ChronoUnit.MINUTES))); - qm.persist(integrityMetaExisting); - - IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); - initializer.contextInitialized(null); - // kafka event is dispatched - assertThat(kafkaMockProducer.history().size()).isZero(); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } -} diff --git a/src/test/java/org/dependencytrack/event/PurlMigratorTest.java b/src/test/java/org/dependencytrack/event/PurlMigratorTest.java new file mode 100644 index 000000000..1191df236 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/PurlMigratorTest.java @@ -0,0 +1,71 @@ +package org.dependencytrack.event; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Date; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.model.FetchStatus.IN_PROGRESS; +import static org.dependencytrack.model.FetchStatus.PROCESSED; + +public class PurlMigratorTest extends AbstractPostgresEnabledTest { + + final Component componentPersisted = new Component(); + + @Before + public void persistComponentData() { + final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); + componentPersisted.setProject(projectA); + componentPersisted.setName("acme-lib-a"); + componentPersisted.setInternal(false); + componentPersisted.setPurlCoordinates("pkg:maven/acme/acme-lib-a@1.0.1"); + componentPersisted.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); + qm.persist(componentPersisted); + kafkaMockProducer.clear(); + } + + @Test + public void testIntegrityMetaInitializerWhenDisabledByDefault() { + PurlMigrator initializer = new PurlMigrator(false); + initializer.contextInitialized(null); + assertThat(qm.getIntegrityMetaComponentCount()).isZero(); + assertThat(kafkaMockProducer.history().size()).isZero(); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataProcessed() { + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + integrityMetaExisting.setStatus(PROCESSED); + qm.persist(integrityMetaExisting); + // data exists in IntegrityMetaComponent so sync will be skipped + PurlMigrator initializer = new PurlMigrator(true); + initializer.contextInitialized(null); + // kafka event is not dispatched + assertThat(kafkaMockProducer.history().size()).isZero(); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } + + + @Test + public void testIntegrityMetaInitializerWithExistingDataFetchedRecently() { + // data exists in IntegrityMetaComponent but last fetched 30 min ago < 1 hour wait time + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + integrityMetaExisting.setStatus(IN_PROGRESS); + integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(30, ChronoUnit.MINUTES))); + qm.persist(integrityMetaExisting); + + PurlMigrator initializer = new PurlMigrator(true); + initializer.contextInitialized(null); + // kafka event is dispatched + assertThat(kafkaMockProducer.history().size()).isZero(); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } +} diff --git a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java index 9593f75bf..2a1d3fda3 100644 --- a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java +++ b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java @@ -1,8 +1,10 @@ package org.dependencytrack.persistence; import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; import org.dependencytrack.model.FetchStatus; import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.model.Project; import org.junit.Test; import java.util.Date; @@ -26,4 +28,37 @@ public void testCreateIntegrityMetadataHandlingConflict() { integrityMeta2.setLastFetch(new Date()); assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); } + + @Test + public void testSynchronizeIntegrityMetaComponent() { + final var project = new Project(); + project.setName("acme-app"); + project.setVersion("1.0.0"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); + component.setName("acme-lib"); + + // without any component in database + qm.synchronizeIntegrityMetaComponent(); + assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).isNull(); + + // with existing component in database + qm.persist(component); + qm.synchronizeIntegrityMetaComponent(); + assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).satisfies( + meta -> { + assertThat(meta.getStatus()).isNull(); + assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/example@1.0.0?type=jar"); + assertThat(meta.getId()).isEqualTo(1L); + assertThat(meta.getMd5()).isNull(); + assertThat(meta.getSha1()).isNull(); + assertThat(meta.getSha256()).isNull(); + assertThat(meta.getLastFetch()).isNull(); + assertThat(meta.getPublishedAt()).isNull(); + } + ); + } } diff --git a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java index 098daca0a..a687b814d 100644 --- a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java +++ b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java @@ -55,39 +55,6 @@ public void testUpdateIntegrityMetaComponent() { assertThat(result.getStatus()).isEqualTo(FetchStatus.PROCESSED); } - @Test - public void testSynchronizeIntegrityMetaComponent() { - final var project = new Project(); - project.setName("acme-app"); - project.setVersion("1.0.0"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - component.setName("acme-lib"); - - // without any component in database - qm.synchronizeIntegrityMetaComponent(); - assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).isNull(); - - // with existing component in database - qm.persist(component); - qm.synchronizeIntegrityMetaComponent(); - assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).satisfies( - meta -> { - assertThat(meta.getStatus()).isNull(); - assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/example@1.0.0?type=jar"); - assertThat(meta.getId()).isEqualTo(1L); - assertThat(meta.getMd5()).isNull(); - assertThat(meta.getSha1()).isNull(); - assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isNull(); - assertThat(meta.getPublishedAt()).isNull(); - } - ); - } - @Test public void testGetIntegrityMetaComponentCount() { var integrityMeta = new IntegrityMetaComponent(); diff --git a/src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java b/src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java new file mode 100644 index 000000000..14f74d757 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java @@ -0,0 +1,77 @@ +package org.dependencytrack.tasks; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.IntegrityMetaInitializerEvent; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; + +import java.sql.Date; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.model.FetchStatus.IN_PROGRESS; + +public class IntegrityMetaInitializerTaskTest extends AbstractPostgresEnabledTest { + + @Rule + public EnvironmentVariables environmentVariables = new EnvironmentVariables(); + + final Component componentPersisted = new Component(); + + @Before + public void persistComponentData() { + environmentVariables.set("INTEGRITY_INITIALIZER_ENABLED", "true"); + final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); + componentPersisted.setProject(projectA); + componentPersisted.setName("acme-lib-a"); + componentPersisted.setInternal(false); + componentPersisted.setPurlCoordinates("pkg:maven/acme/acme-lib-a@1.0.1"); + componentPersisted.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); + qm.persist(componentPersisted); + kafkaMockProducer.clear(); + } + + @Test + public void testIntegrityMetaInitializer() { + final var IntegrityMetaComponent = new IntegrityMetaComponent(); + IntegrityMetaComponent.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); + qm.persist(IntegrityMetaComponent); + new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()) + ); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataFetchedNotRecently() { + // data exists in IntegrityMetaComponent but last fetched 3 hours ago > 1 hour wait time + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + integrityMetaExisting.setStatus(IN_PROGRESS); + integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(3, ChronoUnit.HOURS))); + qm.persist(integrityMetaExisting); + new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); + // kafka event is dispatched + assertThat(kafkaMockProducer.history().size()).isEqualTo(1); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataNotProcessed() { + // data exists in IntegrityMetaComponent but not processed yet + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + qm.persist(integrityMetaExisting); + new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); + // kafka event is dispatched + assertThat(kafkaMockProducer.history().size()).isEqualTo(1); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } +}