Skip to content

Commit

Permalink
end to end test successful
Browse files Browse the repository at this point in the history 
Signed-off-by: mehab <[email protected]>
  • Loading branch information
@mehab
mehab committed Oct 11, 2023
1 parent fd44da4 commit 914a48d
Show file tree
Hide file tree
Showing 12 changed files with 49 additions and 53 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ private void updateIntegrityMetaForPurls(QueryManager qm, List<String> purls) {
private void dispatchPurls(QueryManager qm, List<String> purls) {
for (final var purl : purls) {
ComponentProjection componentProjection = qm.getComponentByPurl(purl);
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal, FetchMeta.FETCH_META_INTEGRITY_DATA));
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(purl, componentProjection.internal, FetchMeta.FETCH_META_INTEGRITY_DATA));
}
}

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
package org.dependencytrack.event.kafka.componentmeta;

public record ComponentProjection(String purlCoordinates, Boolean internal, String purl) {
import com.github.packageurl.PackageURL;

public record ComponentProjection(String purlCoordinates, Boolean internal, PackageURL purl) {
}
4 changes: 1 addition & 3 deletions src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
package org.dependencytrack.event.kafka.componentmeta;

import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import org.dependencytrack.event.kafka.KafkaEventDispatcher;
import org.dependencytrack.persistence.QueryManager;
import org.hyades.proto.repometaanalysis.v1.FetchMeta;

public class HandlerFactory {

public static Handler createHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, FetchMeta fetchMeta) throws MalformedPackageURLException {
PackageURL packageURL = new PackageURL(componentProjection.purl());
boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(packageURL.getType());
boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(componentProjection.purl().getType());
if (result) {
return new SupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchMeta);
} else {
Expand Down
10 changes: 5 additions & 5 deletions src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,19 @@ public SupportedMetaHandler(ComponentProjection componentProjection, QueryManage

@Override
public IntegrityMetaComponent handle() throws MalformedPackageURLException {
IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl());
IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl().toString());
if (persistentIntegrityMetaComponent == null) {
IntegrityMetaComponent integrityMetaComponent = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl()));
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(new PackageURL(componentProjection.purlCoordinates()).canonicalize(), componentProjection.internal(), fetchMeta));
IntegrityMetaComponent integrityMetaComponent = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl().toString()));
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purl().canonicalize(), componentProjection.internal(), fetchMeta));
return integrityMetaComponent;
}
if (persistentIntegrityMetaComponent.getStatus() == null || (persistentIntegrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - persistentIntegrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) {
persistentIntegrityMetaComponent.setLastFetch(Date.from(Instant.now()));
IntegrityMetaComponent updateIntegrityMetaComponent = queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent);
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(new PackageURL(componentProjection.purlCoordinates()).canonicalize(), componentProjection.internal(), fetchMeta));
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purl().canonicalize(), componentProjection.internal(), fetchMeta));
return updateIntegrityMetaComponent;
} else {
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(new PackageURL(componentProjection.purlCoordinates()).canonicalize(), componentProjection.internal(), fetchMeta));
kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purl().canonicalize(), componentProjection.internal(), fetchMeta));
return persistentIntegrityMetaComponent;
}
}
Expand Down
53 changes: 26 additions & 27 deletions src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ private void synchronizeMetaInformationForComponent(final QueryManager queryMana
trx.begin();
pm.makePersistent(repositoryMetaComponentResult);
trx.commit();
break; // this means that transaction was successful and we do not need to retry
}
} catch (JDODataStoreException e) {
// TODO: DataNucleus doesn't map constraint violation exceptions very well,
Expand All @@ -99,8 +100,9 @@ private void synchronizeMetaInformationForComponent(final QueryManager queryMana
}
}
//snychronize integrity meta information if available
IntegrityMetaComponent res = synchronizeIntegrityMetaResult(record, queryManager, purl);
if (res == null) {
if (result.hasIntegrityMeta()) {
synchronizeIntegrityMetaResult(record, queryManager, purl);
} else {
LOGGER.debug("Incoming result for component with purl %s does not include component integrity info".formatted(purl));
}
}
Expand Down Expand Up @@ -146,41 +148,38 @@ private RepositoryMetaComponent createRepositoryMetaResult(Record<String, Analys
}
}

private IntegrityMetaComponent synchronizeIntegrityMetaResult(final Record<String, AnalysisResult> incomingAnalysisResultRecord, QueryManager queryManager, PackageURL purl) {
private void synchronizeIntegrityMetaResult(final Record<String, AnalysisResult> incomingAnalysisResultRecord, QueryManager queryManager, PackageURL purl) {
final AnalysisResult result = incomingAnalysisResultRecord.value();

IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(purl.toString());
if (persistentIntegrityMetaComponent == null) {
persistentIntegrityMetaComponent = new IntegrityMetaComponent();
}

if (persistentIntegrityMetaComponent.getStatus().equals(FetchStatus.PROCESSED)) {
LOGGER.warn("""
Received hash information for %s that has already been processed; Discarding
""".formatted(purl));
return null;
}
if (result.hasIntegrityMeta()) {
if (result.getIntegrityMeta().hasMd5() || result.getIntegrityMeta().hasSha1() || result.getIntegrityMeta().hasSha256()
|| result.getIntegrityMeta().hasSha512() || result.getIntegrityMeta().hasCurrentVersionLastModified()) {
Optional.ofNullable(result.getIntegrityMeta().getMd5()).ifPresent(persistentIntegrityMetaComponent::setMd5);
Optional.ofNullable(result.getIntegrityMeta().getSha1()).ifPresent(persistentIntegrityMetaComponent::setSha1);
Optional.ofNullable(result.getIntegrityMeta().getSha256()).ifPresent(persistentIntegrityMetaComponent::setSha256);
Optional.ofNullable(result.getIntegrityMeta().getSha512()).ifPresent(persistentIntegrityMetaComponent::setSha512);
persistentIntegrityMetaComponent.setPurl(result.getComponent().getPurl());
persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl());
persistentIntegrityMetaComponent.setPublishedAt(result.getIntegrityMeta().hasCurrentVersionLastModified() ? new Date(result.getIntegrityMeta().getCurrentVersionLastModified().getSeconds() * 1000) : null);
persistentIntegrityMetaComponent.setStatus(FetchStatus.PROCESSED);
} else {
persistentIntegrityMetaComponent.setMd5(null);
persistentIntegrityMetaComponent.setSha256(null);
persistentIntegrityMetaComponent.setSha1(null);
persistentIntegrityMetaComponent.setSha512(null);
persistentIntegrityMetaComponent.setPurl(purl.toString());
persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl());
persistentIntegrityMetaComponent.setStatus(FetchStatus.NOT_AVAILABLE);
}
return queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent);
if (result.getIntegrityMeta().hasMd5() || result.getIntegrityMeta().hasSha1() || result.getIntegrityMeta().hasSha256()
|| result.getIntegrityMeta().hasSha512() || result.getIntegrityMeta().hasCurrentVersionLastModified()) {
Optional.ofNullable(result.getIntegrityMeta().getMd5()).ifPresent(persistentIntegrityMetaComponent::setMd5);
Optional.ofNullable(result.getIntegrityMeta().getSha1()).ifPresent(persistentIntegrityMetaComponent::setSha1);
Optional.ofNullable(result.getIntegrityMeta().getSha256()).ifPresent(persistentIntegrityMetaComponent::setSha256);
Optional.ofNullable(result.getIntegrityMeta().getSha512()).ifPresent(persistentIntegrityMetaComponent::setSha512);
persistentIntegrityMetaComponent.setPurl(result.getComponent().getPurl());
persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl());
persistentIntegrityMetaComponent.setPublishedAt(result.getIntegrityMeta().hasCurrentVersionLastModified() ? new Date(result.getIntegrityMeta().getCurrentVersionLastModified().getSeconds() * 1000) : null);
persistentIntegrityMetaComponent.setStatus(FetchStatus.PROCESSED);
} else {
return null;
persistentIntegrityMetaComponent.setMd5(null);
persistentIntegrityMetaComponent.setSha256(null);
persistentIntegrityMetaComponent.setSha1(null);
persistentIntegrityMetaComponent.setSha512(null);
persistentIntegrityMetaComponent.setPurl(purl.toString());
persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl());
persistentIntegrityMetaComponent.setStatus(FetchStatus.NOT_AVAILABLE);
}
queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent);
}

}
2 changes: 1 addition & 1 deletion src/main/java/org/dependencytrack/model/IntegrityMetaComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public void setSha512(String sha512) {
}

@Persistent
@Column(name = "SHA_512", jdbcType = "VARCHAR", length = 128)
@Column(name = "SHA512", jdbcType = "VARCHAR", length = 128)
@Pattern(regexp = "^[0-9a-fA-F]{128}$", message = "The SHA-512 hash must be a valid 128 character HEX number")
private String sha512;

Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/dependencytrack/resources/v1/ComponentResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo
component = qm.createComponent(component, true);
ComponentProjection componentProjection =
new ComponentProjection(component.getPurlCoordinates().toString(),
component.isInternal(), component.getPurl().toString());
component.isInternal(), component.getPurl());
try {
Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
repoMetaHandler.handle();
Expand Down Expand Up @@ -418,7 +418,7 @@ public Response updateComponent(Component jsonComponent) {
component = qm.updateComponent(component, true);
ComponentProjection componentProjection =
new ComponentProjection(component.getPurlCoordinates().toString(),
component.isInternal(), component.getPurl().toString());
component.isInternal(), component.getPurl());
try {

Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -980,12 +980,12 @@ private ComponentRepositoryMetaAnalysisEvent createRepoMetaAnalysisEvent(Compone
IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(component.getPurl().toString());
if (integrityMetaComponent == null) {
qm.getPersistenceManager().makePersistent(AbstractMetaHandler.createIntegrityMetaComponent(component.getPurl().toString()));
return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().canonicalize(), component.isInternal(), FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
return new ComponentRepositoryMetaAnalysisEvent(component.getPurl().canonicalize(), component.isInternal(), FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
}
if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && (Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime()) > TIME_SPAN)) {
integrityMetaComponent.setLastFetch(Date.from(Instant.now()));
qm.getPersistenceManager().makePersistent(integrityMetaComponent);
return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().canonicalize(), component.isInternal(), FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
return new ComponentRepositoryMetaAnalysisEvent(component.getPurl().canonicalize(), component.isInternal(), FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
} else {
return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().canonicalize(), component.isInternal(), FetchMeta.FETCH_META_LATEST_VERSION);
}
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public void createHandlerForSupportedPackageTest() {
KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
try {
PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12");
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString());
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl);
handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION);
assertTrue(handler instanceof SupportedMetaHandler);
} catch (MalformedPackageURLException e) {
Expand All @@ -36,7 +36,7 @@ public void createHandlerForUnSupportedPackageTest() {
KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
try {
PackageURL packageUrl = new PackageURL("pkg:golang/github.com/foo/[email protected]");
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString());
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl);
handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_LATEST_VERSION);
assertTrue(handler instanceof UnSupportedMetaHandler);
} catch (MalformedPackageURLException e) {
Expand Down
6 changes: 3 additions & 3 deletions src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ public void testHandleIntegrityComponentNotInDB() {
KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
try {
PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12");
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString());
IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl());
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl);
IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl().toString());
Assertions.assertNull(integrityMetaComponent);
handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA);
IntegrityMetaComponent result = handler.handle();
Expand All @@ -57,7 +57,7 @@ public void testHandleIntegrityComponentInDBForMoreThanAnHour() {
KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
try {
PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12");
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString());
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl);
var integrityMeta = new IntegrityMetaComponent();
integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12");
integrityMeta.setStatus(FetchStatus.IN_PROGRESS);
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public void testHandleComponentInDb() {
KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
try {
PackageURL packageUrl = new PackageURL("pkg:golang/foo/bar@baz?ping=pong#1/2/3");
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString());
IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl());
ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl);
IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl().toString());
Assertions.assertNull(integrityMetaComponent);
handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_LATEST_VERSION);
handler.handle();
Expand Down
Loading

0 comments on commit 914a48d

Please sign in to comment.