diff --git a/src/main/kotlin/artifact/ArtifactService.kt b/src/main/kotlin/artifact/ArtifactService.kt deleted file mode 100644 index fd5bfd9..0000000 --- a/src/main/kotlin/artifact/ArtifactService.kt +++ /dev/null @@ -1,191 +0,0 @@ -package artifact - -import artifact.model.* -import http.deps.DepsClient -import http.deps.model.DepsTreeResponseDto -import http.deps.model.Node -import io.github.z4kn4fein.semver.toVersion -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.cancel -import org.apache.logging.log4j.kotlin.logger - -class ArtifactService @OptIn(ExperimentalCoroutinesApi::class) constructor( - private val depsClient: DepsClient = DepsClient(), - // It is important to limit the parallelization of the IO scope, which is used to make server - // requests, or else the server at some point will tell us to go away. - private val ioScope: CoroutineScope = CoroutineScope(Dispatchers.IO.limitedParallelism(10)) -) { - - - fun directDependencyPackageReferenceToArtifact( - rootPackage: PackageReferenceDto - ): ArtifactDto? { - - return getDependencyVersionInformation( - packageRef = rootPackage, - seen = mutableSetOf() - ) - } - - private fun getDependencyVersionInformation( - packageRef: PackageReferenceDto, - seen: MutableSet, - ): ArtifactDto? { - return if (seen.contains(packageRef)) { - null - } else { - seen.add(packageRef) - - val transitiveDependencies = packageRef.dependencies.mapNotNull { - getDependencyVersionInformation( - packageRef = it, - seen = seen - ) - } - - return ArtifactDto( - artifactId = packageRef.name, - groupId = packageRef.namespace, - usedVersion = packageRef.version, - transitiveDependencies = transitiveDependencies - ) - } - } - - -// suspend fun simulateUpdateForArtifact( -// ecosystem: String, -// artifactDto: ArtifactDto, -// usedVersion: VersionDto, -// allVersions: List -// ): UpdatePossibilities { -// try { -// logger.info { "Simulate update for ${artifactDto.artifactId}" } -// val highestPossibleMinor = -// getApplicableVersion(usedVersion, allVersions, VersionTypes.Minor) -// val highestPossibleMajor = -// getApplicableVersion(usedVersion, allVersions, VersionTypes.Major) -// val highestPossiblePatch = -// getApplicableVersion(usedVersion, allVersions, VersionTypes.Patch) -// logger.info { "Update possibilities $highestPossiblePatch, $highestPossibleMajor, $highestPossibleMinor" } -// -// // TODO: we need to merge this existing code with the updated version storage data structure -// -// val updatedSubTreeMinor = getDependencyTreeForPkg( -// ecosystem, -// artifactDto.groupId, -// artifactDto.artifactId, -// highestPossibleMinor -// ) -// -// val updatedSubTreeMajor = getDependencyTreeForPkg( -// ecosystem, -// artifactDto.groupId, -// artifactDto.artifactId, -// highestPossibleMajor -// ) -// -// val updatedSubTreePatch = getDependencyTreeForPkg( -// ecosystem, -// artifactDto.groupId, -// artifactDto.artifactId, -// highestPossiblePatch -// ) -// -// return UpdatePossibilities( -// minor = updatedSubTreeMinor, -// major = updatedSubTreeMajor, -// patch = updatedSubTreePatch -// ) -// -// } catch (exception: Exception) { -// logger.warn { "Get update possibilities failed with $exception" } -// return UpdatePossibilities() -// } -// } - - /** - * Returns the highest matching version from the given versions array with the same version type as the - * given version's type. - */ -// private fun getApplicableVersion(version: VersionDto, versions: List, type: VersionTypes): String? { -// val semvers = versions.map { it.versionNumber.toVersion(strict = false) } -// val semver = version.versionNumber.toVersion(strict = false) -// -// val highestVersion = when (type) { -// VersionTypes.Minor -> { -// semvers.filter { it.isStable && it.major == semver.major } -// .maxWithOrNull(compareBy({ it.minor }, { it.patch })) -// } -// -// VersionTypes.Major -> { -// semvers.filter { it.isStable } -// .maxWithOrNull(compareBy({ it.major }, { it.minor }, { it.patch })) -// } -// -// VersionTypes.Patch -> { -// semvers.filter { it.isStable && it.major == semver.major && it.minor == semver.minor } -// .maxByOrNull { it.patch } -// } -// } -// return highestVersion?.toString() -// } - -// private suspend fun getDependencyTreeForPkg( -// ecosystem: String, -// namespace: String = "", -// name: String, -// version: String -// ): DepsTreeResponseDto? { -// -// depsClient.getDepsForPackage( -// ecosystem = ecosystem, -// namespace = namespace, -// name = name, -// version = version -// )?.let { depsTreeResponse -> -// logger.info { "Deps for package $namespace $name retrieved. Node size: ${depsTreeResponse.nodes.size}" } -// -// removeCycles(depsTreeResponse) -// -// return depsTreeResponse -// } -// -// return null -// } - - - - - private fun getCreateArtifactFromVersion( - ecosystem: String, - tree: DepsTreeResponseDto, - idx: Int, - node: Node - ): ArtifactDto { - - val transitiveNodes = tree.edges - .filter { it.fromNode == idx } - .map { Pair(it.toNode, tree.nodes[it.toNode]) } - .map { getCreateArtifactFromVersion(ecosystem, tree, it.first, it.second) } - - val nameAndNamespaceSplit = node.versionKey.name.split("/") - val nameAndNamespace = if (nameAndNamespaceSplit.count() == 2) { - Pair(nameAndNamespaceSplit[0], nameAndNamespaceSplit[1]) - } else { - Pair("", nameAndNamespaceSplit[0]) - } - - return ArtifactDto( - artifactId = nameAndNamespace.second, - groupId = nameAndNamespace.first, - usedVersion = node.versionKey.version, - transitiveDependencies = transitiveNodes, - ) - } - - -} - diff --git a/src/main/kotlin/artifact/model/DependencyGraphs.kt b/src/main/kotlin/artifact/model/DependencyGraphs.kt index 83c27d8..f2fbc61 100644 --- a/src/main/kotlin/artifact/model/DependencyGraphs.kt +++ b/src/main/kotlin/artifact/model/DependencyGraphs.kt @@ -4,10 +4,16 @@ import io.github.z4kn4fein.semver.toVersion import kotlinx.serialization.Serializable @Serializable -data class ArtifactNode( +data class ArtifactNode private constructor( val artifactIdx: Int, // Index of the artifact in the DependencyGraphs' artifacts list - val usedVersion: String, // Version of the linked artifact used in this specific node -) + val usedVersion: String, +) { + companion object { + fun create(artifactIdx: Int, version: String): ArtifactNode { + return ArtifactNode(artifactIdx, ArtifactVersion.validateAndHarmonizeVersionString(version)) + } + } +} @Serializable data class ArtifactNodeEdge( @@ -82,7 +88,7 @@ data class Artifact( ) @Serializable -data class ArtifactVersion( +data class ArtifactVersion private constructor( val versionNumber: String, val releaseDate: Long, val isDefault: Boolean = false @@ -93,6 +99,18 @@ data class ArtifactVersion( } companion object { + fun create(versionNumber: String, releaseDate: Long, isDefault: Boolean = false): ArtifactVersion { + return ArtifactVersion( + releaseDate = releaseDate, + isDefault = false, + versionNumber = validateAndHarmonizeVersionString(versionNumber) + ) + } + + fun validateAndHarmonizeVersionString(version: String): String { + return version.toVersion(strict = false).toString() + } + fun findHighestApplicableVersion( version: String, versions: List, diff --git a/src/main/kotlin/artifact/model/PackageReferenceDto.kt b/src/main/kotlin/artifact/model/PackageReferenceDto.kt deleted file mode 100644 index 588bbf4..0000000 --- a/src/main/kotlin/artifact/model/PackageReferenceDto.kt +++ /dev/null @@ -1,26 +0,0 @@ -package artifact.model - -import org.ossreviewtoolkit.model.PackageReference - -data class PackageReferenceDto( - val name: String, - val namespace: String, - val version: String, - val type: String, - val dependencies: List = listOf() -) { - companion object { - fun initFromPackageRef(packageRef: PackageReference): PackageReferenceDto { - - return PackageReferenceDto( - name = packageRef.id.name, - namespace = packageRef.id.namespace, - version = packageRef.id.version, - type = packageRef.id.type, - dependencies = packageRef.dependencies.map { - initFromPackageRef(it) - } - ) - } - } -} diff --git a/src/main/kotlin/commands/AnalyzeVersions.kt b/src/main/kotlin/commands/AnalyzeVersions.kt deleted file mode 100644 index 281df78..0000000 --- a/src/main/kotlin/commands/AnalyzeVersions.kt +++ /dev/null @@ -1,49 +0,0 @@ -package commands - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.groups.cooccurring -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required -import com.github.ajalt.clikt.parameters.types.path -import commands.options.DbOptions -import kotlinx.coroutines.runBlocking -import util.DbConfig -import util.initSqlLiteDb -import vulnerabilities.VulnerabilityAnalyzer - -class AnalyzeVersions : CliktCommand() { - - private val dbOptions by DbOptions().cooccurring() - - private val inputPath by option( - envvar = "INPUT_PATH", - help = "Path to the folder in which the combined version and vulnerability information are stored." - ) - .path(mustExist = false, canBeFile = false) - .required() - - override fun run() = runBlocking { - - val dbConfig = dbOptions?.let { - DbConfig( - url = it.dbUrl, - userName = it.userName, - password = it.password - ) - } - if (dbConfig != null) { - initSqlLiteDb(dbConfig) - val vulnerabilityAnalyzer = - VulnerabilityAnalyzer(inputPath) - vulnerabilityAnalyzer.dbExport() - } -// val vulnerabilityAnalyzer = -// VulnerabilityAnalyzer(inputPath) -//// vulnerabilityAnalyzer.analyze() -// val histogram = vulnerabilityAnalyzer.histogram() -// Visualizer.createAndStoreHistogram( -// histogram, -// inputPath.resolve("histogram-noOutlierGreater900.png").toString() -// ) - } -} diff --git a/src/main/kotlin/commands/GetVersions.kt b/src/main/kotlin/commands/GetVersions.kt deleted file mode 100644 index 5099e1a..0000000 --- a/src/main/kotlin/commands/GetVersions.kt +++ /dev/null @@ -1,21 +0,0 @@ -//package commands -// -//import com.github.ajalt.clikt.core.CliktCommand -//import com.github.ajalt.clikt.parameters.options.option -//import com.github.ajalt.clikt.parameters.options.required -//import com.github.ajalt.clikt.parameters.types.path -//import kotlinx.coroutines.runBlocking -//import vulnerabilities.VulnerabilityVersionDownloader -// -//class GetVersions : CliktCommand() { -// private val inputPath by option( -// envvar = "INPUT_PATH", help = "Path to the folder in which the vulnerability information are stored." -// ) -// .path(mustExist = false, canBeFile = false) -// .required() -// -// override fun run() = runBlocking { -// val vulnerabilityVersionDownloader = VulnerabilityVersionDownloader() -// vulnerabilityVersionDownloader.storeVersionsForVulnerablePackages(inputPath) -// } -//} diff --git a/src/main/kotlin/commands/options/DbOptions.kt b/src/main/kotlin/commands/options/DbOptions.kt deleted file mode 100644 index 881c544..0000000 --- a/src/main/kotlin/commands/options/DbOptions.kt +++ /dev/null @@ -1,17 +0,0 @@ -package commands.options - -import com.github.ajalt.clikt.parameters.groups.OptionGroup -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required - -class DbOptions : OptionGroup() { - val dbUrl by option( - envvar = "DB_URL", help = "Optional path to store a file based database which contains" + - " version numbers and their release dates." + - "This database is used as a cache and the application works seamlessly without it." + - "If the path doesn't exist it will be created." - ).required() - - val userName by option(envvar = "DB_USER", help = "Username of database user") - val password by option(envvar = "DB_PW", help = "Password for given database user") -} diff --git a/src/main/kotlin/dependencies/DependencyGraphService.kt b/src/main/kotlin/dependencies/DependencyGraphService.kt index 27883f0..ae8a823 100644 --- a/src/main/kotlin/dependencies/DependencyGraphService.kt +++ b/src/main/kotlin/dependencies/DependencyGraphService.kt @@ -3,7 +3,6 @@ package dependencies import artifact.model.* import http.deps.DepsClient import http.deps.model.DepsTreeResponseDto -import io.github.z4kn4fein.semver.toVersion import kotlinx.coroutines.* import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.DependencyGraph @@ -70,13 +69,14 @@ class DependencyGraphService( ) } - simulateUpdates( - DependencyGraphs( - ecosystem = packageManager, - artifacts = uniqueArtifacts.artifacts, - graph = graphs - ) + //TODO: Update function is broken right now and we are not yet settled on how to use it anyways +// simulateUpdates( + DependencyGraphs( + ecosystem = packageManager, + artifacts = uniqueArtifacts.artifacts, + graph = graphs ) +// ) } } @@ -136,10 +136,15 @@ class DependencyGraphService( ecosystem = ecosystem, namespace = artifact.groupId, name = artifact.artifactId - ).mapNotNull { + ).mapNotNull { artifactVersion -> try { - it.versionNumber.toVersion(strict = false) - it + ArtifactVersion.create( + releaseDate = artifactVersion.releaseDate, + isDefault = artifactVersion.isDefault, + // this step harmonizes possibly weired version formats like 2.4 or 5 + // those are parsed to 2.4.0 and 5.0.0 + versionNumber = artifactVersion.versionNumber + ) } catch (e: Exception) { null } @@ -176,9 +181,9 @@ class DependencyGraphService( val idx = uniqueArtifacts.identToIdx[ident] ?: -1 val insertIndex = nodes.count() nodes.add( - ArtifactNode( + ArtifactNode.create( artifactIdx = idx, - usedVersion = packageRef.id.version + version = packageRef.id.version ) ) @@ -208,7 +213,10 @@ class DependencyGraphService( suspend fun simulateUpdates(graphs: DependencyGraphs): DependencyGraphs { - val scopeToVersionToTree: MutableMap> = + // TODO: I believe this data structure is incomplete right now. After running the code it only contains + // the nodes which have update possibilities. However, if we can't update something in the graph it + // stays unchanged and therefor the original graph content needs to be copied + val scopeToVersionToTree: MutableMap> = mutableMapOf() graphs.graph.forEach { (scope, graph) -> @@ -217,13 +225,14 @@ class DependencyGraphService( val artifact = graphs.artifacts[artifactNode.artifactIdx] val currentVersion = artifactNode.usedVersion + val versionTypesToGraph: MutableMap = + mutableMapOf() listOf( ArtifactVersion.VersionTypes.Major, ArtifactVersion.VersionTypes.Minor, ArtifactVersion.VersionTypes.Patch ).forEach { versionTypes -> - val versionTypesToGraph: MutableMap = - mutableMapOf() + ArtifactVersion.findHighestApplicableVersion( version = currentVersion, versions = artifact.versions, updateType = versionTypes @@ -237,8 +246,8 @@ class DependencyGraphService( versionTypesToGraph[versionTypes] = depsTree } } - scopeToVersionToTree[scope] = versionTypesToGraph } + scopeToVersionToTree[scope] = versionTypesToGraph.toMap() } } @@ -256,7 +265,12 @@ class DependencyGraphService( } }.toMutableList() - newArtifacts.removeAll { artifacts.contains(it) } + newArtifacts.removeAll { newArtifact -> + artifacts.any { existingArtifact -> + newArtifact.artifactId == existingArtifact.artifactId && + newArtifact.groupId == existingArtifact.groupId + } + } val newArtifactsWithVersion = queryArtifactsVersions(ecosystem = graphs.ecosystem, artifacts = newArtifacts) newArtifactsWithVersion.forEach { artifact -> @@ -271,9 +285,9 @@ class DependencyGraphService( val nodes = tree.nodes.map { node -> val artifactIdx = artifacts.indexOfFirst { it.groupId == node.getNamespace() && it.artifactId == node.getName() } - ArtifactNode( + ArtifactNode.create( artifactIdx = artifactIdx, - usedVersion = node.versionKey.version + version = node.versionKey.version ) } diff --git a/src/main/kotlin/dependencies/db/AnalyzerResults.kt b/src/main/kotlin/dependencies/db/AnalyzerResults.kt deleted file mode 100644 index fe74573..0000000 --- a/src/main/kotlin/dependencies/db/AnalyzerResults.kt +++ /dev/null @@ -1,20 +0,0 @@ -package dependencies.db - -import dependencies.model.AnalyzerResultDto -import kotlinx.serialization.json.Json -import org.jetbrains.exposed.dao.IntEntity -import org.jetbrains.exposed.dao.IntEntityClass -import org.jetbrains.exposed.dao.id.EntityID -import org.jetbrains.exposed.dao.id.IntIdTable -import org.jetbrains.exposed.sql.json.jsonb - - -object AnalyzerResults : IntIdTable() { - val result = jsonb("result", Json.Default) -} - -class AnalyzerResult(id: EntityID) : IntEntity(id) { - companion object : IntEntityClass(AnalyzerResults) - - var result by AnalyzerResults.result -} diff --git a/src/main/kotlin/dependencies/model/AnalyzerResultDto.kt b/src/main/kotlin/dependencies/model/AnalyzerResultDto.kt index cf7104c..ba26e36 100644 --- a/src/main/kotlin/dependencies/model/AnalyzerResultDto.kt +++ b/src/main/kotlin/dependencies/model/AnalyzerResultDto.kt @@ -1,6 +1,5 @@ package dependencies.model -import artifact.model.DependencyGraphs import kotlinx.serialization.Serializable @Serializable diff --git a/src/main/kotlin/dependencies/model/ProjectDto.kt b/src/main/kotlin/dependencies/model/ProjectDto.kt deleted file mode 100644 index a2f3a01..0000000 --- a/src/main/kotlin/dependencies/model/ProjectDto.kt +++ /dev/null @@ -1,11 +0,0 @@ -package dependencies.model - -import kotlinx.serialization.Serializable - -@Serializable -data class ProjectDto( - val type: String, - val namespace: String, - val name: String, - val version: String -) diff --git a/src/main/kotlin/dependencies/model/RepositoryInfoDto.kt b/src/main/kotlin/dependencies/model/RepositoryInfoDto.kt index 9e50e5e..d2ada3f 100644 --- a/src/main/kotlin/dependencies/model/RepositoryInfoDto.kt +++ b/src/main/kotlin/dependencies/model/RepositoryInfoDto.kt @@ -4,3 +4,12 @@ import kotlinx.serialization.Serializable @Serializable data class RepositoryInfoDto(val url: String, val revision: String, val projects: List) + +@Serializable +data class ProjectDto( + val type: String, + val namespace: String, + val name: String, + val version: String +) + diff --git a/src/main/kotlin/http/deps/DepsClient.kt b/src/main/kotlin/http/deps/DepsClient.kt index 5b57b46..668180c 100644 --- a/src/main/kotlin/http/deps/DepsClient.kt +++ b/src/main/kotlin/http/deps/DepsClient.kt @@ -120,7 +120,7 @@ class DepsClient( private fun versionResponseToDto(version: Version): artifact.model.ArtifactVersion? { return if (version.publishedAt != null) { try { - artifact.model.ArtifactVersion( + artifact.model.ArtifactVersion.create( versionNumber = version.versionKey.version, releaseDate = dateToMs(version.publishedAt), isDefault = version.isDefault ?: false diff --git a/src/main/kotlin/http/deps/model/DepsResponseDto.kt b/src/main/kotlin/http/deps/model/DepsResponseDto.kt index a8b48be..b083649 100644 --- a/src/main/kotlin/http/deps/model/DepsResponseDto.kt +++ b/src/main/kotlin/http/deps/model/DepsResponseDto.kt @@ -10,4 +10,32 @@ data class DepsResponseDto( val packageKey: PackageKey, @SerialName("versions") val versions: List = emptyList() -) \ No newline at end of file +) + +@Serializable +data class Version( + @SerialName("isDefault") + val isDefault: Boolean? = null, + @SerialName("publishedAt") + val publishedAt: String? = null, + @SerialName("versionKey") + val versionKey: VersionKey +) + +@Serializable +data class VersionKey( + @SerialName("name") + val name: String, + @SerialName("system") + val system: String, + @SerialName("version") + val version: String +) + +@Serializable +data class PackageKey( + @SerialName("name") + val name: String, + @SerialName("system") + val system: String +) diff --git a/src/main/kotlin/http/deps/model/DepsTreeResponseDto.kt b/src/main/kotlin/http/deps/model/DepsTreeResponseDto.kt index 08d2ce0..4df73d4 100644 --- a/src/main/kotlin/http/deps/model/DepsTreeResponseDto.kt +++ b/src/main/kotlin/http/deps/model/DepsTreeResponseDto.kt @@ -16,3 +16,48 @@ data class DepsTreeResponseDto( @SerialName("nodes") val nodes: List = listOf() ) + +@Serializable +data class Node( + @SerialName("bundled") + val bundled: Boolean, + @SerialName("errors") + val errors: List = listOf(), + @SerialName("relation") + val relation: String, + @SerialName("versionKey") + val versionKey: VersionKeyX +) { + val namespaceAndName by lazy { + val namespaceAndNameSplit = versionKey.name.split("/") + + if (namespaceAndNameSplit.count() == 2) { + Pair(namespaceAndNameSplit[0], namespaceAndNameSplit[1]) + } else { + Pair("", namespaceAndNameSplit[0]) + } + } + + fun getNamespace(): String = namespaceAndName.first + fun getName(): String = namespaceAndName.second +} + +@Serializable +data class Edge( + @SerialName("fromNode") + val fromNode: Int, + @SerialName("requirement") + val requirement: String?, + @SerialName("toNode") + val toNode: Int +) + +@Serializable +data class VersionKeyX( + @SerialName("name") + val name: String, + @SerialName("system") + val system: String, + @SerialName("version") + val version: String +) diff --git a/src/main/kotlin/http/deps/model/Edge.kt b/src/main/kotlin/http/deps/model/Edge.kt deleted file mode 100644 index f9f8173..0000000 --- a/src/main/kotlin/http/deps/model/Edge.kt +++ /dev/null @@ -1,15 +0,0 @@ -package http.deps.model - - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -data class Edge( - @SerialName("fromNode") - val fromNode: Int, - @SerialName("requirement") - val requirement: String?, - @SerialName("toNode") - val toNode: Int -) \ No newline at end of file diff --git a/src/main/kotlin/http/deps/model/Node.kt b/src/main/kotlin/http/deps/model/Node.kt deleted file mode 100644 index 3f2c499..0000000 --- a/src/main/kotlin/http/deps/model/Node.kt +++ /dev/null @@ -1,30 +0,0 @@ -package http.deps.model - - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -data class Node( - @SerialName("bundled") - val bundled: Boolean, - @SerialName("errors") - val errors: List = listOf(), - @SerialName("relation") - val relation: String, - @SerialName("versionKey") - val versionKey: VersionKeyX -) { - val namespaceAndName by lazy { - val namespaceAndNameSplit = versionKey.name.split("/") - - if (namespaceAndNameSplit.count() == 2) { - Pair(namespaceAndNameSplit[0], namespaceAndNameSplit[1]) - } else { - Pair("", namespaceAndNameSplit[0]) - } - } - - fun getNamespace(): String = namespaceAndName.first - fun getName(): String = namespaceAndName.second -} \ No newline at end of file diff --git a/src/main/kotlin/http/deps/model/PackageKey.kt b/src/main/kotlin/http/deps/model/PackageKey.kt deleted file mode 100644 index d723405..0000000 --- a/src/main/kotlin/http/deps/model/PackageKey.kt +++ /dev/null @@ -1,13 +0,0 @@ -package http.deps.model - - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -data class PackageKey( - @SerialName("name") - val name: String, - @SerialName("system") - val system: String -) \ No newline at end of file diff --git a/src/main/kotlin/http/deps/model/Version.kt b/src/main/kotlin/http/deps/model/Version.kt deleted file mode 100644 index 57e1d7d..0000000 --- a/src/main/kotlin/http/deps/model/Version.kt +++ /dev/null @@ -1,15 +0,0 @@ -package http.deps.model - - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -data class Version( - @SerialName("isDefault") - val isDefault: Boolean? = null, - @SerialName("publishedAt") - val publishedAt: String? = null, - @SerialName("versionKey") - val versionKey: VersionKey -) \ No newline at end of file diff --git a/src/main/kotlin/http/deps/model/VersionKey.kt b/src/main/kotlin/http/deps/model/VersionKey.kt deleted file mode 100644 index 36ebe21..0000000 --- a/src/main/kotlin/http/deps/model/VersionKey.kt +++ /dev/null @@ -1,15 +0,0 @@ -package http.deps.model - - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -data class VersionKey( - @SerialName("name") - val name: String, - @SerialName("system") - val system: String, - @SerialName("version") - val version: String -) \ No newline at end of file diff --git a/src/main/kotlin/http/deps/model/VersionKeyX.kt b/src/main/kotlin/http/deps/model/VersionKeyX.kt deleted file mode 100644 index a2213b4..0000000 --- a/src/main/kotlin/http/deps/model/VersionKeyX.kt +++ /dev/null @@ -1,15 +0,0 @@ -package http.deps.model - - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -data class VersionKeyX( - @SerialName("name") - val name: String, - @SerialName("system") - val system: String, - @SerialName("version") - val version: String -) \ No newline at end of file diff --git a/src/main/kotlin/util/DbHelper.kt b/src/main/kotlin/util/DbHelper.kt deleted file mode 100644 index 09fa136..0000000 --- a/src/main/kotlin/util/DbHelper.kt +++ /dev/null @@ -1,52 +0,0 @@ -package util - -import dependencies.db.AnalyzerResults -import kotlinx.coroutines.Dispatchers -import org.jetbrains.exposed.sql.Database -import org.jetbrains.exposed.sql.SchemaUtils -import org.jetbrains.exposed.sql.StdOutSqlLogger -import org.jetbrains.exposed.sql.addLogger -import org.jetbrains.exposed.sql.transactions.TransactionManager -import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction -import org.jetbrains.exposed.sql.transactions.transaction -import vulnerabilities.entity.Versions -import vulnerabilities.entity.Vulnerabilities -import vulnerabilities.entity.VulnerabilitiesVersions -import java.sql.Connection - -suspend fun dbQuery(block: suspend () -> T): T = - newSuspendedTransaction(Dispatchers.IO) { block() } - -data class DbConfig(val url: String, val userName: String? = "", val password: String? = "") - -fun initSqlLiteDb(dbConfig: DbConfig) { - Database.connect( - url = dbConfig.url, - driver = "org.sqlite.JDBC", - ) - TransactionManager.manager.defaultIsolationLevel = - Connection.TRANSACTION_SERIALIZABLE - - transaction { - addLogger(StdOutSqlLogger) - SchemaUtils.create(Vulnerabilities) - SchemaUtils.create(Versions) - SchemaUtils.create(VulnerabilitiesVersions) - } -} - -fun initDatabase(dbConfig: DbConfig) { - - Database.connect( - url = dbConfig.url, - driver = "org.postgresql.Driver", - user = dbConfig.userName ?: "", - password = dbConfig.password ?: "" - ) - - - transaction { - addLogger(StdOutSqlLogger) - SchemaUtils.create(AnalyzerResults) - } -} diff --git a/src/main/kotlin/util/StoreResultsHelper.kt b/src/main/kotlin/util/StoreResultsHelper.kt index 382866c..96fa120 100644 --- a/src/main/kotlin/util/StoreResultsHelper.kt +++ b/src/main/kotlin/util/StoreResultsHelper.kt @@ -8,21 +8,8 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json import java.io.File -import java.nio.file.Path import java.util.* -data class StorageConfig( - val dbConfig: DbConfig?, - val outputPath: Path?, - val storeAnalyzerResultsInFile: Boolean, - val storeAnalyzerResultsInDb: Boolean, - val storeLibyearResultsInFile: Boolean, - val storeLibyearResultsInDb: Boolean, - val storeLibyearGraphs: Boolean, -) { - val storeAnalyzerResults = storeAnalyzerResultsInFile || storeAnalyzerResultsInDb -} - class StoreResultHelper { companion object { @@ -75,100 +62,3 @@ class StoreResultHelper { } } } - -// -//suspend fun storeResults( -// results: List = listOf(), -// aggregatedResults: AggregatedResults, -// config: StorageConfig -//) { -// -// if (config.dbConfig != null) { -// initDatabase(config.dbConfig) -// } -// -// if (config.storeAnalyzerResultsInFile) { -// results.forEach { analyzerResult -> -// if (config.storeAnalyzerResults) { -// -// val outputFile = config.outputPath?.resolve("${Date().time}-graphResult.json")?.toFile() -// withContext(Dispatchers.IO) { -// outputFile?.createNewFile() -// val json = Json { prettyPrint = false } -// val jsonString = -// json.encodeToString(DependencyGraphsDto.serializer(), analyzerResult.dependencyGraphDtos) -// outputFile?.writeText(jsonString) -// } -// } -// if (config.storeAnalyzerResultsInDb) { -// dbQuery { -// AnalyzerResult.new { -// result = analyzerResult -// } -// } -// } -// } -// } -// -// if (config.storeLibyearResultsInFile) { -// val outputFileAggregate = -// config.outputPath?.resolve("graphResultAggregate-${Date().time}.json")?.toFile() -// -// withContext(Dispatchers.IO) { -// outputFileAggregate?.createNewFile() -// val json = Json { prettyPrint = false } -// -// val jsonString = -// json.encodeToString( -// AggregatedResults.serializer(), -// aggregatedResults -// ) -// outputFileAggregate?.writeText(jsonString) -// } -// } -// -// if (config.storeLibyearGraphs) { -// val graphResults: MutableMap> = mutableMapOf() -// aggregatedResults.results.forEach { -// it.packageManagerToScopes.forEach { (packageManager, scopeToLibyears) -> -// scopeToLibyears.forEach { (scope, libyearResult) -> -// val keyTransitive = "$packageManager-$scope-transitive" -// val keyDirect = "$packageManager-$scope-direct" -// val keyDirectPerDependency = "$packageManager-$scope-direct-per-dependency" -// val keyTransitivePerDependency = "$packageManager-$scope-transitive-per-dependency" -// val keyDirectDependencies = "$packageManager-$scope-direct-number-dependencies" -// val keyTransitiveDependencies = "$packageManager-$scope-transitive-number-dependencies" -// fun addToResult(key: String, value: Long) { -// if (!graphResults.contains(key)) { -// graphResults[key] = mutableListOf(value) -// } else { -// graphResults[key]?.add(value) -// } -// } -// addToResult(keyDirect, libyearResult.direct.libyears) -// addToResult(keyTransitive, libyearResult.transitive.libyears) -// addToResult( -// keyDirectPerDependency, -// libyearResult.direct.libyears / libyearResult.direct.numberOfDependencies -// ) -// addToResult( -// keyTransitivePerDependency, -// libyearResult.transitive.libyears / libyearResult.transitive.numberOfDependencies -// ) -// addToResult(keyTransitiveDependencies, libyearResult.transitive.numberOfDependencies.toLong()) -// addToResult(keyDirectDependencies, libyearResult.direct.numberOfDependencies.toLong()) -// } -// } -// } -// -// graphResults.forEach { (name, values) -> -// Visualizer.createAndStoreLineDiagram( -// outputFilePath = config.outputPath?.resolve("$name.png")?.toAbsolutePath().toString(), -// name = name, -// values = values.toList(), -// ) -// } -// -// } -//} - diff --git a/src/main/kotlin/vulnerabilities/VulnerabilityAnalyzer.kt b/src/main/kotlin/vulnerabilities/VulnerabilityAnalyzer.kt index 18984e4..fbeb5c6 100644 --- a/src/main/kotlin/vulnerabilities/VulnerabilityAnalyzer.kt +++ b/src/main/kotlin/vulnerabilities/VulnerabilityAnalyzer.kt @@ -1,233 +1,233 @@ -package vulnerabilities - -import kotlinx.datetime.Instant -import kotlinx.datetime.TimeZone -import kotlinx.datetime.toLocalDateTime -import org.jetbrains.exposed.sql.SizedCollection -import util.dbQuery -import vulnerabilities.dto.osv.OpenSourceVulnerabilityFormat -import vulnerabilities.entity.Version -import vulnerabilities.entity.Vulnerability -import java.nio.file.Path -import kotlin.time.Duration - -data class VulnerabilityStatistics( - // how many vulnerabilities are in our dataset - val numberOfVulnerabilities: Int, - // what is the time difference between the report date - // of a vulnerability and the release date of the oldest - // version to which the vulnerability applies (one entry - // per vulnerability) - val timeDifferences: MutableList, - // see timeDifferences, just counting how many versions - // are mapped to one vulnerability - val versionDifferences: MutableList -) - -class VulnerabilityAnalyzer(private val filePath: Path) { - - - private val allVulnerabilities: MutableList = - VulnerabilityHelper.readVulnerabilitiesFromFile(filePath).toMutableList() - - - init { - removeIncompleteData() - } - - suspend fun dbExport() { - dbQuery { - val packageToVersions: MutableMap> = mutableMapOf() - allVulnerabilities.forEach { vul -> - val vulPublishedDate = Instant.parse(vul.published) - val id = vul.id - vul.affected.forEach { affected -> - affected.ranges.forEach { range -> - val introduced = range.events.find { it.introduced != null } - val fixed = range.events.find { it.fixed != null } - if (introduced != null) { - val pkgName = affected.packageX.name - val vulnerability = Vulnerability.new { - vulnerabilityId = id - publishedDate = vulPublishedDate.toLocalDateTime(timeZone = TimeZone.UTC).date - packageName = pkgName - introducedVersion = introduced.introduced!! - fixedVersion = fixed?.fixed - } - val versionEntities = if (packageToVersions.contains(pkgName)) { - packageToVersions[pkgName]!! - } else { - val vers = SizedCollection(affected.packageX.versions.map { pkgVersion -> - Version.new { - versionNumber = pkgVersion.versionNumber - publishedDate = Instant.parse(pkgVersion.released) - .toLocalDateTime(timeZone = TimeZone.UTC).date - } - }) - packageToVersions[pkgName] = vers - vers - } - - vulnerability.versions = versionEntities - } - } - - } - - } -// val vul = allVulnerabilities.first() - -// allVulnerabilities.forEach { -// Vulnerability.new { -// vulnerability = it +//package vulnerabilities +// +//import kotlinx.datetime.Instant +//import kotlinx.datetime.TimeZone +//import kotlinx.datetime.toLocalDateTime +//import org.jetbrains.exposed.sql.SizedCollection +//import util.dbQuery +//import vulnerabilities.dto.osv.OpenSourceVulnerabilityFormat +//import vulnerabilities.entity.Version +//import vulnerabilities.entity.Vulnerability +//import java.nio.file.Path +//import kotlin.time.Duration +// +//data class VulnerabilityStatistics( +// // how many vulnerabilities are in our dataset +// val numberOfVulnerabilities: Int, +// // what is the time difference between the report date +// // of a vulnerability and the release date of the oldest +// // version to which the vulnerability applies (one entry +// // per vulnerability) +// val timeDifferences: MutableList, +// // see timeDifferences, just counting how many versions +// // are mapped to one vulnerability +// val versionDifferences: MutableList +//) +// +//class VulnerabilityAnalyzer(private val filePath: Path) { +// +// +// private val allVulnerabilities: MutableList = +// VulnerabilityHelper.readVulnerabilitiesFromFile(filePath).toMutableList() +// +// +// init { +// removeIncompleteData() +// } +// +// suspend fun dbExport() { +// dbQuery { +// val packageToVersions: MutableMap> = mutableMapOf() +// allVulnerabilities.forEach { vul -> +// val vulPublishedDate = Instant.parse(vul.published) +// val id = vul.id +// vul.affected.forEach { affected -> +// affected.ranges.forEach { range -> +// val introduced = range.events.find { it.introduced != null } +// val fixed = range.events.find { it.fixed != null } +// if (introduced != null) { +// val pkgName = affected.packageX.name +// val vulnerability = Vulnerability.new { +// vulnerabilityId = id +// publishedDate = vulPublishedDate.toLocalDateTime(timeZone = TimeZone.UTC).date +// packageName = pkgName +// introducedVersion = introduced.introduced!! +// fixedVersion = fixed?.fixed +// } +// val versionEntities = if (packageToVersions.contains(pkgName)) { +// packageToVersions[pkgName]!! +// } else { +// val vers = SizedCollection(affected.packageX.versions.map { pkgVersion -> +// Version.new { +// versionNumber = pkgVersion.versionNumber +// publishedDate = Instant.parse(pkgVersion.released) +// .toLocalDateTime(timeZone = TimeZone.UTC).date +// } +// }) +// packageToVersions[pkgName] = vers +// vers +// } +// +// vulnerability.versions = versionEntities +// } +// } +// // } +// // } - } - } - - fun histogram(): Map { - var versionsWithoutMatch = 0 - val durations = allVulnerabilities.flatMap { vulnerability -> - val publishedDate = Instant.parse(vulnerability.published) - vulnerability.affected.flatMap { affected -> - val versions = affected.packageX.versions - affected.ranges.flatMap { range -> - range.events.mapNotNull { event -> - val introducedVersion = event.introduced - val version = versions.find { it.versionNumber == introducedVersion } - if (version == null) { - versionsWithoutMatch += 1 - } - version?.released?.let { - val versionReleaseDate = Instant.parse(it) - return@mapNotNull publishedDate.minus(versionReleaseDate).inWholeDays - } - return@mapNotNull null - } - } - } - } - val sortedNumbers = durations.sorted() - - val size = sortedNumbers.size - val index50 = (size * 0.5).toInt() - val index70 = (size * 0.7).toInt() - val index90 = (size * 0.9).toInt() - - val percentile50 = sortedNumbers[index50] - val percentile70 = sortedNumbers[index70] - val percentile90 = sortedNumbers[index90] - - println("50% percentile $percentile50, 70% percentile $percentile70, 90% percentile $percentile90") - - println("Versions without match $versionsWithoutMatch") - val noOutlier = durations.toMutableList() - println("previous count ${noOutlier.count()}") - noOutlier.removeAll { it > 900L } - println("after remove count ${noOutlier.count()}") - val occurrencesMap = mutableMapOf() - - - for (value in noOutlier) { - val intervalStart = (value / 7) - occurrencesMap[intervalStart] = occurrencesMap.getOrDefault(intervalStart, 0) + 1 - } - - - return occurrencesMap - } - - fun analyze() { - - - //TODO: only use vulnerabilities with a specific "introduced" value (precise, but small dataset) - var noIntroduced = allVulnerabilities.count() - allVulnerabilities.forEach { - it.affected.forEach { - it.ranges.forEach { - it.events.removeAll { it.introduced == "0" } - } - it.ranges.removeAll { it.events.isEmpty() } - } - } - allVulnerabilities.removeAll { it.affected.none { it.ranges.isNotEmpty() } } - noIntroduced -= allVulnerabilities.count() - - println( - "Vulnerabilities with no concrete introduced value $noIntroduced.\n" + - "Remaining ${allVulnerabilities.count()}" - ) - val diffs: MutableList = mutableListOf() -// allVulnerabilities.forEach { vulnerability -> -// // Using kotlinx.datetime.Instant -// val publishedInstant = Instant.parse(vulnerability.published) -// -// // each distinct package is counted separately for the same vulnerability -//// val distinctAffected = vulnerability.affected.distinctBy { it.packageX.name } -// vulnerability.affected.forEach { affected -> -// try { -// // TODO: this is strange -// val smallest = -// vulnerability.affected.minOf { it.ranges.minOf { it.events.minOf { it.introduced } } } -// affected.packageX.versions.find { it.versionNumber == smallest }.let { smallestVersion -> -// smallestVersion?.let { -// val versionReleaseDate = Instant.parse(it.released) -// diffs.add(publishedInstant.minus(versionReleaseDate)) +//// val vul = allVulnerabilities.first() +// +//// allVulnerabilities.forEach { +//// Vulnerability.new { +//// vulnerability = it +//// } +//// } +// } +// } +// +// fun histogram(): Map { +// var versionsWithoutMatch = 0 +// val durations = allVulnerabilities.flatMap { vulnerability -> +// val publishedDate = Instant.parse(vulnerability.published) +// vulnerability.affected.flatMap { affected -> +// val versions = affected.packageX.versions +// affected.ranges.flatMap { range -> +// range.events.mapNotNull { event -> +// val introducedVersion = event.introduced +// val version = versions.find { it.versionNumber == introducedVersion } +// if (version == null) { +// versionsWithoutMatch += 1 +// } +// version?.released?.let { +// val versionReleaseDate = Instant.parse(it) +// return@mapNotNull publishedDate.minus(versionReleaseDate).inWholeDays // } +// return@mapNotNull null // } -// } catch (exception: Exception) { -// println("calculation failed with $exception") // } // } // } - val avgDays = diffs.sumOf { it.inWholeDays } / diffs.count() - println("Avg days after version release until published vulnerability $avgDays") - - //TODO: use all vulnerabilities for which we have version information and make the time diff between - // the vulnerability found date and the latest release before the vulnerability was found (optimistic) - - //TODO: make the diff between the report time and the oldest minor or patch version (maybe realistic?) -// val haveIntroducedValue = allVulnerabilities.filter { vul -> -// vul.affected.any { affected -> -// affected.ranges?.any { range -> -// range?.events?.any { event -> -// event?.introduced != null && event.introduced != "0" -// } ?: false -// } ?: false +// val sortedNumbers = durations.sorted() +// +// val size = sortedNumbers.size +// val index50 = (size * 0.5).toInt() +// val index70 = (size * 0.7).toInt() +// val index90 = (size * 0.9).toInt() +// +// val percentile50 = sortedNumbers[index50] +// val percentile70 = sortedNumbers[index70] +// val percentile90 = sortedNumbers[index90] +// +// println("50% percentile $percentile50, 70% percentile $percentile70, 90% percentile $percentile90") +// +// println("Versions without match $versionsWithoutMatch") +// val noOutlier = durations.toMutableList() +// println("previous count ${noOutlier.count()}") +// noOutlier.removeAll { it > 900L } +// println("after remove count ${noOutlier.count()}") +// val occurrencesMap = mutableMapOf() +// +// +// for (value in noOutlier) { +// val intervalStart = (value / 7) +// occurrencesMap[intervalStart] = occurrencesMap.getOrDefault(intervalStart, 0) + 1 +// } +// +// +// return occurrencesMap +// } +// +// fun analyze() { +// +// +// //TODO: only use vulnerabilities with a specific "introduced" value (precise, but small dataset) +// var noIntroduced = allVulnerabilities.count() +// allVulnerabilities.forEach { +// it.affected.forEach { +// it.ranges.forEach { +// it.events.removeAll { it.introduced == "0" } +// } +// it.ranges.removeAll { it.events.isEmpty() } +// } +// } +// allVulnerabilities.removeAll { it.affected.none { it.ranges.isNotEmpty() } } +// noIntroduced -= allVulnerabilities.count() +// +// println( +// "Vulnerabilities with no concrete introduced value $noIntroduced.\n" + +// "Remaining ${allVulnerabilities.count()}" +// ) +// val diffs: MutableList = mutableListOf() +//// allVulnerabilities.forEach { vulnerability -> +//// // Using kotlinx.datetime.Instant +//// val publishedInstant = Instant.parse(vulnerability.published) +//// +//// // each distinct package is counted separately for the same vulnerability +////// val distinctAffected = vulnerability.affected.distinctBy { it.packageX.name } +//// vulnerability.affected.forEach { affected -> +//// try { +//// // TODO: this is strange +//// val smallest = +//// vulnerability.affected.minOf { it.ranges.minOf { it.events.minOf { it.introduced } } } +//// affected.packageX.versions.find { it.versionNumber == smallest }.let { smallestVersion -> +//// smallestVersion?.let { +//// val versionReleaseDate = Instant.parse(it.released) +//// diffs.add(publishedInstant.minus(versionReleaseDate)) +//// } +//// } +//// } catch (exception: Exception) { +//// println("calculation failed with $exception") +//// } +//// } +//// } +// val avgDays = diffs.sumOf { it.inWholeDays } / diffs.count() +// println("Avg days after version release until published vulnerability $avgDays") +// +// //TODO: use all vulnerabilities for which we have version information and make the time diff between +// // the vulnerability found date and the latest release before the vulnerability was found (optimistic) +// +// //TODO: make the diff between the report time and the oldest minor or patch version (maybe realistic?) +//// val haveIntroducedValue = allVulnerabilities.filter { vul -> +//// vul.affected.any { affected -> +//// affected.ranges?.any { range -> +//// range?.events?.any { event -> +//// event?.introduced != null && event.introduced != "0" +//// } ?: false +//// } ?: false +//// } +//// } +// } +// +// private fun removeIncompleteData() { +// +// println( +// "Started with ${allVulnerabilities.size} vulnerabilities. \n " + +// "Removing incomplete data." +// ) +// var noIntroducedValue = allVulnerabilities.count() +// +// allVulnerabilities.forEach { +// it.affected.forEach { +// it.ranges.forEach { +// it.events.removeAll { it.introduced == "-1" } +// } +// it.ranges.removeAll { it.events.isEmpty() } // } // } - } - - private fun removeIncompleteData() { - - println( - "Started with ${allVulnerabilities.size} vulnerabilities. \n " + - "Removing incomplete data." - ) - var noIntroducedValue = allVulnerabilities.count() - - allVulnerabilities.forEach { - it.affected.forEach { - it.ranges.forEach { - it.events.removeAll { it.introduced == "-1" } - } - it.ranges.removeAll { it.events.isEmpty() } - } - } - - allVulnerabilities.removeAll { it.affected.none { it.ranges.isNotEmpty() } } - - noIntroducedValue -= allVulnerabilities.count() - var noVersions = allVulnerabilities.count() - allVulnerabilities.removeAll { it.affected.none { it.packageX.versions?.isNotEmpty() ?: true } } - noVersions -= allVulnerabilities.count() - - val incompleteInformationCount = noVersions + noIntroducedValue - - println( - "Vulnerabilities with no introduced value ($noIntroducedValue) or no version information ($noVersions)\n" + - "Removed vulnerabilities: $incompleteInformationCount, remaining ${allVulnerabilities.size}. " - ) - } - -} \ No newline at end of file +// +// allVulnerabilities.removeAll { it.affected.none { it.ranges.isNotEmpty() } } +// +// noIntroducedValue -= allVulnerabilities.count() +// var noVersions = allVulnerabilities.count() +// allVulnerabilities.removeAll { it.affected.none { it.packageX.versions?.isNotEmpty() ?: true } } +// noVersions -= allVulnerabilities.count() +// +// val incompleteInformationCount = noVersions + noIntroducedValue +// +// println( +// "Vulnerabilities with no introduced value ($noIntroducedValue) or no version information ($noVersions)\n" + +// "Removed vulnerabilities: $incompleteInformationCount, remaining ${allVulnerabilities.size}. " +// ) +// } +// +//} \ No newline at end of file diff --git a/src/test/kotlin/artifact/ArtifactServiceTest.kt b/src/test/kotlin/artifact/ArtifactServiceTest.kt index 5701617..c5f0dee 100644 --- a/src/test/kotlin/artifact/ArtifactServiceTest.kt +++ b/src/test/kotlin/artifact/ArtifactServiceTest.kt @@ -1,6 +1,5 @@ package artifact -import artifact.model.PackageReferenceDto import http.deps.DepsClient import io.ktor.client.* import io.ktor.client.engine.mock.* @@ -61,49 +60,49 @@ class ArtifactServiceTest { fun transitiveDependencies() = runTest { - val artifactService = ArtifactService( +// val artifactService = ArtifactService( +// +// depsClient = apiClient +// ) - depsClient = apiClient - ) - - val packageDto = PackageReferenceDto( - name = "vite", - namespace = "", type = "NPM", version = "5.0.4", - dependencies = listOf( - PackageReferenceDto( - name = "vite", - namespace = "", - type = "NPM", - version = "3.1.1" - ), - PackageReferenceDto( - name = "vite", - namespace = "", - type = "NPM", - version = "3.1.0-beta.2", - dependencies = listOf( - PackageReferenceDto( - name = "vite", - namespace = "", - type = "NPM", - version = "3.1.0-beta.1" - ), - ) - ), - PackageReferenceDto( - name = "vite", - namespace = "", - type = "NPM", - version = "5.0.4" - ), - ) - ) - - val artifact = artifactService.directDependencyPackageReferenceToArtifact( - packageDto - ) - assertEquals( 3, artifact?.transitiveDependencies?.count()) - assertEquals( 1, artifact?.transitiveDependencies?.get(1)?.transitiveDependencies?.count()) +// val packageDto = PackageReferenceDto( +// name = "vite", +// namespace = "", type = "NPM", version = "5.0.4", +// dependencies = listOf( +// PackageReferenceDto( +// name = "vite", +// namespace = "", +// type = "NPM", +// version = "3.1.1" +// ), +// PackageReferenceDto( +// name = "vite", +// namespace = "", +// type = "NPM", +// version = "3.1.0-beta.2", +// dependencies = listOf( +// PackageReferenceDto( +// name = "vite", +// namespace = "", +// type = "NPM", +// version = "3.1.0-beta.1" +// ), +// ) +// ), +// PackageReferenceDto( +// name = "vite", +// namespace = "", +// type = "NPM", +// version = "5.0.4" +// ), +// ) +// ) +// +// val artifact = artifactService.directDependencyPackageReferenceToArtifact( +// packageDto +// ) +// assertEquals( 3, artifact?.transitiveDependencies?.count()) +// assertEquals( 1, artifact?.transitiveDependencies?.get(1)?.transitiveDependencies?.count()) } } \ No newline at end of file diff --git a/src/test/kotlin/dependencies/DependencyGraphServiceTest.kt b/src/test/kotlin/dependencies/DependencyGraphServiceTest.kt index 480ed66..72d1f45 100644 --- a/src/test/kotlin/dependencies/DependencyGraphServiceTest.kt +++ b/src/test/kotlin/dependencies/DependencyGraphServiceTest.kt @@ -2,6 +2,10 @@ package dependencies import artifact.model.ArtifactVersion import http.deps.DepsClient +import http.deps.model.DepsTreeResponseDto +import http.deps.model.Edge +import http.deps.model.Node +import http.deps.model.VersionKeyX import io.mockk.coEvery import io.mockk.mockk import kotlinx.coroutines.test.runTest @@ -35,7 +39,7 @@ class DependencyGraphServiceTest { val refConfig = DependencyReference(2, dependencies = sortedSetOf(refLang, refCollections, refCollectionsLast)) val refCsv = DependencyReference(3, dependencies = sortedSetOf(refConfig)) val fragments = sortedSetOf(DependencyGraph.DEPENDENCY_REFERENCE_COMPARATOR, refCsv) - val scopeMap = mapOf("scope" to listOf(RootDependencyIndex(3))) + val scopeMap = mapOf("scope" to listOf(RootDependencyIndex(3), RootDependencyIndex(1))) return DependencyGraph(ids, fragments, scopeMap) } @@ -46,12 +50,12 @@ class DependencyGraphServiceTest { val mockDepsClient = mockk() coEvery { mockDepsClient.getVersionsForPackage(any(), any(), any()) } returns listOf( - ArtifactVersion(versionNumber = "3.11", releaseDate = 0L), - ArtifactVersion(versionNumber = "4.4.3", releaseDate = 0L), - ArtifactVersion(versionNumber = "2.4", releaseDate = 0L), - ArtifactVersion(versionNumber = "5", releaseDate = 0L), - ArtifactVersion(versionNumber = "2.11", releaseDate = 0L), - ArtifactVersion(versionNumber = "5.1.2", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "3.11", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "4.4.3", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "2.4", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "5", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "2.11", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "5.1.2", releaseDate = 0L), ) coEvery { mockDepsClient.getDepsForPackage(any(), any(), any(), any()) } returns null @@ -70,7 +74,7 @@ class DependencyGraphServiceTest { val transformedGraph = transformedGraphs.first().graph.values.first() - assertEquals(5, transformedGraph.nodes.count()) + assertEquals(6, transformedGraph.nodes.count()) // Check if the edges connect the correct nodes which are linked to the correct artifacts assertEquals(4, transformedGraph.edges.count()) @@ -80,7 +84,7 @@ class DependencyGraphServiceTest { assertEquals("commons-configuration", rootArtifact.artifactId) val directDeps = transformedGraph.linkedDirectDependencies - assertEquals(1, directDeps.count()) + assertEquals(2, directDeps.count()) assertEquals(1, directDeps.first().children.count()) assertEquals(3, directDeps.first().children.first().children.count()) } @@ -90,15 +94,59 @@ class DependencyGraphServiceTest { val mockDepsClient = mockk() coEvery { mockDepsClient.getVersionsForPackage(any(), any(), any()) } returns listOf( - ArtifactVersion(versionNumber = "3.11", releaseDate = 0L), - ArtifactVersion(versionNumber = "4.4.3", releaseDate = 0L), - ArtifactVersion(versionNumber = "2.4", releaseDate = 0L), - ArtifactVersion(versionNumber = "5", releaseDate = 0L), - ArtifactVersion(versionNumber = "2.11", releaseDate = 0L), - ArtifactVersion(versionNumber = "5.1.2", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "3.11", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "3.12", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "4.4.3", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "2.4", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "5", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "2.11", releaseDate = 0L), + ArtifactVersion.create(versionNumber = "5.1.2", releaseDate = 0L), ) coEvery { mockDepsClient.getDepsForPackage(any(), any(), any(), any()) } returns null + coEvery { + mockDepsClient.getDepsForPackage( + ecosystem = "npm", + groupId = "org.apache.commons", + artifactId = "commons-lang3", + version = "3.12.0" + ) + } returns DepsTreeResponseDto( + nodes = listOf( + Node( + bundled = false, + relation = "", + versionKey = VersionKeyX( + name = "org.apache.commons/commons-lang3", + system = "npm", + version = "3.12" + ) + ), + Node( + bundled = false, + relation = "", + versionKey = VersionKeyX( + name = "org.apache.commons/commons-configuration", // existing artifact + system = "npm", + version = "2.11" + ) + ), + Node( + bundled = false, + relation = "", + versionKey = VersionKeyX( + name = "new/artifact", // new artifact + system = "npm", + version = "5.1.2" + ) + ), + ), + edges = mutableListOf( + Edge(fromNode = 0, toNode = 1, requirement = null), + Edge(fromNode = 0, toNode = 2, requirement = null), + ), + error = null + ) val dependencyGraphService = DependencyGraphService(depsClient = mockDepsClient) val graph = setupTree() @@ -107,5 +155,7 @@ class DependencyGraphServiceTest { mapOf("npm" to graph) ) + println(transformedGraphs.first().graphs) + } } \ No newline at end of file