From cda894b9aba905af75d256169648f60821b5f334 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 20 Dec 2022 18:21:21 +0100 Subject: [PATCH] stop using `Pkg.Types.Project` (#78) --- Project.toml | 2 +- README.md | 4 ++ src/RegistryTools.jl | 8 --- src/register.jl | 158 +++++++++---------------------------------- src/types.jl | 28 +++++++- src/utils.jl | 4 +- test/regedit.jl | 46 +++++-------- 7 files changed, 80 insertions(+), 170 deletions(-) diff --git a/Project.toml b/Project.toml index e5cd517..7a8ffaf 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "RegistryTools" uuid = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4" authors = ["Stefan Karpinski "] -version = "1.9.1" +version = "2.0.0" [deps] AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f" diff --git a/README.md b/README.md index 22bb1d3..d74e2ee 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,7 @@ [![codecov](https://codecov.io/gh/JuliaRegistries/RegistryTools.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaRegistries/RegistryTools.jl) Functionality for modifying Julia registry files. + +#### Updating to 2.0 + +RegistryTools.jl changed the API in 2.0 from taking a `Pkg.Types.Project` to just taking a path to a project file instead. diff --git a/src/RegistryTools.jl b/src/RegistryTools.jl index d362f21..6a10ae3 100644 --- a/src/RegistryTools.jl +++ b/src/RegistryTools.jl @@ -11,14 +11,6 @@ using Pkg: Pkg, TOML, GitTools using UUIDs const DEFAULT_REGISTRY_URL = "https://github.com/JuliaRegistries/General" -const PKG_HAS_WEAK = hasfield(Pkg.Types.Project, :_deps_weak) - -function __init__() - if !PKG_HAS_WEAK - @warn "Running Registrator on a Julia version that does not support weak dependencies. " * - "Weak dependencies will not be registered." - end -end include("Compress.jl") include("builtin_pkgs.jl") diff --git a/src/register.jl b/src/register.jl index 5c16faf..124e203 100644 --- a/src/register.jl +++ b/src/register.jl @@ -1,59 +1,3 @@ -""" -Given a remote repo URL and a git tree spec, get a `Project` object -for the project file in that tree and a hash string for the tree. -""" -# function get_project(remote_url::AbstractString, tree_spec::AbstractString) -# # TODO?: use raw file downloads for GitHub/GitLab -# mktempdir(mkpath("packages")) do tmp -# # bare clone the package repo -# @debug("bare clone the package repo") -# repo = LibGit2.clone(remote_url, joinpath(tmp, "repo"), isbare=true) -# tree = try -# LibGit2.GitObject(repo, tree_spec) -# catch err -# err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow(err) -# error("$remote_url: git object $(repr(tree_spec)) could not be found") -# end -# tree isa LibGit2.GitTree || (tree = LibGit2.peel(LibGit2.GitTree, tree)) -# -# # check out the requested tree -# @debug("check out the requested tree") -# tree_path = abspath(tmp, "tree") -# GC.@preserve tree_path begin -# opts = LibGit2.CheckoutOptions( -# checkout_strategy = LibGit2.Consts.CHECKOUT_FORCE, -# target_directory = Base.unsafe_convert(Cstring, tree_path) -# ) -# LibGit2.checkout_tree(repo, tree, options=opts) -# end -# -# # look for a project file in the tree -# @debug("look for a project file in the tree") -# project_file = Pkg.Types.projectfile_path(tree_path) -# project_file !== nothing && isfile(project_file) || -# error("$remote_url: git tree $(repr(tree_spec)) has no project file") -# -# # parse the project file -# @debug("parse the project file") -# project = Pkg.Types.read_project(project_file) -# project.name === nothing && -# error("$remote_url $(repr(tree_spec)): package has no name") -# project.uuid === nothing && -# error("$remote_url $(repr(tree_spec)): package has no UUID") -# project.version === nothing && -# error("$remote_url $(repr(tree_spec)): package has no version") -# -# return project, string(LibGit2.GitHash(tree)) -# end -# end - -function getdeps(pkg) - if PKG_HAS_WEAK - return merge(pkg.deps, pkg._deps_weak) - end - return pkg.deps -end - # These can compromise the integrity of the registry and cannot be # opted out of. const mandatory_errors = [:version_exists, @@ -105,7 +49,7 @@ struct RegBranch metadata::Dict{String, Any} # "error", "warning", kind etc. - function RegBranch(pkg::Pkg.Types.Project, branch::AbstractString) + function RegBranch(pkg::Project, branch::AbstractString) new(pkg.name, pkg.version, branch, Dict{String,Any}()) end end @@ -282,7 +226,7 @@ function versionrange(lo::VersionBound, hi::VersionBound) return VersionRange(lo, hi) end -function find_package_in_registry(pkg::Pkg.Types.Project, +function find_package_in_registry(pkg::Project, registry_file::AbstractString, registry_path::AbstractString, registry_data::RegistryData, @@ -346,7 +290,7 @@ function check_package!(package_repo::AbstractString, return package_repo end -function update_package_file(pkg::Pkg.Types.Project, +function update_package_file(pkg::Project, package_repo::AbstractString, subdir::AbstractString, package_path::AbstractString) @@ -371,7 +315,7 @@ function get_versions_file(package_path::AbstractString) return filename, data end -function check_versions!(pkg::Pkg.Types.Project, +function check_versions!(pkg::Project, versions_data::Dict{String, Any}, status::ReturnStatus) versions = sort!([VersionNumber(v) for v in keys(versions_data)]) @@ -379,7 +323,7 @@ function check_versions!(pkg::Pkg.Types.Project, return versions end -function update_versions_file(pkg::Pkg.Types.Project, +function update_versions_file(pkg::Project, versions_file::AbstractString, versions_data::Dict{String, Any}, tree_hash::AbstractString) @@ -408,12 +352,10 @@ function update_versions_file(pkg::Pkg.Types.Project, end end -function check_deps!(pkg::Pkg.Types.Project, +function check_deps!(pkg::Project, regdata::Vector{RegistryData}, status::ReturnStatus) - depses = [getdeps(pkg)] - PKG_HAS_WEAK && push!(depses, pkg.weakdeps) - for deps in depses + for deps in [pkg.deps, pkg.weakdeps] if pkg.name in keys(deps) err = :package_self_dep @debug(err) @@ -433,12 +375,10 @@ end # Versions.toml after update. This is handled with the `old_versions' # argument and the assumption that Versions.toml has been updated with # the new version before calling this function. -function update_deps_file(pkg::Pkg.Types.Project, +function update_deps_file(pkg::Project, package_path::AbstractString, old_versions::Vector{VersionNumber}) - file_depses = [("Deps.toml", getdeps(pkg))] - PKG_HAS_WEAK && push!(file_depses, ("WeakDeps.toml", pkg.weakdeps)) - for (file, deps) in file_depses + for (file, deps) in [("Deps.toml", pkg.deps), ("WeakDeps.toml", pkg.weakdeps)] deps_file = joinpath(package_path, file) if isfile(deps_file) deps_data = Compress.load(deps_file, old_versions) @@ -453,17 +393,12 @@ function update_deps_file(pkg::Pkg.Types.Project, end end -function check_compat!(pkg::Pkg.Types.Project, +function check_compat!(pkg::Project, regdata::Vector{RegistryData}, - regpaths::Vector{String}, status::ReturnStatus) if haskey(pkg.compat, "julia") - if Base.VERSION >= v"1.7-" - ver = pkg.compat["julia"].val - else - ver = Pkg.Types.semver_spec(pkg.compat["julia"]) - end - if any(map(x -> !isempty(intersect(Pkg.Types.VersionRange("0-0.6"), x)), ver.ranges)) + ver = Pkg.Types.semver_spec(pkg.compat["julia"]) + if any(map(x -> !isempty(intersect(VersionRange("0-0.6"), x)), ver.ranges)) err = :julia_before_07_in_compat @debug(err) add!(status, err) @@ -476,9 +411,9 @@ function check_compat!(pkg::Pkg.Types.Project, # entries not mentioned in deps, nor in extras. invalid_compats = [] for name in keys(pkg.compat) - indeps = haskey(getdeps(pkg), name) - inextras = haskey(pkg.extras, name) - inweaks = PKG_HAS_WEAK ? haskey(pkg.weakdeps, name) : false + indeps = haskey(pkg.deps, name) + inextras = haskey(pkg.extras, name) + inweaks = haskey(pkg.weakdeps, name) if !(indeps || inextras || inweaks || name == "julia") push!(invalid_compats, name) end @@ -500,11 +435,11 @@ function check_compat!(pkg::Pkg.Types.Project, # anyway. for name in keys(pkg.compat) if name != "julia" - indeps = haskey(getdeps(pkg), name) + indeps = haskey(pkg.deps, name) inextras = haskey(pkg.extras, name) if indeps - uuidofdep = string(getdeps(pkg)[name]) + uuidofdep = string(pkg.deps[name]) findpackageerror!(name, uuidofdep, regdata, status) elseif inextras uuidofdep = string(pkg.extras[name]) @@ -520,14 +455,12 @@ end # See the comments for `update_deps_file` for the rationale for the # `old_versions` argument. -function update_compat_file(pkg::Pkg.Types.Project, +function update_compat_file(pkg::Project, package_path::AbstractString, old_versions::Vector{VersionNumber}) @debug("update package data: compat file") - file_depses = [("Compat.toml", getdeps(pkg))] - PKG_HAS_WEAK && push!(file_depses, ("WeakCompat.toml", pkg.weakdeps)) - for (file, deps) in file_depses + for (file, deps) in [("Compat.toml", pkg.deps), ("WeakCompat.toml", pkg.weakdeps)] compat_file = joinpath(package_path, file) if isfile(compat_file) compat_data = Compress.load(compat_file, old_versions) @@ -546,11 +479,8 @@ function update_compat_file(pkg::Pkg.Types.Project, continue end - if Base.VERSION >= v"1.7-" - spec = version.val - else - spec = Pkg.Types.semver_spec(version) - end + spec = Pkg.Types.semver_spec(version) + # The call to `map(versionrange, )` can be removed # once Pkg is updated to a version including # https://github.com/JuliaLang/Pkg.jl/pull/1181 @@ -568,6 +498,7 @@ function update_compat_file(pkg::Pkg.Types.Project, end function get_registrator_tree_sha() + # TODO: Rewrite this to not use as much Pkg internals # If Registrator is in the manifest, return its tree-sha. # Otherwise return the tree-sha for RegistryTools. manifest = Pkg.Types.Context().env.manifest @@ -581,7 +512,7 @@ function get_registrator_tree_sha() return "unknown" end -function check_and_update_registry_files(pkg, package_repo, tree_hash, +function check_and_update_registry_files(pkg::Project, package_repo, tree_hash, registry_path, registry_deps_paths, status; subdir = "") # find package in registry @@ -617,8 +548,7 @@ function check_and_update_registry_files(pkg, package_repo, tree_hash, # update package data: compat file @debug("check compat section") - regpaths = [registry_path; registry_deps_paths] - check_compat!(pkg, regdata, regpaths, status) + check_compat!(pkg, regdata, status) haserror(status) && return update_compat_file(pkg, package_path, old_versions) end @@ -633,7 +563,7 @@ errors or warnings that occurred. # Arguments * `package_repo::AbstractString`: The git repository URL for the package to be registered. If empty, keep the stored repository URL. -* `pkg::Pkg.Types.Project`: the parsed (Julia)Project.toml file for the package to be registered +* `pkg::String`: the path of the project file for the package to be registered * `tree_hash::AbstractString`: the tree hash (not commit hash) of the package revision to be registered # Keyword Arguments @@ -647,7 +577,7 @@ errors or warnings that occurred. * `gitconfig::Dict=Dict()`: dictionary of configuration options for the `git` command """ function register( - package_repo::AbstractString, pkg::Pkg.Types.Project, tree_hash::AbstractString; + package_repo::AbstractString, pkg::Union{String, Project}, tree_hash::AbstractString; registry::AbstractString = DEFAULT_REGISTRY_URL, registry_fork::AbstractString = registry, registry_deps::Vector{<:AbstractString} = AbstractString[], @@ -659,6 +589,10 @@ function register( cache::RegistryCache=REGISTRY_CACHE, gitconfig::Dict = Dict() ) + if pkg isa String + pkg = Project(pkg) + end + # get info from package registry @debug("get info from package registry") if !isempty(package_repo) @@ -736,36 +670,6 @@ function register( return set_metadata!(regbr, status) end -struct RegisterParams - package_repo::String - pkg::Pkg.Types.Project - tree_sha::String - registry::String - registry_fork::String - registry_deps::Vector{<:String} - subdir::String - push::Bool - gitconfig::Dict - - function RegisterParams(package_repo::AbstractString, - pkg::Pkg.Types.Project, - tree_sha::AbstractString; - registry::AbstractString=DEFAULT_REGISTRY_URL, - registry_fork::AbstractString=registry, - registry_deps::Vector{<:AbstractString}=[], - subdir::AbstractString="", - push::Bool=false, - gitconfig::Dict=Dict(),) - new(package_repo, pkg, tree_sha, registry, registry_fork, - registry_deps, subdir, push, gitconfig,) - end -end - -register(regp::RegisterParams) = register(regp.package_repo, regp.pkg, regp.tree_sha; - registry=regp.registry, registry_fork=regp.registry_fork, - registry_deps=regp.registry_deps, - subdir=regp.subdir, push=regp.push, gitconfig=regp.gitconfig,) - """ find_registered_version(pkg, registry_path) @@ -773,7 +677,7 @@ If the package and version specified by `pkg` exists in the registry at `registry_path`, return its tree hash. Otherwise return the empty string. """ -function find_registered_version(pkg::Pkg.Types.Project, +function find_registered_version(pkg::Project, registry_path::AbstractString) registry_file = joinpath(registry_path, "Registry.toml") registry_data = parse_registry(registry_file) diff --git a/src/types.jl b/src/types.jl index 7c2f5fb..fd7641f 100644 --- a/src/types.jl +++ b/src/types.jl @@ -28,6 +28,30 @@ function get_registry_default_branch(git::Cmd) strip(split(lines[idx], ":")[2]) end +struct Project + name::Union{Nothing, String} + uuid::Union{Nothing, UUID} + version::Union{Nothing, VersionNumber} + deps::Dict{String, String} + weakdeps::Dict{String, String} + compat::Dict{String, String} + extras::Dict{String, String} +end + +Project(project_file::String) = Project(isfile(project_file) ? TOML.parsefile(project_file) : Dict()) +function Project(d::Dict) + name = get(d, "name", nothing) + uuid = get(d, "uuid", nothing) + uuid !== nothing && (uuid = UUID(uuid)) + version = get(d, "version", nothing) + version !== nothing && (version = VersionNumber(version)) + deps = get(Dict, d, "deps") + weakdeps = get(Dict, d, "weakdeps") + compat = get(Dict, d, "compat") + extras = get(Dict, d, "extras") + Project(name, uuid, version, deps, weakdeps, compat, extras) +end + """ RegEdit.get_registry(registry_url) @@ -160,7 +184,7 @@ end # Not using `joinpath` here since we don't want backslashes in # Registry.toml when running on Windows. -function package_relpath(pkg::Pkg.Types.Project) +function package_relpath(pkg::Project) return package_relpath(pkg.name) end function package_relpath(pkg_name::AbstractString) @@ -169,7 +193,7 @@ function package_relpath(pkg_name::AbstractString) return join(path_components, path_separator) end -function Base.push!(reg::RegistryData, pkg::Pkg.Types.Project) +function Base.push!(reg::RegistryData, pkg::Project) reg.packages[string(pkg.uuid)] = Dict( "name" => pkg.name, "path" => package_relpath(pkg) ) diff --git a/src/utils.jl b/src/utils.jl index ce3cdf8..c327ec2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -12,11 +12,11 @@ function gitcmd(path::AbstractString, gitconfig::Dict) end """ - registration_branch(pkg::Pkg.Types.Project; url::AbstractString) -> String + registration_branch(pkg::RegistryTools.Project; url::AbstractString) -> String Generate the name for the registry branch used to register the package version. """ -function registration_branch(pkg::Pkg.Types.Project; url::AbstractString) +function registration_branch(pkg::Project; url::AbstractString) url_hash = bytes2hex(SHA.sha256(url)) url_hash_trunc = url_hash[1:10] return "registrator-$(lowercase(pkg.name))-$(string(pkg.uuid)[1:8])-v$(pkg.version)-$(url_hash_trunc)" diff --git a/test/regedit.jl b/test/regedit.jl index 0cd85f7..7642a8e 100644 --- a/test/regedit.jl +++ b/test/regedit.jl @@ -1,4 +1,5 @@ using RegistryTools: DEFAULT_REGISTRY_URL, + Project, parse_registry, showsafe, registration_branch, @@ -7,7 +8,6 @@ using RegistryTools: DEFAULT_REGISTRY_URL, using LibGit2 import Pkg using Pkg.TOML -using Pkg.Types: Project using Test @@ -140,7 +140,6 @@ end @testset "check_version!" begin import RegistryTools: ReturnStatus, check_version!, haserror - import Pkg.Types: Project function hascheck(status::ReturnStatus, check) return check in (c.id for c in status.triggered_checks) end @@ -276,7 +275,7 @@ end @testset "versions_file" begin import RegistryTools: get_versions_file, update_versions_file, check_versions! mktempdir(@__DIR__) do temp_dir - pkg = Project(version = v"1.0.0") + pkg = Project(Dict("version" => "1.0.0")) tree_hash = repeat("0", 40) status = ReturnStatus() @@ -307,8 +306,8 @@ end import RegistryTools: update_deps_file mktempdir(@__DIR__) do temp_dir uuid = Base.UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") - deps = Dict("Test" => uuid) - pkg = Project(version = v"1.0.0", deps = deps) + deps = Dict("Test" => string(uuid)) + pkg = Project(Dict("version" => "1.0.0", "deps" => deps)) update_versions_file(pkg, joinpath(temp_dir, "Versions.toml"), Dict{String, Any}(), repeat("0", 40)) update_deps_file(pkg, temp_dir, VersionNumber[]) @@ -325,12 +324,7 @@ end import RegistryTools: update_compat_file mktempdir(@__DIR__) do temp_dir compat = Dict("julia" => "1.3") - if Base.VERSION >= v"1.7-" - compat2 = Dict((k, Pkg.Types.Compat(Pkg.Types.semver_spec(v), v)) for (k, v) in compat) - pkg = Project(version = v"1.0.0", compat = compat2) - else - pkg = Project(version = v"1.0.0", compat = compat) - end + pkg = Project(Dict("version" => "1.0.0", "compat" => compat)) update_versions_file(pkg, joinpath(temp_dir, "Versions.toml"), Dict{String, Any}(), repeat("0", 40)) update_compat_file(pkg, temp_dir, VersionNumber[]) @@ -347,7 +341,6 @@ end import RegistryTools: RegistryData, ReturnStatus, haserror, write_registry, check_and_update_registry_files, RegBranch, set_metadata! - import Pkg.Types: read_project registry_update_tests = [ # Add one simple package. @@ -497,9 +490,7 @@ end kind = "New package", labels = String["new package"])) - # Compat for non-dependency. On Julia 1.2 and later this - # gives an error already in `read_project`, so only run it - # on Julia 1.1. + # Compat for non-dependency. (project_files = ["Example15"], skip_for_newer_julia = true, status = Symbol[:new_package, :new_package_label, @@ -566,7 +557,7 @@ end # Register some packages. for project in test_data.project_files project_file = joinpath(projects_path, "$(project).toml") - pkg = read_project(project_file) + pkg = Project(project_file) # Create status object. status = ReturnStatus() regbr = RegBranch(pkg, "") @@ -605,7 +596,7 @@ function create_and_populate_registry(registry_path, registry_name, # Add a package. projects_path = joinpath(@__DIR__, "project_files") project_file = joinpath(projects_path, "$(package).toml") - pkg = read_project(project_file) + pkg = Project(project_file) package_repo = "http://example.com/$(pkg.name).git" tree_hash = repeat("0", 40) registry_deps_paths = String[] @@ -638,7 +629,7 @@ end "Example1") @test !haserror(status) - registry2_path = joinpath(temp_dir, "Registry2") + registry2_path = joinpath(@__DIR__, temp_dir, "Registry2") status = create_and_populate_registry(registry2_path, "Registry2", "a5a8be26-c942-4674-beee-533a0e81ac1d", "Dep1") @@ -646,7 +637,7 @@ end projects_path = joinpath(@__DIR__, "project_files") project_file = joinpath(projects_path, "Example18.toml") - pkg = read_project(project_file) + pkg = Project(project_file) package_repo = "http://example.com/$(pkg.name).git" tree_hash = repeat("0", 40) registry_repo = "file://$(registry1_path)" @@ -679,9 +670,8 @@ end # Add a package. projects_path = joinpath(@__DIR__, "project_files") project_file = joinpath(projects_path, "Example1.toml") - pkg = read_project(project_file) + pkg = Project(project_file) @test find_registered_version(pkg, registry_path) == "" - package_repo = string("http://example.com/$(pkg.name).git") tree_hash = "7dd821daaae58ddf9fee53e00aa1aab33794d130" registry_deps_paths = String[] @@ -692,13 +682,13 @@ end @test find_registered_version(pkg, registry_path) == tree_hash project_file = joinpath(projects_path, "Example2.toml") - pkg = read_project(project_file) + pkg = Project(project_file) @test find_registered_version(pkg, registry_path) == "" end end @testset "Sorted (recursive) TOML.print for Versions.toml file" begin - pkg = Project(version = v"2.0.0") + pkg = Project(Dict("version" => "2.0.0")) # versions_file = "Versions.toml" version_data = Dict{String,Any}( "1.0.0" => Dict("yanked"=>true,"git-tree-sha1"=>"b04b6c6bfd3a607aa1b85362b4854ef612137f3e"), @@ -726,7 +716,6 @@ end @testset "subdirectory" begin import RegistryTools: ReturnStatus, check_and_update_registry_files - import Pkg.Types: read_project mktempdir(@__DIR__) do temp_dir registry_path = joinpath(temp_dir, "registry") @@ -738,7 +727,7 @@ end "d4e2f5cd-0f48-4704-9988-f1754e755b45") # Register a package without subdir (default). project_file = joinpath(projects_path, "Example1.toml") - pkg = read_project(project_file) + pkg = Project(project_file) status = ReturnStatus() package_repo = "http://example.com/Example1.git" check_and_update_registry_files(pkg, package_repo, tree_hash, @@ -754,7 +743,7 @@ end # Register a package with subdir. project_file = joinpath(projects_path, "Dep1.toml") - pkg = read_project(project_file) + pkg = Project(project_file) status = ReturnStatus() package_repo = "http://example.com/BigRepo.git" check_and_update_registry_files(pkg, package_repo, tree_hash, @@ -772,10 +761,8 @@ end end end -if RegistryTools.PKG_HAS_WEAK @testset "weakdeps" begin import RegistryTools: ReturnStatus, check_and_update_registry_files - import Pkg.Types: read_project temp_dir = mktempdir(; cleanup=false) mktempdir(@__DIR__) do temp_dir @@ -787,7 +774,7 @@ if RegistryTools.PKG_HAS_WEAK create_empty_registry(registry_path, "TestRegistry", "d4e2f5cd-0f48-4704-9988-f1754e755b45") project_file = joinpath(projects_path, "Example19.toml") - pkg = read_project(project_file) + pkg = Project(project_file) status = ReturnStatus() package_repo = "http://example.com/Example1.git" check_and_update_registry_files(pkg, package_repo, tree_hash, @@ -813,7 +800,6 @@ if RegistryTools.PKG_HAS_WEAK """ end end -end @testset "The `RegistryTools.package_relpath` function" begin @test RegistryTools.package_relpath("Example") == "E/Example"