diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3f3ccdca3..fa17e737ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Build Singularity run: | @@ -126,7 +126,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Fetch deps run: sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup @@ -148,7 +148,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Fetch deps run: sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup @@ -192,7 +192,7 @@ jobs: if: env.run_tests uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Fetch deps if: env.run_tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 462a72e71a..d6b4a535b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Singularity Changelog +## v3.8.2 - [2021-08-31] + +### Bug fixes + + - Fix regression when files `source`d from `%environment` contain `\` escaped + shell builtins (fixes issue with `source` of conda profile.d script). + - `singularity delete` will use the correct library service when the hostname + is specified in the `library://` URI. + - `singularity build` will use the correct library service when the hostname + is specified in the `library://` URI / definition file. + - Call `debootstrap` with correct Debian arch when it is not identical to the + value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`. + - When destination is ommitted in `%files` entry in definition file, ensure + globbed files are copied to correct resolved path. + - Return an error if `--tokenfile` used for `remote login` to an OCI registry, + as this is not supported. + - Ensure repeated `remote login` to same URI does not create duplicate entries + in `~/.singularity/remote.yaml`. + - Properly escape single quotes in Docker `CMD` / `ENTRYPOINT` translation. + - Use host uid when choosing unsquashfs flags, to avoid selinux xattr errors + with `--fakeroot` on non-EL/Fedora distributions with recent squashfs-tools. + - Updated the modified golang-x-crypto module with the latest upstream + version. + ## v3.8.1 - [2021-08-12] ### Bug Fixes @@ -13,6 +37,7 @@ - Prevent garbage collection from closing the container image file descriptor. - Update to Arch Linux pacman.conf URL and remove file size verification. + - Avoid panic when mountinfo line has a blank field. ## v3.8.0 - [2021-06-15] diff --git a/INSTALL.md b/INSTALL.md index c6231a3798..6a95e6c387 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -40,8 +40,8 @@ First, download the Golang archive to `/tmp`, then extract the archive to `/usr/ _**NOTE:** if you are updating Go from a older version, make sure you remove `/usr/local/go` before reinstalling it._ -``` -$ export VERSION=1.16.6 OS=linux ARCH=amd64 # change this as you need +```sh +$ export VERSION=1.16.7 OS=linux ARCH=amd64 # change this as you need $ wget -O /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz https://dl.google.com/go/go${VERSION}.${OS}-${ARCH}.tar.gz && \ sudo tar -C /usr/local -xzf /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz @@ -87,7 +87,7 @@ $ mkdir -p ${GOPATH}/src/github.com/hpcng && \ To build a stable version of Singularity, check out a [release tag](https://github.com/hpcng/singularity/tags) before compiling: ``` -$ git checkout v3.8.1 +$ git checkout v3.8.2 ``` ## Compiling Singularity @@ -130,7 +130,7 @@ as shown above. Then download the latest and use it to install the RPM like this: ``` -$ export VERSION=3.8.1 # this is the singularity version, change as you need +$ export VERSION=3.8.2 # this is the singularity version, change as you need $ wget https://github.com/hpcng/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \ rpmbuild -tb singularity-${VERSION}.tar.gz && \ @@ -146,7 +146,7 @@ tarball and use it to install Singularity: $ cd $GOPATH/src/github.com/hpcng/singularity && \ ./mconfig && \ make -C builddir rpm && \ - sudo rpm -ivh ~/rpmbuild/RPMS/x86_64/singularity-3.8.1*.x86_64.rpm # or whatever version you built + sudo rpm -ivh ~/rpmbuild/RPMS/x86_64/singularity-3.8.2*.x86_64.rpm # or whatever version you built ``` To build an rpm with an alternative install prefix set RPMPREFIX on the diff --git a/cmd/internal/cli/delete.go b/cmd/internal/cli/delete.go index 6e6c2b7a75..0e4174ff97 100644 --- a/cmd/internal/cli/delete.go +++ b/cmd/internal/cli/delete.go @@ -97,6 +97,18 @@ var deleteImageCmd = &cobra.Command{ sylog.Fatalf("Error parsing library ref: %v", err) } + if deleteLibraryURI != "" && imageRef.Host != "" { + sylog.Fatalf("Conflicting arguments; do not use --library with a library URI containing host name") + } + + var libraryURI string + if deleteLibraryURI != "" { + libraryURI = deleteLibraryURI + } else if imageRef.Host != "" { + // override libraryURI if ref contains host name + libraryURI = "https://" + imageRef.Host + } + r := fmt.Sprintf("%s:%s", imageRef.Path, imageRef.Tags[0]) if !deleteForce { @@ -109,7 +121,7 @@ var deleteImageCmd = &cobra.Command{ } } - libraryConfig, err := getLibraryClientConfig(deleteLibraryURI) + libraryConfig, err := getLibraryClientConfig(libraryURI) if err != nil { sylog.Fatalf("Error while getting library client config: %v", err) } diff --git a/cmd/internal/cli/remote.go b/cmd/internal/cli/remote.go index 90c6d73ad6..fb48c77772 100644 --- a/cmd/internal/cli/remote.go +++ b/cmd/internal/cli/remote.go @@ -67,7 +67,7 @@ var remoteTokenFileFlag = cmdline.Flag{ Value: &loginTokenFile, DefaultValue: "", Name: "tokenfile", - Usage: "path to the file holding token", + Usage: "path to the file holding auth token for login (remote endpoints only)", } // --no-login @@ -86,7 +86,7 @@ var remoteLoginUsernameFlag = cmdline.Flag{ DefaultValue: "", Name: "username", ShortHand: "u", - Usage: "username to authenticate with (leave it empty for token authentication)", + Usage: "username to authenticate with (required for Docker/OCI registry login)", EnvKeys: []string{"LOGIN_USERNAME"}, } @@ -97,7 +97,7 @@ var remoteLoginPasswordFlag = cmdline.Flag{ DefaultValue: "", Name: "password", ShortHand: "p", - Usage: "password to authenticate with", + Usage: "password / token to authenticate with", EnvKeys: []string{"LOGIN_PASSWORD"}, } diff --git a/docs/remote.go b/docs/remote.go index 69e690a73e..df942c7fed 100644 --- a/docs/remote.go +++ b/docs/remote.go @@ -14,23 +14,43 @@ const ( RemoteUse string = `remote [remote options...]` RemoteShort string = `Manage singularity remote endpoints, keyservers and OCI/Docker registry credentials` RemoteLong string = ` - The 'remote' commands allow you to manage Singularity remote endpoints, keyservers - and OCI/Docker registry credentials through its subcommands. The remote configuration - is stored in $HOME/.singularity/remotes.yaml by default.` + The 'remote' command allows you to manage Singularity remote endpoints, + standalone keyservers and OCI/Docker registry credentials through its + subcommands. + + A 'remote endpoint' is the Sylabs Cloud, a Singularity Enterprise installation, + or a compatible group of services. The remote endpoint is a single address, + e.g. 'cloud.sylabs.io' through which linked library, builder and keystore + sevices will be automatically discovered. + + To configure a remote endpoint you must 'remote add' it. You can 'remote login' if + you will be performing actions needing authentication. Switch between + configured remote endpoints with the 'remote use' command. The active remote + endpoint will be used for remote builds, key operations, and 'library://' pull + and push. You can also 'remote logout' from and 'remote remove' an endpoint that + is no longer required. + + To configure credentials for OCI registries that should be used when pulling or + pushing from/to 'docker://'' or 'oras://' URIs, use the 'remote login' command + only. You do not have to 'remote add' OCI registries. To remove credentials + 'remote logout' with the same URI. You do not need to 'remote remove' OCI + credentials. + + The remote configuration is stored in $HOME/.singularity/remotes.yaml by default.` RemoteExample string = ` All group commands have their own help output: - $ singularity help remote list - $ singularity remote list` + $ singularity help remote list + $ singularity remote list` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // remote add command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RemoteAddUse string = `add [add options...] ` - RemoteAddShort string = `Create a new singularity remote endpoint` + RemoteAddShort string = `Add a new singularity remote endpoint` RemoteAddLong string = ` - The 'remote add' command allows you to create a new remote endpoint to be - be used for singularity remote services. Authentication with a newly created - endpoint will occur automatically.` + The 'remote add' command allows you to add a new remote endpoint to be + be used for singularity remote services. Authentication with a newly created + endpoint will occur automatically.` RemoteAddExample string = ` $ singularity remote add SylabsCloud cloud.sylabs.io` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -57,28 +77,39 @@ const ( // remote list command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RemoteListUse string = `list` - RemoteListShort string = `List all singularity remote endpoints and services that are configured` + RemoteListShort string = `List all singularity remote endpoints, keyservers, and OCI credentials that are configured` RemoteListLong string = ` - The 'remote list' command lists all remote endpoints configured for use. If a remote - is in use, its name will be encompassed by brackets.` + The 'remote list' command lists all remote endpoints, keyservers, and OCI registry + credentials configured for use. + + The current remote is indicated by 'YES' in the 'ACTIVE' column and can be changed + with the 'remote use' command.` RemoteListExample string = ` $ singularity remote list` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // remote login command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RemoteLoginUse string = `login [login options...] ` - RemoteLoginShort string = `Log into a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials` + RemoteLoginShort string = `Login to a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials` RemoteLoginLong string = ` The 'remote login' command allows you to set credentials for a specific endpoint, - an OCI/Docker registry or a keyserver. This command can produce a link directing you to - the token service you can use to generate a valid token. If no endpoint or registry is - specified, it will try the default remote endpoint (SylabsCloud).` + an OCI/Docker registry or a keyserver. + + If no endpoint or registry is specified, the command will login to the currently + active remote endpoint. This is cloud.sylabs.io by default.` RemoteLoginExample string = ` To log in to an endpoint: $ singularity remote login SylabsCloud To login in to a docker/OCI registry: - $ singularity remote login --username foo --password bar docker://docker.io` + $ singularity remote login --username foo docker://docker.io + $ singularity remote login --username foo oras://myregistry.example.com + + Note that many cloud OCI registries use token based authentication. The token + should be specified as the password for login. A username is still required. E.g. + when using a standard Azure identity and token to login to an ACR registry the + username '00000000-0000-0000-0000-000000000000' is required. Consult your provider + documentation for detail of their login requirements.` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // remote logout command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -87,7 +118,8 @@ const ( RemoteLogoutLong string = ` The 'remote logout' command allows you to log out from a singularity specific endpoint, an OCI/Docker registry or a keyserver. If no endpoint or service is specified, it will - try the default remote endpoint (SylabsCloud).` + logout from the current active remote endpoint. + ` RemoteLogoutExample string = ` To log out from an endpoint $ singularity remote logout SylabsCloud diff --git a/e2e/delete/delete.go b/e2e/delete/delete.go index 14b9c0be27..fbd98c317b 100644 --- a/e2e/delete/delete.go +++ b/e2e/delete/delete.go @@ -23,6 +23,7 @@ func (c ctx) testDeleteCmd(t *testing.T) { args []string agree string expectExit int + expect e2e.SingularityCmdResultOp }{ { name: "delete unauthorized arch", @@ -66,6 +67,13 @@ func (c ctx) testDeleteCmd(t *testing.T) { agree: "y", expectExit: 255, }, + { + name: "delete host in uri", + args: []string{"library://library.example.com/test/default/test:v0.0.3"}, + agree: "y", + expectExit: 255, + expect: e2e.ExpectError(e2e.ContainMatch, "dial tcp: lookup library.example.com: no such host"), + }, } for _, tt := range tests { @@ -76,7 +84,7 @@ func (c ctx) testDeleteCmd(t *testing.T) { e2e.WithCommand("delete"), e2e.WithArgs(tt.args...), e2e.WithStdin(bytes.NewBufferString(tt.agree)), - e2e.ExpectExit(tt.expectExit), + e2e.ExpectExit(tt.expectExit, tt.expect), ) } } diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index a9a822c439..7d1167e9de 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -397,6 +397,19 @@ func (c ctx) testDockerRegistry(t *testing.T) { } } +// https://github.com/sylabs/singularity/issues/233 +func (c ctx) testDockerCMDQuotes(t *testing.T) { + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("run"), + e2e.WithArgs("docker://sylabsio/issue233"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "Test run"), + ), + ) +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := ctx{ @@ -410,5 +423,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "pulls": c.testDockerPulls, "registry": c.testDockerRegistry, "whiteout symlink": c.testDockerWhiteoutSymlink, + "cmd quotes": c.testDockerCMDQuotes, } } diff --git a/e2e/env/env.go b/e2e/env/env.go index f78bfc5941..6f50e62e6b 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -444,8 +444,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "environment manipulation": c.singularityEnv, "environment option": c.singularityEnvOption, "environment file": c.singularityEnvFile, - "issue 5057": c.issue5057, // https://github.com/sylabs/hpcng/issues/5057 - "issue 5426": c.issue5426, // https://github.com/sylabs/hpcng/issues/5426 + "issue 5057": c.issue5057, // https://github.com/hpcng/issues/5057 + "issue 5426": c.issue5426, // https://github.com/hpcng/issues/5426 "issue 43": c.issue43, // https://github.com/sylabs/singularity/issues/43 + "issue 274": c.issue274, // https://github.com/sylabs/singularity/issues/274 } } diff --git a/e2e/env/regressions.go b/e2e/env/regressions.go index 6eaf3e7c5e..71d45258b5 100644 --- a/e2e/env/regressions.go +++ b/e2e/env/regressions.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path" + "path/filepath" "testing" "github.com/hpcng/singularity/e2e/internal/e2e" @@ -121,3 +122,58 @@ func (c ctx) issue43(t *testing.T) { ), ) } + +// https://github.com/sylabs/singularity/issues/274 +// The conda profile.d script must be able to be source'd from %environment. +// This has been broken by changes to mvdan.cc/sh interacting badly with our +// custom internalExecHandler. +// The test is quite heavyweight, but is warranted IMHO to ensure that conda +// environment activation works as expected, as this is a common use-case +// for SingularityCE. +func (c ctx) issue274(t *testing.T) { + imageDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "issue274-", "") + defer cleanup(t) + imagePath := filepath.Join(imageDir, "container") + + // Create a minimal conda environment on the current miniconda3 base. + // Source the conda profile.d code and activate the env from `%environment`. + def := `Bootstrap: docker +From: continuumio/miniconda3:latest + +%post + + . /opt/conda/etc/profile.d/conda.sh + conda create -n env python=3 + +%environment + + source /opt/conda/etc/profile.d/conda.sh + conda activate env +` + defFile, err := e2e.WriteTempFile(imageDir, "deffile", def) + if err != nil { + t.Fatalf("Unable to create test definition file: %v", err) + } + + c.env.RunSingularity( + t, + e2e.AsSubtest("build"), + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("build"), + e2e.WithArgs(imagePath, defFile), + e2e.ExpectExit(0), + ) + // An exec of `conda info` in the container should show environment active, no errors. + // I.E. the `%environment` section should have worked. + c.env.RunSingularity( + t, + e2e.AsSubtest("exec"), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("exec"), + e2e.WithArgs(imagePath, "conda", "info"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "active environment : env"), + e2e.ExpectError(e2e.ExactMatch, ""), + ), + ) +} diff --git a/e2e/imgbuild/imgbuild.go b/e2e/imgbuild/imgbuild.go index 13e2885bb5..2c7c4f0233 100644 --- a/e2e/imgbuild/imgbuild.go +++ b/e2e/imgbuild/imgbuild.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -1356,6 +1356,32 @@ func (c imgBuildTests) buildBindMount(t *testing.T) { } } +func (c imgBuildTests) buildLibraryHost(t *testing.T) { + e2e.EnsureImage(t, c.env) + + tmpdir, cleanup := c.tempDir(t, "build-libraryhost-test") + defer cleanup() + + // Library hostname in the From URI + // The hostname is invalid, and we should get an error to that effect. + definition := "Bootstrap: library\nFrom: library.example.com/test/test/test:latest\n" + + defFile := e2e.RawDefFile(t, tmpdir, strings.NewReader(definition)) + imagePath := filepath.Join(tmpdir, "image-libaryhost") + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("build"), + e2e.WithArgs("-F", imagePath, defFile), + e2e.PostRun(func(t *testing.T) { + os.Remove(defFile) + }), + e2e.ExpectExit(255, + e2e.ExpectError(e2e.ContainMatch, "dial tcp: lookup library.example.com: no such host"), + ), + ) +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := imgBuildTests{ @@ -1374,6 +1400,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "build and update sandbox": c.buildUpdateSandbox, // build/update sandbox "fingerprint check": c.buildWithFingerprint, // definition file includes fingerprint check "build with bind mount": c.buildBindMount, // build image with bind mount + "library host": c.buildLibraryHost, // build image with hostname in library URI "issue 3848": c.issue3848, // https://github.com/hpcng/singularity/issues/3848 "issue 4203": c.issue4203, // https://github.com/hpcng/singularity/issues/4203 "issue 4407": c.issue4407, // https://github.com/hpcng/singularity/issues/4407 diff --git a/e2e/remote/remote.go b/e2e/remote/remote.go index 17c3d59e75..9443ca50c5 100644 --- a/e2e/remote/remote.go +++ b/e2e/remote/remote.go @@ -412,17 +412,17 @@ func (c ctx) remoteTestFlag(t *testing.T) { { name: "add help", cmdArgs: []string{"add", "--help"}, - expectedOutput: "Create a new singularity remote endpoint", + expectedOutput: "Add a new singularity remote endpoint", }, { name: "list help", cmdArgs: []string{"list", "--help"}, - expectedOutput: "List all singularity remote endpoints and services that are configured", + expectedOutput: "List all singularity remote endpoints, keyservers, and OCI credentials that are configured", }, { name: "login help", cmdArgs: []string{"login", "--help"}, - expectedOutput: "Log into a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials", + expectedOutput: "Login to a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials", }, { name: "remove help", @@ -601,6 +601,62 @@ func (c ctx) remoteLoginPushPrivate(t *testing.T) { } } +// Repeated logins with same URI should not create duplicate remote.yaml entries. +// If we login twice, and logout once we should not see the URI in list. +// See https://github.com/sylabs/singularity/issues/214 +func (c ctx) remoteLoginRepeated(t *testing.T) { + e2e.EnsureRegistry(t) + e2e.EnsureImage(t, c.env) + + var ( + registry = fmt.Sprintf("oras://%s", c.env.TestRegistry) + ) + + tests := []struct { + name string + command string + args []string + expectExit int + resultOp e2e.SingularityCmdResultOp + }{ + { + name: "FirstLogin", + command: "remote login", + args: []string{"-u", e2e.DefaultUsername, "-p", e2e.DefaultPassword, registry}, + expectExit: 0, + }, + { + name: "SecondLogin", + command: "remote login", + args: []string{"-u", e2e.DefaultUsername, "-p", e2e.DefaultPassword, registry}, + expectExit: 0, + }, + { + name: "logout", + command: "remote logout", + args: []string{registry}, + expectExit: 0, + }, + { + name: "list", + command: "remote list", + expectExit: 0, + resultOp: e2e.ExpectOutput(e2e.UnwantedContainMatch, registry), + }, + } + + for _, tt := range tests { + c.env.RunSingularity( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand(tt.command), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(tt.expectExit, tt.resultOp), + ) + } +} + func (c ctx) remoteKeyserver(t *testing.T) { var ( sylabsKeyserver = "https://keys.sylabs.io" @@ -897,6 +953,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "use": c.remoteUse, "oci login basic": np(c.remoteBasicLogin), "oci login push private": np(c.remoteLoginPushPrivate), + "oci login repeated": np(c.remoteLoginRepeated), "keyserver": np(c.remoteKeyserver), "use exclusive": np(c.remoteUseExclusive), } diff --git a/go.mod b/go.mod index 8ce79b288e..f30710f74f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gorilla/handlers v1.4.0 // indirect github.com/gorilla/websocket v1.4.2 - github.com/hpcng/sif v1.5.1 + github.com/hpcng/sif v1.6.0 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kr/pty v1.1.8 github.com/opencontainers/go-digest v1.0.0 @@ -45,7 +45,7 @@ require ( github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 github.com/sylabs/json-resp v0.8.0 - github.com/sylabs/scs-build-client v0.1.6 + github.com/sylabs/scs-build-client v0.2.1 github.com/sylabs/scs-key-client v0.6.2 github.com/sylabs/scs-library-client v1.0.5 github.com/urfave/cli v1.22.5 // indirect @@ -55,7 +55,7 @@ require ( github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.6 // indirect github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c gopkg.in/yaml.v2 v2.4.0 @@ -69,5 +69,5 @@ replace ( github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible - golang.org/x/crypto => github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9 + golang.org/x/crypto => github.com/hpcng/golang-x-crypto v0.0.0-20210830200829-e6b35e3fb874 ) diff --git a/go.sum b/go.sum index b27731b12c..d9c227b86b 100644 --- a/go.sum +++ b/go.sum @@ -522,10 +522,10 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9 h1:t8KUun6NE+Kv500O2sNQ6LIFvUz7QX+RJGOVoJFlyHM= -github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9/go.mod h1:YRpvuFxEV8iMqW8qf0+X+dTRn3labfXfykTzp8hRkSg= -github.com/hpcng/sif v1.5.1 h1:ZKsjRc4Wykos3pB3ASiKXK9iep9+k78Fmo2r79xHbDE= -github.com/hpcng/sif v1.5.1/go.mod h1:Zm0L455Vvf9YZv6QyuTmQUgl9fcf/ST9r3/v9FbGLYo= +github.com/hpcng/golang-x-crypto v0.0.0-20210830200829-e6b35e3fb874 h1:mGKQ3LEDpGH3MVbBWPXw/m2xJ7s9XEnga1q5fYgK/RY= +github.com/hpcng/golang-x-crypto v0.0.0-20210830200829-e6b35e3fb874/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +github.com/hpcng/sif v1.6.0 h1:CEYAU+PY0AFmg+G8SwJJmNA4Ixs6DOcgf/pIkVd8QBA= +github.com/hpcng/sif v1.6.0/go.mod h1:czlq6ECeYaGHAYmZdKjAXB0S5VrfQXcT3Momvszz2qY= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -850,8 +850,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/sylabs/json-resp v0.7.1/go.mod h1:3O9ze5PoYxia6106m5x+5G5ohnW1AJX/FURjCNovtzA= github.com/sylabs/json-resp v0.8.0 h1:bZ932uaF220aPqT0+x/vakoaGCGNbpLCjUFm1f+JKlY= github.com/sylabs/json-resp v0.8.0/go.mod h1:bUGV9cqShOyxz7RxBq03Yt9raKGfELKrfN6Yac3lfiw= -github.com/sylabs/scs-build-client v0.1.6 h1:ZzcS8trPF+jFNovh9o6aeiJnMtClKNyrby9DPEkrAAQ= -github.com/sylabs/scs-build-client v0.1.6/go.mod h1:XFO5Aq0NXsO1A41JBNMt3tWwxmddV8UrId4wmmoLrIU= +github.com/sylabs/scs-build-client v0.2.1 h1:/Hk9SI+hNPPjuacNk0aFvi6u7tFdNVtPzz+ZdO3NMQc= +github.com/sylabs/scs-build-client v0.2.1/go.mod h1:/kuKpMv3JnFd2IXCmX1+OCYxBxv+zUUWv92Az3QvwfE= github.com/sylabs/scs-key-client v0.6.2 h1:PTdmkE50rFMrSty9usl8wnzaW+JPK0rjXF4cdIQ3MEo= github.com/sylabs/scs-key-client v0.6.2/go.mod h1:WoTk47nNRrk0uLfblxUFekxDIIVPY4l2/9evME5fgX0= github.com/sylabs/scs-library-client v1.0.5 h1:u8ueCIQaa38/B0axtKUzdWSQ6uLXL9HvLQGNkB69Oe0= diff --git a/internal/app/singularity/remote_login.go b/internal/app/singularity/remote_login.go index e093eff802..b46a8ac067 100644 --- a/internal/app/singularity/remote_login.go +++ b/internal/app/singularity/remote_login.go @@ -70,6 +70,9 @@ func RemoteLogin(usrConfigFile string, args *LoginArgs) (err error) { } } else { // services (oci registry, single keyserver etc.) + if args.Tokenfile != "" { + return fmt.Errorf("--tokenfile is only supported for login to a remote endpoint, not OCI (docker/oras) or keyservers") + } if err := c.Login(args.Name, args.Username, args.Password, args.Insecure); err != nil { return fmt.Errorf("while login to %s: %s", args.Name, err) } diff --git a/internal/pkg/build/files/copy.go b/internal/pkg/build/files/copy.go index 748455b0cc..f2bf172f66 100644 --- a/internal/pkg/build/files/copy.go +++ b/internal/pkg/build/files/copy.go @@ -18,20 +18,20 @@ import ( ) // makeParentDir ensures existence of the expected destination directory for the cp command -// based on the supplied path and the number of source paths to copy -func makeParentDir(path string, numSrcPaths int) error { +// based on the supplied path. +func makeParentDir(path string) error { _, err := os.Stat(path) if !os.IsNotExist(err) { return nil } - // if path ends with a trailing '/' or if there are multiple source paths to copy - // always ensure the full path exists as a directory because 'cp' is expecting a - // dir in these cases - if strings.HasSuffix(path, "/") || numSrcPaths > 1 { + // if path ends with a trailing '/' always ensure the full path exists as a directory + // because 'cp' is expecting a dir in these cases + if strings.HasSuffix(path, "/") { if err := os.MkdirAll(filepath.Clean(path), 0755); err != nil { return fmt.Errorf("while creating full path: %s", err) } + return nil } // only make parent directory @@ -44,7 +44,8 @@ func makeParentDir(path string, numSrcPaths int) error { // CopyFromHost should be used to copy files into the rootfs from the host fs. // src is a path relative to CWD on the host, or an absolute path on the host. -// dstRel is a destination path inside dstRootfs +// dstRel is a destination path inside dstRootfs. +// An empty dstRel "" means copy the src file to the same path in the rootfs. // All symlinks encountered in the copy will be dereferenced (cp -L behavior). func CopyFromHost(src, dstRel, dstRootfs string) error { // resolve any bash globbing in filepath @@ -53,43 +54,48 @@ func CopyFromHost(src, dstRel, dstRootfs string) error { return fmt.Errorf("while expanding source path with bash: %s: %s", src, err) } - // Resolve our destination within the container rootfs - dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRel) - if err != nil { - return fmt.Errorf("while resolving destination: %s: %s", dstRel, err) - } + for _, srcGlobbed := range paths { + // If the dstRel is "" then we are copying to the full source path, appended to the rootfs prefix + dstRelGlobbed := dstRel + if dstRel == "" { + dstRelGlobbed = srcGlobbed + } - // Create any parent dirs for dst that don't already exist - if err := makeParentDir(dstResolved, len(paths)); err != nil { - return fmt.Errorf("while creating parent dir: %v", err) - } + // Resolve our destination within the container rootfs + dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRelGlobbed) + if err != nil { + return fmt.Errorf("while resolving destination: %s: %s", dstRelGlobbed, err) + } + + // Create any parent dirs for dst that don't already exist + if err := makeParentDir(dstResolved); err != nil { + return fmt.Errorf("while creating parent dir: %v", err) + } + + args := []string{"-fLr", srcGlobbed, dstResolved} + var output, stderr bytes.Buffer + // copy each file into bundle rootfs + copy := exec.Command("/bin/cp", args...) + copy.Stdout = &output + copy.Stderr = &stderr + if err := copy.Run(); err != nil { + return fmt.Errorf("while copying %s to %s: %v: %s", paths, dstResolved, args, stderr.String()) + } - args := []string{"-fLr"} - // append file(s) to be copied - args = append(args, paths...) - // append dst as last arg - args = append(args, dstResolved) - - var output, stderr bytes.Buffer - // copy each file into bundle rootfs - copy := exec.Command("/bin/cp", args...) - copy.Stdout = &output - copy.Stderr = &stderr - if err := copy.Run(); err != nil { - return fmt.Errorf("while copying %s to %s: %s: %s", paths, dstResolved, err, stderr.String()) } return nil } // CopyFromStage should be used to copy files into the rootfs from a previous stage. -// The srcRel and dstRel are src / dst paths relative to the srcRootfs and dstRootfs. +// The src and dst are paths relative to the srcRootfs and dstRootfs. +// An empty dst "" means copy the src file to the same path in the dst rootfs. // Symlinks are only dereferenced for the specified source or files that resolve // directly from a specified glob pattern. Any additional links inside a directory // being copied are not dereferenced. -func CopyFromStage(srcRel, dstRel, srcRootfs, dstRootfs string) error { +func CopyFromStage(src, dst, srcRootfs, dstRootfs string) error { // An absolute path is required for globbing... but with no symlink resolution or // path cleaning yet. - srcAbs := joinKeepSlash(srcRootfs, srcRel) + srcAbs := joinKeepSlash(srcRootfs, src) // resolve any bash globbing in filepath paths, err := expandPath(srcAbs) @@ -107,24 +113,31 @@ func CopyFromStage(srcRel, dstRel, srcRootfs, dstRootfs string) error { if err != nil { return fmt.Errorf("while resolving source: %s: %s", srcGlobbedRel, err) } + + // If the dst is "" then we are copying to the same path in dstRootfs, as src is in srcRootfs. + dstGlobbed := dst + if dst == "" { + dstGlobbed = srcGlobbedRel + } + // Resolve the destination path, keeping any final slash + dstResolved, err := secureJoinKeepSlash(dstRootfs, dstGlobbed) + if err != nil { + return fmt.Errorf("while resolving destination: %s: %s", dstGlobbed, err) + } + // Create any parent dirs for dstResolved that don't already exist. + if err := makeParentDir(dstResolved); err != nil { + return fmt.Errorf("while creating parent dir: %v", err) + } + // If we are copying into a directory then we must use the original source filename, // for the destination filename, not the one that was resolved out. // I.E. if copying `/opt/view` to `/opt/` where `/opt/view links-> /opt/.view/abc123` // we want to create `/opt/view` in the dest, not `/opt/abc123`. - dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRel) - if err != nil { - return fmt.Errorf("while resolving destination: %s: %s", dstRel, err) - } if fs.IsDir(dstResolved) { _, srcName := path.Split(srcGlobbedRel) dstResolved = path.Join(dstResolved, srcName) } - // Create any parent dirs for dst that don't already exist. - if err := makeParentDir(dstResolved, len(paths)); err != nil { - return fmt.Errorf("while creating parent dir: %v", err) - } - // Set flags for cp to perform a recursive copy without further symlink dereference. args := []string{"-fr", srcResolved, dstResolved} var output, stderr bytes.Buffer diff --git a/internal/pkg/build/files/copy_test.go b/internal/pkg/build/files/copy_test.go index a7a55bae68..95d2a1ede1 100644 --- a/internal/pkg/build/files/copy_test.go +++ b/internal/pkg/build/files/copy_test.go @@ -25,37 +25,16 @@ func TestMakeParentDir(t *testing.T) { }{ { name: "basic", - srcNum: 1, path: "basic/path", parent: true, }, { name: "trailing slash", - srcNum: 1, path: "trailing/slash/", parent: false, }, - { - name: "multiple", - srcNum: 2, - path: "multiple/files", - parent: false, - }, - { - name: "multiple trailing slash", - srcNum: 2, - path: "multiple/trailing/slash/", - parent: false, - }, { name: "exists", - srcNum: 1, - path: "", // this will create a path of just the testdir, which will always exist - parent: false, - }, - { - name: "exists multiple", - srcNum: 2, path: "", // this will create a path of just the testdir, which will always exist parent: false, }, @@ -73,7 +52,7 @@ func TestMakeParentDir(t *testing.T) { // concatenate test path with directory, do not use a join function so that we do not remove a trailing slash path := dir + "/" + tt.path - if err := makeParentDir(path, tt.srcNum); err != nil { + if err := makeParentDir(path); err != nil { t.Errorf("") } @@ -131,6 +110,13 @@ func TestCopyFromHost(t *testing.T) { if err := os.Mkdir(srcSpaceDir, 0755); err != nil { t.Fatal(err) } + srcGlob := filepath.Join(dir, "src*") + // Nested File (to test multi level glob) + srcFileNested := filepath.Join(dir, "srcDir/srcFileNested") + if err := ioutil.WriteFile(srcFileNested, []byte(sourceFileContent), 0644); err != nil { + t.Fatal(err) + } + srcFileNestedGlob := filepath.Join(dir, "srcDi?/srcFil?Nested") // Source Symlinks srcFileLinkAbs := filepath.Join(dir, "srcFileLinkAbs") if err := os.Symlink(srcFile, srcFileLinkAbs); err != nil { @@ -162,7 +148,7 @@ func TestCopyFromHost(t *testing.T) { name: "SrcFileNoDest", src: srcFile, dst: "", - expectPath: "srcFile", + expectPath: srcFile, expectFile: true, }, { @@ -189,7 +175,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcFileSpace", src: srcSpaceFile, - dst: "", + dst: "src File", expectPath: "src File", expectFile: true, }, @@ -207,9 +193,30 @@ func TestCopyFromHost(t *testing.T) { expectPath: "dstDir/srcFile", expectFile: true, }, + { + name: "srcFileGlobNoDest", + src: srcFileGlob, + dst: "", + expectPath: srcFile, + expectFile: true, + }, + { + name: "srcFileNestedGlob", + src: srcFileNestedGlob, + dst: "dstDir/", + expectPath: "dstDir/srcFileNested", + expectFile: true, + }, + { + name: "srcFileNestedGlobNoDest", + src: srcFileNestedGlob, + dst: "", + expectPath: srcFileNested, + expectFile: true, + }, { name: "dstRestricted", - src: srcFileGlob, + src: srcFile, // Will be restricted to `/` in the rootfs and should copy to there OK dst: "../../../../", expectPath: "srcFile", @@ -220,7 +227,7 @@ func TestCopyFromHost(t *testing.T) { name: "SrcDirNoDest", src: srcDir, dst: "", - expectPath: "srcDir", + expectPath: srcDir, expectDir: true, }, { @@ -247,7 +254,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcDirSpace", src: srcSpaceDir, - dst: "", + dst: "src Dir", expectPath: "src Dir", expectDir: true, }, @@ -265,11 +272,18 @@ func TestCopyFromHost(t *testing.T) { expectPath: "dstDir/srcDir", expectDir: true, }, + { + name: "srcDirGlobNoDest", + src: srcDirGlob, + dst: "", + expectPath: srcDir, + expectDir: true, + }, // Source is a Symlink { name: "srcFileLinkRel", src: srcFileLinkRel, - dst: "", + dst: "srcFileLinkRel", expectPath: "srcFileLinkRel", // Copied the file, not the link itself expectFile: true, @@ -277,7 +291,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcFileLinkAbs", src: srcFileLinkAbs, - dst: "", + dst: "srcFileLinkAbs", expectPath: "srcFileLinkAbs", // Copied the file, not the link itself expectFile: true, @@ -285,7 +299,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcDirLinkRel", src: srcDirLinkRel, - dst: "", + dst: "srcDirLinkRel", expectPath: "srcDirLinkRel", // Copied the dir, not the link itself expectDir: true, @@ -293,11 +307,27 @@ func TestCopyFromHost(t *testing.T) { { name: "srcDirLinkAbs", src: srcDirLinkAbs, - dst: "", + dst: "srcDirLinkAbs", expectPath: "srcDirLinkAbs", // Copied the dir, not the link itself expectDir: true, }, + // issue 261 - multiple globbed sources, with no dest + // both srcfile and srcdir should be copied for glob of "src*" + { + name: "srcDirGlobNoDestMulti1", + src: srcGlob, + dst: "", + expectPath: srcDir, + expectDir: true, + }, + { + name: "srcDirGlobNoDestMulti2", + src: srcGlob, + dst: "", + expectPath: srcFile, + expectFile: true, + }, } for _, tt := range tests { @@ -390,7 +420,7 @@ func TestCopyFromHostNested(t *testing.T) { defer os.RemoveAll(dstDir) // Copy our source innerDir over into the destination dir - if err := CopyFromHost(innerDir, "", dstDir); err != nil { + if err := CopyFromHost(innerDir, "innerDir", dstDir); err != nil { t.Errorf("unexpected failure copying directory: %s", err) } @@ -477,6 +507,11 @@ func TestCopyFromStage(t *testing.T) { if err := os.Mkdir(srcSpaceDir, 0755); err != nil { t.Fatal(err) } + // Nested File (to test multi level glob) + srcFileNested := filepath.Join(srcRoot, "srcDir/srcFileNested") + if err := ioutil.WriteFile(srcFileNested, []byte(sourceFileContent), 0644); err != nil { + t.Fatal(err) + } // Source Symlinks // Note the absolute links are absolute paths inside the srcRoot srcFileLinkAbs := filepath.Join(srcRoot, "srcFileLinkAbs") @@ -561,6 +596,27 @@ func TestCopyFromStage(t *testing.T) { expectPath: "dstDir/srcFile", expectFile: true, }, + { + name: "srcFileGlobNoDest", + srcRel: "srcF?*", + dstRel: "", + expectPath: "srcFile", + expectFile: true, + }, + { + name: "srcFileNestedGlob", + srcRel: "srcDi?/srcFil?Nested", + dstRel: "dstDir/", + expectPath: "dstDir/srcFileNested", + expectFile: true, + }, + { + name: "srcFileNestedGlobNoDest", + srcRel: "srcDi?/srcFil?Nested", + dstRel: "", + expectPath: "srcDir/srcFileNested", + expectFile: true, + }, { name: "dstRestricted", srcRel: "srcFile", @@ -627,6 +683,13 @@ func TestCopyFromStage(t *testing.T) { expectPath: "dstDir/srcDir", expectDir: true, }, + { + name: "srcDirGlobNoDest", + srcRel: "srcD?*", + dstRel: "", + expectPath: "srcDir", + expectDir: true, + }, // Source is a Symlink { name: "srcFileLinkRel", @@ -660,6 +723,22 @@ func TestCopyFromStage(t *testing.T) { // Copied the dir, not the link itself expectDir: true, }, + // issue 261 - multiple globbed sources, with no dest + // both srcfile and srcdir should be copied for glob of "src*" + { + name: "srcDirGlobNoDestMulti1", + srcRel: "src*", + dstRel: "", + expectPath: "srcDir", + expectDir: true, + }, + { + name: "srcDirGlobNoDestMulti2", + srcRel: "src*", + dstRel: "", + expectPath: "srcFile", + expectFile: true, + }, } for _, tt := range tests { diff --git a/internal/pkg/build/sources/conveyorPacker_debootstrap.go b/internal/pkg/build/sources/conveyorPacker_debootstrap.go index 11a228a69f..b6ce55b516 100644 --- a/internal/pkg/build/sources/conveyorPacker_debootstrap.go +++ b/internal/pkg/build/sources/conveyorPacker_debootstrap.go @@ -22,6 +22,19 @@ import ( "github.com/hpcng/singularity/pkg/util/namespaces" ) +// debootstrapArchs is a map of GO Archs to official Debian ports +// https://www.debian.org/ports/ +var debootstrapArchs = map[string]string{ + "386": "i386", + "amd64": "amd64", + "arm": "armhf", + "arm64": "arm64", + "ppc64le": "ppc64el", + "mipsle": "mipsel", + "mips64le": "mips64el", + "s390x": "s390x", +} + // DebootstrapConveyorPacker holds stuff that needs to be packed into the bundle type DebootstrapConveyorPacker struct { b *types.Bundle @@ -48,6 +61,12 @@ func (cp *DebootstrapConveyorPacker) Get(ctx context.Context, b *types.Bundle) ( return fmt.Errorf("you must be root to build with debootstrap") } + // Debian port arch values do not always match GOARCH values, so we need to look it up. + debArch, ok := debootstrapArchs[runtime.GOARCH] + if !ok { + return fmt.Errorf("Debian arch not known for GOARCH %s", runtime.GOARCH) + } + insideUserNs, setgroupsAllowed := namespaces.IsInsideUserNamespace(os.Getpid()) if insideUserNs && setgroupsAllowed { umountFn, err := cp.prepareFakerootEnv(ctx) @@ -60,7 +79,7 @@ func (cp *DebootstrapConveyorPacker) Get(ctx context.Context, b *types.Bundle) ( } // run debootstrap command - cmd := exec.Command(debootstrapPath, `--variant=minbase`, `--exclude=openssl,udev,debconf-i18n,e2fsprogs`, `--include=apt,`+cp.include, `--arch=`+runtime.GOARCH, cp.osversion, cp.b.RootfsPath, cp.mirrorurl) + cmd := exec.Command(debootstrapPath, `--variant=minbase`, `--exclude=openssl,udev,debconf-i18n,e2fsprogs`, `--include=apt,`+cp.include, `--arch=`+debArch, cp.osversion, cp.b.RootfsPath, cp.mirrorurl) sylog.Debugf("\n\tDebootstrap Path: %s\n\tIncludes: apt(default),%s\n\tDetected Arch: %s\n\tOSVersion: %s\n\tMirrorURL: %s\n", debootstrapPath, cp.include, runtime.GOARCH, cp.osversion, cp.mirrorurl) diff --git a/internal/pkg/build/sources/conveyorPacker_library.go b/internal/pkg/build/sources/conveyorPacker_library.go index 96045a42f2..38807326e9 100644 --- a/internal/pkg/build/sources/conveyorPacker_library.go +++ b/internal/pkg/build/sources/conveyorPacker_library.go @@ -50,13 +50,22 @@ func (cp *LibraryConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err libraryURL = customLib } - sylog.Debugf("LibraryURL: %v", libraryURL) - sylog.Debugf("LibraryRef: %v", b.Recipe.Header["from"]) - imageRef, err := library.NormalizeLibraryRef(b.Recipe.Header["from"]) if err != nil { return fmt.Errorf("error parsing libraryRef: %v", err) } + + if imageRef.Host != "" { + if b.Opts.NoHTTPS { + libraryURL = "http://" + imageRef.Host + } else { + libraryURL = "https://" + imageRef.Host + } + } + + sylog.Debugf("LibraryURL: %v", libraryURL) + sylog.Debugf("LibraryRef: %v", imageRef.String()) + libraryConfig := &client.Config{ BaseURL: libraryURL, AuthToken: authToken, diff --git a/internal/pkg/build/sources/conveyorPacker_oci.go b/internal/pkg/build/sources/conveyorPacker_oci.go index c627f0ad25..690bc5fc15 100644 --- a/internal/pkg/build/sources/conveyorPacker_oci.go +++ b/internal/pkg/build/sources/conveyorPacker_oci.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -324,7 +324,9 @@ func (cp *OCIConveyorPacker) insertRunScript() (err error) { } if len(cp.imgConfig.Entrypoint) > 0 { - _, err = f.WriteString("OCI_ENTRYPOINT='" + shell.ArgsQuoted(cp.imgConfig.Entrypoint) + "'\n") + _, err = f.WriteString("OCI_ENTRYPOINT='" + + shell.EscapeSingleQuotes(shell.ArgsQuoted(cp.imgConfig.Entrypoint)) + + "'\n") if err != nil { return } @@ -336,7 +338,9 @@ func (cp *OCIConveyorPacker) insertRunScript() (err error) { } if len(cp.imgConfig.Cmd) > 0 { - _, err = f.WriteString("OCI_CMD='" + shell.ArgsQuoted(cp.imgConfig.Cmd) + "'\n") + _, err = f.WriteString("OCI_CMD='" + + shell.EscapeSingleQuotes(shell.ArgsQuoted(cp.imgConfig.Cmd)) + + "'\n") if err != nil { return } diff --git a/internal/pkg/build/stage.go b/internal/pkg/build/stage.go index e76345883d..030b5199ee 100644 --- a/internal/pkg/build/stage.go +++ b/internal/pkg/build/stage.go @@ -174,11 +174,6 @@ func (s *stage) copyFilesFrom(b *Build) error { sylog.Warningf("Attempt to copy file with no name, skipping.") continue } - // dest = source if not specified - if transfer.Dst == "" { - transfer.Dst = transfer.Src - } - // copy each file into bundle rootfs sylog.Infof("Copying %v to %v", transfer.Src, transfer.Dst) if err := files.CopyFromStage(transfer.Src, transfer.Dst, srcRootfsPath, dstRootfsPath); err != nil { @@ -207,10 +202,6 @@ func (s *stage) copyFiles() error { sylog.Warningf("Attempt to copy file with no name, skipping.") continue } - // dest = source if not specified - if transfer.Dst == "" { - transfer.Dst = transfer.Src - } // copy each file into bundle rootfs sylog.Infof("Copying %v to %v", transfer.Src, transfer.Dst) if err := files.CopyFromHost(transfer.Src, transfer.Dst, s.b.RootfsPath); err != nil { diff --git a/internal/pkg/cgroups/cgroups_linux_test.go b/internal/pkg/cgroups/cgroups_linux_test.go index aeeb32ee07..d69c8bd16d 100644 --- a/internal/pkg/cgroups/cgroups_linux_test.go +++ b/internal/pkg/cgroups/cgroups_linux_test.go @@ -12,7 +12,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "strconv" "strings" "testing" @@ -55,9 +54,12 @@ func TestCgroups(t *testing.T) { manager := &Manager{Pid: pid, Path: path} cgroupsToml := "example/cgroups.toml" - // A different pageSize is needed on pp64le - if runtime.GOARCH == "ppc64le" { - cgroupsToml = "example/cgroups-ppc64le.toml" + // Some systems, e.g. ppc64le may not have a 2MB page size, so don't + // apply a 2MB hugetlb limit if that's the case. + _, err = os.Stat("/sys/fs/cgroup/hugetlb/hugetlb.2MB.limit_in_bytes") + if os.IsNotExist(err) { + t.Log("No hugetlb.2MB.limit_in_bytes - using alternate cgroups test file") + cgroupsToml = "example/cgroups-no-hugetlb.toml" } if err := manager.ApplyFromFile(cgroupsToml); err != nil { diff --git a/internal/pkg/cgroups/example/cgroups-ppc64le.toml b/internal/pkg/cgroups/example/cgroups-no-hugetlb.toml similarity index 97% rename from internal/pkg/cgroups/example/cgroups-ppc64le.toml rename to internal/pkg/cgroups/example/cgroups-no-hugetlb.toml index 539bd9b79d..0f0d99f155 100644 --- a/internal/pkg/cgroups/example/cgroups-ppc64le.toml +++ b/internal/pkg/cgroups/example/cgroups-no-hugetlb.toml @@ -102,9 +102,9 @@ # Hugetlb limit (in bytes) # - pagesize: the hugepage size # - limit: the limit of "hugepagesize" hugetlb usage -[[hugepageLimits]] - limit = 9223372036854771712 - pageSize = "16MB" +# [[hugepageLimits]] +# limit = 9223372036854771712 +# pageSize = "16MB" # Network restriction configuration diff --git a/internal/pkg/remote/credential/login_handler.go b/internal/pkg/remote/credential/login_handler.go index 5f5f41df45..01015087b6 100644 --- a/internal/pkg/remote/credential/login_handler.go +++ b/internal/pkg/remote/credential/login_handler.go @@ -44,7 +44,7 @@ func init() { // function just return the password provided as argument. func ensurePassword(password string) (string, error) { if password == "" { - question := "Password (or token when username is empty): " + question := "Password / Token: " input, err := interactive.AskQuestionNoEcho(question) if err != nil { return "", fmt.Errorf("failed to read password: %s", err) diff --git a/internal/pkg/remote/remote.go b/internal/pkg/remote/remote.go index b30daf4f34..411e494bd1 100644 --- a/internal/pkg/remote/remote.go +++ b/internal/pkg/remote/remote.go @@ -257,6 +257,18 @@ func (c *Config) Login(uri, username, password string, insecure bool) error { if err != nil { return err } + + // Remove any existing remote.yaml entry for the same URI. + // Older versions of Singularity can create duplicate entries with same URI, + // so loop must handle removing multiple matches (#214). + for i := 0; i < len(c.Credentials); i++ { + cred := c.Credentials[i] + if remoteutil.SameURI(cred.URI, uri) { + c.Credentials = append(c.Credentials[:i], c.Credentials[i+1:]...) + i = -1 + } + } + c.Credentials = append(c.Credentials, credConfig) return nil } @@ -266,13 +278,16 @@ func (c *Config) Logout(uri string) error { if err := credential.Manager.Logout(uri); err != nil { return err } - for i, cred := range c.Credentials { + // Older versions of Singularity can create duplicate entries with same URI, + // so loop must handle removing multiple matches (#214). + for i := 0; i < len(c.Credentials); i++ { + cred := c.Credentials[i] if remoteutil.SameURI(cred.URI, uri) { c.Credentials = append(c.Credentials[:i], c.Credentials[i+1:]...) - return nil + i = -1 } } - return fmt.Errorf("%s is not configured", uri) + return nil } // Rename an existing remote diff --git a/internal/pkg/runtime/engine/singularity/process_linux.go b/internal/pkg/runtime/engine/singularity/process_linux.go index 19eb0fd205..580db12e2d 100644 --- a/internal/pkg/runtime/engine/singularity/process_linux.go +++ b/internal/pkg/runtime/engine/singularity/process_linux.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -646,7 +646,7 @@ func injectEnvHandler(senv map[string]string) interpreter.OpenHandler { b.WriteString(fmt.Sprintf(snippet, key, value+":/.singularity.d/libs")) continue } - b.WriteString(fmt.Sprintf(snippet, key, shell.EscapeQuotes(value))) + b.WriteString(fmt.Sprintf(snippet, key, shell.EscapeDoubleQuotes(value))) } }) diff --git a/internal/pkg/util/shell/escape.go b/internal/pkg/util/shell/escape.go index bf1141f2c3..d0e531880e 100644 --- a/internal/pkg/util/shell/escape.go +++ b/internal/pkg/util/shell/escape.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018, Sylabs, Inc. All rights reserved. +// Copyright (c) 2018-2021 Sylabs, Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please // consult LICENSE.md file distributed with the sources of this project regarding // your rights to use or distribute this software. @@ -16,7 +16,8 @@ func ArgsQuoted(a []string) (quoted string) { return } -// Escape performs escaping of shell quotes, backticks and $ characters +// Escape performs escaping of shell double quotes, backticks and $ characters. +// Does not escape single quotes - apply EscapeSingleQuotes separately for this. func Escape(s string) string { escaped := strings.Replace(s, `\`, `\\`, -1) escaped = strings.Replace(escaped, `"`, `\"`, -1) @@ -25,7 +26,12 @@ func Escape(s string) string { return escaped } -// EscapeQuotes performs escaping of double quotes only -func EscapeQuotes(s string) string { +// EscapeDoubleQuotes performs shell escaping of double quotes only +func EscapeDoubleQuotes(s string) string { return strings.Replace(s, `"`, `\"`, -1) } + +// EscapeSingleQuotes performs shell escaping of single quotes only +func EscapeSingleQuotes(s string) string { + return strings.Replace(s, `'`, `'"'"'`, -1) +} diff --git a/internal/pkg/util/shell/escape_test.go b/internal/pkg/util/shell/escape_test.go index 19526b9223..5cfc9ad05b 100644 --- a/internal/pkg/util/shell/escape_test.go +++ b/internal/pkg/util/shell/escape_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018, Sylabs, Inc. All rights reserved. +// Copyright (c) 2018-2021 Sylabs, Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please // consult LICENSE file distributed with the sources of this project regarding // your rights to use or distribute this software. @@ -52,8 +52,8 @@ func TestEscape(t *testing.T) { } -func TestEscapeQuotes(t *testing.T) { - var escapeQuotesTests = []struct { +func TestEscapeDoubleQuotes(t *testing.T) { + escapeQuotesTests := []struct { input string expected string }{ @@ -64,7 +64,27 @@ func TestEscapeQuotes(t *testing.T) { for _, test := range escapeQuotesTests { t.Run(test.input, func(t *testing.T) { - escaped := EscapeQuotes(test.input) + escaped := EscapeDoubleQuotes(test.input) + if escaped != test.expected { + t.Errorf("got %s, expected %s", escaped, test.expected) + } + }) + } +} + +func TestEscapeSingleQuotes(t *testing.T) { + escapeQuotesTests := []struct { + input string + expected string + }{ + {`Hello`, `Hello`}, + {`'Hello'`, `'"'"'Hello'"'"'`}, + {`Hell'o`, `Hell'"'"'o`}, + } + + for _, test := range escapeQuotesTests { + t.Run(test.input, func(t *testing.T) { + escaped := EscapeSingleQuotes(test.input) if escaped != test.expected { t.Errorf("got %s, expected %s", escaped, test.expected) } diff --git a/internal/pkg/util/shell/interpreter/interpreter.go b/internal/pkg/util/shell/interpreter/interpreter.go index d4d677c426..ad9ecb5344 100644 --- a/internal/pkg/util/shell/interpreter/interpreter.go +++ b/internal/pkg/util/shell/interpreter/interpreter.go @@ -166,7 +166,16 @@ func (s *Shell) internalExecHandler() interp.ExecHandlerFunc { if err != nil { return err } - return s.runner.Run(ctx, node) + + // We run individual syntax.Stmt rather than the parsed syntax.File as the latter + // implies an `exit`, and causes https://github.com/sylabs/singularity/issues/274 + // with the exit/trap changes in https://github.com/mvdan/sh/commit/fb5052e7a0109c9ef5553a310c05f3b8c04cca5f + for _, stmt := range node.Stmts { + if err := s.runner.Run(ctx, stmt); err != nil { + return err + } + } + return nil } } return defaultExecHandler(ctx, args) diff --git a/internal/pkg/util/uri/uri.go b/internal/pkg/util/uri/uri.go index 284ff772c1..51824eb7e2 100644 --- a/internal/pkg/util/uri/uri.go +++ b/internal/pkg/util/uri/uri.go @@ -54,7 +54,7 @@ func IsValid(source string) (valid bool, err error) { } // GetName turns a transport:ref URI into a name containing the top-level identifier -// of the image. For example, docker://godlovedc/lolcow returns lolcow +// of the image. For example, docker://sylabsio/lolcow returns lolcow // // Returns "" when not in transport:ref format func GetName(uri string) string { diff --git a/internal/pkg/util/uri/uri_test.go b/internal/pkg/util/uri/uri_test.go index 2d23887298..2057b8166b 100644 --- a/internal/pkg/util/uri/uri_test.go +++ b/internal/pkg/util/uri/uri_test.go @@ -17,8 +17,8 @@ func Test_GetName(t *testing.T) { }{ {"docker basic", "docker://ubuntu", "ubuntu_latest.sif"}, {"docker scoped", "docker://user/image", "image_latest.sif"}, - {"dave's magical lolcow", "docker://godlovedc/lolcow", "lolcow_latest.sif"}, - {"docker w/ tags", "docker://godlovedc/lolcow:3.7", "lolcow_3.7.sif"}, + {"dave's magical lolcow", "docker://sylabs.io/lolcow", "lolcow_latest.sif"}, + {"docker w/ tags", "docker://sylabs.io/lolcow:3.7", "lolcow_3.7.sif"}, } for _, tt := range tests { @@ -39,8 +39,8 @@ func Test_Split(t *testing.T) { }{ {"docker basic", "docker://ubuntu", "docker", "//ubuntu"}, {"docker scoped", "docker://user/image", "docker", "//user/image"}, - {"dave's magical lolcow", "docker://godlovedc/lolcow", "docker", "//godlovedc/lolcow"}, - {"docker with tags", "docker://godlovedc/lolcow:latest", "docker", "//godlovedc/lolcow:latest"}, + {"dave's magical lolcow", "docker://sylabs.io/lolcow", "docker", "//sylabs.io/lolcow"}, + {"docker with tags", "docker://sylabs.io/lolcow:latest", "docker", "//sylabs.io/lolcow:latest"}, {"library basic", "library://image", "library", "//image"}, {"library scoped", "library://collection/image", "library", "//collection/image"}, {"without transport", "ubuntu", "", "ubuntu"}, diff --git a/pkg/image/unpacker/squashfs.go b/pkg/image/unpacker/squashfs.go index ea13f313ba..9bfdf3ceaa 100644 --- a/pkg/image/unpacker/squashfs.go +++ b/pkg/image/unpacker/squashfs.go @@ -110,7 +110,11 @@ func (s *Squashfs) extract(files []string, reader io.Reader, dest string) (err e // 2. Must check (user) xattrs are supported on the FS as unsquashfs >=4.4 will give a non-zero error code if // it cannot set them, e.g. on tmpfs (#5668) opts := []string{} - rootless := os.Geteuid() != 0 + hostuid, err := namespaces.HostUID() + if err != nil { + return fmt.Errorf("could not get host UID: %s", err) + } + rootless := hostuid != 0 // Do we support user xattrs? ok, err := TestUserXattr(filepath.Dir(dest)) @@ -129,15 +133,11 @@ func (s *Squashfs) extract(files []string, reader io.Reader, dest string) (err e // non real root users could not create pseudo devices so we compare // the host UID (to include fake root user) and apply a filter at extraction (#5690) - hostuid, err := namespaces.HostUID() - if err != nil { - return fmt.Errorf("could not get host UID: %s", err) - } filter := "" // exclude dev directory only if there no specific files provided for extraction // as globbing won't work with POSIX regex enabled - if hostuid != 0 && len(files) == 0 { + if rootless && len(files) == 0 { sylog.Debugf("Excluding /dev directory during root filesystem extraction (non root user)") // filter requires POSIX regex opts = append(opts, "-r") diff --git a/pkg/util/fs/proc/proc_linux_test.go b/pkg/util/fs/proc/proc_linux_test.go index 765f8954b0..5b0452ba58 100644 --- a/pkg/util/fs/proc/proc_linux_test.go +++ b/pkg/util/fs/proc/proc_linux_test.go @@ -72,7 +72,8 @@ var mountInfoData = `22 28 0:21 / /sys rw,nosuid,nodev,noexec,relatime shared:7 90 47 0:47 / /proc/sys/fs/binfmt_misc rw,relatime shared:34 - binfmt_misc binfmt_misc rw 381 26 0:54 / /run/user/1000 rw,nosuid,nodev,relatime shared:245 - tmpfs tmpfs rw,size=1635868k,mode=700,uid=1000,gid=1000 363 381 0:52 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:233 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 -579 28 0:65 / /tmp/squashfs rw,relatime - squashfs /dev/loop0 rw` +579 28 0:65 / /tmp/squashfs rw,relatime - squashfs /dev/loop0 rw +377 314 0:80 / /dev/shm rw,relatime - tmpfs rw,mode=750,uid=174988` var expectedMap = map[string][]string{ "/": { @@ -205,6 +206,19 @@ func TestGetMountInfo(t *testing.T) { SuperOptions: []string{"rw"}, Options: []string{"rw", "relatime"}, }, + // github.com/hpcng/singularity/issues/6048 + // Empty 9th field (source) in mountinfo line + { + ParentID: "314", + ID: "377", + Dev: "0:80", + Root: "/", + Fields: "", + FSType: "tmpfs", + Source: "", + SuperOptions: []string{"rw", "rw,mode=750,uid=174988"}, + Options: []string{"rw", "relatime"}, + }, } for _, c := range check {