From d5541a86ae436ef25a13a0bf3479efb4e68849ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 05:20:49 +0000 Subject: [PATCH 01/30] Bump github.com/opencontainers/selinux from 1.8.3 to 1.8.4 Bumps [github.com/opencontainers/selinux](https://github.com/opencontainers/selinux) from 1.8.3 to 1.8.4. - [Release notes](https://github.com/opencontainers/selinux/releases) - [Commits](https://github.com/opencontainers/selinux/compare/v1.8.3...v1.8.4) --- updated-dependencies: - dependency-name: github.com/opencontainers/selinux dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dce12cf52f..a83a4048c3 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/opencontainers/image-spec v1.0.2-0.20210331164927-859973e32cca github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b - github.com/opencontainers/selinux v1.8.3 + github.com/opencontainers/selinux v1.8.4 github.com/opencontainers/umoci v0.4.7 github.com/pelletier/go-toml v1.9.3 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 8df995e99b..4d8ccfc464 100644 --- a/go.sum +++ b/go.sum @@ -706,8 +706,8 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b/go. github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.8.3 h1:tzZR7AuKB5gU1+53uBkoG4XdIFGZzvJTOVoNbRQI8/4= -github.com/opencontainers/selinux v1.8.3/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= +github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= +github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/opencontainers/umoci v0.4.7 h1:mbIbtMpZ3v9oMpKaLopnWoLykgmnixeLzq51EzAX5nQ= github.com/opencontainers/umoci v0.4.7/go.mod h1:lgJ4bnwJezsN1o/5d7t/xdRPvmf8TvBko5kKYJsYvgo= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= From 22e80a349cfacb9ad266faf36dad5da619dc4314 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 05:25:37 +0000 Subject: [PATCH 02/30] chore(deps): bump github.com/google/uuid from 1.2.0 to 1.3.0 Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.2.0 to 1.3.0. - [Release notes](https://github.com/google/uuid/releases) - [Commits](https://github.com/google/uuid/compare/v1.2.0...v1.3.0) --- updated-dependencies: - dependency-name: github.com/google/uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1b3a6bc262..a198f4e8b1 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/godbus/dbus v4.1.0+incompatible // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/uuid v1.2.0 + github.com/google/uuid v1.3.0 github.com/gorilla/handlers v1.4.0 // indirect github.com/gorilla/websocket v1.4.2 github.com/hpcng/sif v1.5.1 diff --git a/go.sum b/go.sum index 4215c72482..93bd29268c 100644 --- a/go.sum +++ b/go.sum @@ -470,8 +470,9 @@ github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWU github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= From af9d9e24859154370ce8e4459c8e0796a4b472ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 05:22:39 +0000 Subject: [PATCH 03/30] chore(deps): bump github.com/containers/image/v5 from 5.15.0 to 5.15.2 Bumps [github.com/containers/image/v5](https://github.com/containers/image) from 5.15.0 to 5.15.2. - [Release notes](https://github.com/containers/image/releases) - [Commits](https://github.com/containers/image/compare/v5.15.0...v5.15.2) --- updated-dependencies: - dependency-name: github.com/containers/image/v5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 98a7e4944a..ab9cb569b9 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/containerd/containerd v1.5.5 github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 - github.com/containers/image/v5 v5.15.0 + github.com/containers/image/v5 v5.15.2 github.com/cyphar/filepath-securejoin v0.2.3 github.com/fatih/color v1.12.0 github.com/garyburd/redigo v1.6.0 // indirect diff --git a/go.sum b/go.sum index d161bb9cae..d47d195384 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -253,8 +254,8 @@ github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containers/image/v5 v5.15.0 h1:NduhN20ptHNlf0uRny5iTJa2OodB9SLMEB4hKKbzBBs= -github.com/containers/image/v5 v5.15.0/go.mod h1:gzdBcooi6AFdiqfzirUqv90hUyHyI0MMdaqKzACKr2s= +github.com/containers/image/v5 v5.15.2 h1:DKicmVr0h1HGkzs9muoErX+fVbV9sV9W5TyMy5perLE= +github.com/containers/image/v5 v5.15.2/go.mod h1:8jejVSzTDfyPwr/HXp9rri34n/vbdavYk6IzTiB3TBw= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= @@ -262,8 +263,8 @@ github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgU github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/containers/ocicrypt v1.1.2 h1:Ez+GAMP/4GLix5Ywo/fL7O0nY771gsBIigiqUm1aXz0= github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/storage v1.33.0 h1:sTk1Mfz3uSNg7cxeaDb0Ld8/UV+8pZEOQjvysjJuzX8= -github.com/containers/storage v1.33.0/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= +github.com/containers/storage v1.34.1 h1:PsBGMH7hwuQ3MOr7qTgPznFrE8ebfIbwQbg2gKvg0lE= +github.com/containers/storage v1.34.1/go.mod h1:FY2TcbfgCLMU4lYoKnlZeZXeH353TOTbpDEA+sAcqAY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -559,8 +560,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= From 84446800a5f4fbde540c1614bf994b781899bfb9 Mon Sep 17 00:00:00 2001 From: Adam Hughes Date: Tue, 10 Aug 2021 16:01:04 +0000 Subject: [PATCH 04/30] chore: bump Go to 1.16.7 --- .github/workflows/ci.yml | 8 ++++---- INSTALL.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3f3ccdca3..fa17e737ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Build Singularity run: | @@ -126,7 +126,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Fetch deps run: sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup @@ -148,7 +148,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Fetch deps run: sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup @@ -192,7 +192,7 @@ jobs: if: env.run_tests uses: actions/setup-go@v2 with: - go-version: 1.16.4 + go-version: 1.16.7 - name: Fetch deps if: env.run_tests diff --git a/INSTALL.md b/INSTALL.md index c6231a3798..cbbbe3b888 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -40,8 +40,8 @@ First, download the Golang archive to `/tmp`, then extract the archive to `/usr/ _**NOTE:** if you are updating Go from a older version, make sure you remove `/usr/local/go` before reinstalling it._ -``` -$ export VERSION=1.16.6 OS=linux ARCH=amd64 # change this as you need +```sh +$ export VERSION=1.16.7 OS=linux ARCH=amd64 # change this as you need $ wget -O /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz https://dl.google.com/go/go${VERSION}.${OS}-${ARCH}.tar.gz && \ sudo tar -C /usr/local -xzf /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz From 3e971c4feaa4c880e13c542c96d111b663c8cc74 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 3 Aug 2021 13:09:48 -0500 Subject: [PATCH 05/30] fix: use correct Debian arch for debootstrap Sometime the Debian arch string is not identical to the `runtime.GOARCH` value for a platform. Map from `runtime.GOARCH` to the Debian arch to address this. Fixes: singularity-ce #204 --- CHANGELOG.md | 7 +++++++ .../sources/conveyorPacker_debootstrap.go | 21 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 462a72e71a..9389359506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Singularity Changelog +### Changes since last release + +### Bug fixes + + - Call `debootstrap` with correct Debian arch when it is not identical to the + value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`. + ## v3.8.1 - [2021-08-12] ### Bug Fixes diff --git a/internal/pkg/build/sources/conveyorPacker_debootstrap.go b/internal/pkg/build/sources/conveyorPacker_debootstrap.go index 11a228a69f..b6ce55b516 100644 --- a/internal/pkg/build/sources/conveyorPacker_debootstrap.go +++ b/internal/pkg/build/sources/conveyorPacker_debootstrap.go @@ -22,6 +22,19 @@ import ( "github.com/hpcng/singularity/pkg/util/namespaces" ) +// debootstrapArchs is a map of GO Archs to official Debian ports +// https://www.debian.org/ports/ +var debootstrapArchs = map[string]string{ + "386": "i386", + "amd64": "amd64", + "arm": "armhf", + "arm64": "arm64", + "ppc64le": "ppc64el", + "mipsle": "mipsel", + "mips64le": "mips64el", + "s390x": "s390x", +} + // DebootstrapConveyorPacker holds stuff that needs to be packed into the bundle type DebootstrapConveyorPacker struct { b *types.Bundle @@ -48,6 +61,12 @@ func (cp *DebootstrapConveyorPacker) Get(ctx context.Context, b *types.Bundle) ( return fmt.Errorf("you must be root to build with debootstrap") } + // Debian port arch values do not always match GOARCH values, so we need to look it up. + debArch, ok := debootstrapArchs[runtime.GOARCH] + if !ok { + return fmt.Errorf("Debian arch not known for GOARCH %s", runtime.GOARCH) + } + insideUserNs, setgroupsAllowed := namespaces.IsInsideUserNamespace(os.Getpid()) if insideUserNs && setgroupsAllowed { umountFn, err := cp.prepareFakerootEnv(ctx) @@ -60,7 +79,7 @@ func (cp *DebootstrapConveyorPacker) Get(ctx context.Context, b *types.Bundle) ( } // run debootstrap command - cmd := exec.Command(debootstrapPath, `--variant=minbase`, `--exclude=openssl,udev,debconf-i18n,e2fsprogs`, `--include=apt,`+cp.include, `--arch=`+runtime.GOARCH, cp.osversion, cp.b.RootfsPath, cp.mirrorurl) + cmd := exec.Command(debootstrapPath, `--variant=minbase`, `--exclude=openssl,udev,debconf-i18n,e2fsprogs`, `--include=apt,`+cp.include, `--arch=`+debArch, cp.osversion, cp.b.RootfsPath, cp.mirrorurl) sylog.Debugf("\n\tDebootstrap Path: %s\n\tIncludes: apt(default),%s\n\tDetected Arch: %s\n\tOSVersion: %s\n\tMirrorURL: %s\n", debootstrapPath, cp.include, runtime.GOARCH, cp.osversion, cp.mirrorurl) From a1055ebbfbc12b87349b831080f917cabbba6f1c Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 30 Jul 2021 10:09:27 -0500 Subject: [PATCH 06/30] Copy globbed files to correct destination in `%files` When the destination of a line in the `%files` section is ommitted, the file should be copied to the same full path within the destination rootfs as it is in the source. Previously we naively set dst = src in this case, but src might be a globbing pattern. In this case the file copy destination was the glob pattern, not the matched path(s) after globbing. Handle a blank "" dest in the CopyFromHost / CopyFromStage functions, where globbing occurs. Copy to the correct full path after globbing. Update tests to capture this behavior. Fixes #196 --- CHANGELOG.md | 2 + internal/pkg/build/files/copy.go | 93 ++++++++++++---------- internal/pkg/build/files/copy_test.go | 110 ++++++++++++++++++-------- internal/pkg/build/stage.go | 9 --- 4 files changed, 132 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9389359506..d47c692da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Call `debootstrap` with correct Debian arch when it is not identical to the value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`. + - When destination is ommitted in `%files` entry in definition file, ensure + globbed files are copied to correct resolved path. ## v3.8.1 - [2021-08-12] diff --git a/internal/pkg/build/files/copy.go b/internal/pkg/build/files/copy.go index 748455b0cc..8811f76d8b 100644 --- a/internal/pkg/build/files/copy.go +++ b/internal/pkg/build/files/copy.go @@ -18,20 +18,20 @@ import ( ) // makeParentDir ensures existence of the expected destination directory for the cp command -// based on the supplied path and the number of source paths to copy -func makeParentDir(path string, numSrcPaths int) error { +// based on the supplied path. +func makeParentDir(path string) error { _, err := os.Stat(path) if !os.IsNotExist(err) { return nil } - // if path ends with a trailing '/' or if there are multiple source paths to copy - // always ensure the full path exists as a directory because 'cp' is expecting a - // dir in these cases - if strings.HasSuffix(path, "/") || numSrcPaths > 1 { + // if path ends with a trailing '/' always ensure the full path exists as a directory + // because 'cp' is expecting a dir in these cases + if strings.HasSuffix(path, "/") { if err := os.MkdirAll(filepath.Clean(path), 0755); err != nil { return fmt.Errorf("while creating full path: %s", err) } + return nil } // only make parent directory @@ -44,7 +44,8 @@ func makeParentDir(path string, numSrcPaths int) error { // CopyFromHost should be used to copy files into the rootfs from the host fs. // src is a path relative to CWD on the host, or an absolute path on the host. -// dstRel is a destination path inside dstRootfs +// dstRel is a destination path inside dstRootfs. +// An empty dstRel "" means copy the src file to the same path in the rootfs. // All symlinks encountered in the copy will be dereferenced (cp -L behavior). func CopyFromHost(src, dstRel, dstRootfs string) error { // resolve any bash globbing in filepath @@ -53,43 +54,47 @@ func CopyFromHost(src, dstRel, dstRootfs string) error { return fmt.Errorf("while expanding source path with bash: %s: %s", src, err) } - // Resolve our destination within the container rootfs - dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRel) - if err != nil { - return fmt.Errorf("while resolving destination: %s: %s", dstRel, err) - } + for _, srcGlobbed := range paths { + // If the dstRel is "" then we are copying to the full source path, appended to the rootfs prefix + if dstRel == "" { + dstRel = srcGlobbed + } - // Create any parent dirs for dst that don't already exist - if err := makeParentDir(dstResolved, len(paths)); err != nil { - return fmt.Errorf("while creating parent dir: %v", err) - } + // Resolve our destination within the container rootfs + dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRel) + if err != nil { + return fmt.Errorf("while resolving destination: %s: %s", dstRel, err) + } + + // Create any parent dirs for dst that don't already exist + if err := makeParentDir(dstResolved); err != nil { + return fmt.Errorf("while creating parent dir: %v", err) + } + + args := []string{"-fLr", srcGlobbed, dstResolved} + var output, stderr bytes.Buffer + // copy each file into bundle rootfs + copy := exec.Command("/bin/cp", args...) + copy.Stdout = &output + copy.Stderr = &stderr + if err := copy.Run(); err != nil { + return fmt.Errorf("while copying %s to %s: %v: %s", paths, dstResolved, args, stderr.String()) + } - args := []string{"-fLr"} - // append file(s) to be copied - args = append(args, paths...) - // append dst as last arg - args = append(args, dstResolved) - - var output, stderr bytes.Buffer - // copy each file into bundle rootfs - copy := exec.Command("/bin/cp", args...) - copy.Stdout = &output - copy.Stderr = &stderr - if err := copy.Run(); err != nil { - return fmt.Errorf("while copying %s to %s: %s: %s", paths, dstResolved, err, stderr.String()) } return nil } // CopyFromStage should be used to copy files into the rootfs from a previous stage. -// The srcRel and dstRel are src / dst paths relative to the srcRootfs and dstRootfs. +// The src and dst are paths relative to the srcRootfs and dstRootfs. +// An empty dst "" means copy the src file to the same path in the dst rootfs. // Symlinks are only dereferenced for the specified source or files that resolve // directly from a specified glob pattern. Any additional links inside a directory // being copied are not dereferenced. -func CopyFromStage(srcRel, dstRel, srcRootfs, dstRootfs string) error { +func CopyFromStage(src, dst, srcRootfs, dstRootfs string) error { // An absolute path is required for globbing... but with no symlink resolution or // path cleaning yet. - srcAbs := joinKeepSlash(srcRootfs, srcRel) + srcAbs := joinKeepSlash(srcRootfs, src) // resolve any bash globbing in filepath paths, err := expandPath(srcAbs) @@ -107,24 +112,30 @@ func CopyFromStage(srcRel, dstRel, srcRootfs, dstRootfs string) error { if err != nil { return fmt.Errorf("while resolving source: %s: %s", srcGlobbedRel, err) } + + // If the dst is "" then we are copying to the same path in dstRootfs, as src is in srcRootfs. + if dst == "" { + dst = srcGlobbedRel + } + // Resolve the destination path, keeping any final slash + dstResolved, err := secureJoinKeepSlash(dstRootfs, dst) + if err != nil { + return fmt.Errorf("while resolving destination: %s: %s", dst, err) + } + // Create any parent dirs for dstResolved that don't already exist. + if err := makeParentDir(dstResolved); err != nil { + return fmt.Errorf("while creating parent dir: %v", err) + } + // If we are copying into a directory then we must use the original source filename, // for the destination filename, not the one that was resolved out. // I.E. if copying `/opt/view` to `/opt/` where `/opt/view links-> /opt/.view/abc123` // we want to create `/opt/view` in the dest, not `/opt/abc123`. - dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRel) - if err != nil { - return fmt.Errorf("while resolving destination: %s: %s", dstRel, err) - } if fs.IsDir(dstResolved) { _, srcName := path.Split(srcGlobbedRel) dstResolved = path.Join(dstResolved, srcName) } - // Create any parent dirs for dst that don't already exist. - if err := makeParentDir(dstResolved, len(paths)); err != nil { - return fmt.Errorf("while creating parent dir: %v", err) - } - // Set flags for cp to perform a recursive copy without further symlink dereference. args := []string{"-fr", srcResolved, dstResolved} var output, stderr bytes.Buffer diff --git a/internal/pkg/build/files/copy_test.go b/internal/pkg/build/files/copy_test.go index a7a55bae68..525230e27a 100644 --- a/internal/pkg/build/files/copy_test.go +++ b/internal/pkg/build/files/copy_test.go @@ -25,37 +25,16 @@ func TestMakeParentDir(t *testing.T) { }{ { name: "basic", - srcNum: 1, path: "basic/path", parent: true, }, { name: "trailing slash", - srcNum: 1, path: "trailing/slash/", parent: false, }, - { - name: "multiple", - srcNum: 2, - path: "multiple/files", - parent: false, - }, - { - name: "multiple trailing slash", - srcNum: 2, - path: "multiple/trailing/slash/", - parent: false, - }, { name: "exists", - srcNum: 1, - path: "", // this will create a path of just the testdir, which will always exist - parent: false, - }, - { - name: "exists multiple", - srcNum: 2, path: "", // this will create a path of just the testdir, which will always exist parent: false, }, @@ -73,7 +52,7 @@ func TestMakeParentDir(t *testing.T) { // concatenate test path with directory, do not use a join function so that we do not remove a trailing slash path := dir + "/" + tt.path - if err := makeParentDir(path, tt.srcNum); err != nil { + if err := makeParentDir(path); err != nil { t.Errorf("") } @@ -131,6 +110,12 @@ func TestCopyFromHost(t *testing.T) { if err := os.Mkdir(srcSpaceDir, 0755); err != nil { t.Fatal(err) } + // Nested File (to test multi level glob) + srcFileNested := filepath.Join(dir, "srcDir/srcFileNested") + if err := ioutil.WriteFile(srcFileNested, []byte(sourceFileContent), 0644); err != nil { + t.Fatal(err) + } + srcFileNestedGlob := filepath.Join(dir, "srcDi?/srcFil?Nested") // Source Symlinks srcFileLinkAbs := filepath.Join(dir, "srcFileLinkAbs") if err := os.Symlink(srcFile, srcFileLinkAbs); err != nil { @@ -162,7 +147,7 @@ func TestCopyFromHost(t *testing.T) { name: "SrcFileNoDest", src: srcFile, dst: "", - expectPath: "srcFile", + expectPath: srcFile, expectFile: true, }, { @@ -189,7 +174,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcFileSpace", src: srcSpaceFile, - dst: "", + dst: "src File", expectPath: "src File", expectFile: true, }, @@ -207,9 +192,30 @@ func TestCopyFromHost(t *testing.T) { expectPath: "dstDir/srcFile", expectFile: true, }, + { + name: "srcFileGlobNoDest", + src: srcFileGlob, + dst: "", + expectPath: srcFile, + expectFile: true, + }, + { + name: "srcFileNestedGlob", + src: srcFileNestedGlob, + dst: "dstDir/", + expectPath: "dstDir/srcFileNested", + expectFile: true, + }, + { + name: "srcFileNestedGlobNoDest", + src: srcFileNestedGlob, + dst: "", + expectPath: srcFileNested, + expectFile: true, + }, { name: "dstRestricted", - src: srcFileGlob, + src: srcFile, // Will be restricted to `/` in the rootfs and should copy to there OK dst: "../../../../", expectPath: "srcFile", @@ -220,7 +226,7 @@ func TestCopyFromHost(t *testing.T) { name: "SrcDirNoDest", src: srcDir, dst: "", - expectPath: "srcDir", + expectPath: srcDir, expectDir: true, }, { @@ -247,7 +253,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcDirSpace", src: srcSpaceDir, - dst: "", + dst: "src Dir", expectPath: "src Dir", expectDir: true, }, @@ -265,11 +271,18 @@ func TestCopyFromHost(t *testing.T) { expectPath: "dstDir/srcDir", expectDir: true, }, + { + name: "srcDirGlobNoDest", + src: srcDirGlob, + dst: "", + expectPath: srcDir, + expectDir: true, + }, // Source is a Symlink { name: "srcFileLinkRel", src: srcFileLinkRel, - dst: "", + dst: "srcFileLinkRel", expectPath: "srcFileLinkRel", // Copied the file, not the link itself expectFile: true, @@ -277,7 +290,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcFileLinkAbs", src: srcFileLinkAbs, - dst: "", + dst: "srcFileLinkAbs", expectPath: "srcFileLinkAbs", // Copied the file, not the link itself expectFile: true, @@ -285,7 +298,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcDirLinkRel", src: srcDirLinkRel, - dst: "", + dst: "srcDirLinkRel", expectPath: "srcDirLinkRel", // Copied the dir, not the link itself expectDir: true, @@ -293,7 +306,7 @@ func TestCopyFromHost(t *testing.T) { { name: "srcDirLinkAbs", src: srcDirLinkAbs, - dst: "", + dst: "srcDirLinkAbs", expectPath: "srcDirLinkAbs", // Copied the dir, not the link itself expectDir: true, @@ -390,7 +403,7 @@ func TestCopyFromHostNested(t *testing.T) { defer os.RemoveAll(dstDir) // Copy our source innerDir over into the destination dir - if err := CopyFromHost(innerDir, "", dstDir); err != nil { + if err := CopyFromHost(innerDir, "innerDir", dstDir); err != nil { t.Errorf("unexpected failure copying directory: %s", err) } @@ -477,6 +490,11 @@ func TestCopyFromStage(t *testing.T) { if err := os.Mkdir(srcSpaceDir, 0755); err != nil { t.Fatal(err) } + // Nested File (to test multi level glob) + srcFileNested := filepath.Join(srcRoot, "srcDir/srcFileNested") + if err := ioutil.WriteFile(srcFileNested, []byte(sourceFileContent), 0644); err != nil { + t.Fatal(err) + } // Source Symlinks // Note the absolute links are absolute paths inside the srcRoot srcFileLinkAbs := filepath.Join(srcRoot, "srcFileLinkAbs") @@ -561,6 +579,27 @@ func TestCopyFromStage(t *testing.T) { expectPath: "dstDir/srcFile", expectFile: true, }, + { + name: "srcFileGlobNoDest", + srcRel: "srcF?*", + dstRel: "", + expectPath: "srcFile", + expectFile: true, + }, + { + name: "srcFileNestedGlob", + srcRel: "srcDi?/srcFil?Nested", + dstRel: "dstDir/", + expectPath: "dstDir/srcFileNested", + expectFile: true, + }, + { + name: "srcFileNestedGlobNoDest", + srcRel: "srcDi?/srcFil?Nested", + dstRel: "", + expectPath: "srcDir/srcFileNested", + expectFile: true, + }, { name: "dstRestricted", srcRel: "srcFile", @@ -627,6 +666,13 @@ func TestCopyFromStage(t *testing.T) { expectPath: "dstDir/srcDir", expectDir: true, }, + { + name: "srcDirGlobNoDest", + srcRel: "srcD?*", + dstRel: "", + expectPath: "srcDir", + expectDir: true, + }, // Source is a Symlink { name: "srcFileLinkRel", diff --git a/internal/pkg/build/stage.go b/internal/pkg/build/stage.go index e76345883d..030b5199ee 100644 --- a/internal/pkg/build/stage.go +++ b/internal/pkg/build/stage.go @@ -174,11 +174,6 @@ func (s *stage) copyFilesFrom(b *Build) error { sylog.Warningf("Attempt to copy file with no name, skipping.") continue } - // dest = source if not specified - if transfer.Dst == "" { - transfer.Dst = transfer.Src - } - // copy each file into bundle rootfs sylog.Infof("Copying %v to %v", transfer.Src, transfer.Dst) if err := files.CopyFromStage(transfer.Src, transfer.Dst, srcRootfsPath, dstRootfsPath); err != nil { @@ -207,10 +202,6 @@ func (s *stage) copyFiles() error { sylog.Warningf("Attempt to copy file with no name, skipping.") continue } - // dest = source if not specified - if transfer.Dst == "" { - transfer.Dst = transfer.Src - } // copy each file into bundle rootfs sylog.Infof("Copying %v to %v", transfer.Src, transfer.Dst) if err := files.CopyFromHost(transfer.Src, transfer.Dst, s.b.RootfsPath); err != nil { From 1d3ad986e91a9b0668177dbf34a9942d71e2ba9b Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 12 Aug 2021 12:50:11 -0500 Subject: [PATCH 07/30] fix: honor hostname in library:// URI for `singularity delete` `singularity delete` would accept a hostname in the library:// URI provided, but would perform the request against the default remote. Fix this, so it is handled in the same way as for push/pull. Fixes singularity-ce #243 --- CHANGELOG.md | 4 +++- cmd/internal/cli/delete.go | 14 +++++++++++++- e2e/delete/delete.go | 10 +++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d47c692da9..c75764fc8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,11 @@ # Singularity Changelog -### Changes since last release +## Changes since last release ### Bug fixes + - `singularity delete` will use the correct library service when the hostname + is specified in the `library://` URI. - Call `debootstrap` with correct Debian arch when it is not identical to the value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`. - When destination is ommitted in `%files` entry in definition file, ensure diff --git a/cmd/internal/cli/delete.go b/cmd/internal/cli/delete.go index 6e6c2b7a75..0e4174ff97 100644 --- a/cmd/internal/cli/delete.go +++ b/cmd/internal/cli/delete.go @@ -97,6 +97,18 @@ var deleteImageCmd = &cobra.Command{ sylog.Fatalf("Error parsing library ref: %v", err) } + if deleteLibraryURI != "" && imageRef.Host != "" { + sylog.Fatalf("Conflicting arguments; do not use --library with a library URI containing host name") + } + + var libraryURI string + if deleteLibraryURI != "" { + libraryURI = deleteLibraryURI + } else if imageRef.Host != "" { + // override libraryURI if ref contains host name + libraryURI = "https://" + imageRef.Host + } + r := fmt.Sprintf("%s:%s", imageRef.Path, imageRef.Tags[0]) if !deleteForce { @@ -109,7 +121,7 @@ var deleteImageCmd = &cobra.Command{ } } - libraryConfig, err := getLibraryClientConfig(deleteLibraryURI) + libraryConfig, err := getLibraryClientConfig(libraryURI) if err != nil { sylog.Fatalf("Error while getting library client config: %v", err) } diff --git a/e2e/delete/delete.go b/e2e/delete/delete.go index 14b9c0be27..fbd98c317b 100644 --- a/e2e/delete/delete.go +++ b/e2e/delete/delete.go @@ -23,6 +23,7 @@ func (c ctx) testDeleteCmd(t *testing.T) { args []string agree string expectExit int + expect e2e.SingularityCmdResultOp }{ { name: "delete unauthorized arch", @@ -66,6 +67,13 @@ func (c ctx) testDeleteCmd(t *testing.T) { agree: "y", expectExit: 255, }, + { + name: "delete host in uri", + args: []string{"library://library.example.com/test/default/test:v0.0.3"}, + agree: "y", + expectExit: 255, + expect: e2e.ExpectError(e2e.ContainMatch, "dial tcp: lookup library.example.com: no such host"), + }, } for _, tt := range tests { @@ -76,7 +84,7 @@ func (c ctx) testDeleteCmd(t *testing.T) { e2e.WithCommand("delete"), e2e.WithArgs(tt.args...), e2e.WithStdin(bytes.NewBufferString(tt.agree)), - e2e.ExpectExit(tt.expectExit), + e2e.ExpectExit(tt.expectExit, tt.expect), ) } } From 8f97185719a8b578182304fa9c6d1a7b380aa916 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 5 Aug 2021 10:46:52 -0500 Subject: [PATCH 08/30] fix: --tokenfile not supported for oci login so give error Fixes #211 --- CHANGELOG.md | 2 ++ internal/app/singularity/remote_login.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c75764fc8a..b8ae8bca66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`. - When destination is ommitted in `%files` entry in definition file, ensure globbed files are copied to correct resolved path. + - Return an error if `--tokenfile` used for `remote login` to an OCI registry, + as this is not supported. ## v3.8.1 - [2021-08-12] diff --git a/internal/app/singularity/remote_login.go b/internal/app/singularity/remote_login.go index e093eff802..b46a8ac067 100644 --- a/internal/app/singularity/remote_login.go +++ b/internal/app/singularity/remote_login.go @@ -70,6 +70,9 @@ func RemoteLogin(usrConfigFile string, args *LoginArgs) (err error) { } } else { // services (oci registry, single keyserver etc.) + if args.Tokenfile != "" { + return fmt.Errorf("--tokenfile is only supported for login to a remote endpoint, not OCI (docker/oras) or keyservers") + } if err := c.Login(args.Name, args.Username, args.Password, args.Insecure); err != nil { return fmt.Errorf("while login to %s: %s", args.Name, err) } From 076c7c68e44819e5c0ba648733b58a770697e234 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 5 Aug 2021 10:48:14 -0500 Subject: [PATCH 09/30] docs: revise CLI help for remote login Be much more explicit in what a remote endpoint is vs OCI credentials. Generally expand the descriptive text. --- cmd/internal/cli/remote.go | 6 +- docs/remote.go | 68 ++++++++++++++----- .../pkg/remote/credential/login_handler.go | 2 +- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/cmd/internal/cli/remote.go b/cmd/internal/cli/remote.go index 90c6d73ad6..fb48c77772 100644 --- a/cmd/internal/cli/remote.go +++ b/cmd/internal/cli/remote.go @@ -67,7 +67,7 @@ var remoteTokenFileFlag = cmdline.Flag{ Value: &loginTokenFile, DefaultValue: "", Name: "tokenfile", - Usage: "path to the file holding token", + Usage: "path to the file holding auth token for login (remote endpoints only)", } // --no-login @@ -86,7 +86,7 @@ var remoteLoginUsernameFlag = cmdline.Flag{ DefaultValue: "", Name: "username", ShortHand: "u", - Usage: "username to authenticate with (leave it empty for token authentication)", + Usage: "username to authenticate with (required for Docker/OCI registry login)", EnvKeys: []string{"LOGIN_USERNAME"}, } @@ -97,7 +97,7 @@ var remoteLoginPasswordFlag = cmdline.Flag{ DefaultValue: "", Name: "password", ShortHand: "p", - Usage: "password to authenticate with", + Usage: "password / token to authenticate with", EnvKeys: []string{"LOGIN_PASSWORD"}, } diff --git a/docs/remote.go b/docs/remote.go index 69e690a73e..df942c7fed 100644 --- a/docs/remote.go +++ b/docs/remote.go @@ -14,23 +14,43 @@ const ( RemoteUse string = `remote [remote options...]` RemoteShort string = `Manage singularity remote endpoints, keyservers and OCI/Docker registry credentials` RemoteLong string = ` - The 'remote' commands allow you to manage Singularity remote endpoints, keyservers - and OCI/Docker registry credentials through its subcommands. The remote configuration - is stored in $HOME/.singularity/remotes.yaml by default.` + The 'remote' command allows you to manage Singularity remote endpoints, + standalone keyservers and OCI/Docker registry credentials through its + subcommands. + + A 'remote endpoint' is the Sylabs Cloud, a Singularity Enterprise installation, + or a compatible group of services. The remote endpoint is a single address, + e.g. 'cloud.sylabs.io' through which linked library, builder and keystore + sevices will be automatically discovered. + + To configure a remote endpoint you must 'remote add' it. You can 'remote login' if + you will be performing actions needing authentication. Switch between + configured remote endpoints with the 'remote use' command. The active remote + endpoint will be used for remote builds, key operations, and 'library://' pull + and push. You can also 'remote logout' from and 'remote remove' an endpoint that + is no longer required. + + To configure credentials for OCI registries that should be used when pulling or + pushing from/to 'docker://'' or 'oras://' URIs, use the 'remote login' command + only. You do not have to 'remote add' OCI registries. To remove credentials + 'remote logout' with the same URI. You do not need to 'remote remove' OCI + credentials. + + The remote configuration is stored in $HOME/.singularity/remotes.yaml by default.` RemoteExample string = ` All group commands have their own help output: - $ singularity help remote list - $ singularity remote list` + $ singularity help remote list + $ singularity remote list` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // remote add command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RemoteAddUse string = `add [add options...] ` - RemoteAddShort string = `Create a new singularity remote endpoint` + RemoteAddShort string = `Add a new singularity remote endpoint` RemoteAddLong string = ` - The 'remote add' command allows you to create a new remote endpoint to be - be used for singularity remote services. Authentication with a newly created - endpoint will occur automatically.` + The 'remote add' command allows you to add a new remote endpoint to be + be used for singularity remote services. Authentication with a newly created + endpoint will occur automatically.` RemoteAddExample string = ` $ singularity remote add SylabsCloud cloud.sylabs.io` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -57,28 +77,39 @@ const ( // remote list command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RemoteListUse string = `list` - RemoteListShort string = `List all singularity remote endpoints and services that are configured` + RemoteListShort string = `List all singularity remote endpoints, keyservers, and OCI credentials that are configured` RemoteListLong string = ` - The 'remote list' command lists all remote endpoints configured for use. If a remote - is in use, its name will be encompassed by brackets.` + The 'remote list' command lists all remote endpoints, keyservers, and OCI registry + credentials configured for use. + + The current remote is indicated by 'YES' in the 'ACTIVE' column and can be changed + with the 'remote use' command.` RemoteListExample string = ` $ singularity remote list` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // remote login command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RemoteLoginUse string = `login [login options...] ` - RemoteLoginShort string = `Log into a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials` + RemoteLoginShort string = `Login to a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials` RemoteLoginLong string = ` The 'remote login' command allows you to set credentials for a specific endpoint, - an OCI/Docker registry or a keyserver. This command can produce a link directing you to - the token service you can use to generate a valid token. If no endpoint or registry is - specified, it will try the default remote endpoint (SylabsCloud).` + an OCI/Docker registry or a keyserver. + + If no endpoint or registry is specified, the command will login to the currently + active remote endpoint. This is cloud.sylabs.io by default.` RemoteLoginExample string = ` To log in to an endpoint: $ singularity remote login SylabsCloud To login in to a docker/OCI registry: - $ singularity remote login --username foo --password bar docker://docker.io` + $ singularity remote login --username foo docker://docker.io + $ singularity remote login --username foo oras://myregistry.example.com + + Note that many cloud OCI registries use token based authentication. The token + should be specified as the password for login. A username is still required. E.g. + when using a standard Azure identity and token to login to an ACR registry the + username '00000000-0000-0000-0000-000000000000' is required. Consult your provider + documentation for detail of their login requirements.` // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // remote logout command // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -87,7 +118,8 @@ const ( RemoteLogoutLong string = ` The 'remote logout' command allows you to log out from a singularity specific endpoint, an OCI/Docker registry or a keyserver. If no endpoint or service is specified, it will - try the default remote endpoint (SylabsCloud).` + logout from the current active remote endpoint. + ` RemoteLogoutExample string = ` To log out from an endpoint $ singularity remote logout SylabsCloud diff --git a/internal/pkg/remote/credential/login_handler.go b/internal/pkg/remote/credential/login_handler.go index 5f5f41df45..01015087b6 100644 --- a/internal/pkg/remote/credential/login_handler.go +++ b/internal/pkg/remote/credential/login_handler.go @@ -44,7 +44,7 @@ func init() { // function just return the password provided as argument. func ensurePassword(password string) (string, error) { if password == "" { - question := "Password (or token when username is empty): " + question := "Password / Token: " input, err := interactive.AskQuestionNoEcho(question) if err != nil { return "", fmt.Errorf("failed to read password: %s", err) From 6a219e0838c23a1b20ba66ed4104d2084c490fd6 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 5 Aug 2021 09:34:05 -0500 Subject: [PATCH 10/30] fix: avoid duplicate `remote.yaml` entries from multiple login Repeated `remote login` with the same URI can cause duplicate `Credentials` entries in `~/remote.yaml`. Ensure a `remote login` with an existing configured URI replaces the existing entry. Make `remote logout` remove all entries to clean up the result of this bug. Fixes #214 --- CHANGELOG.md | 2 ++ e2e/remote/remote.go | 57 +++++++++++++++++++++++++++++++++++ internal/pkg/remote/remote.go | 21 +++++++++++-- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8ae8bca66..b813a42355 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ globbed files are copied to correct resolved path. - Return an error if `--tokenfile` used for `remote login` to an OCI registry, as this is not supported. + - Ensure repeated `remote login` to same URI does not create duplicate entries + in `~/.singularity/remote.yaml`. ## v3.8.1 - [2021-08-12] diff --git a/e2e/remote/remote.go b/e2e/remote/remote.go index 17c3d59e75..8c4609ead5 100644 --- a/e2e/remote/remote.go +++ b/e2e/remote/remote.go @@ -601,6 +601,62 @@ func (c ctx) remoteLoginPushPrivate(t *testing.T) { } } +// Repeated logins with same URI should not create duplicate remote.yaml entries. +// If we login twice, and logout once we should not see the URI in list. +// See https://github.com/sylabs/singularity/issues/214 +func (c ctx) remoteLoginRepeated(t *testing.T) { + e2e.EnsureRegistry(t) + e2e.EnsureImage(t, c.env) + + var ( + registry = fmt.Sprintf("oras://%s", c.env.TestRegistry) + ) + + tests := []struct { + name string + command string + args []string + expectExit int + resultOp e2e.SingularityCmdResultOp + }{ + { + name: "FirstLogin", + command: "remote login", + args: []string{"-u", e2e.DefaultUsername, "-p", e2e.DefaultPassword, registry}, + expectExit: 0, + }, + { + name: "SecondLogin", + command: "remote login", + args: []string{"-u", e2e.DefaultUsername, "-p", e2e.DefaultPassword, registry}, + expectExit: 0, + }, + { + name: "logout", + command: "remote logout", + args: []string{registry}, + expectExit: 0, + }, + { + name: "list", + command: "remote list", + expectExit: 0, + resultOp: e2e.ExpectOutput(e2e.UnwantedContainMatch, registry), + }, + } + + for _, tt := range tests { + c.env.RunSingularity( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand(tt.command), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(tt.expectExit, tt.resultOp), + ) + } +} + func (c ctx) remoteKeyserver(t *testing.T) { var ( sylabsKeyserver = "https://keys.sylabs.io" @@ -897,6 +953,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "use": c.remoteUse, "oci login basic": np(c.remoteBasicLogin), "oci login push private": np(c.remoteLoginPushPrivate), + "oci login repeated": np(c.remoteLoginRepeated), "keyserver": np(c.remoteKeyserver), "use exclusive": np(c.remoteUseExclusive), } diff --git a/internal/pkg/remote/remote.go b/internal/pkg/remote/remote.go index b30daf4f34..411e494bd1 100644 --- a/internal/pkg/remote/remote.go +++ b/internal/pkg/remote/remote.go @@ -257,6 +257,18 @@ func (c *Config) Login(uri, username, password string, insecure bool) error { if err != nil { return err } + + // Remove any existing remote.yaml entry for the same URI. + // Older versions of Singularity can create duplicate entries with same URI, + // so loop must handle removing multiple matches (#214). + for i := 0; i < len(c.Credentials); i++ { + cred := c.Credentials[i] + if remoteutil.SameURI(cred.URI, uri) { + c.Credentials = append(c.Credentials[:i], c.Credentials[i+1:]...) + i = -1 + } + } + c.Credentials = append(c.Credentials, credConfig) return nil } @@ -266,13 +278,16 @@ func (c *Config) Logout(uri string) error { if err := credential.Manager.Logout(uri); err != nil { return err } - for i, cred := range c.Credentials { + // Older versions of Singularity can create duplicate entries with same URI, + // so loop must handle removing multiple matches (#214). + for i := 0; i < len(c.Credentials); i++ { + cred := c.Credentials[i] if remoteutil.SameURI(cred.URI, uri) { c.Credentials = append(c.Credentials[:i], c.Credentials[i+1:]...) - return nil + i = -1 } } - return fmt.Errorf("%s is not configured", uri) + return nil } // Rename an existing remote From 2911931743606a5ab3b01247688bee3f132c8f0d Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 6 Aug 2021 14:00:59 -0500 Subject: [PATCH 11/30] Fix e2e for usages messages from #217 --- e2e/remote/remote.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/remote/remote.go b/e2e/remote/remote.go index 8c4609ead5..9443ca50c5 100644 --- a/e2e/remote/remote.go +++ b/e2e/remote/remote.go @@ -412,17 +412,17 @@ func (c ctx) remoteTestFlag(t *testing.T) { { name: "add help", cmdArgs: []string{"add", "--help"}, - expectedOutput: "Create a new singularity remote endpoint", + expectedOutput: "Add a new singularity remote endpoint", }, { name: "list help", cmdArgs: []string{"list", "--help"}, - expectedOutput: "List all singularity remote endpoints and services that are configured", + expectedOutput: "List all singularity remote endpoints, keyservers, and OCI credentials that are configured", }, { name: "login help", cmdArgs: []string{"login", "--help"}, - expectedOutput: "Log into a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials", + expectedOutput: "Login to a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials", }, { name: "remove help", From ab4eca82b371ce38bce64e208bd1d13945ef98b2 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 16 Aug 2021 12:52:59 -0500 Subject: [PATCH 12/30] Ensure lolcow references point to non-fortune image The Sylabs lolcow image has been modified so that it no longer uses `fortune`, which produced offensive output. Ensure all references to a lolcow image are to the fixed Sylabs version. Fixes #248 --- internal/pkg/util/uri/uri.go | 2 +- internal/pkg/util/uri/uri_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/pkg/util/uri/uri.go b/internal/pkg/util/uri/uri.go index 284ff772c1..51824eb7e2 100644 --- a/internal/pkg/util/uri/uri.go +++ b/internal/pkg/util/uri/uri.go @@ -54,7 +54,7 @@ func IsValid(source string) (valid bool, err error) { } // GetName turns a transport:ref URI into a name containing the top-level identifier -// of the image. For example, docker://godlovedc/lolcow returns lolcow +// of the image. For example, docker://sylabsio/lolcow returns lolcow // // Returns "" when not in transport:ref format func GetName(uri string) string { diff --git a/internal/pkg/util/uri/uri_test.go b/internal/pkg/util/uri/uri_test.go index 2d23887298..2057b8166b 100644 --- a/internal/pkg/util/uri/uri_test.go +++ b/internal/pkg/util/uri/uri_test.go @@ -17,8 +17,8 @@ func Test_GetName(t *testing.T) { }{ {"docker basic", "docker://ubuntu", "ubuntu_latest.sif"}, {"docker scoped", "docker://user/image", "image_latest.sif"}, - {"dave's magical lolcow", "docker://godlovedc/lolcow", "lolcow_latest.sif"}, - {"docker w/ tags", "docker://godlovedc/lolcow:3.7", "lolcow_3.7.sif"}, + {"dave's magical lolcow", "docker://sylabs.io/lolcow", "lolcow_latest.sif"}, + {"docker w/ tags", "docker://sylabs.io/lolcow:3.7", "lolcow_3.7.sif"}, } for _, tt := range tests { @@ -39,8 +39,8 @@ func Test_Split(t *testing.T) { }{ {"docker basic", "docker://ubuntu", "docker", "//ubuntu"}, {"docker scoped", "docker://user/image", "docker", "//user/image"}, - {"dave's magical lolcow", "docker://godlovedc/lolcow", "docker", "//godlovedc/lolcow"}, - {"docker with tags", "docker://godlovedc/lolcow:latest", "docker", "//godlovedc/lolcow:latest"}, + {"dave's magical lolcow", "docker://sylabs.io/lolcow", "docker", "//sylabs.io/lolcow"}, + {"docker with tags", "docker://sylabs.io/lolcow:latest", "docker", "//sylabs.io/lolcow:latest"}, {"library basic", "library://image", "library", "//image"}, {"library scoped", "library://collection/image", "library", "//collection/image"}, {"without transport", "ubuntu", "", "ubuntu"}, From 63f623cbc64e56842d489e8a89a3adc2c29371df Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 17 Aug 2021 14:20:47 -0500 Subject: [PATCH 13/30] fix: honor hostname in library URI for `singularity build` Fixes sylabs/singularity#246 --- CHANGELOG.md | 2 ++ e2e/imgbuild/imgbuild.go | 29 ++++++++++++++++++- .../build/sources/conveyorPacker_library.go | 15 ++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b813a42355..3066868b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - `singularity delete` will use the correct library service when the hostname is specified in the `library://` URI. + - `singularity build` will use the correct library service when the hostname + is specified in the `library://` URI / definition file. - Call `debootstrap` with correct Debian arch when it is not identical to the value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`. - When destination is ommitted in `%files` entry in definition file, ensure diff --git a/e2e/imgbuild/imgbuild.go b/e2e/imgbuild/imgbuild.go index 13e2885bb5..2c7c4f0233 100644 --- a/e2e/imgbuild/imgbuild.go +++ b/e2e/imgbuild/imgbuild.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -1356,6 +1356,32 @@ func (c imgBuildTests) buildBindMount(t *testing.T) { } } +func (c imgBuildTests) buildLibraryHost(t *testing.T) { + e2e.EnsureImage(t, c.env) + + tmpdir, cleanup := c.tempDir(t, "build-libraryhost-test") + defer cleanup() + + // Library hostname in the From URI + // The hostname is invalid, and we should get an error to that effect. + definition := "Bootstrap: library\nFrom: library.example.com/test/test/test:latest\n" + + defFile := e2e.RawDefFile(t, tmpdir, strings.NewReader(definition)) + imagePath := filepath.Join(tmpdir, "image-libaryhost") + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("build"), + e2e.WithArgs("-F", imagePath, defFile), + e2e.PostRun(func(t *testing.T) { + os.Remove(defFile) + }), + e2e.ExpectExit(255, + e2e.ExpectError(e2e.ContainMatch, "dial tcp: lookup library.example.com: no such host"), + ), + ) +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := imgBuildTests{ @@ -1374,6 +1400,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "build and update sandbox": c.buildUpdateSandbox, // build/update sandbox "fingerprint check": c.buildWithFingerprint, // definition file includes fingerprint check "build with bind mount": c.buildBindMount, // build image with bind mount + "library host": c.buildLibraryHost, // build image with hostname in library URI "issue 3848": c.issue3848, // https://github.com/hpcng/singularity/issues/3848 "issue 4203": c.issue4203, // https://github.com/hpcng/singularity/issues/4203 "issue 4407": c.issue4407, // https://github.com/hpcng/singularity/issues/4407 diff --git a/internal/pkg/build/sources/conveyorPacker_library.go b/internal/pkg/build/sources/conveyorPacker_library.go index 96045a42f2..38807326e9 100644 --- a/internal/pkg/build/sources/conveyorPacker_library.go +++ b/internal/pkg/build/sources/conveyorPacker_library.go @@ -50,13 +50,22 @@ func (cp *LibraryConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err libraryURL = customLib } - sylog.Debugf("LibraryURL: %v", libraryURL) - sylog.Debugf("LibraryRef: %v", b.Recipe.Header["from"]) - imageRef, err := library.NormalizeLibraryRef(b.Recipe.Header["from"]) if err != nil { return fmt.Errorf("error parsing libraryRef: %v", err) } + + if imageRef.Host != "" { + if b.Opts.NoHTTPS { + libraryURL = "http://" + imageRef.Host + } else { + libraryURL = "https://" + imageRef.Host + } + } + + sylog.Debugf("LibraryURL: %v", libraryURL) + sylog.Debugf("LibraryRef: %v", imageRef.String()) + libraryConfig := &client.Config{ BaseURL: libraryURL, AuthToken: authToken, From 4f2ef4205c821c11a669008810d7890233b94f80 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 30 Jul 2021 10:50:58 -0500 Subject: [PATCH 14/30] Add test and changelog entry for mountinfo splitting fix --- CHANGELOG.md | 1 + pkg/util/fs/proc/proc_linux_test.go | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3066868b2e..b4fe4c540f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ as this is not supported. - Ensure repeated `remote login` to same URI does not create duplicate entries in `~/.singularity/remote.yaml`. + - Avoid panic when mountinfo line has a blank field. ## v3.8.1 - [2021-08-12] diff --git a/pkg/util/fs/proc/proc_linux_test.go b/pkg/util/fs/proc/proc_linux_test.go index 765f8954b0..25d69fd00a 100644 --- a/pkg/util/fs/proc/proc_linux_test.go +++ b/pkg/util/fs/proc/proc_linux_test.go @@ -72,7 +72,8 @@ var mountInfoData = `22 28 0:21 / /sys rw,nosuid,nodev,noexec,relatime shared:7 90 47 0:47 / /proc/sys/fs/binfmt_misc rw,relatime shared:34 - binfmt_misc binfmt_misc rw 381 26 0:54 / /run/user/1000 rw,nosuid,nodev,relatime shared:245 - tmpfs tmpfs rw,size=1635868k,mode=700,uid=1000,gid=1000 363 381 0:52 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:233 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 -579 28 0:65 / /tmp/squashfs rw,relatime - squashfs /dev/loop0 rw` +579 28 0:65 / /tmp/squashfs rw,relatime - squashfs /dev/loop0 rw +377 314 0:80 / /dev/shm rw,relatime - tmpfs rw,mode=750,uid=174988` var expectedMap = map[string][]string{ "/": { @@ -205,6 +206,19 @@ func TestGetMountInfo(t *testing.T) { SuperOptions: []string{"rw"}, Options: []string{"rw", "relatime"}, }, + // github.com/hpcng/singularity/issues/6048 + // Empty 9th field (source) in mountinfo line + { + ParentID: "314", + ID: "377", + Dev: "0:80", + Root: "/", + Fields: "", + FSType: "tmpfs", + Source: "", + SuperOptions: []string{"rw","rw,mode=750,uid=174988"}, + Options: []string{"rw", "relatime"}, + }, } for _, c := range check { From 50e8c77981c8760db676658c6c3d932842c34537 Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Wed, 25 Aug 2021 16:20:21 -0500 Subject: [PATCH 15/30] move mountinfo panic fix message to 3.8.1 where it was implemented but not documented --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fe4c540f..ba8f31e6cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,6 @@ as this is not supported. - Ensure repeated `remote login` to same URI does not create duplicate entries in `~/.singularity/remote.yaml`. - - Avoid panic when mountinfo line has a blank field. ## v3.8.1 - [2021-08-12] @@ -31,6 +30,7 @@ - Prevent garbage collection from closing the container image file descriptor. - Update to Arch Linux pacman.conf URL and remove file size verification. + - Avoid panic when mountinfo line has a blank field. ## v3.8.0 - [2021-06-15] From d069e7b955066c419ab410e41ace15505fb24ccd Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 3 Aug 2021 12:47:38 -0500 Subject: [PATCH 16/30] testing: skip hugetlb cgroup limit for non-2MB page size If the system page size is not 2MB just test cgroups without a hugetlb limit, rather than putting in more complex logic to work up configs for different page sizes. Fixes #9 --- internal/pkg/cgroups/cgroups_linux_test.go | 10 ++++++---- .../{cgroups-ppc64le.toml => cgroups-no-hugetlb.toml} | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) rename internal/pkg/cgroups/example/{cgroups-ppc64le.toml => cgroups-no-hugetlb.toml} (97%) diff --git a/internal/pkg/cgroups/cgroups_linux_test.go b/internal/pkg/cgroups/cgroups_linux_test.go index aeeb32ee07..d69c8bd16d 100644 --- a/internal/pkg/cgroups/cgroups_linux_test.go +++ b/internal/pkg/cgroups/cgroups_linux_test.go @@ -12,7 +12,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "strconv" "strings" "testing" @@ -55,9 +54,12 @@ func TestCgroups(t *testing.T) { manager := &Manager{Pid: pid, Path: path} cgroupsToml := "example/cgroups.toml" - // A different pageSize is needed on pp64le - if runtime.GOARCH == "ppc64le" { - cgroupsToml = "example/cgroups-ppc64le.toml" + // Some systems, e.g. ppc64le may not have a 2MB page size, so don't + // apply a 2MB hugetlb limit if that's the case. + _, err = os.Stat("/sys/fs/cgroup/hugetlb/hugetlb.2MB.limit_in_bytes") + if os.IsNotExist(err) { + t.Log("No hugetlb.2MB.limit_in_bytes - using alternate cgroups test file") + cgroupsToml = "example/cgroups-no-hugetlb.toml" } if err := manager.ApplyFromFile(cgroupsToml); err != nil { diff --git a/internal/pkg/cgroups/example/cgroups-ppc64le.toml b/internal/pkg/cgroups/example/cgroups-no-hugetlb.toml similarity index 97% rename from internal/pkg/cgroups/example/cgroups-ppc64le.toml rename to internal/pkg/cgroups/example/cgroups-no-hugetlb.toml index 539bd9b79d..0f0d99f155 100644 --- a/internal/pkg/cgroups/example/cgroups-ppc64le.toml +++ b/internal/pkg/cgroups/example/cgroups-no-hugetlb.toml @@ -102,9 +102,9 @@ # Hugetlb limit (in bytes) # - pagesize: the hugepage size # - limit: the limit of "hugepagesize" hugetlb usage -[[hugepageLimits]] - limit = 9223372036854771712 - pageSize = "16MB" +# [[hugepageLimits]] +# limit = 9223372036854771712 +# pageSize = "16MB" # Network restriction configuration From fb37ecb017891383ae7c9f5856c8a750d080529a Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 17 Aug 2021 13:37:21 -0500 Subject: [PATCH 17/30] fix: Escape ' in docker CMD / ENTRYPOINT for runscript --- CHANGELOG.md | 1 + e2e/docker/docker.go | 16 ++++++++++- .../pkg/build/sources/conveyorPacker_oci.go | 10 +++++-- .../engine/singularity/process_linux.go | 4 +-- internal/pkg/util/shell/escape.go | 14 +++++++--- internal/pkg/util/shell/escape_test.go | 28 ++++++++++++++++--- pkg/util/fs/proc/proc_linux_test.go | 2 +- 7 files changed, 60 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8f31e6cf..bf5f9ea679 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ as this is not supported. - Ensure repeated `remote login` to same URI does not create duplicate entries in `~/.singularity/remote.yaml`. + - Properly escape single quotes in Docker `CMD` / `ENTRYPOINT` translation. ## v3.8.1 - [2021-08-12] diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index a9a822c439..7d1167e9de 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -397,6 +397,19 @@ func (c ctx) testDockerRegistry(t *testing.T) { } } +// https://github.com/sylabs/singularity/issues/233 +func (c ctx) testDockerCMDQuotes(t *testing.T) { + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("run"), + e2e.WithArgs("docker://sylabsio/issue233"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "Test run"), + ), + ) +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := ctx{ @@ -410,5 +423,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "pulls": c.testDockerPulls, "registry": c.testDockerRegistry, "whiteout symlink": c.testDockerWhiteoutSymlink, + "cmd quotes": c.testDockerCMDQuotes, } } diff --git a/internal/pkg/build/sources/conveyorPacker_oci.go b/internal/pkg/build/sources/conveyorPacker_oci.go index c627f0ad25..690bc5fc15 100644 --- a/internal/pkg/build/sources/conveyorPacker_oci.go +++ b/internal/pkg/build/sources/conveyorPacker_oci.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -324,7 +324,9 @@ func (cp *OCIConveyorPacker) insertRunScript() (err error) { } if len(cp.imgConfig.Entrypoint) > 0 { - _, err = f.WriteString("OCI_ENTRYPOINT='" + shell.ArgsQuoted(cp.imgConfig.Entrypoint) + "'\n") + _, err = f.WriteString("OCI_ENTRYPOINT='" + + shell.EscapeSingleQuotes(shell.ArgsQuoted(cp.imgConfig.Entrypoint)) + + "'\n") if err != nil { return } @@ -336,7 +338,9 @@ func (cp *OCIConveyorPacker) insertRunScript() (err error) { } if len(cp.imgConfig.Cmd) > 0 { - _, err = f.WriteString("OCI_CMD='" + shell.ArgsQuoted(cp.imgConfig.Cmd) + "'\n") + _, err = f.WriteString("OCI_CMD='" + + shell.EscapeSingleQuotes(shell.ArgsQuoted(cp.imgConfig.Cmd)) + + "'\n") if err != nil { return } diff --git a/internal/pkg/runtime/engine/singularity/process_linux.go b/internal/pkg/runtime/engine/singularity/process_linux.go index 19eb0fd205..580db12e2d 100644 --- a/internal/pkg/runtime/engine/singularity/process_linux.go +++ b/internal/pkg/runtime/engine/singularity/process_linux.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -646,7 +646,7 @@ func injectEnvHandler(senv map[string]string) interpreter.OpenHandler { b.WriteString(fmt.Sprintf(snippet, key, value+":/.singularity.d/libs")) continue } - b.WriteString(fmt.Sprintf(snippet, key, shell.EscapeQuotes(value))) + b.WriteString(fmt.Sprintf(snippet, key, shell.EscapeDoubleQuotes(value))) } }) diff --git a/internal/pkg/util/shell/escape.go b/internal/pkg/util/shell/escape.go index bf1141f2c3..9d4131c3c3 100644 --- a/internal/pkg/util/shell/escape.go +++ b/internal/pkg/util/shell/escape.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018, Sylabs, Inc. All rights reserved. +// Copyright (c) 2018-2021 Sylabs, Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please // consult LICENSE.md file distributed with the sources of this project regarding // your rights to use or distribute this software. @@ -16,7 +16,8 @@ func ArgsQuoted(a []string) (quoted string) { return } -// Escape performs escaping of shell quotes, backticks and $ characters +// Escape performs escaping of shell double quotes, backticks and $ characters. +// Does not escape single quotes - apply EscapeSingleQuotes separately for this. func Escape(s string) string { escaped := strings.Replace(s, `\`, `\\`, -1) escaped = strings.Replace(escaped, `"`, `\"`, -1) @@ -25,7 +26,12 @@ func Escape(s string) string { return escaped } -// EscapeQuotes performs escaping of double quotes only -func EscapeQuotes(s string) string { +// EscapeQuotes performs shell escaping of double quotes only +func EscapeDoubleQuotes(s string) string { return strings.Replace(s, `"`, `\"`, -1) } + +// EscapeSingleQuotes performs shell escaping of single quotes only +func EscapeSingleQuotes(s string) string { + return strings.Replace(s, `'`, `'"'"'`, -1) +} diff --git a/internal/pkg/util/shell/escape_test.go b/internal/pkg/util/shell/escape_test.go index 19526b9223..5cfc9ad05b 100644 --- a/internal/pkg/util/shell/escape_test.go +++ b/internal/pkg/util/shell/escape_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018, Sylabs, Inc. All rights reserved. +// Copyright (c) 2018-2021 Sylabs, Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please // consult LICENSE file distributed with the sources of this project regarding // your rights to use or distribute this software. @@ -52,8 +52,8 @@ func TestEscape(t *testing.T) { } -func TestEscapeQuotes(t *testing.T) { - var escapeQuotesTests = []struct { +func TestEscapeDoubleQuotes(t *testing.T) { + escapeQuotesTests := []struct { input string expected string }{ @@ -64,7 +64,27 @@ func TestEscapeQuotes(t *testing.T) { for _, test := range escapeQuotesTests { t.Run(test.input, func(t *testing.T) { - escaped := EscapeQuotes(test.input) + escaped := EscapeDoubleQuotes(test.input) + if escaped != test.expected { + t.Errorf("got %s, expected %s", escaped, test.expected) + } + }) + } +} + +func TestEscapeSingleQuotes(t *testing.T) { + escapeQuotesTests := []struct { + input string + expected string + }{ + {`Hello`, `Hello`}, + {`'Hello'`, `'"'"'Hello'"'"'`}, + {`Hell'o`, `Hell'"'"'o`}, + } + + for _, test := range escapeQuotesTests { + t.Run(test.input, func(t *testing.T) { + escaped := EscapeSingleQuotes(test.input) if escaped != test.expected { t.Errorf("got %s, expected %s", escaped, test.expected) } diff --git a/pkg/util/fs/proc/proc_linux_test.go b/pkg/util/fs/proc/proc_linux_test.go index 25d69fd00a..5b0452ba58 100644 --- a/pkg/util/fs/proc/proc_linux_test.go +++ b/pkg/util/fs/proc/proc_linux_test.go @@ -216,7 +216,7 @@ func TestGetMountInfo(t *testing.T) { Fields: "", FSType: "tmpfs", Source: "", - SuperOptions: []string{"rw","rw,mode=750,uid=174988"}, + SuperOptions: []string{"rw", "rw,mode=750,uid=174988"}, Options: []string{"rw", "relatime"}, }, } From 0aae4b5eb277245fc8aea3d98322a3744b578847 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 18 Aug 2021 15:25:04 -0500 Subject: [PATCH 18/30] Markdown doc changes for 3.8.2 --- INSTALL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index cbbbe3b888..6a95e6c387 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -87,7 +87,7 @@ $ mkdir -p ${GOPATH}/src/github.com/hpcng && \ To build a stable version of Singularity, check out a [release tag](https://github.com/hpcng/singularity/tags) before compiling: ``` -$ git checkout v3.8.1 +$ git checkout v3.8.2 ``` ## Compiling Singularity @@ -130,7 +130,7 @@ as shown above. Then download the latest and use it to install the RPM like this: ``` -$ export VERSION=3.8.1 # this is the singularity version, change as you need +$ export VERSION=3.8.2 # this is the singularity version, change as you need $ wget https://github.com/hpcng/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \ rpmbuild -tb singularity-${VERSION}.tar.gz && \ @@ -146,7 +146,7 @@ tarball and use it to install Singularity: $ cd $GOPATH/src/github.com/hpcng/singularity && \ ./mconfig && \ make -C builddir rpm && \ - sudo rpm -ivh ~/rpmbuild/RPMS/x86_64/singularity-3.8.1*.x86_64.rpm # or whatever version you built + sudo rpm -ivh ~/rpmbuild/RPMS/x86_64/singularity-3.8.2*.x86_64.rpm # or whatever version you built ``` To build an rpm with an alternative install prefix set RPMPREFIX on the From 3ae281b003744b50c006b3d81bc046269e21d6ca Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 18 Aug 2021 16:17:12 -0500 Subject: [PATCH 19/30] Failing test for issue #261 multi-glob copy --- internal/pkg/build/files/copy_test.go | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/pkg/build/files/copy_test.go b/internal/pkg/build/files/copy_test.go index 525230e27a..95d2a1ede1 100644 --- a/internal/pkg/build/files/copy_test.go +++ b/internal/pkg/build/files/copy_test.go @@ -110,6 +110,7 @@ func TestCopyFromHost(t *testing.T) { if err := os.Mkdir(srcSpaceDir, 0755); err != nil { t.Fatal(err) } + srcGlob := filepath.Join(dir, "src*") // Nested File (to test multi level glob) srcFileNested := filepath.Join(dir, "srcDir/srcFileNested") if err := ioutil.WriteFile(srcFileNested, []byte(sourceFileContent), 0644); err != nil { @@ -311,6 +312,22 @@ func TestCopyFromHost(t *testing.T) { // Copied the dir, not the link itself expectDir: true, }, + // issue 261 - multiple globbed sources, with no dest + // both srcfile and srcdir should be copied for glob of "src*" + { + name: "srcDirGlobNoDestMulti1", + src: srcGlob, + dst: "", + expectPath: srcDir, + expectDir: true, + }, + { + name: "srcDirGlobNoDestMulti2", + src: srcGlob, + dst: "", + expectPath: srcFile, + expectFile: true, + }, } for _, tt := range tests { @@ -706,6 +723,22 @@ func TestCopyFromStage(t *testing.T) { // Copied the dir, not the link itself expectDir: true, }, + // issue 261 - multiple globbed sources, with no dest + // both srcfile and srcdir should be copied for glob of "src*" + { + name: "srcDirGlobNoDestMulti1", + srcRel: "src*", + dstRel: "", + expectPath: "srcDir", + expectDir: true, + }, + { + name: "srcDirGlobNoDestMulti2", + srcRel: "src*", + dstRel: "", + expectPath: "srcFile", + expectFile: true, + }, } for _, tt := range tests { From 8a1deecd526451af36f60d92cc27a5d85d5cd16f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 18 Aug 2021 16:17:29 -0500 Subject: [PATCH 20/30] fix: always check dest for globbed %files When the source in a `%files` entry is a glob that matches multiple files, and the destination is `""`, make sure we always set the correct real destination for each file. Fixes #261 --- internal/pkg/build/files/copy.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/pkg/build/files/copy.go b/internal/pkg/build/files/copy.go index 8811f76d8b..f2bf172f66 100644 --- a/internal/pkg/build/files/copy.go +++ b/internal/pkg/build/files/copy.go @@ -56,14 +56,15 @@ func CopyFromHost(src, dstRel, dstRootfs string) error { for _, srcGlobbed := range paths { // If the dstRel is "" then we are copying to the full source path, appended to the rootfs prefix + dstRelGlobbed := dstRel if dstRel == "" { - dstRel = srcGlobbed + dstRelGlobbed = srcGlobbed } // Resolve our destination within the container rootfs - dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRel) + dstResolved, err := secureJoinKeepSlash(dstRootfs, dstRelGlobbed) if err != nil { - return fmt.Errorf("while resolving destination: %s: %s", dstRel, err) + return fmt.Errorf("while resolving destination: %s: %s", dstRelGlobbed, err) } // Create any parent dirs for dst that don't already exist @@ -114,13 +115,14 @@ func CopyFromStage(src, dst, srcRootfs, dstRootfs string) error { } // If the dst is "" then we are copying to the same path in dstRootfs, as src is in srcRootfs. + dstGlobbed := dst if dst == "" { - dst = srcGlobbedRel + dstGlobbed = srcGlobbedRel } // Resolve the destination path, keeping any final slash - dstResolved, err := secureJoinKeepSlash(dstRootfs, dst) + dstResolved, err := secureJoinKeepSlash(dstRootfs, dstGlobbed) if err != nil { - return fmt.Errorf("while resolving destination: %s: %s", dst, err) + return fmt.Errorf("while resolving destination: %s: %s", dstGlobbed, err) } // Create any parent dirs for dstResolved that don't already exist. if err := makeParentDir(dstResolved); err != nil { From fba6d34e842f173960c1153bee4e1352049af0ca Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 19 Aug 2021 08:41:04 -0500 Subject: [PATCH 21/30] fix: Check for *host* non-root uid when setting unsquashfs flags We were previously setting `--user-xattrs` / `--no-xattrs` based on the current euid when calling `unsquashfs`. However, in a non-setuid install `--fakeroot` build the euid is 0, so the flags will not be set. On distributions other than EL / Fedora (i.e. Debian / Ubuntu which are not selinux native), this can cause failed extraction. Decide if we are performing a rootless extraction based on the host uid, outside of any namespace that is in play. Fixes: #266 --- CHANGELOG.md | 2 ++ pkg/image/unpacker/squashfs.go | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5f9ea679..fb95c1c4ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ - Ensure repeated `remote login` to same URI does not create duplicate entries in `~/.singularity/remote.yaml`. - Properly escape single quotes in Docker `CMD` / `ENTRYPOINT` translation. + - Use host uid when choosing unsquashfs flags, to avoid selinux xattr errors + with `--fakeroot` on non-EL/Fedora distributions with recent squashfs-tools. ## v3.8.1 - [2021-08-12] diff --git a/pkg/image/unpacker/squashfs.go b/pkg/image/unpacker/squashfs.go index ea13f313ba..9bfdf3ceaa 100644 --- a/pkg/image/unpacker/squashfs.go +++ b/pkg/image/unpacker/squashfs.go @@ -110,7 +110,11 @@ func (s *Squashfs) extract(files []string, reader io.Reader, dest string) (err e // 2. Must check (user) xattrs are supported on the FS as unsquashfs >=4.4 will give a non-zero error code if // it cannot set them, e.g. on tmpfs (#5668) opts := []string{} - rootless := os.Geteuid() != 0 + hostuid, err := namespaces.HostUID() + if err != nil { + return fmt.Errorf("could not get host UID: %s", err) + } + rootless := hostuid != 0 // Do we support user xattrs? ok, err := TestUserXattr(filepath.Dir(dest)) @@ -129,15 +133,11 @@ func (s *Squashfs) extract(files []string, reader io.Reader, dest string) (err e // non real root users could not create pseudo devices so we compare // the host UID (to include fake root user) and apply a filter at extraction (#5690) - hostuid, err := namespaces.HostUID() - if err != nil { - return fmt.Errorf("could not get host UID: %s", err) - } filter := "" // exclude dev directory only if there no specific files provided for extraction // as globbing won't work with POSIX regex enabled - if hostuid != 0 && len(files) == 0 { + if rootless && len(files) == 0 { sylog.Debugf("Excluding /dev directory during root filesystem extraction (non root user)") // filter requires POSIX regex opts = append(opts, "-r") From 758c3eb36b77b23d4cb4a90954cc22e5b5dbd75e Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Wed, 25 Aug 2021 16:54:43 -0500 Subject: [PATCH 22/30] fix EscapeDoubleQuotes name in comment caught by revive --- internal/pkg/util/shell/escape.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/util/shell/escape.go b/internal/pkg/util/shell/escape.go index 9d4131c3c3..d0e531880e 100644 --- a/internal/pkg/util/shell/escape.go +++ b/internal/pkg/util/shell/escape.go @@ -26,7 +26,7 @@ func Escape(s string) string { return escaped } -// EscapeQuotes performs shell escaping of double quotes only +// EscapeDoubleQuotes performs shell escaping of double quotes only func EscapeDoubleQuotes(s string) string { return strings.Replace(s, `"`, `\"`, -1) } From 927ab88a441bdabc3a39e9c2fd57b497fad68e37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 05:23:12 +0000 Subject: [PATCH 23/30] chore(deps): bump github.com/sylabs/scs-build-client from 0.1.6 to 0.2.1 Bumps [github.com/sylabs/scs-build-client](https://github.com/sylabs/scs-build-client) from 0.1.6 to 0.2.1. - [Release notes](https://github.com/sylabs/scs-build-client/releases) - [Commits](https://github.com/sylabs/scs-build-client/compare/v0.1.6...v0.2.1) --- updated-dependencies: - dependency-name: github.com/sylabs/scs-build-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index ab9cb569b9..c7cbacbb87 100644 --- a/go.mod +++ b/go.mod @@ -44,8 +44,8 @@ require ( github.com/seccomp/libseccomp-golang v0.9.1 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 - github.com/sylabs/json-resp v0.7.1 - github.com/sylabs/scs-build-client v0.1.6 + github.com/sylabs/json-resp v0.8.0 + github.com/sylabs/scs-build-client v0.2.1 github.com/sylabs/scs-key-client v0.6.2 github.com/sylabs/scs-library-client v1.0.5 github.com/urfave/cli v1.22.5 // indirect diff --git a/go.sum b/go.sum index d47d195384..cdbc66e982 100644 --- a/go.sum +++ b/go.sum @@ -847,10 +847,11 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/sylabs/json-resp v0.7.1 h1:tYxJer8FqLNIWx3mEbYQgrwLL7HR4gyPAfzyeUwEznM= github.com/sylabs/json-resp v0.7.1/go.mod h1:3O9ze5PoYxia6106m5x+5G5ohnW1AJX/FURjCNovtzA= -github.com/sylabs/scs-build-client v0.1.6 h1:ZzcS8trPF+jFNovh9o6aeiJnMtClKNyrby9DPEkrAAQ= -github.com/sylabs/scs-build-client v0.1.6/go.mod h1:XFO5Aq0NXsO1A41JBNMt3tWwxmddV8UrId4wmmoLrIU= +github.com/sylabs/json-resp v0.8.0 h1:bZ932uaF220aPqT0+x/vakoaGCGNbpLCjUFm1f+JKlY= +github.com/sylabs/json-resp v0.8.0/go.mod h1:bUGV9cqShOyxz7RxBq03Yt9raKGfELKrfN6Yac3lfiw= +github.com/sylabs/scs-build-client v0.2.1 h1:/Hk9SI+hNPPjuacNk0aFvi6u7tFdNVtPzz+ZdO3NMQc= +github.com/sylabs/scs-build-client v0.2.1/go.mod h1:/kuKpMv3JnFd2IXCmX1+OCYxBxv+zUUWv92Az3QvwfE= github.com/sylabs/scs-key-client v0.6.2 h1:PTdmkE50rFMrSty9usl8wnzaW+JPK0rjXF4cdIQ3MEo= github.com/sylabs/scs-key-client v0.6.2/go.mod h1:WoTk47nNRrk0uLfblxUFekxDIIVPY4l2/9evME5fgX0= github.com/sylabs/scs-library-client v1.0.5 h1:u8ueCIQaa38/B0axtKUzdWSQ6uLXL9HvLQGNkB69Oe0= From 11d25abe6438e32919bbe5029346a29cf5045d79 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 26 Aug 2021 09:58:17 -0500 Subject: [PATCH 24/30] fix: interpret individual Stmt in internalExecHandler Changes to exit handling with the implementation of trap in mvdan.cc/sh mean that we need to interpret individual `syntax.Stmt`, rather than a full parsed `syntax.File` when we are handling built-ins prefixed with `\` in `internalExecHandler`. Fixes sylabs/singularity#274 --- CHANGELOG.md | 2 ++ internal/pkg/util/shell/interpreter/interpreter.go | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb95c1c4ef..e9ea8a6f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Bug fixes + - Fix regression when files `source`d from `%environment` contain `\` escaped + shell builtins (fixes issue with `source` of conda profile.d script). - `singularity delete` will use the correct library service when the hostname is specified in the `library://` URI. - `singularity build` will use the correct library service when the hostname diff --git a/internal/pkg/util/shell/interpreter/interpreter.go b/internal/pkg/util/shell/interpreter/interpreter.go index d4d677c426..45c61c2bb8 100644 --- a/internal/pkg/util/shell/interpreter/interpreter.go +++ b/internal/pkg/util/shell/interpreter/interpreter.go @@ -166,7 +166,18 @@ func (s *Shell) internalExecHandler() interp.ExecHandlerFunc { if err != nil { return err } - return s.runner.Run(ctx, node) + + // We run individual syntax.Stmt rather than the parsed syntax.File as the latter + // implies an `exit`, and causes https://github.com/sylabs/singularity/issues/274 + // with the exit/trap changes in https://github.com/mvdan/sh/commit/fb5052e7a0109c9ef5553a310c05f3b8c04cca5f + for _, stmt := range node.Stmts { + err := s.runner.Run(ctx, stmt) + if err != nil { + return err + } + } + return nil + } } return defaultExecHandler(ctx, args) From 8b046442f49840f4ba80e8d1f84ea488b5012c39 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 26 Aug 2021 10:47:29 -0500 Subject: [PATCH 25/30] Add e2e test for conda env activation from `%environment` --- e2e/env/env.go | 1 + e2e/env/regressions.go | 56 +++++++++++++++++++ .../pkg/util/shell/interpreter/interpreter.go | 4 +- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/e2e/env/env.go b/e2e/env/env.go index f78bfc5941..dd0b833848 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -447,5 +447,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "issue 5057": c.issue5057, // https://github.com/sylabs/hpcng/issues/5057 "issue 5426": c.issue5426, // https://github.com/sylabs/hpcng/issues/5426 "issue 43": c.issue43, // https://github.com/sylabs/singularity/issues/43 + "issue 274": c.issue274, // https://github.com/sylabs/singularity/issues/274 } } diff --git a/e2e/env/regressions.go b/e2e/env/regressions.go index 6eaf3e7c5e..71d45258b5 100644 --- a/e2e/env/regressions.go +++ b/e2e/env/regressions.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path" + "path/filepath" "testing" "github.com/hpcng/singularity/e2e/internal/e2e" @@ -121,3 +122,58 @@ func (c ctx) issue43(t *testing.T) { ), ) } + +// https://github.com/sylabs/singularity/issues/274 +// The conda profile.d script must be able to be source'd from %environment. +// This has been broken by changes to mvdan.cc/sh interacting badly with our +// custom internalExecHandler. +// The test is quite heavyweight, but is warranted IMHO to ensure that conda +// environment activation works as expected, as this is a common use-case +// for SingularityCE. +func (c ctx) issue274(t *testing.T) { + imageDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "issue274-", "") + defer cleanup(t) + imagePath := filepath.Join(imageDir, "container") + + // Create a minimal conda environment on the current miniconda3 base. + // Source the conda profile.d code and activate the env from `%environment`. + def := `Bootstrap: docker +From: continuumio/miniconda3:latest + +%post + + . /opt/conda/etc/profile.d/conda.sh + conda create -n env python=3 + +%environment + + source /opt/conda/etc/profile.d/conda.sh + conda activate env +` + defFile, err := e2e.WriteTempFile(imageDir, "deffile", def) + if err != nil { + t.Fatalf("Unable to create test definition file: %v", err) + } + + c.env.RunSingularity( + t, + e2e.AsSubtest("build"), + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("build"), + e2e.WithArgs(imagePath, defFile), + e2e.ExpectExit(0), + ) + // An exec of `conda info` in the container should show environment active, no errors. + // I.E. the `%environment` section should have worked. + c.env.RunSingularity( + t, + e2e.AsSubtest("exec"), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("exec"), + e2e.WithArgs(imagePath, "conda", "info"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "active environment : env"), + e2e.ExpectError(e2e.ExactMatch, ""), + ), + ) +} diff --git a/internal/pkg/util/shell/interpreter/interpreter.go b/internal/pkg/util/shell/interpreter/interpreter.go index 45c61c2bb8..ad9ecb5344 100644 --- a/internal/pkg/util/shell/interpreter/interpreter.go +++ b/internal/pkg/util/shell/interpreter/interpreter.go @@ -171,13 +171,11 @@ func (s *Shell) internalExecHandler() interp.ExecHandlerFunc { // implies an `exit`, and causes https://github.com/sylabs/singularity/issues/274 // with the exit/trap changes in https://github.com/mvdan/sh/commit/fb5052e7a0109c9ef5553a310c05f3b8c04cca5f for _, stmt := range node.Stmts { - err := s.runner.Run(ctx, stmt) - if err != nil { + if err := s.runner.Run(ctx, stmt); err != nil { return err } } return nil - } } return defaultExecHandler(ctx, args) From c81e8d86d6c93022aca07957eb2c96881a3bb930 Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Mon, 30 Aug 2021 14:15:36 -0500 Subject: [PATCH 26/30] fix url in comments --- e2e/env/env.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/env/env.go b/e2e/env/env.go index dd0b833848..6f50e62e6b 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -444,8 +444,8 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "environment manipulation": c.singularityEnv, "environment option": c.singularityEnvOption, "environment file": c.singularityEnvFile, - "issue 5057": c.issue5057, // https://github.com/sylabs/hpcng/issues/5057 - "issue 5426": c.issue5426, // https://github.com/sylabs/hpcng/issues/5426 + "issue 5057": c.issue5057, // https://github.com/hpcng/issues/5057 + "issue 5426": c.issue5426, // https://github.com/hpcng/issues/5426 "issue 43": c.issue43, // https://github.com/sylabs/singularity/issues/43 "issue 274": c.issue274, // https://github.com/sylabs/singularity/issues/274 } From b824245e91cfb80ada56dc1d2b2e26cf77123f15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 05:22:00 +0000 Subject: [PATCH 27/30] chore(deps): bump github.com/hpcng/sif from 1.5.1 to 1.6.0 Bumps [github.com/hpcng/sif](https://github.com/hpcng/sif) from 1.5.1 to 1.6.0. - [Release notes](https://github.com/hpcng/sif/releases) - [Commits](https://github.com/hpcng/sif/compare/v1.5.1...v1.6.0) --- updated-dependencies: - dependency-name: github.com/hpcng/sif dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c7cbacbb87..f97a573b59 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gorilla/handlers v1.4.0 // indirect github.com/gorilla/websocket v1.4.2 - github.com/hpcng/sif v1.5.1 + github.com/hpcng/sif v1.6.0 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kr/pty v1.1.8 github.com/opencontainers/go-digest v1.0.0 @@ -55,7 +55,7 @@ require ( github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.6 // indirect github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index cdbc66e982..807553db87 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9 h1:t8KUun6NE+Kv500O2sNQ6LIFvUz7QX+RJGOVoJFlyHM= github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9/go.mod h1:YRpvuFxEV8iMqW8qf0+X+dTRn3labfXfykTzp8hRkSg= -github.com/hpcng/sif v1.5.1 h1:ZKsjRc4Wykos3pB3ASiKXK9iep9+k78Fmo2r79xHbDE= -github.com/hpcng/sif v1.5.1/go.mod h1:Zm0L455Vvf9YZv6QyuTmQUgl9fcf/ST9r3/v9FbGLYo= +github.com/hpcng/sif v1.6.0 h1:CEYAU+PY0AFmg+G8SwJJmNA4Ixs6DOcgf/pIkVd8QBA= +github.com/hpcng/sif v1.6.0/go.mod h1:czlq6ECeYaGHAYmZdKjAXB0S5VrfQXcT3Momvszz2qY= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= From dcb9b390cc39de453ece0e7f27096ab4839452ec Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Tue, 31 Aug 2021 11:08:48 -0500 Subject: [PATCH 28/30] update golang-x-crypto to latest version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f97a573b59..f30710f74f 100644 --- a/go.mod +++ b/go.mod @@ -69,5 +69,5 @@ replace ( github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible - golang.org/x/crypto => github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9 + golang.org/x/crypto => github.com/hpcng/golang-x-crypto v0.0.0-20210830200829-e6b35e3fb874 ) diff --git a/go.sum b/go.sum index 807553db87..d9c227b86b 100644 --- a/go.sum +++ b/go.sum @@ -522,8 +522,8 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9 h1:t8KUun6NE+Kv500O2sNQ6LIFvUz7QX+RJGOVoJFlyHM= -github.com/hpcng/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9/go.mod h1:YRpvuFxEV8iMqW8qf0+X+dTRn3labfXfykTzp8hRkSg= +github.com/hpcng/golang-x-crypto v0.0.0-20210830200829-e6b35e3fb874 h1:mGKQ3LEDpGH3MVbBWPXw/m2xJ7s9XEnga1q5fYgK/RY= +github.com/hpcng/golang-x-crypto v0.0.0-20210830200829-e6b35e3fb874/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= github.com/hpcng/sif v1.6.0 h1:CEYAU+PY0AFmg+G8SwJJmNA4Ixs6DOcgf/pIkVd8QBA= github.com/hpcng/sif v1.6.0/go.mod h1:czlq6ECeYaGHAYmZdKjAXB0S5VrfQXcT3Momvszz2qY= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= From 162321329fc09ceccbca69101ddfa0b25502fc6b Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Tue, 31 Aug 2021 11:12:26 -0500 Subject: [PATCH 29/30] add message about update to golang-x-crypto --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9ea8a6f7e..18d5b8a35c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ - Properly escape single quotes in Docker `CMD` / `ENTRYPOINT` translation. - Use host uid when choosing unsquashfs flags, to avoid selinux xattr errors with `--fakeroot` on non-EL/Fedora distributions with recent squashfs-tools. + - Updated the modified golang-x-crypto module with the latest upstream + version. ## v3.8.1 - [2021-08-12] From 3d8c116d43fc72fb6f7e334b82c160d24fe699c3 Mon Sep 17 00:00:00 2001 From: Krishna Muriki Date: Tue, 31 Aug 2021 22:44:14 -0700 Subject: [PATCH 30/30] Change log edits for release v3.8.2 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d5b8a35c..d6b4a535b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Singularity Changelog -## Changes since last release +## v3.8.2 - [2021-08-31] ### Bug fixes