From f8d6c23211d0ebb78ce62e4c2b9d4d7ba6312936 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 12:36:08 -0400 Subject: [PATCH 01/37] feat: run ASH image using non-root user --- Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6ed2b97..df8f328 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,6 +134,25 @@ COPY ./appsec_cfn_rules /ash/appsec_cfn_rules/ COPY ./ash-multi /ash/ash COPY ./__version__ /ash/__version__ +RUN chmod +x /ash/ash + +# +# Create a non-root user in the container and run as this user +# +# And add GitHub's public fingerprints to known_hosts inside the image to prevent fingerprint +# confirmation requests unexpectedly +# +ARG ASHUSER_HOME=/home/ash-user +RUN adduser --disabled-password --disabled-login \ + --uid 500 --gid 100 \ + ash-user && \ + mkdir -p ${ASHUSER_HOME}/.ssh && \ + echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ + echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts + +USER 500 + # # Flag ASH as local execution mode since we are running in a container already # From 13c903cc68dd2979f0eeb3599c8e2cd99f0b4e54 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 13:29:59 -0400 Subject: [PATCH 02/37] feat: run ASH image using non-root user --- Dockerfile | 14 ++++++++++++-- ash | 8 ++++++++ ash-multi | 11 +++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index df8f328..cef6ced 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,16 @@ ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye FROM ${BASE_IMAGE} +# +# Collect the host UID/GID which built the image in order to set the UID/GID of the non-root user +# to match the UID/GID of the builder of the container. +# +# Set the default values to non-zero to avoid someone inadvertantly creating a container that runs as root +# if/when they did not specify these as --build-arg arguments on the ${OCI_RUNNER} build command. +# +ARG UID=500 +ARG GID=100 + # # Setting timezone in the container to UTC to ensure logged times are universal. # @@ -144,14 +154,14 @@ RUN chmod +x /ash/ash # ARG ASHUSER_HOME=/home/ash-user RUN adduser --disabled-password --disabled-login \ - --uid 500 --gid 100 \ + --uid ${UID} --gid ${GID} \ ash-user && \ mkdir -p ${ASHUSER_HOME}/.ssh && \ echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts -USER 500 +USER ${UID} # # Flag ASH as local execution mode since we are running in a container already diff --git a/ash b/ash index a47a4b0..5fb6123 100755 --- a/ash +++ b/ash @@ -77,6 +77,12 @@ if [[ "${OUTPUT_DIR_SPECIFIED}" == "YES" ]]; then OUTPUT_DIR="$(cd "$OUTPUT_DIR"; pwd)" fi +# +# Gather the UID and GID of the caller +# +HOST_UID=$(id -u) +HOST_GID=$(id -g) + # Resolve the OCI_RUNNER RESOLVED_OCI_RUNNER=${OCI_RUNNER:-$(command -v finch || command -v docker || command -v nerdctl || command -v podman)} @@ -95,6 +101,8 @@ else if [ "${NO_BUILD}" = "NO" ]; then echo "Building image ${ASH_IMAGE_NAME} -- this may take a few minutes during the first build..." ${RESOLVED_OCI_RUNNER} build \ + --build-arg UID="${HOST_UID}" \ + --build-arg GID="${HOST_GID}" \ --tag ${ASH_IMAGE_NAME} \ --file "${ASH_ROOT_DIR}/Dockerfile" \ ${DOCKER_EXTRA_ARGS} \ diff --git a/ash-multi b/ash-multi index bbc8fbc..d0b2c2a 100755 --- a/ash-multi +++ b/ash-multi @@ -162,7 +162,18 @@ validate_input() { if [ -d "${OUTPUT_DIR}/work" ]; then rm -rf "${OUTPUT_DIR}/work" fi + echo ">>> ID is" + id + echo "<<< ID" + echo ">>> permissions on ${OUTPUT_DIR} are" + ls -aln "${OUTPUT_DIR}" + echo "<<< permissions" + echo ">>> before mkdir" mkdir -p "${OUTPUT_DIR}/work" + echo "<<< after mkdir" + echo ">>> permissions on ${OUTPUT_DIR} are" + ls -aln "${OUTPUT_DIR}" + echo "<<< permissions" OUTPUT_DIR=$(cd "${OUTPUT_DIR}"; pwd) # Transform any relative path to absolute CFNRULES_LOCATION=$(cd "${CFNRULES_LOCATION}"; pwd) # Transform any relative path to absolute UTILS_LOCATION=$(cd "${UTILS_LOCATION}"; pwd) # Transform any relative path to absolute From ed5ff23d2d5ea3f70af7016b08497b27719e5172 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 14:21:29 -0400 Subject: [PATCH 03/37] feat: run ASH image using non-root user --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index cef6ced..c0b38fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,9 +153,10 @@ RUN chmod +x /ash/ash # confirmation requests unexpectedly # ARG ASHUSER_HOME=/home/ash-user -RUN adduser --disabled-password --disabled-login \ - --uid ${UID} --gid ${GID} \ - ash-user && \ +RUN addgroup --gid ${GID} ash-group || \ + adduser --disabled-password --disabled-login \ + --uid ${UID} --gid ${GID} \ + ash-user && \ mkdir -p ${ASHUSER_HOME}/.ssh && \ echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ From 28f304a0a49874f717bb6c843ecc9873b667b0b2 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 14:46:51 -0400 Subject: [PATCH 04/37] feat: run ASH image using non-root user --- Dockerfile | 2 +- ash-multi | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c0b38fd..6cb0d77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -144,7 +144,7 @@ COPY ./appsec_cfn_rules /ash/appsec_cfn_rules/ COPY ./ash-multi /ash/ash COPY ./__version__ /ash/__version__ -RUN chmod +x /ash/ash +RUN chmod -R +r /ash && chmod +x /ash/ash # # Create a non-root user in the container and run as this user diff --git a/ash-multi b/ash-multi index d0b2c2a..5ab71cc 100755 --- a/ash-multi +++ b/ash-multi @@ -449,6 +449,9 @@ export _ASH_RUN_DIR="${_ASH_RUN_DIR:-$(mktemp -d -p /tmp ash-run-scan.XXXX)}" # echo -e "\n${LPURPLE}ASH version ${GREEN}$VERSION${NC}\n" +echo ">>> setting debug on" +set -x + # nosemgrep IFS=$'\n' # Support directories with spaces, make the loop iterate over newline instead of space pushd . >/dev/null 2>&1 From 6654e553a67bc9ef3c3d463cb03de2f155d1d630 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 15:08:08 -0400 Subject: [PATCH 05/37] feat: run ASH image using non-root user --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6cb0d77..d25964e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,6 +168,7 @@ USER ${UID} # Flag ASH as local execution mode since we are running in a container already # ENV _ASH_EXEC_MODE="local" +ENV HOME=${ASHUSER_HOME} # # Append /ash to PATH to allow calling `ash` directly From 55f30df80bcd3b2be3f0459593a51789ff4daefb Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 15:21:32 -0400 Subject: [PATCH 06/37] feat: run ASH image using non-root user --- ash-multi | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ash-multi b/ash-multi index 5ab71cc..991804c 100755 --- a/ash-multi +++ b/ash-multi @@ -94,8 +94,18 @@ map_extensions_and_files() { cd "${_ASH_SOURCE_DIR}" # On local mode, this configuration will only affect the current container - git config --global --add safe.directory "${_ASH_SOURCE_DIR}" >/dev/null 2>&1 - git config --global --add safe.directory "${_ASH_RUN_DIR}" >/dev/null 2>&1 + # git config --global --add safe.directory "${_ASH_SOURCE_DIR}" >/dev/null 2>&1 + # git config --global --add safe.directory "${_ASH_RUN_DIR}" >/dev/null 2>&1 + echo "PATH is ${PATH}, git is $(type git)" + echo ">>> info on ${_ASH_SOURCE_DIR}" + ls -aln "${_ASH_SOURCE_DIR}" + echo "<<< info on ${_ASH_SOURCE_DIR}" + echo ">>> info on ${_ASH_RUN_DIR}" + ls -aln "${_ASH_RUN_DIR}" + echo "<<< info on ${_ASH_RUN_DIR}" + git --version + git config --global --add safe.directory "${_ASH_SOURCE_DIR}" || echo "return code $? from git config" + git config --global --add safe.directory "${_ASH_RUN_DIR}" || echo "return code $? from git config" if [[ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" == "true" ]]; then git clone "${_ASH_SOURCE_DIR}" "${_ASH_RUN_DIR}" echo "Repository cloned successfully." From 89ad81d2220e3872d4229692dc52fd2d0709d3b6 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 15:39:08 -0400 Subject: [PATCH 07/37] feat: run ASH image using non-root user --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d25964e..e9d1c80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -160,7 +160,8 @@ RUN addgroup --gid ${GID} ash-group || \ mkdir -p ${ASHUSER_HOME}/.ssh && \ echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ - echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts \ + chown -R ${UID}:${GID} ${ASHUSER_HOME} USER ${UID} From f2bf01885a06202645834a57cc541e202d74b379 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 15:47:54 -0400 Subject: [PATCH 08/37] feat: run ASH image using non-root user --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e9d1c80..1412599 100644 --- a/Dockerfile +++ b/Dockerfile @@ -160,7 +160,7 @@ RUN addgroup --gid ${GID} ash-group || \ mkdir -p ${ASHUSER_HOME}/.ssh && \ echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ - echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts \ + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ chown -R ${UID}:${GID} ${ASHUSER_HOME} USER ${UID} From ad8ccd81a449f2aacef1e198ec4221ec0575a48e Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 15:57:36 -0400 Subject: [PATCH 09/37] feat: run ASH image using non-root user --- Dockerfile | 6 +++++- ash-multi | 28 ++-------------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1412599..179843f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -165,11 +165,15 @@ RUN addgroup --gid ${GID} ash-group || \ USER ${UID} +# +# Set the HOME environment variable to be the HOME folder for the non-root user +# +ENV HOME=${ASHUSER_HOME} + # # Flag ASH as local execution mode since we are running in a container already # ENV _ASH_EXEC_MODE="local" -ENV HOME=${ASHUSER_HOME} # # Append /ash to PATH to allow calling `ash` directly diff --git a/ash-multi b/ash-multi index 991804c..bbc8fbc 100755 --- a/ash-multi +++ b/ash-multi @@ -94,18 +94,8 @@ map_extensions_and_files() { cd "${_ASH_SOURCE_DIR}" # On local mode, this configuration will only affect the current container - # git config --global --add safe.directory "${_ASH_SOURCE_DIR}" >/dev/null 2>&1 - # git config --global --add safe.directory "${_ASH_RUN_DIR}" >/dev/null 2>&1 - echo "PATH is ${PATH}, git is $(type git)" - echo ">>> info on ${_ASH_SOURCE_DIR}" - ls -aln "${_ASH_SOURCE_DIR}" - echo "<<< info on ${_ASH_SOURCE_DIR}" - echo ">>> info on ${_ASH_RUN_DIR}" - ls -aln "${_ASH_RUN_DIR}" - echo "<<< info on ${_ASH_RUN_DIR}" - git --version - git config --global --add safe.directory "${_ASH_SOURCE_DIR}" || echo "return code $? from git config" - git config --global --add safe.directory "${_ASH_RUN_DIR}" || echo "return code $? from git config" + git config --global --add safe.directory "${_ASH_SOURCE_DIR}" >/dev/null 2>&1 + git config --global --add safe.directory "${_ASH_RUN_DIR}" >/dev/null 2>&1 if [[ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" == "true" ]]; then git clone "${_ASH_SOURCE_DIR}" "${_ASH_RUN_DIR}" echo "Repository cloned successfully." @@ -172,18 +162,7 @@ validate_input() { if [ -d "${OUTPUT_DIR}/work" ]; then rm -rf "${OUTPUT_DIR}/work" fi - echo ">>> ID is" - id - echo "<<< ID" - echo ">>> permissions on ${OUTPUT_DIR} are" - ls -aln "${OUTPUT_DIR}" - echo "<<< permissions" - echo ">>> before mkdir" mkdir -p "${OUTPUT_DIR}/work" - echo "<<< after mkdir" - echo ">>> permissions on ${OUTPUT_DIR} are" - ls -aln "${OUTPUT_DIR}" - echo "<<< permissions" OUTPUT_DIR=$(cd "${OUTPUT_DIR}"; pwd) # Transform any relative path to absolute CFNRULES_LOCATION=$(cd "${CFNRULES_LOCATION}"; pwd) # Transform any relative path to absolute UTILS_LOCATION=$(cd "${UTILS_LOCATION}"; pwd) # Transform any relative path to absolute @@ -459,9 +438,6 @@ export _ASH_RUN_DIR="${_ASH_RUN_DIR:-$(mktemp -d -p /tmp ash-run-scan.XXXX)}" # echo -e "\n${LPURPLE}ASH version ${GREEN}$VERSION${NC}\n" -echo ">>> setting debug on" -set -x - # nosemgrep IFS=$'\n' # Support directories with spaces, make the loop iterate over newline instead of space pushd . >/dev/null 2>&1 From 82df7b01e9b8136400feb0986574fae4b461dc93 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Mon, 29 Apr 2024 16:01:46 -0400 Subject: [PATCH 10/37] feat: run ASH image using non-root user --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 179843f..c7599b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ #checkov:skip=CKV_DOCKER_7: Base image is using a non-latest version tag by default, Checkov is unable to parse due to the use of ARG -#checkov:skip=CKV_DOCKER_3: ASH is focused on mounting source code into the container and scanning it, not running services. Setting USER breaks the ability for certain scanners to work correctly. # # Enable BASE_IMAGE as an overrideable ARG for proxy cache + private registry support # From f681a3bc0d4c55473ca8f2f7a69dee5a8dde8645 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Tue, 30 Apr 2024 09:15:44 -0400 Subject: [PATCH 11/37] feat: run ASH image using non-root user --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index c7599b2..f2887b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -161,6 +161,8 @@ RUN addgroup --gid ${GID} ash-group || \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ chown -R ${UID}:${GID} ${ASHUSER_HOME} +# Setting default WORKDIR to ${ASHUSER_HOME} +WORKDIR ${ASHUSER_HOME} USER ${UID} From f23f3924bfe5178c03ccb1c8afbb55891095fd92 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Wed, 1 May 2024 10:27:27 -0400 Subject: [PATCH 12/37] feat: run ASH image using non-root user --- .github/workflows/ash-build-and-scan.yml | 2 +- Dockerfile | 4 ++-- ash | 22 ++++++++++++++++++++-- ash-multi | 7 ++++++- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ash-build-and-scan.yml b/.github/workflows/ash-build-and-scan.yml index 81168cc..2e46d09 100644 --- a/.github/workflows/ash-build-and-scan.yml +++ b/.github/workflows/ash-build-and-scan.yml @@ -69,7 +69,7 @@ jobs: set +e # Run ASH against itself - ./ash --source-dir $(pwd) --output-dir ash_output --debug | \ + ./ash --source-dir $(pwd) --output-dir ash_output --container-uid 1001 --container-gid 123 --debug | \ tee ash_stdout.txt # cat the output contents to build the summary markdown diff --git a/Dockerfile b/Dockerfile index f2887b6..f2c4c21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,7 +153,7 @@ RUN chmod -R +r /ash && chmod +x /ash/ash # ARG ASHUSER_HOME=/home/ash-user RUN addgroup --gid ${GID} ash-group || \ - adduser --disabled-password --disabled-login \ + adduser --disabled-password --disabled-login --gecos "" \ --uid ${UID} --gid ${GID} \ ash-user && \ mkdir -p ${ASHUSER_HOME}/.ssh && \ @@ -164,7 +164,7 @@ RUN addgroup --gid ${GID} ash-group || \ # Setting default WORKDIR to ${ASHUSER_HOME} WORKDIR ${ASHUSER_HOME} -USER ${UID} +USER ${UID}:${GID} # # Set the HOME environment variable to be the HOME folder for the non-root user diff --git a/ash b/ash index 5fb6123..6892d40 100755 --- a/ash +++ b/ash @@ -9,6 +9,8 @@ export ASH_IMAGE_NAME=${ASH_IMAGE_NAME:-"automated-security-helper:local"} SOURCE_DIR="" OUTPUT_DIR="" OUTPUT_DIR_SPECIFIED="NO" +CONTAINER_UID_SPECIFIED="NO" +CONTAINER_GID_SPEICIFED="NO" DOCKER_EXTRA_ARGS="" ASH_ARGS="" NO_BUILD="NO" @@ -37,6 +39,16 @@ while (("$#")); do shift OCI_RUNNER="$1" ;; + --container-uid | -u) + shift + CONTAINER_UID_SPECIFIED="YES" + CONTAINER_UID="$1" + ;; + --container-gid | -u) + shift + CONTAINER_GID_SPECIFIED="YES" + CONTAINER_GID="$1" + ;; --no-build) NO_BUILD="YES" ;; @@ -99,10 +111,16 @@ else # Build the image if the --no-build flag is not set if [ "${NO_BUILD}" = "NO" ]; then + if [[ ${CONTAINER_UID_SPECIFIED} = "YES" ]]; then + CONTAINER_UID_OPTION="--build-arg UID=${CONTAINER_UID}" # add readonly source mount when --output-dir is specified + fi + if [[ ${CONTAINER_GID_SPECIFIED} = "YES" ]]; then + CONTAINER_GID_OPTION="--build-arg GID=${CONTAINER_GID}" # add readonly source mount when --output-dir is specified + fi echo "Building image ${ASH_IMAGE_NAME} -- this may take a few minutes during the first build..." ${RESOLVED_OCI_RUNNER} build \ - --build-arg UID="${HOST_UID}" \ - --build-arg GID="${HOST_GID}" \ + ${CONTAINER_UID_OPTION} \ + ${CONTAINER_GID_OPTION} \ --tag ${ASH_IMAGE_NAME} \ --file "${ASH_ROOT_DIR}/Dockerfile" \ ${DOCKER_EXTRA_ARGS} \ diff --git a/ash-multi b/ash-multi index bbc8fbc..4d27673 100755 --- a/ash-multi +++ b/ash-multi @@ -97,7 +97,7 @@ map_extensions_and_files() { git config --global --add safe.directory "${_ASH_SOURCE_DIR}" >/dev/null 2>&1 git config --global --add safe.directory "${_ASH_RUN_DIR}" >/dev/null 2>&1 if [[ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" == "true" ]]; then - git clone "${_ASH_SOURCE_DIR}" "${_ASH_RUN_DIR}" + git clone "${_ASH_SOURCE_DIR}" "${_ASH_RUN_DIR}" >/dev/null 2>&1 echo "Repository cloned successfully." _ASH_SOURCE_DIR="${_ASH_RUN_DIR}" else @@ -438,6 +438,11 @@ export _ASH_RUN_DIR="${_ASH_RUN_DIR:-$(mktemp -d -p /tmp ash-run-scan.XXXX)}" # echo -e "\n${LPURPLE}ASH version ${GREEN}$VERSION${NC}\n" +# +# Print out the uid/gid values in effect +# +echo -e "\n${LPURPLE}ASH running with UID/GID: ${CYAN}$(id -u):$(id -g)${NC}\n" + # nosemgrep IFS=$'\n' # Support directories with spaces, make the loop iterate over newline instead of space pushd . >/dev/null 2>&1 From 683bf4283719a2c26c96bef0722c8331b9554ffb Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Wed, 1 May 2024 10:49:30 -0400 Subject: [PATCH 13/37] feat: run ASH image using non-root user --- Dockerfile | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f2c4c21..0c1d625 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,18 @@ ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye FROM ${BASE_IMAGE} # -# Collect the host UID/GID which built the image in order to set the UID/GID of the non-root user -# to match the UID/GID of the builder of the container. +# Allow for UID and GID to be set as build arguments to this Dockerfile. # -# Set the default values to non-zero to avoid someone inadvertantly creating a container that runs as root -# if/when they did not specify these as --build-arg arguments on the ${OCI_RUNNER} build command. +# Set the default values of UID/GID to non-zero (and above typical low-number UID/GID values which +# are defined by default). 500 and 100 are not special values or depended upon. +# Their selection as default values is only to set the default for this container to be non-zero. +# +# The actions performed in this image (CMD command) do not require root privileges. +# Thus, UID/GID are set non-zero to avoid someone inadvertantly running this image in a container +# that runs privileged and as as root(UID 0). +# +# If the environment in which the container will run requires a specific UID/GID, then --build-arg +# arguments on the ${OCI_RUNNER} build command can be used to over-ride these values. # ARG UID=500 ARG GID=100 From 46e15b1d460e41659f94198d74df2cfbd1ac2ecf Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Wed, 1 May 2024 11:08:35 -0400 Subject: [PATCH 14/37] feat: run ASH image using non-root user --- Dockerfile | 5 +++-- ash | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0c1d625..5dce560 100644 --- a/Dockerfile +++ b/Dockerfile @@ -159,8 +159,9 @@ RUN chmod -R +r /ash && chmod +x /ash/ash # confirmation requests unexpectedly # ARG ASHUSER_HOME=/home/ash-user -RUN addgroup --gid ${GID} ash-group || \ - adduser --disabled-password --disabled-login --gecos "" \ +# ignore a failure to add the group +RUN addgroup --gid ${GID} ash-group || : +RUN adduser --disabled-password --disabled-login \ --uid ${UID} --gid ${GID} \ ash-user && \ mkdir -p ${ASHUSER_HOME}/.ssh && \ diff --git a/ash b/ash index 6892d40..e211639 100755 --- a/ash +++ b/ash @@ -125,7 +125,6 @@ else --file "${ASH_ROOT_DIR}/Dockerfile" \ ${DOCKER_EXTRA_ARGS} \ "${ASH_ROOT_DIR}" - eval $build_cmd fi # Run the image if the --no-run flag is not set From 40f49b77e7cdbe64053ef9ed9f92ac27cf3dc699 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Wed, 1 May 2024 11:16:13 -0400 Subject: [PATCH 15/37] feat: run ASH image using non-root user --- ash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ash b/ash index e211639..7b18297 100755 --- a/ash +++ b/ash @@ -111,6 +111,8 @@ else # Build the image if the --no-build flag is not set if [ "${NO_BUILD}" = "NO" ]; then + CONTAINER_UID_OPTION="" + CONTAINER_GID_OPTION="" if [[ ${CONTAINER_UID_SPECIFIED} = "YES" ]]; then CONTAINER_UID_OPTION="--build-arg UID=${CONTAINER_UID}" # add readonly source mount when --output-dir is specified fi From 0f748d76e650505fdd9f6bd6293d760df233173e Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Fri, 3 May 2024 14:07:55 -0400 Subject: [PATCH 16/37] chore: fix auto-merge updates --- Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a61596..1d6377a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -150,6 +150,9 @@ COPY ./appsec_cfn_rules /ash/appsec_cfn_rules/ COPY ./ash-multi /ash/ash COPY ./__version__ /ash/__version__ +# +# Make sure the ash script is executable +# RUN chmod -R +r /ash && chmod +x /ash/ash # @@ -179,11 +182,6 @@ USER ${UID}:${GID} # ENV HOME=${ASHUSER_HOME} -# -# Make sure the ash script is executable -# -RUN chmod +x /ash/ash - # # Flag ASH as local execution mode since we are running in a container already # From e41adbbc13990c8ab73eacf75c40f22ee449cd7e Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Tue, 4 Jun 2024 16:23:24 -0400 Subject: [PATCH 17/37] fix: always set UID/GID build-arg values when building the ASH container image --- ash | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ash b/ash index 7b18297..6f9c207 100755 --- a/ash +++ b/ash @@ -114,10 +114,14 @@ else CONTAINER_UID_OPTION="" CONTAINER_GID_OPTION="" if [[ ${CONTAINER_UID_SPECIFIED} = "YES" ]]; then - CONTAINER_UID_OPTION="--build-arg UID=${CONTAINER_UID}" # add readonly source mount when --output-dir is specified + CONTAINER_UID_OPTION="--build-arg UID=${CONTAINER_UID}" # set the UID build-arg if --container-uid is specified + else + CONTAINER_UID_OPTION="--build-arg UID=${HOST_UID}" # set the UID build-arg to the caller's UID if --container-uid is not specified fi if [[ ${CONTAINER_GID_SPECIFIED} = "YES" ]]; then - CONTAINER_GID_OPTION="--build-arg GID=${CONTAINER_GID}" # add readonly source mount when --output-dir is specified + CONTAINER_GID_OPTION="--build-arg GID=${CONTAINER_GID}" # set the GID build-arg if --container-gid is specified + else + CONTAINER_GID_OPTION="--build-arg GID=${HOST_GID}" # set the GID build-arg to the caller's GID if --container-uid is not specified fi echo "Building image ${ASH_IMAGE_NAME} -- this may take a few minutes during the first build..." ${RESOLVED_OCI_RUNNER} build \ From 26c83de411ed8b8813fbd610dfffb76cac332fa5 Mon Sep 17 00:00:00 2001 From: Tim Hahn Date: Tue, 4 Jun 2024 16:34:52 -0400 Subject: [PATCH 18/37] chore: fix EOL characters in file --- requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 669a118..82fbc73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -# Documentation static site generator & deployment tool -mkdocs>=1.1.2 -# Material theme for mkdocs -mkdocs-material>=5.4.0 -# awesome-pages plugin for mkdocs -mkdocs-awesome-pages-plugin==2.8.0 -# Additional mkdocs extensions for prettifying the output +# Documentation static site generator & deployment tool +mkdocs>=1.1.2 +# Material theme for mkdocs +mkdocs-material>=5.4.0 +# awesome-pages plugin for mkdocs +mkdocs-awesome-pages-plugin==2.8.0 +# Additional mkdocs extensions for prettifying the output pymdown-extensions From a2d8da85016e411d0141bea93826187738346f3f Mon Sep 17 00:00:00 2001 From: Madison Steiner Date: Sun, 22 Sep 2024 08:56:04 -0700 Subject: [PATCH 19/37] Add additional checks for build expiry and ignoring Checkov/NPM Audit in offline mode --- Dockerfile | 2 + ash | 1 + ash-multi | 7 +++ helper_dockerfiles/Dockerfile-js | 4 ++ helper_dockerfiles/Dockerfile-yaml | 5 ++ utils/js-docker-execute.sh | 77 ++++++++++++++++-------------- utils/yaml-docker-execute.sh | 4 +- 7 files changed, 62 insertions(+), 38 deletions(-) diff --git a/Dockerfile b/Dockerfile index c18a0de..a335c93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,11 @@ RUN poetry build FROM ${BASE_IMAGE} as ash SHELL ["/bin/bash", "-c"] +ARG BUILD_DATE_EPOCH="-1" ARG OFFLINE="NO" ARG OFFLINE_SEMGREP_RULESETS="p/ci" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" ENV OFFLINE="${OFFLINE}" ENV OFFLINE_AT_BUILD_TIME="${OFFLINE}" ENV OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" diff --git a/ash b/ash index 1b622bc..89cc495 100755 --- a/ash +++ b/ash @@ -122,6 +122,7 @@ else --file "${ASH_ROOT_DIR}/Dockerfile" \ --build-arg OFFLINE="${OFFLINE}" \ --build-arg OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" \ + --build-arg BUILD_DATE="$(date +%s)" \ ${DOCKER_EXTRA_ARGS} \ "${ASH_ROOT_DIR}" eval $build_cmd diff --git a/ash-multi b/ash-multi index 4398b2c..c2b7fea 100755 --- a/ash-multi +++ b/ash-multi @@ -160,6 +160,13 @@ validate_input() { echo -e "${RED}Please ensure --offline is specified at build time to be able to run in offline mode.${NC}" exit 1 fi + if [[ ${OFFLINE} == "YES" ]]; then + if [[ ${BUILD_DATE_EPOCH} == -1 ]]; then + echo -e "${RED}Please ensure you have pulled in the latest ASH changes and rebuilt your container since v1.5.2." + elif [[ "$(($(date +%s)-"${BUILD_DATE_EPOCH}"))" -gt "604800" ]]; then # Greater than 7 days + echo -e "${RED}Your build is over 7 days old. Please rebuild your ASH offline container." + fi + fi if [[ -z ${PRESERVE_FILE} ]]; then AGGREGATED_RESULTS_REPORT_FILENAME="aggregated_results.txt"; else AGGREGATED_RESULTS_REPORT_FILENAME="aggregated_results-$(date +%s).txt"; fi if [[ -z ${FORCE_REBUILD} ]]; then FORCE_REBUILD="false"; fi if [[ -z ${SOURCE_DIR} ]]; then SOURCE_DIR="$(pwd)"; else SOURCE_DIR=$(cd "${SOURCE_DIR}"; pwd); fi # Transform any relative path to absolute diff --git a/helper_dockerfiles/Dockerfile-js b/helper_dockerfiles/Dockerfile-js index f97c251..9c3297c 100644 --- a/helper_dockerfiles/Dockerfile-js +++ b/helper_dockerfiles/Dockerfile-js @@ -1,6 +1,10 @@ # Get NPM Image FROM public.ecr.aws/docker/library/node:18.0.0 +ARG OFFLINE="NO" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" +ENV OFFLINE="${OFFLINE}" # # Make sure the default dirs are initialized # diff --git a/helper_dockerfiles/Dockerfile-yaml b/helper_dockerfiles/Dockerfile-yaml index 9807630..0b49108 100644 --- a/helper_dockerfiles/Dockerfile-yaml +++ b/helper_dockerfiles/Dockerfile-yaml @@ -1,5 +1,10 @@ # Get Ubuntu Image FROM --platform=linux/amd64 public.ecr.aws/bitnami/python:3.11 +ARG OFFLINE="NO" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" + +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" +ENV OFFLINE="${OFFLINE}" ENV TZ=Europe/London RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/utils/js-docker-execute.sh b/utils/js-docker-execute.sh index 90a614d..f87b1d6 100755 --- a/utils/js-docker-execute.sh +++ b/utils/js-docker-execute.sh @@ -58,45 +58,48 @@ if [[ "${ASH_OUTPUT_FORMAT:-text}" != "text" ]]; then AUDIT_ARGS="--json ${AUDIT_ARGS}" fi -for i in "${!scan_paths[@]}"; -do - scan_path=${scan_paths[$i]} - cd ${scan_path} - for file in $(find . \ - -iname "package-lock.json" -o \ - -iname "pnpm-lock.yaml" -o \ - -iname "yarn.lock"); +if [[ $OFFLINE == "YES" ]]; then + debug_echo "[js] JavaScript package auditing is not available in offline mode" +else + for i in "${!scan_paths[@]}"; do - path="$(dirname -- $file)" - cd $path - - audit_command="npm" - - case $file in - "./package-lock.json") - audit_command="npm" - ;; - "./pnpm-lock.yaml") - audit_command="pnpm" - ;; - "./yarn.lock") - audit_command="yarn" - ;; - esac - - echo -e "\n>>>>>> Begin ${audit_command} audit output for ${scan_path} >>>>>>\n" >> ${REPORT_PATH} - - eval "${audit_command} audit ${AUDIT_ARGS} >> ${REPORT_PATH} 2>&1" - - NRC=$? - RC=$(bumprc $RC $NRC) - - cd ${scan_path} - - echo -e "\n<<<<<< End ${audit_command} audit output for ${scan_path} <<<<<<\n" >> ${REPORT_PATH} + scan_path=${scan_paths[$i]} + cd ${scan_path} + for file in $(find . \ + -iname "package-lock.json" -o \ + -iname "pnpm-lock.yaml" -o \ + -iname "yarn.lock"); + do + path="$(dirname -- $file)" + cd $path + + audit_command="npm" + + case $file in + "./package-lock.json") + audit_command="npm" + ;; + "./pnpm-lock.yaml") + audit_command="pnpm" + ;; + "./yarn.lock") + audit_command="yarn" + ;; + esac + + echo -e "\n>>>>>> Begin ${audit_command} audit output for ${scan_path} >>>>>>\n" >> ${REPORT_PATH} + + eval "${audit_command} audit ${AUDIT_ARGS} >> ${REPORT_PATH} 2>&1" + + NRC=$? + RC=$(bumprc $RC $NRC) + + cd ${scan_path} + + echo -e "\n<<<<<< End ${audit_command} audit output for ${scan_path} <<<<<<\n" >> ${REPORT_PATH} + done done -done - +fi # cd back to the original SOURCE_DIR in case path changed during scan cd ${_ASH_SOURCE_DIR} diff --git a/utils/yaml-docker-execute.sh b/utils/yaml-docker-execute.sh index 3856ed5..2f44c8d 100644 --- a/utils/yaml-docker-execute.sh +++ b/utils/yaml-docker-execute.sh @@ -101,7 +101,9 @@ do -not -path "./.external_modules/*") 2>/dev/null)) checkov_files=( ${checkov_files[@]} ${cfn_files[@]} ) - if [ "${#checkov_files[@]}" -gt 0 ]; then + if [[ $OFFLINE == "YES" ]]; then + debug_echo "[yaml] Checkov scans are not available in offline mode" + elif [ "${#checkov_files[@]}" -gt 0 ]; then echo "found ${#checkov_files[@]} files to scan. Starting checkov scans ..." >> ${REPORT_PATH} ##HACK Overcomes the String length limitation default of 10000 characters so false negatives cannot occur from large resource policies. ##Vendor Issue: https://github.com/bridgecrewio/checkov/issues/5627 From a47771b6cfeb7368a61d95262ee91450eae33ff6 Mon Sep 17 00:00:00 2001 From: Madison Steiner Date: Sun, 22 Sep 2024 08:58:30 -0700 Subject: [PATCH 20/37] Fail on old builds --- ash-multi | 1 + 1 file changed, 1 insertion(+) diff --git a/ash-multi b/ash-multi index c2b7fea..5e11cee 100755 --- a/ash-multi +++ b/ash-multi @@ -165,6 +165,7 @@ validate_input() { echo -e "${RED}Please ensure you have pulled in the latest ASH changes and rebuilt your container since v1.5.2." elif [[ "$(($(date +%s)-"${BUILD_DATE_EPOCH}"))" -gt "604800" ]]; then # Greater than 7 days echo -e "${RED}Your build is over 7 days old. Please rebuild your ASH offline container." + exit 1 fi fi if [[ -z ${PRESERVE_FILE} ]]; then AGGREGATED_RESULTS_REPORT_FILENAME="aggregated_results.txt"; else AGGREGATED_RESULTS_REPORT_FILENAME="aggregated_results-$(date +%s).txt"; fi From c1f3980bfe876c2b9280a38ed975cf2afd3127dc Mon Sep 17 00:00:00 2001 From: Madison Steiner Date: Wed, 9 Oct 2024 12:56:40 -0700 Subject: [PATCH 21/37] Add override for build expiration seconds --- ash-multi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ash-multi b/ash-multi index 5e11cee..a1edcce 100755 --- a/ash-multi +++ b/ash-multi @@ -163,7 +163,7 @@ validate_input() { if [[ ${OFFLINE} == "YES" ]]; then if [[ ${BUILD_DATE_EPOCH} == -1 ]]; then echo -e "${RED}Please ensure you have pulled in the latest ASH changes and rebuilt your container since v1.5.2." - elif [[ "$(($(date +%s)-"${BUILD_DATE_EPOCH}"))" -gt "604800" ]]; then # Greater than 7 days + elif [[ "$(($(date +%s)-"${BUILD_DATE_EPOCH}"))" -gt "${BUILD_EXPIRATION_SECONDS:-604800}" ]]; then # Greater than 7 days or specified as BUILD_EXPIRATION_SECONDS env var echo -e "${RED}Your build is over 7 days old. Please rebuild your ASH offline container." exit 1 fi From 6116e5a428d829cbfe76e4ce8598ed3006a6d33b Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Wed, 23 Oct 2024 09:59:19 -0500 Subject: [PATCH 22/37] feat: #comment updated yaml-docker-execute.sh to run Checkov differently in offline mode so findings are still captured --- utils/yaml-docker-execute.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/utils/yaml-docker-execute.sh b/utils/yaml-docker-execute.sh index 2f44c8d..48c3a70 100644 --- a/utils/yaml-docker-execute.sh +++ b/utils/yaml-docker-execute.sh @@ -69,12 +69,19 @@ CFNNAG_ARGS="--print-suppression --rule-directory ${_ASH_CFNRULES_LOCATION}" debug_echo "[yaml] ASH_OUTPUT_FORMAT: '${ASH_OUTPUT_FORMAT:-text}'" if [[ "${ASH_OUTPUT_FORMAT:-text}" != "text" ]]; then debug_echo "[yaml] Output format is not 'text', setting output format options to JSON to enable easy translation into desired output format" - CHECKOV_ARGS="--output=json" + CHECKOV_ARGS="${CHECKOV_ARGS} --output=json" CFNNAG_ARGS="--output-format json ${CFNNAG_ARGS}" else CFNNAG_ARGS="--output-format txt ${CFNNAG_ARGS}" fi +if [[ $OFFLINE == "YES" ]]; then + debug_echo "[yaml] Adding --skip-download to prevent connection to Prisma Cloud during offline mode for Checkov scans" + CHECKOV_ARGS="${CHECKOV_ARGS} --skip-download" +else + CHECKOV_ARGS="${CHECKOV_ARGS} --download-external-modules True" +fi + for i in "${!scan_paths[@]}"; do scan_path=${scan_paths[$i]} @@ -101,9 +108,7 @@ do -not -path "./.external_modules/*") 2>/dev/null)) checkov_files=( ${checkov_files[@]} ${cfn_files[@]} ) - if [[ $OFFLINE == "YES" ]]; then - debug_echo "[yaml] Checkov scans are not available in offline mode" - elif [ "${#checkov_files[@]}" -gt 0 ]; then + if [ "${#checkov_files[@]}" -gt 0 ]; then echo "found ${#checkov_files[@]} files to scan. Starting checkov scans ..." >> ${REPORT_PATH} ##HACK Overcomes the String length limitation default of 10000 characters so false negatives cannot occur from large resource policies. ##Vendor Issue: https://github.com/bridgecrewio/checkov/issues/5627 @@ -116,7 +121,9 @@ do # # Run the checkov scan on the file # - checkov ${CHECKOV_ARGS} --download-external-modules True -f "${file}" >> ${REPORT_PATH} 2>&1 + checkov_call="checkov ${CHECKOV_ARGS} -f '${file}'" + debug_echo "[yaml] Running checkov ${checkov_call}" + eval $checkov_call >> ${REPORT_PATH} 2>&1 CHRC=$? echo "<<<<<< end checkov result for ${file1} <<<<<<" >> ${REPORT_PATH} RC=$(bumprc $RC $CHRC) From e8b462184e05b3d9107de22a4c6698f0661696fc Mon Sep 17 00:00:00 2001 From: Rafael Pereyra Date: Mon, 28 Oct 2024 14:10:09 -0400 Subject: [PATCH 23/37] Fixed Grype database permissions --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1635396..6830f03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,7 @@ ENV OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" # ARG UID=500 ARG GID=100 +ARG ASHUSER_HOME=/home/ash-user # # Setting timezone in the container to UTC to ensure logged times are universal. @@ -135,7 +136,6 @@ RUN echo "gem: --no-document" >> /etc/gemrc && \ # # Grype/Syft/Semgrep # -ENV HOME="/root" ENV GRYPE_DB_CACHE_DIR="${HOME}/.grype" ENV SEMGREP_RULES_CACHE_DIR="${HOME}/.semgrep" @@ -204,7 +204,6 @@ RUN chmod -R +r /ash && chmod +x /ash/ash # And add GitHub's public fingerprints to known_hosts inside the image to prevent fingerprint # confirmation requests unexpectedly # -ARG ASHUSER_HOME=/home/ash-user # ignore a failure to add the group RUN addgroup --gid ${GID} ash-group || : RUN adduser --disabled-password --disabled-login \ @@ -225,6 +224,12 @@ USER ${UID}:${GID} # ENV HOME=${ASHUSER_HOME} +# +# Set the location for Grype DB inside ASH_USER folder +# +ENV GRYPE_DB_CACHE_DIR=${HOME}/.grypedb +RUN mkdir -p ${GRYPE_DB_CACHE_DIR} + # # Flag ASH as local execution mode since we are running in a container already # From 9e02015599172619c4b4f3f3d78905bc078010f8 Mon Sep 17 00:00:00 2001 From: Rafael Pereyra Date: Mon, 28 Oct 2024 16:54:07 -0400 Subject: [PATCH 24/37] Added environment variable for ash user and group --- Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6830f03..3da04dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,9 @@ ENV OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" # ARG UID=500 ARG GID=100 -ARG ASHUSER_HOME=/home/ash-user +ARG ASH_USER=ash-user +ARG ASH_GROUP=ash-group +ARG ASHUSER_HOME=/home/${ASH_USER} # # Setting timezone in the container to UTC to ensure logged times are universal. @@ -205,10 +207,10 @@ RUN chmod -R +r /ash && chmod +x /ash/ash # confirmation requests unexpectedly # # ignore a failure to add the group -RUN addgroup --gid ${GID} ash-group || : +RUN addgroup --gid ${GID} ${ASH_GROUP} || : RUN adduser --disabled-password --disabled-login \ --uid ${UID} --gid ${GID} \ - ash-user && \ + ${ASH_USER} && \ mkdir -p ${ASHUSER_HOME}/.ssh && \ echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ @@ -223,7 +225,8 @@ USER ${UID}:${GID} # Set the HOME environment variable to be the HOME folder for the non-root user # ENV HOME=${ASHUSER_HOME} - +ENV ASH_USER=${ASH_USER} +ENV ASH_GROUP=${ASH_GROUP} # # Set the location for Grype DB inside ASH_USER folder # From 450311ffb74643041cc95dee97dbcec0cb8f0dcc Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 22:15:57 -0500 Subject: [PATCH 25/37] updated Dockerfile to multi-stage build to support generating CI targeted images --- Dockerfile | 118 ++++++++++++++++++++--------------------------------- 1 file changed, 45 insertions(+), 73 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3da04dc..7afa4fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,63 +1,27 @@ #checkov:skip=CKV_DOCKER_7: Base image is using a non-latest version tag by default, Checkov is unable to parse due to the use of ARG -# -# Enable BASE_IMAGE as an overrideable ARG for proxy cache + private registry support -# ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye - +# First stage: Build poetry requirements FROM ${BASE_IMAGE} as poetry-reqs - ENV PYTHONDONTWRITEBYTECODE 1 - RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y \ - python3-venv && \ + apt-get install -y python3-venv && \ rm -rf /var/lib/apt/lists/* - RUN python3 -m pip install -U pip poetry - WORKDIR /src - -COPY pyproject.toml pyproject.toml -COPY poetry.lock poetry.lock -COPY README.md README.md +COPY pyproject.toml poetry.lock README.md ./ COPY src/ src/ - RUN poetry build - -FROM ${BASE_IMAGE} as ash +# Second stage: Core ASH image +FROM ${BASE_IMAGE} as core SHELL ["/bin/bash", "-c"] ARG OFFLINE="NO" ARG OFFLINE_SEMGREP_RULESETS="p/ci" - ENV OFFLINE="${OFFLINE}" ENV OFFLINE_AT_BUILD_TIME="${OFFLINE}" ENV OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" -# -# Allow for UID and GID to be set as build arguments to this Dockerfile. -# -# Set the default values of UID/GID to non-zero (and above typical low-number UID/GID values which -# are defined by default). 500 and 100 are not special values or depended upon. -# Their selection as default values is only to set the default for this container to be non-zero. -# -# The actions performed in this image (CMD command) do not require root privileges. -# Thus, UID/GID are set non-zero to avoid someone inadvertantly running this image in a container -# that runs privileged and as as root(UID 0). -# -# If the environment in which the container will run requires a specific UID/GID, then --build-arg -# arguments on the ${OCI_RUNNER} build command can be used to over-ride these values. -# -ARG UID=500 -ARG GID=100 -ARG ASH_USER=ash-user -ARG ASH_GROUP=ash-group -ARG ASHUSER_HOME=/home/${ASH_USER} - -# -# Setting timezone in the container to UTC to ensure logged times are universal. -# ENV TZ=UTC RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -136,10 +100,11 @@ RUN echo "gem: --no-document" >> /etc/gemrc && \ # # -# Grype/Syft/Semgrep +# Grype/Syft/Semgrep - Also sets default location env vars for root user for CI compat # -ENV GRYPE_DB_CACHE_DIR="${HOME}/.grype" -ENV SEMGREP_RULES_CACHE_DIR="${HOME}/.semgrep" +ENV GRYPE_DB_CACHE_DIR="/deps/.grype" +ENV SEMGREP_RULES_CACHE_DIR="/deps/.semgrep" +RUN mkdir -p ${GRYPE_DB_CACHE_DIR} ${SEMGREP_RULES_CACHE_DIR} RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | \ sh -s -- -b /usr/local/bin @@ -198,7 +163,40 @@ RUN python3 -m pip install *.whl && rm *.whl # # Make sure the ash script is executable # -RUN chmod -R +r /ash && chmod +x /ash/ash +RUN chmod -R 777 /ash /src /out /deps && chmod +x /ash/ash + +# +# Flag ASH as local execution mode since we are running in a container already +# +ENV _ASH_EXEC_MODE="local" + +# +# Append /ash to PATH to allow calling `ash` directly +# +ENV PATH="$PATH:/ash" + + +# CI stage -- any customizations specific to CI platform compatibility should be added +# in this stage if it is not applicable to ASH outside of CI usage +FROM core as ci + +ENV ASH_TARGET=ci + + +# Final stage: Local development final version. This image contains all dependencies +# for ASH, but ensures it is launched as a non-root user for local runs. +# Running as a non-root user impacts the ability to run ASH reliably across CI +# platforms and other orchestrators where the initialization and launch of the image +# is not configurable for customizing the running UID/GID. +FROM core as non-root + +ENV ASH_TARGET=non-root + +ARG UID=500 +ARG GID=100 +ARG ASH_USER=ash-user +ARG ASH_GROUP=ash-group +ARG ASHUSER_HOME=/home/${ASH_USER} # # Create a non-root user in the container and run as this user @@ -222,41 +220,15 @@ WORKDIR ${ASHUSER_HOME} USER ${UID}:${GID} # -# Set the HOME environment variable to be the HOME folder for the non-root user +# Set the HOME environment variable to be the HOME folder for the non-root user, +# along with any additional details that were set to root user values by default # ENV HOME=${ASHUSER_HOME} ENV ASH_USER=${ASH_USER} ENV ASH_GROUP=${ASH_GROUP} -# -# Set the location for Grype DB inside ASH_USER folder -# -ENV GRYPE_DB_CACHE_DIR=${HOME}/.grypedb -RUN mkdir -p ${GRYPE_DB_CACHE_DIR} - -# -# Flag ASH as local execution mode since we are running in a container already -# -ENV _ASH_EXEC_MODE="local" - -# -# Append /ash to PATH to allow calling `ash` directly -# -ENV PATH="$PATH:/ash" -# nosemgrep HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ CMD type ash || exit 1 -# -# The ENTRYPOINT needs to be NULL for CI platform support -# This needs to be an empty array ([ ]), as nerdctl-based runners will attempt to -# resolve an empty string in PATH, unlike Docker which treats an empty string the -# same as a literal NULL -# ENTRYPOINT [ ] - -# -# CMD will be run when invoking it via `$OCI_RUNNER run ...`, but will -# be overridden during CI execution when used as the job image directly. -# CMD [ "ash" ] From 7072698ccd2cfbf16c3861d79579c4f9436ebf2f Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 23:00:15 -0500 Subject: [PATCH 26/37] updated help docs in code and readme --- Dockerfile | 4 ++-- README.md | 50 +++++++++++++++++++++++++++++----------------- ash | 15 ++++++++++---- ash-multi | 58 ++++++++++++++++++++++++++++++++++-------------------- 4 files changed, 82 insertions(+), 45 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7afa4fa..4b5abac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -183,8 +183,8 @@ FROM core as ci ENV ASH_TARGET=ci -# Final stage: Local development final version. This image contains all dependencies -# for ASH, but ensures it is launched as a non-root user for local runs. +# Final stage: Non-root user final version. This image contains all dependencies +# for ASH from the `core` stage, but ensures it is launched as a non-root user. # Running as a non-root user impacts the ability to run ASH reliably across CI # platforms and other orchestrators where the initialization and launch of the image # is not configurable for customizing the running UID/GID. diff --git a/README.md b/README.md index 6f1c8bd..69d49c3 100644 --- a/README.md +++ b/README.md @@ -208,25 +208,39 @@ ash --source-dir . --ext py ```text NAME: - ash + ash SYNOPSIS: - ash [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir + ash [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir OPTIONS: - -v | --version Prints version number. - - -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. - --source-dir Path to the directory containing the code/files you wish to scan. Defaults to $(pwd) - --output-dir Path to the directory that will contain the report of the scans. Defaults to $(pwd) - --ext | -extension Force a file extension to scan. Defaults to identify files automatically. - --offline Build ASH for offline execution. Defaults to false. - --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Defaults to 'p/ci'. - --force Rebuild the Docker images of the scanning tools, to make sure software is up-to-date. - --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. - --debug Print ASH debug log information where applicable. - -q | --quiet Don't print verbose text about the build process. - -c | --no-color Don't print colorized output. - -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. - -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. + --source-dir Path to the directory containing the code/files you wish to scan. Defaults to $(pwd) + --output-dir Path to the directory that will contain the report of the scans. Defaults to $(pwd) + + --format Output format of the aggregated_results file segments. + Options: text, json + Default: text + + --target Specify the target stage of the ASH image to build. + Options: non-root, ci + Default: non-root + + --offline Build ASH for offline execution. Defaults to false. + --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. + Default: p/ci + + --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. + --ext | -extension Force a file extension to scan. Defaults to identify files automatically. + + -d | --debug Print ASH debug log information where applicable. + -q | --quiet Don't print verbose text about the build process. + -v | --version Prints version number. + + -c | --no-color Don't print colorized output. + -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. + -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. + -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. + +INFO: + For more information, please visit https://github.com/awslabs/automated-security-helper ``` ## FAQ @@ -260,7 +274,7 @@ OPTIONS: - Q: How to run `ash` in an environment without internet connectivity/with an airgap? - A: From your environment which does have internet connectivity, build the ASH image using `--offline` and `--offline-semgrep-rulesets` to specify what resources to package into the image. Environment variable `$ASH_IMAGE_NAME` controls the name of the image. After building, push to your container repository of choice which will be available within the airgapped environment. When you go to execute ASH in your offline environment, passing `--no-build` to `ash` alongside `--offline` and `--offline-semgrep-rulesets` will use your offline image and skip the build. Specify `$ASH_IMAGE_NAME` to override ASH's container image to the previously-built image available within your airgapped environment. + A: From your environment which does have internet connectivity, build the ASH image using `--offline` and `--offline-semgrep-rulesets` to specify what resources to package into the image. Environment variable `$ASH_IMAGE_NAME` controls the name of the image. After building, push to your container repository of choice which will be available within the airgapped environment. When you go to execute ASH in your offline environment, passing `--no-build` to `ash` alongside `--offline` and `--offline-semgrep-rulesets` will use your offline image and skip the build. Specify `$ASH_IMAGE_NAME` to override ASH's container image to the previously-built image available within your airgapped environment. ## Feedback diff --git a/ash b/ash index fd4237d..74ce43a 100755 --- a/ash +++ b/ash @@ -5,7 +5,6 @@ # Resolve the absolute path of the parent of the script directory (ASH repo root) export ASH_ROOT_DIR="$(cd "$(dirname "$0")"; pwd)" export ASH_UTILS_DIR="${ASH_ROOT_DIR}/utils" -export ASH_IMAGE_NAME=${ASH_IMAGE_NAME:-"automated-security-helper:local"} # Set local variables SOURCE_DIR="" @@ -22,6 +21,7 @@ NO_RUN="NO" DEBUG="NO" OFFLINE="NO" OFFLINE_SEMGREP_RULESETS="p/ci" +TARGET_STAGE="non-root" # Parse arguments while (("$#")); do case $1 in @@ -69,13 +69,17 @@ while (("$#")); do --no-run) NO_RUN="YES" ;; - --debug) + --debug|-d) DEBUG="YES" ;; --format) shift OUTPUT_FORMAT="$1" ;; + --target) + shift + TARGET_STAGE="$1" + ;; --help | -h) source "${ASH_ROOT_DIR}/ash-multi" --help exit 0 @@ -95,6 +99,8 @@ while (("$#")); do shift done +export ASH_IMAGE_NAME=${ASH_IMAGE_NAME:-"automated-security-helper:${TARGET_STAGE}"} + # Default to the pwd if [ "${SOURCE_DIR}" = "" ]; then SOURCE_DIR="$(pwd)" @@ -138,12 +144,12 @@ else CONTAINER_GID_OPTION="" if [[ ${CONTAINER_UID_SPECIFIED} = "YES" ]]; then CONTAINER_UID_OPTION="--build-arg UID=${CONTAINER_UID}" # set the UID build-arg if --container-uid is specified - else + elif [[ "${HOST_UID}" != "" ]]; then CONTAINER_UID_OPTION="--build-arg UID=${HOST_UID}" # set the UID build-arg to the caller's UID if --container-uid is not specified fi if [[ ${CONTAINER_GID_SPECIFIED} = "YES" ]]; then CONTAINER_GID_OPTION="--build-arg GID=${CONTAINER_GID}" # set the GID build-arg if --container-gid is specified - else + elif [[ "${HOST_GID}" != "" ]]; then CONTAINER_GID_OPTION="--build-arg GID=${HOST_GID}" # set the GID build-arg to the caller's GID if --container-uid is not specified fi echo "Building image ${ASH_IMAGE_NAME} -- this may take a few minutes during the first build..." @@ -151,6 +157,7 @@ else ${CONTAINER_UID_OPTION} \ ${CONTAINER_GID_OPTION} \ --tag ${ASH_IMAGE_NAME} \ + --target ${TARGET_STAGE} \ --file "${ASH_ROOT_DIR}/Dockerfile" \ --build-arg OFFLINE="${OFFLINE}" \ --build-arg OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" \ diff --git a/ash-multi b/ash-multi index f2c1ff3..0921c30 100755 --- a/ash-multi +++ b/ash-multi @@ -49,27 +49,43 @@ version_check() { } print_usage() { - echo "NAME:" - echo -e "\t$(basename $0)" - echo "SYNOPSIS:" - echo -e "\t$(basename $0) [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir" - echo "OPTIONS:" - echo -e "\t-v | --version Prints version number.\n" - echo -e "\t-p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions." - echo -e "\t--source-dir Path to the directory containing the code/files you wish to scan. Defaults to \$(pwd)" - echo -e "\t--output-dir Path to the directory that will contain the report of the scans. Defaults to \$(pwd)" - echo -e "\t--format Output format of the aggregated_results file segments. Defaults to text. Use json instead to enable parseable output." - echo -e "\t--ext | -extension Force a file extension to scan. Defaults to identify files automatically." - echo -e "\t--offline Build ASH for offline execution. Defaults to false." - echo -e "\t--offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Defaults to 'p/ci'." - echo -e "\t--force Rebuild the Docker images of the scanning tools, to make sure software is up-to-date." - echo -e "\t--no-cleanup Don't cleanup the work directory where temp reports are stored during scans." - echo -e "\t--debug Print ASH debug log information where applicable." - echo -e "\t-q | --quiet Don't print verbose text about the build process." - echo -e "\t-c | --no-color Don't print colorized output." - echo -e "\t-s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes." - echo -e "\t-o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools.\n" - echo -e "For more information please visit https://github.com/awslabs/automated-security-helper" + local cmd=$(basename "$0") + cat <<-EOF +NAME: + ${cmd} +SYNOPSIS: + ${cmd} [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir +OPTIONS: + --source-dir Path to the directory containing the code/files you wish to scan. Defaults to \$(pwd) + --output-dir Path to the directory that will contain the report of the scans. Defaults to \$(pwd) + + --format Output format of the aggregated_results file segments. + Options: text, json + Default: text + + --target Specify the target stage of the ASH image to build. + Options: non-root, ci + Default: non-root + + --offline Build ASH for offline execution. Defaults to false. + --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. + Default: p/ci + + --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. + --ext | -extension Force a file extension to scan. Defaults to identify files automatically. + + -d | --debug Print ASH debug log information where applicable. + -q | --quiet Don't print verbose text about the build process. + -v | --version Prints version number. + + -c | --no-color Don't print colorized output. + -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. + -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. + -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. + +INFO: + For more information, please visit https://github.com/awslabs/automated-security-helper +EOF } # Find all files in the source directory. Method to list the files will be different if the source is a git repository or not From 8e0915bc649f26c1ec163fa98159c268c1241564 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 23:03:29 -0500 Subject: [PATCH 27/37] moved documentation workflow to main one to make debugging failed pipelines easier, reduce unnecessary complexity, etc --- .github/workflows/ash-build-and-scan.yml | 44 +++++++++++++++++ .github/workflows/docs-build-and-deploy.yml | 55 --------------------- 2 files changed, 44 insertions(+), 55 deletions(-) delete mode 100644 .github/workflows/docs-build-and-deploy.yml diff --git a/.github/workflows/ash-build-and-scan.yml b/.github/workflows/ash-build-and-scan.yml index 2e46d09..93ea40c 100644 --- a/.github/workflows/ash-build-and-scan.yml +++ b/.github/workflows/ash-build-and-scan.yml @@ -15,6 +15,8 @@ permissions: id-token: write security-events: write pull-requests: write +env: + PYTHON_VERSION: "3.12" jobs: build: strategy: @@ -152,3 +154,45 @@ jobs: name: ash_output path: ash_output if-no-files-found: error + + build-docs: + name: Build documentation + needs: [] + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref != 'refs/heads/main') + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Build documentation + run: mkdocs build --clean + + deploy-docs: + name: Deploy documentation + needs: [] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Deploy documentation + run: mkdocs gh-deploy --clean --force diff --git a/.github/workflows/docs-build-and-deploy.yml b/.github/workflows/docs-build-and-deploy.yml deleted file mode 100644 index b7e3284..0000000 --- a/.github/workflows/docs-build-and-deploy.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build and Deploy Documentation - -on: - pull_request: - branches: - - main - push: - branches: - - main - -env: - PYTHON_VERSION: "3.12" - -jobs: - build: - name: Build documentation - needs: [] - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref != 'refs/heads/main') - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Build documentation - run: mkdocs build --clean - - deploy: - name: Deploy documentation - needs: [] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Deploy documentation - run: mkdocs gh-deploy --clean --force From 7e29e52dd48867a84d71d0c33d3dd94c469dd9d0 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 23:06:11 -0500 Subject: [PATCH 28/37] renamed pipeline --- .github/workflows/ash-build-and-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ash-build-and-scan.yml b/.github/workflows/ash-build-and-scan.yml index 93ea40c..c0bcdff 100644 --- a/.github/workflows/ash-build-and-scan.yml +++ b/.github/workflows/ash-build-and-scan.yml @@ -1,4 +1,4 @@ -name: Build & run ASH against itself +name: ASH - Core Pipeline on: push: branches: From 8daf3ddde1da089ffe131b662c883ef69fcf8744 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 23:09:59 -0500 Subject: [PATCH 29/37] updated help docs in code and readme --- ash-multi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ash-multi b/ash-multi index 0921c30..4954b84 100755 --- a/ash-multi +++ b/ash-multi @@ -67,7 +67,9 @@ OPTIONS: Options: non-root, ci Default: non-root - --offline Build ASH for offline execution. Defaults to false. + --offline Build ASH for offline execution. + Default: false + --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Default: p/ci From 04ef4d8aebdc0231c3c98424985a510531f27e39 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 23:11:09 -0500 Subject: [PATCH 30/37] updated help docs in code and readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 69d49c3..abdb1c6 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,7 @@ ash --source-dir . --ext py ## Synopsis ```text +$ ash --help NAME: ash SYNOPSIS: @@ -223,7 +224,9 @@ OPTIONS: Options: non-root, ci Default: non-root - --offline Build ASH for offline execution. Defaults to false. + --offline Build ASH for offline execution. + Default: false + --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Default: p/ci From c6596c6676556e8e00bb942fdf09edf5b63bc318 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 29 Oct 2024 23:12:43 -0500 Subject: [PATCH 31/37] updated help docs in code and readme --- README.md | 8 ++++---- ash-multi | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index abdb1c6..c14c1b3 100644 --- a/README.md +++ b/README.md @@ -233,15 +233,15 @@ OPTIONS: --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. --ext | -extension Force a file extension to scan. Defaults to identify files automatically. - -d | --debug Print ASH debug log information where applicable. - -q | --quiet Don't print verbose text about the build process. - -v | --version Prints version number. - -c | --no-color Don't print colorized output. -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. + -d | --debug Print ASH debug log information where applicable. + -q | --quiet Don't print verbose text about the build process. + -v | --version Prints version number. + INFO: For more information, please visit https://github.com/awslabs/automated-security-helper ``` diff --git a/ash-multi b/ash-multi index 4954b84..dbb78fd 100755 --- a/ash-multi +++ b/ash-multi @@ -76,15 +76,15 @@ OPTIONS: --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. --ext | -extension Force a file extension to scan. Defaults to identify files automatically. - -d | --debug Print ASH debug log information where applicable. - -q | --quiet Don't print verbose text about the build process. - -v | --version Prints version number. - -c | --no-color Don't print colorized output. -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. + -d | --debug Print ASH debug log information where applicable. + -q | --quiet Don't print verbose text about the build process. + -v | --version Prints version number. + INFO: For more information, please visit https://github.com/awslabs/automated-security-helper EOF From aef7c492e41cbd96bcc2ed7b3186c2e8081e75fe Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Thu, 31 Oct 2024 13:11:31 -0500 Subject: [PATCH 32/37] feat: #comment adjusted permissions based on feedback --- Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b5abac..5bd3a98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -163,7 +163,7 @@ RUN python3 -m pip install *.whl && rm *.whl # # Make sure the ash script is executable # -RUN chmod -R 777 /ash /src /out /deps && chmod +x /ash/ash +RUN chmod -R 755 /ash && chmod -R 777 /src /out /deps # # Flag ASH as local execution mode since we are running in a container already @@ -212,8 +212,14 @@ RUN adduser --disabled-password --disabled-login \ mkdir -p ${ASHUSER_HOME}/.ssh && \ echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ - echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ - chown -R ${UID}:${GID} ${ASHUSER_HOME} + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts + +# Change ownership and permissions now that we are running with a non-root +# user by default. +RUN chown -R ${UID}:${GID} ${ASHUSER_HOME} /src /out /deps && \ + chgrp -R ${GID} ${ASHUSER_HOME} /src /out /deps && \ + chmod 750 -R ${ASHUSER_HOME} /src /out /deps + # Setting default WORKDIR to ${ASHUSER_HOME} WORKDIR ${ASHUSER_HOME} From 245584ab821d9e83241dc8acc1a59c2e0dfae5a1 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Fri, 1 Nov 2024 08:17:03 -0500 Subject: [PATCH 33/37] feat: #comment adjusted permissions based on feedback --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5bd3a98..55d8d71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -217,7 +217,6 @@ RUN adduser --disabled-password --disabled-login \ # Change ownership and permissions now that we are running with a non-root # user by default. RUN chown -R ${UID}:${GID} ${ASHUSER_HOME} /src /out /deps && \ - chgrp -R ${GID} ${ASHUSER_HOME} /src /out /deps && \ chmod 750 -R ${ASHUSER_HOME} /src /out /deps # Setting default WORKDIR to ${ASHUSER_HOME} From 5e5c50a150bebc71f3b1a358710dbc8ec021971c Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Fri, 1 Nov 2024 08:25:50 -0500 Subject: [PATCH 34/37] feat: #comment bumped version to 1.6.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f31c393..1e34dd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [tool.poetry] name = "automated-security-helper" -version = "1.5.1" +version = "1.6.0" description = "" authors = ["Nate Ferrell ", "Nathan Bates "] license = "Apache-2.0" From be0474d082bcd06451014572ba0c83b849160bcd Mon Sep 17 00:00:00 2001 From: Rafael Pereyra Date: Tue, 12 Nov 2024 16:01:29 -0500 Subject: [PATCH 35/37] Added Changelog information --- CHANGELOG.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a32634a..5a19411 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,19 @@ # Automated Security Helper - CHANGELOG - -- [v1.5.1](#v151) +- [v1.6.0](#v160) - [What's Changed](#whats-changed) -- [v1.5.0](#v150) +- [v1.5.1](#v151) - [What's Changed](#whats-changed-1) +- [v1.5.0](#v150) + - [What's Changed](#whats-changed-2) - [New Contributors](#new-contributors) - [v1.4.1](#v141) - - [What's Changed](#whats-changed-2) -- [v1.4.0](#v140) - [What's Changed](#whats-changed-3) -- [v1.3.3](#v133) +- [v1.4.0](#v140) - [What's Changed](#whats-changed-4) -- [v1.3.2](#v132) +- [v1.3.3](#v133) - [What's Changed](#whats-changed-5) +- [v1.3.2](#v132) + - [What's Changed](#whats-changed-6) - [New Contributors](#new-contributors-1) - [1.3.0 - 2024-04-17](#130---2024-04-17) - [Features](#features) @@ -25,6 +26,14 @@ - [1.0.5-e-06Mar2023](#105-e-06mar2023) - [1.0.1-e-10Jan2023](#101-e-10jan2023) +## v1.6.0 + +### What's Changed + +- Run ash as non-root user to comply with the security scans (ASH on ASH) and best practices. +- Create a CI version of the docker file that still runs as root to comply with the different requirements from building platforms where UID/GID cannot be modified and there are additional agents installed at runtime that requires elevated privileges. + +**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/v1.5.1...v1.6.0 ## v1.5.1 From bb4c68f304244a333ef98ac53a76d489dca0dd42 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Mon, 2 Dec 2024 13:50:01 -0600 Subject: [PATCH 36/37] feat: #comment updated changelog, CI documentation, README, version, memory settings for Node, script help functions --- CHANGELOG.md | 51 +++++++++++++++------ Dockerfile | 2 + README.md | 7 ++- ash | 2 +- ash-multi | 7 ++- docs/content/tutorials/running-ash-in-ci.md | 18 ++++++++ pyproject.toml | 2 +- 7 files changed, 71 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a19411..4d03db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,25 @@ # Automated Security Helper - CHANGELOG -- [v1.6.0](#v160) - - [What's Changed](#whats-changed) +- [v2.0.0](#v200) + - [Breaking Changes](#breaking-changes) + - [Features](#features) + - [Fixes](#fixes) - [v1.5.1](#v151) - - [What's Changed](#whats-changed-1) + - [What's Changed](#whats-changed) - [v1.5.0](#v150) - - [What's Changed](#whats-changed-2) + - [What's Changed](#whats-changed-1) - [New Contributors](#new-contributors) - [v1.4.1](#v141) - - [What's Changed](#whats-changed-3) + - [What's Changed](#whats-changed-2) - [v1.4.0](#v140) - - [What's Changed](#whats-changed-4) + - [What's Changed](#whats-changed-3) - [v1.3.3](#v133) - - [What's Changed](#whats-changed-5) + - [What's Changed](#whats-changed-4) - [v1.3.2](#v132) - - [What's Changed](#whats-changed-6) + - [What's Changed](#whats-changed-5) - [New Contributors](#new-contributors-1) - [1.3.0 - 2024-04-17](#130---2024-04-17) - - [Features](#features) - - [Fixes](#fixes) + - [Features](#features-1) + - [Fixes](#fixes-1) - [Maintenance / Internal](#maintenance--internal) - [1.2.0-e-06Mar2024](#120-e-06mar2024) - [1.1.0-e-01Dec2023](#110-e-01dec2023) @@ -26,14 +28,35 @@ - [1.0.5-e-06Mar2023](#105-e-06mar2023) - [1.0.1-e-10Jan2023](#101-e-10jan2023) -## v1.6.0 +## v2.0.0 -### What's Changed +### Breaking Changes + +- Building ASH images for use in CI platforms (or other orchestration platforms that may require elevated access within the container) now requires targeting the `ci` stage of the `Dockerfile`: + +_via `ash` CLI_ + +```sh +ash --no-run --build-target ci +``` + +_via `docker` or other OCI CLI_ -- Run ash as non-root user to comply with the security scans (ASH on ASH) and best practices. +```sh +docker build --tag automated-security-helper:ci --target ci . +``` + +### Features + +- Run ASH as non-root user to align with security best practices. - Create a CI version of the docker file that still runs as root to comply with the different requirements from building platforms where UID/GID cannot be modified and there are additional agents installed at runtime that requires elevated privileges. -**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/v1.5.1...v1.6.0 +### Fixes + +- Offline mode now skips NPM/PNPM/Yarn Audit checks (requires connection to registry to pull package information) +- NPM install during image build now restricts available memory to prevent segmentation fault + +**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/v1.5.1...v2.0.0 ## v1.5.1 diff --git a/Dockerfile b/Dockerfile index 55d8d71..9a0fc43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -144,6 +144,8 @@ RUN mkdir -p /src && \ # # Update NPM to latest COPY ./utils/cdk-nag-scan /ash/utils/cdk-nag-scan/ +# Limit memory size available for Node to prevent segmentation faults during npm install +ENV NODE_OPTIONS=--max_old_space_size=512 RUN npm install -g npm pnpm yarn && \ cd /ash/utils/cdk-nag-scan && \ npm install --quiet diff --git a/README.md b/README.md index c14c1b3..96ada5f 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ OPTIONS: Options: text, json Default: text - --target Specify the target stage of the ASH image to build. + --build-target Specify the target stage of the ASH image to build. Options: non-root, ci Default: non-root @@ -230,6 +230,11 @@ OPTIONS: --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Default: p/ci + --no-build Skip rebuild of the ASH container image, run a scan only. + Requires an existing ASH image to be present in the image cache. + --no-run Skip running a scan with ASH, build a new ASH container image only. + Useful when needing to build an ASH image for publishing to a private image registry. + --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. --ext | -extension Force a file extension to scan. Defaults to identify files automatically. diff --git a/ash b/ash index 74ce43a..df2c7ba 100755 --- a/ash +++ b/ash @@ -76,7 +76,7 @@ while (("$#")); do shift OUTPUT_FORMAT="$1" ;; - --target) + --build-target) shift TARGET_STAGE="$1" ;; diff --git a/ash-multi b/ash-multi index dbb78fd..d3dc413 100755 --- a/ash-multi +++ b/ash-multi @@ -63,7 +63,7 @@ OPTIONS: Options: text, json Default: text - --target Specify the target stage of the ASH image to build. + --build-target Specify the target stage of the ASH image to build. Options: non-root, ci Default: non-root @@ -73,6 +73,11 @@ OPTIONS: --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Default: p/ci + --no-build Skip rebuild of the ASH container image, run a scan only. + Requires an existing ASH image to be present in the image cache. + --no-run Skip running a scan with ASH, build a new ASH container image only. + Useful when needing to build an ASH image for publishing to a private image registry. + --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. --ext | -extension Force a file extension to scan. Defaults to identify files automatically. diff --git a/docs/content/tutorials/running-ash-in-ci.md b/docs/content/tutorials/running-ash-in-ci.md index fc86999..ff5984f 100644 --- a/docs/content/tutorials/running-ash-in-ci.md +++ b/docs/content/tutorials/running-ash-in-ci.md @@ -2,6 +2,24 @@ ASH supports running in CI environments as an executable container (e.g. via `docker run`) as well as via Container Job mechanisms, depending on CI platform support. +### Building ASH Container Images for CI Usage + +Building ASH images for use in CI platforms (or other orchestration platforms that may require elevated access within the container) requires targeting the `ci` stage of the `Dockerfile`. This can be done via one of the following methods from the root of the ASH repository: + +_via `ash` CLI_ + +```sh +ash --no-run --build-target ci +``` + +_via `docker` or other OCI CLI_ + +```sh +docker build --tag automated-security-helper:ci --target ci . +``` + +### Examples + Within the CI folder, there are multiple examples of running ASH scans in various CI platforms. All examples include the following: * ASH repository is cloned from GitHub alongside the repository to be scanned. diff --git a/pyproject.toml b/pyproject.toml index 1e34dd7..2b69184 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [tool.poetry] name = "automated-security-helper" -version = "1.6.0" +version = "2.0.0" description = "" authors = ["Nate Ferrell ", "Nathan Bates "] license = "Apache-2.0" From d4e28a8c6d10104de26f1a74bbd5a524b0f78992 Mon Sep 17 00:00:00 2001 From: Nate Ferrell Date: Tue, 3 Dec 2024 15:41:18 -0600 Subject: [PATCH 37/37] feat: #comment updated AS casing in Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index aaac070..84bb7a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye # First stage: Build poetry requirements -FROM ${BASE_IMAGE} as poetry-reqs +FROM ${BASE_IMAGE} AS poetry-reqs ENV PYTHONDONTWRITEBYTECODE 1 RUN apt-get update && \ apt-get upgrade -y && \ @@ -15,7 +15,7 @@ COPY src/ src/ RUN poetry build # Second stage: Core ASH image -FROM ${BASE_IMAGE} as core +FROM ${BASE_IMAGE} AS core SHELL ["/bin/bash", "-c"] ARG BUILD_DATE_EPOCH="-1" ARG OFFLINE="NO" @@ -183,7 +183,7 @@ ENV PATH="$PATH:/ash" # CI stage -- any customizations specific to CI platform compatibility should be added # in this stage if it is not applicable to ASH outside of CI usage -FROM core as ci +FROM core AS ci ENV ASH_TARGET=ci @@ -193,7 +193,7 @@ ENV ASH_TARGET=ci # Running as a non-root user impacts the ability to run ASH reliably across CI # platforms and other orchestrators where the initialization and launch of the image # is not configurable for customizing the running UID/GID. -FROM core as non-root +FROM core AS non-root ENV ASH_TARGET=non-root