diff --git a/.github/workflows/ash-build-and-scan.yml b/.github/workflows/ash-build-and-scan.yml index 81168cc..c0bcdff 100644 --- a/.github/workflows/ash-build-and-scan.yml +++ b/.github/workflows/ash-build-and-scan.yml @@ -1,4 +1,4 @@ -name: Build & run ASH against itself +name: ASH - Core Pipeline on: push: branches: @@ -15,6 +15,8 @@ permissions: id-token: write security-events: write pull-requests: write +env: + PYTHON_VERSION: "3.12" jobs: build: strategy: @@ -69,7 +71,7 @@ jobs: set +e # Run ASH against itself - ./ash --source-dir $(pwd) --output-dir ash_output --debug | \ + ./ash --source-dir $(pwd) --output-dir ash_output --container-uid 1001 --container-gid 123 --debug | \ tee ash_stdout.txt # cat the output contents to build the summary markdown @@ -152,3 +154,45 @@ jobs: name: ash_output path: ash_output if-no-files-found: error + + build-docs: + name: Build documentation + needs: [] + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref != 'refs/heads/main') + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Build documentation + run: mkdocs build --clean + + deploy-docs: + name: Deploy documentation + needs: [] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Deploy documentation + run: mkdocs gh-deploy --clean --force diff --git a/.github/workflows/docs-build-and-deploy.yml b/.github/workflows/docs-build-and-deploy.yml deleted file mode 100644 index b7e3284..0000000 --- a/.github/workflows/docs-build-and-deploy.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build and Deploy Documentation - -on: - pull_request: - branches: - - main - push: - branches: - - main - -env: - PYTHON_VERSION: "3.12" - -jobs: - build: - name: Build documentation - needs: [] - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref != 'refs/heads/main') - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Build documentation - run: mkdocs build --clean - - deploy: - name: Deploy documentation - needs: [] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Deploy documentation - run: mkdocs gh-deploy --clean --force diff --git a/CHANGELOG.md b/CHANGELOG.md index a32634a..4d03db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Automated Security Helper - CHANGELOG - +- [v2.0.0](#v200) + - [Breaking Changes](#breaking-changes) + - [Features](#features) + - [Fixes](#fixes) - [v1.5.1](#v151) - [What's Changed](#whats-changed) - [v1.5.0](#v150) @@ -15,8 +18,8 @@ - [What's Changed](#whats-changed-5) - [New Contributors](#new-contributors-1) - [1.3.0 - 2024-04-17](#130---2024-04-17) - - [Features](#features) - - [Fixes](#fixes) + - [Features](#features-1) + - [Fixes](#fixes-1) - [Maintenance / Internal](#maintenance--internal) - [1.2.0-e-06Mar2024](#120-e-06mar2024) - [1.1.0-e-01Dec2023](#110-e-01dec2023) @@ -25,6 +28,35 @@ - [1.0.5-e-06Mar2023](#105-e-06mar2023) - [1.0.1-e-10Jan2023](#101-e-10jan2023) +## v2.0.0 + +### Breaking Changes + +- Building ASH images for use in CI platforms (or other orchestration platforms that may require elevated access within the container) now requires targeting the `ci` stage of the `Dockerfile`: + +_via `ash` CLI_ + +```sh +ash --no-run --build-target ci +``` + +_via `docker` or other OCI CLI_ + +```sh +docker build --tag automated-security-helper:ci --target ci . +``` + +### Features + +- Run ASH as non-root user to align with security best practices. +- Create a CI version of the docker file that still runs as root to comply with the different requirements from building platforms where UID/GID cannot be modified and there are additional agents installed at runtime that requires elevated privileges. + +### Fixes + +- Offline mode now skips NPM/PNPM/Yarn Audit checks (requires connection to registry to pull package information) +- NPM install during image build now restricts available memory to prevent segmentation fault + +**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/v1.5.1...v2.0.0 ## v1.5.1 diff --git a/Dockerfile b/Dockerfile index c18a0de..84bb7a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,30 @@ #checkov:skip=CKV_DOCKER_7: Base image is using a non-latest version tag by default, Checkov is unable to parse due to the use of ARG -#checkov:skip=CKV_DOCKER_3: ASH is focused on mounting source code into the container and scanning it, not running services. Setting USER breaks the ability for certain scanners to work correctly. -# -# Enable BASE_IMAGE as an overrideable ARG for proxy cache + private registry support -# ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye - -FROM ${BASE_IMAGE} as poetry-reqs - +# First stage: Build poetry requirements +FROM ${BASE_IMAGE} AS poetry-reqs ENV PYTHONDONTWRITEBYTECODE 1 - RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y \ - python3-venv && \ + apt-get install -y python3-venv && \ rm -rf /var/lib/apt/lists/* - RUN python3 -m pip install -U pip poetry - WORKDIR /src - -COPY pyproject.toml pyproject.toml -COPY poetry.lock poetry.lock -COPY README.md README.md +COPY pyproject.toml poetry.lock README.md ./ COPY src/ src/ - RUN poetry build - -FROM ${BASE_IMAGE} as ash +# Second stage: Core ASH image +FROM ${BASE_IMAGE} AS core SHELL ["/bin/bash", "-c"] +ARG BUILD_DATE_EPOCH="-1" ARG OFFLINE="NO" ARG OFFLINE_SEMGREP_RULESETS="p/ci" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" ENV OFFLINE="${OFFLINE}" ENV OFFLINE_AT_BUILD_TIME="${OFFLINE}" ENV OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" -# -# Setting timezone in the container to UTC to ensure logged times are universal. -# ENV TZ=UTC RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -117,11 +103,11 @@ RUN echo "gem: --no-document" >> /etc/gemrc && \ # # -# Grype/Syft/Semgrep +# Grype/Syft/Semgrep - Also sets default location env vars for root user for CI compat # -ENV HOME="/root" -ENV GRYPE_DB_CACHE_DIR="${HOME}/.grype" -ENV SEMGREP_RULES_CACHE_DIR="${HOME}/.semgrep" +ENV GRYPE_DB_CACHE_DIR="/deps/.grype" +ENV SEMGREP_RULES_CACHE_DIR="/deps/.semgrep" +RUN mkdir -p ${GRYPE_DB_CACHE_DIR} ${SEMGREP_RULES_CACHE_DIR} RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | \ sh -s -- -b /usr/local/bin @@ -161,6 +147,8 @@ RUN mkdir -p /src && \ # # Update NPM to latest COPY ./utils/cdk-nag-scan /ash/utils/cdk-nag-scan/ +# Limit memory size available for Node to prevent segmentation faults during npm install +ENV NODE_OPTIONS=--max_old_space_size=512 RUN npm install -g npm pnpm yarn && \ cd /ash/utils/cdk-nag-scan && \ npm install --quiet @@ -180,7 +168,7 @@ RUN python3 -m pip install *.whl && rm *.whl # # Make sure the ash script is executable # -RUN chmod +x /ash/ash +RUN chmod -R 755 /ash && chmod -R 777 /src /out /deps # # Flag ASH as local execution mode since we are running in a container already @@ -192,20 +180,65 @@ ENV _ASH_EXEC_MODE="local" # ENV PATH="$PATH:/ash" -# nosemgrep -HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ - CMD type ash || exit 1 + +# CI stage -- any customizations specific to CI platform compatibility should be added +# in this stage if it is not applicable to ASH outside of CI usage +FROM core AS ci + +ENV ASH_TARGET=ci + + +# Final stage: Non-root user final version. This image contains all dependencies +# for ASH from the `core` stage, but ensures it is launched as a non-root user. +# Running as a non-root user impacts the ability to run ASH reliably across CI +# platforms and other orchestrators where the initialization and launch of the image +# is not configurable for customizing the running UID/GID. +FROM core AS non-root + +ENV ASH_TARGET=non-root + +ARG UID=500 +ARG GID=100 +ARG ASH_USER=ash-user +ARG ASH_GROUP=ash-group +ARG ASHUSER_HOME=/home/${ASH_USER} # -# The ENTRYPOINT needs to be NULL for CI platform support -# This needs to be an empty array ([ ]), as nerdctl-based runners will attempt to -# resolve an empty string in PATH, unlike Docker which treats an empty string the -# same as a literal NULL +# Create a non-root user in the container and run as this user # -ENTRYPOINT [ ] +# And add GitHub's public fingerprints to known_hosts inside the image to prevent fingerprint +# confirmation requests unexpectedly +# +# ignore a failure to add the group +RUN addgroup --gid ${GID} ${ASH_GROUP} || : +RUN adduser --disabled-password --disabled-login \ + --uid ${UID} --gid ${GID} \ + ${ASH_USER} && \ + mkdir -p ${ASHUSER_HOME}/.ssh && \ + echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ + echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \ + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts + +# Change ownership and permissions now that we are running with a non-root +# user by default. +RUN chown -R ${UID}:${GID} ${ASHUSER_HOME} /src /out /deps && \ + chmod 750 -R ${ASHUSER_HOME} /src /out /deps + +# Setting default WORKDIR to ${ASHUSER_HOME} +WORKDIR ${ASHUSER_HOME} + +USER ${UID}:${GID} # -# CMD will be run when invoking it via `$OCI_RUNNER run ...`, but will -# be overridden during CI execution when used as the job image directly. +# Set the HOME environment variable to be the HOME folder for the non-root user, +# along with any additional details that were set to root user values by default # +ENV HOME=${ASHUSER_HOME} +ENV ASH_USER=${ASH_USER} +ENV ASH_GROUP=${ASH_GROUP} + +HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ + CMD type ash || exit 1 + +ENTRYPOINT [ ] CMD [ "ash" ] diff --git a/README.md b/README.md index 6f1c8bd..96ada5f 100644 --- a/README.md +++ b/README.md @@ -207,26 +207,48 @@ ash --source-dir . --ext py ## Synopsis ```text +$ ash --help NAME: - ash + ash SYNOPSIS: - ash [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir + ash [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir OPTIONS: - -v | --version Prints version number. - - -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. - --source-dir Path to the directory containing the code/files you wish to scan. Defaults to $(pwd) - --output-dir Path to the directory that will contain the report of the scans. Defaults to $(pwd) - --ext | -extension Force a file extension to scan. Defaults to identify files automatically. - --offline Build ASH for offline execution. Defaults to false. - --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Defaults to 'p/ci'. - --force Rebuild the Docker images of the scanning tools, to make sure software is up-to-date. - --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. - --debug Print ASH debug log information where applicable. - -q | --quiet Don't print verbose text about the build process. - -c | --no-color Don't print colorized output. - -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. - -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. + --source-dir Path to the directory containing the code/files you wish to scan. Defaults to $(pwd) + --output-dir Path to the directory that will contain the report of the scans. Defaults to $(pwd) + + --format Output format of the aggregated_results file segments. + Options: text, json + Default: text + + --build-target Specify the target stage of the ASH image to build. + Options: non-root, ci + Default: non-root + + --offline Build ASH for offline execution. + Default: false + + --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. + Default: p/ci + + --no-build Skip rebuild of the ASH container image, run a scan only. + Requires an existing ASH image to be present in the image cache. + --no-run Skip running a scan with ASH, build a new ASH container image only. + Useful when needing to build an ASH image for publishing to a private image registry. + + --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. + --ext | -extension Force a file extension to scan. Defaults to identify files automatically. + + -c | --no-color Don't print colorized output. + -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. + -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. + -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. + + -d | --debug Print ASH debug log information where applicable. + -q | --quiet Don't print verbose text about the build process. + -v | --version Prints version number. + +INFO: + For more information, please visit https://github.com/awslabs/automated-security-helper ``` ## FAQ @@ -260,7 +282,7 @@ OPTIONS: - Q: How to run `ash` in an environment without internet connectivity/with an airgap? - A: From your environment which does have internet connectivity, build the ASH image using `--offline` and `--offline-semgrep-rulesets` to specify what resources to package into the image. Environment variable `$ASH_IMAGE_NAME` controls the name of the image. After building, push to your container repository of choice which will be available within the airgapped environment. When you go to execute ASH in your offline environment, passing `--no-build` to `ash` alongside `--offline` and `--offline-semgrep-rulesets` will use your offline image and skip the build. Specify `$ASH_IMAGE_NAME` to override ASH's container image to the previously-built image available within your airgapped environment. + A: From your environment which does have internet connectivity, build the ASH image using `--offline` and `--offline-semgrep-rulesets` to specify what resources to package into the image. Environment variable `$ASH_IMAGE_NAME` controls the name of the image. After building, push to your container repository of choice which will be available within the airgapped environment. When you go to execute ASH in your offline environment, passing `--no-build` to `ash` alongside `--offline` and `--offline-semgrep-rulesets` will use your offline image and skip the build. Specify `$ASH_IMAGE_NAME` to override ASH's container image to the previously-built image available within your airgapped environment. ## Feedback diff --git a/ash b/ash index 1b622bc..8b0ce7c 100755 --- a/ash +++ b/ash @@ -5,12 +5,13 @@ # Resolve the absolute path of the parent of the script directory (ASH repo root) export ASH_ROOT_DIR="$(cd "$(dirname "$0")"; pwd)" export ASH_UTILS_DIR="${ASH_ROOT_DIR}/utils" -export ASH_IMAGE_NAME=${ASH_IMAGE_NAME:-"automated-security-helper:local"} # Set local variables SOURCE_DIR="" OUTPUT_DIR="" OUTPUT_DIR_SPECIFIED="NO" +CONTAINER_UID_SPECIFIED="NO" +CONTAINER_GID_SPEICIFED="NO" OUTPUT_FORMAT="text" DOCKER_EXTRA_ARGS="${DOCKER_EXTRA_ARGS:-}" DOCKER_RUN_EXTRA_ARGS="" @@ -20,6 +21,7 @@ NO_RUN="NO" DEBUG="NO" OFFLINE="NO" OFFLINE_SEMGREP_RULESETS="p/ci" +TARGET_STAGE="non-root" # Parse arguments while (("$#")); do case $1 in @@ -51,19 +53,33 @@ while (("$#")); do shift OCI_RUNNER="$1" ;; + --container-uid | -u) + shift + CONTAINER_UID_SPECIFIED="YES" + CONTAINER_UID="$1" + ;; + --container-gid | -u) + shift + CONTAINER_GID_SPECIFIED="YES" + CONTAINER_GID="$1" + ;; --no-build) NO_BUILD="YES" ;; --no-run) NO_RUN="YES" ;; - --debug) + --debug|-d) DEBUG="YES" ;; --format) shift OUTPUT_FORMAT="$1" ;; + --build-target) + shift + TARGET_STAGE="$1" + ;; --help | -h) source "${ASH_ROOT_DIR}/ash-multi" --help exit 0 @@ -83,6 +99,8 @@ while (("$#")); do shift done +export ASH_IMAGE_NAME=${ASH_IMAGE_NAME:-"automated-security-helper:${TARGET_STAGE}"} + # Default to the pwd if [ "${SOURCE_DIR}" = "" ]; then SOURCE_DIR="$(pwd)" @@ -95,6 +113,12 @@ if [[ "${OUTPUT_DIR_SPECIFIED}" == "YES" ]]; then OUTPUT_DIR="$(cd "$OUTPUT_DIR"; pwd)" fi +# +# Gather the UID and GID of the caller +# +HOST_UID=$(id -u) +HOST_GID=$(id -g) + # Resolve any offline mode flags if [[ "${OFFLINE}" == "YES" ]]; then DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} --network=none" @@ -116,15 +140,30 @@ else # Build the image if the --no-build flag is not set if [ "${NO_BUILD}" = "NO" ]; then + CONTAINER_UID_OPTION="" + CONTAINER_GID_OPTION="" + if [[ ${CONTAINER_UID_SPECIFIED} = "YES" ]]; then + CONTAINER_UID_OPTION="--build-arg UID=${CONTAINER_UID}" # set the UID build-arg if --container-uid is specified + elif [[ "${HOST_UID}" != "" ]]; then + CONTAINER_UID_OPTION="--build-arg UID=${HOST_UID}" # set the UID build-arg to the caller's UID if --container-uid is not specified + fi + if [[ ${CONTAINER_GID_SPECIFIED} = "YES" ]]; then + CONTAINER_GID_OPTION="--build-arg GID=${CONTAINER_GID}" # set the GID build-arg if --container-gid is specified + elif [[ "${HOST_GID}" != "" ]]; then + CONTAINER_GID_OPTION="--build-arg GID=${HOST_GID}" # set the GID build-arg to the caller's GID if --container-uid is not specified + fi echo "Building image ${ASH_IMAGE_NAME} -- this may take a few minutes during the first build..." ${RESOLVED_OCI_RUNNER} build \ + ${CONTAINER_UID_OPTION} \ + ${CONTAINER_GID_OPTION} \ --tag ${ASH_IMAGE_NAME} \ + --target ${TARGET_STAGE} \ --file "${ASH_ROOT_DIR}/Dockerfile" \ --build-arg OFFLINE="${OFFLINE}" \ --build-arg OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}" \ + --build-arg BUILD_DATE="$(date +%s)" \ ${DOCKER_EXTRA_ARGS} \ "${ASH_ROOT_DIR}" - eval $build_cmd fi # Run the image if the --no-run flag is not set diff --git a/ash-multi b/ash-multi index 4398b2c..67968a4 100755 --- a/ash-multi +++ b/ash-multi @@ -49,27 +49,50 @@ version_check() { } print_usage() { - echo "NAME:" - echo -e "\t$(basename $0)" - echo "SYNOPSIS:" - echo -e "\t$(basename $0) [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir" - echo "OPTIONS:" - echo -e "\t-v | --version Prints version number.\n" - echo -e "\t-p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions." - echo -e "\t--source-dir Path to the directory containing the code/files you wish to scan. Defaults to \$(pwd)" - echo -e "\t--output-dir Path to the directory that will contain the report of the scans. Defaults to \$(pwd)" - echo -e "\t--format Output format of the aggregated_results file segments. Defaults to text. Use json instead to enable parseable output." - echo -e "\t--ext | -extension Force a file extension to scan. Defaults to identify files automatically." - echo -e "\t--offline Build ASH for offline execution. Defaults to false." - echo -e "\t--offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. Defaults to 'p/ci'." - echo -e "\t--force Rebuild the Docker images of the scanning tools, to make sure software is up-to-date." - echo -e "\t--no-cleanup Don't cleanup the work directory where temp reports are stored during scans." - echo -e "\t--debug Print ASH debug log information where applicable." - echo -e "\t-q | --quiet Don't print verbose text about the build process." - echo -e "\t-c | --no-color Don't print colorized output." - echo -e "\t-s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes." - echo -e "\t-o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools.\n" - echo -e "For more information please visit https://github.com/awslabs/automated-security-helper" + local cmd=$(basename "$0") + cat <<-EOF +NAME: + ${cmd} +SYNOPSIS: + ${cmd} [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir +OPTIONS: + --source-dir Path to the directory containing the code/files you wish to scan. Defaults to \$(pwd) + --output-dir Path to the directory that will contain the report of the scans. Defaults to \$(pwd) + + --format Output format of the aggregated_results file segments. + Options: text, json + Default: text + + --build-target Specify the target stage of the ASH image to build. + Options: non-root, ci + Default: non-root + + --offline Build ASH for offline execution. + Default: false + + --offline-semgrep-rulesets Specify Semgrep rulesets for use in ASH offline mode. + Default: p/ci + + --no-build Skip rebuild of the ASH container image, run a scan only. + Requires an existing ASH image to be present in the image cache. + --no-run Skip running a scan with ASH, build a new ASH container image only. + Useful when needing to build an ASH image for publishing to a private image registry. + + --no-cleanup Don't cleanup the work directory where temp reports are stored during scans. + --ext | -extension Force a file extension to scan. Defaults to identify files automatically. + + -c | --no-color Don't print colorized output. + -s | --single-process Run ash scanners serially rather than as separate, parallel sub-processes. + -o | --oci-runner Use the specified OCI runner instead of docker to run the containerized tools. + -p | --preserve-report Add timestamp to the final report file to avoid overwriting it after multiple executions. + + -d | --debug Print ASH debug log information where applicable. + -q | --quiet Don't print verbose text about the build process. + -v | --version Prints version number. + +INFO: + For more information, please visit https://github.com/awslabs/automated-security-helper +EOF } # Find all files in the source directory. Method to list the files will be different if the source is a git repository or not @@ -101,7 +124,7 @@ map_extensions_and_files() { git config --global --add safe.directory "${_ASH_SOURCE_DIR}" >/dev/null 2>&1 git config --global --add safe.directory "${_ASH_RUN_DIR}" >/dev/null 2>&1 if [[ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" == "true" ]]; then - git clone "${_ASH_SOURCE_DIR}" "${_ASH_RUN_DIR}" + git clone "${_ASH_SOURCE_DIR}" "${_ASH_RUN_DIR}" >/dev/null 2>&1 echo "Repository cloned successfully." _ASH_SOURCE_DIR="${_ASH_RUN_DIR}" else @@ -160,6 +183,14 @@ validate_input() { echo -e "${RED}Please ensure --offline is specified at build time to be able to run in offline mode.${NC}" exit 1 fi + if [[ ${OFFLINE} == "YES" ]]; then + if [[ ${BUILD_DATE_EPOCH} == -1 ]]; then + echo -e "${RED}Please ensure you have pulled in the latest ASH changes and rebuilt your container since v1.5.2." + elif [[ "$(($(date +%s)-"${BUILD_DATE_EPOCH}"))" -gt "${BUILD_EXPIRATION_SECONDS:-604800}" ]]; then # Greater than 7 days or specified as BUILD_EXPIRATION_SECONDS env var + echo -e "${RED}Your build is over 7 days old. Please rebuild your ASH offline container." + exit 1 + fi + fi if [[ -z ${PRESERVE_FILE} ]]; then AGGREGATED_RESULTS_REPORT_FILENAME="aggregated_results.txt"; else AGGREGATED_RESULTS_REPORT_FILENAME="aggregated_results-$(date +%s).txt"; fi if [[ -z ${FORCE_REBUILD} ]]; then FORCE_REBUILD="false"; fi if [[ -z ${SOURCE_DIR} ]]; then SOURCE_DIR="$(pwd)"; else SOURCE_DIR=$(cd "${SOURCE_DIR}"; pwd); fi # Transform any relative path to absolute @@ -460,6 +491,11 @@ export _ASH_OFFLINE_SEMGREP_RULESETS=${_ASH_OFFLINE_SEMGREP_RULESETS:-${OFFLINE_ # echo -e "\n${LPURPLE}ASH version ${GREEN}$VERSION${NC}\n" +# +# Print out the uid/gid values in effect +# +echo -e "\n${LPURPLE}ASH running with UID/GID: ${CYAN}$(id -u):$(id -g)${NC}\n" + # nosemgrep IFS=$'\n' # Support directories with spaces, make the loop iterate over newline instead of space pushd . >/dev/null 2>&1 diff --git a/docs/content/tutorials/running-ash-in-ci.md b/docs/content/tutorials/running-ash-in-ci.md index fc86999..ff5984f 100644 --- a/docs/content/tutorials/running-ash-in-ci.md +++ b/docs/content/tutorials/running-ash-in-ci.md @@ -2,6 +2,24 @@ ASH supports running in CI environments as an executable container (e.g. via `docker run`) as well as via Container Job mechanisms, depending on CI platform support. +### Building ASH Container Images for CI Usage + +Building ASH images for use in CI platforms (or other orchestration platforms that may require elevated access within the container) requires targeting the `ci` stage of the `Dockerfile`. This can be done via one of the following methods from the root of the ASH repository: + +_via `ash` CLI_ + +```sh +ash --no-run --build-target ci +``` + +_via `docker` or other OCI CLI_ + +```sh +docker build --tag automated-security-helper:ci --target ci . +``` + +### Examples + Within the CI folder, there are multiple examples of running ASH scans in various CI platforms. All examples include the following: * ASH repository is cloned from GitHub alongside the repository to be scanned. diff --git a/helper_dockerfiles/Dockerfile-js b/helper_dockerfiles/Dockerfile-js index f97c251..9c3297c 100644 --- a/helper_dockerfiles/Dockerfile-js +++ b/helper_dockerfiles/Dockerfile-js @@ -1,6 +1,10 @@ # Get NPM Image FROM public.ecr.aws/docker/library/node:18.0.0 +ARG OFFLINE="NO" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" +ENV OFFLINE="${OFFLINE}" # # Make sure the default dirs are initialized # diff --git a/helper_dockerfiles/Dockerfile-yaml b/helper_dockerfiles/Dockerfile-yaml index 9807630..0b49108 100644 --- a/helper_dockerfiles/Dockerfile-yaml +++ b/helper_dockerfiles/Dockerfile-yaml @@ -1,5 +1,10 @@ # Get Ubuntu Image FROM --platform=linux/amd64 public.ecr.aws/bitnami/python:3.11 +ARG OFFLINE="NO" +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" + +ENV BUILD_DATE_EPOCH="${BUILD_DATE_EPOCH}" +ENV OFFLINE="${OFFLINE}" ENV TZ=Europe/London RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/pyproject.toml b/pyproject.toml index f31c393..2b69184 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [tool.poetry] name = "automated-security-helper" -version = "1.5.1" +version = "2.0.0" description = "" authors = ["Nate Ferrell ", "Nathan Bates "] license = "Apache-2.0" diff --git a/utils/js-docker-execute.sh b/utils/js-docker-execute.sh index 90a614d..f87b1d6 100755 --- a/utils/js-docker-execute.sh +++ b/utils/js-docker-execute.sh @@ -58,45 +58,48 @@ if [[ "${ASH_OUTPUT_FORMAT:-text}" != "text" ]]; then AUDIT_ARGS="--json ${AUDIT_ARGS}" fi -for i in "${!scan_paths[@]}"; -do - scan_path=${scan_paths[$i]} - cd ${scan_path} - for file in $(find . \ - -iname "package-lock.json" -o \ - -iname "pnpm-lock.yaml" -o \ - -iname "yarn.lock"); +if [[ $OFFLINE == "YES" ]]; then + debug_echo "[js] JavaScript package auditing is not available in offline mode" +else + for i in "${!scan_paths[@]}"; do - path="$(dirname -- $file)" - cd $path - - audit_command="npm" - - case $file in - "./package-lock.json") - audit_command="npm" - ;; - "./pnpm-lock.yaml") - audit_command="pnpm" - ;; - "./yarn.lock") - audit_command="yarn" - ;; - esac - - echo -e "\n>>>>>> Begin ${audit_command} audit output for ${scan_path} >>>>>>\n" >> ${REPORT_PATH} - - eval "${audit_command} audit ${AUDIT_ARGS} >> ${REPORT_PATH} 2>&1" - - NRC=$? - RC=$(bumprc $RC $NRC) - - cd ${scan_path} - - echo -e "\n<<<<<< End ${audit_command} audit output for ${scan_path} <<<<<<\n" >> ${REPORT_PATH} + scan_path=${scan_paths[$i]} + cd ${scan_path} + for file in $(find . \ + -iname "package-lock.json" -o \ + -iname "pnpm-lock.yaml" -o \ + -iname "yarn.lock"); + do + path="$(dirname -- $file)" + cd $path + + audit_command="npm" + + case $file in + "./package-lock.json") + audit_command="npm" + ;; + "./pnpm-lock.yaml") + audit_command="pnpm" + ;; + "./yarn.lock") + audit_command="yarn" + ;; + esac + + echo -e "\n>>>>>> Begin ${audit_command} audit output for ${scan_path} >>>>>>\n" >> ${REPORT_PATH} + + eval "${audit_command} audit ${AUDIT_ARGS} >> ${REPORT_PATH} 2>&1" + + NRC=$? + RC=$(bumprc $RC $NRC) + + cd ${scan_path} + + echo -e "\n<<<<<< End ${audit_command} audit output for ${scan_path} <<<<<<\n" >> ${REPORT_PATH} + done done -done - +fi # cd back to the original SOURCE_DIR in case path changed during scan cd ${_ASH_SOURCE_DIR} diff --git a/utils/yaml-docker-execute.sh b/utils/yaml-docker-execute.sh index 3856ed5..48c3a70 100644 --- a/utils/yaml-docker-execute.sh +++ b/utils/yaml-docker-execute.sh @@ -69,12 +69,19 @@ CFNNAG_ARGS="--print-suppression --rule-directory ${_ASH_CFNRULES_LOCATION}" debug_echo "[yaml] ASH_OUTPUT_FORMAT: '${ASH_OUTPUT_FORMAT:-text}'" if [[ "${ASH_OUTPUT_FORMAT:-text}" != "text" ]]; then debug_echo "[yaml] Output format is not 'text', setting output format options to JSON to enable easy translation into desired output format" - CHECKOV_ARGS="--output=json" + CHECKOV_ARGS="${CHECKOV_ARGS} --output=json" CFNNAG_ARGS="--output-format json ${CFNNAG_ARGS}" else CFNNAG_ARGS="--output-format txt ${CFNNAG_ARGS}" fi +if [[ $OFFLINE == "YES" ]]; then + debug_echo "[yaml] Adding --skip-download to prevent connection to Prisma Cloud during offline mode for Checkov scans" + CHECKOV_ARGS="${CHECKOV_ARGS} --skip-download" +else + CHECKOV_ARGS="${CHECKOV_ARGS} --download-external-modules True" +fi + for i in "${!scan_paths[@]}"; do scan_path=${scan_paths[$i]} @@ -114,7 +121,9 @@ do # # Run the checkov scan on the file # - checkov ${CHECKOV_ARGS} --download-external-modules True -f "${file}" >> ${REPORT_PATH} 2>&1 + checkov_call="checkov ${CHECKOV_ARGS} -f '${file}'" + debug_echo "[yaml] Running checkov ${checkov_call}" + eval $checkov_call >> ${REPORT_PATH} 2>&1 CHRC=$? echo "<<<<<< end checkov result for ${file1} <<<<<<" >> ${REPORT_PATH} RC=$(bumprc $RC $CHRC)