diff --git a/.checkov.yml b/.checkov.yml new file mode 100644 index 0000000..8b311d2 --- /dev/null +++ b/.checkov.yml @@ -0,0 +1,31 @@ +--- +quiet: true +compact: false + +skip-check: + # https://github.com/bridgecrewio/checkov/issues/5286 + - CKV_TF_1 + # do not enforce Customer Supplied Encryption Keys (CSEK) + - CKV_GCP_37 + - CKV_GCP_38 + - CKV_GCP_80 + - CKV_GCP_81 + - CKV_GCP_83 + - CKV_GCP_84 + - CKV_GCP_85 + - CKV_GCP_90 + - CKV_GCP_91 + - CKV_GCP_93 + # do not enforce vpc flow logs + - CKV_GCP_26 + - CKV_GCP_61 + # restricted default service account is ok + - CKV_GCP_30 + # do not enforce bucket access log + - CKV_GCP_62 + # do not enforce bucket object versioning + - CKV_GCP_78 + # we want serial port console access + - CKV_GCP_35 + # base64 high entropy check causes too many false positives + - CKV_SECRET_6 diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..c1e87ef --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,16 @@ +--- +# Changes here will be overwritten by Copier +_commit: v2.4.4 +_src_path: gh:remerge/template +project_id: workflows +project_license: private +project_name: GitHub Actions Workflows +project_owner: core +project_type: default +run_workflows_for_all_branches: false +use_ansible: false +use_consul: false +use_golang: false +use_nomad: false +use_python: false +use_terraform: false diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3b57522 --- /dev/null +++ b/.envrc @@ -0,0 +1,20 @@ +# Turns on shell execution strictness. This will force the .envrc +# evaluation context to exit immediately if: +# +# - any command in a pipeline returns a non-zero exit status that is +# not otherwise handled as part of `if`, `while`, or `until` tests, +# return value negation (`!`), or part of a boolean (`&&` or `||`) +# chain. +# - any variable that has not explicitly been set or declared (with +# either `declare` or `local`) is referenced. +strict_env + +# Loads a ".env" file into the current environment +dotenv_if_exists "${PWD}"/../.env +dotenv_if_exists "${PWD}"/.env + +# Add local scripts to PATH +PATH_add "${PWD}/bin" + +# Enforce correct 1Password account +export OP_ACCOUNT=remerge.1password.com diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..30289bd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# By default all files are owned by the project owner +* @remerge/core diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 6ddbd21..dde1812 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,7 +1,8 @@ --- self-hosted-runner: - # Labels of self-hosted runner in array of string labels: - generic - - default + - self-hosted - nomad + - docker + - default diff --git a/.github/workflows/actions-doc.yml b/.github/workflows/actions-doc.yml index 8702532..55bc9e9 100644 --- a/.github/workflows/actions-doc.yml +++ b/.github/workflows/actions-doc.yml @@ -1,3 +1,4 @@ +--- name: Update github actions doc on: workflow_run: @@ -20,8 +21,8 @@ jobs: run: python ./scripts/documentation/main.py -p .github/workflows/ > ./doc.md - uses: remerge/Github-action-confluence-sync@develop with: - from: './doc.md' - to: '2306670708' + from: "./doc.md" + to: "2306670708" cloud: remerge user: ${{secrets.confluence_user_email}} token: ${{secrets.confluence_token}} diff --git a/.github/workflows/actions-lint.yml b/.github/workflows/actions-lint.yml index 9ed30a9..7f87f93 100644 --- a/.github/workflows/actions-lint.yml +++ b/.github/workflows/actions-lint.yml @@ -1,3 +1,4 @@ +--- name: Lint github action on: [push] diff --git a/.github/workflows/check-repos-i-depend-on.yml b/.github/workflows/check-repos-i-depend-on.yml index ca9e1ae..f38d90b 100644 --- a/.github/workflows/check-repos-i-depend-on.yml +++ b/.github/workflows/check-repos-i-depend-on.yml @@ -1,3 +1,4 @@ +--- # we want to run the "go get $dependency && go mod tidy && open PR if needed" # workflow for every dependency every time the main branch gets updated. # this fixes possible merge conflicts in existing branches/PRs and closes them @@ -39,32 +40,32 @@ jobs: outputs: dependencies: ${{ steps.prepare.outputs.dependencies }} steps: - - name: Set up SSH agent - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: ${{ secrets.ssh_key }} + - name: Set up SSH agent + uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: ${{ secrets.ssh_key }} - - name: Set global git config - run: git config --global url.git@github.com:.insteadof https://github.com/ + - name: Set global git config + run: git config --global url.git@github.com:.insteadof https://github.com/ - - name: Check out code - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ inputs.golang_version || env.GO_VERSION }} + - name: Check out code + uses: actions/checkout@v3 - - name: Prepare dependencies - id: prepare - run: | - DELIMITER=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - DEPS=$(go list -m -f '{{if not .Indirect}}{{if not .Main}}{{.Path}}{{end}}{{end}}' all | grep "github.com/remerge/" | awk -F'/' '{print $NF}' | awk '{print $1}' | jq -R . | jq -s .) - { - echo "dependencies<<$DELIMITER" - echo "$DEPS" - echo "$DELIMITER" - } >> "$GITHUB_OUTPUT" + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ inputs.golang_version || env.GO_VERSION }} + + - name: Prepare dependencies + id: prepare + run: | + DELIMITER=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + DEPS=$(go list -m -f '{{if not .Indirect}}{{if not .Main}}{{.Path}}{{end}}{{end}}' all | grep "github.com/remerge/" | awk -F'/' '{print $NF}' | awk '{print $1}' | jq -R . | jq -s .) + { + echo "dependencies<<$DELIMITER" + echo "$DEPS" + echo "$DELIMITER" + } >> "$GITHUB_OUTPUT" notify-self-about-all-dependencies: needs: prepare-dependencies @@ -73,15 +74,15 @@ jobs: dependency: ${{ fromJSON(needs.prepare-dependencies.outputs.dependencies) }} runs-on: ${{ inputs.os }} steps: - - run: echo ${{ matrix.dependency }} - shell: bash + - run: echo ${{ matrix.dependency }} + shell: bash - - name: Check possible dependency update - uses: remerge/check-possible-dependency-update@main - with: - repo_path: "github.com/remerge/${{ matrix.dependency }}" - repo_name: "remerge/${{ matrix.dependency }}" - assign_to: ${{ inputs.assign_to }} - app_id: ${{ secrets.app_id }} - app_key: ${{ secrets.app_key }} - ssh_key: ${{ secrets.ssh_key }} + - name: Check possible dependency update + uses: remerge/check-possible-dependency-update@main + with: + repo_path: "github.com/remerge/${{ matrix.dependency }}" + repo_name: "remerge/${{ matrix.dependency }}" + assign_to: ${{ inputs.assign_to }} + app_id: ${{ secrets.app_id }} + app_key: ${{ secrets.app_key }} + ssh_key: ${{ secrets.ssh_key }} diff --git a/.github/workflows/docker-gcr-push.yml b/.github/workflows/docker-gcr-push.yml index f62d3d4..88dc1d7 100644 --- a/.github/workflows/docker-gcr-push.yml +++ b/.github/workflows/docker-gcr-push.yml @@ -1,3 +1,4 @@ +--- name: Docker build and push to GCR on: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3870721..b32da7a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,3 +1,4 @@ +--- name: Docker build and push to GAR on: diff --git a/.github/workflows/go-build.yml b/.github/workflows/go-build.yml index fa8bc40..4816783 100644 --- a/.github/workflows/go-build.yml +++ b/.github/workflows/go-build.yml @@ -1,3 +1,4 @@ +--- name: Go app build on: @@ -17,7 +18,7 @@ on: description: "Git branches eligible to build" type: string required: false - default: 'production' + default: "production" os: description: "OS version to run the workflow on. If not provided, defaults to 'ubuntu-latest'" type: string @@ -25,8 +26,8 @@ on: required: false outputs: artifact: - description: 'Binary output artifact' - value: '${{ jobs.go-app.outputs.artifact }}' + description: "Binary output artifact" + value: "${{ jobs.go-app.outputs.artifact }}" secrets: ssh_key: description: "SSH key to use" @@ -40,34 +41,34 @@ jobs: GOPRIVATE: ${{ inputs.goprivate }} REPONAME: ${{ github.event.repository.name }} outputs: - artifact: '${{ github.event.repository.name }}.linux.amd64' + artifact: "${{ github.event.repository.name }}.linux.amd64" steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-build- + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-build- - - run: echo "GO_VERSION=$(grep '^go ' < go.mod | awk '{print $2}')" >> "$GITHUB_ENV" - - uses: actions/setup-go@v4 - with: - go-version: ${{ inputs.golang_version || env.GO_VERSION }} - - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: '${{ secrets.ssh_key }}' - - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' + - run: echo "GO_VERSION=$(grep '^go ' < go.mod | awk '{print $2}')" >> "$GITHUB_ENV" + - uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.golang_version || env.GO_VERSION }} + - uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: "${{ secrets.ssh_key }}" + - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' - - run: make dist - env: - CI_COMMIT: ${{ github.sha }} - CI_REPO: ${{ env.REPONAME }} - CI_NUM: ${{ github.run_id }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ env.REPONAME }}.linux.amd64 - path: .build/${{ env.REPONAME }}.linux.amd64 + - run: make dist + env: + CI_COMMIT: ${{ github.sha }} + CI_REPO: ${{ env.REPONAME }} + CI_NUM: ${{ github.run_id }} + - uses: actions/upload-artifact@v3 + with: + name: ${{ env.REPONAME }}.linux.amd64 + path: .build/${{ env.REPONAME }}.linux.amd64 diff --git a/.github/workflows/go-checks.yml b/.github/workflows/go-checks.yml index dd57dae..e25bb70 100644 --- a/.github/workflows/go-checks.yml +++ b/.github/workflows/go-checks.yml @@ -1,3 +1,4 @@ +--- # Runs common checks for Go package: lint (fmt, modules, static, shadowing), # tests and race. @@ -24,7 +25,7 @@ on: lint-target: type: string required: false - default: 'lint' + default: "lint" description: "Lint Make target" no-lint-gen: type: boolean @@ -69,49 +70,49 @@ jobs: name: "lint_and_test" runs-on: ${{ inputs.os }} steps: - - uses: actions/checkout@v3 - - run: echo "GO_VERSION=$(grep '^go ' < go.mod | awk '{print $2}')" >> "$GITHUB_ENV" - - name: go modules cache - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg - key: ${{ runner.os }}-go-mod-${{ github.job }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-mod-${{ github.job }}- - - name: go tools cache - uses: actions/cache@v3 - with: - path: | - ${{ github.workspace }}/.tools - key: ${{ runner.os }}-go-tools-${{ inputs.golang_version || env.GO_VERSION }}-${{ github.job }} - restore-keys: | - ${{ runner.os }}-go-tools-${{ inputs.golang_version || env.GO_VERSION }}-${{ github.job }} - - uses: actions/setup-go@v4 - with: - go-version: ${{ inputs.golang_version || env.GO_VERSION }} - - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - ruby-version: '${{ inputs.lint-gen-ruby-version }}' - if: "${{ !inputs.no-lint-gen && inputs.lint-gen-ruby-version != '' }}" + - uses: actions/checkout@v3 + - run: echo "GO_VERSION=$(grep '^go ' < go.mod | awk '{print $2}')" >> "$GITHUB_ENV" + - name: go modules cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg + key: ${{ runner.os }}-go-mod-${{ github.job }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod-${{ github.job }}- + - name: go tools cache + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }}/.tools + key: ${{ runner.os }}-go-tools-${{ inputs.golang_version || env.GO_VERSION }}-${{ github.job }} + restore-keys: | + ${{ runner.os }}-go-tools-${{ inputs.golang_version || env.GO_VERSION }}-${{ github.job }} + - uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.golang_version || env.GO_VERSION }} + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + ruby-version: "${{ inputs.lint-gen-ruby-version }}" + if: "${{ !inputs.no-lint-gen && inputs.lint-gen-ruby-version != '' }}" - - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: '${{ secrets.ssh_key }}' - if: "${{ inputs.goprivate != '' }}" + - uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: "${{ secrets.ssh_key }}" + if: "${{ inputs.goprivate != '' }}" - - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' - if: "${{ inputs.goprivate != '' }}" + - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' + if: "${{ inputs.goprivate != '' }}" - - run: make ${{ inputs.lint-target }} - - run: make gen && git diff --exit-code --name-only HEAD - if: '!inputs.no-lint-gen' - - run: 'docker-compose -f "${{ inputs.compose-file }}" up -d --build && sleep ${{ inputs.compose-wait }}' - if: "${{ !inputs.no-compose && hashFiles(inputs.compose-file) != '' }}" - - run: make test-nocache - - run: make race-nocache - if: '!inputs.no-race' - - run: 'docker-compose -f "${{ inputs.compose-file }}" down' - if: "${{ always() && !inputs.no-compose && hashFiles(inputs.compose-file) != '' }}" + - run: make ${{ inputs.lint-target }} + - run: make gen && git diff --exit-code --name-only HEAD + if: "${{ !inputs.no-lint-gen }}" + - run: 'docker-compose -f "${{ inputs.compose-file }}" up -d --build && sleep ${{ inputs.compose-wait }}' + if: "${{ !inputs.no-compose && hashFiles(inputs.compose-file) != '' }}" + - run: make test-nocache + - run: make race-nocache + if: "${{ !inputs.no-race }}" + - run: 'docker-compose -f "${{ inputs.compose-file }}" down' + if: "${{ always() && !inputs.no-compose && hashFiles(inputs.compose-file) != '' }}" diff --git a/.github/workflows/go-docker.yml b/.github/workflows/go-docker.yml index 6691a97..eeeb9cb 100644 --- a/.github/workflows/go-docker.yml +++ b/.github/workflows/go-docker.yml @@ -1,3 +1,4 @@ +--- # deprecated; don't use in new projects name: Build go binary and build + push docker image diff --git a/.github/workflows/go-lint-test.yml b/.github/workflows/go-lint-test.yml index 01454f7..c2f5973 100644 --- a/.github/workflows/go-lint-test.yml +++ b/.github/workflows/go-lint-test.yml @@ -1,3 +1,4 @@ +--- name: Go-based application lint and test on: @@ -16,7 +17,7 @@ on: fetch_depth: default: 1 type: string - description: 'Number of commits to fetch. 0 indicates all history for all branches and tags.' + description: "Number of commits to fetch. 0 indicates all history for all branches and tags." required: false os: description: "OS version to run the workflow on. If not provided, defaults to 'ubuntu-latest'" @@ -26,12 +27,12 @@ on: secrets: ssh_key: description: | - SSH key to use to build and test the model. The user of the key must have access to our other private + SSH key to use to build and test the model. The user of the key must have access to our other private repositories. required: true fake_google_service_account_json: description: | - Google service account JSON to use while testing. It should be fake credentials, unless your tests are to + Google service account JSON to use while testing. It should be fake credentials, unless your tests are to be run against a real bucket. required: false diff --git a/.github/workflows/go-modules.yml b/.github/workflows/go-modules.yml index 9a7fcfe..54cb9b9 100644 --- a/.github/workflows/go-modules.yml +++ b/.github/workflows/go-modules.yml @@ -1,3 +1,4 @@ +--- # Check that Go modules dependencies is up-to-date. # This workflow is optional and can be invoked once per week on master branch. @@ -32,24 +33,24 @@ jobs: env: GOPRIVATE: ${{ inputs.goprivate }} steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-modules- - - run: echo "GO_VERSION=$(grep '^go ' < go.mod | awk '{print $2}')" >> "$GITHUB_ENV" - - uses: actions/setup-go@v4 - with: - go-version: ${{ inputs.golang_version || env.GO_VERSION }} - - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: '${{ secrets.ssh_key }}' - - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-modules- + - run: echo "GO_VERSION=$(grep '^go ' < go.mod | awk '{print $2}')" >> "$GITHUB_ENV" + - uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.golang_version || env.GO_VERSION }} + - uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: "${{ secrets.ssh_key }}" + - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' - - run: go get -u - - run: go mod tidy -compat=${{ inputs.golang_version || env.GO_VERSION }} - - run: git diff --exit-code -G"${GOPRIVATE}" go.mod + - run: go get -u + - run: go mod tidy -compat=${{ inputs.golang_version || env.GO_VERSION }} + - run: git diff --exit-code -G"${GOPRIVATE}" go.mod diff --git a/.github/workflows/go-upload-s3.yml b/.github/workflows/go-upload-s3.yml index 8cd4724..75314d6 100644 --- a/.github/workflows/go-upload-s3.yml +++ b/.github/workflows/go-upload-s3.yml @@ -1,3 +1,4 @@ +--- name: Go upload to S3 on: @@ -7,17 +8,17 @@ on: description: "Artifact file name" type: string required: false - default: '${{ github.event.repository.name }}.linux.amd64' + default: "${{ github.event.repository.name }}.linux.amd64" aws_bucket: description: "AWS bucket" type: string required: false - default: 'remerge-artifacts-v2' + default: "remerge-artifacts-v2" directory: description: "Directory in AWS bucket" type: string required: false - default: '${{ github.repository }}/${{ github.head_ref || github.ref_name }}' + default: "${{ github.repository }}/${{ github.head_ref || github.ref_name }}" os: description: "OS version to run the workflow on. If not provided, defaults to 'ubuntu-latest'" type: string @@ -35,17 +36,17 @@ jobs: go-upload-s3: runs-on: ${{ inputs.os }} steps: - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.artifact }} - - run: chmod +x ${{ inputs.artifact }} - - run: tar -czvf ${{ inputs.artifact }}.tar.gz ${{ inputs.artifact }} - - uses: jakejarvis/s3-sync-action@master - with: - args: --acl private --follow-symlinks --delete - env: - AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_key_secret}} - AWS_S3_BUCKET: ${{ inputs.aws_bucket }} - SOURCE_DIR: './' - DEST_DIR: ${{ inputs.directory }} + - uses: actions/download-artifact@v3 + with: + name: ${{ inputs.artifact }} + - run: chmod +x ${{ inputs.artifact }} + - run: tar -czvf ${{ inputs.artifact }}.tar.gz ${{ inputs.artifact }} + - uses: jakejarvis/s3-sync-action@master + with: + args: --acl private --follow-symlinks --delete + env: + AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_key_secret}} + AWS_S3_BUCKET: ${{ inputs.aws_bucket }} + SOURCE_DIR: "./" + DEST_DIR: ${{ inputs.directory }} diff --git a/.github/workflows/handle-possible-dependency-update.yml b/.github/workflows/handle-possible-dependency-update.yml index 53b3a2a..809b778 100644 --- a/.github/workflows/handle-possible-dependency-update.yml +++ b/.github/workflows/handle-possible-dependency-update.yml @@ -1,3 +1,4 @@ +--- name: Handle event that represents a possible dependency update on: @@ -26,7 +27,7 @@ on: env: GOPRIVATE: github.com/remerge - GO111MODULE: on + GO111MODULE: "on" jobs: create-or-update-pull-request: diff --git a/.github/workflows/model-release.yml b/.github/workflows/model-release.yml index 6a127cd..ae05cb3 100644 --- a/.github/workflows/model-release.yml +++ b/.github/workflows/model-release.yml @@ -1,3 +1,4 @@ +--- name: "[DEPRECATED] Model release DAG update" on: @@ -16,7 +17,7 @@ on: secrets: ssh_key: description: | - SSH key to use to build and test the model. The user of the key must have access to our other private + SSH key to use to build and test the model. The user of the key must have access to our other private repositories. required: true google_service_account_json: diff --git a/.github/workflows/nomad-deploy.yml b/.github/workflows/nomad-deploy.yml index ffee4f8..0b6895a 100644 --- a/.github/workflows/nomad-deploy.yml +++ b/.github/workflows/nomad-deploy.yml @@ -1,3 +1,4 @@ +--- name: Deploy to Nomad Cluster on: workflow_call: @@ -50,22 +51,22 @@ jobs: matrix: cluster_detail: ${{fromJson(inputs.cluster_details)}} steps: - - uses: actions/checkout@v3 - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ inputs.image_name }} - tags: | - type=sha + - uses: actions/checkout@v3 + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ inputs.image_name }} + tags: | + type=sha - - name: Deploy to Nomad - if: ${{ github.event_name != 'pull_request' }} - env: - NOMAD_TOKEN: ${{ secrets[matrix.cluster_detail.api_key_name] }} - NOMAD_API: ${{ matrix.cluster_detail.api_url }} - NOMAD_VAR_task_image: ${{ steps.meta.outputs.tags }} - NOMAD_VAR_cluster: ${{ matrix.cluster_detail.cluster }} - NOMAD_VAR_environment: ${{ matrix.cluster_detail.environment }} - run: | - nomad run -token="$NOMAD_TOKEN" -address="$NOMAD_API" -detach nomad.hcl + - name: Deploy to Nomad + if: ${{ github.event_name != 'pull_request' }} + env: + NOMAD_TOKEN: ${{ secrets[matrix.cluster_detail.api_key_name] }} + NOMAD_API: ${{ matrix.cluster_detail.api_url }} + NOMAD_VAR_task_image: ${{ steps.meta.outputs.tags }} + NOMAD_VAR_cluster: ${{ matrix.cluster_detail.cluster }} + NOMAD_VAR_environment: ${{ matrix.cluster_detail.environment }} + run: | + nomad run -token="$NOMAD_TOKEN" -address="$NOMAD_API" -detach nomad.hcl diff --git a/.github/workflows/nomad-pack.yml b/.github/workflows/nomad-pack.yml index 8774eab..8d6e633 100644 --- a/.github/workflows/nomad-pack.yml +++ b/.github/workflows/nomad-pack.yml @@ -1,3 +1,4 @@ +--- name: Plan and deploy to Nomad Cluster using nomad-pack on: workflow_call: diff --git a/.github/workflows/nomad.yml b/.github/workflows/nomad.yml index 3be5432..3b9285c 100644 --- a/.github/workflows/nomad.yml +++ b/.github/workflows/nomad.yml @@ -1,3 +1,4 @@ +--- name: Deploy Application to Nomad Cluster on: workflow_call: diff --git a/.github/workflows/notify-repos-that-depend-on-me.yml b/.github/workflows/notify-repos-that-depend-on-me.yml index 8abb672..e250082 100644 --- a/.github/workflows/notify-repos-that-depend-on-me.yml +++ b/.github/workflows/notify-repos-that-depend-on-me.yml @@ -1,3 +1,4 @@ +--- name: Notify repos that depend on me on: @@ -51,7 +52,7 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ inputs.golang_version || env.GO_VERSION }} - + - name: Prepare dependants id: prepare run: | @@ -75,25 +76,25 @@ jobs: dependant: ${{ fromJSON(needs.prepare-dependants.outputs.dependants) }} runs-on: ${{ inputs.os }} steps: - - run: echo ${{ matrix.dependant }} - shell: bash + - run: echo ${{ matrix.dependant }} + shell: bash - - name: Use our App Key and App ID to get an App Token - id: generate_app_token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.app_id }} - private_key: ${{ secrets.app_key }} + - name: Use our App Key and App ID to get an App Token + id: generate_app_token + uses: tibdex/github-app-token@v1 + with: + app_id: ${{ secrets.app_id }} + private_key: ${{ secrets.app_key }} - - name: Dispatch Event - uses: peter-evans/repository-dispatch@v2 - with: - token: ${{ steps.generate_app_token.outputs.token }} - repository: remerge/${{ matrix.dependant }} - event-type: possible-dependency-update - client-payload: | - { - "repo_path": "github.com/${{ github.repository }}", - "repo_name": "${{ github.repository }}", - "assign_to": "${{ inputs.assign_to }}" - } + - name: Dispatch Event + uses: peter-evans/repository-dispatch@v2 + with: + token: ${{ steps.generate_app_token.outputs.token }} + repository: remerge/${{ matrix.dependant }} + event-type: possible-dependency-update + client-payload: | + { + "repo_path": "github.com/${{ github.repository }}", + "repo_name": "${{ github.repository }}", + "assign_to": "${{ inputs.assign_to }}" + } diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..828606f --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,28 @@ +--- +name: pre-commit + +on: + pull_request: + push: + branches: [main, master] + +permissions: read-all + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Load envrc + uses: HatsuneMiku3939/direnv-action@v1 + + - name: Setup pre-commit + run: python -m pip install pre-commit + + - name: Install project dependencies + run: make install + + - name: Run pre-commit checks + uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/pygo-lint-test.yml b/.github/workflows/pygo-lint-test.yml index b6f712a..d36748c 100644 --- a/.github/workflows/pygo-lint-test.yml +++ b/.github/workflows/pygo-lint-test.yml @@ -1,3 +1,4 @@ +--- name: Python-based model lint and test on: @@ -21,7 +22,7 @@ on: fetch_depth: default: 1 type: string - description: 'Number of commits to fetch. 0 indicates all history for all branches and tags.' + description: "Number of commits to fetch. 0 indicates all history for all branches and tags." required: false os: description: "OS version to run the workflow on. If not provided, defaults to 'ubuntu-latest'" @@ -31,12 +32,12 @@ on: secrets: ssh_key: description: | - SSH key to use to build and test the model. The user of the key must have access to our other private + SSH key to use to build and test the model. The user of the key must have access to our other private repositories. required: true fake_google_service_account_json: description: | - Google service account JSON to use while testing. It should be fake credentials, unless your tests are to + Google service account JSON to use while testing. It should be fake credentials, unless your tests are to be run against a real bucket. required: false pypi_username: @@ -84,7 +85,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ inputs.python_version || env.PYTHON_VERSION }} - cache: 'poetry' + cache: "poetry" - name: Setup SSH uses: webfactory/ssh-agent@v0.6.0 with: diff --git a/.github/workflows/python-library-release.yml b/.github/workflows/python-library-release.yml index eb3663b..6fe5e70 100644 --- a/.github/workflows/python-library-release.yml +++ b/.github/workflows/python-library-release.yml @@ -1,3 +1,4 @@ +--- name: Python Library Release on: @@ -12,27 +13,27 @@ on: description: "Git branches eligible for release" type: string required: false - default: 'production' + default: "production" aws_bucket: description: "AWS bucket" type: string required: false - default: 'remerge-artifacts-v2' + default: "remerge-artifacts-v2" bucket_directory: description: "Directory in AWS bucket" type: string required: false - default: '${{ github.repository }}/${{ github.head_ref || github.ref_name }}' + default: "${{ github.repository }}/${{ github.head_ref || github.ref_name }}" pypi_aws_bucket: description: "AWS bucket for PyPi server" type: string required: false - default: 'remerge-pypi' + default: "remerge-pypi" pypi_bucket_directory: description: "Directory in AWS bucket for PyPi server" type: string required: false - default: '${{ github.repository }}' + default: "${{ github.repository }}" os: description: "OS version to run the workflow on. If not provided, defaults to 'ubuntu-latest'" type: string @@ -45,7 +46,7 @@ on: secrets: ssh_key: description: | - SSH key to use to build and test the model. The user of the key must have access to our other private + SSH key to use to build and test the model. The user of the key must have access to our other private repositories. required: true pypi_username: @@ -80,7 +81,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ inputs.python_version || env.PYTHON_VERSION }} - cache: 'poetry' + cache: "poetry" - name: Setup SSH uses: webfactory/ssh-agent@v0.6.0 with: @@ -130,7 +131,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_id }} AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_key_secret}} AWS_S3_BUCKET: ${{ inputs.aws_bucket }} - SOURCE_DIR: './dist/' + SOURCE_DIR: "./dist/" DEST_DIR: ${{ inputs.bucket_directory }} - name: Upload the artifact to S3 for PyPi @@ -141,7 +142,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_id }} AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_key_secret}} AWS_S3_BUCKET: ${{ inputs.pypi_aws_bucket }} - SOURCE_DIR: './dist/' + SOURCE_DIR: "./dist/" DEST_DIR: ${{ inputs.pypi_bucket_directory }} - name: Trigger the PyPi server refresh run: | diff --git a/.github/workflows/python-lint-test.yml b/.github/workflows/python-lint-test.yml index 6b526f0..269c6d2 100644 --- a/.github/workflows/python-lint-test.yml +++ b/.github/workflows/python-lint-test.yml @@ -1,3 +1,4 @@ +--- name: Python-based application lint and test on: @@ -11,7 +12,7 @@ on: fetch_depth: default: 1 type: string - description: 'Number of commits to fetch. 0 indicates all history for all branches and tags.' + description: "Number of commits to fetch. 0 indicates all history for all branches and tags." required: false os: description: "OS version to run the workflow on. If not provided, defaults to 'ubuntu-latest'" @@ -21,12 +22,12 @@ on: secrets: ssh_key: description: | - SSH key to use to build and test the model. The user of the key must have access to our other private + SSH key to use to build and test the model. The user of the key must have access to our other private repositories. required: true fake_google_service_account_json: description: | - Google service account JSON to use while testing. It should be fake credentials, unless your tests are to + Google service account JSON to use while testing. It should be fake credentials, unless your tests are to be run against a real bucket. required: false pypi_username: @@ -54,7 +55,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ inputs.python_version || env.PYTHON_VERSION }} - cache: 'poetry' + cache: "poetry" - name: Setup SSH uses: webfactory/ssh-agent@v0.6.0 with: diff --git a/.github/workflows/slack-notify.yml b/.github/workflows/slack-notify.yml index d5d3ecb..5edf0c6 100644 --- a/.github/workflows/slack-notify.yml +++ b/.github/workflows/slack-notify.yml @@ -1,3 +1,4 @@ +--- name: slack-notify on: diff --git a/.github/workflows/upstream-check.yml b/.github/workflows/upstream-check.yml index e1a3de7..daaec1b 100644 --- a/.github/workflows/upstream-check.yml +++ b/.github/workflows/upstream-check.yml @@ -1,3 +1,4 @@ +--- # Check that specific directory is up-to-date with specific Git repo. # This workflow is optional and can be invoked once per week on master branch. @@ -38,14 +39,14 @@ jobs: upstream: runs-on: ${{ inputs.os }} steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: '${{ secrets.ssh_key }}' - - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' + - uses: actions/checkout@v3 + with: + persist-credentials: false + - uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: "${{ secrets.ssh_key }}" + - run: 'git config --global url."git@github.com:".insteadOf "https://github.com/"' - - run: git remote add ${{ inputs.upstream_name }} https://${{ inputs.repo }}.git - - run: git fetch -a -p ${{ inputs.upstream_name }} - - run: git diff --exit-code --name-only --relative HEAD:${{ inputs.directory }}/ ${{ inputs.upstream_name }}/${{ inputs.branch }}:./ + - run: git remote add ${{ inputs.upstream_name }} https://${{ inputs.repo }}.git + - run: git fetch -a -p ${{ inputs.upstream_name }} + - run: git diff --exit-code --name-only --relative HEAD:${{ inputs.directory }}/ ${{ inputs.upstream_name }}/${{ inputs.branch }}:./ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1730cf --- /dev/null +++ b/.gitignore @@ -0,0 +1,575 @@ + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Archives.gitignore + +# It's better to unpack these files and commit the raw source because +# git has its own built in compression methods. +*.7z +*.jar +*.rar +*.zip +*.gz +*.gzip +*.tgz +*.bzip +*.bzip2 +*.bz2 +*.xz +*.lzma +*.cab +*.xar + +# Packing-only formats +*.iso +*.tar + +# Package management formats +*.dmg +*.xpi +*.gem +*.egg +*.deb +*.rpm +*.msi +*.msm +*.msp +*.txz + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Backup.gitignore + +*.bak +*.gho +*.ori +*.orig +*.tmp + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Diff.gitignore + +*.patch +*.diff + +## https://raw.githubusercontent.com/github/gitignore/main/Global/GPG.gitignore + +secring.* + + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Images.gitignore + +# JPEG +*.jpg +*.jpeg +*.jpe +*.jif +*.jfif +*.jfi + +# JPEG 2000 +*.jp2 +*.j2k +*.jpf +*.jpx +*.jpm +*.mj2 + +# JPEG XR +*.jxr +*.hdp +*.wdp + +# Graphics Interchange Format +*.gif + +# RAW +*.raw + +# Web P +*.webp + +# Portable Network Graphics +*.png + +# Animated Portable Network Graphics +*.apng + +# Multiple-image Network Graphics +*.mng + +# Tagged Image File Format +*.tiff +*.tif + +# Scalable Vector Graphics +*.svg +*.svgz + +# Portable Document Format +*.pdf + +# X BitMap +*.xbm + +# BMP +*.bmp +*.dib + +# ICO +*.ico + +# 3D Images +*.3dm +*.max + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Linux.gitignore + +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +## https://raw.githubusercontent.com/github/gitignore/main/Global/macOS.gitignore + +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +## https://raw.githubusercontent.com/github/gitignore/main/Global/MicrosoftOffice.gitignore + +*.tmp + +# Word temporary +~$*.doc* + +# Word Auto Backup File +Backup of *.doc* + +# Excel temporary +~$*.xls* + +# Excel Backup File +*.xlk + +# PowerPoint temporary +~$*.ppt* + +# Visio autosave temporary files +*.~vsd* + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Patch.gitignore + +*.orig +*.rej + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Tags.gitignore + +# Ignore tags created by etags, ctags, gtags (GNU global) and cscope +TAGS +.TAGS +!TAGS/ +tags +.tags +!tags/ +gtags.files +GTAGS +GRTAGS +GPATH +GSYMS +cscope.files +cscope.out +cscope.in.out +cscope.po.out + + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Vagrant.gitignore + +# General +.vagrant/ + +# Log files (if you are creating logs in debug mode, uncomment this) +# *.log + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Vim.gitignore + +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +## https://raw.githubusercontent.com/github/gitignore/main/Global/VisualStudioCode.gitignore + +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Windows.gitignore + +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +## https://raw.githubusercontent.com/github/gitignore/main/Global/Xcode.gitignore + +## User settings +xcuserdata/ + +## Xcode 8 and earlier +*.xcscmblueprint +*.xccheckout + +## https://raw.githubusercontent.com/github/gitignore/main/community/OpenSSL.gitignore + +# OpenSSL-related files best not committed + +## Certificate Authority +*.ca + +## Certificate +*.crt + +## Certificate Sign Request +*.csr + +## Certificate +*.der + +## Key database file +*.kdb + +## OSCP request data +*.org + +## PKCS #12 +*.p12 + +## PEM-encoded certificate data +*.pem + +## Random number seed +*.rnd + +## SSLeay data +*.ssleay + +## S/MIME message +*.smime + +## https://raw.githubusercontent.com/github/gitignore/main/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +## https://raw.githubusercontent.com/github/gitignore/main/Terraform.gitignore + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +## https://raw.githubusercontent.com/github/gitignore/main/Go.gitignore + +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# Custom ignores for Go projects +.build/ +.tools/ +cache/ + +# Ignore SSH keys +id_* +*.key + +# Ignore Terraform lock and cache from submodules +*/**/.terraform +*/**/.terraform.lock.hcl + +# Allow all .tfvars files even though the default gitignore file for Terraform +# excludes these. These files should normally not be part of version control as +# they are likely to contain sensitive data, such as password, private keys, and +# other secrets. However, we provide all sensitive variables to Terraform +# through Terraform Cloud and only use .tfvars files for non-sensitive variables +# only. +!*.tfvars +!*.tfvars.json diff --git a/.gitlint b/.gitlint new file mode 100644 index 0000000..763c0bc --- /dev/null +++ b/.gitlint @@ -0,0 +1,2 @@ +[general] +ignore=body-is-missing,title-must-not-contain-word diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..7e21f51 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,4 @@ +--- +MD013: + tables: false + code_blocks: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1c89d1f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,94 @@ +--- +default_stages: + - commit + +default_install_hook_types: + - commit-msg + - pre-commit + - prepare-commit-msg + +repos: + # https://github.com/pre-commit/pre-commit-hooks/tags + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.5.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-toml + - id: check-vcs-permalinks + - id: check-xml + - id: check-yaml + - id: destroyed-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: forbid-submodules + - id: trailing-whitespace + exclude: "^.gitignore$" + + # https://github.com/pre-commit/mirrors-prettier/tags + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.0.3" + hooks: + - id: prettier + exclude: "^project/" + + # https://github.com/igorshubovych/markdownlint-cli/tags + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: "v0.37.0" + hooks: + - id: markdownlint-fix + name: markdownlint + + # https://github.com/adrienverge/yamllint/tags + - repo: https://github.com/adrienverge/yamllint + rev: "v1.32.0" + hooks: + - id: yamllint + entry: yamllint --strict + + # https://github.com/pecigonzalo/pre-commit-shfmt/tags + - repo: https://github.com/pecigonzalo/pre-commit-shfmt + rev: "v2.1.0" + hooks: + - id: shell-fmt-go + + # https://github.com/koalaman/shellcheck-precommit/tags + - repo: https://github.com/koalaman/shellcheck-precommit + rev: "v0.9.0" + hooks: + - id: shellcheck + name: shellcheck + + # https://github.com/rhysd/actionlint/tags + - repo: https://github.com/rhysd/actionlint + rev: "v1.6.26" + hooks: + - id: actionlint-docker + name: actionlint + + # https://github.com/bridgecrewio/checkov/tags + - repo: https://github.com/bridgecrewio/checkov + rev: "3.0.24" + hooks: + - id: checkov + name: checkov + args: [--config-file=.checkov.yml] + + # https://github.com/jorisroovers/gitlint/tags + - repo: https://github.com/jorisroovers/gitlint + rev: "v0.19.1" + hooks: + - id: gitlint + + # https://github.com/get-woke/woke/tags + - repo: https://github.com/get-woke/woke + rev: "v0.19.0" + hooks: + - id: woke-from-source + args: [--config=.woke.yaml] diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..d2ab856 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +# auto generated +.copier-answers.yml diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..b52628c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,25 @@ +{ + "recommendations": [ + "aaron-bond.better-comments", + "bierner.markdown-emoji", + "DavidAnson.vscode-markdownlint", + "DotJoshJohnson.xml", + "eamodio.gitlens", + "esbenp.prettier-vscode", + "github.vscode-github-actions", + "GitHub.vscode-pull-request-github", + "jasonnutter.vscode-codeowners", + "mkhl.direnv", + "mkhl.shfmt", + "ms-azuretools.vscode-docker", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.makefile-tools", + "redhat.vscode-yaml", + "samuelcolvin.jinjahtml", + "sharat.vscode-brewfile", + "tamasfe.even-better-toml", + "timonwong.shellcheck", + "wholroyd.jinja", + "yzhang.markdown-all-in-one" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..3c24a27 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,15 @@ +{ + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[jsonc]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[markdown]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "editor.formatOnSave": true, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "files.trimTrailingWhitespace": true +} diff --git a/.woke.yaml b/.woke.yaml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/.woke.yaml @@ -0,0 +1 @@ +--- diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..2a1db60 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,16 @@ +--- +extends: default + +ignore: | + .copier-answers.yml + .cache/ + .tox/ + +rules: + comments: + min-spaces-from-content: 1 + comments-indentation: disable + line-length: disable + truthy: + allowed-values: ["true", "false"] + check-keys: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..23490db --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Contributing Guide + +The following document describes how to contribute to this repository and the +required setup for your development environment. + +This repository is generated using [`copier`](https://copier.readthedocs.io). +The [template documentation](https://github.com/remerge/template#readme) +explains how to generate and update this repository from the template. + +## Getting Started + +The [template repository](https://github.com/remerge/template) provides a +`make`-based development workflow that can be extended and customized per +project. + +The [template documentation](https://github.com/remerge/template#readme) +explains the default development workflow and all `make` targets in detail. + +To get started quickly clone this repository and use `make install check` to +install project dependencies and ensure that your development environment works. + +The following system dependencies are are not managed by this repository and +need to be installed manually. + +- [docker](https://www.docker.com/products/docker-desktop/) or access to a + working docker host +- [pre-commit](https://pre-commit.com) to run formatting and linting +- [pipx](https://pypa.github.io/pipx/) to install global dependencies +- [direnv](https://direnv.net) to ensure a working environment +- [copier](https://copier.readthedocs.io) to update this repository from the + template + +Most dependencies can be installed using [Homebrew](https://brew.sh): + +```shell +brew install --cask docker +brew install pre-commit pipx direnv copier +``` + +Once `pre-commit` hook is activated (`make pre-commit-install`), +set of formatting and linting routines is run automatically on each commit. +The step could be avoided by providing `--no-verify` flag for `git commit`. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c2b14ec --- /dev/null +++ b/Makefile @@ -0,0 +1,100 @@ +PROJECT_ID := workflows +PROJECT_NAME := GitHub Actions Workflows +PROJECT_REPO := github.com/remerge/$(PROJECT_ID) + +# Provide local development fallbacks if there are no CI variables +DEV_WHOAMI := $(shell whoami) +DEV_COMMIT := dev.$(DEV_WHOAMI).$(shell git rev-parse --short HEAD) + +# Set CI variables from GitHub Actions workflow run +CI_COMMIT ?= $(or $(GITHUB_SHA), $(DEV_COMMIT)) +CI_REPO ?= $(or $(GITHUB_REPOSITORY), $(PROJECT_REPO)) +CI_NUM ?= $(or $(GITHUB_RUN_ID), $(DEV_WHOAMI)) + +# Base URL of our Google Cloud Artifact Registry for Docker images being +# deployed as Nomad service jobs by GitHub Actions workflows or local divert +SERVICES_ARTIFACT_REGISTRY := europe-west4-docker.pkg.dev/artifact-registry-ff9b/services/ + +# Use bash instead of sh as the shell to run commands +# https://www.gnu.org/software/make/manual/html_node/Choosing-the-Shell.html +SHELL = bash + +# Show auto-generated help text when invoking make without a target +# https://www.gnu.org/software/make/manual/html_node/Special-Variables.html +.DEFAULT_GOAL := help + +.PHONY: help +help: ## generate help text from Makefile comments + @grep -hE '^[a-zA-Z_0-9%-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +# https://www.gnu.org/software/make/manual/html_node/Double_002dColon.html +.PHONY: install +install:: ## prepare environment and install dependencies + @: + +.PHONY: update +update:: ## update environment and dependencies + @: + +.PHONY: generate +generate:: ## generate documentation, configuration, schemas, etc + @: + +.PHONY: check +check:: ## run formatters and linters + @: + +.PHONY: test +test:: ## run unit and integration tests + @: + +.PHONY: build +build:: ## run build steps and create artifact + @: + +.PHONY: clean +clean:: ## remove build artifacts and caches + @: + +.PHONY: reset +reset: ## cleanup and reset repository to remote state +ifeq ($(FORCE),1) + git reset --hard @{upstream} + git clean -fdx +else + @echo -e "\u001b[41mThis is a dangerous operation – use \`make reset FORCE=1' to execute\u001b[0m" + @echo "Would execute \`git reset --hard @{upstream}'" + @echo "Would execute \`git clean -fdx'" + @git clean -ndx +endif + +## copier + +.PHONY: copier-copy +copier-copy: ## copy template without merging updates + copier recopy$(if $(FORCE), -f,)$(if $(REF), -r $(REF),) + +.PHONY: copier-update +copier-update: ## update project from copier template + copier update$(if $(FORCE), -f,)$(if $(REF), -r $(REF),) +update:: copier-update + +## pre-commit + +.PHONY: pre-commit-install +pre-commit-install: ## install pre-commit hook + pre-commit install -t pre-commit -t prepare-commit-msg -t commit-msg + +.PHONY: pre-commit-check +pre-commit-check: ## run pre commit hooks + pre-commit run --all-files +check:: pre-commit-check + +.PHONY: pre-commit-clean +pre-commit-clean: ## remove pre-commit and cached repositories + pre-commit uninstall + pre-commit clean +clean:: pre-commit-clean + +-include *.mk diff --git a/README.md b/README.md index 4a33b84..0579e31 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -# workflows -Reusable github action workflows. -https://docs.github.com/en/actions/learn-github-actions/reusing-workflows +# GitHub Actions Workflows -List of available workflows and usabe instruction can be found [here](https://remerge.atlassian.net/wiki/spaces/SRE/pages/2306670708/Reusable+Github+Actions+Workflow). +Reusable github action workflows. + + +List of available workflows and usabe instruction can be found +[here](https://remerge.atlassian.net/wiki/spaces/SRE/pages/2306670708/Reusable+Github+Actions+Workflow). diff --git a/bin/.keep b/bin/.keep new file mode 100644 index 0000000..e69de29 diff --git a/compass.yaml b/compass.yaml new file mode 100644 index 0000000..d4286fd --- /dev/null +++ b/compass.yaml @@ -0,0 +1,31 @@ +--- +name: workflows +description: GitHub Actions Workflows +configVersion: 1 +typeId: OTHER +ownerId: ari:cloud:teams::team/aa671d39-0e9e-45f1-87eb-44533a6f9f4c +fields: + lifecycle: Active + tier: 4 +links: + - name: null + type: REPOSITORY + url: "https://github.com/remerge/workflows" + - name: "" + type: CHAT_CHANNEL + url: "https://remerge.slack.com/archives/C058W7TD90U" + - name: "" + type: PROJECT + url: "https://remerge.atlassian.net/browse/CORE" + - name: "" + type: ON_CALL + url: "https://remerge.app.opsgenie.com/settings/schedule/detail/1a0808b9-5d0c-41bb-9cb8-2b86479aaeab" + - name: README + type: DOCUMENT + url: "https://github.com/remerge/workflows#readme" +relationships: + DEPENDS_ON: + # template + - ari:cloud:compass:1f02c36b-a506-45d6-bf3b-c800a6bbcfa5:component/27e24933-56f2-4b79-98cf-256bc22a164a/46894632-4cd4-405e-ab0e-f02f09ad5ac0 +labels: + - private