From a75387b5606f174a5f82a5fc23018ea50269a900 Mon Sep 17 00:00:00 2001 From: Sudip Bhattarai Date: Sat, 17 Aug 2024 16:22:04 +0545 Subject: [PATCH 1/5] Use bash script instead of ansible --- .../workflows/build-and-deploy-test-stack.yml | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-and-deploy-test-stack.yml b/.github/workflows/build-and-deploy-test-stack.yml index f003258c8..97b8b9ad4 100644 --- a/.github/workflows/build-and-deploy-test-stack.yml +++ b/.github/workflows/build-and-deploy-test-stack.yml @@ -15,39 +15,60 @@ jobs: deploy: name: Deploy app runs-on: ubuntu-latest - env: - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - SENTRY_DSN_FRONTEND: ${{ secrets.SENTRY_DSN_FRONTEND }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} - APP_ENV: test - PDF_API_URL: ${{ secrets.PDF_API_URL}} steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 + - name: Set up SSH and deploy + uses: appleboy/ssh-action@v1.0.3 with: - ssh-private-key: ${{ secrets.TEST_STACK_SSH_KEY }} - - - name: Run Ansible playbook - uses: dawidd6/action-ansible-playbook@v2 - with: - playbook: playbook.yml - directory: ./tests/test-infrastructure + host: ${{ secrets.TEST_STACK_SERVER_IP }} + username: ec2-user key: ${{ secrets.TEST_STACK_SSH_KEY }} - inventory: | - [test_server] - ${{ secrets.TEST_STACK_SERVER_IP }} ansible_user=ec2-user - options: | - --verbose + script: | + REPO_URL="https://github.com/${{ github.repository }}" + DEST_DIR="$HOME/Documents/govtool" + + # Create parent directory if it does not exist + mkdir -p "$(dirname "$DEST_DIR")" + + # Check if $DEST_DIR exists + if [ -d "$DEST_DIR" ]; then + if [ -d "$DEST_DIR/.git" ]; then + cd $DEST_DIR || exit + echo "Updating repository..." + git fetch --all + git checkout --force "$GOVTOOL_TAG" + else + echo "Not a git repository. Re-cloning..." + rm -rf "$DEST_DIR" + git clone "$REPO_URL" "$DEST_DIR" + cd "$DEST_DIR" || exit + git checkout --force "$GOVTOOL_TAG" + fi + else + echo "Directory does not exist. Cloning repository..." + git clone "$REPO_URL" "$DEST_DIR" + cd "$DEST_DIR" || exit + git checkout --force "$GOVTOOL_TAG" + fi + + # Execute the build-and-deploy.sh script + cd $DEST_DIR/tests/test-infrastructure + ./build-and-deploy.sh update-images + env: GOVTOOL_TAG: ${{ github.sha }} + GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} + GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} + GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} + SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} + GTM_ID: ${{ secrets.GTM_ID }} + NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} + SENTRY_DSN_FRONTEND: ${{ secrets.SENTRY_DSN_FRONTEND }} + PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} + APP_ENV: test + PDF_API_URL: ${{ secrets.PDF_API_URL }} From 68c6dea8175811c39c4aa637375c64e66561aa18 Mon Sep 17 00:00:00 2001 From: Sudip Bhattarai Date: Sat, 17 Aug 2024 16:44:13 +0545 Subject: [PATCH 2/5] Expose all required env variables --- .github/workflows/build-and-deploy-test-stack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-deploy-test-stack.yml b/.github/workflows/build-and-deploy-test-stack.yml index 97b8b9ad4..448c4a68f 100644 --- a/.github/workflows/build-and-deploy-test-stack.yml +++ b/.github/workflows/build-and-deploy-test-stack.yml @@ -58,7 +58,7 @@ jobs: # Execute the build-and-deploy.sh script cd $DEST_DIR/tests/test-infrastructure ./build-and-deploy.sh update-images - + envs: GOVTOOL_TAG, GRAFANA_ADMIN_PASSWORD, GRAFANA_SLACK_RECIPIENT, GRAFANA_SLACK_OAUTH_TOKEN, SENTRY_DSN_BACKEND, GTM_ID, NPMRC_TOKEN, SENTRY_DSN_FRONTEND, PIPELINE_URL, USERSNAP_SPACE_API_KEY, APP_ENV, PDF_API_URL env: GOVTOOL_TAG: ${{ github.sha }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} From f84bd48d77f010cc9212da2c4ec30bff15789f74 Mon Sep 17 00:00:00 2001 From: Sudip Bhattarai Date: Sat, 17 Aug 2024 16:52:29 +0545 Subject: [PATCH 3/5] Log docker command --- tests/test-infrastructure/build-and-deploy.sh | 1 + tests/test-infrastructure/build-images.sh | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test-infrastructure/build-and-deploy.sh b/tests/test-infrastructure/build-and-deploy.sh index 4c6e6e166..2244cb73a 100755 --- a/tests/test-infrastructure/build-and-deploy.sh +++ b/tests/test-infrastructure/build-and-deploy.sh @@ -16,6 +16,7 @@ check_env # Build images ./build-images.sh function update-service(){ + echo '> docker' service update --image "$2" "$1" docker service update --image "$2" "$1" } diff --git a/tests/test-infrastructure/build-images.sh b/tests/test-infrastructure/build-images.sh index e86103e7a..94f039e72 100755 --- a/tests/test-infrastructure/build-images.sh +++ b/tests/test-infrastructure/build-images.sh @@ -1,15 +1,24 @@ #!/usr/bin/env bash set -e + +# Define a function to log and execute Docker commands +docker_() { + local cmd="$*" + echo docker "$cmd" + docker $cmd +} + export BASE_IMAGE_NAME="govtool" + BASE_IMAGE_EXISTS=$(docker images -q "$BASE_IMAGE_NAME"/backend-base) if [ -z "$BASE_IMAGE_EXISTS" ]; then echo "Building the base image..." - docker build -t "$BASE_IMAGE_NAME"/backend-base -f ../../govtool/backend/Dockerfile.base ../../govtool/backend + docker_ build -t "$BASE_IMAGE_NAME"/backend-base -f ../../govtool/backend/Dockerfile.base ../../govtool/backend else echo "Base image already exists. Skipping build." fi -docker compose -f ./docker-compose-govtool.yml build -docker compose -f ./docker-compose-govaction-loader.yml build -docker compose -f ./docker-compose-test.yml build \ No newline at end of file +docker_ compose -f ./docker-compose-govtool.yml build +docker_ compose -f ./docker-compose-govaction-loader.yml build +docker_ compose -f ./docker-compose-test.yml build From 94f533a325db7f49ac102c5596824c2ec6941213 Mon Sep 17 00:00:00 2001 From: Sudip Bhattarai Date: Sat, 17 Aug 2024 17:30:43 +0545 Subject: [PATCH 4/5] Increase ssh command execution timeout --- .github/workflows/build-and-deploy-test-stack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-deploy-test-stack.yml b/.github/workflows/build-and-deploy-test-stack.yml index 448c4a68f..9f12f9a6a 100644 --- a/.github/workflows/build-and-deploy-test-stack.yml +++ b/.github/workflows/build-and-deploy-test-stack.yml @@ -27,6 +27,7 @@ jobs: host: ${{ secrets.TEST_STACK_SERVER_IP }} username: ec2-user key: ${{ secrets.TEST_STACK_SSH_KEY }} + command_timeout: 100m ## Haskell container build takes a lot of time. script: | REPO_URL="https://github.com/${{ github.repository }}" DEST_DIR="$HOME/Documents/govtool" From c8bc48b3c92ec815abfa300769730df57da0adfd Mon Sep 17 00:00:00 2001 From: Sudip Bhattarai Date: Tue, 20 Aug 2024 10:28:06 +0545 Subject: [PATCH 5/5] Cleanup all images after build --- .github/workflows/build-and-deploy-test-stack.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-deploy-test-stack.yml b/.github/workflows/build-and-deploy-test-stack.yml index 9f12f9a6a..e735e2c0c 100644 --- a/.github/workflows/build-and-deploy-test-stack.yml +++ b/.github/workflows/build-and-deploy-test-stack.yml @@ -59,6 +59,8 @@ jobs: # Execute the build-and-deploy.sh script cd $DEST_DIR/tests/test-infrastructure ./build-and-deploy.sh update-images + docker system prune + (docker image ls -q | xargs docker image rm --force ) || echo "Images cleaned-up" envs: GOVTOOL_TAG, GRAFANA_ADMIN_PASSWORD, GRAFANA_SLACK_RECIPIENT, GRAFANA_SLACK_OAUTH_TOKEN, SENTRY_DSN_BACKEND, GTM_ID, NPMRC_TOKEN, SENTRY_DSN_FRONTEND, PIPELINE_URL, USERSNAP_SPACE_API_KEY, APP_ENV, PDF_API_URL env: GOVTOOL_TAG: ${{ github.sha }}