From 5b3c9036d77f1df0807987906f91ba4c97daf540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=BChringer?= <51900829+puehringer@users.noreply.github.com> Date: Wed, 24 Jan 2024 14:32:40 +0100 Subject: [PATCH] Revert "Revert "Change postgres hostname to something unique for github runner support"" --- .github/actions/build-node/action.yml | 7 ++- .github/actions/build-python/action.yml | 12 ++++- .github/workflows/build-node-python.yml | 49 +++++++++++++------ .github/workflows/build-node.yml | 4 +- .github/workflows/build-product.yml | 6 +-- .github/workflows/build-push-docker.yml | 4 +- .github/workflows/build-push-helm-chart.yml | 4 +- .github/workflows/build-python.yml | 4 +- .../workflows/build-single-product-part.yml | 4 +- .../build-workspace-product-part.yml | 4 +- .../workflows/check-helm-chart-version.yml | 4 +- .github/workflows/deploy-product.yml | 4 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish-node-python.yml | 8 +-- .github/workflows/publish-node.yml | 4 +- .github/workflows/publish-python.yml | 4 +- .github/workflows/release-product.yml | 10 ++-- .github/workflows/release-source.yml | 4 +- 18 files changed, 87 insertions(+), 51 deletions(-) diff --git a/.github/actions/build-node/action.yml b/.github/actions/build-node/action.yml index 6389b08e..a9b4aee7 100644 --- a/.github/actions/build-node/action.yml +++ b/.github/actions/build-node/action.yml @@ -29,6 +29,10 @@ inputs: run_webpack: default: false required: false + disable_cache: + description: "Set to true to disable caching" + default: false + required: false runs: using: "composite" @@ -59,6 +63,7 @@ runs: name: node-yarn-lock # Enable yarn download cache, @see https://github.com/actions/cache/tree/main/save#always-save-cache and https://github.com/actions/setup-node/issues/325 - name: Restore yarn cache + if: inputs.disable_cache == 'false' uses: actions/cache/restore@v3 with: # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder @@ -71,8 +76,8 @@ runs: run: yarn install --no-immutable --inline-builds shell: bash - name: Save yarn cache + if: inputs.disable_cache == 'false' && steps.install.outcome == 'success' uses: actions/cache/save@v3 - if: steps.install.outcome == 'success' with: # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder path: ~/.yarn/berry/cache/ diff --git a/.github/actions/build-python/action.yml b/.github/actions/build-python/action.yml index 0d97165a..44456203 100644 --- a/.github/actions/build-python/action.yml +++ b/.github/actions/build-python/action.yml @@ -20,16 +20,26 @@ inputs: run_build: default: true required: false + disable_cache: + description: "Set to true to disable caching" + default: false + required: false runs: using: "composite" steps: - - name: Set up python + - name: Set up python (with cache) + if: inputs.disable_cache == 'false' uses: actions/setup-python@v3 with: python-version: ${{ inputs.python_version }} cache: 'pip' # needs wheel for make + - name: Set up python (without cache) + if: inputs.disable_cache == 'true' + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python_version }} - name: install additional requirements run: | pip install setuptools wheel diff --git a/.github/workflows/build-node-python.yml b/.github/workflows/build-node-python.yml index c0068f0c..9809b3e2 100644 --- a/.github/workflows/build-node-python.yml +++ b/.github/workflows/build-node-python.yml @@ -57,6 +57,11 @@ on: type: string required: false default: "ubuntu-22.04-large" + #disable_cache: + # description: "Set to true to disable caching" + # required: false + # type: boolean + # default: false secrets: DATAVISYN_BOT_REPO_TOKEN: required: false @@ -75,7 +80,8 @@ env: PYPI_REGISTRY: "https://upload.pypi.org/legacy/" PYPI_USERNAME: "datavisyn" PYTHON_VERSION: "3.10" - WORKFLOW_BRANCH: "new_deployment" + WORKFLOW_BRANCH: "new_deployment_no_ports" + POSTGRES_HOSTNAME: postgres_${{ inputs.branch }}_${{ github.run_id }} permissions: contents: read @@ -94,12 +100,12 @@ jobs: runs-on: ${{ inputs.runs_on }} steps: - name: Checkout source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} - name: Checkout github-workflows - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -111,6 +117,7 @@ jobs: npm_registry: ${{ env.NPM_REGISTRY }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_webpack: ${{ inputs.node_run_webpack }} + #disable_cache: ${{ inputs.disable_cache }} build-python: name: Python @@ -121,12 +128,12 @@ jobs: runs-on: ${{ inputs.runs_on }} steps: - name: Checkout source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} - name: Checkout github-workflows - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -136,7 +143,7 @@ jobs: with: python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} - + #disable_cache: ${{ inputs.disable_cache }} # If cypress is used, build node and python sequentially as it is avoiding the duplicate install overhead build-node-python-cypress: name: Node, Python, Cypress @@ -161,17 +168,21 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - ports: - - 5432:5432 + --name postgres_${{ inputs.branch }}_${{ github.run_id }} steps: + - name: Set system env variable to github env + run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV" + - name: Connect service to default network + if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != '' + run: docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }} - name: Checkout source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} - name: Checkout github-workflows - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -183,11 +194,13 @@ jobs: npm_registry: ${{ env.NPM_REGISTRY }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_webpack: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install + #disable_cache: ${{ inputs.disable_cache }} - name: Build python uses: ./tmp/github-workflows/.github/actions/build-python with: python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} + #disable_cache: ${{ inputs.disable_cache }} - name: Decrypt .env.enc and /.env.enc run: | yarn run env:decrypt -pass env:ENV_PASSWORD || true @@ -210,6 +223,8 @@ jobs: make start wait-on: "http://localhost:8080, http://localhost:9000/health" env: ${{ secrets.CYPRESS_ENV }} + env: + POSTGRES_HOSTNAME: ${{ env.POSTGRES_HOSTNAME }} - name: Run cypress component tests uses: cypress-io/github-action@v6 with: @@ -257,17 +272,21 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - ports: - - 5432:5432 + --name postgres_${{ inputs.branch }}_${{ github.run_id }} steps: + - name: Set system env variable to github env + run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV" + - name: Connect service to default network + if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != '' + run: docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }} - name: Checkout source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} - name: Checkout github-workflows - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -279,11 +298,13 @@ jobs: npm_registry: ${{ env.NPM_REGISTRY }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_webpack: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install + #disable_cache: ${{ inputs.disable_cache }} - name: Build python uses: ./tmp/github-workflows/.github/actions/build-python with: python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} + #disable_cache: ${{ inputs.disable_cache }} - name: Decrypt .env.enc and /.env.enc run: | yarn run env:decrypt -pass env:ENV_PASSWORD || true diff --git a/.github/workflows/build-node.yml b/.github/workflows/build-node.yml index 1cb2ad43..9449fbb1 100644 --- a/.github/workflows/build-node.yml +++ b/.github/workflows/build-node.yml @@ -37,12 +37,12 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} # has to set because otherwise it will not work # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/build-product.yml b/.github/workflows/build-product.yml index a847cdd7..dcb97e5e 100644 --- a/.github/workflows/build-product.yml +++ b/.github/workflows/build-product.yml @@ -57,9 +57,9 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -118,7 +118,7 @@ jobs: if: ${{ always() && (needs.build-single.result == 'success' || needs.build-single.result == 'skipped') && (needs.build-workspace.result == 'success' || needs.build-workspace.result == 'skipped') && !(needs.build-workspace.result == 'skipped' && needs.build-single.result == 'skipped')}} runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/build-push-docker.yml b/.github/workflows/build-push-docker.yml index 74a6c0a8..f550a407 100644 --- a/.github/workflows/build-push-docker.yml +++ b/.github/workflows/build-push-docker.yml @@ -38,9 +38,9 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/build-push-helm-chart.yml b/.github/workflows/build-push-helm-chart.yml index 0fcb641c..6bc05f17 100644 --- a/.github/workflows/build-push-helm-chart.yml +++ b/.github/workflows/build-push-helm-chart.yml @@ -36,9 +36,9 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml index a7cba16b..00cc3492 100644 --- a/.github/workflows/build-python.yml +++ b/.github/workflows/build-python.yml @@ -37,11 +37,11 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/build-single-product-part.yml b/.github/workflows/build-single-product-part.yml index 90e620fa..6bfa63ab 100644 --- a/.github/workflows/build-single-product-part.yml +++ b/.github/workflows/build-single-product-part.yml @@ -59,11 +59,11 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/build-workspace-product-part.yml b/.github/workflows/build-workspace-product-part.yml index 554b688e..e9207048 100644 --- a/.github/workflows/build-workspace-product-part.yml +++ b/.github/workflows/build-workspace-product-part.yml @@ -57,11 +57,11 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/check-helm-chart-version.yml b/.github/workflows/check-helm-chart-version.yml index 8a2be6b1..53b7b9b2 100644 --- a/.github/workflows/check-helm-chart-version.yml +++ b/.github/workflows/check-helm-chart-version.yml @@ -30,9 +30,9 @@ jobs: runs-on: ubuntu-22.04 steps: # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/deploy-product.yml b/.github/workflows/deploy-product.yml index cd9f663b..d4748750 100644 --- a/.github/workflows/deploy-product.yml +++ b/.github/workflows/deploy-product.yml @@ -47,9 +47,9 @@ jobs: revision_label: ${{ steps.get-revision-label.outputs.revision_label }} steps: # checkout repo to get package.json - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d42b8a98..a0b4f05e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-22.04 if: ${{ github.event_name == 'push' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Full git history needed to get a proper # list of changed files within `super-linter` diff --git a/.github/workflows/publish-node-python.yml b/.github/workflows/publish-node-python.yml index 0d3ba9e7..ca4716b3 100644 --- a/.github/workflows/publish-node-python.yml +++ b/.github/workflows/publish-node-python.yml @@ -44,11 +44,11 @@ jobs: echo "only the main branch can be published." exit 1 # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} # has to set because otherwise it will not work # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -76,9 +76,9 @@ jobs: echo "only the main branch can be published." exit 1 # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/publish-node.yml b/.github/workflows/publish-node.yml index 3c357cd0..8313f8d1 100644 --- a/.github/workflows/publish-node.yml +++ b/.github/workflows/publish-node.yml @@ -35,11 +35,11 @@ jobs: echo "only the main branch can be published." exit 1 # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} # has to set because otherwise it will not work # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml index 392ed3d9..0a16f903 100644 --- a/.github/workflows/publish-python.yml +++ b/.github/workflows/publish-python.yml @@ -34,9 +34,9 @@ jobs: echo "only the main branch can be published." exit 1 # checkout specific source repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/release-product.yml b/.github/workflows/release-product.yml index e1b0c296..1700dff1 100644 --- a/.github/workflows/release-product.yml +++ b/.github/workflows/release-product.yml @@ -33,11 +33,11 @@ jobs: exit 1 env: GITHUB_REF: ${{ github.ref_name }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.DATAVISYN_BOT_REPO_TOKEN }} # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -180,10 +180,10 @@ jobs: run: | echo 'only the main branches can be released.' exit 1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.DATAVISYN_BOT_REPO_TOKEN }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} @@ -266,7 +266,7 @@ jobs: DEVELOP_BRANCH: ${{ steps.get-release-version.outputs.develop_branch_name }} MAIN_BRANCH: ${{ steps.get-release-version.outputs.main_branch_name }} # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }} diff --git a/.github/workflows/release-source.yml b/.github/workflows/release-source.yml index b4fe11b1..6eaf5b60 100644 --- a/.github/workflows/release-source.yml +++ b/.github/workflows/release-source.yml @@ -25,11 +25,11 @@ jobs: echo 'only the develop branch can be released.' exit 1 # checkout specific repository - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ github.token }} # checkout this workflow repository to get actions - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: datavisyn/github-workflows ref: ${{ env.WORKFLOW_BRANCH }}