diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b6f2724..bb30175e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,14 +13,11 @@ on: branches: [ master ] env: - VAULT_ADDR: https://clotho.broadinstitute.org:8200 - VAULT_SERVICE_ACCOUNT_PATH: secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/app-sa - VAULT_CLIENT_SERVICE_ACCOUNT_PATH: secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/client-sa - VAULT_CLOUD_ACCESS_ACCOUNT_PATH: secret/dsde/terra/janitor-test/default/cloud-access-sa - # Where to store the retrieved service accounts credentials for Google integration tests. - SERVICE_ACCOUNT_FILE: src/test/resources/rendered/sa-account.json - SERVICE_ACCOUNT_CLIENT_FILE: src/test/resources/rendered/client-sa-account.json - SERVICE_ACCOUNT_CLOUD_ACCESS_FILE: src/test/resources/rendered/cloud-access-sa-account.json + JANITOR_APP_SA_FILE: src/test/resources/rendered/sa-account.json + JANITOR_CLIENT_SA_FILE: src/test/resources/rendered/client-sa-account.json + JANITOR_CLIENT_SA_TOOLS_FILE: src/test/resources/rendered/tools-client-sa-account.json + JANITOR_CLOUD_ACCESS_SA_FILE: src/test/resources/rendered/cloud-access-sa-account.json + LOCAL_PROPERTIES_DIR: config jobs: unit-test: @@ -45,28 +42,48 @@ jobs: uses: ./.github/actions/bump-skip with: event-name: ${{ github.event_name }} - - name: Pull Vault image + - name: Render secrets for tests if: steps.skiptest.outputs.is-bump == 'no' - run: docker pull vault:1.1.0 - # Currently, there's no way to add capabilities to Docker actions on Git, and Vault needs IPC_LOCK to run. - - name: Get Vault token - if: steps.skiptest.outputs.is-bump == 'no' - id: vault-token-step run: | - VAULT_TOKEN=$(docker run --rm --cap-add IPC_LOCK \ - -e "VAULT_ADDR=${VAULT_ADDR}" \ - vault:1.1.0 \ - vault write -field token \ - auth/approle/login role_id=${{ secrets.VAULT_APPROLE_ROLE_ID }} \ - secret_id=${{ secrets.VAULT_APPROLE_SECRET_ID }}) - echo ::add-mask::$VAULT_TOKEN - echo vault-token=$VAULT_TOKEN >> $GITHUB_OUTPUT - - name: Grant execute permission for render-config - if: steps.skiptest.outputs.is-bump == 'no' - run: chmod +x local-dev/render-config.sh - - name: Render configuration for tests - if: steps.skiptest.outputs.is-bump == 'no' - run: local-dev/render-config.sh ${{ steps.vault-token-step.outputs.vault-token }} + JANITOR_APP_SA_B64=${{ secrets.JANITOR_APP_SA }} + echo ::add-mask::$JANITOR_APP_SA_B64 + JANITOR_APP_SA=$(echo $JANITOR_APP_SA_B64 | base64 --decode) + echo ::add-mask::$JANITOR_APP_SA + echo $JANITOR_APP_SA > $JANITOR_APP_SA_FILE + + JANITOR_CLIENT_SA_B64=${{ secrets.JANITOR_CLIENT_SA }} + echo ::add-mask::$JANITOR_CLIENT_SA_B64 + JANITOR_CLIENT_SA=$(echo $JANITOR_CLIENT_SA_B64 | base64 --decode) + echo ::add-mask::$JANITOR_CLIENT_SA + echo $JANITOR_CLIENT_SA > $JANITOR_CLIENT_SA_FILE + + JANITOR_CLIENT_SA_TOOLS_B64=${{ secrets.JANITOR_CLIENT_SA_TOOLS }} + echo ::add-mask::$JANITOR_CLIENT_SA_TOOLS_B64 + JANITOR_CLIENT_SA_TOOLS=$(echo $JANITOR_CLIENT_SA_TOOLS_B64 | base64 --decode) + echo ::add-mask::$JANITOR_CLIENT_SA_TOOLS + echo $JANITOR_CLIENT_SA_TOOLS > $JANITOR_CLIENT_SA_TOOLS_FILE + + JANITOR_CLOUD_ACCESS_SA_B64=${{ secrets.JANITOR_CLOUD_ACCESS_SA }} + echo ::add-mask::JANITOR_CLOUD_ACCESS_SA_B64 + JANITOR_CLOUD_ACCESS_SA=$(echo $JANITOR_CLOUD_ACCESS_SA_B64 | base64 --decode) + echo ::add-mask::$JANITOR_CLOUD_ACCESS_SA + echo $JANITOR_CLOUD_ACCESS_SA > $JANITOR_CLOUD_ACCESS_SA_FILE + + AZURE_PUBLISHER_CLIENT_ID=${{ secrets.AZURE_PUBLISHER_CLIENT_ID }} + echo ::add-mask::$AZURE_PUBLISHER_CLIENT_ID + AZURE_PUBLISHER_CLIENT_SECRET=${{ secrets.AZURE_PUBLISHER_CLIENT_SECRET }} + echo ::add-mask::$AZURE_PUBLISHER_CLIENT_SECRET + AZURE_PUBLISHER_TENANT_ID=${{ secrets.AZURE_PUBLISHER_TENANT_ID }} + echo ::add-mask::$AZURE_PUBLISHER_TENANT_ID + + mkdir -p "${LOCAL_PROPERTIES_DIR}" + cat << EOF > ${LOCAL_PROPERTIES_DIR}/local-properties.yml + janitor: + azure: + managed-app-client-id: ${AZURE_PUBLISHER_CLIENT_ID} + managed-app-client-secret: ${AZURE_PUBLISHER_CLIENT_SECRET} + managed-app-tenant-id: ${AZURE_PUBLISHER_TENANT_ID} + EOF - name: Initialize Postgres DB if: steps.skiptest.outputs.is-bump == 'no' env: diff --git a/local-dev/render-config.sh b/local-dev/render-config.sh index c9664fea..853abe94 100755 --- a/local-dev/render-config.sh +++ b/local-dev/render-config.sh @@ -3,13 +3,20 @@ # https://github.com/broadinstitute/terraform-ap-deployments/blob/f26945d9d857e879f01671726188cecdc2d7fb10/terra-env/vault_crl_janitor.tf#L43 # TODO(PF-67): Find solution for piping configs and secrets. +# Vault secrets +# TODO: migrate vault secrets to GSM as needed VAULT_TOKEN=${1:-$(cat $HOME/.vault-token)} DSDE_TOOLBOX_DOCKER_IMAGE=broadinstitute/dsde-toolbox:dev VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/app-sa VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/client-sa -VAULT_TOOLS_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/tools/crl_janitor/client-sa VAULT_CLOUD_ACCESS_SERVICE_ACCOUNT_PATH=secret/dsde/terra/janitor-test/default/cloud-access-sa VAULT_AZURE_MANAGED_APP_PUBLISHER_PATH=secret/dsde/terra/azure/common/managed-app-publisher + +# GSM secrets +GSM_CLIENT_SERVICE_ACCOUNT_SECRET=crljanitor-client-sa +GSM_CLIENT_SERVICE_ACCOUNT_PROJECT=broad-dsde-qa + +# Rendered paths SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/sa-account.json CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/client-sa-account.json TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/tools-client-sa-account.json @@ -17,6 +24,7 @@ CLOUD_ACCESS_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resour AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/azure-mananged-app-publisher.json LOCAL_PROPERTIES_DIR="$(dirname $0)"/../config +# Pull secrets from vault docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \ vault read -format json ${VAULT_SERVICE_ACCOUNT_PATH} \ | jq -r .data.key | base64 -d > ${SERVICE_ACCOUNT_OUTPUT_FILE_PATH} @@ -24,10 +32,6 @@ docker run --rm --cap-add IPC_LOCK \ -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \ vault read -format json ${VAULT_CLIENT_SERVICE_ACCOUNT_PATH} \ | jq -r .data.key | base64 -d > ${CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH} -docker run --rm --cap-add IPC_LOCK \ - -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \ - vault read -format json ${VAULT_TOOLS_CLIENT_SERVICE_ACCOUNT_PATH} \ - | jq -r .data.key | base64 -d > ${TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH} docker run --rm --cap-add IPC_LOCK \ -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \ vault read -format json ${VAULT_CLOUD_ACCESS_SERVICE_ACCOUNT_PATH} \ @@ -37,6 +41,10 @@ docker run --rm --cap-add IPC_LOCK \ vault read -format json ${VAULT_AZURE_MANAGED_APP_PUBLISHER_PATH} \ | jq -r .data > ${AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH} +# Pull secrets from GSM +gcloud secrets versions access latest --project $GSM_CLIENT_SERVICE_ACCOUNT_PROJECT --secret $GSM_CLIENT_SERVICE_ACCOUNT_SECRET \ + | jq -r '.key' | base64 -d > "$TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH" + # Write the Azure configuration into the local-properties.yml file mkdir -p "${LOCAL_PROPERTIES_DIR}" AZURE_MANAGED_APP_CLIENT_ID=$(jq -r '."client-id"' ${AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH}) diff --git a/settings.gradle b/settings.gradle index f7f0664c..7876c12b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,4 +2,4 @@ rootProject.name = 'terra-resource-janitor' include 'terra-resource-janitor-client' gradle.ext.projectGroup = 'bio.terra' -gradle.ext.janitorVersion = '0.113.37-SNAPSHOT' +gradle.ext.janitorVersion = '0.114.0-SNAPSHOT' diff --git a/src/main/java/bio/terra/janitor/app/configuration/ApplicationConfiguration.java b/src/main/java/bio/terra/janitor/app/configuration/ApplicationConfiguration.java index 4993e8ba..67c102d9 100644 --- a/src/main/java/bio/terra/janitor/app/configuration/ApplicationConfiguration.java +++ b/src/main/java/bio/terra/janitor/app/configuration/ApplicationConfiguration.java @@ -4,6 +4,7 @@ import bio.terra.janitor.app.StartupInitializer; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; @@ -29,6 +30,7 @@ public ObjectMapper objectMapper() { .registerModule(new Jdk8Module()) .registerModule(new JavaTimeModule()) .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .setDefaultPropertyInclusion(JsonInclude.Include.NON_ABSENT); } diff --git a/src/main/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriber.java b/src/main/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriber.java index 94bfc157..77203d18 100644 --- a/src/main/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriber.java +++ b/src/main/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriber.java @@ -12,7 +12,8 @@ import com.google.cloud.pubsub.v1.MessageReceiver; import com.google.cloud.pubsub.v1.Subscriber; import com.google.common.annotations.VisibleForTesting; -import com.google.pubsub.v1.*; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/test/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriberTest.java b/src/test/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriberTest.java index cd909547..b0566106 100644 --- a/src/test/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriberTest.java +++ b/src/test/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriberTest.java @@ -13,8 +13,11 @@ import bio.terra.janitor.generated.model.CloudResourceUid; import bio.terra.janitor.generated.model.CreateResourceRequestBody; import bio.terra.janitor.generated.model.GoogleBucketUid; +import bio.terra.janitor.generated.model.GoogleProjectUid; import bio.terra.janitor.service.janitor.TrackedResourceService; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonAppend; import com.google.cloud.pubsub.v1.AckReplyConsumer; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; @@ -100,4 +103,56 @@ public void nack() {} PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8("bad json")).build(), consumer)); } + + // See https://broadworkbench.atlassian.net/browse/CORE-104 + @Test + public void receiveMessage_unknownFields() throws Exception { + // Set up base project message + OffsetDateTime publishTime = JanitorDao.currentOffsetDateTime(); + CloudResourceUid resource = + new CloudResourceUid().googleProjectUid(new GoogleProjectUid().projectId("project")); + + // "Extend" the mapper for CloudResourceUid by adding a few extra fields to the JSON with null + // values. + @JsonAppend( + attrs = { + @JsonAppend.Attr(value = "azurePublicIp", include = JsonInclude.Include.ALWAYS), + @JsonAppend.Attr(value = "unknownField", include = JsonInclude.Include.ALWAYS), + @JsonAppend.Attr(value = "someOtherUnknownField", include = JsonInclude.Include.ALWAYS), + }) + abstract class AzurePublicIpMixin {} + objectMapper.addMixIn(CloudResourceUid.class, AzurePublicIpMixin.class); + ByteString data = + ByteString.copyFromUtf8( + objectMapper.writeValueAsString( + new CreateResourceRequestBody() + .resourceUid(resource) + .creation(publishTime) + .expiration(publishTime))); + + // Send the extended JSON to the TrackedResourceSubscriber + AckReplyConsumer consumer = + new AckReplyConsumer() { + @Override + public void ack() {} + + @Override + public void nack() {} + }; + TrackedResourceSubscriber.ResourceReceiver resourceReceiver = + new TrackedResourceSubscriber.ResourceReceiver(objectMapper, trackedResourceService); + resourceReceiver.receiveMessage(PubsubMessage.newBuilder().setData(data).build(), consumer); + + // The bucket message should have been processed + List resources = + janitorDao.retrieveResourcesAndLabels( + TrackedResourceFilter.builder().cloudResourceUid(resource).build()); + assertEquals(1, resources.size()); + TrackedResourceAndLabels trackedResourceAndLabels = resources.get(0); + TrackedResource trackedResource = trackedResourceAndLabels.trackedResource(); + assertEquals(resource, trackedResource.cloudResourceUid()); + assertEquals(publishTime.toInstant(), trackedResource.creation()); + assertEquals(publishTime.toInstant(), trackedResource.expiration()); + assertEquals(TrackedResourceState.READY, trackedResource.trackedResourceState()); + } }