CORE-104: Don't fail on unknown fields (#175)

* CORE-104: Don't fail on unknown fields

* Fix test

* Update test render-config.sh to pull secret from GSM

* Update github actions to use github secrets

* dummy commit to kick tests

* Fix test action

* Render the app sa as well

* Fix GHA

.github/workflows/test.yml

@@ -13,14 +13,11 @@ on:
     branches: [ master ]
 
 env:
-  VAULT_ADDR: https://clotho.broadinstitute.org:8200
-  VAULT_SERVICE_ACCOUNT_PATH: secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/app-sa
-  VAULT_CLIENT_SERVICE_ACCOUNT_PATH: secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/client-sa
-  VAULT_CLOUD_ACCESS_ACCOUNT_PATH: secret/dsde/terra/janitor-test/default/cloud-access-sa
-  # Where to store the retrieved service accounts credentials for Google integration tests.
-  SERVICE_ACCOUNT_FILE: src/test/resources/rendered/sa-account.json
-  SERVICE_ACCOUNT_CLIENT_FILE: src/test/resources/rendered/client-sa-account.json
-  SERVICE_ACCOUNT_CLOUD_ACCESS_FILE: src/test/resources/rendered/cloud-access-sa-account.json
+  JANITOR_APP_SA_FILE: src/test/resources/rendered/sa-account.json
+  JANITOR_CLIENT_SA_FILE: src/test/resources/rendered/client-sa-account.json
+  JANITOR_CLIENT_SA_TOOLS_FILE: src/test/resources/rendered/tools-client-sa-account.json
+  JANITOR_CLOUD_ACCESS_SA_FILE: src/test/resources/rendered/cloud-access-sa-account.json
+  LOCAL_PROPERTIES_DIR: config
 
 jobs:
   unit-test:
@@ -45,28 +42,48 @@ jobs:
         uses: ./.github/actions/bump-skip
         with:
           event-name: ${{ github.event_name }}
-      - name: Pull Vault image
+      - name: Render secrets for tests
         if: steps.skiptest.outputs.is-bump == 'no'
-        run: docker pull vault:1.1.0
-      # Currently, there's no way to add capabilities to Docker actions on Git, and Vault needs IPC_LOCK to run.
-      - name: Get Vault token
-        if: steps.skiptest.outputs.is-bump == 'no'
-        id: vault-token-step
         run: |
-          VAULT_TOKEN=$(docker run --rm --cap-add IPC_LOCK \
-            -e "VAULT_ADDR=${VAULT_ADDR}" \
-            vault:1.1.0 \
-            vault write -field token \
-              auth/approle/login role_id=${{ secrets.VAULT_APPROLE_ROLE_ID }} \
-              secret_id=${{ secrets.VAULT_APPROLE_SECRET_ID }})
-          echo ::add-mask::$VAULT_TOKEN
-          echo vault-token=$VAULT_TOKEN >> $GITHUB_OUTPUT
-      - name: Grant execute permission for render-config
-        if: steps.skiptest.outputs.is-bump == 'no'
-        run: chmod +x local-dev/render-config.sh
-      - name: Render configuration for tests
-        if: steps.skiptest.outputs.is-bump == 'no'
-        run: local-dev/render-config.sh ${{ steps.vault-token-step.outputs.vault-token }}
+          JANITOR_APP_SA_B64=${{ secrets.JANITOR_APP_SA }}
+          echo ::add-mask::$JANITOR_APP_SA_B64
+          JANITOR_APP_SA=$(echo $JANITOR_APP_SA_B64 | base64 --decode)
+          echo ::add-mask::$JANITOR_APP_SA
+          echo $JANITOR_APP_SA > $JANITOR_APP_SA_FILE
+
+          JANITOR_CLIENT_SA_B64=${{ secrets.JANITOR_CLIENT_SA }}
+          echo ::add-mask::$JANITOR_CLIENT_SA_B64
+          JANITOR_CLIENT_SA=$(echo $JANITOR_CLIENT_SA_B64 | base64 --decode)
+          echo ::add-mask::$JANITOR_CLIENT_SA
+          echo $JANITOR_CLIENT_SA > $JANITOR_CLIENT_SA_FILE
+
+          JANITOR_CLIENT_SA_TOOLS_B64=${{ secrets.JANITOR_CLIENT_SA_TOOLS }}
+          echo ::add-mask::$JANITOR_CLIENT_SA_TOOLS_B64
+          JANITOR_CLIENT_SA_TOOLS=$(echo $JANITOR_CLIENT_SA_TOOLS_B64 | base64 --decode)
+          echo ::add-mask::$JANITOR_CLIENT_SA_TOOLS
+          echo $JANITOR_CLIENT_SA_TOOLS > $JANITOR_CLIENT_SA_TOOLS_FILE
+
+          JANITOR_CLOUD_ACCESS_SA_B64=${{ secrets.JANITOR_CLOUD_ACCESS_SA }}
+          echo ::add-mask::JANITOR_CLOUD_ACCESS_SA_B64
+          JANITOR_CLOUD_ACCESS_SA=$(echo $JANITOR_CLOUD_ACCESS_SA_B64 | base64 --decode)
+          echo ::add-mask::$JANITOR_CLOUD_ACCESS_SA
+          echo $JANITOR_CLOUD_ACCESS_SA > $JANITOR_CLOUD_ACCESS_SA_FILE
+
+          AZURE_PUBLISHER_CLIENT_ID=${{ secrets.AZURE_PUBLISHER_CLIENT_ID }}
+          echo ::add-mask::$AZURE_PUBLISHER_CLIENT_ID
+          AZURE_PUBLISHER_CLIENT_SECRET=${{ secrets.AZURE_PUBLISHER_CLIENT_SECRET }}
+          echo ::add-mask::$AZURE_PUBLISHER_CLIENT_SECRET
+          AZURE_PUBLISHER_TENANT_ID=${{ secrets.AZURE_PUBLISHER_TENANT_ID }}
+          echo ::add-mask::$AZURE_PUBLISHER_TENANT_ID
+
+          mkdir -p "${LOCAL_PROPERTIES_DIR}"
+          cat << EOF > ${LOCAL_PROPERTIES_DIR}/local-properties.yml
+          janitor:
+            azure:
+              managed-app-client-id: ${AZURE_PUBLISHER_CLIENT_ID}
+              managed-app-client-secret: ${AZURE_PUBLISHER_CLIENT_SECRET}
+              managed-app-tenant-id: ${AZURE_PUBLISHER_TENANT_ID}
+          EOF
       - name: Initialize Postgres DB
         if: steps.skiptest.outputs.is-bump == 'no'
         env:

local-dev/render-config.sh

@@ -3,31 +3,35 @@
 # https://github.com/broadinstitute/terraform-ap-deployments/blob/f26945d9d857e879f01671726188cecdc2d7fb10/terra-env/vault_crl_janitor.tf#L43
 # TODO(PF-67): Find solution for piping configs and secrets.
 
+# Vault secrets
+# TODO: migrate vault secrets to GSM as needed
 VAULT_TOKEN=${1:-$(cat $HOME/.vault-token)}
 DSDE_TOOLBOX_DOCKER_IMAGE=broadinstitute/dsde-toolbox:dev
 VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/app-sa
 VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/client-sa
-VAULT_TOOLS_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/tools/crl_janitor/client-sa
 VAULT_CLOUD_ACCESS_SERVICE_ACCOUNT_PATH=secret/dsde/terra/janitor-test/default/cloud-access-sa
 VAULT_AZURE_MANAGED_APP_PUBLISHER_PATH=secret/dsde/terra/azure/common/managed-app-publisher
+
+# GSM secrets
+GSM_CLIENT_SERVICE_ACCOUNT_SECRET=crljanitor-client-sa
+GSM_CLIENT_SERVICE_ACCOUNT_PROJECT=broad-dsde-qa
+
+# Rendered paths
 SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/sa-account.json
 CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/client-sa-account.json
 TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/tools-client-sa-account.json
 CLOUD_ACCESS_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/cloud-access-sa-account.json
 AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/azure-mananged-app-publisher.json
 LOCAL_PROPERTIES_DIR="$(dirname $0)"/../config
 
+# Pull secrets from vault
 docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
             vault read -format json ${VAULT_SERVICE_ACCOUNT_PATH} \
             | jq -r .data.key | base64 -d > ${SERVICE_ACCOUNT_OUTPUT_FILE_PATH}
 docker run --rm --cap-add IPC_LOCK \
             -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
             vault read -format json ${VAULT_CLIENT_SERVICE_ACCOUNT_PATH} \
             | jq -r .data.key | base64 -d > ${CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH}
-docker run --rm --cap-add IPC_LOCK \
-            -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
-            vault read -format json ${VAULT_TOOLS_CLIENT_SERVICE_ACCOUNT_PATH} \
-            | jq -r .data.key | base64 -d > ${TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH}
 docker run --rm --cap-add IPC_LOCK \
             -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
             vault read -format json ${VAULT_CLOUD_ACCESS_SERVICE_ACCOUNT_PATH} \
@@ -37,6 +41,10 @@ docker run --rm --cap-add IPC_LOCK \
             vault read -format json ${VAULT_AZURE_MANAGED_APP_PUBLISHER_PATH} \
             | jq -r .data > ${AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH}
 
+# Pull secrets from GSM
+gcloud secrets versions access latest --project $GSM_CLIENT_SERVICE_ACCOUNT_PROJECT --secret $GSM_CLIENT_SERVICE_ACCOUNT_SECRET \
+  | jq -r '.key' | base64 -d > "$TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH"
+
 # Write the Azure configuration into the local-properties.yml file
 mkdir -p "${LOCAL_PROPERTIES_DIR}"
 AZURE_MANAGED_APP_CLIENT_ID=$(jq -r '."client-id"' ${AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH})

settings.gradle

@@ -2,4 +2,4 @@ rootProject.name = 'terra-resource-janitor'
 include 'terra-resource-janitor-client'
 
 gradle.ext.projectGroup = 'bio.terra'
-gradle.ext.janitorVersion = '0.113.37-SNAPSHOT'
+gradle.ext.janitorVersion = '0.114.0-SNAPSHOT'

src/main/java/bio/terra/janitor/app/configuration/ApplicationConfiguration.java

@@ -4,6 +4,7 @@
 
 import bio.terra.janitor.app.StartupInitializer;
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
@@ -29,6 +30,7 @@ public ObjectMapper objectMapper() {
         .registerModule(new Jdk8Module())
         .registerModule(new JavaTimeModule())
         .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
+        .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
         .setDefaultPropertyInclusion(JsonInclude.Include.NON_ABSENT);
   }
 

src/main/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriber.java

@@ -12,7 +12,8 @@
 import com.google.cloud.pubsub.v1.MessageReceiver;
 import com.google.cloud.pubsub.v1.Subscriber;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.pubsub.v1.*;
+import com.google.pubsub.v1.ProjectSubscriptionName;
+import com.google.pubsub.v1.PubsubMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;

src/test/java/bio/terra/janitor/service/pubsub/TrackedResourceSubscriberTest.java

@@ -13,8 +13,11 @@
 import bio.terra.janitor.generated.model.CloudResourceUid;
 import bio.terra.janitor.generated.model.CreateResourceRequestBody;
 import bio.terra.janitor.generated.model.GoogleBucketUid;
+import bio.terra.janitor.generated.model.GoogleProjectUid;
 import bio.terra.janitor.service.janitor.TrackedResourceService;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.annotation.JsonAppend;
 import com.google.cloud.pubsub.v1.AckReplyConsumer;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -100,4 +103,56 @@ public void nack() {}
                 PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8("bad json")).build(),
                 consumer));
   }
+
+  // See https://broadworkbench.atlassian.net/browse/CORE-104
+  @Test
+  public void receiveMessage_unknownFields() throws Exception {
+    // Set up base project message
+    OffsetDateTime publishTime = JanitorDao.currentOffsetDateTime();
+    CloudResourceUid resource =
+        new CloudResourceUid().googleProjectUid(new GoogleProjectUid().projectId("project"));
+
+    // "Extend" the mapper for CloudResourceUid by adding a few extra fields to the JSON with null
+    // values.
+    @JsonAppend(
+        attrs = {
+          @JsonAppend.Attr(value = "azurePublicIp", include = JsonInclude.Include.ALWAYS),
+          @JsonAppend.Attr(value = "unknownField", include = JsonInclude.Include.ALWAYS),
+          @JsonAppend.Attr(value = "someOtherUnknownField", include = JsonInclude.Include.ALWAYS),
+        })
+    abstract class AzurePublicIpMixin {}
+    objectMapper.addMixIn(CloudResourceUid.class, AzurePublicIpMixin.class);
+    ByteString data =
+        ByteString.copyFromUtf8(
+            objectMapper.writeValueAsString(
+                new CreateResourceRequestBody()
+                    .resourceUid(resource)
+                    .creation(publishTime)
+                    .expiration(publishTime)));
+
+    // Send the extended JSON to the TrackedResourceSubscriber
+    AckReplyConsumer consumer =
+        new AckReplyConsumer() {
+          @Override
+          public void ack() {}
+
+          @Override
+          public void nack() {}
+        };
+    TrackedResourceSubscriber.ResourceReceiver resourceReceiver =
+        new TrackedResourceSubscriber.ResourceReceiver(objectMapper, trackedResourceService);
+    resourceReceiver.receiveMessage(PubsubMessage.newBuilder().setData(data).build(), consumer);
+
+    // The bucket message should have been processed
+    List<TrackedResourceAndLabels> resources =
+        janitorDao.retrieveResourcesAndLabels(
+            TrackedResourceFilter.builder().cloudResourceUid(resource).build());
+    assertEquals(1, resources.size());
+    TrackedResourceAndLabels trackedResourceAndLabels = resources.get(0);
+    TrackedResource trackedResource = trackedResourceAndLabels.trackedResource();
+    assertEquals(resource, trackedResource.cloudResourceUid());
+    assertEquals(publishTime.toInstant(), trackedResource.creation());
+    assertEquals(publishTime.toInstant(), trackedResource.expiration());
+    assertEquals(TrackedResourceState.READY, trackedResource.trackedResourceState());
+  }
 }