diff --git a/src/main/java/bio/terra/janitor/app/configuration/CrlConfiguration.java b/src/main/java/bio/terra/janitor/app/configuration/CrlConfiguration.java index b65a38da..f99eccdd 100644 --- a/src/main/java/bio/terra/janitor/app/configuration/CrlConfiguration.java +++ b/src/main/java/bio/terra/janitor/app/configuration/CrlConfiguration.java @@ -12,7 +12,6 @@ import com.azure.core.management.profile.AzureProfile; import com.azure.identity.ClientSecretCredentialBuilder; import com.azure.resourcemanager.compute.ComputeManager; -import com.azure.resourcemanager.containerinstance.ContainerInstanceManager; import com.azure.resourcemanager.msi.MsiManager; import com.azure.resourcemanager.relay.RelayManager; import com.azure.resourcemanager.storage.StorageManager; @@ -108,15 +107,6 @@ public RelayManager buildRelayManager(AzureResourceGroup resourceGroup) { .authenticate(getAzureCredential(), getAzureProfile(resourceGroup)); } - /** - * Creates an Azure {@link ContainerInstanceManager} client for a given managed resource group. - */ - public ContainerInstanceManager buildContainerInstance(AzureResourceGroup resourceGroup) { - return bio.terra.cloudres.azure.resourcemanager.common.Defaults.crlConfigure( - clientConfig, ContainerInstanceManager.configure()) - .authenticate(getAzureCredential(), getAzureProfile(resourceGroup)); - } - /** Creates an Azure {@link MsiManager} client for a given managed resource group. */ public MsiManager buildMsiManager(AzureResourceGroup resourceGroup) { return bio.terra.cloudres.azure.resourcemanager.common.Defaults.crlConfigure( diff --git a/src/main/java/bio/terra/janitor/common/CloudResourceUidVisitor.java b/src/main/java/bio/terra/janitor/common/CloudResourceUidVisitor.java index 98d3987e..6654e6b2 100644 --- a/src/main/java/bio/terra/janitor/common/CloudResourceUidVisitor.java +++ b/src/main/java/bio/terra/janitor/common/CloudResourceUidVisitor.java @@ -1,12 +1,9 @@ package bio.terra.janitor.common; -import bio.terra.janitor.generated.model.AzureContainerInstance; +import bio.terra.janitor.generated.model.AzureDatabase; import bio.terra.janitor.generated.model.AzureDisk; +import bio.terra.janitor.generated.model.AzureKubernetesNamespace; import bio.terra.janitor.generated.model.AzureManagedIdentity; -import bio.terra.janitor.generated.model.AzureNetwork; -import bio.terra.janitor.generated.model.AzureNetworkSecurityGroup; -import bio.terra.janitor.generated.model.AzurePublicIp; -import bio.terra.janitor.generated.model.AzureRelay; import bio.terra.janitor.generated.model.AzureRelayHybridConnection; import bio.terra.janitor.generated.model.AzureStorageContainer; import bio.terra.janitor.generated.model.AzureVirtualMachine; @@ -33,28 +30,22 @@ public interface CloudResourceUidVisitor { R visit(GoogleAiNotebookInstanceUid resource); - R visit(AzurePublicIp resource); - - R visit(AzureNetworkSecurityGroup resource); - - R visit(AzureNetwork resource); - R visit(AzureDisk resource); R visit(AzureVirtualMachine resource); - R visit(AzureRelay resource); - R visit(AzureRelayHybridConnection resource); - R visit(AzureContainerInstance resource); - R visit(TerraWorkspaceUid resource); R visit(AzureManagedIdentity resource); R visit(AzureStorageContainer resource); + R visit(AzureDatabase resource); + + R visit(AzureKubernetesNamespace resource); + R noResourceVisited(CloudResourceUid resource); static R visit(CloudResourceUid resource, CloudResourceUidVisitor visitor) { @@ -70,28 +61,22 @@ static R visit(CloudResourceUid resource, CloudResourceUidVisitor visitor return visitor.visit(resource.getGoogleBigQueryTableUid()); } else if (resource.getGoogleAiNotebookInstanceUid() != null) { return visitor.visit(resource.getGoogleAiNotebookInstanceUid()); - } else if (resource.getAzurePublicIp() != null) { - return visitor.visit(resource.getAzurePublicIp()); - } else if (resource.getAzureNetworkSecurityGroup() != null) { - return visitor.visit(resource.getAzureNetworkSecurityGroup()); - } else if (resource.getAzureNetwork() != null) { - return visitor.visit(resource.getAzureNetwork()); } else if (resource.getAzureDisk() != null) { return visitor.visit(resource.getAzureDisk()); } else if (resource.getAzureVirtualMachine() != null) { return visitor.visit(resource.getAzureVirtualMachine()); - } else if (resource.getAzureRelay() != null) { - return visitor.visit(resource.getAzureRelay()); } else if (resource.getAzureRelayHybridConnection() != null) { return visitor.visit(resource.getAzureRelayHybridConnection()); - } else if (resource.getAzureContainerInstance() != null) { - return visitor.visit(resource.getAzureContainerInstance()); } else if (resource.getTerraWorkspace() != null) { return visitor.visit(resource.getTerraWorkspace()); } else if (resource.getAzureManagedIdentity() != null) { return visitor.visit(resource.getAzureManagedIdentity()); } else if (resource.getAzureStorageContainer() != null) { return visitor.visit(resource.getAzureStorageContainer()); + } else if (resource.getAzureDatabase() != null) { + return visitor.visit(resource.getAzureDatabase()); + } else if (resource.getAzureKubernetesNamespace() != null) { + return visitor.visit(resource.getAzureKubernetesNamespace()); } else { return visitor.noResourceVisited(resource); } diff --git a/src/main/java/bio/terra/janitor/db/ResourceType.java b/src/main/java/bio/terra/janitor/db/ResourceType.java index f832daa1..5f430ff1 100644 --- a/src/main/java/bio/terra/janitor/db/ResourceType.java +++ b/src/main/java/bio/terra/janitor/db/ResourceType.java @@ -15,15 +15,12 @@ public enum ResourceType { GOOGLE_BUCKET, GOOGLE_NOTEBOOK_INSTANCE, GOOGLE_PROJECT, - AZURE_PUBLIC_IP, - AZURE_NETWORK_SECURITY_GROUP, - AZURE_NETWORK, AZURE_DISK, AZURE_VIRTUAL_MACHINE, - AZURE_RELAY, AZURE_RELAY_CONNECTION, - AZURE_CONTAINER_INSTANCE, TERRA_WORKSPACE, AZURE_MANAGED_IDENTITY, - AZURE_STORAGE_CONTAINER + AZURE_STORAGE_CONTAINER, + AZURE_DATABASE, + AZURE_KUBERNETES_NAMESPACE, } diff --git a/src/main/java/bio/terra/janitor/db/ResourceTypeVisitor.java b/src/main/java/bio/terra/janitor/db/ResourceTypeVisitor.java index 8b0a913d..ca891964 100644 --- a/src/main/java/bio/terra/janitor/db/ResourceTypeVisitor.java +++ b/src/main/java/bio/terra/janitor/db/ResourceTypeVisitor.java @@ -38,21 +38,6 @@ public ResourceType visit(GoogleAiNotebookInstanceUid resource) { return GOOGLE_NOTEBOOK_INSTANCE; } - @Override - public ResourceType visit(AzurePublicIp resource) { - return AZURE_PUBLIC_IP; - } - - @Override - public ResourceType visit(AzureNetworkSecurityGroup resource) { - return AZURE_NETWORK_SECURITY_GROUP; - } - - @Override - public ResourceType visit(AzureNetwork resource) { - return AZURE_NETWORK; - } - @Override public ResourceType visit(AzureDisk resource) { return AZURE_DISK; @@ -63,21 +48,11 @@ public ResourceType visit(AzureVirtualMachine resource) { return AZURE_VIRTUAL_MACHINE; } - @Override - public ResourceType visit(AzureRelay resource) { - return AZURE_RELAY; - } - @Override public ResourceType visit(AzureRelayHybridConnection resource) { return AZURE_RELAY_CONNECTION; } - @Override - public ResourceType visit(AzureContainerInstance resource) { - return AZURE_CONTAINER_INSTANCE; - } - @Override public ResourceType visit(TerraWorkspaceUid resource) { return TERRA_WORKSPACE; @@ -93,6 +68,16 @@ public ResourceType visit(AzureStorageContainer resource) { return AZURE_STORAGE_CONTAINER; } + @Override + public ResourceType visit(AzureDatabase resource) { + return AZURE_DATABASE; + } + + @Override + public ResourceType visit(AzureKubernetesNamespace resource) { + return AZURE_KUBERNETES_NAMESPACE; + } + @Override public ResourceType noResourceVisited(CloudResourceUid resource) { throw new InvalidResourceUidException("invalid CloudResourceUid for" + resource); diff --git a/src/main/java/bio/terra/janitor/service/cleanup/FlightSubmissionFactoryImpl.java b/src/main/java/bio/terra/janitor/service/cleanup/FlightSubmissionFactoryImpl.java index 15439391..0de3ef09 100644 --- a/src/main/java/bio/terra/janitor/service/cleanup/FlightSubmissionFactoryImpl.java +++ b/src/main/java/bio/terra/janitor/service/cleanup/FlightSubmissionFactoryImpl.java @@ -30,22 +30,12 @@ public FlightSubmission createSubmission(TrackedResource trackedResource) { return FlightSubmission.create(GoogleAiNotebookInstanceCleanupFlight.class, flightMap); case GOOGLE_PROJECT: return FlightSubmission.create(GoogleProjectCleanupFlight.class, flightMap); - case AZURE_PUBLIC_IP: - return FlightSubmission.create(AzurePublicIpCleanupFlight.class, flightMap); - case AZURE_NETWORK_SECURITY_GROUP: - return FlightSubmission.create(AzureNetworkSecurityGroupCleanupFlight.class, flightMap); - case AZURE_NETWORK: - return FlightSubmission.create(AzureNetworkCleanupFlight.class, flightMap); case AZURE_DISK: return FlightSubmission.create(AzureDiskCleanupFlight.class, flightMap); case AZURE_VIRTUAL_MACHINE: return FlightSubmission.create(AzureVirtualMachineCleanupFlight.class, flightMap); - case AZURE_RELAY: - return FlightSubmission.create(AzureRelayCleanupFlight.class, flightMap); case AZURE_RELAY_CONNECTION: return FlightSubmission.create(AzureRelayHybridConnectionCleanupFlight.class, flightMap); - case AZURE_CONTAINER_INSTANCE: - return FlightSubmission.create(AzureContainerInstanceCleanupFlight.class, flightMap); case TERRA_WORKSPACE: return FlightSubmission.create(TerraWorkspaceCleanupFlight.class, flightMap); case AZURE_MANAGED_IDENTITY: diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureContainerInstanceCleanupFlight.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureContainerInstanceCleanupFlight.java deleted file mode 100644 index 74599f5f..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureContainerInstanceCleanupFlight.java +++ /dev/null @@ -1,24 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.stairway.Flight; -import bio.terra.stairway.FlightMap; -import bio.terra.stairway.RetryRuleFixedInterval; -import org.springframework.context.ApplicationContext; - -/** Flight to clean up an Azure Container Instance. */ -public class AzureContainerInstanceCleanupFlight extends Flight { - public AzureContainerInstanceCleanupFlight(FlightMap inputParameters, Object applicationContext) { - super(inputParameters, applicationContext); - ApplicationContext appContext = (ApplicationContext) applicationContext; - JanitorDao janitorDao = appContext.getBean(JanitorDao.class); - CrlConfiguration crlConfiguration = appContext.getBean(CrlConfiguration.class); - RetryRuleFixedInterval retryRule = - new RetryRuleFixedInterval(/* intervalSeconds =*/ 180, /* maxCount =*/ 5); - - addStep(new InitialCleanupStep(janitorDao)); - addStep(new AzureContainerInstanceCleanupStep(crlConfiguration, janitorDao), retryRule); - addStep(new FinalCleanupStep(janitorDao)); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureContainerInstanceCleanupStep.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureContainerInstanceCleanupStep.java deleted file mode 100644 index 4b29e801..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureContainerInstanceCleanupStep.java +++ /dev/null @@ -1,36 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.janitor.db.ResourceMetadata; -import bio.terra.janitor.generated.model.AzureContainerInstance; -import bio.terra.janitor.generated.model.CloudResourceUid; -import bio.terra.stairway.StepResult; -import bio.terra.stairway.exception.RetryException; -import com.azure.resourcemanager.containerinstance.ContainerInstanceManager; - -public class AzureContainerInstanceCleanupStep extends ResourceCleanupStep { - private final CrlConfiguration crlConfiguration; - - public AzureContainerInstanceCleanupStep( - CrlConfiguration crlConfiguration, JanitorDao janitorDao) { - super(janitorDao); - this.crlConfiguration = crlConfiguration; - } - - @Override - protected StepResult cleanUp(CloudResourceUid resourceUid, ResourceMetadata metadata) - throws InterruptedException, RetryException { - AzureContainerInstance containerInstance = resourceUid.getAzureContainerInstance(); - ContainerInstanceManager containerInstanceManager = - crlConfiguration.buildContainerInstance(containerInstance.getResourceGroup()); - - return AzureUtils.ignoreNotFound( - () -> - containerInstanceManager - .containerGroups() - .deleteByResourceGroup( - containerInstance.getResourceGroup().getResourceGroupName(), - containerInstance.getContainerGroupName())); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkCleanupFlight.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkCleanupFlight.java deleted file mode 100644 index 037dd312..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkCleanupFlight.java +++ /dev/null @@ -1,24 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.stairway.Flight; -import bio.terra.stairway.FlightMap; -import bio.terra.stairway.RetryRuleFixedInterval; -import org.springframework.context.ApplicationContext; - -/** Flight to clean up an Azure network. */ -public class AzureNetworkCleanupFlight extends Flight { - public AzureNetworkCleanupFlight(FlightMap inputParameters, Object applicationContext) { - super(inputParameters, applicationContext); - ApplicationContext appContext = (ApplicationContext) applicationContext; - JanitorDao janitorDao = appContext.getBean(JanitorDao.class); - CrlConfiguration crlConfiguration = appContext.getBean(CrlConfiguration.class); - RetryRuleFixedInterval retryRule = - new RetryRuleFixedInterval(/* intervalSeconds =*/ 180, /* maxCount =*/ 5); - - addStep(new InitialCleanupStep(janitorDao)); - addStep(new AzureNetworkCleanupStep(crlConfiguration, janitorDao), retryRule); - addStep(new FinalCleanupStep(janitorDao)); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkCleanupStep.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkCleanupStep.java deleted file mode 100644 index 8eb16d4c..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkCleanupStep.java +++ /dev/null @@ -1,36 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.janitor.db.ResourceMetadata; -import bio.terra.janitor.generated.model.AzureNetwork; -import bio.terra.janitor.generated.model.CloudResourceUid; -import bio.terra.stairway.StepResult; -import bio.terra.stairway.exception.RetryException; -import com.azure.resourcemanager.compute.ComputeManager; - -/** Step to clean up an Azure network. */ -public class AzureNetworkCleanupStep extends ResourceCleanupStep { - private final CrlConfiguration crlConfiguration; - - public AzureNetworkCleanupStep(CrlConfiguration crlConfiguration, JanitorDao janitorDao) { - super(janitorDao); - this.crlConfiguration = crlConfiguration; - } - - @Override - protected StepResult cleanUp(CloudResourceUid resourceUid, ResourceMetadata metadata) - throws InterruptedException, RetryException { - AzureNetwork network = resourceUid.getAzureNetwork(); - ComputeManager computeManager = - crlConfiguration.buildComputeManager(network.getResourceGroup()); - - return AzureUtils.ignoreNotFound( - () -> - computeManager - .networkManager() - .networks() - .deleteByResourceGroup( - network.getResourceGroup().getResourceGroupName(), network.getNetworkName())); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkSecurityGroupCleanupFlight.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkSecurityGroupCleanupFlight.java deleted file mode 100644 index 8aef27d9..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkSecurityGroupCleanupFlight.java +++ /dev/null @@ -1,25 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.stairway.Flight; -import bio.terra.stairway.FlightMap; -import bio.terra.stairway.RetryRuleFixedInterval; -import org.springframework.context.ApplicationContext; - -/** Flight to clean up an Azure network security group. */ -public class AzureNetworkSecurityGroupCleanupFlight extends Flight { - public AzureNetworkSecurityGroupCleanupFlight( - FlightMap inputParameters, Object applicationContext) { - super(inputParameters, applicationContext); - ApplicationContext appContext = (ApplicationContext) applicationContext; - JanitorDao janitorDao = appContext.getBean(JanitorDao.class); - CrlConfiguration crlConfiguration = appContext.getBean(CrlConfiguration.class); - RetryRuleFixedInterval retryRule = - new RetryRuleFixedInterval(/* intervalSeconds =*/ 180, /* maxCount =*/ 5); - - addStep(new InitialCleanupStep(janitorDao)); - addStep(new AzureNetworkSecurityGroupCleanupStep(crlConfiguration, janitorDao), retryRule); - addStep(new FinalCleanupStep(janitorDao)); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkSecurityGroupCleanupStep.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkSecurityGroupCleanupStep.java deleted file mode 100644 index 74cb0059..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureNetworkSecurityGroupCleanupStep.java +++ /dev/null @@ -1,38 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.janitor.db.ResourceMetadata; -import bio.terra.janitor.generated.model.AzureNetworkSecurityGroup; -import bio.terra.janitor.generated.model.CloudResourceUid; -import bio.terra.stairway.StepResult; -import bio.terra.stairway.exception.RetryException; -import com.azure.resourcemanager.compute.ComputeManager; - -/** Step to clean up an Azure network security group. */ -public class AzureNetworkSecurityGroupCleanupStep extends ResourceCleanupStep { - private final CrlConfiguration crlConfiguration; - - public AzureNetworkSecurityGroupCleanupStep( - CrlConfiguration crlConfiguration, JanitorDao janitorDao) { - super(janitorDao); - this.crlConfiguration = crlConfiguration; - } - - @Override - protected StepResult cleanUp(CloudResourceUid resourceUid, ResourceMetadata metadata) - throws InterruptedException, RetryException { - AzureNetworkSecurityGroup networkSg = resourceUid.getAzureNetworkSecurityGroup(); - ComputeManager computeManager = - crlConfiguration.buildComputeManager(networkSg.getResourceGroup()); - - return AzureUtils.ignoreNotFound( - () -> - computeManager - .networkManager() - .networkSecurityGroups() - .deleteByResourceGroup( - networkSg.getResourceGroup().getResourceGroupName(), - networkSg.getNetworkSecurityGroupName())); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzurePublicIpCleanupFlight.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzurePublicIpCleanupFlight.java deleted file mode 100644 index fd588388..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzurePublicIpCleanupFlight.java +++ /dev/null @@ -1,24 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.stairway.Flight; -import bio.terra.stairway.FlightMap; -import bio.terra.stairway.RetryRuleFixedInterval; -import org.springframework.context.ApplicationContext; - -/** Flight to clean up an Azure public IP. */ -public class AzurePublicIpCleanupFlight extends Flight { - public AzurePublicIpCleanupFlight(FlightMap inputParameters, Object applicationContext) { - super(inputParameters, applicationContext); - ApplicationContext appContext = (ApplicationContext) applicationContext; - JanitorDao janitorDao = appContext.getBean(JanitorDao.class); - CrlConfiguration crlConfiguration = appContext.getBean(CrlConfiguration.class); - RetryRuleFixedInterval retryRule = - new RetryRuleFixedInterval(/* intervalSeconds =*/ 180, /* maxCount =*/ 5); - - addStep(new InitialCleanupStep(janitorDao)); - addStep(new AzurePublicIpCleanupStep(crlConfiguration, janitorDao), retryRule); - addStep(new FinalCleanupStep(janitorDao)); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzurePublicIpCleanupStep.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzurePublicIpCleanupStep.java deleted file mode 100644 index d920aa47..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzurePublicIpCleanupStep.java +++ /dev/null @@ -1,35 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.janitor.db.ResourceMetadata; -import bio.terra.janitor.generated.model.AzurePublicIp; -import bio.terra.janitor.generated.model.CloudResourceUid; -import bio.terra.stairway.StepResult; -import bio.terra.stairway.exception.RetryException; -import com.azure.resourcemanager.compute.ComputeManager; - -/** Step to clean up an Azure public IP. */ -public class AzurePublicIpCleanupStep extends ResourceCleanupStep { - private final CrlConfiguration crlConfiguration; - - public AzurePublicIpCleanupStep(CrlConfiguration crlConfiguration, JanitorDao janitorDao) { - super(janitorDao); - this.crlConfiguration = crlConfiguration; - } - - @Override - protected StepResult cleanUp(CloudResourceUid resourceUid, ResourceMetadata metadata) - throws InterruptedException, RetryException { - AzurePublicIp ip = resourceUid.getAzurePublicIp(); - ComputeManager computeManager = crlConfiguration.buildComputeManager(ip.getResourceGroup()); - - return AzureUtils.ignoreNotFound( - () -> { - computeManager - .networkManager() - .publicIpAddresses() - .deleteByResourceGroup(ip.getResourceGroup().getResourceGroupName(), ip.getIpName()); - }); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureRelayCleanupFlight.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureRelayCleanupFlight.java deleted file mode 100644 index 08f0691c..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureRelayCleanupFlight.java +++ /dev/null @@ -1,24 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.stairway.Flight; -import bio.terra.stairway.FlightMap; -import bio.terra.stairway.RetryRuleFixedInterval; -import org.springframework.context.ApplicationContext; - -/** Flight to clean up an Azure Relay Namespace. */ -public class AzureRelayCleanupFlight extends Flight { - public AzureRelayCleanupFlight(FlightMap inputParameters, Object applicationContext) { - super(inputParameters, applicationContext); - ApplicationContext appContext = (ApplicationContext) applicationContext; - JanitorDao janitorDao = appContext.getBean(JanitorDao.class); - CrlConfiguration crlConfiguration = appContext.getBean(CrlConfiguration.class); - RetryRuleFixedInterval retryRule = - new RetryRuleFixedInterval(/* intervalSeconds =*/ 180, /* maxCount =*/ 5); - - addStep(new InitialCleanupStep(janitorDao)); - addStep(new AzureRelayCleanupStep(crlConfiguration, janitorDao), retryRule); - addStep(new FinalCleanupStep(janitorDao)); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureRelayCleanupStep.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureRelayCleanupStep.java deleted file mode 100644 index 1f0c2200..00000000 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureRelayCleanupStep.java +++ /dev/null @@ -1,35 +0,0 @@ -package bio.terra.janitor.service.cleanup.flight; - -import bio.terra.janitor.app.configuration.CrlConfiguration; -import bio.terra.janitor.db.JanitorDao; -import bio.terra.janitor.db.ResourceMetadata; -import bio.terra.janitor.generated.model.AzureRelay; -import bio.terra.janitor.generated.model.CloudResourceUid; -import bio.terra.stairway.StepResult; -import bio.terra.stairway.exception.RetryException; -import com.azure.resourcemanager.relay.RelayManager; - -/** Step to clean up an Azure Relay. */ -public class AzureRelayCleanupStep extends ResourceCleanupStep { - private final CrlConfiguration crlConfiguration; - - public AzureRelayCleanupStep(CrlConfiguration crlConfiguration, JanitorDao janitorDao) { - super(janitorDao); - this.crlConfiguration = crlConfiguration; - } - - @Override - protected StepResult cleanUp(CloudResourceUid resourceUid, ResourceMetadata metadata) - throws InterruptedException, RetryException { - AzureRelay relay = resourceUid.getAzureRelay(); - - RelayManager relayManager = crlConfiguration.buildRelayManager(relay.getResourceGroup()); - - return AzureUtils.ignoreNotFound( - () -> - relayManager - .namespaces() - .deleteByResourceGroup( - relay.getResourceGroup().getResourceGroupName(), relay.getRelayName())); - } -} diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupFlight.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupFlight.java index 0f4b4723..1b508848 100644 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupFlight.java +++ b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupFlight.java @@ -21,5 +21,4 @@ public AzureStorageContainerCleanupFlight(FlightMap inputParameters, Object appl addStep(new AzureStorageContainerCleanupStep(crlConfiguration, janitorDao), retryRule); addStep(new FinalCleanupStep(janitorDao)); } - } diff --git a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupStep.java b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupStep.java index f7fb48b4..ff2c2473 100644 --- a/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupStep.java +++ b/src/main/java/bio/terra/janitor/service/cleanup/flight/AzureStorageContainerCleanupStep.java @@ -9,13 +9,12 @@ import bio.terra.stairway.exception.RetryException; import com.azure.resourcemanager.storage.StorageManager; - public class AzureStorageContainerCleanupStep extends ResourceCleanupStep { private final CrlConfiguration crlConfiguration; - public AzureStorageContainerCleanupStep(CrlConfiguration crlConfiguration, - JanitorDao janitorDao) { + public AzureStorageContainerCleanupStep( + CrlConfiguration crlConfiguration, JanitorDao janitorDao) { super(janitorDao); this.crlConfiguration = crlConfiguration; } @@ -24,8 +23,8 @@ public AzureStorageContainerCleanupStep(CrlConfiguration crlConfiguration, protected StepResult cleanUp(CloudResourceUid resourceUid, ResourceMetadata metadata) throws InterruptedException, RetryException { AzureStorageContainer storageContainer = resourceUid.getAzureStorageContainer(); - StorageManager storageManager = crlConfiguration.buildStorageManager( - storageContainer.getResourceGroup()); + StorageManager storageManager = + crlConfiguration.buildStorageManager(storageContainer.getResourceGroup()); return AzureUtils.ignoreNotFound( () -> diff --git a/src/main/resources/static/service_openapi.yaml b/src/main/resources/static/service_openapi.yaml index 2eb69b55..035d7237 100644 --- a/src/main/resources/static/service_openapi.yaml +++ b/src/main/resources/static/service_openapi.yaml @@ -377,9 +377,9 @@ components: AzureResourceGroup: type: object required: - - tenantId + - tenantId - subscriptionId - - resourceGroupName + - resourceGroupName properties: tenantId: description: The tenant ID. @@ -391,18 +391,6 @@ components: description: Name of the resource group. type: string - AzurePublicIp: - type: object - required: - - resourceGroup - - ipName - properties: - resourceGroup: - $ref: '#/components/schemas/AzureResourceGroup' - ipName: - description: The user-provided name to identify the IP. - type: string - AzureDisk: type: object required: @@ -415,30 +403,6 @@ components: description: The user-provided name to identify the disk. type: string - AzureNetwork: - type: object - required: - - resourceGroup - - networkName - properties: - resourceGroup: - $ref: '#/components/schemas/AzureResourceGroup' - networkName: - description: The user-provided name to identify the network. - type: string - - AzureNetworkSecurityGroup: - type: object - required: - - resourceGroup - - networkSecurityGroupName - properties: - resourceGroup: - $ref: '#/components/schemas/AzureResourceGroup' - networkSecurityGroupName: - description: The user-provided name to identify the network security group. - type: string - AzureVirtualMachine: type: object required: @@ -451,18 +415,6 @@ components: description: The user-provided name to identify the VM. type: string - AzureRelay: - type: object - required: - - resourceGroup - - relayName - properties: - resourceGroup: - $ref: '#/components/schemas/AzureResourceGroup' - relayName: - description: The user-provided name to identify the relay. - type: string - AzureRelayHybridConnection: type: object required: @@ -479,77 +431,6 @@ components: description: The user-provided name to identify the Relay HybridConnection. type: string - AzureContainerInstance: - type: object - required: - - resourceGroup - - containerGroupName - - containers - - operatingSystemTypes - properties: - resourceGroup: - $ref: '#/components/schemas/AzureResourceGroup' - containerGroupName: - description: The name of the container group. - type: string - containers: - description: The containers within the container group. - type: array - items: - type: object - required: - - name - - image - properties: - name: - description: The user-provided name of the container instance. - type: string - image: - description: The name of the image used to create the container instance. - type: string - command: - description: The commands to execute within the container instance in exec form. - type: array - items: - type: string - ports: - description: The exposed ports on the container instance. - type: array - items: - type: object - required: - - protocol - - port - properties: - protocol: - description: The protocol associated with the port. - type: string - enum: [TCP, UDP] - port: - type: integer - description: The port number exposed within the container group. - environmentVariables: - description: The environment variables to set in the container instance. - type: array - items: - type: object - required: - - name - properties: - name: - description: The name of the environment variable. - type: string - value: - description: The value of the environment variable. - type: string - secureValue: - description: The value of the secure environment variable. - type: string - operatingSystemTypes: - description: The operating system type required by the containers in the container group. - type: string - enum: [Windows, Linux] - AzureBatchPool: type: object required: @@ -614,6 +495,38 @@ components: description: The user-provided name to identify the managed identity. type: string + AzureDatabase: + type: object + required: + - resourceGroup + - serverName + - databaseName + properties: + resourceGroup: + $ref: '#/components/schemas/AzureResourceGroup' + serverName: + description: Name of the Postgres server + type: string + databaseName: + description: Name of the database + type: string + + AzureKubernetesNamespace: + type: object + required: + - resourceGroup + - clusterName + - namespaceName + properties: + resourceGroup: + $ref: '#/components/schemas/AzureResourceGroup' + clusterName: + description: Name of the AKS cluster + type: string + namespaceName: + description: Namespace name + type: string + # A cloud resource unique identifier. Each CloudResourceUid represents exactly one cloud resource. # We are not doing polymorphism at this moment because of lack of support in swagger-codegen or openApiGenerator # Now CloudResourceUid contains all possible cloud resource types, and we will assume there is only one being set @@ -633,22 +546,12 @@ components: $ref: '#/components/schemas/GoogleBucketUid' googleProjectUid: $ref: '#/components/schemas/GoogleProjectUid' - azurePublicIp: - $ref: '#/components/schemas/AzurePublicIp' azureDisk: $ref: '#/components/schemas/AzureDisk' - azureNetwork: - $ref: '#/components/schemas/AzureNetwork' - azureNetworkSecurityGroup: - $ref: '#/components/schemas/AzureNetworkSecurityGroup' azureVirtualMachine: $ref: '#/components/schemas/AzureVirtualMachine' - azureRelay: - $ref: '#/components/schemas/AzureRelay' azureRelayHybridConnection: $ref: '#/components/schemas/AzureRelayHybridConnection' - azureContainerInstance: - $ref: '#/components/schemas/AzureContainerInstance' azureBatchPool: $ref: '#/components/schemas/AzureBatchPool' terraWorkspace: @@ -657,6 +560,10 @@ components: $ref: '#/components/schemas/AzureManagedIdentity' azureStorageContainer: $ref: '#/components/schemas/AzureStorageContainer' + azureDatabase: + $ref: '#/components/schemas/AzureDatabase' + azureKubernetesNamespace: + $ref: '#/components/schemas/AzureKubernetesNamespace' responses: ErrorResponse: diff --git a/src/test/java/bio/terra/janitor/db/BackwardsCompatibilityTest.java b/src/test/java/bio/terra/janitor/db/BackwardsCompatibilityTest.java index 42968b5f..d22e2834 100644 --- a/src/test/java/bio/terra/janitor/db/BackwardsCompatibilityTest.java +++ b/src/test/java/bio/terra/janitor/db/BackwardsCompatibilityTest.java @@ -28,19 +28,15 @@ public void resourceType() { assertEquals( ResourceType.GOOGLE_NOTEBOOK_INSTANCE, ResourceType.valueOf("GOOGLE_NOTEBOOK_INSTANCE")); assertEquals(ResourceType.GOOGLE_PROJECT, ResourceType.valueOf("GOOGLE_PROJECT")); - assertEquals(ResourceType.AZURE_PUBLIC_IP, ResourceType.valueOf("AZURE_PUBLIC_IP")); - assertEquals( - ResourceType.AZURE_NETWORK_SECURITY_GROUP, - ResourceType.valueOf("AZURE_NETWORK_SECURITY_GROUP")); - assertEquals(ResourceType.AZURE_NETWORK, ResourceType.valueOf("AZURE_NETWORK")); assertEquals(ResourceType.AZURE_DISK, ResourceType.valueOf("AZURE_DISK")); assertEquals(ResourceType.AZURE_VIRTUAL_MACHINE, ResourceType.valueOf("AZURE_VIRTUAL_MACHINE")); - assertEquals(ResourceType.AZURE_RELAY, ResourceType.valueOf("AZURE_RELAY")); assertEquals( ResourceType.AZURE_RELAY_CONNECTION, ResourceType.valueOf("AZURE_RELAY_CONNECTION")); - assertEquals( - ResourceType.AZURE_CONTAINER_INSTANCE, ResourceType.valueOf("AZURE_CONTAINER_INSTANCE")); assertEquals(ResourceType.TERRA_WORKSPACE, ResourceType.valueOf("TERRA_WORKSPACE")); + assertEquals(ResourceType.AZURE_DATABASE, ResourceType.valueOf("AZURE_DATABASE")); + assertEquals( + ResourceType.AZURE_KUBERNETES_NAMESPACE, + ResourceType.valueOf("AZURE_KUBERNETES_NAMESPACE")); } /** diff --git a/src/test/java/bio/terra/janitor/integration/TrackResourceIntegrationTest.java b/src/test/java/bio/terra/janitor/integration/TrackResourceIntegrationTest.java index ce69626b..5a90ee75 100644 --- a/src/test/java/bio/terra/janitor/integration/TrackResourceIntegrationTest.java +++ b/src/test/java/bio/terra/janitor/integration/TrackResourceIntegrationTest.java @@ -9,7 +9,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import bio.terra.cloudres.azure.resourcemanager.common.Defaults; import bio.terra.cloudres.google.api.services.common.OperationCow; import bio.terra.cloudres.google.api.services.common.OperationUtils; import bio.terra.cloudres.google.bigquery.BigQueryCow; @@ -22,13 +21,8 @@ import bio.terra.janitor.app.configuration.TrackResourcePubsubConfiguration; import bio.terra.janitor.common.BaseIntegrationTest; import bio.terra.janitor.db.JanitorDao; -import bio.terra.janitor.generated.model.AzureContainerInstance; import bio.terra.janitor.generated.model.AzureDisk; import bio.terra.janitor.generated.model.AzureManagedIdentity; -import bio.terra.janitor.generated.model.AzureNetwork; -import bio.terra.janitor.generated.model.AzureNetworkSecurityGroup; -import bio.terra.janitor.generated.model.AzurePublicIp; -import bio.terra.janitor.generated.model.AzureRelay; import bio.terra.janitor.generated.model.AzureRelayHybridConnection; import bio.terra.janitor.generated.model.AzureStorageContainer; import bio.terra.janitor.generated.model.AzureVirtualMachine; @@ -55,17 +49,12 @@ import com.azure.resourcemanager.compute.models.KnownLinuxVirtualMachineImage; import com.azure.resourcemanager.compute.models.VirtualMachine; import com.azure.resourcemanager.compute.models.VirtualMachineSizeTypes; -import com.azure.resourcemanager.containerinstance.ContainerInstanceManager; -import com.azure.resourcemanager.containerinstance.models.ContainerGroup; import com.azure.resourcemanager.msi.MsiManager; import com.azure.resourcemanager.msi.models.Identity; import com.azure.resourcemanager.network.models.Network; import com.azure.resourcemanager.network.models.NetworkInterface; -import com.azure.resourcemanager.network.models.NetworkSecurityGroup; -import com.azure.resourcemanager.network.models.PublicIpAddress; import com.azure.resourcemanager.relay.RelayManager; import com.azure.resourcemanager.relay.models.HybridConnection; -import com.azure.resourcemanager.relay.models.RelayNamespace; import com.azure.resourcemanager.storage.StorageManager; import com.azure.resourcemanager.storage.models.BlobContainer; import com.azure.resourcemanager.storage.models.PublicAccess; @@ -110,19 +99,15 @@ @AutoConfigureMockMvc public class TrackResourceIntegrationTest extends BaseIntegrationTest { - @Autowired - private TrackResourcePubsubConfiguration trackResourcePubsubConfiguration; - @Autowired - private TestConfiguration testConfiguration; - @Autowired - private MockMvc mvc; + @Autowired private TrackResourcePubsubConfiguration trackResourcePubsubConfiguration; + @Autowired private TestConfiguration testConfiguration; + @Autowired private MockMvc mvc; @Autowired @Qualifier(OBJECT_MAPPER) private ObjectMapper objectMapper; - @Autowired - private CrlConfiguration crlConfiguration; + @Autowired private CrlConfiguration crlConfiguration; private Publisher publisher; @@ -133,11 +118,9 @@ public class TrackResourceIntegrationTest extends BaseIntegrationTest { private String projectId; private ComputeManager computeManager; private RelayManager relayManager; - private ContainerInstanceManager containerInstanceManager; private MsiManager msiManager; private StorageManager storageManager; - @MockBean - private WorkspaceManagerService mockWorkspaceManagerService; + @MockBean private WorkspaceManagerService mockWorkspaceManagerService; private static final Map DEFAULT_LABELS = ImmutableMap.of("key1", "value1", "key2", "value2"); @@ -189,13 +172,12 @@ public void setUp() throws Exception { relayManager = crlConfiguration.buildRelayManager(testConfiguration.getAzureResourceGroup()); - containerInstanceManager = - crlConfiguration.buildContainerInstance(testConfiguration.getAzureResourceGroup()); - msiManager = crlConfiguration.buildMsiManager(testConfiguration.getAzureResourceGroup()); - storageManager = crlConfiguration.buildStorageManager( - testConfiguration.getAzureResourceGroup()); + storageManager = + crlConfiguration.buildStorageManager(testConfiguration.getAzureResourceGroup()); + + // TODO: test DB and namespace } @AfterEach @@ -225,9 +207,7 @@ public void subscribeAndCleanupResource_googleBucket() throws Exception { assertNull(storageCow.get(blobId)); } - /** - * Try to let Janitor cleanup a Bucket that is already deleted in cloud. - */ + /** Try to let Janitor cleanup a Bucket that is already deleted in cloud. */ @Test public void subscribeAndCleanupResource_alreadyDeletedBucket() throws Exception { // Creates bucket and verify. @@ -268,9 +248,7 @@ public void subscribeAndCleanupResource_googleBlob() throws Exception { storageCow.delete(bucketName); } - /** - * Try to let Janitor cleanup a Blob that is already deleted in cloud. - */ + /** Try to let Janitor cleanup a Blob that is already deleted in cloud. */ @Test public void subscribeAndCleanupResource_alreadyDeletedBlob() throws Exception { // Creates Blob and verify. @@ -533,7 +511,7 @@ public void subscribeAndCleanupResource_neverCreatedGoogleProject_withMetadataOk // by metadata, we can successfully recognize that the project never existed. CreateResourceRequestBody request = newExpiredCreateResourceMessage( - resource, JanitorDao.currentOffsetDateTime(), /*resourceMetadata=*/ null) + resource, JanitorDao.currentOffsetDateTime(), /*resourceMetadata=*/ null) .resourceMetadata( new ResourceMetadata() .googleProjectParent(testConfiguration.getParentResourceId())); @@ -554,230 +532,37 @@ public void subscribeAndCleanupResource_neverCreatedGoogleProject_withoutMetadat } @Test - public void subscribeAndCleanupResource_azurePublicIp() throws Exception { - // Creates IP - String ipName = randomNameWithUnderscore(); - PublicIpAddress createdIp = - computeManager - .networkManager() - .publicIpAddresses() - .define(ipName) - .withRegion(Region.US_EAST) - .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .withDynamicIP() - .withTag("janitor.integration.test", "true") - .create(); - - // Verify resources are created in Azure - assertEquals( - ipName, computeManager.networkManager().publicIpAddresses().getById(createdIp.id()).name()); - - CloudResourceUid ipUid = - new CloudResourceUid() - .azurePublicIp( - new AzurePublicIp() - .ipName(ipName) - .resourceGroup(testConfiguration.getAzureResourceGroup())); - - // Publish a message to cleanup the IP. - publishAndVerify(ipUid, ResourceState.DONE); - - // Resource is removed - ManagementException ipDeleted = - assertThrows( - ManagementException.class, - () -> computeManager.networkManager().publicIpAddresses().getById(createdIp.id())); - assertEquals("ResourceNotFound", ipDeleted.getValue().getCode()); - } - - @Test - public void subscribeAndCleanupResource_azureRelayAndHybridConnections() throws Exception { - // Creates IP - String relayName = randomRelayNameSpace(); - RelayNamespace createdNameSpace = - relayManager - .namespaces() - .define(relayName) - .withRegion(Region.US_EAST) - .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .create(); - - // Verify resources are created in Azure - assertEquals(relayName, relayManager.namespaces().getById(createdNameSpace.id()).name()); - + public void subscribeAndCleanupResource_azureRelayHybridConnections() throws Exception { String hybridConnectionName = randomNameWithUnderscore(); HybridConnection createdHc = relayManager .hybridConnections() .define(hybridConnectionName) - .withExistingNamespace(testConfiguration.getAzureManagedResourceGroupName(), relayName) + .withExistingNamespace( + testConfiguration.getAzureManagedResourceGroupName(), + testConfiguration.getAzureRelayNamespace()) .create(); assertEquals( hybridConnectionName, relayManager.hybridConnections().getById(createdHc.id()).name()); - CloudResourceUid relayUid = - new CloudResourceUid() - .azureRelay( - new AzureRelay() - .relayName(relayName) - .resourceGroup(testConfiguration.getAzureResourceGroup())); - CloudResourceUid hcUid = new CloudResourceUid() .azureRelayHybridConnection( new AzureRelayHybridConnection() .hybridConnectionName(hybridConnectionName) - .namespace(relayName) + .namespace(testConfiguration.getAzureRelayNamespace()) .resourceGroup(testConfiguration.getAzureResourceGroup())); - // Publish a message to cleanup the IP. + // Publish a message to cleanup the hybrid connection. publishAndVerify(hcUid, ResourceState.DONE); + // Resource is removed ManagementException removeHc = assertThrows( ManagementException.class, () -> relayManager.hybridConnections().getById(createdHc.id())); assertEquals("EntityNotFound", removeHc.getValue().getCode()); - - // Publish a message to cleanup the IP. - publishAndVerify(relayUid, ResourceState.DONE); - - // Resource is removed - ManagementException removeRelay = - assertThrows( - ManagementException.class, - () -> relayManager.namespaces().getById(createdNameSpace.id())); - assertEquals("NotFound", removeRelay.getValue().getCode()); - } - - @Test - public void subscribeAndCleanupResource_azureContainerInstance() throws Exception { - String containerGroupName = randomName(); - // create container group - ContainerGroup createdContainerInstance = - containerInstanceManager - .containerGroups() - .define(containerGroupName) - .withRegion(Region.US_EAST) - .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .withLinux() - .withPublicImageRegistryOnly() - .withoutVolume() - .defineContainerInstance("test-container-instance") - .withImage("busybox") - .withoutPorts() - .attach() - .create(); - - // Verify resources are created in Azure - assertEquals( - containerGroupName, - containerInstanceManager.containerGroups().getById(createdContainerInstance.id()).name()); - - CloudResourceUid containerGroupUid = - new CloudResourceUid() - .azureContainerInstance( - new AzureContainerInstance() - .containerGroupName(containerGroupName) - .resourceGroup(testConfiguration.getAzureResourceGroup())); - - // publish and verify to clean up the container group - publishAndVerify(containerGroupUid, ResourceState.DONE); - - // Resource is removed - ManagementException removeContainerGroup = - assertThrows( - ManagementException.class, - () -> - containerInstanceManager.containerGroups().getById(createdContainerInstance.id())); - assertEquals("ResourceNotFound", removeContainerGroup.getValue().getCode()); - } - - @Test - public void subscribeAndCleanupResource_azureNetworkSecurityGroup() throws Exception { - // Creates network security group - String networkSgName = randomNameWithUnderscore(); - NetworkSecurityGroup createdNetworkSg = - computeManager - .networkManager() - .networkSecurityGroups() - .define(networkSgName) - .withRegion(Region.US_EAST) - .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .withTag("janitor.integration.test", "true") - .create(); - - // Verify resources are created in Azure - assertEquals( - networkSgName, - computeManager - .networkManager() - .networkSecurityGroups() - .getById(createdNetworkSg.id()) - .name()); - - CloudResourceUid networkSgUid = - new CloudResourceUid() - .azureNetworkSecurityGroup( - new AzureNetworkSecurityGroup() - .networkSecurityGroupName(networkSgName) - .resourceGroup(testConfiguration.getAzureResourceGroup())); - - // Publish a message to cleanup the network security group. - publishAndVerify(networkSgUid, ResourceState.DONE); - - // Resource is removed - ManagementException networkSgDeleted = - assertThrows( - ManagementException.class, - () -> - computeManager - .networkManager() - .networkSecurityGroups() - .getById(createdNetworkSg.id())); - assertEquals("ResourceNotFound", networkSgDeleted.getValue().getCode()); - } - - @Test - public void subscribeAndCleanupResource_azureNetwork() throws Exception { - // Creates network - String networkName = randomNameWithUnderscore(); - Network createdNetwork = - computeManager - .networkManager() - .networks() - .define(networkName) - .withRegion(Region.US_EAST) - .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .withTag("janitor.integration.test", "true") - .withAddressSpace("10.0.0.0/16") - .defineSubnet("mysubnet") - .withAddressPrefix("10.0.0.0/24") - .attach() - .create(); - - // Verify resources are created in Azure - assertEquals( - networkName, - computeManager.networkManager().networks().getById(createdNetwork.id()).name()); - - CloudResourceUid networkUid = - new CloudResourceUid() - .azureNetwork( - new AzureNetwork() - .networkName(networkName) - .resourceGroup(testConfiguration.getAzureResourceGroup())); - - // Publish a message to cleanup the network. - publishAndVerify(networkUid, ResourceState.DONE); - - // Resource is removed - ManagementException networkDeleted = - assertThrows( - ManagementException.class, - () -> computeManager.networkManager().networks().getById(createdNetwork.id())); - assertEquals("ResourceNotFound", networkDeleted.getValue().getCode()); } @Test @@ -788,7 +573,7 @@ public void subscribeAndCleanupResource_azureDisk() throws Exception { computeManager .disks() .define(diskName) - .withRegion(Region.US_EAST) + .withRegion(Region.US_SOUTH_CENTRAL) .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) .withData() .withSizeInGB(500) @@ -817,46 +602,39 @@ public void subscribeAndCleanupResource_azureDisk() throws Exception { @Test public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception { - // Creates network - String networkName = randomNameWithUnderscore(); - String subnetName = randomNameWithUnderscore(); - Network createdNetwork = - computeManager - .networkManager() - .networks() - .define(networkName) - .withRegion(Region.US_EAST) - .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .withTag("janitor.integration.test", "true") - .withAddressSpace("10.0.0.0/16") - .defineSubnet(subnetName) - .withAddressPrefix("10.0.0.0/24") - .attach() - .create(); - // Creates disk String diskName = randomNameWithUnderscore(); Disk createdDisk = computeManager .disks() .define(diskName) - .withRegion(Region.US_EAST) + .withRegion(Region.US_SOUTH_CENTRAL) .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) .withData() .withSizeInGB(500) .withTag("janitor.integration.test", "true") .create(); + // Resolve network + Network network = + computeManager + .networkManager() + .networks() + .getByResourceGroup( + testConfiguration.getAzureManagedResourceGroupName(), + testConfiguration.getAzureVnetName()); + + // Create nic String nicName = randomNameWithUnderscore(); NetworkInterface createdNetworkInterface = computeManager .networkManager() .networkInterfaces() .define(nicName) - .withRegion(Region.US_EAST) + .withRegion(Region.US_SOUTH_CENTRAL) .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) - .withExistingPrimaryNetwork(createdNetwork) - .withSubnet(subnetName) + .withExistingPrimaryNetwork(network) + .withSubnet("COMPUTE_SUBNET") .withPrimaryPrivateIPAddressDynamic() .withTag("janitor.integration.test", "true") .create(); @@ -867,7 +645,7 @@ public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception { computeManager .virtualMachines() .define(vmName) - .withRegion(Region.US_EAST) + .withRegion(Region.US_SOUTH_CENTRAL) .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) .withExistingPrimaryNetworkInterface(createdNetworkInterface) .withPopularLinuxImage(KnownLinuxVirtualMachineImage.CENTOS_8_3) @@ -879,20 +657,9 @@ public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception { .create(); // Verify resources are created in Azure - assertEquals( - networkName, - computeManager.networkManager().networks().getById(createdNetwork.id()).name()); - assertEquals(diskName, computeManager.disks().getById(createdDisk.id()).name()); - assertEquals(vmName, computeManager.virtualMachines().getById(createdVm.id()).name()); - CloudResourceUid networkUid = - new CloudResourceUid() - .azureNetwork( - new AzureNetwork() - .networkName(networkName) - .resourceGroup(testConfiguration.getAzureResourceGroup())); CloudResourceUid diskUid = new CloudResourceUid() .azureDisk( @@ -906,10 +673,9 @@ public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception { .vmName(vmName) .resourceGroup(testConfiguration.getAzureResourceGroup())); - // Publish messages to cleanup the vm, network, and disk. + // Publish messages to cleanup the vm and disk. publishAndVerify(vmUid, ResourceState.DONE); publishAndVerify(diskUid, ResourceState.DONE); - publishAndVerify(networkUid, ResourceState.DONE); // All resources are removed ManagementException diskDeleted = @@ -917,12 +683,6 @@ public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception { ManagementException.class, () -> computeManager.disks().getById(createdDisk.id())); assertEquals("ResourceNotFound", diskDeleted.getValue().getCode()); - ManagementException networkDeleted = - assertThrows( - ManagementException.class, - () -> computeManager.networkManager().networks().getById(createdNetwork.id())); - assertEquals("ResourceNotFound", networkDeleted.getValue().getCode()); - ManagementException vmDeleted = assertThrows( ManagementException.class, @@ -930,9 +690,7 @@ public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception { assertEquals("ResourceNotFound", vmDeleted.getValue().getCode()); } - /** - * Clean up a fake WSM workspace. - */ + /** Clean up a fake WSM workspace. */ @Test public void subscribeAndCleanupResource_terraWorkspace() throws Exception { // Cleaning up workspaces relies on domain-wide delegation to impersonate test users. The tools @@ -953,9 +711,7 @@ public void subscribeAndCleanupResource_terraWorkspace() throws Exception { publishAndVerify(resource, ResourceState.DONE, metadata); } - /** - * Try to clean up an already deleted workspace, should succeed. - */ + /** Try to clean up an already deleted workspace, should succeed. */ @Test public void subscribeAndCleanupResource_alreadyDeletedTerraWorkspace() throws Exception { UUID fakeWorkspaceId = UUID.randomUUID(); @@ -986,7 +742,7 @@ public void subscribeAndCleanupResource_azureManagedIdentity() throws Exception msiManager .identities() .define(identityName) - .withRegion(Region.US_EAST) + .withRegion(Region.US_SOUTH_CENTRAL) .withExistingResourceGroup(testConfiguration.getAzureManagedResourceGroupName()) .withTag("janitor.integration.test", "true") .create(); @@ -1013,22 +769,29 @@ public void subscribeAndCleanupResource_azureManagedIdentity() throws Exception @Test public void subscribeAndCleanupResource_azureStorageContainer() throws Exception { - String storageAccountName = "teststgacctdonotdelete"; String storageContainerName = randomName(); // create storage container - BlobContainer createdStorageContainer = storageManager - .blobContainers() - .defineContainer(storageContainerName) - .withExistingStorageAccount( - testConfiguration.getAzureManagedResourceGroupName(), storageAccountName) - .withPublicAccess(PublicAccess.NONE) - .create(); + BlobContainer createdStorageContainer = + storageManager + .blobContainers() + .defineContainer(storageContainerName) + .withExistingStorageAccount( + testConfiguration.getAzureManagedResourceGroupName(), + testConfiguration.getAzureStorageAccountName()) + .withPublicAccess(PublicAccess.NONE) + .create(); // verify container is created in Azure - assertEquals(storageContainerName, storageManager.blobContainers().get( - testConfiguration.getAzureManagedResourceGroupName(), storageAccountName, - createdStorageContainer.name()).name()); + assertEquals( + storageContainerName, + storageManager + .blobContainers() + .get( + testConfiguration.getAzureManagedResourceGroupName(), + testConfiguration.getAzureStorageAccountName(), + createdStorageContainer.name()) + .name()); // publish and verify cleanup of storage container by Janitor publishAndVerify( @@ -1036,18 +799,26 @@ public void subscribeAndCleanupResource_azureStorageContainer() throws Exception .azureStorageContainer( new AzureStorageContainer() .storageContainerName(storageContainerName) - .storageAccountName(storageAccountName) - .resourceGroup(testConfiguration.getAzureResourceGroup())) - , ResourceState.DONE); + .storageAccountName(testConfiguration.getAzureStorageAccountName()) + .resourceGroup(testConfiguration.getAzureResourceGroup())), + ResourceState.DONE); // verify storage container is no longer present in Azure - ManagementException removeStorageContainer = assertThrows(ManagementException.class, () -> - storageManager.blobContainers().get( - testConfiguration.getAzureManagedResourceGroupName(), storageAccountName, - createdStorageContainer.name())); + ManagementException removeStorageContainer = + assertThrows( + ManagementException.class, + () -> + storageManager + .blobContainers() + .get( + testConfiguration.getAzureManagedResourceGroupName(), + testConfiguration.getAzureStorageAccountName(), + createdStorageContainer.name())); assertEquals("ContainerNotFound", removeStorageContainer.getValue().getCode()); } + // TODO add test cases for database, k8s namespace in the next PR + private void publishAndVerify(CloudResourceUid resource, ResourceState expectedState) throws Exception { publishAndVerify(resource, expectedState, null); @@ -1083,9 +854,7 @@ private void publishAndVerify(CreateResourceRequestBody request, ResourceState e assertEquals(expectedState, trackedResourceInfo.getState()); } - /** - * Returns a new {@link CreateResourceRequestBody} for a resource that is ready for cleanup. - */ + /** Returns a new {@link CreateResourceRequestBody} for a resource that is ready for cleanup. */ private CreateResourceRequestBody newExpiredCreateResourceMessage( CloudResourceUid resource, OffsetDateTime now, ResourceMetadata metadata) { return new CreateResourceRequestBody() @@ -1130,9 +899,7 @@ private void createNotebookInstance(InstanceName instanceName) assertNull(operation.getOperation().getError()); } - /** - * Creates an {@link Instance} that's ready to be created. - */ + /** Creates an {@link Instance} that's ready to be created. */ private static Instance defaultInstance() { return new Instance() // A VM or Container image is required. @@ -1142,46 +909,34 @@ private static Instance defaultInstance() { .setMachineType("e2-standard-2"); } - /** - * Generates a random name to use for a cloud resource. - */ + /** Generates a random name to use for a cloud resource. */ private static String randomName() { return UUID.randomUUID().toString(); } - /** - * Generates a random name to use for a cloud resource. - */ + /** Generates a random name to use for a cloud resource. */ private static String randomRelayNameSpace() { return "a" + randomName().substring(0, 8) + "b"; } - /** - * Generates a random name to and replace '-' with '_'. - */ + /** Generates a random name to and replace '-' with '_'. */ private static String randomNameWithUnderscore() { return UUID.randomUUID().toString().replace('-', '_'); } - /** - * Generates a random project id start with a letter and 30 characters long. - */ + /** Generates a random project id start with a letter and 30 characters long. */ private static String randomProjectId() { // Project ids must starting with a letter and be no more than 30 characters long. return "p" + randomName().substring(0, 29); } - /** - * Generates a random notebook instance id. - */ + /** Generates a random notebook instance id. */ private static String randomNotebookInstanceId() { // Instance ids must start with a letter, be all lower case letters, numbers, and dashses. return "n" + randomName().toLowerCase(); } - /** - * Poll from get resource endpoint until it gets resources from Janitor in the expected state. - */ + /** Poll from get resource endpoint until it gets resources from Janitor in the expected state. */ private TrackedResourceInfoList pollUntilResourceState( CloudResourceUid resource, ResourceState expectedState, Duration period, int maxNumPolls) throws Exception { diff --git a/src/test/java/bio/terra/janitor/integration/common/configuration/TestConfiguration.java b/src/test/java/bio/terra/janitor/integration/common/configuration/TestConfiguration.java index a8b5f92b..96798ed8 100644 --- a/src/test/java/bio/terra/janitor/integration/common/configuration/TestConfiguration.java +++ b/src/test/java/bio/terra/janitor/integration/common/configuration/TestConfiguration.java @@ -52,6 +52,18 @@ public class TestConfiguration { /** Name of the Azure managed resource group to create resources within. */ private String azureManagedResourceGroupName; + /** Name of the static Azure storage account. */ + private String azureStorageAccountName; + + /** Name of the static Azure Relay namespace. */ + private String azureRelayNamespace; + + /** Name of the status Azure postgres flex server. */ + private String azurePostgresServerName; + + /** Name of the static Azure vnet. */ + private String azureVnetName; + public String getTrackResourceTopicId() { return trackResourceTopicId; } @@ -128,6 +140,38 @@ public void setAzureManagedResourceGroupName(String azureManagedResourceGroupNam this.azureManagedResourceGroupName = azureManagedResourceGroupName; } + public String getAzureStorageAccountName() { + return azureStorageAccountName; + } + + public void setAzureStorageAccountName(String azureStorageAccountName) { + this.azureStorageAccountName = azureStorageAccountName; + } + + public String getAzureRelayNamespace() { + return azureRelayNamespace; + } + + public void setAzureRelayNamespace(String azureRelayNamespace) { + this.azureRelayNamespace = azureRelayNamespace; + } + + public String getAzurePostgresServerName() { + return azurePostgresServerName; + } + + public void setAzurePostgresServerName(String azurePostgresServerName) { + this.azurePostgresServerName = azurePostgresServerName; + } + + public String getAzureVnetName() { + return azureVnetName; + } + + public void setAzureVnetName(String azureVnetName) { + this.azureVnetName = azureVnetName; + } + /** * Janitor Client {@link ServiceAccountCredentials} which has permission to publish message to * Janitor. diff --git a/src/test/resources/application-integration.yml b/src/test/resources/application-integration.yml index b764a13b..07b57649 100644 --- a/src/test/resources/application-integration.yml +++ b/src/test/resources/application-integration.yml @@ -17,6 +17,12 @@ janitor: resource-credential-file-path: rendered/cloud-access-sa-account.json resource-project-id: terra-janitor-test track-resource-topic-id: crljanitor-toolsalpha-pubsub-topic - azure-tenant-id: 0cb7a640-45a2-4ed6-be9f-63519f86e04b - azure-subscription-id: 3efc5bdf-be0e-44e7-b1d7-c08931e3c16c - azure-managed-resource-group-name: mrg-terra-integration-test-20211118 + # Reusing static MRG from https://github.com/broadinstitute/terra-github-workflows/blob/main/.github/workflows/attach-billing-project-to-landing-zone.yaml + azure-tenant-id: fad90753-2022-4456-9b0a-c7e5b934e408 + azure-subscription-id: f557c728-871d-408c-a28b-eb6b2141a087 + azure-managed-resource-group-name: e2e-90b7728d-dfcd-4d64-9082-58bf63031c64 + azure-storage-account-name: lze1e654d6c6562ee363b56d + azure-relay-namespace: lza4d2630f51887e75b724e5a75d0634c4be23b5ad97dbedc2 + azure-postgres-server-name: lz1e3aee11b147c43091902a440b1d0eb9c0e873ab499fb06af81b16551e0cd + azure-vnet-name: lzd6da2d9d8904cad604d7cf3fd092afcf75242cb8ba6aa345974876fa9f1c26 +