From d9495deb87494ef0a21360df3f0bb756c5010539 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Sun, 19 May 2024 18:03:18 -0700 Subject: [PATCH 1/8] Fix bug with branch deletion in PR checklist --- .github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md | 1 + .github/PULL_REQUEST_TEMPLATE/prod-hotfix.md | 1 - .github/pull_request_template.md.template.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md index 00348cb9b2..a07fdf6d00 100644 --- a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md @@ -78,6 +78,7 @@ Connected issue: #0000 - [ ] Reviewed build logs for anomalies on GitLab `anvilprod` - [ ] Ran `_select anvilprod.shared && make -C terraform/shared apply` or this PR is not labeled `deploy:shared` - [ ] Deleted PR branch from GitHub +- [ ] Deleted PR branch from GitLab `anvilprod` - [ ] Moved connected issue to *Merged stable* column on ZenHub - [ ] Moved promoted issues from *Merged lower* to *Merged stable* column on ZenHub - [ ] Moved promoted issues from *Lower* to *Stable* column on ZenHub diff --git a/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md b/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md index 2a468fda36..5d9e665db6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md +++ b/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md @@ -62,7 +62,6 @@ Connected issue: #0000 - [ ] Build passes on GitLab `prod` - [ ] Reviewed build logs for anomalies on GitLab `prod` - [ ] Deleted PR branch from GitHub -- [ ] Deleted PR branch from GitLab `prod` ### Operator (reindex) diff --git a/.github/pull_request_template.md.template.py b/.github/pull_request_template.md.template.py index 40f099c466..0ae92e4434 100644 --- a/.github/pull_request_template.md.template.py +++ b/.github/pull_request_template.md.template.py @@ -879,7 +879,7 @@ def emit(t: T, target_branch: str): 'content': f'Deleted PR branch from GitLab `{d}`' } for d, s in t.target_deployments(target_branch).items() - if t is not t.promotion + if s is not None ), *iif(t is T.promotion, [ { From d540325c61f1f56bfd70e9ae48126e111f5a79cd Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Sun, 19 May 2024 23:01:29 -0700 Subject: [PATCH 2/8] Add function to clone current PyCharm project --- environment | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/environment b/environment index 5c3a19ea76..fdbe20e5db 100644 --- a/environment +++ b/environment @@ -102,8 +102,8 @@ _logout() { } _logout_completely() { - # We don't use `&&` between function invocations because failing to log out of - # one realm shouldn't prevent us from attempting to log out of the others. + # We don't use `&&` between function invocations because failing to log out of + # one realm shouldn't prevent us from attempting to log out of the others. _logout_google _logout _logout_docker_ecr @@ -258,6 +258,64 @@ _revenv() { fi } +_clone() { + if [[ -z "$1" || -z "$2" ]]; then + echo "Need two arguments: the name of the deployment to select in the" + echo "project clone and the name of the branch to check out in it." + return 1 + fi + if [ -n "$VIRTUAL_ENV" ]; then + echo "Run 'deactivate' first" + return 2 + fi + if [ "$(basename "$PWD")" != azul ]; then + echo "The name of the current project directory must be 'azul'" + return 3 + fi + deployment="$1" + branch="$2" + ( + set -e + git worktree add "../azul.${deployment}" "${branch}" + cd "../azul.${deployment}" + (cd terraform/gitlab/vpn && git submodule update --init easy-rsa) + rsync -rvlm \ + -f '+ */' \ + -f '+ environment.local.py' \ + -f '+ /deployments/*.local/environment*.py' \ + -f '- *' \ + ../azul/ \ + . + source environment + _link "${deployment}" + _refresh + make virtualenv + source .venv/bin/activate + make requirements envhook + deactivate + rsync -av ../azul/.idea . + mv .idea/azul.iml ".idea/azul.${deployment}.iml" + sed -e '/ Date: Mon, 20 May 2024 23:46:12 -0700 Subject: [PATCH 3/8] Replace AZUL_IS_SANDBOX with naming convention --- UPGRADING.rst | 6 ++++++ deployments/anvilbox/environment.py | 2 -- deployments/hammerbox/environment.py | 2 -- deployments/sandbox/environment.py | 2 -- environment.py | 6 ------ src/azul/__init__.py | 2 +- 6 files changed, 7 insertions(+), 13 deletions(-) diff --git a/UPGRADING.rst b/UPGRADING.rst index ef6472cb01..7a7460d952 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -20,6 +20,12 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#6239 Promotions fail in hammerbox with dirty requirements +========================================================== + +Remove the variable ``AZUL_IS_SANDBOX`` from all personal deployments. + + #4655 All bucket names should default to qualified_bucket_name() ================================================================ diff --git a/deployments/anvilbox/environment.py b/deployments/anvilbox/environment.py index 6ec73b93fd..163ac9c546 100644 --- a/deployments/anvilbox/environment.py +++ b/deployments/anvilbox/environment.py @@ -112,8 +112,6 @@ def env() -> Mapping[str, Optional[str]]: # 'AZUL_DEPLOYMENT_STAGE': 'anvilbox' if is_sandbox else None, - 'AZUL_IS_SANDBOX': str(int(is_sandbox)), - # This deployment uses a subdomain of the `anvildev` deployment's # domain. # diff --git a/deployments/hammerbox/environment.py b/deployments/hammerbox/environment.py index c592f4b3cc..77ee27062a 100644 --- a/deployments/hammerbox/environment.py +++ b/deployments/hammerbox/environment.py @@ -749,8 +749,6 @@ def env() -> Mapping[str, Optional[str]]: # 'AZUL_DEPLOYMENT_STAGE': 'hammerbox' if is_sandbox else None, - 'AZUL_IS_SANDBOX': str(int(is_sandbox)), - # This deployment uses a subdomain of the `anvilprod` deployment's # domain. # diff --git a/deployments/sandbox/environment.py b/deployments/sandbox/environment.py index 11d6e9373d..026958e0e2 100644 --- a/deployments/sandbox/environment.py +++ b/deployments/sandbox/environment.py @@ -235,8 +235,6 @@ def env() -> Mapping[str, Optional[str]]: # 'AZUL_DEPLOYMENT_STAGE': 'sandbox' if is_sandbox else None, - 'AZUL_IS_SANDBOX': str(int(is_sandbox)), - # This deployment uses a subdomain of the `dev` deployment's domain. # 'AZUL_DOMAIN_NAME': 'dev.singlecell.gi.ucsc.edu', diff --git a/environment.py b/environment.py index bcfbedebc8..cde02053bb 100644 --- a/environment.py +++ b/environment.py @@ -840,12 +840,6 @@ def env() -> Mapping[str, Optional[str]]: # 'azul_browser_sites': json.dumps({}), - # 1 if current deployment is a main deployment with the sole purpose of - # testing feature branches in GitLab before they are merged to the - # develop branch, 0 otherwise. Personal deployments have this set to 0. - # - 'AZUL_IS_SANDBOX': '0', - # A list of names of AWS IAM roles that should be given permission to # manage incidents with AWS support as defined in CIS rule 1.20: # diff --git a/src/azul/__init__.py b/src/azul/__init__.py index 854f2a5978..d44fafce4a 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -1049,7 +1049,7 @@ def is_sandbox_deployment(self) -> bool: Returns True if the current deployment is a shared deployment primarily used for testing feature branches. """ - return self._boolean(self.environ['AZUL_IS_SANDBOX']) + return 'box' in self.deployment_stage @property def is_sandbox_or_personal_deployment(self) -> bool: From 6b6537fde59f6faf31fe9bc589405d29abe111f9 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 20 May 2024 23:47:24 -0700 Subject: [PATCH 4/8] Fix invalid deployment name in test --- test/test_check_branch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_check_branch.py b/test/test_check_branch.py index 9b602ae1d0..4d22af5cb6 100644 --- a/test/test_check_branch.py +++ b/test/test_check_branch.py @@ -38,8 +38,8 @@ def expect_exception(branch, deployment, message): "Detached head cannot be deployed to 'prod', " "only personal deployments.") - check_branch('prod', 'hannes.local') - check_branch('develop', 'hannes.local') + check_branch('prod', 'hannes') + check_branch('develop', 'hannes') expect_exception('prod', 'dev', "Branch 'prod' cannot be deployed to 'dev', " From 2bb4098e3f6429a6b75ad06899fe7a98854d07e9 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Fri, 24 May 2024 11:03:53 -0700 Subject: [PATCH 5/8] Eliminate reimport --- src/azul/indexer/index_service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/azul/indexer/index_service.py b/src/azul/indexer/index_service.py index ea69c424b7..0a82592c15 100644 --- a/src/azul/indexer/index_service.py +++ b/src/azul/indexer/index_service.py @@ -45,7 +45,6 @@ CatalogName, cache, config, - freeze, ) from azul.deployment import ( aws, @@ -89,6 +88,9 @@ from azul.indexer.transform import ( Transformer, ) +from azul.json_freeze import ( + freeze, +) from azul.logging import ( silenced_es_logger, ) From 82a19855cc73ffcf323ea36d54948d35e2f354f8 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Tue, 21 May 2024 23:21:06 -0700 Subject: [PATCH 6/8] Fix: Promotions fail in hammerbox with dirty requirements (#6239) --- .gitlab-ci.yml | 2 +- scripts/update_subgraph_counts.py | 2 +- src/azul/__init__.py | 84 +++++++++++++++++---- src/azul/__main__.py | 30 ++++++-- src/azul/portal_service.py | 2 +- terraform/gitlab/gitlab.tf.json.template.py | 2 +- test/integration_test.py | 6 +- 7 files changed, 100 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b646efebb..512c3c35ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ test: timeout: 1h 30m script: - make format - - test "$AZUL_IS_SANDBOX" = 1 && make requirements_update + - python -m azul -t config.is_lower_sandbox_deployment && make requirements_update - make -C lambdas openapi - make environment.boot - make -C .github diff --git a/scripts/update_subgraph_counts.py b/scripts/update_subgraph_counts.py index 5916ee2660..905a2cf9bf 100644 --- a/scripts/update_subgraph_counts.py +++ b/scripts/update_subgraph_counts.py @@ -82,7 +82,7 @@ def default_common_prefix(self) -> str: try: func = getattr(environment, 'common_prefix') except AttributeError: - assert not config.is_sandbox_or_personal_deployment, environment.__path__ + assert not config.is_sandbox_or_personal_deployment(), environment.__path__ return '' else: return func(self.count) diff --git a/src/azul/__init__.py b/src/azul/__init__.py index d44fafce4a..4b4cb97827 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -1009,7 +1009,7 @@ def shared_deployments_for_branch(self, except KeyError: return None if branch is None else deployments.get(None) - def is_shared_deployment(self, deployment: Optional[str] = None) -> bool: + def is_shared_deployment(self, deployment: str | None = None) -> bool: """ Returns `True` if the deployment of the specified name is a shared deployment, or `False` if it is a personal deployment. If no argument is @@ -1026,16 +1026,17 @@ def is_shared_deployment(self, deployment: Optional[str] = None) -> bool: #: unstable_branches = {'develop', None} - @property - def is_stable_deployment(self) -> bool: + def is_stable_deployment(self, deployment: str | None = None) -> bool: """ - Returns `True` if the current deployment must be kept functional for - public use at all times. + Returns `True` if the deployment of the specified name must be kept + functional for public use at all times. If no argument is passed, or if + the argument is `None`, the current deployment's name is used instead. """ - if self.is_sandbox_deployment: + if deployment is None: + deployment = self.deployment_stage + if self.is_sandbox_deployment(deployment): return False else: - deployment = self.deployment_stage branches = set( branch for branch, deployments in self._shared_deployments.items() @@ -1043,17 +1044,72 @@ def is_stable_deployment(self) -> bool: ) return bool(branches) and branches.isdisjoint(self.unstable_branches) - @property - def is_sandbox_deployment(self) -> bool: + def is_sandbox_deployment(self, deployment: str | None = None) -> bool: + """ + Returns `True` if the deployment of the specified name is a shared + deployment primarily used for testing branches prior to merging. If no + argument is passed, or if the argument is `None`, the current + deployment's name is used instead. + """ + if deployment is None: + deployment = self.deployment_stage + return 'box' in deployment + + def is_personal_deployment(self, deployment: str | None = None) -> bool: + """ + Returns `True` if the deployment of the specified name is a deployment + managed by an individual developer. If no argument is passed, or if the + argument is `None`, the current deployment's name is used instead. + """ + return not self.is_shared_deployment(deployment) + + def is_sandbox_or_personal_deployment(self, + deployment: str | None = None + ) -> bool: + """ + Returns `True` if the deployment of the specified name is managed by an + individual developer or if it is a shared deployment primarily used for + testing branches prior to merging. If no argument is passed, or if the + argument is `None`, the current deployment's name is used instead. """ - Returns True if the current deployment is a shared deployment primarily - used for testing feature branches. + return ( + self.is_sandbox_deployment(deployment) + or self.is_personal_deployment(deployment) + ) + + def is_main_deployment(self, deployment: str | None = None) -> bool: + """ + Returns `True` if the deployment of the specified name is a main + deployment. If no argument is passed, or if the argument is `None`, the + current deployment's name is used instead. Main deployments are deployed + from long-lived (as opposed to feature) branches and serve some + public-facing purpose, be that testing (a lower deployment) or + production (a stable deployment). """ - return 'box' in self.deployment_stage + return not self.is_sandbox_or_personal_deployment(deployment) + + def is_lower_deployment(self, deployment: str | None = None) -> bool: + """ + Returns `True` if the current deployment is a main deployment that is + not stable. + """ + if deployment is None: + deployment = self.deployment_stage + return ( + self.is_main_deployment(deployment) + and not self.is_stable_deployment(deployment) + ) @property - def is_sandbox_or_personal_deployment(self) -> bool: - return self.is_sandbox_deployment or not self.is_shared_deployment() + def is_lower_sandbox_deployment(self) -> bool: + """ + Returns `True` if the current deployment is a sandbox for a lower + deployment. + """ + return ( + self.is_sandbox_deployment() + and self.is_lower_deployment(self.main_deployment_stage) + ) class BrowserSite(TypedDict): domain: str diff --git a/src/azul/__main__.py b/src/azul/__main__.py index 7bacf2a659..37befb1f1c 100644 --- a/src/azul/__main__.py +++ b/src/azul/__main__.py @@ -1,8 +1,8 @@ """ -Extract information from Azul config and print to standard output. - -Usage example: python -m azul 'docker.resolve_docker_image_for_launch("pycharm")' +Evaluate an expression after 'from azul import config, docker' and either print +the result or return it via the process exit status. """ +import argparse import logging import sys @@ -16,8 +16,24 @@ log = logging.getLogger(__name__) configure_script_logging() +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('expression', + help='the Python expression to evaluate') +group = parser.add_mutually_exclusive_group() +for status in (True, False): + lower = str(status).lower() + group.add_argument('--' + lower, '-' + lower[0], + dest='status', + default=None, + action='store_' + lower, + help=f'do not print the result of the evaluation but instead ' + f'exit with a status of 0 if the result is {status}-ish or ' + f'a non-zero exit status otherwise.') +args = parser.parse_args(sys.argv[1:]) locals = dict(config=config, docker=docker) -expression = sys.argv[1] -result = str(eval(expression, dict(__builtins__={}), locals)) -log.info('Expression str(%s) evaluated to %r', expression, result) -print(result) +result = eval(args.expression, dict(__builtins__={}), locals) +log.info('Expression %r evaluated to %r', args.expression, result) +if args.status is None: + print(result) +else: + sys.exit(0 if bool(result) == args.status else 1) diff --git a/src/azul/portal_service.py b/src/azul/portal_service.py index 6de0bcb958..f69fe82bf0 100644 --- a/src/azul/portal_service.py +++ b/src/azul/portal_service.py @@ -285,4 +285,4 @@ def _db_url(self) -> str: @property def _expiration_tag(self) -> tuple[str, str]: - return 'expires', str(config.is_sandbox_or_personal_deployment).lower() + return 'expires', str(config.is_sandbox_or_personal_deployment()).lower() diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 6b1523f127..4e7b41ee8b 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -187,7 +187,7 @@ vpn_subnet = config.vpn_subnet -split_tunnel = not config.is_stable_deployment +split_tunnel = not config.is_stable_deployment() # The public key of that keypair # diff --git a/test/integration_test.py b/test/integration_test.py index 3e261cda16..3dddb4eed5 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -305,7 +305,7 @@ def _list_partition_bundles(self, fqids = self.azul_client.list_bundles(catalog, source, partition_prefix) num_bundles = len(fqids) partition = f'Partition {effective_prefix!r} of source {source.spec}' - if not config.is_sandbox_or_personal_deployment: + if not config.is_sandbox_or_personal_deployment(): # For sources that use partitioning, 512 is the desired partition # size. In practice, we observe the reindex succeeding with sizes # >700 without the partition size becoming a limiting factor. From @@ -532,7 +532,7 @@ def _reset_indexer(self): self.azul_client.reset_indexer(catalogs=config.integration_test_catalogs, # Can't purge the queues in stable deployment as # they may contain work for non-IT catalogs. - purge_queues=not config.is_stable_deployment, + purge_queues=not config.is_stable_deployment(), delete_indices=True, create_indices=True) @@ -1754,7 +1754,7 @@ def test_expiration_tagging(self): # FIXME: Re-enable when SlowDown error can be avoided # https://github.com/DataBiosphere/azul/issues/4285 @unittest.skip('Test disabled. FIXME #4285') -@unittest.skipUnless(config.is_sandbox_or_personal_deployment, +@unittest.skipUnless(config.is_sandbox_or_personal_deployment(), 'Test would pollute portal DB') class PortalRegistrationIntegrationTest(PortalTestCase, AlwaysTearDownTestCase): From 5e002c156c74b37158188350d325e50438c76326 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Fri, 24 May 2024 09:42:32 -0700 Subject: [PATCH 7/8] Refactor determination of deployment types --- .gitlab-ci.yml | 2 +- scripts/check_branch.py | 16 +- scripts/update_subgraph_counts.py | 2 +- src/azul/__init__.py | 197 ++++++++++---------- src/azul/portal_service.py | 2 +- terraform/gitlab/gitlab.tf.json.template.py | 2 +- test/integration_test.py | 6 +- 7 files changed, 112 insertions(+), 115 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 512c3c35ae..343f21ad74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ test: timeout: 1h 30m script: - make format - - python -m azul -t config.is_lower_sandbox_deployment && make requirements_update + - python -m azul -t config.deployment.is_lower_sandbox && make requirements_update - make -C lambdas openapi - make environment.boot - make -C .github diff --git a/scripts/check_branch.py b/scripts/check_branch.py index 2b27ced067..d75c886330 100644 --- a/scripts/check_branch.py +++ b/scripts/check_branch.py @@ -18,24 +18,28 @@ def default_deployment(branch: Optional[str]) -> Optional[str]: deployments = config.shared_deployments_for_branch(branch) - return None if deployments is None else deployments[0] + return None if deployments is None else deployments[0].name class BranchDeploymentMismatch(Exception): def __init__(self, branch: Optional[str], - deployment: str, - allowed: Optional[Sequence[str]] + deployment: config.Deployment, + allowed: Optional[Sequence[config.Deployment]] ) -> None: branch = 'Detached head' if branch is None else f'Branch {branch!r}' - allowed = '' if allowed is None else f'one of {set(allowed)!r} or ' - super().__init__(f'{branch} cannot be deployed to {deployment!r}, ' + if allowed is None: + allowed = '' + else: + allowed = f'one of {set(d.name for d in allowed)!r} or ' + super().__init__(f'{branch} cannot be deployed to {deployment.name!r}, ' f'only {allowed}personal deployments.') def check_branch(branch: Optional[str], deployment: str) -> None: - if config.is_shared_deployment(deployment): + deployment = config.Deployment(deployment) + if deployment.is_shared: deployments = config.shared_deployments_for_branch(branch) if deployments is None or deployment not in deployments: raise BranchDeploymentMismatch(branch, deployment, deployments) diff --git a/scripts/update_subgraph_counts.py b/scripts/update_subgraph_counts.py index 905a2cf9bf..b389dce52f 100644 --- a/scripts/update_subgraph_counts.py +++ b/scripts/update_subgraph_counts.py @@ -82,7 +82,7 @@ def default_common_prefix(self) -> str: try: func = getattr(environment, 'common_prefix') except AttributeError: - assert not config.is_sandbox_or_personal_deployment(), environment.__path__ + assert config.deployment.is_main, environment.__path__ return '' else: return func(self.count) diff --git a/src/azul/__init__.py b/src/azul/__init__.py index 4b4cb97827..33d4aab21a 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -47,12 +47,12 @@ from azul.collections import ( atuple, ) -from azul.json_freeze import ( - freeze, -) from azul.types import ( JSON, ) +from azul.vendored.frozendict import ( + frozendict, +) log = logging.getLogger(__name__) @@ -975,7 +975,7 @@ def private_api(self) -> bool: return self._boolean(self.environ['AZUL_PRIVATE_API']) @property - def _shared_deployments(self) -> Mapping[Optional[str], Sequence[str]]: + def _shared_deployments(self) -> Mapping[Optional[str], Sequence['Deployment']]: """ Maps a branch name to a sequence of names of shared deployments the branch can be deployed to. The key of None signifies any other branch @@ -987,14 +987,14 @@ def _shared_deployments(self) -> Mapping[Optional[str], Sequence[str]]: deployments = json.loads(self.environ['azul_shared_deployments']) require(all(isinstance(v, list) and v for v in deployments.values()), 'Invalid value for azul_shared_deployments') - return freeze({ - k if k else None: v + return frozendict( + (k if k else None, tuple(self.Deployment(n) for n in v)) for k, v in deployments.items() - }) + ) def shared_deployments_for_branch(self, - branch: Optional[str] - ) -> Optional[Sequence[str]]: + branch: str | None, + ) -> Sequence['Deployment'] | None: """ The list of names of shared deployments the given branch can be deployed to or `None` of no such deployments exist. An argument of `None` @@ -1009,107 +1009,100 @@ def shared_deployments_for_branch(self, except KeyError: return None if branch is None else deployments.get(None) - def is_shared_deployment(self, deployment: str | None = None) -> bool: - """ - Returns `True` if the deployment of the specified name is a shared - deployment, or `False` if it is a personal deployment. If no argument is - passed, or if the argument is `None`, the current deployment's name is - used instead. - """ - if deployment is None: - deployment = self.deployment_stage - return deployment in set(chain.from_iterable(self._shared_deployments.values())) + @property + def deployment(self) -> 'Deployment': + return self.Deployment(self.deployment_stage) - #: The set of branches that are used for development and that are usually - #: deployed to personal, lower and main deployments, but never stable ones. - #: The set member ``None`` represents a feature branch or detached HEAD. - #: - unstable_branches = {'develop', None} + @attr.s(frozen=True, kw_only=False, auto_attribs=True) + class Deployment: + name: str - def is_stable_deployment(self, deployment: str | None = None) -> bool: - """ - Returns `True` if the deployment of the specified name must be kept - functional for public use at all times. If no argument is passed, or if - the argument is `None`, the current deployment's name is used instead. - """ - if deployment is None: - deployment = self.deployment_stage - if self.is_sandbox_deployment(deployment): - return False - else: - branches = set( - branch - for branch, deployments in self._shared_deployments.items() - if deployment in deployments - ) - return bool(branches) and branches.isdisjoint(self.unstable_branches) + @property + def is_shared(self) -> bool: + """ + ``True`` if this deployment is a shared deployment, or ``False`` if + it is a personal deployment. + """ + return self in set(chain.from_iterable(config._shared_deployments.values())) + + #: The set of branches that are used for development and that are + #: usually deployed to personal, lower and main deployments, but never + #: stable ones. The set member ``None`` represents a feature branch or + #: detached HEAD. + #: + unstable_branches = {'develop', None} - def is_sandbox_deployment(self, deployment: str | None = None) -> bool: - """ - Returns `True` if the deployment of the specified name is a shared - deployment primarily used for testing branches prior to merging. If no - argument is passed, or if the argument is `None`, the current - deployment's name is used instead. - """ - if deployment is None: - deployment = self.deployment_stage - return 'box' in deployment + @property + def is_stable(self) -> bool: + """ + ``True`` if this deployment must be kept functional for public use + at all times. + """ + if self.is_sandbox: + return False + else: + branches = set( + branch + for branch, deployments in config._shared_deployments.items() + if self in deployments + ) + return bool(branches) and branches.isdisjoint(self.unstable_branches) - def is_personal_deployment(self, deployment: str | None = None) -> bool: - """ - Returns `True` if the deployment of the specified name is a deployment - managed by an individual developer. If no argument is passed, or if the - argument is `None`, the current deployment's name is used instead. - """ - return not self.is_shared_deployment(deployment) + @property + def is_sandbox(self) -> bool: + """ + ``True`` if this deployment is a shared deployment primarily used + for testing branches prior to merging. + """ + return 'box' in self.name - def is_sandbox_or_personal_deployment(self, - deployment: str | None = None - ) -> bool: - """ - Returns `True` if the deployment of the specified name is managed by an - individual developer or if it is a shared deployment primarily used for - testing branches prior to merging. If no argument is passed, or if the - argument is `None`, the current deployment's name is used instead. - """ - return ( - self.is_sandbox_deployment(deployment) - or self.is_personal_deployment(deployment) - ) + @property + def is_personal(self) -> bool: + """ + ``True`` if this deployment is managed by an individual developer. + """ + return not self.is_shared - def is_main_deployment(self, deployment: str | None = None) -> bool: - """ - Returns `True` if the deployment of the specified name is a main - deployment. If no argument is passed, or if the argument is `None`, the - current deployment's name is used instead. Main deployments are deployed - from long-lived (as opposed to feature) branches and serve some - public-facing purpose, be that testing (a lower deployment) or - production (a stable deployment). - """ - return not self.is_sandbox_or_personal_deployment(deployment) + @property + def is_sandbox_or_personal(self) -> bool: + """ + ``True`` if this deployment is managed by an individual developer or + is a shared deployment primarily used for testing branches prior to + merging. + """ + return self.is_sandbox or self.is_personal - def is_lower_deployment(self, deployment: str | None = None) -> bool: - """ - Returns `True` if the current deployment is a main deployment that is - not stable. - """ - if deployment is None: - deployment = self.deployment_stage - return ( - self.is_main_deployment(deployment) - and not self.is_stable_deployment(deployment) - ) + @property + def is_main(self) -> bool: + """ + ``True`` if this deployment is a main deployment. - @property - def is_lower_sandbox_deployment(self) -> bool: - """ - Returns `True` if the current deployment is a sandbox for a lower - deployment. - """ - return ( - self.is_sandbox_deployment() - and self.is_lower_deployment(self.main_deployment_stage) - ) + Main deployments are deployed from long-lived (as opposed to + feature) branches and serve some public-facing purpose, be that + testing (a lower deployment) or production (a stable deployment). + """ + return not self.is_sandbox_or_personal + + @property + def is_lower(self) -> bool: + """ + ``True`` if this deployment is an unstable main deployment. + """ + return self.is_main and not self.is_stable + + @property + def is_lower_sandbox(self) -> bool: + """ + ``True`` if this deployment is a sandbox for a lower deployment. + + Note: This method currently only works for the current deployment, + i.e., the one created obtained from ``config.deployment`` + """ + require(self.name == config.deployment_stage, exception=NotImplementedError) + return ( + self.is_sandbox + and config.Deployment(config.main_deployment_stage).is_lower + ) class BrowserSite(TypedDict): domain: str diff --git a/src/azul/portal_service.py b/src/azul/portal_service.py index f69fe82bf0..c3793dfbb1 100644 --- a/src/azul/portal_service.py +++ b/src/azul/portal_service.py @@ -285,4 +285,4 @@ def _db_url(self) -> str: @property def _expiration_tag(self) -> tuple[str, str]: - return 'expires', str(config.is_sandbox_or_personal_deployment()).lower() + return 'expires', str(config.deployment.is_sandbox_or_personal).lower() diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 4e7b41ee8b..2a1ccb5535 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -187,7 +187,7 @@ vpn_subnet = config.vpn_subnet -split_tunnel = not config.is_stable_deployment() +split_tunnel = not config.deployment.is_stable # The public key of that keypair # diff --git a/test/integration_test.py b/test/integration_test.py index 3dddb4eed5..52ca9ac4b2 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -305,7 +305,7 @@ def _list_partition_bundles(self, fqids = self.azul_client.list_bundles(catalog, source, partition_prefix) num_bundles = len(fqids) partition = f'Partition {effective_prefix!r} of source {source.spec}' - if not config.is_sandbox_or_personal_deployment(): + if not config.deployment.is_sandbox_or_personal: # For sources that use partitioning, 512 is the desired partition # size. In practice, we observe the reindex succeeding with sizes # >700 without the partition size becoming a limiting factor. From @@ -532,7 +532,7 @@ def _reset_indexer(self): self.azul_client.reset_indexer(catalogs=config.integration_test_catalogs, # Can't purge the queues in stable deployment as # they may contain work for non-IT catalogs. - purge_queues=not config.is_stable_deployment(), + purge_queues=not config.deployment.is_stable, delete_indices=True, create_indices=True) @@ -1754,7 +1754,7 @@ def test_expiration_tagging(self): # FIXME: Re-enable when SlowDown error can be avoided # https://github.com/DataBiosphere/azul/issues/4285 @unittest.skip('Test disabled. FIXME #4285') -@unittest.skipUnless(config.is_sandbox_or_personal_deployment(), +@unittest.skipUnless(config.deployment.is_sandbox_or_personal, 'Test would pollute portal DB') class PortalRegistrationIntegrationTest(PortalTestCase, AlwaysTearDownTestCase): From 3df842e327342c46bcb6b2f6e8bb22e05e1e652b Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Sat, 25 May 2024 10:59:16 -0700 Subject: [PATCH 8/8] Reorder members to eliminate forward references --- src/azul/__init__.py | 78 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/azul/__init__.py b/src/azul/__init__.py index 33d4aab21a..f74400a62c 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -974,45 +974,6 @@ def domain_name(self) -> str: def private_api(self) -> bool: return self._boolean(self.environ['AZUL_PRIVATE_API']) - @property - def _shared_deployments(self) -> Mapping[Optional[str], Sequence['Deployment']]: - """ - Maps a branch name to a sequence of names of shared deployments the - branch can be deployed to. The key of None signifies any other branch - not mapped explicitly, or a detached head. - """ - # FIXME: Eliminate local import - # https://github.com/DataBiosphere/azul/issues/3133 - import json - deployments = json.loads(self.environ['azul_shared_deployments']) - require(all(isinstance(v, list) and v for v in deployments.values()), - 'Invalid value for azul_shared_deployments') - return frozendict( - (k if k else None, tuple(self.Deployment(n) for n in v)) - for k, v in deployments.items() - ) - - def shared_deployments_for_branch(self, - branch: str | None, - ) -> Sequence['Deployment'] | None: - """ - The list of names of shared deployments the given branch can be deployed - to or `None` of no such deployments exist. An argument of `None` - indicates a detached head. If a list is returned, it will not be empty - and the first element denotes the default deployment. The default - deployment is the one that GitLab deploys a branch to when it builds a - commit on that branch. - """ - deployments = self._shared_deployments - try: - return deployments[branch] - except KeyError: - return None if branch is None else deployments.get(None) - - @property - def deployment(self) -> 'Deployment': - return self.Deployment(self.deployment_stage) - @attr.s(frozen=True, kw_only=False, auto_attribs=True) class Deployment: name: str @@ -1104,6 +1065,45 @@ def is_lower_sandbox(self) -> bool: and config.Deployment(config.main_deployment_stage).is_lower ) + @property + def deployment(self) -> Deployment: + return self.Deployment(self.deployment_stage) + + @property + def _shared_deployments(self) -> Mapping[Optional[str], Sequence[Deployment]]: + """ + Maps a branch name to a sequence of names of shared deployments the + branch can be deployed to. The key of None signifies any other branch + not mapped explicitly, or a detached head. + """ + # FIXME: Eliminate local import + # https://github.com/DataBiosphere/azul/issues/3133 + import json + deployments = json.loads(self.environ['azul_shared_deployments']) + require(all(isinstance(v, list) and v for v in deployments.values()), + 'Invalid value for azul_shared_deployments') + return frozendict( + (k if k else None, tuple(self.Deployment(n) for n in v)) + for k, v in deployments.items() + ) + + def shared_deployments_for_branch(self, + branch: str | None, + ) -> Sequence[Deployment] | None: + """ + The list of names of shared deployments the given branch can be deployed + to or `None` of no such deployments exist. An argument of `None` + indicates a detached head. If a list is returned, it will not be empty + and the first element denotes the default deployment. The default + deployment is the one that GitLab deploys a branch to when it builds a + commit on that branch. + """ + deployments = self._shared_deployments + try: + return deployments[branch] + except KeyError: + return None if branch is None else deployments.get(None) + class BrowserSite(TypedDict): domain: str bucket: str