diff --git a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md
index 00348cb9b2..a07fdf6d00 100644
--- a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md
+++ b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md
@@ -78,6 +78,7 @@ Connected issue: #0000
- [ ] Reviewed build logs for anomalies on GitLab `anvilprod`
- [ ] Ran `_select anvilprod.shared && make -C terraform/shared apply` or this PR is not labeled `deploy:shared`
- [ ] Deleted PR branch from GitHub
+- [ ] Deleted PR branch from GitLab `anvilprod`
- [ ] Moved connected issue to *Merged stable* column on ZenHub
- [ ] Moved promoted issues from *Merged lower* to *Merged stable* column on ZenHub
- [ ] Moved promoted issues from *Lower* to *Stable* column on ZenHub
diff --git a/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md b/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md
index 2a468fda36..5d9e665db6 100644
--- a/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md
+++ b/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md
@@ -62,7 +62,6 @@ Connected issue: #0000
- [ ] Build passes on GitLab `prod`
- [ ] Reviewed build logs for anomalies on GitLab `prod`
- [ ] Deleted PR branch from GitHub
-- [ ] Deleted PR branch from GitLab `prod`
### Operator (reindex)
diff --git a/.github/pull_request_template.md.template.py b/.github/pull_request_template.md.template.py
index 40f099c466..0ae92e4434 100644
--- a/.github/pull_request_template.md.template.py
+++ b/.github/pull_request_template.md.template.py
@@ -879,7 +879,7 @@ def emit(t: T, target_branch: str):
'content': f'Deleted PR branch from GitLab `{d}`'
}
for d, s in t.target_deployments(target_branch).items()
- if t is not t.promotion
+ if s is not None
),
*iif(t is T.promotion, [
{
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1b646efebb..343f21ad74 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -77,7 +77,7 @@ test:
timeout: 1h 30m
script:
- make format
- - test "$AZUL_IS_SANDBOX" = 1 && make requirements_update
+ - python -m azul -t config.deployment.is_lower_sandbox && make requirements_update
- make -C lambdas openapi
- make environment.boot
- make -C .github
diff --git a/UPGRADING.rst b/UPGRADING.rst
index ef6472cb01..7a7460d952 100644
--- a/UPGRADING.rst
+++ b/UPGRADING.rst
@@ -20,6 +20,12 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't
have too many entries in this file.
+#6239 Promotions fail in hammerbox with dirty requirements
+==========================================================
+
+Remove the variable ``AZUL_IS_SANDBOX`` from all personal deployments.
+
+
#4655 All bucket names should default to qualified_bucket_name()
================================================================
diff --git a/deployments/anvilbox/environment.py b/deployments/anvilbox/environment.py
index 6ec73b93fd..163ac9c546 100644
--- a/deployments/anvilbox/environment.py
+++ b/deployments/anvilbox/environment.py
@@ -112,8 +112,6 @@ def env() -> Mapping[str, Optional[str]]:
#
'AZUL_DEPLOYMENT_STAGE': 'anvilbox' if is_sandbox else None,
- 'AZUL_IS_SANDBOX': str(int(is_sandbox)),
-
# This deployment uses a subdomain of the `anvildev` deployment's
# domain.
#
diff --git a/deployments/hammerbox/environment.py b/deployments/hammerbox/environment.py
index c592f4b3cc..77ee27062a 100644
--- a/deployments/hammerbox/environment.py
+++ b/deployments/hammerbox/environment.py
@@ -749,8 +749,6 @@ def env() -> Mapping[str, Optional[str]]:
#
'AZUL_DEPLOYMENT_STAGE': 'hammerbox' if is_sandbox else None,
- 'AZUL_IS_SANDBOX': str(int(is_sandbox)),
-
# This deployment uses a subdomain of the `anvilprod` deployment's
# domain.
#
diff --git a/deployments/sandbox/environment.py b/deployments/sandbox/environment.py
index 11d6e9373d..026958e0e2 100644
--- a/deployments/sandbox/environment.py
+++ b/deployments/sandbox/environment.py
@@ -235,8 +235,6 @@ def env() -> Mapping[str, Optional[str]]:
#
'AZUL_DEPLOYMENT_STAGE': 'sandbox' if is_sandbox else None,
- 'AZUL_IS_SANDBOX': str(int(is_sandbox)),
-
# This deployment uses a subdomain of the `dev` deployment's domain.
#
'AZUL_DOMAIN_NAME': 'dev.singlecell.gi.ucsc.edu',
diff --git a/environment b/environment
index 5c3a19ea76..fdbe20e5db 100644
--- a/environment
+++ b/environment
@@ -102,8 +102,8 @@ _logout() {
}
_logout_completely() {
- # We don't use `&&` between function invocations because failing to log out of
- # one realm shouldn't prevent us from attempting to log out of the others.
+ # We don't use `&&` between function invocations because failing to log out of
+ # one realm shouldn't prevent us from attempting to log out of the others.
_logout_google
_logout
_logout_docker_ecr
@@ -258,6 +258,64 @@ _revenv() {
fi
}
+_clone() {
+ if [[ -z "$1" || -z "$2" ]]; then
+ echo "Need two arguments: the name of the deployment to select in the"
+ echo "project clone and the name of the branch to check out in it."
+ return 1
+ fi
+ if [ -n "$VIRTUAL_ENV" ]; then
+ echo "Run 'deactivate' first"
+ return 2
+ fi
+ if [ "$(basename "$PWD")" != azul ]; then
+ echo "The name of the current project directory must be 'azul'"
+ return 3
+ fi
+ deployment="$1"
+ branch="$2"
+ (
+ set -e
+ git worktree add "../azul.${deployment}" "${branch}"
+ cd "../azul.${deployment}"
+ (cd terraform/gitlab/vpn && git submodule update --init easy-rsa)
+ rsync -rvlm \
+ -f '+ */' \
+ -f '+ environment.local.py' \
+ -f '+ /deployments/*.local/environment*.py' \
+ -f '- *' \
+ ../azul/ \
+ .
+ source environment
+ _link "${deployment}"
+ _refresh
+ make virtualenv
+ source .venv/bin/activate
+ make requirements envhook
+ deactivate
+ rsync -av ../azul/.idea .
+ mv .idea/azul.iml ".idea/azul.${deployment}.iml"
+ sed -e '/ Mapping[str, Optional[str]]:
#
'azul_browser_sites': json.dumps({}),
- # 1 if current deployment is a main deployment with the sole purpose of
- # testing feature branches in GitLab before they are merged to the
- # develop branch, 0 otherwise. Personal deployments have this set to 0.
- #
- 'AZUL_IS_SANDBOX': '0',
-
# A list of names of AWS IAM roles that should be given permission to
# manage incidents with AWS support as defined in CIS rule 1.20:
#
diff --git a/scripts/check_branch.py b/scripts/check_branch.py
index 2b27ced067..d75c886330 100644
--- a/scripts/check_branch.py
+++ b/scripts/check_branch.py
@@ -18,24 +18,28 @@
def default_deployment(branch: Optional[str]) -> Optional[str]:
deployments = config.shared_deployments_for_branch(branch)
- return None if deployments is None else deployments[0]
+ return None if deployments is None else deployments[0].name
class BranchDeploymentMismatch(Exception):
def __init__(self,
branch: Optional[str],
- deployment: str,
- allowed: Optional[Sequence[str]]
+ deployment: config.Deployment,
+ allowed: Optional[Sequence[config.Deployment]]
) -> None:
branch = 'Detached head' if branch is None else f'Branch {branch!r}'
- allowed = '' if allowed is None else f'one of {set(allowed)!r} or '
- super().__init__(f'{branch} cannot be deployed to {deployment!r}, '
+ if allowed is None:
+ allowed = ''
+ else:
+ allowed = f'one of {set(d.name for d in allowed)!r} or '
+ super().__init__(f'{branch} cannot be deployed to {deployment.name!r}, '
f'only {allowed}personal deployments.')
def check_branch(branch: Optional[str], deployment: str) -> None:
- if config.is_shared_deployment(deployment):
+ deployment = config.Deployment(deployment)
+ if deployment.is_shared:
deployments = config.shared_deployments_for_branch(branch)
if deployments is None or deployment not in deployments:
raise BranchDeploymentMismatch(branch, deployment, deployments)
diff --git a/scripts/update_subgraph_counts.py b/scripts/update_subgraph_counts.py
index 5916ee2660..b389dce52f 100644
--- a/scripts/update_subgraph_counts.py
+++ b/scripts/update_subgraph_counts.py
@@ -82,7 +82,7 @@ def default_common_prefix(self) -> str:
try:
func = getattr(environment, 'common_prefix')
except AttributeError:
- assert not config.is_sandbox_or_personal_deployment, environment.__path__
+ assert config.deployment.is_main, environment.__path__
return ''
else:
return func(self.count)
diff --git a/src/azul/__init__.py b/src/azul/__init__.py
index 854f2a5978..f74400a62c 100644
--- a/src/azul/__init__.py
+++ b/src/azul/__init__.py
@@ -47,12 +47,12 @@
from azul.collections import (
atuple,
)
-from azul.json_freeze import (
- freeze,
-)
from azul.types import (
JSON,
)
+from azul.vendored.frozendict import (
+ frozendict,
+)
log = logging.getLogger(__name__)
@@ -974,8 +974,103 @@ def domain_name(self) -> str:
def private_api(self) -> bool:
return self._boolean(self.environ['AZUL_PRIVATE_API'])
+ @attr.s(frozen=True, kw_only=False, auto_attribs=True)
+ class Deployment:
+ name: str
+
+ @property
+ def is_shared(self) -> bool:
+ """
+ ``True`` if this deployment is a shared deployment, or ``False`` if
+ it is a personal deployment.
+ """
+ return self in set(chain.from_iterable(config._shared_deployments.values()))
+
+ #: The set of branches that are used for development and that are
+ #: usually deployed to personal, lower and main deployments, but never
+ #: stable ones. The set member ``None`` represents a feature branch or
+ #: detached HEAD.
+ #:
+ unstable_branches = {'develop', None}
+
+ @property
+ def is_stable(self) -> bool:
+ """
+ ``True`` if this deployment must be kept functional for public use
+ at all times.
+ """
+ if self.is_sandbox:
+ return False
+ else:
+ branches = set(
+ branch
+ for branch, deployments in config._shared_deployments.items()
+ if self in deployments
+ )
+ return bool(branches) and branches.isdisjoint(self.unstable_branches)
+
+ @property
+ def is_sandbox(self) -> bool:
+ """
+ ``True`` if this deployment is a shared deployment primarily used
+ for testing branches prior to merging.
+ """
+ return 'box' in self.name
+
+ @property
+ def is_personal(self) -> bool:
+ """
+ ``True`` if this deployment is managed by an individual developer.
+ """
+ return not self.is_shared
+
+ @property
+ def is_sandbox_or_personal(self) -> bool:
+ """
+ ``True`` if this deployment is managed by an individual developer or
+ is a shared deployment primarily used for testing branches prior to
+ merging.
+ """
+ return self.is_sandbox or self.is_personal
+
+ @property
+ def is_main(self) -> bool:
+ """
+ ``True`` if this deployment is a main deployment.
+
+ Main deployments are deployed from long-lived (as opposed to
+ feature) branches and serve some public-facing purpose, be that
+ testing (a lower deployment) or production (a stable deployment).
+ """
+ return not self.is_sandbox_or_personal
+
+ @property
+ def is_lower(self) -> bool:
+ """
+ ``True`` if this deployment is an unstable main deployment.
+ """
+ return self.is_main and not self.is_stable
+
+ @property
+ def is_lower_sandbox(self) -> bool:
+ """
+ ``True`` if this deployment is a sandbox for a lower deployment.
+
+ Note: This method currently only works for the current deployment,
+ i.e., the one created obtained from ``config.deployment``
+ """
+ require(self.name == config.deployment_stage, exception=NotImplementedError)
+ return (
+ self.is_sandbox
+ and config.Deployment(config.main_deployment_stage).is_lower
+ )
+
+ @property
+ def deployment(self) -> Deployment:
+ return self.Deployment(self.deployment_stage)
+
@property
- def _shared_deployments(self) -> Mapping[Optional[str], Sequence[str]]:
+ def _shared_deployments(self) -> Mapping[Optional[str], Sequence[Deployment]]:
"""
Maps a branch name to a sequence of names of shared deployments the
branch can be deployed to. The key of None signifies any other branch
@@ -987,14 +1082,14 @@ def _shared_deployments(self) -> Mapping[Optional[str], Sequence[str]]:
deployments = json.loads(self.environ['azul_shared_deployments'])
require(all(isinstance(v, list) and v for v in deployments.values()),
'Invalid value for azul_shared_deployments')
- return freeze({
- k if k else None: v
+ return frozendict(
+ (k if k else None, tuple(self.Deployment(n) for n in v))
for k, v in deployments.items()
- })
+ )
def shared_deployments_for_branch(self,
- branch: Optional[str]
- ) -> Optional[Sequence[str]]:
+ branch: str | None,
+ ) -> Sequence[Deployment] | None:
"""
The list of names of shared deployments the given branch can be deployed
to or `None` of no such deployments exist. An argument of `None`
@@ -1009,52 +1104,6 @@ def shared_deployments_for_branch(self,
except KeyError:
return None if branch is None else deployments.get(None)
- def is_shared_deployment(self, deployment: Optional[str] = None) -> bool:
- """
- Returns `True` if the deployment of the specified name is a shared
- deployment, or `False` if it is a personal deployment. If no argument is
- passed, or if the argument is `None`, the current deployment's name is
- used instead.
- """
- if deployment is None:
- deployment = self.deployment_stage
- return deployment in set(chain.from_iterable(self._shared_deployments.values()))
-
- #: The set of branches that are used for development and that are usually
- #: deployed to personal, lower and main deployments, but never stable ones.
- #: The set member ``None`` represents a feature branch or detached HEAD.
- #:
- unstable_branches = {'develop', None}
-
- @property
- def is_stable_deployment(self) -> bool:
- """
- Returns `True` if the current deployment must be kept functional for
- public use at all times.
- """
- if self.is_sandbox_deployment:
- return False
- else:
- deployment = self.deployment_stage
- branches = set(
- branch
- for branch, deployments in self._shared_deployments.items()
- if deployment in deployments
- )
- return bool(branches) and branches.isdisjoint(self.unstable_branches)
-
- @property
- def is_sandbox_deployment(self) -> bool:
- """
- Returns True if the current deployment is a shared deployment primarily
- used for testing feature branches.
- """
- return self._boolean(self.environ['AZUL_IS_SANDBOX'])
-
- @property
- def is_sandbox_or_personal_deployment(self) -> bool:
- return self.is_sandbox_deployment or not self.is_shared_deployment()
-
class BrowserSite(TypedDict):
domain: str
bucket: str
diff --git a/src/azul/__main__.py b/src/azul/__main__.py
index 7bacf2a659..37befb1f1c 100644
--- a/src/azul/__main__.py
+++ b/src/azul/__main__.py
@@ -1,8 +1,8 @@
"""
-Extract information from Azul config and print to standard output.
-
-Usage example: python -m azul 'docker.resolve_docker_image_for_launch("pycharm")'
+Evaluate an expression after 'from azul import config, docker' and either print
+the result or return it via the process exit status.
"""
+import argparse
import logging
import sys
@@ -16,8 +16,24 @@
log = logging.getLogger(__name__)
configure_script_logging()
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('expression',
+ help='the Python expression to evaluate')
+group = parser.add_mutually_exclusive_group()
+for status in (True, False):
+ lower = str(status).lower()
+ group.add_argument('--' + lower, '-' + lower[0],
+ dest='status',
+ default=None,
+ action='store_' + lower,
+ help=f'do not print the result of the evaluation but instead '
+ f'exit with a status of 0 if the result is {status}-ish or '
+ f'a non-zero exit status otherwise.')
+args = parser.parse_args(sys.argv[1:])
locals = dict(config=config, docker=docker)
-expression = sys.argv[1]
-result = str(eval(expression, dict(__builtins__={}), locals))
-log.info('Expression str(%s) evaluated to %r', expression, result)
-print(result)
+result = eval(args.expression, dict(__builtins__={}), locals)
+log.info('Expression %r evaluated to %r', args.expression, result)
+if args.status is None:
+ print(result)
+else:
+ sys.exit(0 if bool(result) == args.status else 1)
diff --git a/src/azul/indexer/index_service.py b/src/azul/indexer/index_service.py
index ea69c424b7..0a82592c15 100644
--- a/src/azul/indexer/index_service.py
+++ b/src/azul/indexer/index_service.py
@@ -45,7 +45,6 @@
CatalogName,
cache,
config,
- freeze,
)
from azul.deployment import (
aws,
@@ -89,6 +88,9 @@
from azul.indexer.transform import (
Transformer,
)
+from azul.json_freeze import (
+ freeze,
+)
from azul.logging import (
silenced_es_logger,
)
diff --git a/src/azul/portal_service.py b/src/azul/portal_service.py
index 6de0bcb958..c3793dfbb1 100644
--- a/src/azul/portal_service.py
+++ b/src/azul/portal_service.py
@@ -285,4 +285,4 @@ def _db_url(self) -> str:
@property
def _expiration_tag(self) -> tuple[str, str]:
- return 'expires', str(config.is_sandbox_or_personal_deployment).lower()
+ return 'expires', str(config.deployment.is_sandbox_or_personal).lower()
diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py
index 6b1523f127..2a1ccb5535 100644
--- a/terraform/gitlab/gitlab.tf.json.template.py
+++ b/terraform/gitlab/gitlab.tf.json.template.py
@@ -187,7 +187,7 @@
vpn_subnet = config.vpn_subnet
-split_tunnel = not config.is_stable_deployment
+split_tunnel = not config.deployment.is_stable
# The public key of that keypair
#
diff --git a/test/integration_test.py b/test/integration_test.py
index 3e261cda16..52ca9ac4b2 100644
--- a/test/integration_test.py
+++ b/test/integration_test.py
@@ -305,7 +305,7 @@ def _list_partition_bundles(self,
fqids = self.azul_client.list_bundles(catalog, source, partition_prefix)
num_bundles = len(fqids)
partition = f'Partition {effective_prefix!r} of source {source.spec}'
- if not config.is_sandbox_or_personal_deployment:
+ if not config.deployment.is_sandbox_or_personal:
# For sources that use partitioning, 512 is the desired partition
# size. In practice, we observe the reindex succeeding with sizes
# >700 without the partition size becoming a limiting factor. From
@@ -532,7 +532,7 @@ def _reset_indexer(self):
self.azul_client.reset_indexer(catalogs=config.integration_test_catalogs,
# Can't purge the queues in stable deployment as
# they may contain work for non-IT catalogs.
- purge_queues=not config.is_stable_deployment,
+ purge_queues=not config.deployment.is_stable,
delete_indices=True,
create_indices=True)
@@ -1754,7 +1754,7 @@ def test_expiration_tagging(self):
# FIXME: Re-enable when SlowDown error can be avoided
# https://github.com/DataBiosphere/azul/issues/4285
@unittest.skip('Test disabled. FIXME #4285')
-@unittest.skipUnless(config.is_sandbox_or_personal_deployment,
+@unittest.skipUnless(config.deployment.is_sandbox_or_personal,
'Test would pollute portal DB')
class PortalRegistrationIntegrationTest(PortalTestCase, AlwaysTearDownTestCase):
diff --git a/test/test_check_branch.py b/test/test_check_branch.py
index 9b602ae1d0..4d22af5cb6 100644
--- a/test/test_check_branch.py
+++ b/test/test_check_branch.py
@@ -38,8 +38,8 @@ def expect_exception(branch, deployment, message):
"Detached head cannot be deployed to 'prod', "
"only personal deployments.")
- check_branch('prod', 'hannes.local')
- check_branch('develop', 'hannes.local')
+ check_branch('prod', 'hannes')
+ check_branch('develop', 'hannes')
expect_exception('prod', 'dev',
"Branch 'prod' cannot be deployed to 'dev', "