diff --git a/rfc/portal-registration-api.yaml b/attic/rfc/portal-registration-api.yaml similarity index 100% rename from rfc/portal-registration-api.yaml rename to attic/rfc/portal-registration-api.yaml diff --git a/scripts/manage_portals.py b/attic/scripts/manage_portals.py similarity index 100% rename from scripts/manage_portals.py rename to attic/scripts/manage_portals.py diff --git a/scripts/reset_portal_db.py b/attic/scripts/reset_portal_db.py similarity index 100% rename from scripts/reset_portal_db.py rename to attic/scripts/reset_portal_db.py diff --git a/src/azul/portal_service.py b/attic/src/azul/portal_service.py similarity index 100% rename from src/azul/portal_service.py rename to attic/src/azul/portal_service.py diff --git a/src/azul/version_service.py b/attic/src/azul/version_service.py similarity index 100% rename from src/azul/version_service.py rename to attic/src/azul/version_service.py diff --git a/test/service/test_portal_service.py b/attic/test/service/test_portal_service.py similarity index 100% rename from test/service/test_portal_service.py rename to attic/test/service/test_portal_service.py diff --git a/test/service/test_version_service.py b/attic/test/service/test_version_service.py similarity index 100% rename from test/service/test_version_service.py rename to attic/test/service/test_version_service.py diff --git a/test/version_table_test_case.py b/attic/test/version_table_test_case.py similarity index 100% rename from test/version_table_test_case.py rename to attic/test/version_table_test_case.py diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 824f1350a0..0add4e1f8b 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -90,9 +90,6 @@ from azul.plugins.metadata.hca.indexer.transform import ( value_and_unit, ) -from azul.portal_service import ( - PortalService, -) from azul.service import ( FileUrlFunc, ) @@ -231,7 +228,7 @@ # changes and reset the minor version to zero. Otherwise, increment only # the minor version for backwards compatible changes. A backwards # compatible change is one that does not require updates to clients. - 'version': '9.3' + 'version': '10.0' }, 'tags': [ { @@ -757,42 +754,6 @@ def fmt_error(err_description, params): } -@app.route( - '/integrations', - methods=['GET'], - cors=True, - method_spec=deprecated_method_spec -) -def get_integrations(): - query_params = app.current_request.query_params or {} - validate_params(query_params, - entity_type=Mandatory(str), - integration_type=Mandatory(str), - entity_ids=str) - try: - entity_ids = query_params['entity_ids'] - except KeyError: - # Case where parameter is absent (do not filter using entity_id field) - entity_ids = None - else: - if entity_ids: - # Case where parameter is present and non-empty (filter for matching id value) - entity_ids = {entity_id.strip() for entity_id in entity_ids.split(',')} - else: - # Case where parameter is present but empty (filter for missing entity_id field, - # i.e., there are no acceptable id values) - entity_ids = set() - - entity_type = query_params['entity_type'] - integration_type = query_params['integration_type'] - - portal_service = PortalService() - body = portal_service.list_integrations(entity_type, integration_type, entity_ids) - return Response(status_code=200, - headers={'content-type': 'application/json'}, - body=json.dumps(body)) - - @app.route( '/index/catalogs', methods=['GET'], diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index 84d45c885e..d40dfd3ca4 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "azul-service-dev", "description": "\n# Overview\n\nAzul is a REST web service for querying metadata associated with\nboth experimental and analysis data from a data repository. In order\nto deliver response times that make it suitable for interactive use\ncases, the set of metadata properties that it exposes for sorting,\nfiltering, and aggregation is limited. Azul provides a uniform view\nof the metadata over a range of diverse schemas, effectively\nshielding clients from changes in the schemas as they occur over\ntime. It does so, however, at the expense of detail in the set of\nmetadata properties it exposes and in the accuracy with which it\naggregates them.\n\nAzul denormalizes and aggregates metadata into several different\nindices for selected entity types. Metadata entities can be queried\nusing the [Index](#operations-tag-Index) endpoints.\n\nA set of indices forms a catalog. There is a default catalog called\n`dcp2` which will be used unless a\ndifferent catalog name is specified using the `catalog` query\nparameter. Metadata from different catalogs is completely\nindependent: a response obtained by querying one catalog does not\nnecessarily correlate to a response obtained by querying another\none. Two catalogs can contain metadata from the same sources or\ndifferent sources. It is only guaranteed that the body of a\nresponse by any given endpoint adheres to one schema,\nindependently of which catalog was specified in the request.\n\nAzul provides the ability to download data and metadata via the\n[Manifests](#operations-tag-Manifests) endpoints. The\n`curl` format manifests can be used to\ndownload data files. Other formats provide various views of the\nmetadata. Manifests can be generated for a selection of files using\nfilters. These filters are interchangeable with the filters used by\nthe [Index](#operations-tag-Index) endpoints.\n\nAzul also provides a [summary](#operations-Index-get_index_summary)\nview of indexed data.\n\n## Data model\n\nAny index, when queried, returns a JSON array of hits. Each hit\nrepresents a metadata entity. Nested in each hit is a summary of the\nproperties of entities associated with the hit. An entity is\nassociated either by a direct edge in the original metadata graph,\nor indirectly as a series of edges. The nested properties are\ngrouped by the type of the associated entity. The properties of all\ndata files associated with a particular sample, for example, are\nlisted under `hits[*].files` in a `/index/samples` response. It is\nimportant to note that while each _hit_ represents a discrete\nentity, the properties nested within that hit are the result of an\naggregation over potentially many associated entities.\n\nTo illustrate this, consider a data file that is part of two\nprojects (a project is a group of related experiments, typically by\none laboratory, institution or consortium). Querying the `files`\nindex for this file yields a hit looking something like:\n\n```\n{\n \"projects\": [\n {\n \"projectTitle\": \"Project One\"\n \"laboratory\": ...,\n ...\n },\n {\n \"projectTitle\": \"Project Two\"\n \"laboratory\": ...,\n ...\n }\n ],\n \"files\": [\n {\n \"format\": \"pdf\",\n \"name\": \"Team description.pdf\",\n ...\n }\n ]\n}\n```\n\nThis example hit contains two kinds of nested entities (a hit in an\nactual Azul response will contain more): There are the two projects\nentities, and the file itself. These nested entities contain\nselected metadata properties extracted in a consistent way. This\nmakes filtering and sorting simple.\n\nAlso notice that there is only one file. When querying a particular\nindex, the corresponding entity will always be a singleton like\nthis.\n", - "version": "9.3" + "version": "10.0" }, "tags": [ { @@ -21,10 +21,6 @@ { "name": "Auxiliary", "description": "\nDescribes various aspects of the Azul service\n" - }, - { - "name": "Deprecated", - "description": "\nEndpoints that should not be used and that will be removed\n" } ], "paths": { @@ -765,20 +761,6 @@ } } }, - "/integrations": { - "get": { - "summary": "This endpoint will be removed in the future.", - "tags": [ - "Deprecated" - ], - "deprecated": true, - "responses": { - "504": { - "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" - } - } - } - }, "/index/catalogs": { "get": { "summary": "List all available catalogs.", diff --git a/src/azul/__init__.py b/src/azul/__init__.py index 973f6340b5..935580a9dd 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -1462,9 +1462,6 @@ def github_access_token(self) -> str: def gitlab_access_token(self) -> Optional[str]: return self.environ.get('azul_gitlab_access_token') - def portal_db_object_key(self, catalog_source: str) -> str: - return f'azul/{self.deployment_stage}/portals/{catalog_source}-db.json' - @property def lambda_layer_key(self) -> str: return 'lambda_layers' diff --git a/src/azul/plugins/__init__.py b/src/azul/plugins/__init__.py index 1f9c19da54..2d3fddd158 100644 --- a/src/azul/plugins/__init__.py +++ b/src/azul/plugins/__init__.py @@ -60,7 +60,6 @@ ) from azul.types import ( JSON, - JSONs, MutableJSON, get_generic_type_params, ) @@ -686,13 +685,6 @@ def fetch_bundle(self, bundle_fqid: BUNDLE_FQID) -> BUNDLE: """ raise NotImplementedError - @abstractmethod - def portal_db(self) -> JSONs: - """ - Returns integrations data object - """ - raise NotImplementedError - @abstractmethod def drs_client(self, authentication: Authentication | None = None diff --git a/src/azul/plugins/repository/canned/__init__.py b/src/azul/plugins/repository/canned/__init__.py index a554c6c3b0..cec0316cd6 100644 --- a/src/azul/plugins/repository/canned/__init__.py +++ b/src/azul/plugins/repository/canned/__init__.py @@ -20,7 +20,6 @@ import time from typing import ( AbstractSet, - Sequence, ) from furl import ( @@ -200,9 +199,6 @@ def fetch_bundle(self, bundle_fqid: CannedBundleFQID) -> CannedBundle: time.time() - now, bundle.uuid, bundle.version) return bundle - def portal_db(self) -> Sequence[JSON]: - return [] - def _construct_file_url(self, url: furl, file_name: str) -> furl: """ >>> plugin = Plugin(_sources=set()) diff --git a/src/azul/plugins/repository/dss/__init__.py b/src/azul/plugins/repository/dss/__init__.py index 4e5809f4a6..f6736794e1 100644 --- a/src/azul/plugins/repository/dss/__init__.py +++ b/src/azul/plugins/repository/dss/__init__.py @@ -3,12 +3,8 @@ from typing import ( AbstractSet, NoReturn, - Sequence, ) import urllib -from urllib.parse import ( - quote, -) from uuid import ( UUID, uuid5, @@ -186,212 +182,6 @@ def _prefix_clause(self, prefix): } ] if prefix else [] - def portal_db(self) -> Sequence[JSON]: - """ - A hardcoded example database for use during development of the - integrations API implementation - """ - return [ - { - "portal_id": "7bc432a6-0fcf-4c19-b6e6-4cb6231279b3", - "portal_name": "Terra Portal", - "portal_icon": "https://app.terra.bio/favicon.png", - "contact_email": "", - "organization_name": "Broad Institute", - "portal_description": "Terra is a cloud-native platform for biomedical researchers to access data, " - "run analysis tools, and collaborate.", - "integrations": [ - { - "integration_id": "b87b7f30-2e60-4ca5-9a6f-00ebfcd35f35", - "integration_type": "get_manifest", - "entity_type": "file", - "title": "Populate a Terra workspace with data files matching the current filter selection", - "manifest_type": "full", - "portal_url_template": "https://app.terra.bio/#import-data?url={manifest_url}" - } - ] - }, - { - "portal_id": "9852dece-443d-42e8-869c-17b9a86d447e", - "portal_name": "Single Cell Portal", - "portal_icon": "https://singlecell.broadinstitute.org/single_cell/assets/" - "scp_favicon-1e5be59fdd577f7e7e275109b800364728b01b4ffc54a41e9e32117f3d5d9aa6.ico", - "contact_email": "", - "organization_name": "Broad Institute", - "portal_description": "Reducing barriers and accelerating single-cell research.", - "integrations": [ - { - "integration_id": "977854a0-2eea-4fec-9459-d4807fe79f0c", - "integration_type": "get", - "entity_type": "project", - "title": "Visualize in SCP", - "entity_ids": { - "dev": ["bc2229e7-e330-435a-8c2a-4275741f2c2d"], - "staging": ["bc2229e7-e330-435a-8c2a-4275741f2c2d", "259f9041-b72f-45ce-894d-b645add2e620"], - "integration": ["bc2229e7-e330-435a-8c2a-4275741f2c2d"], - "prod": ["c4077b3c-5c98-4d26-a614-246d12c2e5d7"] - }, - "portal_url": "https://singlecell.broadinstitute.org/single_cell/study/SCP495" - }, - # https://docs.google.com/document/d/1HBOPe6h_RjxltfbPenKsNmoN3MVAtkhVKJ_LGG1DBkA/edit# - # https://github.com/HumanCellAtlas/data-browser/issues/545#issuecomment-528092658 - # { - # "integration_id": "f62f5202-55c3-4dfa-bedd-ba4d2c4fb6c9", - # "integration_type": "get_entity", - # "entity_type": "project", - # "title": "", - # "allow_head": False, - # "portal_url_template": "https://singlecell.broadinstitute.org/hca-project/{entity_id}" - # } - ] - }, - { - "portal_id": "f58bdc5e-98cd-4df4-80a4-7372dc035e87", - "portal_name": "Single Cell Expression Atlas", - "portal_icon": "https://ebi.emblstatic.net/web_guidelines/EBI-Framework/v1.3/images/logos/EMBL-EBI/" - "favicons/favicon.ico", - "contact_email": "Irene Papatheodorou irenep@ebi.ac.uk", - "organization_name": "European Bioinformatics Institute", - "portal_description": "Single Cell Expression Atlas annotates publicly available single cell " - "RNA-Seq experiments with ontology identifiers and re-analyses them using " - "standardised pipelines available through SCXA-Workflows, our collection of " - "RNA-Seq analysis pipelines, which is available at " - "https://github.com/ebi-gene-expression-group/scxa-workflows . The browser " - "enables visualisation of clusters of cells, their annotations and supports " - "searches for gene expression within and across studies.", - "integrations": [ - { - "integration_id": "dbfe9394-a326-4574-9632-fbadb51a7b1a", - "integration_type": "get", - "entity_type": "project", - "title": "Single-cell transcriptome analysis of precursors of human CD4+ " - "cytotoxic T lymphocytes", - "entity_ids": { - "staging": ["519b58ef-6462-4ed3-8c0d-375b54f53c31"], - "integration": ["90bd6933-40c0-48d4-8d76-778c103bf545"], - "prod": ["90bd6933-40c0-48d4-8d76-778c103bf545"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-GEOD-106540/results/tsne" - }, - { - "integration_id": "081a6a90-29b6-4100-9c42-17a50014ea03", - "integration_type": "get", - "entity_type": "project", - "title": "Reconstructing the human first trimester fetal-maternal interface using single cell " - "transcriptomics - 10x data", - "entity_ids": { - "staging": [], - "integration": ["f83165c5-e2ea-4d15-a5cf-33f3550bffde"], - "prod": ["f83165c5-e2ea-4d15-a5cf-33f3550bffde"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-MTAB-6701/results/tsne" - }, - { - "integration_id": "f0886c45-e339-4f22-8f6b-a715db1943e3", - "integration_type": "get", - "entity_type": "project", - "title": "Reconstructing the human first trimester fetal-maternal interface using single cell " - "transcriptomics - Smartseq 2 data", - "entity_ids": { - "staging": [], - "integration": ["f83165c5-e2ea-4d15-a5cf-33f3550bffde"], - "prod": ["f83165c5-e2ea-4d15-a5cf-33f3550bffde"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-MTAB-6678/results/tsne" - }, - { - "integration_id": "f13ddf2d-d913-492b-9ea8-2de4b1881c26", - "integration_type": "get", - "entity_type": "project", - "title": "Single cell transcriptome analysis of human pancreas", - "entity_ids": { - "staging": ["b1f3afcb-f061-4862-b6c2-ace971595d22", "08e7b6ba-5825-47e9-be2d-7978533c5f8c"], - "integration": ["cddab57b-6868-4be4-806f-395ed9dd635a"], - "prod": ["cddab57b-6868-4be4-806f-395ed9dd635a"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-GEOD-81547/results/tsne" - }, - { - "integration_id": "5ef44133-e71f-4f52-893b-3b200d5fb99b", - "integration_type": "get", - "entity_type": "project", - "title": "Single-cell RNA-seq analysis of 1,732 cells throughout a 125-day differentiation " - "protocol that converted H1 human embryonic stem cells to a variety of " - "ventrally-derived cell types", - "entity_ids": { - "staging": ["019a935b-ea35-4d83-be75-e1a688179328"], - "integration": ["2043c65a-1cf8-4828-a656-9e247d4e64f1"], - "prod": ["2043c65a-1cf8-4828-a656-9e247d4e64f1"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-GEOD-93593/results/tsne" - }, - { - "integration_id": "d43464c0-38c6-402d-bdec-8972d71005c5 ", - "integration_type": "get", - "entity_type": "project", - "title": "Single-cell RNA-seq analysis of human pancreas from healthy individuals and type 2 " - "diabetes patients", - "entity_ids": { - "staging": ["a5ae0428-476c-46d2-a9f2-aad955b149aa"], - "integration": ["ae71be1d-ddd8-4feb-9bed-24c3ddb6e1ad"], - "prod": ["ae71be1d-ddd8-4feb-9bed-24c3ddb6e1ad"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-MTAB-5061/results/tsne" - }, - { - "integration_id": "60912ae7-e88f-48bf-8b33-27daccade2b6", - "integration_type": "get", - "entity_type": "project", - "title": "Single-cell RNA-seq analysis of 20 organs and tissues from individual mice creates " - "a Tabula muris", - "entity_ids": { - "staging": ["2cd14cf5-f8e0-4c97-91a2-9e8957f41ea8"], - "integration": ["e0009214-c0a0-4a7b-96e2-d6a83e966ce0"], - "prod": ["e0009214-c0a0-4a7b-96e2-d6a83e966ce0"] - }, - "portal_url": "https://www.ebi.ac.uk/gxa/sc/experiments/E-ENAD-15/results/tsne" - }, - ], - }, - { - "portal_id": "2e05f611-16fb-4bf3-b860-aa500f0256de", - "portal_name": "Xena", - "portal_icon": "https://xena.ucsc.edu/icons-9ac0cb8372f662ad72d747b981120f73/favicon.ico", - "contact_email": "", - "organization_name": "UCSC", - "portal_description": "", - "integrations": [ - { - "integration_id": integration_id, - "integration_type": "get", - "entity_type": "project", - "title": title, - "entity_ids": { - "staging": [], - "integration": [project_uuid], - "prod": [project_uuid] - }, - "portal_url": "https://singlecell.xenabrowser.net/datapages/?cohort=" + quote(title) - } for integration_id, project_uuid, title in ( - # @formatter:off - ("73aa70fe-e40a-48da-9fa4-bea4c4d2ae74", "4a95101c-9ffc-4f30-a809-f04518a23803", "HCA Human Tissue T cell Activation"), # noqa E501 - ("c36e46c2-34d6-4129-853b-60256bc0af8d", "8185730f-4113-40d3-9cc3-929271784c2b", "HCA Adult Retina (Wong)"), # noqa E501 - ("dedb2f00-b92f-4f81-8633-6f58edcbf3f7", "005d611a-14d5-4fbf-846e-571a1f874f70", "HCA HPSI human cerebral organoids"), # noqa E501 - ("ced58994-05c2-4a2d-87b1-fff4faf2ca93", "cc95ff89-2e68-4a08-a234-480eca21ce79", "HCA Census of Immune Cells"), # noqa E501 - ("65e1465e-0641-4770-abbb-fde8bc0582aa", "4d6f6c96-2a83-43d8-8fe1-0f53bffd4674", "HCA Single Cell Liver Landscape"), # noqa E501 - ("39c69c7d-a245-4460-a045-6bd055564cca", "c4077b3c-5c98-4d26-a614-246d12c2e5d7", "HCA Tissue stability"), # noqa E501 - ("9cd98133-1a86-4937-94dc-4e9a68a36192", "091cf39b-01bc-42e5-9437-f419a66c8a45", "HCA Human Hematopoietic Profiling"), # noqa E501 - ("0f760ab0-2f81-40c8-a838-6a6c508bdd59", "f83165c5-e2ea-4d15-a5cf-33f3550bffde", "HCA Fetal Maternal Interface"), # noqa E501 - ("23ce54c0-58b5-4617-9b61-53ac020c1087", "cddab57b-6868-4be4-806f-395ed9dd635a", "HCA Human Pancreas"), # noqa E501 - ("980577ba-02d5-4edb-9662-24f2d1dc351a", "2043c65a-1cf8-4828-a656-9e247d4e64f1", "HCA Human Interneuron Development"), # noqa E501 - ("1dae24a2-998e-47d3-a570-26c36f2b073e", "abe1a013-af7a-45ed-8c26-f3793c24a1f4", "HCA Kidney Single Cell Atlas"), # noqa E501 - ("66cff7c4-8e79-4e84-a269-2c0970b49392", "f8aa201c-4ff1-45a4-890e-840d63459ca2", "HCA Human Colonic Mesenchyme IBD") # noqa E501 - # @formatter:on - ) - ] - } - ] - def _direct_file_url(self, file_uuid: str, *, diff --git a/src/azul/plugins/repository/tdr.py b/src/azul/plugins/repository/tdr.py index 0ca79cd786..0810918eda 100644 --- a/src/azul/plugins/repository/tdr.py +++ b/src/azul/plugins/repository/tdr.py @@ -8,7 +8,6 @@ from typing import ( AbstractSet, Callable, - Sequence, TypeVar, ) @@ -199,9 +198,6 @@ def fetch_bundle(self, bundle_fqid: TDRBundleFQID) -> TDR_BUNDLE: time.time() - now, bundle.uuid, bundle.version) return bundle - def portal_db(self) -> Sequence[JSON]: - return [] - @classmethod def format_version(cls, version: datetime.datetime) -> str: return format_dcp2_datetime(version) diff --git a/terraform/dynamo.tf.json.template.py b/terraform/dynamo.tf.json.template.py index 28ed796f06..6f6b1f7de3 100644 --- a/terraform/dynamo.tf.json.template.py +++ b/terraform/dynamo.tf.json.template.py @@ -7,29 +7,12 @@ from azul.terraform import ( emit_tf, ) -from azul.version_service import ( - VersionService, -) emit_tf( { "resource": [ { "aws_dynamodb_table": { - "object_versions": { - "name": config.dynamo_object_version_table_name, - "billing_mode": "PAY_PER_REQUEST", - "point_in_time_recovery": { - "enabled": True - }, - "hash_key": VersionService.key_name, - "attribute": [ - { - "name": VersionService.key_name, - "type": "S" - } - ] - }, "sources_cache_by_auth": { "name": config.dynamo_sources_cache_table_name, "billing_mode": "PAY_PER_REQUEST", diff --git a/test/integration_test.py b/test/integration_test.py index b3667471da..80978f707e 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -43,7 +43,6 @@ TypedDict, cast, ) -import unittest from unittest import ( mock, ) @@ -161,9 +160,6 @@ BundleType, TDRAnvilBundleFQID, ) -from azul.portal_service import ( - PortalService, -) from azul.service.async_manifest_service import ( Token, ) @@ -1706,94 +1702,6 @@ def test_azul_client_error_handling(self): notifications) -class PortalTestCase(IntegrationTestCase): - - @cached_property - def portal_service(self) -> PortalService: - return PortalService() - - -class PortalExpirationIntegrationTest(PortalTestCase): - - def test_expiration_tagging(self): - # This will upload the default DB if it is missing - self.portal_service.read() - s3_client = self.portal_service.client - response = s3_client.get_object_tagging(Bucket=self.portal_service.bucket, - Key=self.portal_service.object_key) - tags = [(tag['Key'], tag['Value']) for tag in response['TagSet']] - self.assertIn(self.portal_service._expiration_tag, tags) - - -# FIXME: Re-enable when SlowDown error can be avoided -# https://github.com/DataBiosphere/azul/issues/4285 -@unittest.skip('Test disabled. FIXME #4285') -@unittest.skipUnless(config.deployment.is_sandbox_or_personal, - 'Test would pollute portal DB') -class PortalRegistrationIntegrationTest(PortalTestCase, AlwaysTearDownTestCase): - - @property - def expected_db(self) -> JSONs: - return self.portal_service.default_db - - def setUp(self) -> None: - self.old_db = self.portal_service.read() - self.portal_service.overwrite(self.expected_db) - - def test_concurrent_portal_db_crud(self): - """ - Use multithreading to simulate multiple users simultaneously modifying - the portals database. - """ - - n_threads = 4 - n_tasks = n_threads * 5 - n_ops = 5 - - entry_format = 'task={};op={}' - - running = True - - def run(thread_count): - for op_count in range(n_ops): - if not running: - break - mock_entry = { - 'portal_id': 'foo', - 'integrations': [ - { - 'integration_id': 'bar', - 'entity_type': 'project', - 'integration_type': 'get', - 'entity_ids': ['baz'] - } - ], - 'mock-count': entry_format.format(thread_count, op_count) - } - self.portal_service._crud(lambda db: [*db, mock_entry]) - - with ThreadPoolExecutor(max_workers=n_threads) as executor: - futures = [executor.submit(run, i) for i in range(n_tasks)] - try: - self.assertTrue(all(f.result() is None for f in futures)) - finally: - running = False - - new_db = self.portal_service.read() - - old_entries = [portal for portal in new_db if 'mock-count' not in portal] - self.assertEqual(old_entries, self.expected_db) - mock_counts = [portal['mock-count'] for portal in new_db if 'mock-count' in portal] - self.assertEqual(len(mock_counts), len(set(mock_counts))) - self.assertEqual(set(mock_counts), { - entry_format.format(i, j) - for i in range(n_tasks) for j in range(n_ops) - }) - - def tearDown(self) -> None: - self.portal_service.overwrite(self.old_db) - - class OpenAPIIntegrationTest(AzulTestCase): def test_openapi(self): diff --git a/test/service/test_request_validation.py b/test/service/test_request_validation.py index a1d8f9ea2e..3c00ca330d 100644 --- a/test/service/test_request_validation.py +++ b/test/service/test_request_validation.py @@ -281,13 +281,9 @@ def test_bad_query_params(self): for entity_type in ('files', 'bundles', 'samples'): url = self.base_url.set(path=('index', entity_type)) with self.subTest(entity_type=entity_type): - with self.subTest(test='extra parameter'): - url.args = dict(catalog=self.catalog, - some_nonexistent_filter=1) - self.assertBadRequest(url, 'Unknown query parameter `some_nonexistent_filter`') - with self.subTest(test='missing required parameter'): - url = self.base_url.set(path='/integrations') - self.assertBadRequest(url, 'Missing required query parameters `entity_type`, `integration_type`') + url.args = dict(catalog=self.catalog, + some_nonexistent_filter=1) + self.assertBadRequest(url, 'Unknown query parameter `some_nonexistent_filter`') def test_bad_catalog_param(self): for path in (*('/index/' + e for e in ('summary', 'files')), diff --git a/test/service/test_response.py b/test/service/test_response.py index 2274c050cd..d3c6f46d02 100644 --- a/test/service/test_response.py +++ b/test/service/test_response.py @@ -3615,182 +3615,6 @@ def test_sorted_responses(self): self.assertEqual(200, response.status_code) -class TestPortalIntegrationResponse(DCP1CannedBundleTestCase, LocalAppTestCase): - - @classmethod - def lambda_name(cls) -> str: - return "service" - - maxDiff = None - - # Mocked DB content for the tests - _portal_integrations_db = [ - { - "portal_id": "9852dece-443d-42e8-869c-17b9a86d447e", - "integrations": [ - { - "integration_id": "b87b7f30-2e60-4ca5-9a6f-00ebfcd35f35", - "integration_type": "get_manifest", - "entity_type": "file", - "manifest_type": "full", - }, - { - "integration_id": "977854a0-2eea-4fec-9459-d4807fe79f0c", - "integration_type": "get", - "entity_type": "project", - "entity_ids": ["c4077b3c-5c98-4d26-a614-246d12c2e5d7"] - } - ] - }, - { - "portal_id": "f58bdc5e-98cd-4df4-80a4-7372dc035e87", - "integrations": [ - { - "integration_id": "e8b3ca4f-bcf5-42eb-b58c-de6d7e0fe138", - "integration_type": "get", - "entity_type": "project", - "entity_ids": ["c4077b3c-5c98-4d26-a614-246d12c2e5d7"] - }, - { - "integration_id": "dbfe9394-a326-4574-9632-fbadb51a7b1a", - "integration_type": "get", - "entity_type": "project", - "entity_ids": ["90bd6933-40c0-48d4-8d76-778c103bf545"] - }, - { - "integration_id": "f13ddf2d-d913-492b-9ea8-2de4b1881c26", - "integration_type": "get", - "entity_type": "project", - "entity_ids": ["cddab57b-6868-4be4-806f-395ed9dd635a"] - }, - { - "integration_id": "224b1d42-b939-4d10-8a8f-2b2ac304b813", - "integration_type": "get", - "entity_type": "project", - # NO entity_ids field - } - ] - } - ] - - def _mock_portal_crud(self, operation): - operation(self._portal_integrations_db) - - def _get_integrations(self, params: dict) -> dict: - url = self.base_url.set(path='/integrations', args=params) - response = requests.get(str(url)) - response.raise_for_status() - return response.json() - - @classmethod - def _extract_integration_ids(cls, response_json): - return [ - integration['integration_id'] - for portal in response_json - for integration in portal['integrations'] - ] - - @mock.patch('azul.portal_service.PortalService._crud') - def test_integrations(self, portal_crud): - """ - Verify requests specifying `integration_type` and `entity_type` only - return integrations matching those types - """ - test_cases = [ - ('get_manifest', 'file', ['b87b7f30-2e60-4ca5-9a6f-00ebfcd35f35']), - ('get', 'bundle', []), - ( - 'get', - 'project', - [ - '977854a0-2eea-4fec-9459-d4807fe79f0c', - 'e8b3ca4f-bcf5-42eb-b58c-de6d7e0fe138', - 'dbfe9394-a326-4574-9632-fbadb51a7b1a', - 'f13ddf2d-d913-492b-9ea8-2de4b1881c26', - '224b1d42-b939-4d10-8a8f-2b2ac304b813' - ] - ) - ] - portal_crud.side_effect = self._mock_portal_crud - with mock.patch.object(type(config), 'dss_deployment_stage', 'prod'): - for integration_type, entity_type, expected_integration_ids in test_cases: - params = dict(integration_type=integration_type, entity_type=entity_type) - with self.subTest(**params): - response_json = self._get_integrations(params) - found_integration_ids = self._extract_integration_ids(response_json) - self.assertEqual(len(expected_integration_ids), len(found_integration_ids)) - self.assertEqual(set(expected_integration_ids), set(found_integration_ids)) - self.assertTrue(all(isinstance(integration.get('entity_ids', []), list) - for portal in response_json - for integration in portal['integrations'])) - - @mock.patch('azul.portal_service.PortalService._crud') - def test_integrations_by_entity_ids(self, portal_crud): - """ - Verify requests specifying `entity_ids` only return integrations - matching those entity_ids - """ - - # 224b1d42-b939-4d10-8a8f-2b2ac304b813 must appear in every test since it lacks the entity_ids field - test_cases = [ - # One project entity id specified by one integration - ( - 'cddab57b-6868-4be4-806f-395ed9dd635a', - [ - 'f13ddf2d-d913-492b-9ea8-2de4b1881c26', - '224b1d42-b939-4d10-8a8f-2b2ac304b813' - ] - ), - # Two project entity ids specified by two different integrations - ( - 'cddab57b-6868-4be4-806f-395ed9dd635a, 90bd6933-40c0-48d4-8d76-778c103bf545', - [ - 'f13ddf2d-d913-492b-9ea8-2de4b1881c26', - 'dbfe9394-a326-4574-9632-fbadb51a7b1a', - '224b1d42-b939-4d10-8a8f-2b2ac304b813' - ] - ), - # One project entity id specified by two different integrations - ( - 'c4077b3c-5c98-4d26-a614-246d12c2e5d7', - [ - '977854a0-2eea-4fec-9459-d4807fe79f0c', - 'e8b3ca4f-bcf5-42eb-b58c-de6d7e0fe138', - '224b1d42-b939-4d10-8a8f-2b2ac304b813' - ] - ), - # Blank entity id, to match integrations lacking the entity_id field - ( - '', - [ - '224b1d42-b939-4d10-8a8f-2b2ac304b813' - ] - ), - # No entity id, accepting all integrations - ( - None, - [ - 'f13ddf2d-d913-492b-9ea8-2de4b1881c26', - 'dbfe9394-a326-4574-9632-fbadb51a7b1a', - '977854a0-2eea-4fec-9459-d4807fe79f0c', - 'e8b3ca4f-bcf5-42eb-b58c-de6d7e0fe138', - '224b1d42-b939-4d10-8a8f-2b2ac304b813' - ] - ) - ] - - portal_crud.side_effect = self._mock_portal_crud - with mock.patch.object(type(config), 'dss_deployment_stage', 'prod'): - for entity_ids, integration_ids in test_cases: - params = dict(integration_type='get', entity_type='project') - if entity_ids is not None: - params['entity_ids'] = entity_ids - with self.subTest(**params): - response_json = self._get_integrations(params) - found_integration_ids = self._extract_integration_ids(response_json) - self.assertEqual(set(integration_ids), set(found_integration_ids)) - - class TestListCatalogsResponse(DCP1CannedBundleTestCase, LocalAppTestCase): maxDiff = None