remove database backend #489

.flake8

@@ -10,7 +10,7 @@ per-file-ignores =
   test/*: S101
   util/icat_db_generator.py: S311
   datagateway_api/wsgi.py:E402,F401
-  datagateway_api/src/datagateway_api/database/models.py: N815,A003
+  datagateway_api/src/datagateway_api/icat/models.py: N815,A003
   datagateway_api/src/datagateway_api/icat/filters.py: C901
   datagateway_api/src/search_api/models.py: B950
 enable-extensions=G

.github/workflows/ci-build.yml

@@ -382,13 +382,67 @@ jobs:
       # Checkout DataGateway API and setup Python
       - name: Check out repo
         uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v3.5.3
+      # Setup Java & Python
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Setup Java
+        uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0
+        with:
+          distribution: 'temurin'
+          java-version: 8
+
+       # ICAT Ansible clone and install dependencies
+      - name: Checkout icat-ansible
+        if: success()
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          repository: icatproject-contrib/icat-ansible
+          ref: master
+          path: icat-ansible
+      - name: Install Ansible
+        run: pip install -r icat-ansible/requirements.txt
+     # Prep for running the playbook
+      - name: Create hosts file
+        run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
+      - name: Prepare vault pass
+        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
+      - name: Move vault to directory it'll get detected by Ansible
+        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
+      - name: Replace default payara user with Actions user
+        run: |
+          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
+
+      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
+      - name: Change hostname to localhost
+        run: sudo hostname -b localhost
+
+      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
+      - name: Remove existing mysql
+        run: |
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          sudo apt-get remove --purge "mysql*"
+          sudo rm -rf /var/lib/mysql* /etc/mysql
+
+      # Create local instance of ICAT
+      - name: Run ICAT Ansible Playbook
+        run: |
+          ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
+
+      # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests
+      - name: Add anon user to rootUserNames
+        run: |
+          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
+      - name: Apply rootUserNames change
+        run: |
+          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
+      - name: Reinstall ICAT Server
+        run: |
+          cd /home/runner/install/icat.server/ && ./setup -vv install
 
-      # Create virtual environment and install DataGateway API
+        # Create virtual environment and install DataGateway API
       - name: Create and activate virtual environment
         run: |
           python3 -m venv dg-api-venv
@@ -407,9 +461,6 @@ jobs:
           '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example
       - name: Create config.yaml
         run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml
-      # These sections are removed so the API doesn't try to (and fail) to connect to an ICAT stack on startup
-      - name: Remove DataGateway API and Search API sections from config
-        run: yq -i 'del(.datagateway_api, .search_api)' datagateway_api/config.yaml
       - name: Create search_api_mapping.json
         run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
 

datagateway_api/config.yaml.example

@@ -1,11 +1,9 @@
 ---
 datagateway_api:
   extension: "/"
-  backend: "python_icat"
   client_cache_size: 5
   client_pool_init_size: 2
   client_pool_max_size: 5
-  db_url: "mysql+pymysql://icatdbuser:icatdbuserpw@localhost:3306/icatdb"
   icat_url: "https://localhost:8181"
   icat_check_cert: false
   use_reader_for_performance:

datagateway_api/src/api_start_utils.py

@@ -9,14 +9,8 @@
 from flask_swagger_ui import get_swaggerui_blueprint
 
 from datagateway_api.src.common.config import Config
-
-# Only attempt to create a DataGateway API backend if the datagateway_api object
-# is present in the config. This ensures that the API does not error on startup
-# due to an AttributeError exception being thrown if the object is missing.
-if Config.config.datagateway_api:
-    from datagateway_api.src.datagateway_api.backends import create_backend
-from datagateway_api.src.datagateway_api.database.helpers import db  # noqa: I202
 from datagateway_api.src.datagateway_api.icat.icat_client_pool import create_client_pool
+from datagateway_api.src.datagateway_api.icat.python_icat import PythonICAT
 from datagateway_api.src.resources.entities.entity_endpoint import (
     get_count_endpoint,
     get_endpoint,
@@ -112,21 +106,6 @@ def create_app_infrastructure(flask_app):
     CORS(flask_app)
     flask_app.url_map.strict_slashes = False
     api = CustomErrorHandledApi(flask_app)
-
-    if Config.config.datagateway_api is not None:
-        try:
-            backend_type = flask_app.config["TEST_BACKEND"]
-            Config.config.datagateway_api.set_backend_type(backend_type)
-        except KeyError:
-            backend_type = Config.config.datagateway_api.backend
-
-        if backend_type == "db":
-            flask_app.config[
-                "SQLALCHEMY_DATABASE_URI"
-            ] = Config.config.datagateway_api.db_url
-            flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-            db.init_app(flask_app)
-
     specs = []
     if Config.config.datagateway_api is not None:
         configure_datagateway_api_swaggerui_blueprint(flask_app)
@@ -148,25 +127,18 @@ def create_api_endpoints(flask_app, api, specs):
         datagateway_api_spec = next(
             (spec for spec in specs if spec.title == "DataGateway API"), None,
         )
-        try:
-            backend_type = flask_app.config["TEST_BACKEND"]
-            Config.config.datagateway_api.set_backend_type(backend_type)
-        except KeyError:
-            backend_type = Config.config.datagateway_api.backend
 
-        backend = create_backend(backend_type)
+        python_icat = PythonICAT()
 
-        icat_client_pool = None
-        if backend_type == "python_icat":
-            # Create client pool
-            icat_client_pool = create_client_pool()
+        # Create client pool
+        icat_client_pool = create_client_pool()
 
         datagateway_api_extension = Config.config.datagateway_api.extension
         for entity_name in endpoints:
             get_endpoint_resource = get_endpoint(
                 entity_name,
                 endpoints[entity_name],
-                backend,
+                python_icat,
                 client_pool=icat_client_pool,
             )
             api.add_resource(
@@ -179,7 +151,7 @@ def create_api_endpoints(flask_app, api, specs):
             get_id_endpoint_resource = get_id_endpoint(
                 entity_name,
                 endpoints[entity_name],
-                backend,
+                python_icat,
                 client_pool=icat_client_pool,
             )
             api.add_resource(
@@ -192,7 +164,7 @@ def create_api_endpoints(flask_app, api, specs):
             get_count_endpoint_resource = get_count_endpoint(
                 entity_name,
                 endpoints[entity_name],
-                backend,
+                python_icat,
                 client_pool=icat_client_pool,
             )
             api.add_resource(
@@ -205,7 +177,7 @@ def create_api_endpoints(flask_app, api, specs):
             get_find_one_endpoint_resource = get_find_one_endpoint(
                 entity_name,
                 endpoints[entity_name],
-                backend,
+                python_icat,
                 client_pool=icat_client_pool,
             )
             api.add_resource(
@@ -217,7 +189,7 @@ def create_api_endpoints(flask_app, api, specs):
 
         # Session endpoint
         session_endpoint_resource = session_endpoints(
-            backend, client_pool=icat_client_pool,
+            python_icat, client_pool=icat_client_pool,
         )
         api.add_resource(
             session_endpoint_resource,
@@ -227,7 +199,7 @@ def create_api_endpoints(flask_app, api, specs):
         datagateway_api_spec.path(resource=session_endpoint_resource, api=api)
 
         # Ping endpoint
-        ping_resource = ping_endpoint(backend, client_pool=icat_client_pool)
+        ping_resource = ping_endpoint(python_icat, client_pool=icat_client_pool)
         api.add_resource(ping_resource, f"{datagateway_api_extension}/ping")
         datagateway_api_spec.path(resource=ping_resource, api=api)
 

datagateway_api/src/common/config.py

@@ -48,16 +48,12 @@ class UseReaderForPerformance(BaseModel):
 class DataGatewayAPI(BaseModel):
     """
     Configuration model class that implements pydantic's BaseModel class to allow for
-    validation of the DataGatewayAPI config data using Python type annotations. It takes
-    the backend into account, meaning only the config options for the backend used are
-    required.
+    validation of the DataGatewayAPI config data using Python type annotations.
     """
 
-    backend: StrictStr
     client_cache_size: Optional[StrictInt]
     client_pool_init_size: Optional[StrictInt]
     client_pool_max_size: Optional[StrictInt]
-    db_url: Optional[StrictStr]
     extension: StrictStr
     icat_check_cert: Optional[StrictBool]
     icat_url: Optional[StrictStr]
@@ -68,23 +64,6 @@ class DataGatewayAPI(BaseModel):
     def __getitem__(self, item):
         return getattr(self, item)
 
-    @validator("db_url", always=True)
-    def require_db_config_value(cls, value, values):  # noqa: B902, N805
-        """
-        By default the `db_url` config field is optional so that it does not have to be
-        present in the config file if `backend` is set to `python_icat`. However, if the
-        `backend` is set to `db`, this validator esentially makes the `db_url` config
-        field mandatory. This means that an error is raised, at which point the
-        application exits, if a `db_url` config value is not present in the config file.
-
-        :param cls: :class:`DataGatewayAPI` pointer
-        :param value: The value of the given config field
-        :param values: The config field values loaded before the given config field
-        """
-        if "backend" in values and values["backend"] == "db" and value is None:
-            raise TypeError("field required")
-        return value
-
     @validator(
         "client_cache_size",
         "client_pool_init_size",
@@ -93,44 +72,25 @@ def require_db_config_value(cls, value, values):  # noqa: B902, N805
         "icat_url",
         always=True,
     )
-    def require_icat_config_value(cls, value, values):  # noqa: B902, N805
+    def require_icat_config_value(cls, value):  # noqa: B902, N805
         """
-        By default the above config fields that are passed to the `@validator` decorator
-        are optional so that they do not have to be present in the config file if
-        `backend` is set to `db`. However, if the `backend` is set to `python_icat`,
-        this validator esentially makes these config fields mandatory. This means that
-        an error is raised, at which point the application exits, if any of these config
-        values are not present in the config file.
+        Validates that the required config fields for the `python_icat`
+        are present and not None. If any of these config values are missing,
+        an error is raised, causing the application to exit.
 
         :param cls: :class:`DataGatewayAPI` pointer
         :param value: The value of the given config field
-        :param values: The config field values loaded before the given config field
         """
-
-        if "backend" in values and values["backend"] == "python_icat" and value is None:
-            raise TypeError("field required")
+        if value is None:
+            raise TypeError("Field required for `python_icat`.")
         return value
 
-    def set_backend_type(self, backend_type):
-        """
-        This setter is used as a way for automated tests to set the backend type. The
-        API can detect if the Flask app setup is from an automated test by checking the
-        app's config for a `TEST_BACKEND`. If this value exists (a KeyError will be
-        raised when the API is run normally, which will then grab the backend type from
-        `config.yaml`), it needs to be set using this function. This is required because
-        creating filters in the `QueryFilterFactory` is backend-specific so the backend
-        type must be fetched. This must be done using this module (rather than directly
-        importing and checking the Flask app's config) to avoid circular import issues.
-        """
-        self.backend = backend_type
-
     class Config:
         """
         The behaviour of the BaseModel class can be controlled via this class.
         """
 
         # Enables assignment validation on the BaseModel fields. Useful for when the
-        # backend type is changed using the set_backend_type function.
         validate_assignment = True
 
 

datagateway_api/src/common/date_handler.py

@@ -32,8 +32,7 @@ def is_str_a_date(potential_date):
     @staticmethod
     def str_to_datetime_object(data):
         """
-        Convert a string to a `datetime.datetime` object. This is commonly used when
-        storing user input in ICAT (using the Python ICAT backend).
+        Convert a string to a `datetime.datetime` object.
 
         Python 3.7+ has support for `datetime.fromisoformat()` which would be a more
         elegant solution to this conversion operation since dates are converted into ISO

datagateway_api/src/common/filter_order_handler.py

@@ -176,7 +176,7 @@ def clear_python_icat_order_filters(self):
     def manage_icat_filters(self, filters, query):
         """
         Utility function to call other functions in this class, used to manage filters
-        when using the Python ICAT backend. These steps are the same with the different
+        when using the Python ICAT. These steps are the same with the different
         types of requests that utilise filters, therefore this function helps to reduce
         code duplication
 

datagateway_api/src/common/filters.py

@@ -21,8 +21,8 @@ class WhereFilter(QueryFilter):
     precedence = 1
 
     def __init__(self, field, value, operation):
-        # The field is set to None as a precaution but this should be set by the
-        # individual backend since backends deal with this data differently
+        # The field is set to None as a precaution but this should be set
+        # when initialising Python ICAT
         self.field = None
         self.value = value
         self.operation = operation

datagateway_api/src/common/helpers.py

@@ -17,7 +17,7 @@
     FilterError,
     MissingCredentialsError,
 )
-from datagateway_api.src.datagateway_api.database import models
+from datagateway_api.src.datagateway_api.icat import models
 from datagateway_api.src.resources.entities.entity_endpoint_dict import endpoints
 
 log = logging.getLogger()
@@ -137,7 +137,7 @@ def get_entity_object_from_name(entity_name):
     :param entity_name: Name of the entity to fetch a version from this model
     :type entity_name: :class:`str`
     :return: Object of the entity requested (e.g.
-        :class:`.datagateway_api.database.models.INVESTIGATIONINSTRUMENT`)
+        :class:`.datagateway_api.icat.models.INVESTIGATIONINSTRUMENT`)
     :raises: KeyError: If an entity model cannot be found as a class in this model
     """
     try: