From 02a8a95dde8885c5abd9a1a42a55fcff8635556f Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Feb 2023 14:59:17 +0100 Subject: [PATCH 1/7] Bump versions of pre-commit Was getting ``` RuntimeError: The Poetry configuration is invalid: - [extras.pipfile_deprecated_finder.2] 'pip-shims<=0.3.4' does not match '^[a-zA-Z-_.0-9]+$' ``` --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea0c78b8..2b4641ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/adrienverge/yamllint - rev: "v1.26.0" + rev: "v1.29.0" hooks: - id: yamllint - repo: https://github.com/asottile/setup-cfg-fmt @@ -23,7 +23,7 @@ repos: hooks: - id: black-jupyter - repo: https://github.com/PyCQA/isort - rev: "5.9.3" + rev: 5.12.0 hooks: - id: isort # TODO renable when errors are fixed/ignored @@ -78,7 +78,7 @@ repos: rev: 1.1.0 hooks: - id: nbqa-isort - additional_dependencies: [isort==5.9.3] + additional_dependencies: [isort==5.11.2] - id: nbqa-mypy additional_dependencies: [mypy==0.910, types-python-dateutil] # TODO renable when errors are fixed/ignored From adc25cf9749862de22a87bcaf4a86bd372cacb69 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 7 Mar 2023 09:11:40 +0100 Subject: [PATCH 2/7] Use latest grpc4bmi --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b8e8e3e7..2dfcfbb0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = basic_modeling_interface cftime esmvaltool>=2.4.0 - grpc4bmi>=0.2.12,<0.3 + grpc4bmi>=0.3 grpcio hydrostats matplotlib>=3.5.0 From 13eff27fe65572dd05cab9b466edfb7f4c5d19f9 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 7 Mar 2023 10:24:01 +0100 Subject: [PATCH 3/7] Repalce basic_modeling_interface with bmipy --- docs/conf.py | 2 +- setup.cfg | 4 ++-- src/ewatercycle/models/abstract.py | 10 ++++++++-- src/ewatercycle/models/hype.py | 3 --- tests/models/test_abstract.py | 4 ++-- tests/models/test_hype.py | 2 +- tests/models/test_lisflood.py | 2 +- tests/models/test_pcrglobwb.py | 2 +- tests/models/test_wflow.py | 2 +- 9 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 247fa281..0cc75beb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -191,7 +191,7 @@ def setup(app): # noqa: D103 ] autodoc_mock_imports = [ - "basic_modeling_interface", + "bmipy", "cftime", "dask", "esmvalcore", diff --git a/setup.cfg b/setup.cfg index 2dfcfbb0..458afab6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,10 +37,10 @@ packages = find: install_requires = Fiona Shapely - basic_modeling_interface + bmipy cftime esmvaltool>=2.4.0 - grpc4bmi>=0.3 + grpc4bmi@git+https://github.com/eWaterCycle/grpc4bmi@bmi2 grpcio hydrostats matplotlib>=3.5.0 diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 74904dc2..08815309 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -1,14 +1,20 @@ import logging +from pathlib import Path import textwrap from abc import ABCMeta, abstractmethod from datetime import datetime -from typing import Any, ClassVar, Generic, Iterable, Optional, Set, Tuple, TypeVar +from typing import Any, ClassVar, Generic, Iterable, Optional, Set, Tuple, TypeVar, Union import numpy as np import xarray as xr -from basic_modeling_interface import Bmi +from bmipy import Bmi from cftime import num2date +from grpc4bmi.bmi_client_docker import BmiClientDocker +from grpc4bmi.bmi_client_singularity import BmiClientSingularity +from grpc4bmi.bmi_memoized import MemoizedBmi +from grpc4bmi.bmi_optionaldest import OptionalDestBmi +from ewatercycle import CFG from ewatercycle.forcing import DefaultForcing from ewatercycle.parameter_sets import ParameterSet diff --git a/src/ewatercycle/models/hype.py b/src/ewatercycle/models/hype.py index e7ac7ca6..d9923d2e 100644 --- a/src/ewatercycle/models/hype.py +++ b/src/ewatercycle/models/hype.py @@ -4,10 +4,7 @@ import types from typing import Any, Iterable, Optional, Tuple -import numpy as np import xarray as xr -from basic_modeling_interface import Bmi -from cftime import num2date from dateutil.parser import parse from dateutil.tz import UTC from grpc4bmi.bmi_client_docker import BmiClientDocker diff --git a/tests/models/test_abstract.py b/tests/models/test_abstract.py index 730c1041..1a074e17 100644 --- a/tests/models/test_abstract.py +++ b/tests/models/test_abstract.py @@ -7,7 +7,7 @@ import numpy as np import pytest import xarray as xr -from basic_modeling_interface import Bmi +from bmipy import Bmi from numpy.testing import assert_array_equal from ewatercycle import CFG @@ -66,7 +66,7 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: @pytest.fixture -@patch("basic_modeling_interface.Bmi") +@patch("bmipy.Bmi") def bmi(MockedBmi): mocked_bmi = MockedBmi() mocked_bmi.get_start_time.return_value = 42.0 diff --git a/tests/models/test_hype.py b/tests/models/test_hype.py index 2fba2442..653e1c1d 100644 --- a/tests/models/test_hype.py +++ b/tests/models/test_hype.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi +from bmipy import Bmi from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG diff --git a/tests/models/test_lisflood.py b/tests/models/test_lisflood.py index cf7f7046..11e27d0a 100644 --- a/tests/models/test_lisflood.py +++ b/tests/models/test_lisflood.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi +from bmipy import Bmi from grpc4bmi.bmi_client_singularity import BmiClientSingularity from numpy.testing import assert_array_equal diff --git a/tests/models/test_pcrglobwb.py b/tests/models/test_pcrglobwb.py index 09383dc1..54a61f7b 100644 --- a/tests/models/test_pcrglobwb.py +++ b/tests/models/test_pcrglobwb.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi +from bmipy import Bmi from grpc import FutureTimeoutError from grpc4bmi.bmi_client_singularity import BmiClientSingularity diff --git a/tests/models/test_wflow.py b/tests/models/test_wflow.py index af4c35ee..88944589 100644 --- a/tests/models/test_wflow.py +++ b/tests/models/test_wflow.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi +from bmipy import Bmi from grpc import FutureTimeoutError from grpc4bmi.bmi_client_singularity import BmiClientSingularity From 1681801c24b0240cd414d4e9245bd2d0aebdfda4 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 7 Mar 2023 10:24:39 +0100 Subject: [PATCH 4/7] Start to centralize way to start container --- src/ewatercycle/models/abstract.py | 42 ++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 08815309..130b0176 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -1,9 +1,19 @@ import logging -from pathlib import Path import textwrap from abc import ABCMeta, abstractmethod from datetime import datetime -from typing import Any, ClassVar, Generic, Iterable, Optional, Set, Tuple, TypeVar, Union +from pathlib import Path +from typing import ( + Any, + ClassVar, + Generic, + Iterable, + Optional, + Set, + Tuple, + TypeVar, + Union, +) import numpy as np import xarray as xr @@ -287,3 +297,31 @@ def _check_version(self): f"Supplied version {self.version} is not supported by this model. " f"Available versions are {self.available_versions}." ) + + # TODO centralize way to start container + def _start_container( + self, + image: str, + work_dir: Union[str, Path], + image_port=55555, + timeout=300, + delay=0, + ) -> None: + if CFG["container_engine"] == "docker": + bmi = BmiClientDocker( + image=image, + image_port=image_port, + work_dir=str(work_dir), + timeout=timeout, + delay=delay, + ) + elif CFG["container_engine"] == "singularity": + bmi = BmiClientSingularity( + image=image, + work_dir=str(work_dir), + timeout=timeout, + delay=delay, + ) + else: + raise ValueError(f"Unknown container technology: {CFG['container_engine']}") + self.bmi = OptionalDestBmi(MemoizedBmi(bmi)) From 367e180a48855f180434b232e6b2e1997f138374 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 7 Mar 2023 14:12:18 +0100 Subject: [PATCH 5/7] Move starting of container to own module + each model declares versions/images in same way --- src/ewatercycle/config/_lisflood_versions.py | 17 +--- src/ewatercycle/container.py | 98 ++++++++++++++++++++ src/ewatercycle/forcing/_lisvap.py | 9 +- src/ewatercycle/models/abstract.py | 49 +--------- src/ewatercycle/models/hype.py | 38 ++------ src/ewatercycle/models/lisflood.py | 37 ++------ src/ewatercycle/models/marrmot.py | 95 +++++-------------- src/ewatercycle/models/pcrglobwb.py | 76 ++++----------- src/ewatercycle/models/wflow.py | 79 ++++++---------- 9 files changed, 199 insertions(+), 299 deletions(-) create mode 100644 src/ewatercycle/container.py diff --git a/src/ewatercycle/config/_lisflood_versions.py b/src/ewatercycle/config/_lisflood_versions.py index f050f6b6..f80bf779 100644 --- a/src/ewatercycle/config/_lisflood_versions.py +++ b/src/ewatercycle/config/_lisflood_versions.py @@ -1,19 +1,10 @@ -""" -Versions of Lisflood container images -""" -from pathlib import Path +"""Versions of Lisflood container images.""" -version_images = { +from ewatercycle.container import VersionImages + +version_images: VersionImages = { "20.10": { "docker": "ewatercycle/lisflood-grpc4bmi:20.10", "singularity": "ewatercycle-lisflood-grpc4bmi_20.10.sif", } } - - -def get_docker_image(version): - return version_images[version]["docker"] - - -def get_singularity_image(version, singularity_dir: Path): - return singularity_dir / version_images[version]["singularity"] diff --git a/src/ewatercycle/container.py b/src/ewatercycle/container.py new file mode 100644 index 00000000..e4b8c3ff --- /dev/null +++ b/src/ewatercycle/container.py @@ -0,0 +1,98 @@ +"""Container utilities.""" +from pathlib import Path +from typing import Iterable, Literal, Mapping, Optional, TypedDict, Union + +from bmipy import Bmi +from grpc import FutureTimeoutError +from grpc4bmi.bmi_client_docker import BmiClientDocker +from grpc4bmi.bmi_client_singularity import BmiClientSingularity +from grpc4bmi.bmi_memoized import MemoizedBmi +from grpc4bmi.bmi_optionaldest import OptionalDestBmi + +from ewatercycle import CFG + + +class VersionImage(TypedDict): + """Image container for each container technology.""" + + docker: str + """Docker image name.""" + singularity: str + """Singularity image name. Should be *.sif file in CFG["singularity_dir"].""" + + +VersionImages = Mapping[str, VersionImage] +"""Image containers for each version.""" + + +def start_container( + work_dir: Union[str, Path], + version_image: VersionImage, + input_dirs: Optional[Iterable[str]] = None, + image_port=55555, + timeout=None, + delay=0, +) -> Bmi: + """Start container with model inside. + + The `ewatercycle.CFG['container_engine']` value determines + the engine used to start a container. + + Args: + work_dir: Work directory + version_image: Image name for each container engine. + input_dirs: Additional directories to mount inside container. + image_port: Docker port inside container where grpc4bmi server is running. + timeout: Number of seconds to wait for grpc connection. + delay: Number of seconds to wait before connecting. + + Raises: + ValueError: When unknown container technology is requested. + TimeoutError: When model inside container did not start quickly enough. + + Returns: + _description_ + """ + engine: Literal["docker", "singularity"] = CFG["container_engine"] + image = version_image[engine] + if input_dirs is None: + input_dirs = [] + if engine == "docker": + try: + bmi = BmiClientDocker( + image=image, + image_port=image_port, + work_dir=str(work_dir), + input_dirs=input_dirs, + timeout=timeout, + delay=delay, + ) + except FutureTimeoutError as exc: + # https://github.com/eWaterCycle/grpc4bmi/issues/95 + # https://github.com/eWaterCycle/grpc4bmi/issues/100 + raise TimeoutError( + "Couldn't spawn container within allocated time limit " + "({timeout} seconds). You may try pulling the docker image with" + f" `docker pull {image}` and then try again." + ) from exc + elif engine == "singularity": + image = CFG["singularity_dir"] / image + try: + bmi = BmiClientSingularity( + image=image, + work_dir=str(work_dir), + input_dirs=input_dirs, + timeout=timeout, + delay=delay, + ) + except FutureTimeoutError as exc: + docker_image = version_image["docker"] + raise TimeoutError( + "Couldn't spawn container within allocated time limit " + "({timeout} seconds). You may try pulling the docker image with" + f" `singularity build {image} " + f"docker://{docker_image}` and then try again." + ) from exc + else: + raise ValueError(f"Unknown container technology: {CFG['container_engine']}") + return OptionalDestBmi(MemoizedBmi(bmi)) diff --git a/src/ewatercycle/forcing/_lisvap.py b/src/ewatercycle/forcing/_lisvap.py index a761a4a2..5865caa4 100644 --- a/src/ewatercycle/forcing/_lisvap.py +++ b/src/ewatercycle/forcing/_lisvap.py @@ -23,12 +23,12 @@ import os import subprocess -from typing import Dict, Tuple +from typing import Dict, Literal, Tuple from ewatercycle import CFG from ewatercycle.parametersetdb.config import XmlConfig -from ..config._lisflood_versions import get_docker_image, get_singularity_image +from ..config._lisflood_versions import version_images from ..util import get_time @@ -49,9 +49,9 @@ def lisvap( mask_map, forcing_dir, ) - + engine: Literal["docker", "singularity"] = CFG["container_engine"] + image = version_images[version][engine] if CFG["container_engine"].lower() == "singularity": - image = get_singularity_image(version, CFG["singularity_dir"]) args = [ "singularity", "exec", @@ -62,7 +62,6 @@ def lisvap( image, ] elif CFG["container_engine"].lower() == "docker": - image = get_docker_image(version) args = [ "docker", "run", diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 130b0176..81c0221d 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -1,30 +1,15 @@ +"""Abstract class of a eWaterCycle model.""" import logging import textwrap from abc import ABCMeta, abstractmethod from datetime import datetime -from pathlib import Path -from typing import ( - Any, - ClassVar, - Generic, - Iterable, - Optional, - Set, - Tuple, - TypeVar, - Union, -) +from typing import Any, ClassVar, Generic, Iterable, Optional, Tuple, TypeVar import numpy as np import xarray as xr from bmipy import Bmi from cftime import num2date -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity -from grpc4bmi.bmi_memoized import MemoizedBmi -from grpc4bmi.bmi_optionaldest import OptionalDestBmi -from ewatercycle import CFG from ewatercycle.forcing import DefaultForcing from ewatercycle.parameter_sets import ParameterSet @@ -185,7 +170,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: @property @abstractmethod def parameters(self) -> Iterable[Tuple[str, Any]]: - """Default values for the setup() inputs""" + """Default values for the setup() inputs.""" @property def start_time(self) -> float: @@ -297,31 +282,3 @@ def _check_version(self): f"Supplied version {self.version} is not supported by this model. " f"Available versions are {self.available_versions}." ) - - # TODO centralize way to start container - def _start_container( - self, - image: str, - work_dir: Union[str, Path], - image_port=55555, - timeout=300, - delay=0, - ) -> None: - if CFG["container_engine"] == "docker": - bmi = BmiClientDocker( - image=image, - image_port=image_port, - work_dir=str(work_dir), - timeout=timeout, - delay=delay, - ) - elif CFG["container_engine"] == "singularity": - bmi = BmiClientSingularity( - image=image, - work_dir=str(work_dir), - timeout=timeout, - delay=delay, - ) - else: - raise ValueError(f"Unknown container technology: {CFG['container_engine']}") - self.bmi = OptionalDestBmi(MemoizedBmi(bmi)) diff --git a/src/ewatercycle/models/hype.py b/src/ewatercycle/models/hype.py index d9923d2e..303473f2 100644 --- a/src/ewatercycle/models/hype.py +++ b/src/ewatercycle/models/hype.py @@ -1,3 +1,4 @@ +"""eWaterCycle wrapper around Hype BMI.""" import datetime import logging import shutil @@ -7,10 +8,9 @@ import xarray as xr from dateutil.parser import parse from dateutil.tz import UTC -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._hype import HypeForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -_version_images = { +_version_images: VersionImages = { "feb2021": { "docker": "ewatercycle/hype-grpc4bmi:feb2021", "singularity": "ewatercycle-hype-grpc4bmi_feb2021.sif", @@ -41,6 +41,7 @@ class Hype(AbstractModel[HypeForcing]): """ available_versions = tuple(_version_images.keys()) + """Show supported Hype versions in eWaterCycle""" def __init__( self, @@ -49,7 +50,6 @@ def __init__( forcing: Optional[HypeForcing] = None, ): super().__init__(version, parameter_set, forcing) - assert version in _version_images self._setup_default_config() def _setup_default_config(self): @@ -152,8 +152,10 @@ def setup( # type: ignore cfg_file.write_text(self._cfg, encoding="cp437") # start container - work_dir = str(cfg_dir_as_path) - self.bmi = _start_container(self.version, work_dir) + self.bmi = start_container( + version_image=_version_images[self.version], + work_dir=cfg_dir_as_path, + ) since = self._start.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -164,7 +166,7 @@ def get_time_units(_self): self.bmi.get_time_units = types.MethodType(get_time_units, self.bmi) - return str(cfg_file), work_dir + return str(cfg_file), str(cfg_dir_as_path) @property def parameters(self) -> Iterable[Tuple[str, Any]]: @@ -217,26 +219,6 @@ def _setup_cfg_dir(cfg_dir: Optional[str] = None): return work_dir -def _start_container(version: str, work_dir: str): - if CFG["container_engine"].lower() == "singularity": - image = CFG["singularity_dir"] / _version_images[version]["singularity"] - return BmiClientSingularity( - image=str(image), - work_dir=work_dir, - ) - elif CFG["container_engine"].lower() == "docker": - image = _version_images[version]["docker"] - return BmiClientDocker( - image=image, - image_port=55555, # TODO needed? - work_dir=work_dir, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) - - def _get_code_in_cfg(content: str, code: str): lines = content.splitlines() for line in lines: @@ -262,5 +244,5 @@ def _set_code_in_cfg(content: str, code: str, value: str) -> str: def _get_hype_time(value: str) -> datetime.datetime: - """Converts `yyyy-mm-dd [HH:MM]` string to datetime object""" + """Converts `yyyy-mm-dd [HH:MM]` string to datetime object.""" return parse(value).replace(tzinfo=UTC) diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index 0a5a7d3a..4094f661 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -8,15 +8,10 @@ import numpy as np import xarray as xr from cftime import num2date -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG -from ewatercycle.config._lisflood_versions import ( - get_docker_image, - get_singularity_image, - version_images, -) +from ewatercycle.config._lisflood_versions import version_images +from ewatercycle.container import start_container from ewatercycle.forcing._lisflood import LisfloodForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -115,27 +110,13 @@ def setup( # type: ignore # If not relative add dir input_dirs.append(str(mask_map.parent)) - if CFG["container_engine"].lower() == "singularity": - image = get_singularity_image(self.version, CFG["singularity_dir"]) - self.bmi = BmiClientSingularity( - image=str(image), - input_dirs=input_dirs, - work_dir=str(cfg_dir_as_path), - timeout=300, - ) - elif CFG["container_engine"].lower() == "docker": - image = get_docker_image(self.version) - self.bmi = BmiClientDocker( - image=image, - image_port=55555, - input_dirs=input_dirs, - work_dir=str(cfg_dir_as_path), - timeout=300, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) + self.bmi = start_container( + version_image=version_images[self.version], + work_dir=cfg_dir_as_path, + input_dirs=input_dirs, + timeout=300, + ) + return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index 57ea84bd..6b5584df 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -10,10 +10,9 @@ import scipy.io as sio import xarray as xr from cftime import num2date -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._marrmot import MarrmotForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.util import get_time, to_absolute_path @@ -52,6 +51,14 @@ def _generate_cfg_dir(cfg_dir: Optional[Path] = None) -> Path: return cfg_dir +_version_images: VersionImages = { + "2020.11": { + "docker": "ewatercycle/pcrg-grpc4bmi:setters", + "singularity": "ewatercycle-marrmot-grpc4bmi_2020.11.sif", + } +} + + class MarrmotM01(AbstractModel[MarrmotForcing]): """eWaterCycle implementation of Marrmot Collie River 1 (traditional bucket) model. @@ -72,7 +79,7 @@ class MarrmotM01(AbstractModel[MarrmotForcing]): model_name = "m_01_collie1_1p_1s" """Name of model in Matlab code.""" - available_versions = ("2020.11",) + available_versions = tuple(_version_images.keys()) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 @@ -82,18 +89,6 @@ def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 self.solver = Solver() self._check_forcing(forcing) - self._set_singularity_image() - self._set_docker_image() - - def _set_docker_image(self): - images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} - self.docker_image = images[self.version] - - def _set_singularity_image(self): - images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} - if CFG.get("singularity_dir"): - self.singularity_image = CFG["singularity_dir"] / images[self.version] - # unable to subclass with more specialized arguments so ignore type def setup( # type: ignore self, @@ -143,27 +138,13 @@ def setup( # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG["container_engine"].lower() == "singularity": - message = f"The singularity image {self.singularity_image} does not exist." - assert self.singularity_image.exists(), message - self.bmi = BmiClientSingularity( - image=str(self.singularity_image), - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - elif CFG["container_engine"].lower() == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) + self.bmi = start_container( + version_image=_version_images[self.version], + work_dir=cfg_dir_as_path, + timeout=300, + delay=delay, + ) + return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): @@ -328,7 +309,7 @@ class MarrmotM14(AbstractModel[MarrmotForcing]): model_name = "m_14_topmodel_7p_2s" """Name of model in Matlab code.""" - available_versions = ("2020.11",) + available_versions = tuple(_version_images.keys()) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 @@ -338,18 +319,6 @@ def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 self.solver = Solver() self._check_forcing(forcing) - self._set_singularity_image() - self._set_docker_image() - - def _set_docker_image(self): - images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} - self.docker_image = images[self.version] - - def _set_singularity_image(self): - images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} - if CFG.get("singularity_dir"): - self.singularity_image = CFG["singularity_dir"] / images[self.version] - # unable to subclass with more specialized arguments so ignore type def setup( # type: ignore self, @@ -418,27 +387,13 @@ def setup( # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG["container_engine"].lower() == "singularity": - message = f"The singularity image {self.singularity_image} does not exist." - assert self.singularity_image.exists(), message - self.bmi = BmiClientSingularity( - image=str(self.singularity_image), - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - elif CFG["container_engine"].lower() == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) + self.bmi = start_container( + version_image=_version_images[self.version], + work_dir=cfg_dir_as_path, + timeout=300, + delay=delay, + ) + return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): diff --git a/src/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py index 72265a72..e73e5d73 100644 --- a/src/ewatercycle/models/pcrglobwb.py +++ b/src/ewatercycle/models/pcrglobwb.py @@ -8,11 +8,9 @@ import numpy as np import xarray as xr from cftime import num2date -from grpc import FutureTimeoutError -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._pcrglobwb import PCRGlobWBForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -21,6 +19,13 @@ logger = logging.getLogger(__name__) +_version_images: VersionImages = { + "setters": { + "docker": "ewatercycle/pcrg-grpc4bmi:setters", + "singularity": "ewatercycle-pcrg-grpc4bmi_setters.sif", + } +} + class PCRGlobWB(AbstractModel[PCRGlobWBForcing]): """eWaterCycle implementation of PCRGlobWB hydrological model. @@ -34,7 +39,7 @@ class PCRGlobWB(AbstractModel[PCRGlobWBForcing]): """ - available_versions = ("setters",) + available_versions = tuple(_version_images.keys()) def __init__( # noqa: D107 self, @@ -43,22 +48,8 @@ def __init__( # noqa: D107 forcing: Optional[PCRGlobWBForcing] = None, ): super().__init__(version, parameter_set, forcing) - self._set_docker_image() self._setup_default_config() - def _set_docker_image(self): - images = { - "setters": "ewatercycle/pcrg-grpc4bmi:setters", - } - self.docker_image = images[self.version] - - def _singularity_image(self, singularity_dir): - images = { - "setters": "ewatercycle-pcrg-grpc4bmi_setters.sif", - } - image = singularity_dir / images[self.version] - return str(image) - def _setup_work_dir(self, cfg_dir: Optional[str] = None): if cfg_dir: self.work_dir = to_absolute_path(cfg_dir) @@ -130,19 +121,17 @@ def setup(self, cfg_dir: Optional[str] = None, **kwargs) -> Tuple[str, str]: # cfg_file = self._export_config() work_dir = self.work_dir - try: - self._start_container() - except FutureTimeoutError as exc: - # https://github.com/eWaterCycle/grpc4bmi/issues/95 - # https://github.com/eWaterCycle/grpc4bmi/issues/100 - raise ValueError( - "Couldn't spawn container within allocated time limit " - "(300 seconds). You may try pulling the docker image with" - f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} " - f"docker://{self.docker_image}` if you're using singularity," - " and then try again." - ) from exc + additional_input_dirs = [] + if self.parameter_set: + additional_input_dirs.append(str(self.parameter_set.directory)) + if self.forcing: + additional_input_dirs.append(str(self.forcing.directory)) + self.bmi = start_container( + version_image=_version_images[self.version], + work_dir=self.work_dir, + input_dirs=additional_input_dirs, + timeout=300, + ) return str(cfg_file), str(work_dir) @@ -191,31 +180,6 @@ def _export_config(self) -> PathLike: self.cfg_file = new_cfg_file return self.cfg_file - def _start_container(self): - additional_input_dirs = [str(self.parameter_set.directory)] - if self.forcing: - additional_input_dirs.append(self.forcing.directory) - - if CFG["container_engine"] == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(self.work_dir), - input_dirs=additional_input_dirs, - timeout=300, - ) - elif CFG["container_engine"] == "singularity": - self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG["singularity_dir"]), - work_dir=str(self.work_dir), - input_dirs=additional_input_dirs, - timeout=300, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) - def _coords_to_indices( self, name: str, lat: Iterable[float], lon: Iterable[float] ) -> Iterable[int]: diff --git a/src/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py index dd4998af..e71d8967 100644 --- a/src/ewatercycle/models/wflow.py +++ b/src/ewatercycle/models/wflow.py @@ -9,11 +9,9 @@ import numpy as np import xarray as xr from cftime import num2date -from grpc import FutureTimeoutError -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._wflow import WflowForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -22,6 +20,25 @@ logger = logging.getLogger(__name__) +_version_images: VersionImages = { + # "2019.1": { + # "docker":"ewatercycle/wflow-grpc4bmi:2019.1", + # "singularity": "ewatercycle-wflow-grpc4bmi_2019.1.sif", + # }, # no good ini file + "2020.1.1": { + "docker": "ewatercycle/wflow-grpc4bmi:2020.1.1", + "singularity": "ewatercycle-wflow-grpc4bmi_2020.1.1.sif", + }, + "2020.1.2": { + "docker": "ewatercycle/wflow-grpc4bmi:2020.1.2", + "singularity": "ewatercycle-wflow-grpc4bmi_2020.1.2.sif", + }, + "2020.1.3": { + "docker": "ewatercycle/wflow-grpc4bmi:2020.1.3", + "singularity": "ewatercycle-wflow-grpc4bmi_2020.1.3.sif", + }, +} + class Wflow(AbstractModel[WflowForcing]): """Create an instance of the Wflow model class. @@ -34,7 +51,7 @@ class Wflow(AbstractModel[WflowForcing]): If None, it is assumed that forcing is included with the parameter_set. """ - available_versions = ("2020.1.1", "2020.1.2", "2020.1.3") + available_versions = tuple(_version_images.keys()) """Show supported WFlow versions in eWaterCycle""" def __init__( # noqa: D107 @@ -44,27 +61,8 @@ def __init__( # noqa: D107 forcing: Optional[WflowForcing] = None, ): super().__init__(version, parameter_set, forcing) - self._set_docker_image() self._setup_default_config() - def _set_docker_image(self): - images = { - # "2019.1": "ewatercycle/wflow-grpc4bmi:2019.1", # no good ini file - "2020.1.1": "ewatercycle/wflow-grpc4bmi:2020.1.1", - "2020.1.2": "ewatercycle/wflow-grpc4bmi:2020.1.2", - "2020.1.3": "ewatercycle/wflow-grpc4bmi:2020.1.3", - } - self.docker_image = images[self.version] - - def _singularity_image(self, singularity_dir): - images = { - "2020.1.1": "ewatercycle-wflow-grpc4bmi_2020.1.1.sif", - "2020.1.2": "ewatercycle-wflow-grpc4bmi_2020.1.2.sif", - "2020.1.3": "ewatercycle-wflow-grpc4bmi_2020.1.3.sif", - } - image = singularity_dir / images[self.version] - return str(image) - def _setup_default_config(self): config_file = self.parameter_set.config forcing = self.forcing @@ -124,19 +122,11 @@ def setup(self, cfg_dir: Optional[str] = None, **kwargs) -> Tuple[str, str]: # with updated_cfg_file.open("w") as filename: cfg.write(filename) - try: - self._start_container() - except FutureTimeoutError as exc: - # https://github.com/eWaterCycle/grpc4bmi/issues/95 - # https://github.com/eWaterCycle/grpc4bmi/issues/100 - raise ValueError( - "Couldn't spawn container within allocated time limit " - "(300 seconds). You may try pulling the docker image with" - f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} " - f"docker://{self.docker_image}` if you're using singularity," - " and then try again." - ) from exc + self.bmi = start_container( + version_image=_version_images[self.version], + work_dir=self.work_dir, + timeout=300, + ) return ( str(updated_cfg_file), @@ -164,23 +154,6 @@ def _setup_working_directory(self, cfg_dir: Optional[str] = None): ) shutil.copy(src=forcing_path, dst=self.work_dir) - def _start_container(self): - if CFG["container_engine"] == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(self.work_dir), - timeout=300, - ) - elif CFG["container_engine"] == "singularity": - self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG["singularity_dir"]), - work_dir=str(self.work_dir), - timeout=300, - ) - else: - raise ValueError(f"Unknown container technology: {CFG['container_engine']}") - def _coords_to_indices( self, name: str, lat: Iterable[float], lon: Iterable[float] ) -> Iterable[int]: From 0b25c6d8c00e3c6d7fb32307516015a6d366741e Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 7 Mar 2023 15:00:35 +0100 Subject: [PATCH 6/7] Fix tests --- src/ewatercycle/container.py | 6 +- src/ewatercycle/forcing/_lisvap.py | 1 + src/ewatercycle/models/marrmot.py | 2 +- tests/models/fake_models.py | 137 +++++++++++++++++++++++++++++ tests/models/test_hype.py | 12 ++- tests/models/test_lisflood.py | 5 +- tests/models/test_pcrglobwb.py | 7 +- tests/models/test_wflow.py | 10 ++- 8 files changed, 166 insertions(+), 14 deletions(-) create mode 100644 tests/models/fake_models.py diff --git a/src/ewatercycle/container.py b/src/ewatercycle/container.py index e4b8c3ff..fe9c5389 100644 --- a/src/ewatercycle/container.py +++ b/src/ewatercycle/container.py @@ -72,11 +72,11 @@ def start_container( # https://github.com/eWaterCycle/grpc4bmi/issues/100 raise TimeoutError( "Couldn't spawn container within allocated time limit " - "({timeout} seconds). You may try pulling the docker image with" + f"({timeout} seconds). You may try pulling the docker image with" f" `docker pull {image}` and then try again." ) from exc elif engine == "singularity": - image = CFG["singularity_dir"] / image + image = str(CFG["singularity_dir"] / image) try: bmi = BmiClientSingularity( image=image, @@ -89,7 +89,7 @@ def start_container( docker_image = version_image["docker"] raise TimeoutError( "Couldn't spawn container within allocated time limit " - "({timeout} seconds). You may try pulling the docker image with" + f"({timeout} seconds). You may try pulling the docker image with" f" `singularity build {image} " f"docker://{docker_image}` and then try again." ) from exc diff --git a/src/ewatercycle/forcing/_lisvap.py b/src/ewatercycle/forcing/_lisvap.py index 5865caa4..1765206b 100644 --- a/src/ewatercycle/forcing/_lisvap.py +++ b/src/ewatercycle/forcing/_lisvap.py @@ -52,6 +52,7 @@ def lisvap( engine: Literal["docker", "singularity"] = CFG["container_engine"] image = version_images[version][engine] if CFG["container_engine"].lower() == "singularity": + image = CFG["singularity_dir"] / image args = [ "singularity", "exec", diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index 6b5584df..a0439e85 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -53,7 +53,7 @@ def _generate_cfg_dir(cfg_dir: Optional[Path] = None) -> Path: _version_images: VersionImages = { "2020.11": { - "docker": "ewatercycle/pcrg-grpc4bmi:setters", + "docker": "ewatercycle/marrmot-grpc4bmi:2020.11", "singularity": "ewatercycle-marrmot-grpc4bmi_2020.11.sif", } } diff --git a/tests/models/fake_models.py b/tests/models/fake_models.py new file mode 100644 index 00000000..002e223f --- /dev/null +++ b/tests/models/fake_models.py @@ -0,0 +1,137 @@ +"""Fake BMI models.""" +import numpy as np +from bmipy import Bmi + + +class SomeException(Exception): + pass + + +class FailingModel(Bmi): + def __init__(self, exc=SomeException()): + self.exc = exc + + def initialize(self, filename): + raise self.exc + + def update(self): + raise self.exc + + def update_until(self, time: float) -> None: + raise self.exc + + def finalize(self): + raise self.exc + + def get_component_name(self): + raise self.exc + + def get_input_item_count(self) -> int: + raise self.exc + + def get_output_item_count(self) -> int: + raise self.exc + + def get_input_var_names(self): + raise self.exc + + def get_output_var_names(self): + raise self.exc + + def get_start_time(self): + raise self.exc + + def get_current_time(self): + raise self.exc + + def get_end_time(self): + raise self.exc + + def get_time_step(self): + raise self.exc + + def get_time_units(self): + raise self.exc + + def get_var_type(self, name): + raise self.exc + + def get_var_units(self, name): + raise self.exc + + def get_var_itemsize(self, name): + raise self.exc + + def get_var_nbytes(self, name): + raise self.exc + + def get_var_grid(self, name): + raise self.exc + + def get_value(self, name, dest): + raise self.exc + + def get_value_ptr(self, name): + raise self.exc + + def get_value_at_indices(self, name, dest, inds): + raise self.exc + + def set_value(self, name, src): + raise self.exc + + def set_value_at_indices(self, name, inds, src): + raise self.exc + + def get_grid_shape(self, grid, shape): + raise self.exc + + def get_grid_x(self, grid, x): + raise self.exc + + def get_grid_y(self, grid, y): + raise self.exc + + def get_grid_z(self, grid, z): + raise self.exc + + def get_grid_spacing(self, grid, spacing): + raise self.exc + + def get_grid_origin(self, grid, origin): + raise self.exc + + def get_grid_rank(self, grid): + raise self.exc + + def get_grid_size(self, grid): + raise self.exc + + def get_grid_type(self, grid): + raise self.exc + + def get_var_location(self, name: str) -> str: + raise self.exc + + def get_grid_node_count(self, grid: int) -> int: + raise self.exc + + def get_grid_edge_count(self, grid: int) -> int: + raise self.exc + + def get_grid_face_count(self, grid: int) -> int: + raise self.exc + + def get_grid_edge_nodes(self, grid: int, edge_nodes: np.ndarray) -> np.ndarray: + raise self.exc + + def get_grid_face_nodes(self, grid: int, face_nodes: np.ndarray) -> np.ndarray: + raise self.exc + + def get_grid_nodes_per_face( + self, grid: int, nodes_per_face: np.ndarray + ) -> np.ndarray: + raise self.exc + + def get_grid_face_edges(self, grid: int, face_edges: np.ndarray) -> np.ndarray: + raise self.exc diff --git a/tests/models/test_hype.py b/tests/models/test_hype.py index 653e1c1d..8db9663b 100644 --- a/tests/models/test_hype.py +++ b/tests/models/test_hype.py @@ -12,6 +12,7 @@ from ewatercycle.forcing import load_foreign from ewatercycle.models.hype import Hype, _set_code_in_cfg from ewatercycle.parameter_sets import ParameterSet +from tests.models.fake_models import FailingModel @pytest.fixture @@ -93,6 +94,9 @@ def test_setup_container(self, model_with_setup, tmp_path): mocked_constructor.assert_called_once_with( image=f"{tmp_path}/ewatercycle-hype-grpc4bmi_feb2021.sif", work_dir=f"{tmp_path}/hype_20210102_030405", + input_dirs=[], + timeout=None, + delay=0, ) def test_setup_parameter_set_files(self, model_with_setup): @@ -141,7 +145,7 @@ def test_get_value_as_xarray(self, model): model.get_value_as_xarray("comp outflow olake") def test_get_value_at_coords(self, model): - class MockedBmi(Bmi): + class MockedBmi(FailingModel): """Pretend to be a real BMI model.""" def get_var_grid(self, name): @@ -192,6 +196,9 @@ def test_setup_container(self, model_with_setup, tmp_path): mocked_constructor.assert_called_once_with( image=f"{tmp_path}/ewatercycle-hype-grpc4bmi_feb2021.sif", work_dir=f"{tmp_path}/myworkdir", + input_dirs=[], + timeout=None, + delay=0, ) def test_setup_parameter_set_files(self, model_with_setup): @@ -329,6 +336,9 @@ def test_setup_container(self, model_with_setup, tmp_path): mocked_constructor.assert_called_once_with( image=f"{tmp_path}/ewatercycle-hype-grpc4bmi_feb2021.sif", work_dir=f"{tmp_path}/hype_20210102_030405", + input_dirs=[], + timeout=None, + delay=0, ) def test_setup_forcing_files(self, model_with_setup): diff --git a/tests/models/test_lisflood.py b/tests/models/test_lisflood.py index 11e27d0a..1b32637d 100644 --- a/tests/models/test_lisflood.py +++ b/tests/models/test_lisflood.py @@ -15,6 +15,7 @@ from ewatercycle.models.lisflood import Lisflood from ewatercycle.parameter_sets import ParameterSet, example_parameter_sets from ewatercycle.parametersetdb.config import XmlConfig +from tests.models.fake_models import FailingModel @pytest.fixture @@ -110,6 +111,7 @@ def test_setup(self, model_with_setup, tmp_path): ], work_dir=f"{tmp_path}/lisflood_20210102_030405", timeout=300, + delay=0, ) # Check content config file @@ -210,6 +212,7 @@ def test_setup(self, model_with_setup, tmp_path): ], work_dir=f"{tmp_path}/lisflood_20210102_030405", timeout=300, + delay=0, ) # Check content config file @@ -248,7 +251,7 @@ def test_parameters_after_setup( assert model.parameters == expected_parameters -class MockedBmi(Bmi): +class MockedBmi(FailingModel): """Mimic a real use case with realistic shape and abitrary high precision.""" def get_var_grid(self, name): diff --git a/tests/models/test_pcrglobwb.py b/tests/models/test_pcrglobwb.py index 54a61f7b..5e633915 100644 --- a/tests/models/test_pcrglobwb.py +++ b/tests/models/test_pcrglobwb.py @@ -5,7 +5,6 @@ import numpy as np import pytest -from bmipy import Bmi from grpc import FutureTimeoutError from grpc4bmi.bmi_client_singularity import BmiClientSingularity @@ -13,9 +12,10 @@ from ewatercycle.forcing import load_foreign from ewatercycle.models import PCRGlobWB from ewatercycle.parameter_sets import ParameterSet, example_parameter_sets +from tests.models.fake_models import FailingModel -class MockedBmi(Bmi): +class MockedBmi(FailingModel): """Pretend to be a real BMI model.""" def initialize(self, config_file): @@ -102,13 +102,12 @@ def test_setup_withtimeoutexception(model, tmp_path): with patch.object( BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() ), patch("datetime.datetime") as mocked_datetime, pytest.raises( - ValueError + TimeoutError ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert "docker pull ewatercycle/pcrg-grpc4bmi:setters" in msg sif = tmp_path / "ewatercycle-pcrg-grpc4bmi_setters.sif" assert f"build {sif} docker://ewatercycle/pcrg-grpc4bmi:setters" in msg diff --git a/tests/models/test_wflow.py b/tests/models/test_wflow.py index 88944589..f766aea8 100644 --- a/tests/models/test_wflow.py +++ b/tests/models/test_wflow.py @@ -6,7 +6,6 @@ import numpy as np import pytest -from bmipy import Bmi from grpc import FutureTimeoutError from grpc4bmi.bmi_client_singularity import BmiClientSingularity @@ -14,11 +13,15 @@ from ewatercycle.models import Wflow from ewatercycle.parameter_sets import ParameterSet from ewatercycle.parametersetdb.config import CaseConfigParser +from tests.models.fake_models import FailingModel -class MockedBmi(Bmi): +class MockedBmi(FailingModel): """Pretend to be a real BMI model.""" + def get_component_name(self) -> str: + return "mocked" + def initialize(self, config_file): pass @@ -156,13 +159,12 @@ def test_setup_withtimeoutexception(model, tmp_path): with patch.object( BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() ), patch("datetime.datetime") as mocked_datetime, pytest.raises( - ValueError + TimeoutError ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert "docker pull ewatercycle/wflow-grpc4bmi:2020.1.1" in msg sif = tmp_path / "ewatercycle-wflow-grpc4bmi_2020.1.1.sif" assert f"build {sif} docker://ewatercycle/wflow-grpc4bmi:2020.1.1" in msg From 93c571e599d710ed6bd04d2a4284df2d5383a3d9 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 7 Mar 2023 15:16:10 +0100 Subject: [PATCH 7/7] Simplify typings --- src/ewatercycle/container.py | 27 +++++++++++++-------------- src/ewatercycle/forcing/_lisvap.py | 5 +++-- src/ewatercycle/models/hype.py | 2 +- src/ewatercycle/models/lisflood.py | 2 +- src/ewatercycle/models/marrmot.py | 4 ++-- src/ewatercycle/models/pcrglobwb.py | 2 +- src/ewatercycle/models/wflow.py | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/ewatercycle/container.py b/src/ewatercycle/container.py index fe9c5389..7033aa32 100644 --- a/src/ewatercycle/container.py +++ b/src/ewatercycle/container.py @@ -1,6 +1,6 @@ """Container utilities.""" from pathlib import Path -from typing import Iterable, Literal, Mapping, Optional, TypedDict, Union +from typing import Dict, Iterable, Literal, Mapping, Optional, Union from bmipy import Bmi from grpc import FutureTimeoutError @@ -11,23 +11,22 @@ from ewatercycle import CFG +ContainerEngines = Literal["docker", "singularity"] +"""Supported container engines.""" -class VersionImage(TypedDict): - """Image container for each container technology.""" +ImageForContainerEngines = Dict[ContainerEngines, str] +"""Container image name for each container engine.""" - docker: str - """Docker image name.""" - singularity: str - """Singularity image name. Should be *.sif file in CFG["singularity_dir"].""" +VersionImages = Mapping[str, ImageForContainerEngines] +"""Dictionary of versions of a model. - -VersionImages = Mapping[str, VersionImage] -"""Image containers for each version.""" +Each version has the image name for each container engine. +""" def start_container( work_dir: Union[str, Path], - version_image: VersionImage, + image_engine: ImageForContainerEngines, input_dirs: Optional[Iterable[str]] = None, image_port=55555, timeout=None, @@ -53,8 +52,8 @@ def start_container( Returns: _description_ """ - engine: Literal["docker", "singularity"] = CFG["container_engine"] - image = version_image[engine] + engine: ContainerEngines = CFG["container_engine"] + image = image_engine[engine] if input_dirs is None: input_dirs = [] if engine == "docker": @@ -86,7 +85,7 @@ def start_container( delay=delay, ) except FutureTimeoutError as exc: - docker_image = version_image["docker"] + docker_image = image_engine["docker"] raise TimeoutError( "Couldn't spawn container within allocated time limit " f"({timeout} seconds). You may try pulling the docker image with" diff --git a/src/ewatercycle/forcing/_lisvap.py b/src/ewatercycle/forcing/_lisvap.py index 1765206b..5af60ed2 100644 --- a/src/ewatercycle/forcing/_lisvap.py +++ b/src/ewatercycle/forcing/_lisvap.py @@ -23,9 +23,10 @@ import os import subprocess -from typing import Dict, Literal, Tuple +from typing import Dict, Tuple from ewatercycle import CFG +from ewatercycle.container import ContainerEngines from ewatercycle.parametersetdb.config import XmlConfig from ..config._lisflood_versions import version_images @@ -49,7 +50,7 @@ def lisvap( mask_map, forcing_dir, ) - engine: Literal["docker", "singularity"] = CFG["container_engine"] + engine: ContainerEngines = CFG["container_engine"] image = version_images[version][engine] if CFG["container_engine"].lower() == "singularity": image = CFG["singularity_dir"] / image diff --git a/src/ewatercycle/models/hype.py b/src/ewatercycle/models/hype.py index 303473f2..69d84d2d 100644 --- a/src/ewatercycle/models/hype.py +++ b/src/ewatercycle/models/hype.py @@ -153,7 +153,7 @@ def setup( # type: ignore # start container self.bmi = start_container( - version_image=_version_images[self.version], + image_engine=_version_images[self.version], work_dir=cfg_dir_as_path, ) diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index 4094f661..68f004d3 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -111,7 +111,7 @@ def setup( # type: ignore input_dirs.append(str(mask_map.parent)) self.bmi = start_container( - version_image=version_images[self.version], + image_engine=version_images[self.version], work_dir=cfg_dir_as_path, input_dirs=input_dirs, timeout=300, diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index a0439e85..fd15c0d2 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -139,7 +139,7 @@ def setup( # type: ignore config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) self.bmi = start_container( - version_image=_version_images[self.version], + image_engine=_version_images[self.version], work_dir=cfg_dir_as_path, timeout=300, delay=delay, @@ -388,7 +388,7 @@ def setup( # type: ignore config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) self.bmi = start_container( - version_image=_version_images[self.version], + image_engine=_version_images[self.version], work_dir=cfg_dir_as_path, timeout=300, delay=delay, diff --git a/src/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py index e73e5d73..65c4bc20 100644 --- a/src/ewatercycle/models/pcrglobwb.py +++ b/src/ewatercycle/models/pcrglobwb.py @@ -127,7 +127,7 @@ def setup(self, cfg_dir: Optional[str] = None, **kwargs) -> Tuple[str, str]: # if self.forcing: additional_input_dirs.append(str(self.forcing.directory)) self.bmi = start_container( - version_image=_version_images[self.version], + image_engine=_version_images[self.version], work_dir=self.work_dir, input_dirs=additional_input_dirs, timeout=300, diff --git a/src/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py index e71d8967..44fde922 100644 --- a/src/ewatercycle/models/wflow.py +++ b/src/ewatercycle/models/wflow.py @@ -123,7 +123,7 @@ def setup(self, cfg_dir: Optional[str] = None, **kwargs) -> Tuple[str, str]: # cfg.write(filename) self.bmi = start_container( - version_image=_version_images[self.version], + image_engine=_version_images[self.version], work_dir=self.work_dir, timeout=300, )