diff --git a/autonomy/cli/deploy.py b/autonomy/cli/deploy.py
index 5df9376ff2..0058100f33 100644
--- a/autonomy/cli/deploy.py
+++ b/autonomy/cli/deploy.py
@@ -22,7 +22,7 @@
import os
import shutil
from pathlib import Path
-from typing import Optional, cast
+from typing import List, Optional, cast
import click
from aea import AEA_DIR
@@ -40,6 +40,7 @@
build_and_deploy_from_token,
build_deployment,
run_deployment,
+ run_host_deployment,
stop_deployment,
)
from autonomy.cli.helpers.env import load_env_file
@@ -60,6 +61,7 @@
from autonomy.deploy.constants import INFO, LOGGING_LEVELS
from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator
def _validate_packages_path(path: Optional[Path] = None) -> Path:
@@ -130,12 +132,18 @@ def deploy_group(
default=1,
help="Number of services.",
)
+@click.option(
+ "--localhost",
+ "deployment_type",
+ flag_value=HostDeploymentGenerator.deployment_type,
+ help="Use localhost as a backend.",
+)
@click.option(
"--docker",
"deployment_type",
flag_value=DockerComposeGenerator.deployment_type,
default=True,
- help="Use docker as a backend.",
+ help="Use docker as a backend. (default)",
)
@click.option(
"--kubernetes",
@@ -215,6 +223,13 @@ def deploy_group(
help="Set agent memory usage limit.",
default=DEFAULT_AGENT_MEMORY_LIMIT,
)
+@click.option(
+ "--mkdir",
+ type=str,
+ help="Directory names to create in the build directory.",
+ default=[],
+ multiple=True,
+)
@registry_flag()
@password_option(confirmation_prompt=True)
@image_author_option
@@ -226,6 +241,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo
output_dir: Optional[Path],
dev_mode: bool,
registry: str,
+ mkdir: List[str],
number_of_agents: Optional[int] = None,
number_of_services: int = 1,
password: Optional[str] = None,
@@ -290,6 +306,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo
use_acn=use_acn,
use_tm_testnet_setup=use_tm_testnet_setup,
image_author=image_author,
+ mkdir=mkdir,
resources={
"agent": {
"limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit},
@@ -329,16 +346,42 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo
default=False,
help="Run service in the background.",
)
+@click.option(
+ "--localhost",
+ "deployment_type",
+ flag_value="localhost",
+ help="Use localhost as a backend.",
+)
+@click.option(
+ "--docker",
+ "deployment_type",
+ flag_value="docker",
+ help="Use docker as a backend. (default)",
+ default=True,
+)
def run(
- build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool = False
+ build_dir: Path,
+ no_recreate: bool,
+ remove_orphans: bool,
+ detach: bool,
+ deployment_type: str,
) -> None:
"""Run deployment."""
build_dir = Path(build_dir or Path.cwd()).absolute()
- if not (build_dir / DockerComposeGenerator.output_name).exists():
+ deployment = (
+ HostDeploymentGenerator
+ if deployment_type == "localhost"
+ else DockerComposeGenerator
+ )
+ if not (build_dir / deployment.output_name).exists():
raise click.ClickException(
f"Deployment configuration does not exist @ {build_dir}"
)
- run_deployment(build_dir, no_recreate, remove_orphans, detach=detach)
+ click.echo(f"Running build @ {build_dir}")
+ if deployment_type == "localhost":
+ run_host_deployment(build_dir, detach)
+ else:
+ run_deployment(build_dir, no_recreate, remove_orphans, detach=detach)
@deploy_group.command(name="stop")
diff --git a/autonomy/cli/helpers/deployment.py b/autonomy/cli/helpers/deployment.py
index 8853124b59..30bf805679 100644
--- a/autonomy/cli/helpers/deployment.py
+++ b/autonomy/cli/helpers/deployment.py
@@ -18,13 +18,18 @@
# ------------------------------------------------------------------------------
"""Deployment helpers."""
+import json
import os
+import platform
import shutil
+import subprocess # nosec
+import sys
import time
from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional, Tuple
import click
+from aea.configurations.constants import AGENT
from aea.configurations.data_types import PublicId
from aea.helpers.base import cd
from compose.cli import main as docker_compose
@@ -45,20 +50,27 @@
from autonomy.deploy.build import generate_deployment
from autonomy.deploy.constants import (
AGENT_KEYS_DIR,
+ AGENT_VARS_CONFIG_FILE,
BENCHMARKS_DIR,
+ DEATTACH_WINDOWS_FLAG,
INFO,
LOG_DIR,
PERSISTENT_DATA_DIR,
+ TENDERMINT_FLASK_APP_PATH,
+ TENDERMINT_VARS_CONFIG_FILE,
TM_STATE_DIR,
VENVS_DIR,
)
from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.utils import check_tendermint_version
from autonomy.deploy.image import build_image
-def _build_dirs(build_dir: Path) -> None:
+def _build_dirs(build_dir: Path, mkdir: List[str]) -> None:
"""Build necessary directories."""
+ mkdirs = [(new_dir_name,) for new_dir_name in mkdir]
+
for dir_path in [
(PERSISTENT_DATA_DIR,),
(PERSISTENT_DATA_DIR, LOG_DIR),
@@ -66,9 +78,9 @@ def _build_dirs(build_dir: Path) -> None:
(PERSISTENT_DATA_DIR, BENCHMARKS_DIR),
(PERSISTENT_DATA_DIR, VENVS_DIR),
(AGENT_KEYS_DIR,),
- ]:
+ ] + mkdirs:
path = Path(build_dir, *dir_path)
- path.mkdir()
+ path.mkdir(exist_ok=True, parents=True)
# TOFIX: remove this safely
try:
os.chown(path, 1000, 1000)
@@ -119,7 +131,6 @@ def run_deployment(
detach: bool = False,
) -> None:
"""Run deployment."""
- click.echo(f"Running build @ {build_dir}")
try:
project = _load_compose_project(build_dir=build_dir)
commands = docker_compose.TopLevelCommand(project=project)
@@ -163,6 +174,65 @@ def run_deployment(
stop_deployment(build_dir=build_dir)
+def _get_deattached_creation_flags() -> int:
+ """Get Popen creation flag based on the platform."""
+ return DEATTACH_WINDOWS_FLAG if platform.system() == "Windows" else 0
+
+
+def _start_localhost_agent(working_dir: Path, detach: bool) -> None:
+ """Start localhost agent process."""
+ env = json.loads((working_dir / AGENT_VARS_CONFIG_FILE).read_text())
+ process_fn: Callable = subprocess.Popen if detach else subprocess.run # type: ignore[assignment]
+ process = process_fn( # pylint: disable=subprocess-run-check # nosec
+ args=[sys.executable, "-m", "aea.cli", "run"],
+ cwd=working_dir / AGENT,
+ env={**os.environ, **env},
+ creationflags=_get_deattached_creation_flags(), # Detach process from the main process
+ )
+ (working_dir / "agent.pid").write_text(
+ data=str(process.pid),
+ )
+
+
+def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen:
+ """Start localhost tendermint process."""
+ check_tendermint_version()
+ env = json.loads((working_dir / TENDERMINT_VARS_CONFIG_FILE).read_text())
+ flask_app_path = Path(__file__).parents[3] / TENDERMINT_FLASK_APP_PATH
+ process = subprocess.Popen( # pylint: disable=consider-using-with # nosec
+ args=[
+ "flask",
+ "run",
+ "--host",
+ "localhost",
+ "--port",
+ "8080",
+ ],
+ cwd=working_dir,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ env={**os.environ, **env, "FLASK_APP": f"{flask_app_path}:create_server"},
+ creationflags=_get_deattached_creation_flags(), # Detach process from the main process
+ )
+ (working_dir / "tendermint.pid").write_text(
+ data=str(process.pid),
+ )
+ return process
+
+
+def run_host_deployment(build_dir: Path, detach: bool = False) -> None:
+ """Run host deployment."""
+ tm_process = _start_localhost_tendermint(build_dir)
+ try:
+ _start_localhost_agent(build_dir, detach)
+ except Exception as e: # pylint: disable=broad-except
+ click.echo(e)
+ tm_process.terminate()
+ finally:
+ if not detach:
+ tm_process.terminate()
+
+
def stop_deployment(build_dir: Path) -> None:
"""Stop running deployment."""
try:
@@ -196,6 +266,7 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals
resources: Optional[Resources] = None,
service_hash_id: Optional[str] = None,
service_offset: int = 0,
+ mkdir: Optional[List[str]] = None,
) -> None:
"""Build deployment."""
@@ -209,8 +280,6 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals
click.echo(f"Building deployment @ {build_dir}")
build_dir.mkdir()
- _build_dirs(build_dir)
-
if service_hash_id is None:
service_hash_id = build_hash_id()
@@ -237,6 +306,9 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals
image_author=image_author,
resources=resources,
)
+ if mkdir is not None:
+ _build_dirs(build_dir, mkdir)
+
click.echo(report)
diff --git a/autonomy/deploy/base.py b/autonomy/deploy/base.py
index 1ea737f508..49d31b1cac 100644
--- a/autonomy/deploy/base.py
+++ b/autonomy/deploy/base.py
@@ -66,6 +66,7 @@
KUBERNETES_DEPLOYMENT = "kubernetes"
DOCKER_COMPOSE_DEPLOYMENT = "docker-compose"
+LOCALHOST_DEPLOYMENT = "localhost"
LOOPBACK = "127.0.0.1"
LOCALHOST = "localhost"
@@ -466,14 +467,14 @@ def _try_update_tendermint_params(
"""Try update the tendermint parameters"""
is_kubernetes_deployment = self.deplopyment_type == KUBERNETES_DEPLOYMENT
+ is_localhost_deployment = self.deplopyment_type == LOCALHOST_DEPLOYMENT
def _update_tendermint_params(
param_args: Dict,
idx: int,
- is_kubernetes_deployment: bool = False,
) -> None:
"""Update tendermint params"""
- if is_kubernetes_deployment:
+ if is_kubernetes_deployment or is_localhost_deployment:
param_args[TENDERMINT_URL_PARAM] = TENDERMINT_NODE_LOCAL
param_args[TENDERMINT_COM_URL_PARAM] = TENDERMINT_COM_LOCAL
else:
@@ -504,7 +505,6 @@ def _update_tendermint_params(
_update_tendermint_params(
param_args=param_args,
idx=0,
- is_kubernetes_deployment=is_kubernetes_deployment,
)
else:
param_args = self._get_config_from_json_path(
@@ -513,7 +513,6 @@ def _update_tendermint_params(
_update_tendermint_params(
param_args=param_args,
idx=0,
- is_kubernetes_deployment=is_kubernetes_deployment,
)
return
@@ -530,7 +529,6 @@ def _update_tendermint_params(
_update_tendermint_params(
param_args=param_args,
idx=agent_idx,
- is_kubernetes_deployment=is_kubernetes_deployment,
)
except KeyError: # pragma: nocover
logging.warning(
@@ -611,9 +609,9 @@ def _update_abci_connection_config(
processed_overrides = deepcopy(overrides)
if self.service.number_of_agents == 1:
processed_overrides["config"]["host"] = (
- LOOPBACK
- if self.deplopyment_type == KUBERNETES_DEPLOYMENT
- else self.get_abci_container_name(index=0)
+ self.get_abci_container_name(index=0)
+ if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT
+ else LOOPBACK
)
processed_overrides["config"]["port"] = processed_overrides["config"].get(
"port", DEFAULT_ABCI_PORT
@@ -627,9 +625,9 @@ def _update_abci_connection_config(
for idx, override in processed_overrides.items():
override["config"]["host"] = (
- LOOPBACK
- if self.deplopyment_type == KUBERNETES_DEPLOYMENT
- else self.get_abci_container_name(index=idx)
+ self.get_abci_container_name(index=idx)
+ if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT
+ else LOOPBACK
)
override["config"]["port"] = override["config"].get(
"port", DEFAULT_ABCI_PORT
diff --git a/autonomy/deploy/build.py b/autonomy/deploy/build.py
index df65f13d9e..d6b11d94da 100644
--- a/autonomy/deploy/build.py
+++ b/autonomy/deploy/build.py
@@ -24,11 +24,13 @@
from autonomy.deploy.constants import DEPLOYMENT_REPORT, INFO
from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator
DEPLOYMENT_OPTIONS: Dict[str, Type[BaseDeploymentGenerator]] = {
"kubernetes": KubernetesGenerator,
"docker-compose": DockerComposeGenerator,
+ "localhost": HostDeploymentGenerator,
}
@@ -56,9 +58,14 @@ def generate_deployment( # pylint: disable=too-many-arguments, too-many-locals
resources: Optional[Resources] = None,
) -> str:
"""Generate the deployment for the service."""
- if dev_mode and type_of_deployment != DockerComposeGenerator.deployment_type:
+ if type_of_deployment == HostDeploymentGenerator.deployment_type:
+ if number_of_agents is not None and number_of_agents > 1:
+ raise RuntimeError(
+ "Host deployment currently only supports single agent deployments"
+ )
+ elif dev_mode:
raise RuntimeError(
- "Development mode can only be used with docker-compose deployments"
+ "Development mode can only be used with localhost deployments"
)
service_builder = ServiceBuilder.from_dir(
diff --git a/autonomy/deploy/constants.py b/autonomy/deploy/constants.py
index 00c553b647..ccb8bff62e 100644
--- a/autonomy/deploy/constants.py
+++ b/autonomy/deploy/constants.py
@@ -19,6 +19,7 @@
"""Constants for generating deployments environment."""
+from pathlib import Path
from string import Template
@@ -34,6 +35,23 @@
DEPLOYMENT_KEY_DIRECTORY = "agent_keys"
DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA = "agent_{agent_n}"
KUBERNETES_AGENT_KEY_NAME = DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA + "_private_key.yaml"
+TENDERMINT_BIN_UNIX = "tendermint"
+TENDERMINT_BIN_WINDOWS = "tendermint.exe"
+TENDERMINT_VARS_CONFIG_FILE = "tendermint.json"
+AGENT_VARS_CONFIG_FILE = "agent.json"
+TENDERMINT_FLASK_APP_PATH = (
+ Path("autonomy") / "deploy" / "generators" / "localhost" / "tendermint" / "app.py"
+)
+DEATTACH_WINDOWS_FLAG = 0x00000008
+
+TM_ENV_TMHOME = "TMHOME"
+TM_ENV_TMSTATE = "TMSTATE"
+TM_ENV_PROXY_APP = "PROXY_APP"
+TM_ENV_P2P_LADDR = "P2P_LADDR"
+TM_ENV_RPC_LADDR = "RPC_LADDR"
+TM_ENV_PROXY_APP = "PROXY_APP"
+TM_ENV_CREATE_EMPTY_BLOCKS = "CREATE_EMPTY_BLOCKS"
+TM_ENV_USE_GRPC = "USE_GRPC"
DEFAULT_ENCODING = "utf-8"
diff --git a/autonomy/deploy/generators/localhost/__init__.py b/autonomy/deploy/generators/localhost/__init__.py
new file mode 100644
index 0000000000..d0103678f9
--- /dev/null
+++ b/autonomy/deploy/generators/localhost/__init__.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+# Copyright 2024 Valory AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Localhost Deployment Generator."""
diff --git a/autonomy/deploy/generators/localhost/base.py b/autonomy/deploy/generators/localhost/base.py
new file mode 100644
index 0000000000..07825db954
--- /dev/null
+++ b/autonomy/deploy/generators/localhost/base.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+# Copyright 2024 Valory AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Localhost Deployment Generator."""
+import json
+import subprocess # nosec
+import typing as t
+from pathlib import Path
+
+from aea.configurations.constants import (
+ DEFAULT_LEDGER,
+ LEDGER,
+ PRIVATE_KEY,
+ PRIVATE_KEY_PATH_SCHEMA,
+)
+
+from autonomy.deploy.base import BaseDeploymentGenerator
+from autonomy.deploy.constants import (
+ AGENT_VARS_CONFIG_FILE,
+ DEFAULT_ENCODING,
+ TENDERMINT_VARS_CONFIG_FILE,
+ TM_ENV_CREATE_EMPTY_BLOCKS,
+ TM_ENV_P2P_LADDR,
+ TM_ENV_PROXY_APP,
+ TM_ENV_RPC_LADDR,
+ TM_ENV_TMHOME,
+ TM_ENV_TMSTATE,
+ TM_ENV_USE_GRPC,
+ TM_STATE_DIR,
+)
+from autonomy.deploy.generators.localhost.utils import (
+ check_tendermint_version,
+ setup_agent,
+)
+
+
+class HostDeploymentGenerator(BaseDeploymentGenerator):
+ """Localhost deployment."""
+
+ output_name: str = AGENT_VARS_CONFIG_FILE
+ deployment_type: str = "localhost"
+
+ @property
+ def agent_dir(self) -> Path:
+ """Path to the agent directory."""
+ return self.build_dir / "agent"
+
+ def generate_config_tendermint(self) -> "HostDeploymentGenerator":
+ """Generate tendermint configuration."""
+ tmhome = str(self.build_dir / "node")
+ tendermint_executable = check_tendermint_version()
+ # if executable found, setup its configs
+ subprocess.run( # pylint: disable=subprocess-run-check # nosec
+ args=[
+ tendermint_executable,
+ "--home",
+ tmhome,
+ "init",
+ ],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+
+ # TODO: Dynamic port allocation
+ params = {
+ TM_ENV_TMHOME: tmhome,
+ TM_ENV_TMSTATE: str(self.build_dir / TM_STATE_DIR),
+ TM_ENV_P2P_LADDR: "tcp://localhost:26656",
+ TM_ENV_RPC_LADDR: "tcp://localhost:26657",
+ TM_ENV_PROXY_APP: "tcp://localhost:26658",
+ TM_ENV_CREATE_EMPTY_BLOCKS: "true",
+ TM_ENV_USE_GRPC: "false",
+ }
+ (self.build_dir / TENDERMINT_VARS_CONFIG_FILE).write_text(
+ json.dumps(params, indent=2),
+ )
+
+ return self
+
+ def generate(
+ self,
+ image_version: t.Optional[str] = None,
+ use_hardhat: bool = False,
+ use_acn: bool = False,
+ ) -> "HostDeploymentGenerator":
+ """Generate agent and tendermint configurations"""
+ agent = self.service_builder.generate_agent(agent_n=0)
+ self.output = json.dumps(
+ {key: f"{value}" for key, value in agent.items()}, indent=2
+ )
+ (self.build_dir / self.output_name).write_text(
+ json.dumps(self.output, indent=2),
+ )
+
+ return self
+
+ def _populate_keys(self) -> None:
+ """Populate the keys directory"""
+ kp, *_ = t.cast(t.List[t.Dict[str, str]], self.service_builder.keys)
+ key = kp[PRIVATE_KEY]
+ ledger = kp.get(LEDGER, DEFAULT_LEDGER)
+ keys_file = self.build_dir / PRIVATE_KEY_PATH_SCHEMA.format(ledger)
+ keys_file.write_text(key, encoding=DEFAULT_ENCODING)
+ setup_agent(self.build_dir, self.agent_dir, keys_file)
+
+ def _populate_keys_multiledger(self) -> None:
+ """Populate the keys directory with multiple set of keys"""
+
+ def populate_private_keys(self) -> "HostDeploymentGenerator":
+ """Populate the private keys to the build directory for host mapping."""
+ if self.service_builder.multiledger:
+ self._populate_keys_multiledger()
+ else:
+ self._populate_keys()
+ return self
+
+ def write_config(self) -> "BaseDeploymentGenerator":
+ """Write output to build dir"""
+ super().write_config()
+ return self
diff --git a/autonomy/deploy/generators/localhost/tendermint/__init__.py b/autonomy/deploy/generators/localhost/tendermint/__init__.py
new file mode 100644
index 0000000000..184e128148
--- /dev/null
+++ b/autonomy/deploy/generators/localhost/tendermint/__init__.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+# Copyright 2021-2024 Valory AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Tendermint flask app."""
diff --git a/autonomy/deploy/generators/localhost/tendermint/app.py b/autonomy/deploy/generators/localhost/tendermint/app.py
new file mode 100644
index 0000000000..2154181f62
--- /dev/null
+++ b/autonomy/deploy/generators/localhost/tendermint/app.py
@@ -0,0 +1,355 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+# Copyright 2021-2022 Valory AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""HTTP server to control the tendermint execution environment."""
+import json
+import logging
+import os
+import re
+import shutil
+import stat
+import traceback
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple, cast
+
+import requests
+from flask import Flask, Response, jsonify, request
+from werkzeug.exceptions import InternalServerError, NotFound
+
+from autonomy.deploy.constants import (
+ TM_ENV_CREATE_EMPTY_BLOCKS,
+ TM_ENV_P2P_LADDR,
+ TM_ENV_PROXY_APP,
+ TM_ENV_RPC_LADDR,
+ TM_ENV_TMHOME,
+ TM_ENV_USE_GRPC,
+)
+
+
+try:
+ from .tendermint import ( # type: ignore
+ DEFAULT_P2P_LISTEN_ADDRESS,
+ DEFAULT_RPC_LISTEN_ADDRESS,
+ TendermintNode,
+ TendermintParams,
+ )
+except ImportError:
+ from tendermint import ( # type: ignore
+ DEFAULT_P2P_LISTEN_ADDRESS,
+ DEFAULT_RPC_LISTEN_ADDRESS,
+ TendermintNode,
+ TendermintParams,
+ )
+
+ENCODING = "utf-8"
+DEFAULT_LOG_FILE = "log.log"
+IS_DEV_MODE = os.environ.get("DEV_MODE", "0") == "1"
+CONFIG_OVERRIDE = [
+ ("fast_sync = true", "fast_sync = false"),
+ ("max_num_outbound_peers = 10", "max_num_outbound_peers = 0"),
+ ("pex = true", "pex = false"),
+]
+DOCKER_INTERNAL_HOST = "host.docker.internal"
+TM_STATUS_ENDPOINT = "http://localhost:26657/status"
+
+logging.basicConfig(
+ filename=os.environ.get("LOG_FILE", DEFAULT_LOG_FILE),
+ level=logging.DEBUG,
+ format="%(asctime)s %(levelname)s %(name)s %(threadName)s : %(message)s", # noqa : W1309
+)
+
+
+def load_genesis() -> Any:
+ """Load genesis file."""
+ return json.loads(
+ Path(os.environ["TMHOME"], "config", "genesis.json").read_text(
+ encoding=ENCODING
+ )
+ )
+
+
+def get_defaults() -> Dict[str, str]:
+ """Get defaults from genesis file."""
+ genesis = load_genesis()
+ return dict(genesis_time=genesis.get("genesis_time"))
+
+
+def override_config_toml() -> None:
+ """Update sync method."""
+
+ config_path = str(Path(os.environ["TMHOME"]) / "config" / "config.toml")
+ logging.info(config_path)
+ with open(config_path, "r", encoding=ENCODING) as fp:
+ config = fp.read()
+
+ for old, new in CONFIG_OVERRIDE:
+ config = config.replace(old, new)
+
+ with open(config_path, "w+", encoding=ENCODING) as fp:
+ fp.write(config)
+
+
+def update_peers(validators: List[Dict], config_path: Path) -> None:
+ """Fix peers."""
+
+ config_text = config_path.read_text(encoding="utf-8")
+
+ new_peer_string = 'persistent_peers = "'
+ for peer in validators:
+ hostname = peer["hostname"]
+ if hostname in ("localhost", "0.0.0.0"): # nosec
+ # This (tendermint) node will be running in a docker container and no other node
+ # will be running in the same container. If we receive either localhost or 0.0.0.0,
+ # we make an assumption that the address belongs to a node running on the
+ # same machine with a different docker container and different p2p port so,
+ # we replace the hostname with the docker's internal host url.
+ hostname = DOCKER_INTERNAL_HOST
+ new_peer_string += (
+ peer["peer_id"] + "@" + hostname + ":" + str(peer["p2p_port"]) + ","
+ )
+ new_peer_string = new_peer_string[:-1] + '"\n'
+
+ updated_config = re.sub('persistent_peers = ".*\n', new_peer_string, config_text)
+ config_path.write_text(updated_config, encoding="utf-8")
+
+
+def update_external_address(external_address: str, config_path: Path) -> None:
+ """Update the external address."""
+ config_text = config_path.read_text(encoding="utf-8")
+ new_external_address = f'external_address = "{external_address}"\n'
+ updated_config = re.sub(
+ 'external_address = ".*\n', new_external_address, config_text
+ )
+ config_path.write_text(updated_config, encoding="utf-8")
+
+
+def update_genesis_config(data: Dict) -> None:
+ """Update genesis.json file for the tendermint node."""
+
+ genesis_file = Path(os.environ["TMHOME"]) / "config" / "genesis.json"
+ genesis_data = {}
+ genesis_data["genesis_time"] = data["genesis_config"]["genesis_time"]
+ genesis_data["chain_id"] = data["genesis_config"]["chain_id"]
+ genesis_data["initial_height"] = "0"
+ genesis_data["consensus_params"] = data["genesis_config"]["consensus_params"]
+ genesis_data["validators"] = [
+ {
+ "address": validator["address"],
+ "pub_key": validator["pub_key"],
+ "power": validator["power"],
+ "name": validator["name"],
+ }
+ for validator in data["validators"]
+ ]
+ genesis_data["app_hash"] = ""
+ genesis_file.write_text(json.dumps(genesis_data, indent=2), encoding=ENCODING)
+
+
+class PeriodDumper:
+ """Dumper for tendermint data."""
+
+ resets: int
+ dump_dir: Path
+ logger: logging.Logger
+
+ def __init__(self, logger: logging.Logger, dump_dir: Optional[Path] = None) -> None:
+ """Initialize object."""
+
+ self.resets = 0
+ self.logger = logger
+ self.dump_dir = dump_dir or Path(os.environ.get("TMSTATE") or "/tm_state")
+
+ if self.dump_dir.is_dir():
+ shutil.rmtree(str(self.dump_dir), onerror=self.readonly_handler)
+ self.dump_dir.mkdir(exist_ok=True)
+
+ @staticmethod
+ def readonly_handler(
+ func: Callable, path: str, execinfo: Any # pylint: disable=unused-argument
+ ) -> None:
+ """If permission is readonly, we change and retry."""
+ try:
+ os.chmod(path, stat.S_IWRITE)
+ func(path)
+ except (FileNotFoundError, OSError):
+ return
+
+ def dump_period(self) -> None:
+ """Dump tendermint run data for replay"""
+ store_dir = self.dump_dir / f"period_{self.resets}"
+ store_dir.mkdir(exist_ok=True)
+ try:
+ shutil.copytree(
+ os.environ["TMHOME"], str(store_dir / ("node" + os.environ["ID"]))
+ )
+ self.logger.info(f"Dumped data for period {self.resets}")
+ except OSError as e:
+ self.logger.info(
+ f"Error occurred while dumping data for period {self.resets}: {e}"
+ )
+ self.resets += 1
+
+
+def create_app( # pylint: disable=too-many-statements
+ dump_dir: Optional[Path] = None,
+ debug: bool = False,
+) -> Tuple[Flask, TendermintNode]:
+ """Create the Tendermint server app"""
+ write_to_log = os.environ.get("WRITE_TO_LOG", "false").lower() == "true"
+ tendermint_params = TendermintParams(
+ proxy_app=os.environ[TM_ENV_PROXY_APP],
+ rpc_laddr=os.environ.get(TM_ENV_RPC_LADDR, DEFAULT_RPC_LISTEN_ADDRESS),
+ p2p_laddr=os.environ.get(TM_ENV_P2P_LADDR, DEFAULT_P2P_LISTEN_ADDRESS),
+ consensus_create_empty_blocks=os.environ[TM_ENV_CREATE_EMPTY_BLOCKS] == "true",
+ home=os.environ[TM_ENV_TMHOME],
+ use_grpc=os.environ[TM_ENV_USE_GRPC] == "true",
+ )
+
+ app = Flask(__name__)
+ period_dumper = PeriodDumper(logger=app.logger, dump_dir=dump_dir)
+ tendermint_node = TendermintNode(
+ tendermint_params,
+ logger=app.logger,
+ write_to_log=write_to_log,
+ )
+ tendermint_node.init()
+ override_config_toml()
+ tendermint_node.start(debug=debug)
+
+ @app.get("/params")
+ def get_params() -> Dict:
+ """Get tendermint params."""
+ try:
+ priv_key_file = (
+ Path(os.environ["TMHOME"]) / "config" / "priv_validator_key.json"
+ )
+ priv_key_data = json.loads(priv_key_file.read_text(encoding=ENCODING))
+ del priv_key_data["priv_key"]
+ status = requests.get(TM_STATUS_ENDPOINT).json()
+ priv_key_data["peer_id"] = status["result"]["node_info"]["id"]
+ return {
+ "params": priv_key_data,
+ "status": True,
+ "error": None,
+ }
+ except (FileNotFoundError, json.JSONDecodeError):
+ return {"params": {}, "status": False, "error": traceback.format_exc()}
+
+ @app.post("/params")
+ def update_params() -> Dict:
+ """Update validator params."""
+
+ try:
+ data: Dict = json.loads(request.get_data().decode(ENCODING))
+ cast(logging.Logger, app.logger).debug( # pylint: disable=no-member
+ f"Data update requested with data={data}"
+ )
+
+ cast(logging.Logger, app.logger).info( # pylint: disable=no-member
+ "Updating genesis config."
+ )
+ update_genesis_config(data=data)
+
+ cast(logging.Logger, app.logger).info( # pylint: disable=no-member
+ "Updating peristent peers."
+ )
+ config_path = Path(os.environ["TMHOME"]) / "config" / "config.toml"
+ update_peers(
+ validators=data["validators"],
+ config_path=config_path,
+ )
+ update_external_address(
+ external_address=data["external_address"],
+ config_path=config_path,
+ )
+
+ return {"status": True, "error": None}
+ except (FileNotFoundError, json.JSONDecodeError, PermissionError):
+ return {"status": False, "error": traceback.format_exc()}
+
+ @app.route("/gentle_reset")
+ def gentle_reset() -> Tuple[Any, int]:
+ """Reset the tendermint node gently."""
+ try:
+ tendermint_node.stop()
+ tendermint_node.start()
+ return jsonify({"message": "Reset successful.", "status": True}), 200
+ except Exception as e: # pylint: disable=W0703
+ return jsonify({"message": f"Reset failed: {e}", "status": False}), 200
+
+ @app.route("/app_hash")
+ def app_hash() -> Tuple[Any, int]:
+ """Get the app hash."""
+ try:
+ non_routable, loopback = "0.0.0.0", "127.0.0.1" # nosec
+ endpoint = f"{tendermint_params.rpc_laddr.replace('tcp', 'http').replace(non_routable, loopback)}/block"
+ height = request.args.get("height")
+ params = {"height": height} if height is not None else None
+ res = requests.get(endpoint, params)
+ app_hash_ = res.json()["result"]["block"]["header"]["app_hash"]
+ return jsonify({"app_hash": app_hash_}), res.status_code
+ except Exception as e: # pylint: disable=W0703
+ return (
+ jsonify({"error": f"Could not get the app hash: {str(e)}"}),
+ 200,
+ )
+
+ @app.route("/hard_reset")
+ def hard_reset() -> Tuple[Any, int]:
+ """Reset the node forcefully, and prune the blocks"""
+ try:
+ tendermint_node.stop()
+ if IS_DEV_MODE:
+ period_dumper.dump_period()
+
+ return_code = tendermint_node.prune_blocks()
+ if return_code:
+ tendermint_node.start()
+ raise RuntimeError("Could not perform `unsafe-reset-all` successfully!")
+ defaults = get_defaults()
+ tendermint_node.reset_genesis_file(
+ request.args.get("genesis_time", defaults["genesis_time"]),
+ # default should be 1: https://github.com/tendermint/tendermint/pull/5191/files
+ request.args.get("initial_height", "1"),
+ request.args.get("period_count", "0"),
+ )
+ tendermint_node.start()
+ return jsonify({"message": "Reset successful.", "status": True}), 200
+ except Exception as e: # pylint: disable=W0703
+ return jsonify({"message": f"Reset failed: {e}", "status": False}), 200
+
+ @app.errorhandler(404) # type: ignore
+ def handle_notfound(e: NotFound) -> Response:
+ """Handle server error."""
+ cast(logging.Logger, app.logger).info(e) # pylint: disable=E
+ return Response("Not Found", status=404, mimetype="application/json")
+
+ @app.errorhandler(500) # type: ignore
+ def handle_server_error(e: InternalServerError) -> Response:
+ """Handle server error."""
+ cast(logging.Logger, app.logger).info(e) # pylint: disable=E
+ return Response("Error Closing Node", status=500, mimetype="application/json")
+
+ return app, tendermint_node
+
+
+def create_server() -> Any:
+ """Function to retrieve just the app to be used by flask entry point."""
+ flask_app, _ = create_app()
+ return flask_app
diff --git a/autonomy/deploy/generators/localhost/tendermint/tendermint.py b/autonomy/deploy/generators/localhost/tendermint/tendermint.py
new file mode 100644
index 0000000000..626ab8e9a3
--- /dev/null
+++ b/autonomy/deploy/generators/localhost/tendermint/tendermint.py
@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+# Copyright 2021-2023 Valory AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Tendermint manager."""
+import json
+import logging
+import os
+import platform
+import signal
+import subprocess # nosec:
+import sys
+from logging import Logger
+from pathlib import Path
+from threading import Event, Thread
+from typing import Any, Dict, List, Optional
+
+
+_TCP = "tcp://"
+ENCODING = "utf-8"
+DEFAULT_P2P_LISTEN_ADDRESS = f"{_TCP}0.0.0.0:26656"
+DEFAULT_RPC_LISTEN_ADDRESS = f"{_TCP}0.0.0.0:26657"
+DEFAULT_TENDERMINT_LOG_FILE = "tendermint.log"
+
+
+class StoppableThread(
+ Thread,
+):
+ """Thread class with a stop() method."""
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ """Initialise the thread."""
+ super().__init__(*args, **kwargs)
+ self._stop_event = Event()
+
+ def stop(self) -> None:
+ """Set the stop event."""
+ self._stop_event.set()
+
+ def stopped(self) -> bool:
+ """Check if the thread is stopped."""
+ return self._stop_event.is_set()
+
+
+class TendermintParams: # pylint: disable=too-few-public-methods
+ """Tendermint node parameters."""
+
+ def __init__( # pylint: disable=too-many-arguments
+ self,
+ proxy_app: str,
+ rpc_laddr: str = DEFAULT_RPC_LISTEN_ADDRESS,
+ p2p_laddr: str = DEFAULT_P2P_LISTEN_ADDRESS,
+ p2p_seeds: Optional[List[str]] = None,
+ consensus_create_empty_blocks: bool = True,
+ home: Optional[str] = None,
+ use_grpc: bool = False,
+ ):
+ """
+ Initialize the parameters to the Tendermint node.
+
+ :param proxy_app: ABCI address.
+ :param rpc_laddr: RPC address.
+ :param p2p_laddr: P2P address.
+ :param p2p_seeds: P2P seeds.
+ :param consensus_create_empty_blocks: if true, Tendermint node creates empty blocks.
+ :param home: Tendermint's home directory.
+ :param use_grpc: Whether to use a gRPC server, or TCP
+ """
+
+ self.proxy_app = proxy_app
+ self.rpc_laddr = rpc_laddr
+ self.p2p_laddr = p2p_laddr
+ self.p2p_seeds = p2p_seeds
+ self.consensus_create_empty_blocks = consensus_create_empty_blocks
+ self.home = home
+ self.use_grpc = use_grpc
+
+ def __str__(self) -> str:
+ """Get the string representation."""
+ return (
+ f"{self.__class__.__name__}("
+ f" proxy_app={self.proxy_app},\n"
+ f" rpc_laddr={self.rpc_laddr},\n"
+ f" p2p_laddr={self.p2p_laddr},\n"
+ f" p2p_seeds={self.p2p_seeds},\n"
+ f" consensus_create_empty_blocks={self.consensus_create_empty_blocks},\n"
+ f" home={self.home},\n"
+ ")"
+ )
+
+ def build_node_command(self, debug: bool = False) -> List[str]:
+ """Build the 'node' command."""
+ p2p_seeds = ",".join(self.p2p_seeds) if self.p2p_seeds else ""
+ cmd = [
+ "tendermint",
+ "node",
+ f"--proxy_app={self.proxy_app}",
+ f"--rpc.laddr={self.rpc_laddr}",
+ f"--p2p.laddr={self.p2p_laddr}",
+ f"--p2p.seeds={p2p_seeds}",
+ f"--consensus.create_empty_blocks={str(self.consensus_create_empty_blocks).lower()}",
+ f"--abci={'grpc' if self.use_grpc else 'socket'}",
+ ]
+ if debug:
+ cmd.append("--log_level=debug")
+ if self.home is not None: # pragma: nocover
+ cmd += ["--home", self.home]
+ return cmd
+
+ @staticmethod
+ def get_node_command_kwargs() -> Dict:
+ """Get the node command kwargs"""
+ kwargs = {
+ "bufsize": 1,
+ "universal_newlines": True,
+ "stdout": subprocess.PIPE,
+ "stderr": subprocess.STDOUT,
+ }
+ if platform.system() == "Windows": # pragma: nocover
+ kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore
+ else:
+ kwargs["preexec_fn"] = os.setsid # type: ignore
+ return kwargs
+
+
+class TendermintNode:
+ """A class to manage a Tendermint node."""
+
+ def __init__(
+ self,
+ params: TendermintParams,
+ logger: Optional[Logger] = None,
+ write_to_log: bool = False,
+ ):
+ """
+ Initialize a Tendermint node.
+
+ :param params: the parameters.
+ :param logger: the logger.
+ :param write_to_log: Write to log file.
+ """
+ self.params = params
+ self._process: Optional[subprocess.Popen] = None
+ self._monitoring: Optional[StoppableThread] = None
+ self._stopping = False
+ self.logger = logger or logging.getLogger()
+ self.log_file = os.environ.get("LOG_FILE", DEFAULT_TENDERMINT_LOG_FILE)
+ self.write_to_log = write_to_log
+
+ def _build_init_command(self) -> List[str]:
+ """Build the 'init' command."""
+ cmd = [
+ "tendermint",
+ "init",
+ ]
+ if self.params.home is not None: # pragma: nocover
+ cmd += ["--home", self.params.home]
+ return cmd
+
+ def init(self) -> None:
+ """Initialize Tendermint node."""
+ cmd = self._build_init_command()
+ subprocess.call(cmd) # nosec
+
+ def _monitor_tendermint_process(
+ self,
+ ) -> None:
+ """Check server status."""
+ if self._monitoring is None:
+ raise ValueError("Monitoring is not running")
+ self.log("Monitoring thread started\n")
+ while not self._monitoring.stopped():
+ try:
+ if self._process is not None and self._process.stdout is not None:
+ line = self._process.stdout.readline()
+ self.log(line)
+ for trigger in [
+ # this occurs when we lose connection from the tm side
+ "RPC HTTP server stopped",
+ # whenever the node is stopped because of a closed connection
+ # from on any of the tendermint modules (abci, p2p, rpc, etc)
+ # we restart the node
+ "Stopping abci.socketClient for error: read message: EOF",
+ ]:
+ if self._monitoring.stopped():
+ break
+ if line.find(trigger) >= 0:
+ self._stop_tm_process()
+ # we can only reach this step if monitoring was activated
+ # so we make sure that after reset the monitoring continues
+ self._start_tm_process()
+ self.log(
+ f"Restarted the HTTP RPC server, as a connection was dropped with message:\n\t\t {line}\n"
+ )
+ except Exception as e: # pylint: disable=broad-except
+ self.log(f"Error!: {str(e)}")
+ self.log("Monitoring thread terminated\n")
+
+ def _start_tm_process(self, debug: bool = False) -> None:
+ """Start a Tendermint node process."""
+ if self._process is not None or self._stopping: # pragma: nocover
+ return
+ cmd = self.params.build_node_command(debug)
+ kwargs = self.params.get_node_command_kwargs()
+ self.log(f"Starting Tendermint: {cmd}\n")
+ self._process = (
+ subprocess.Popen( # nosec # pylint: disable=consider-using-with,W1509
+ cmd, **kwargs
+ )
+ )
+ self.log("Tendermint process started\n")
+
+ def _start_monitoring_thread(self) -> None:
+ """Start a monitoring thread."""
+ self._monitoring = StoppableThread(target=self._monitor_tendermint_process)
+ self._monitoring.start()
+
+ def start(self, debug: bool = False) -> None:
+ """Start a Tendermint node process."""
+ self._start_tm_process(debug)
+ self._start_monitoring_thread()
+
+ def _stop_tm_process(self) -> None:
+ """Stop a Tendermint node process."""
+ if self._process is None or self._stopping:
+ return
+
+ self._stopping = True
+ if platform.system() == "Windows":
+ self._win_stop_tm()
+ else:
+ # this will raise an exception if the process
+ # is not terminated within the specified timeout
+ self._unix_stop_tm()
+
+ self._stopping = False
+ self._process = None
+ self.log("Tendermint process stopped\n")
+
+ def _win_stop_tm(self) -> None:
+ """Stop a Tendermint node process on Windows."""
+ os.kill(self._process.pid, signal.CTRL_C_EVENT) # type: ignore # pylint: disable=no-member
+ try:
+ self._process.wait(timeout=5) # type: ignore
+ except subprocess.TimeoutExpired: # nosec
+ os.kill(self._process.pid, signal.CTRL_BREAK_EVENT) # type: ignore # pylint: disable=no-member
+
+ def _unix_stop_tm(self) -> None:
+ """Stop a Tendermint node process on Unix."""
+ self._process.send_signal(signal.SIGTERM) # type: ignore
+ try:
+ self._process.wait(timeout=5) # type: ignore
+ except subprocess.TimeoutExpired: # nosec
+ self.log("Tendermint process did not stop gracefully\n")
+
+ # if the process is still running poll will return None
+ poll = self._process.poll() # type: ignore
+ if poll is not None:
+ return
+
+ self._process.terminate() # type: ignore
+ self._process.wait(3) # type: ignore
+
+ def _stop_monitoring_thread(self) -> None:
+ """Stop a monitoring process."""
+ if self._monitoring is not None:
+ self._monitoring.stop() # set stop event
+ self._monitoring.join()
+
+ def stop(self) -> None:
+ """Stop a Tendermint node process."""
+ self._stop_tm_process()
+ self._stop_monitoring_thread()
+
+ @staticmethod
+ def _write_to_console(line: str) -> None:
+ """Write line to console."""
+ sys.stdout.write(str(line))
+ sys.stdout.flush()
+
+ def _write_to_file(self, line: str) -> None:
+ """Write line to console."""
+ with open(self.log_file, "a", encoding=ENCODING) as file:
+ file.write(line)
+
+ def log(self, line: str) -> None:
+ """Open and write a line to the log file."""
+ self._write_to_console(line=line)
+ if self.write_to_log:
+ self._write_to_file(line=line)
+
+ def prune_blocks(self) -> int:
+ """Prune blocks from the Tendermint state"""
+ return subprocess.call( # nosec:
+ ["tendermint", "--home", str(self.params.home), "unsafe-reset-all"]
+ )
+
+ def reset_genesis_file(
+ self, genesis_time: str, initial_height: str, period_count: str
+ ) -> None:
+ """Reset genesis file."""
+
+ genesis_file = Path(str(self.params.home), "config", "genesis.json")
+ genesis_config = json.loads(genesis_file.read_text(encoding=ENCODING))
+ genesis_config["genesis_time"] = genesis_time
+ genesis_config["initial_height"] = initial_height
+ # chain id should be max 50 chars.
+ # this means that the app would theoretically break when a 40-digit period is reached
+ genesis_config["chain_id"] = f"autonolas-{period_count}"
+ genesis_file.write_text(json.dumps(genesis_config, indent=2), encoding=ENCODING)
diff --git a/autonomy/deploy/generators/localhost/utils.py b/autonomy/deploy/generators/localhost/utils.py
new file mode 100644
index 0000000000..d11c29cd69
--- /dev/null
+++ b/autonomy/deploy/generators/localhost/utils.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+# Copyright 2024 Valory AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Localhost Deployment utilities."""
+import json
+import os
+import platform
+import shutil
+import subprocess # nosec
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from aea.configurations.constants import AGENT
+
+from autonomy.chain.config import ChainType
+from autonomy.deploy.constants import (
+ BENCHMARKS_DIR,
+ TENDERMINT_BIN_UNIX,
+ TENDERMINT_BIN_WINDOWS,
+)
+
+
+LOCAL_TENDERMINT_VERSION = "0.34.19"
+
+
+def check_tendermint_version() -> Path:
+ """Check tendermint version."""
+ tendermint_executable = Path(str(shutil.which(TENDERMINT_BIN_UNIX)))
+ if platform.system() == "Windows":
+ tendermint_executable = (
+ Path(os.path.dirname(sys.executable)) / TENDERMINT_BIN_WINDOWS
+ )
+
+ if ( # check tendermint version
+ tendermint_executable is None
+ or (
+ current_version := subprocess.check_output( # nosec
+ [tendermint_executable, "version"]
+ )
+ .strip()
+ .decode()
+ )
+ != LOCAL_TENDERMINT_VERSION
+ ):
+ raise FileNotFoundError(
+ f"Please install tendermint version {LOCAL_TENDERMINT_VERSION} "
+ f"or build and run via docker by using the --docker flag."
+ + f"\nYour tendermint version is: {current_version}"
+ if current_version
+ else ""
+ )
+
+ return tendermint_executable
+
+
+def _run_aea_cmd(
+ args: List[str],
+ cwd: Optional[Path] = None,
+ stdout: Optional[int] = None,
+ stderr: Optional[int] = subprocess.PIPE,
+ ignore_error: Optional[str] = None,
+ **kwargs: Any,
+) -> None:
+ """Run an aea command in a subprocess."""
+ result = subprocess.run( # pylint: disable=subprocess-run-check # nosec
+ args=[sys.executable, "-m", "aea.cli", *args],
+ cwd=cwd,
+ stdout=stdout,
+ stderr=stderr,
+ **kwargs,
+ )
+ if result.returncode != 0:
+ result_error = result.stderr.decode()
+ if ignore_error and ignore_error not in result_error:
+ raise RuntimeError(f"Error running: {args} @ {cwd}\n{result_error}")
+
+
+def _prepare_agent_env(working_dir: Path) -> Dict[str, Any]:
+ """Prepare agent env, add keys, run aea commands."""
+ env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
+
+ # TODO: Dynamic port allocation, backport to service builder
+ env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost"
+ env["CONNECTION_ABCI_CONFIG_PORT"] = "26658"
+
+ for var in env:
+ # Fix tendermint connection params
+ if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"):
+ env[var] = "http://localhost:8080"
+
+ if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"):
+ env[var] = "http://localhost:26657"
+
+ if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"):
+ env[var] = "localhost:26656"
+
+ if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"):
+ benchmarks_dir = working_dir / BENCHMARKS_DIR
+ benchmarks_dir.mkdir(exist_ok=True, parents=True)
+ env[var] = str(benchmarks_dir.resolve())
+
+ (working_dir / "agent.json").write_text(
+ json.dumps(env, indent=2),
+ encoding="utf-8",
+ )
+
+ return env
+
+
+def setup_agent(working_dir: Path, agent_dir: Path, keys_file: Path) -> None:
+ """Setup locally deployed agent."""
+ env = _prepare_agent_env(working_dir)
+
+ _run_aea_cmd(
+ ["fetch", env["AEA_AGENT"], "--alias", AGENT],
+ cwd=working_dir,
+ )
+
+ # add private keys
+ shutil.copy(keys_file, agent_dir)
+ _run_aea_cmd(
+ ["add-key", ChainType.ETHEREUM.value],
+ cwd=agent_dir,
+ ignore_error="already present",
+ )
+ _run_aea_cmd(["issue-certificates"], cwd=agent_dir)
diff --git a/docs/api/cli/deploy.md b/docs/api/cli/deploy.md
index 5635726c15..4adbc1453c 100644
--- a/docs/api/cli/deploy.md
+++ b/docs/api/cli/deploy.md
@@ -53,12 +53,18 @@ Deploy an agent service.
default=1,
help="Number of services.",
)
+@click.option(
+ "--localhost",
+ "deployment_type",
+ flag_value=HostDeploymentGenerator.deployment_type,
+ help="Use localhost as a backend.",
+)
@click.option(
"--docker",
"deployment_type",
flag_value=DockerComposeGenerator.deployment_type,
default=True,
- help="Use docker as a backend.",
+ help="Use docker as a backend. (default)",
)
@click.option(
"--kubernetes",
@@ -140,6 +146,13 @@ Deploy an agent service.
help="Set agent memory usage limit.",
default=DEFAULT_AGENT_MEMORY_LIMIT,
)
+@click.option(
+ "--mkdir",
+ type=str,
+ help="Directory names to create in the build directory.",
+ default=[],
+ multiple=True,
+)
@registry_flag()
@password_option(confirmation_prompt=True)
@image_author_option
@@ -151,6 +164,7 @@ def build_deployment_command(
output_dir: Optional[Path],
dev_mode: bool,
registry: str,
+ mkdir: List[str],
number_of_agents: Optional[int] = None,
number_of_services: int = 1,
password: Optional[str] = None,
@@ -200,10 +214,21 @@ Build deployment setup for n agents.
default=False,
help="Run service in the background.",
)
-def run(build_dir: Path,
- no_recreate: bool,
- remove_orphans: bool,
- detach: bool = False) -> None
+@click.option(
+ "--localhost",
+ "deployment_type",
+ flag_value="localhost",
+ help="Use localhost as a backend.",
+)
+@click.option(
+ "--docker",
+ "deployment_type",
+ flag_value="docker",
+ help="Use docker as a backend. (default)",
+ default=True,
+)
+def run(build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool,
+ deployment_type: str) -> None
```
Run deployment.
diff --git a/docs/api/cli/helpers/deployment.md b/docs/api/cli/helpers/deployment.md
index 116d2c0164..8ee8eebf8b 100644
--- a/docs/api/cli/helpers/deployment.md
+++ b/docs/api/cli/helpers/deployment.md
@@ -17,6 +17,16 @@ def run_deployment(build_dir: Path,
Run deployment.
+
+
+#### run`_`host`_`deployment
+
+```python
+def run_host_deployment(build_dir: Path, detach: bool = False) -> None
+```
+
+Run host deployment.
+
#### stop`_`deployment
@@ -51,7 +61,8 @@ def build_deployment(keys_file: Path,
image_author: Optional[str] = None,
resources: Optional[Resources] = None,
service_hash_id: Optional[str] = None,
- service_offset: int = 0) -> None
+ service_offset: int = 0,
+ mkdir: Optional[List[str]] = None) -> None
```
Build deployment.
diff --git a/docs/api/deploy/generators/localhost/base.md b/docs/api/deploy/generators/localhost/base.md
new file mode 100644
index 0000000000..6adfa8c3c2
--- /dev/null
+++ b/docs/api/deploy/generators/localhost/base.md
@@ -0,0 +1,69 @@
+
+
+# autonomy.deploy.generators.localhost.base
+
+Localhost Deployment Generator.
+
+
+
+## HostDeploymentGenerator Objects
+
+```python
+class HostDeploymentGenerator(BaseDeploymentGenerator)
+```
+
+Localhost deployment.
+
+
+
+#### agent`_`dir
+
+```python
+@property
+def agent_dir() -> Path
+```
+
+Path to the agent directory.
+
+
+
+#### generate`_`config`_`tendermint
+
+```python
+def generate_config_tendermint() -> "HostDeploymentGenerator"
+```
+
+Generate tendermint configuration.
+
+
+
+#### generate
+
+```python
+def generate(image_version: t.Optional[str] = None,
+ use_hardhat: bool = False,
+ use_acn: bool = False) -> "HostDeploymentGenerator"
+```
+
+Generate agent and tendermint configurations
+
+
+
+#### populate`_`private`_`keys
+
+```python
+def populate_private_keys() -> "HostDeploymentGenerator"
+```
+
+Populate the private keys to the build directory for host mapping.
+
+
+
+#### write`_`config
+
+```python
+def write_config() -> "BaseDeploymentGenerator"
+```
+
+Write output to build dir
+
diff --git a/docs/api/deploy/generators/localhost/tendermint/app.md b/docs/api/deploy/generators/localhost/tendermint/app.md
new file mode 100644
index 0000000000..44e16adaad
--- /dev/null
+++ b/docs/api/deploy/generators/localhost/tendermint/app.md
@@ -0,0 +1,128 @@
+
+
+# autonomy.deploy.generators.localhost.tendermint.app
+
+HTTP server to control the tendermint execution environment.
+
+
+
+#### load`_`genesis
+
+```python
+def load_genesis() -> Any
+```
+
+Load genesis file.
+
+
+
+#### get`_`defaults
+
+```python
+def get_defaults() -> Dict[str, str]
+```
+
+Get defaults from genesis file.
+
+
+
+#### override`_`config`_`toml
+
+```python
+def override_config_toml() -> None
+```
+
+Update sync method.
+
+
+
+#### update`_`peers
+
+```python
+def update_peers(validators: List[Dict], config_path: Path) -> None
+```
+
+Fix peers.
+
+
+
+#### update`_`external`_`address
+
+```python
+def update_external_address(external_address: str, config_path: Path) -> None
+```
+
+Update the external address.
+
+
+
+#### update`_`genesis`_`config
+
+```python
+def update_genesis_config(data: Dict) -> None
+```
+
+Update genesis.json file for the tendermint node.
+
+
+
+## PeriodDumper Objects
+
+```python
+class PeriodDumper()
+```
+
+Dumper for tendermint data.
+
+
+
+#### `__`init`__`
+
+```python
+def __init__(logger: logging.Logger, dump_dir: Optional[Path] = None) -> None
+```
+
+Initialize object.
+
+
+
+#### readonly`_`handler
+
+```python
+@staticmethod
+def readonly_handler(func: Callable, path: str, execinfo: Any) -> None
+```
+
+If permission is readonly, we change and retry.
+
+
+
+#### dump`_`period
+
+```python
+def dump_period() -> None
+```
+
+Dump tendermint run data for replay
+
+
+
+#### create`_`app
+
+```python
+def create_app(dump_dir: Optional[Path] = None,
+ debug: bool = False) -> Tuple[Flask, TendermintNode]
+```
+
+Create the Tendermint server app
+
+
+
+#### create`_`server
+
+```python
+def create_server() -> Any
+```
+
+Function to retrieve just the app to be used by flask entry point.
+
diff --git a/docs/api/deploy/generators/localhost/tendermint/tendermint.md b/docs/api/deploy/generators/localhost/tendermint/tendermint.md
new file mode 100644
index 0000000000..739c25d315
--- /dev/null
+++ b/docs/api/deploy/generators/localhost/tendermint/tendermint.md
@@ -0,0 +1,202 @@
+
+
+# autonomy.deploy.generators.localhost.tendermint.tendermint
+
+Tendermint manager.
+
+
+
+## StoppableThread Objects
+
+```python
+class StoppableThread(Thread)
+```
+
+Thread class with a stop() method.
+
+
+
+#### `__`init`__`
+
+```python
+def __init__(*args: Any, **kwargs: Any) -> None
+```
+
+Initialise the thread.
+
+
+
+#### stop
+
+```python
+def stop() -> None
+```
+
+Set the stop event.
+
+
+
+#### stopped
+
+```python
+def stopped() -> bool
+```
+
+Check if the thread is stopped.
+
+
+
+## TendermintParams Objects
+
+```python
+class TendermintParams()
+```
+
+Tendermint node parameters.
+
+
+
+#### `__`init`__`
+
+```python
+def __init__(proxy_app: str,
+ rpc_laddr: str = DEFAULT_RPC_LISTEN_ADDRESS,
+ p2p_laddr: str = DEFAULT_P2P_LISTEN_ADDRESS,
+ p2p_seeds: Optional[List[str]] = None,
+ consensus_create_empty_blocks: bool = True,
+ home: Optional[str] = None,
+ use_grpc: bool = False)
+```
+
+Initialize the parameters to the Tendermint node.
+
+**Arguments**:
+
+- `proxy_app`: ABCI address.
+- `rpc_laddr`: RPC address.
+- `p2p_laddr`: P2P address.
+- `p2p_seeds`: P2P seeds.
+- `consensus_create_empty_blocks`: if true, Tendermint node creates empty blocks.
+- `home`: Tendermint's home directory.
+- `use_grpc`: Whether to use a gRPC server, or TCP
+
+
+
+#### `__`str`__`
+
+```python
+def __str__() -> str
+```
+
+Get the string representation.
+
+
+
+#### build`_`node`_`command
+
+```python
+def build_node_command(debug: bool = False) -> List[str]
+```
+
+Build the 'node' command.
+
+
+
+#### get`_`node`_`command`_`kwargs
+
+```python
+@staticmethod
+def get_node_command_kwargs() -> Dict
+```
+
+Get the node command kwargs
+
+
+
+## TendermintNode Objects
+
+```python
+class TendermintNode()
+```
+
+A class to manage a Tendermint node.
+
+
+
+#### `__`init`__`
+
+```python
+def __init__(params: TendermintParams,
+ logger: Optional[Logger] = None,
+ write_to_log: bool = False)
+```
+
+Initialize a Tendermint node.
+
+**Arguments**:
+
+- `params`: the parameters.
+- `logger`: the logger.
+- `write_to_log`: Write to log file.
+
+
+
+#### init
+
+```python
+def init() -> None
+```
+
+Initialize Tendermint node.
+
+
+
+#### start
+
+```python
+def start(debug: bool = False) -> None
+```
+
+Start a Tendermint node process.
+
+
+
+#### stop
+
+```python
+def stop() -> None
+```
+
+Stop a Tendermint node process.
+
+
+
+#### log
+
+```python
+def log(line: str) -> None
+```
+
+Open and write a line to the log file.
+
+
+
+#### prune`_`blocks
+
+```python
+def prune_blocks() -> int
+```
+
+Prune blocks from the Tendermint state
+
+
+
+#### reset`_`genesis`_`file
+
+```python
+def reset_genesis_file(genesis_time: str, initial_height: str,
+ period_count: str) -> None
+```
+
+Reset genesis file.
+
diff --git a/docs/api/deploy/generators/localhost/utils.md b/docs/api/deploy/generators/localhost/utils.md
new file mode 100644
index 0000000000..97a0e8b1a7
--- /dev/null
+++ b/docs/api/deploy/generators/localhost/utils.md
@@ -0,0 +1,26 @@
+
+
+# autonomy.deploy.generators.localhost.utils
+
+Localhost Deployment utilities.
+
+
+
+#### check`_`tendermint`_`version
+
+```python
+def check_tendermint_version() -> Path
+```
+
+Check tendermint version.
+
+
+
+#### setup`_`agent
+
+```python
+def setup_agent(working_dir: Path, agent_dir: Path, keys_file: Path) -> None
+```
+
+Setup locally deployed agent.
+
diff --git a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py
index 3420891da0..5a2c71c0fa 100644
--- a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py
+++ b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py
@@ -28,8 +28,18 @@
from unittest import mock
import yaml
+from aea.cli.registry.settings import REGISTRY_LOCAL
from aea.cli.utils.config import get_default_author_from_cli_config
-from aea.configurations.constants import DEFAULT_ENV_DOTFILE, PACKAGES
+from aea.cli.utils.constants import CLI_CONFIG_PATH, DEFAULT_CLI_CONFIG
+from aea.configurations.constants import (
+ CONNECTION,
+ CONTRACT,
+ DEFAULT_AEA_CONFIG_FILE,
+ DEFAULT_ENV_DOTFILE,
+ PACKAGES,
+ PROTOCOL,
+ SKILL,
+)
from aea_test_autonomy.configurations import (
ETHEREUM_ENCRYPTED_KEYS,
ETHEREUM_ENCRYPTION_PASSWORD,
@@ -39,6 +49,7 @@
DEFAULT_BUILD_FOLDER,
DEFAULT_DOCKER_IMAGE_AUTHOR,
DOCKER_COMPOSE_YAML,
+ VALORY,
)
from autonomy.deploy.base import (
DEFAULT_AGENT_CPU_LIMIT,
@@ -49,12 +60,26 @@
build_hash_id,
)
from autonomy.deploy.constants import (
+ AGENT_VARS_CONFIG_FILE,
DEBUG,
DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA,
DEPLOYMENT_KEY_DIRECTORY,
+ INFO,
KUBERNETES_AGENT_KEY_NAME,
+ PERSISTENT_DATA_DIR,
+ TENDERMINT_VARS_CONFIG_FILE,
+ TM_ENV_CREATE_EMPTY_BLOCKS,
+ TM_ENV_P2P_LADDR,
+ TM_ENV_PROXY_APP,
+ TM_ENV_RPC_LADDR,
+ TM_ENV_TMHOME,
+ TM_ENV_TMSTATE,
+ TM_ENV_USE_GRPC,
+ TM_STATE_DIR,
)
from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
+from autonomy.deploy.generators.localhost.utils import _run_aea_cmd
+from autonomy.replay.agent import ETHEREUM_PRIVATE_KEY_FILE
from tests.conftest import ROOT_DIR, skip_docker_tests
from tests.test_autonomy.base import get_dummy_service_config
@@ -145,6 +170,57 @@ def load_and_check_docker_compose_file(
return docker_compose
+ @staticmethod
+ def check_localhost_build(build_dir: Path) -> None:
+ """Check localhost build directory."""
+ build_tree = list(map(lambda x: x.name, build_dir.iterdir()))
+ assert all(
+ [
+ child in build_tree
+ for child in {
+ ".certs",
+ DEFAULT_AEA_CONFIG_FILE,
+ DEPLOYMENT_KEY_DIRECTORY,
+ ETHEREUM_PRIVATE_KEY_FILE,
+ PERSISTENT_DATA_DIR,
+ AGENT_VARS_CONFIG_FILE,
+ "data",
+ "node",
+ TENDERMINT_VARS_CONFIG_FILE,
+ }
+ ]
+ )
+
+ def load_and_check_localhost_build(self, path: Path) -> None:
+ """Load localhost build config."""
+ with open(path / TENDERMINT_VARS_CONFIG_FILE, "r", encoding="utf-8") as fp:
+ assert json.load(fp) == {
+ TM_ENV_TMHOME: (
+ self.t / "register_reset" / DEFAULT_BUILD_FOLDER / "node"
+ ).as_posix(),
+ TM_ENV_TMSTATE: (
+ self.t / "register_reset" / DEFAULT_BUILD_FOLDER / TM_STATE_DIR
+ ).as_posix(),
+ TM_ENV_P2P_LADDR: "tcp://localhost:26656",
+ TM_ENV_RPC_LADDR: "tcp://localhost:26657",
+ TM_ENV_PROXY_APP: "tcp://localhost:26658",
+ TM_ENV_CREATE_EMPTY_BLOCKS: "true",
+ TM_ENV_USE_GRPC: "false",
+ }
+ with open(path / AGENT_VARS_CONFIG_FILE, "r", encoding="utf-8") as fp:
+ assert json.load(fp) == {
+ "ID": "0",
+ "AEA_AGENT": "valory/register_reset:0.1.0:bafybeia4pxlphcvco3ttlv3tytklriwvuwxxxr5m2tdry32yc5vogxtm7u",
+ "LOG_LEVEL": INFO,
+ "AEA_PASSWORD": "",
+ "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_ADDRESS": "http://host.docker.internal:8545",
+ "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_CHAIN_ID": "31337",
+ "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_POA_CHAIN": "False",
+ "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_DEFAULT_GAS_PRICE_STRATEGY": "eip1559",
+ "CONNECTION_ABCI_CONFIG_HOST": "localhost",
+ "CONNECTION_ABCI_CONFIG_PORT": "26658",
+ }
+
@staticmethod
def check_docker_compose_build(
build_dir: Path,
@@ -163,6 +239,73 @@ def check_docker_compose_build(
)
+class TestLocalhostBuilds(BaseDeployBuildTest):
+ """Test localhost builds."""
+
+ def setup(self) -> None:
+ """Setup test for localhost deployment."""
+ super().setup()
+ shutil.copy(
+ ROOT_DIR
+ / PACKAGES
+ / VALORY
+ / "agents"
+ / "register_reset"
+ / DEFAULT_AEA_CONFIG_FILE,
+ self.t / "register_reset",
+ )
+ aea_cli_config = DEFAULT_CLI_CONFIG
+ aea_cli_config["registry_config"]["settings"][REGISTRY_LOCAL][
+ "default_packages_path"
+ ] = (ROOT_DIR / PACKAGES).as_posix()
+ Path(CLI_CONFIG_PATH).write_text(yaml.dump(aea_cli_config))
+
+ with open(self.t / "register_reset" / DEFAULT_AEA_CONFIG_FILE, "r") as fp:
+ agent_config = next(yaml.safe_load_all(fp))
+ agent_config["private_key_paths"]["ethereum"] = ETHEREUM_PRIVATE_KEY_FILE
+ with open(self.t / "register_reset" / DEFAULT_AEA_CONFIG_FILE, "w") as fp:
+ yaml.dump(agent_config, fp)
+
+ # add all the components
+ for component_type in (CONNECTION, CONTRACT, SKILL, PROTOCOL):
+ for component_name in agent_config[component_type + "s"]:
+ _run_aea_cmd(
+ [
+ "--skip-consistency-check",
+ "add",
+ component_type,
+ component_name,
+ "--mixed",
+ ],
+ cwd=self.t / "register_reset",
+ ignore_error="already exists",
+ )
+
+ # prepare ethereum private key
+ with open(self.t / "register_reset" / ETHEREUM_PRIVATE_KEY_FILE, "w") as fp:
+ fp.write( # mock private key
+ "0x0000000000000000000000000000000000000000000000000000000000000001"
+ )
+
+ def test_localhost_build(
+ self,
+ ) -> None:
+ """Test that the build command works."""
+
+ build_dir = self.t / "register_reset" / DEFAULT_BUILD_FOLDER
+ with mock.patch("os.chown"), OS_ENV_PATCH:
+ result = self.run_cli(
+ (str(self.keys_file), "--localhost", "--mkdir", "data")
+ )
+
+ assert result.exit_code == 0, result.output
+ assert build_dir.exists()
+ assert (build_dir / "data").exists()
+
+ self.check_localhost_build(build_dir=build_dir)
+ self.load_and_check_localhost_build(path=build_dir)
+
+
class TestDockerComposeBuilds(BaseDeployBuildTest):
"""Test docker-compose build."""
diff --git a/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py
index e1b57e7a66..1679db0738 100644
--- a/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py
+++ b/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py
@@ -20,12 +20,25 @@
"""Test `run` command."""
+import json
import os
+import shutil
from unittest import mock
-from autonomy.constants import DOCKER_COMPOSE_YAML
+from aea.configurations.constants import DEFAULT_AEA_CONFIG_FILE, PACKAGES
+from autonomy.constants import DEFAULT_BUILD_FOLDER, DOCKER_COMPOSE_YAML, VALORY
+from autonomy.deploy.base import ServiceBuilder
+from autonomy.deploy.constants import (
+ AGENT_VARS_CONFIG_FILE,
+ TENDERMINT_VARS_CONFIG_FILE,
+)
+
+from tests.conftest import ROOT_DIR
from tests.test_autonomy.test_cli.base import BaseCliTest
+from tests.test_autonomy.test_cli.test_deploy.test_build.test_deployment import (
+ OS_ENV_PATCH,
+)
class TestRun(BaseCliTest):
@@ -51,6 +64,51 @@ def test_run(
assert result.exit_code == 0, result.output
assert "Running build @" in result.output
+ def test_run_local(self) -> None:
+ """Test that `deploy run` works on localhost."""
+ super().setup()
+
+ # setup the service keys and packages
+ self.keys_file = self.t / "keys.json"
+ shutil.copytree(ROOT_DIR / PACKAGES, self.t / PACKAGES)
+ shutil.copy(
+ ROOT_DIR / "deployments" / "keys" / "hardhat_keys.json", self.keys_file
+ )
+ shutil.copytree(
+ self.t / PACKAGES / "valory" / "services" / "register_reset",
+ self.t / "register_reset",
+ )
+ with OS_ENV_PATCH:
+ self.spec = ServiceBuilder.from_dir(
+ self.t / "register_reset",
+ self.keys_file,
+ )
+ os.chdir(self.t / "register_reset")
+
+ # setup aea-config.yaml
+ shutil.copy(
+ ROOT_DIR
+ / PACKAGES
+ / VALORY
+ / "agents"
+ / "register_reset"
+ / DEFAULT_AEA_CONFIG_FILE,
+ self.t / "register_reset",
+ )
+
+ # setup agent.json and tendermint.json
+ os.mkdir(build_path := self.t / "register_reset" / DEFAULT_BUILD_FOLDER)
+ with open(build_path / TENDERMINT_VARS_CONFIG_FILE, "w") as fp:
+ json.dump({}, fp)
+ with open(build_path / AGENT_VARS_CONFIG_FILE, "w") as fp:
+ json.dump({}, fp)
+ with mock.patch("autonomy.cli.helpers.deployment.subprocess.run"), mock.patch(
+ "autonomy.cli.helpers.deployment.subprocess.Popen"
+ ), mock.patch("autonomy.cli.helpers.deployment.check_tendermint_version"):
+ result = self.run_cli(("--localhost", "--build-dir", build_path.as_posix()))
+ assert result.exit_code == 0, result.output
+ assert "Running build @" in result.output
+
def test_missing_config_file(
self,
) -> None:
diff --git a/tests/test_autonomy/test_deploy/test_deployment_generators.py b/tests/test_autonomy/test_deploy/test_deployment_generators.py
index 5b2de7fccb..ab5f0ea036 100644
--- a/tests/test_autonomy/test_deploy/test_deployment_generators.py
+++ b/tests/test_autonomy/test_deploy/test_deployment_generators.py
@@ -33,6 +33,7 @@
from autonomy.deploy.base import BaseDeploymentGenerator, ServiceBuilder
from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator
from tests.conftest import ROOT_DIR
@@ -52,7 +53,10 @@ def get_dummy_service() -> Service:
@skip_docker_tests
-@pytest.mark.parametrize("generator_cls", (DockerComposeGenerator, KubernetesGenerator))
+@pytest.mark.parametrize(
+ "generator_cls",
+ (DockerComposeGenerator, KubernetesGenerator, HostDeploymentGenerator),
+)
@pytest.mark.parametrize("image_version", [None, "0.1.0"])
@pytest.mark.parametrize("use_hardhat", [False, True])
@pytest.mark.parametrize("use_acn", [False, True])
@@ -84,5 +88,9 @@ def test_versioning(
)
deployment_generator.generate(**generate_kwargs)
- expected = f"valory/oar-oracle:{image_version or AGENT.hash}"
+ oar_image = "oar-"
+ if generator_cls == HostDeploymentGenerator:
+ oar_image = ""
+ image_version = f"latest:{AGENT.hash}"
+ expected = f"valory/{oar_image}oracle:{image_version or AGENT.hash}"
assert expected in deployment_generator.output
diff --git a/tests/test_docs/test_commands.py b/tests/test_docs/test_commands.py
index cef4d4a03e..196c9c6e3c 100644
--- a/tests/test_docs/test_commands.py
+++ b/tests/test_docs/test_commands.py
@@ -168,6 +168,7 @@ def test_validate_doc_commands() -> None:
if cmd in skips:
continue
+ print(cmd)
assert validator.validate(cmd, str(file_)), cmd