diff --git a/autonomy/cli/deploy.py b/autonomy/cli/deploy.py index 5df9376ff2..0058100f33 100644 --- a/autonomy/cli/deploy.py +++ b/autonomy/cli/deploy.py @@ -22,7 +22,7 @@ import os import shutil from pathlib import Path -from typing import Optional, cast +from typing import List, Optional, cast import click from aea import AEA_DIR @@ -40,6 +40,7 @@ build_and_deploy_from_token, build_deployment, run_deployment, + run_host_deployment, stop_deployment, ) from autonomy.cli.helpers.env import load_env_file @@ -60,6 +61,7 @@ from autonomy.deploy.constants import INFO, LOGGING_LEVELS from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator def _validate_packages_path(path: Optional[Path] = None) -> Path: @@ -130,12 +132,18 @@ def deploy_group( default=1, help="Number of services.", ) +@click.option( + "--localhost", + "deployment_type", + flag_value=HostDeploymentGenerator.deployment_type, + help="Use localhost as a backend.", +) @click.option( "--docker", "deployment_type", flag_value=DockerComposeGenerator.deployment_type, default=True, - help="Use docker as a backend.", + help="Use docker as a backend. (default)", ) @click.option( "--kubernetes", @@ -215,6 +223,13 @@ def deploy_group( help="Set agent memory usage limit.", default=DEFAULT_AGENT_MEMORY_LIMIT, ) +@click.option( + "--mkdir", + type=str, + help="Directory names to create in the build directory.", + default=[], + multiple=True, +) @registry_flag() @password_option(confirmation_prompt=True) @image_author_option @@ -226,6 +241,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo output_dir: Optional[Path], dev_mode: bool, registry: str, + mkdir: List[str], number_of_agents: Optional[int] = None, number_of_services: int = 1, password: Optional[str] = None, @@ -290,6 +306,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo use_acn=use_acn, use_tm_testnet_setup=use_tm_testnet_setup, image_author=image_author, + mkdir=mkdir, resources={ "agent": { "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit}, @@ -329,16 +346,42 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo default=False, help="Run service in the background.", ) +@click.option( + "--localhost", + "deployment_type", + flag_value="localhost", + help="Use localhost as a backend.", +) +@click.option( + "--docker", + "deployment_type", + flag_value="docker", + help="Use docker as a backend. (default)", + default=True, +) def run( - build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool = False + build_dir: Path, + no_recreate: bool, + remove_orphans: bool, + detach: bool, + deployment_type: str, ) -> None: """Run deployment.""" build_dir = Path(build_dir or Path.cwd()).absolute() - if not (build_dir / DockerComposeGenerator.output_name).exists(): + deployment = ( + HostDeploymentGenerator + if deployment_type == "localhost" + else DockerComposeGenerator + ) + if not (build_dir / deployment.output_name).exists(): raise click.ClickException( f"Deployment configuration does not exist @ {build_dir}" ) - run_deployment(build_dir, no_recreate, remove_orphans, detach=detach) + click.echo(f"Running build @ {build_dir}") + if deployment_type == "localhost": + run_host_deployment(build_dir, detach) + else: + run_deployment(build_dir, no_recreate, remove_orphans, detach=detach) @deploy_group.command(name="stop") diff --git a/autonomy/cli/helpers/deployment.py b/autonomy/cli/helpers/deployment.py index 8853124b59..30bf805679 100644 --- a/autonomy/cli/helpers/deployment.py +++ b/autonomy/cli/helpers/deployment.py @@ -18,13 +18,18 @@ # ------------------------------------------------------------------------------ """Deployment helpers.""" +import json import os +import platform import shutil +import subprocess # nosec +import sys import time from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Tuple import click +from aea.configurations.constants import AGENT from aea.configurations.data_types import PublicId from aea.helpers.base import cd from compose.cli import main as docker_compose @@ -45,20 +50,27 @@ from autonomy.deploy.build import generate_deployment from autonomy.deploy.constants import ( AGENT_KEYS_DIR, + AGENT_VARS_CONFIG_FILE, BENCHMARKS_DIR, + DEATTACH_WINDOWS_FLAG, INFO, LOG_DIR, PERSISTENT_DATA_DIR, + TENDERMINT_FLASK_APP_PATH, + TENDERMINT_VARS_CONFIG_FILE, TM_STATE_DIR, VENVS_DIR, ) from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.utils import check_tendermint_version from autonomy.deploy.image import build_image -def _build_dirs(build_dir: Path) -> None: +def _build_dirs(build_dir: Path, mkdir: List[str]) -> None: """Build necessary directories.""" + mkdirs = [(new_dir_name,) for new_dir_name in mkdir] + for dir_path in [ (PERSISTENT_DATA_DIR,), (PERSISTENT_DATA_DIR, LOG_DIR), @@ -66,9 +78,9 @@ def _build_dirs(build_dir: Path) -> None: (PERSISTENT_DATA_DIR, BENCHMARKS_DIR), (PERSISTENT_DATA_DIR, VENVS_DIR), (AGENT_KEYS_DIR,), - ]: + ] + mkdirs: path = Path(build_dir, *dir_path) - path.mkdir() + path.mkdir(exist_ok=True, parents=True) # TOFIX: remove this safely try: os.chown(path, 1000, 1000) @@ -119,7 +131,6 @@ def run_deployment( detach: bool = False, ) -> None: """Run deployment.""" - click.echo(f"Running build @ {build_dir}") try: project = _load_compose_project(build_dir=build_dir) commands = docker_compose.TopLevelCommand(project=project) @@ -163,6 +174,65 @@ def run_deployment( stop_deployment(build_dir=build_dir) +def _get_deattached_creation_flags() -> int: + """Get Popen creation flag based on the platform.""" + return DEATTACH_WINDOWS_FLAG if platform.system() == "Windows" else 0 + + +def _start_localhost_agent(working_dir: Path, detach: bool) -> None: + """Start localhost agent process.""" + env = json.loads((working_dir / AGENT_VARS_CONFIG_FILE).read_text()) + process_fn: Callable = subprocess.Popen if detach else subprocess.run # type: ignore[assignment] + process = process_fn( # pylint: disable=subprocess-run-check # nosec + args=[sys.executable, "-m", "aea.cli", "run"], + cwd=working_dir / AGENT, + env={**os.environ, **env}, + creationflags=_get_deattached_creation_flags(), # Detach process from the main process + ) + (working_dir / "agent.pid").write_text( + data=str(process.pid), + ) + + +def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen: + """Start localhost tendermint process.""" + check_tendermint_version() + env = json.loads((working_dir / TENDERMINT_VARS_CONFIG_FILE).read_text()) + flask_app_path = Path(__file__).parents[3] / TENDERMINT_FLASK_APP_PATH + process = subprocess.Popen( # pylint: disable=consider-using-with # nosec + args=[ + "flask", + "run", + "--host", + "localhost", + "--port", + "8080", + ], + cwd=working_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env={**os.environ, **env, "FLASK_APP": f"{flask_app_path}:create_server"}, + creationflags=_get_deattached_creation_flags(), # Detach process from the main process + ) + (working_dir / "tendermint.pid").write_text( + data=str(process.pid), + ) + return process + + +def run_host_deployment(build_dir: Path, detach: bool = False) -> None: + """Run host deployment.""" + tm_process = _start_localhost_tendermint(build_dir) + try: + _start_localhost_agent(build_dir, detach) + except Exception as e: # pylint: disable=broad-except + click.echo(e) + tm_process.terminate() + finally: + if not detach: + tm_process.terminate() + + def stop_deployment(build_dir: Path) -> None: """Stop running deployment.""" try: @@ -196,6 +266,7 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals resources: Optional[Resources] = None, service_hash_id: Optional[str] = None, service_offset: int = 0, + mkdir: Optional[List[str]] = None, ) -> None: """Build deployment.""" @@ -209,8 +280,6 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals click.echo(f"Building deployment @ {build_dir}") build_dir.mkdir() - _build_dirs(build_dir) - if service_hash_id is None: service_hash_id = build_hash_id() @@ -237,6 +306,9 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals image_author=image_author, resources=resources, ) + if mkdir is not None: + _build_dirs(build_dir, mkdir) + click.echo(report) diff --git a/autonomy/deploy/base.py b/autonomy/deploy/base.py index 1ea737f508..49d31b1cac 100644 --- a/autonomy/deploy/base.py +++ b/autonomy/deploy/base.py @@ -66,6 +66,7 @@ KUBERNETES_DEPLOYMENT = "kubernetes" DOCKER_COMPOSE_DEPLOYMENT = "docker-compose" +LOCALHOST_DEPLOYMENT = "localhost" LOOPBACK = "127.0.0.1" LOCALHOST = "localhost" @@ -466,14 +467,14 @@ def _try_update_tendermint_params( """Try update the tendermint parameters""" is_kubernetes_deployment = self.deplopyment_type == KUBERNETES_DEPLOYMENT + is_localhost_deployment = self.deplopyment_type == LOCALHOST_DEPLOYMENT def _update_tendermint_params( param_args: Dict, idx: int, - is_kubernetes_deployment: bool = False, ) -> None: """Update tendermint params""" - if is_kubernetes_deployment: + if is_kubernetes_deployment or is_localhost_deployment: param_args[TENDERMINT_URL_PARAM] = TENDERMINT_NODE_LOCAL param_args[TENDERMINT_COM_URL_PARAM] = TENDERMINT_COM_LOCAL else: @@ -504,7 +505,6 @@ def _update_tendermint_params( _update_tendermint_params( param_args=param_args, idx=0, - is_kubernetes_deployment=is_kubernetes_deployment, ) else: param_args = self._get_config_from_json_path( @@ -513,7 +513,6 @@ def _update_tendermint_params( _update_tendermint_params( param_args=param_args, idx=0, - is_kubernetes_deployment=is_kubernetes_deployment, ) return @@ -530,7 +529,6 @@ def _update_tendermint_params( _update_tendermint_params( param_args=param_args, idx=agent_idx, - is_kubernetes_deployment=is_kubernetes_deployment, ) except KeyError: # pragma: nocover logging.warning( @@ -611,9 +609,9 @@ def _update_abci_connection_config( processed_overrides = deepcopy(overrides) if self.service.number_of_agents == 1: processed_overrides["config"]["host"] = ( - LOOPBACK - if self.deplopyment_type == KUBERNETES_DEPLOYMENT - else self.get_abci_container_name(index=0) + self.get_abci_container_name(index=0) + if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT + else LOOPBACK ) processed_overrides["config"]["port"] = processed_overrides["config"].get( "port", DEFAULT_ABCI_PORT @@ -627,9 +625,9 @@ def _update_abci_connection_config( for idx, override in processed_overrides.items(): override["config"]["host"] = ( - LOOPBACK - if self.deplopyment_type == KUBERNETES_DEPLOYMENT - else self.get_abci_container_name(index=idx) + self.get_abci_container_name(index=idx) + if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT + else LOOPBACK ) override["config"]["port"] = override["config"].get( "port", DEFAULT_ABCI_PORT diff --git a/autonomy/deploy/build.py b/autonomy/deploy/build.py index df65f13d9e..d6b11d94da 100644 --- a/autonomy/deploy/build.py +++ b/autonomy/deploy/build.py @@ -24,11 +24,13 @@ from autonomy.deploy.constants import DEPLOYMENT_REPORT, INFO from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator DEPLOYMENT_OPTIONS: Dict[str, Type[BaseDeploymentGenerator]] = { "kubernetes": KubernetesGenerator, "docker-compose": DockerComposeGenerator, + "localhost": HostDeploymentGenerator, } @@ -56,9 +58,14 @@ def generate_deployment( # pylint: disable=too-many-arguments, too-many-locals resources: Optional[Resources] = None, ) -> str: """Generate the deployment for the service.""" - if dev_mode and type_of_deployment != DockerComposeGenerator.deployment_type: + if type_of_deployment == HostDeploymentGenerator.deployment_type: + if number_of_agents is not None and number_of_agents > 1: + raise RuntimeError( + "Host deployment currently only supports single agent deployments" + ) + elif dev_mode: raise RuntimeError( - "Development mode can only be used with docker-compose deployments" + "Development mode can only be used with localhost deployments" ) service_builder = ServiceBuilder.from_dir( diff --git a/autonomy/deploy/constants.py b/autonomy/deploy/constants.py index 00c553b647..ccb8bff62e 100644 --- a/autonomy/deploy/constants.py +++ b/autonomy/deploy/constants.py @@ -19,6 +19,7 @@ """Constants for generating deployments environment.""" +from pathlib import Path from string import Template @@ -34,6 +35,23 @@ DEPLOYMENT_KEY_DIRECTORY = "agent_keys" DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA = "agent_{agent_n}" KUBERNETES_AGENT_KEY_NAME = DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA + "_private_key.yaml" +TENDERMINT_BIN_UNIX = "tendermint" +TENDERMINT_BIN_WINDOWS = "tendermint.exe" +TENDERMINT_VARS_CONFIG_FILE = "tendermint.json" +AGENT_VARS_CONFIG_FILE = "agent.json" +TENDERMINT_FLASK_APP_PATH = ( + Path("autonomy") / "deploy" / "generators" / "localhost" / "tendermint" / "app.py" +) +DEATTACH_WINDOWS_FLAG = 0x00000008 + +TM_ENV_TMHOME = "TMHOME" +TM_ENV_TMSTATE = "TMSTATE" +TM_ENV_PROXY_APP = "PROXY_APP" +TM_ENV_P2P_LADDR = "P2P_LADDR" +TM_ENV_RPC_LADDR = "RPC_LADDR" +TM_ENV_PROXY_APP = "PROXY_APP" +TM_ENV_CREATE_EMPTY_BLOCKS = "CREATE_EMPTY_BLOCKS" +TM_ENV_USE_GRPC = "USE_GRPC" DEFAULT_ENCODING = "utf-8" diff --git a/autonomy/deploy/generators/localhost/__init__.py b/autonomy/deploy/generators/localhost/__init__.py new file mode 100644 index 0000000000..d0103678f9 --- /dev/null +++ b/autonomy/deploy/generators/localhost/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2024 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Localhost Deployment Generator.""" diff --git a/autonomy/deploy/generators/localhost/base.py b/autonomy/deploy/generators/localhost/base.py new file mode 100644 index 0000000000..07825db954 --- /dev/null +++ b/autonomy/deploy/generators/localhost/base.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2024 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Localhost Deployment Generator.""" +import json +import subprocess # nosec +import typing as t +from pathlib import Path + +from aea.configurations.constants import ( + DEFAULT_LEDGER, + LEDGER, + PRIVATE_KEY, + PRIVATE_KEY_PATH_SCHEMA, +) + +from autonomy.deploy.base import BaseDeploymentGenerator +from autonomy.deploy.constants import ( + AGENT_VARS_CONFIG_FILE, + DEFAULT_ENCODING, + TENDERMINT_VARS_CONFIG_FILE, + TM_ENV_CREATE_EMPTY_BLOCKS, + TM_ENV_P2P_LADDR, + TM_ENV_PROXY_APP, + TM_ENV_RPC_LADDR, + TM_ENV_TMHOME, + TM_ENV_TMSTATE, + TM_ENV_USE_GRPC, + TM_STATE_DIR, +) +from autonomy.deploy.generators.localhost.utils import ( + check_tendermint_version, + setup_agent, +) + + +class HostDeploymentGenerator(BaseDeploymentGenerator): + """Localhost deployment.""" + + output_name: str = AGENT_VARS_CONFIG_FILE + deployment_type: str = "localhost" + + @property + def agent_dir(self) -> Path: + """Path to the agent directory.""" + return self.build_dir / "agent" + + def generate_config_tendermint(self) -> "HostDeploymentGenerator": + """Generate tendermint configuration.""" + tmhome = str(self.build_dir / "node") + tendermint_executable = check_tendermint_version() + # if executable found, setup its configs + subprocess.run( # pylint: disable=subprocess-run-check # nosec + args=[ + tendermint_executable, + "--home", + tmhome, + "init", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # TODO: Dynamic port allocation + params = { + TM_ENV_TMHOME: tmhome, + TM_ENV_TMSTATE: str(self.build_dir / TM_STATE_DIR), + TM_ENV_P2P_LADDR: "tcp://localhost:26656", + TM_ENV_RPC_LADDR: "tcp://localhost:26657", + TM_ENV_PROXY_APP: "tcp://localhost:26658", + TM_ENV_CREATE_EMPTY_BLOCKS: "true", + TM_ENV_USE_GRPC: "false", + } + (self.build_dir / TENDERMINT_VARS_CONFIG_FILE).write_text( + json.dumps(params, indent=2), + ) + + return self + + def generate( + self, + image_version: t.Optional[str] = None, + use_hardhat: bool = False, + use_acn: bool = False, + ) -> "HostDeploymentGenerator": + """Generate agent and tendermint configurations""" + agent = self.service_builder.generate_agent(agent_n=0) + self.output = json.dumps( + {key: f"{value}" for key, value in agent.items()}, indent=2 + ) + (self.build_dir / self.output_name).write_text( + json.dumps(self.output, indent=2), + ) + + return self + + def _populate_keys(self) -> None: + """Populate the keys directory""" + kp, *_ = t.cast(t.List[t.Dict[str, str]], self.service_builder.keys) + key = kp[PRIVATE_KEY] + ledger = kp.get(LEDGER, DEFAULT_LEDGER) + keys_file = self.build_dir / PRIVATE_KEY_PATH_SCHEMA.format(ledger) + keys_file.write_text(key, encoding=DEFAULT_ENCODING) + setup_agent(self.build_dir, self.agent_dir, keys_file) + + def _populate_keys_multiledger(self) -> None: + """Populate the keys directory with multiple set of keys""" + + def populate_private_keys(self) -> "HostDeploymentGenerator": + """Populate the private keys to the build directory for host mapping.""" + if self.service_builder.multiledger: + self._populate_keys_multiledger() + else: + self._populate_keys() + return self + + def write_config(self) -> "BaseDeploymentGenerator": + """Write output to build dir""" + super().write_config() + return self diff --git a/autonomy/deploy/generators/localhost/tendermint/__init__.py b/autonomy/deploy/generators/localhost/tendermint/__init__.py new file mode 100644 index 0000000000..184e128148 --- /dev/null +++ b/autonomy/deploy/generators/localhost/tendermint/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2021-2024 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Tendermint flask app.""" diff --git a/autonomy/deploy/generators/localhost/tendermint/app.py b/autonomy/deploy/generators/localhost/tendermint/app.py new file mode 100644 index 0000000000..2154181f62 --- /dev/null +++ b/autonomy/deploy/generators/localhost/tendermint/app.py @@ -0,0 +1,355 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2021-2022 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""HTTP server to control the tendermint execution environment.""" +import json +import logging +import os +import re +import shutil +import stat +import traceback +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, cast + +import requests +from flask import Flask, Response, jsonify, request +from werkzeug.exceptions import InternalServerError, NotFound + +from autonomy.deploy.constants import ( + TM_ENV_CREATE_EMPTY_BLOCKS, + TM_ENV_P2P_LADDR, + TM_ENV_PROXY_APP, + TM_ENV_RPC_LADDR, + TM_ENV_TMHOME, + TM_ENV_USE_GRPC, +) + + +try: + from .tendermint import ( # type: ignore + DEFAULT_P2P_LISTEN_ADDRESS, + DEFAULT_RPC_LISTEN_ADDRESS, + TendermintNode, + TendermintParams, + ) +except ImportError: + from tendermint import ( # type: ignore + DEFAULT_P2P_LISTEN_ADDRESS, + DEFAULT_RPC_LISTEN_ADDRESS, + TendermintNode, + TendermintParams, + ) + +ENCODING = "utf-8" +DEFAULT_LOG_FILE = "log.log" +IS_DEV_MODE = os.environ.get("DEV_MODE", "0") == "1" +CONFIG_OVERRIDE = [ + ("fast_sync = true", "fast_sync = false"), + ("max_num_outbound_peers = 10", "max_num_outbound_peers = 0"), + ("pex = true", "pex = false"), +] +DOCKER_INTERNAL_HOST = "host.docker.internal" +TM_STATUS_ENDPOINT = "http://localhost:26657/status" + +logging.basicConfig( + filename=os.environ.get("LOG_FILE", DEFAULT_LOG_FILE), + level=logging.DEBUG, + format="%(asctime)s %(levelname)s %(name)s %(threadName)s : %(message)s", # noqa : W1309 +) + + +def load_genesis() -> Any: + """Load genesis file.""" + return json.loads( + Path(os.environ["TMHOME"], "config", "genesis.json").read_text( + encoding=ENCODING + ) + ) + + +def get_defaults() -> Dict[str, str]: + """Get defaults from genesis file.""" + genesis = load_genesis() + return dict(genesis_time=genesis.get("genesis_time")) + + +def override_config_toml() -> None: + """Update sync method.""" + + config_path = str(Path(os.environ["TMHOME"]) / "config" / "config.toml") + logging.info(config_path) + with open(config_path, "r", encoding=ENCODING) as fp: + config = fp.read() + + for old, new in CONFIG_OVERRIDE: + config = config.replace(old, new) + + with open(config_path, "w+", encoding=ENCODING) as fp: + fp.write(config) + + +def update_peers(validators: List[Dict], config_path: Path) -> None: + """Fix peers.""" + + config_text = config_path.read_text(encoding="utf-8") + + new_peer_string = 'persistent_peers = "' + for peer in validators: + hostname = peer["hostname"] + if hostname in ("localhost", "0.0.0.0"): # nosec + # This (tendermint) node will be running in a docker container and no other node + # will be running in the same container. If we receive either localhost or 0.0.0.0, + # we make an assumption that the address belongs to a node running on the + # same machine with a different docker container and different p2p port so, + # we replace the hostname with the docker's internal host url. + hostname = DOCKER_INTERNAL_HOST + new_peer_string += ( + peer["peer_id"] + "@" + hostname + ":" + str(peer["p2p_port"]) + "," + ) + new_peer_string = new_peer_string[:-1] + '"\n' + + updated_config = re.sub('persistent_peers = ".*\n', new_peer_string, config_text) + config_path.write_text(updated_config, encoding="utf-8") + + +def update_external_address(external_address: str, config_path: Path) -> None: + """Update the external address.""" + config_text = config_path.read_text(encoding="utf-8") + new_external_address = f'external_address = "{external_address}"\n' + updated_config = re.sub( + 'external_address = ".*\n', new_external_address, config_text + ) + config_path.write_text(updated_config, encoding="utf-8") + + +def update_genesis_config(data: Dict) -> None: + """Update genesis.json file for the tendermint node.""" + + genesis_file = Path(os.environ["TMHOME"]) / "config" / "genesis.json" + genesis_data = {} + genesis_data["genesis_time"] = data["genesis_config"]["genesis_time"] + genesis_data["chain_id"] = data["genesis_config"]["chain_id"] + genesis_data["initial_height"] = "0" + genesis_data["consensus_params"] = data["genesis_config"]["consensus_params"] + genesis_data["validators"] = [ + { + "address": validator["address"], + "pub_key": validator["pub_key"], + "power": validator["power"], + "name": validator["name"], + } + for validator in data["validators"] + ] + genesis_data["app_hash"] = "" + genesis_file.write_text(json.dumps(genesis_data, indent=2), encoding=ENCODING) + + +class PeriodDumper: + """Dumper for tendermint data.""" + + resets: int + dump_dir: Path + logger: logging.Logger + + def __init__(self, logger: logging.Logger, dump_dir: Optional[Path] = None) -> None: + """Initialize object.""" + + self.resets = 0 + self.logger = logger + self.dump_dir = dump_dir or Path(os.environ.get("TMSTATE") or "/tm_state") + + if self.dump_dir.is_dir(): + shutil.rmtree(str(self.dump_dir), onerror=self.readonly_handler) + self.dump_dir.mkdir(exist_ok=True) + + @staticmethod + def readonly_handler( + func: Callable, path: str, execinfo: Any # pylint: disable=unused-argument + ) -> None: + """If permission is readonly, we change and retry.""" + try: + os.chmod(path, stat.S_IWRITE) + func(path) + except (FileNotFoundError, OSError): + return + + def dump_period(self) -> None: + """Dump tendermint run data for replay""" + store_dir = self.dump_dir / f"period_{self.resets}" + store_dir.mkdir(exist_ok=True) + try: + shutil.copytree( + os.environ["TMHOME"], str(store_dir / ("node" + os.environ["ID"])) + ) + self.logger.info(f"Dumped data for period {self.resets}") + except OSError as e: + self.logger.info( + f"Error occurred while dumping data for period {self.resets}: {e}" + ) + self.resets += 1 + + +def create_app( # pylint: disable=too-many-statements + dump_dir: Optional[Path] = None, + debug: bool = False, +) -> Tuple[Flask, TendermintNode]: + """Create the Tendermint server app""" + write_to_log = os.environ.get("WRITE_TO_LOG", "false").lower() == "true" + tendermint_params = TendermintParams( + proxy_app=os.environ[TM_ENV_PROXY_APP], + rpc_laddr=os.environ.get(TM_ENV_RPC_LADDR, DEFAULT_RPC_LISTEN_ADDRESS), + p2p_laddr=os.environ.get(TM_ENV_P2P_LADDR, DEFAULT_P2P_LISTEN_ADDRESS), + consensus_create_empty_blocks=os.environ[TM_ENV_CREATE_EMPTY_BLOCKS] == "true", + home=os.environ[TM_ENV_TMHOME], + use_grpc=os.environ[TM_ENV_USE_GRPC] == "true", + ) + + app = Flask(__name__) + period_dumper = PeriodDumper(logger=app.logger, dump_dir=dump_dir) + tendermint_node = TendermintNode( + tendermint_params, + logger=app.logger, + write_to_log=write_to_log, + ) + tendermint_node.init() + override_config_toml() + tendermint_node.start(debug=debug) + + @app.get("/params") + def get_params() -> Dict: + """Get tendermint params.""" + try: + priv_key_file = ( + Path(os.environ["TMHOME"]) / "config" / "priv_validator_key.json" + ) + priv_key_data = json.loads(priv_key_file.read_text(encoding=ENCODING)) + del priv_key_data["priv_key"] + status = requests.get(TM_STATUS_ENDPOINT).json() + priv_key_data["peer_id"] = status["result"]["node_info"]["id"] + return { + "params": priv_key_data, + "status": True, + "error": None, + } + except (FileNotFoundError, json.JSONDecodeError): + return {"params": {}, "status": False, "error": traceback.format_exc()} + + @app.post("/params") + def update_params() -> Dict: + """Update validator params.""" + + try: + data: Dict = json.loads(request.get_data().decode(ENCODING)) + cast(logging.Logger, app.logger).debug( # pylint: disable=no-member + f"Data update requested with data={data}" + ) + + cast(logging.Logger, app.logger).info( # pylint: disable=no-member + "Updating genesis config." + ) + update_genesis_config(data=data) + + cast(logging.Logger, app.logger).info( # pylint: disable=no-member + "Updating peristent peers." + ) + config_path = Path(os.environ["TMHOME"]) / "config" / "config.toml" + update_peers( + validators=data["validators"], + config_path=config_path, + ) + update_external_address( + external_address=data["external_address"], + config_path=config_path, + ) + + return {"status": True, "error": None} + except (FileNotFoundError, json.JSONDecodeError, PermissionError): + return {"status": False, "error": traceback.format_exc()} + + @app.route("/gentle_reset") + def gentle_reset() -> Tuple[Any, int]: + """Reset the tendermint node gently.""" + try: + tendermint_node.stop() + tendermint_node.start() + return jsonify({"message": "Reset successful.", "status": True}), 200 + except Exception as e: # pylint: disable=W0703 + return jsonify({"message": f"Reset failed: {e}", "status": False}), 200 + + @app.route("/app_hash") + def app_hash() -> Tuple[Any, int]: + """Get the app hash.""" + try: + non_routable, loopback = "0.0.0.0", "127.0.0.1" # nosec + endpoint = f"{tendermint_params.rpc_laddr.replace('tcp', 'http').replace(non_routable, loopback)}/block" + height = request.args.get("height") + params = {"height": height} if height is not None else None + res = requests.get(endpoint, params) + app_hash_ = res.json()["result"]["block"]["header"]["app_hash"] + return jsonify({"app_hash": app_hash_}), res.status_code + except Exception as e: # pylint: disable=W0703 + return ( + jsonify({"error": f"Could not get the app hash: {str(e)}"}), + 200, + ) + + @app.route("/hard_reset") + def hard_reset() -> Tuple[Any, int]: + """Reset the node forcefully, and prune the blocks""" + try: + tendermint_node.stop() + if IS_DEV_MODE: + period_dumper.dump_period() + + return_code = tendermint_node.prune_blocks() + if return_code: + tendermint_node.start() + raise RuntimeError("Could not perform `unsafe-reset-all` successfully!") + defaults = get_defaults() + tendermint_node.reset_genesis_file( + request.args.get("genesis_time", defaults["genesis_time"]), + # default should be 1: https://github.com/tendermint/tendermint/pull/5191/files + request.args.get("initial_height", "1"), + request.args.get("period_count", "0"), + ) + tendermint_node.start() + return jsonify({"message": "Reset successful.", "status": True}), 200 + except Exception as e: # pylint: disable=W0703 + return jsonify({"message": f"Reset failed: {e}", "status": False}), 200 + + @app.errorhandler(404) # type: ignore + def handle_notfound(e: NotFound) -> Response: + """Handle server error.""" + cast(logging.Logger, app.logger).info(e) # pylint: disable=E + return Response("Not Found", status=404, mimetype="application/json") + + @app.errorhandler(500) # type: ignore + def handle_server_error(e: InternalServerError) -> Response: + """Handle server error.""" + cast(logging.Logger, app.logger).info(e) # pylint: disable=E + return Response("Error Closing Node", status=500, mimetype="application/json") + + return app, tendermint_node + + +def create_server() -> Any: + """Function to retrieve just the app to be used by flask entry point.""" + flask_app, _ = create_app() + return flask_app diff --git a/autonomy/deploy/generators/localhost/tendermint/tendermint.py b/autonomy/deploy/generators/localhost/tendermint/tendermint.py new file mode 100644 index 0000000000..626ab8e9a3 --- /dev/null +++ b/autonomy/deploy/generators/localhost/tendermint/tendermint.py @@ -0,0 +1,325 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2021-2023 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Tendermint manager.""" +import json +import logging +import os +import platform +import signal +import subprocess # nosec: +import sys +from logging import Logger +from pathlib import Path +from threading import Event, Thread +from typing import Any, Dict, List, Optional + + +_TCP = "tcp://" +ENCODING = "utf-8" +DEFAULT_P2P_LISTEN_ADDRESS = f"{_TCP}0.0.0.0:26656" +DEFAULT_RPC_LISTEN_ADDRESS = f"{_TCP}0.0.0.0:26657" +DEFAULT_TENDERMINT_LOG_FILE = "tendermint.log" + + +class StoppableThread( + Thread, +): + """Thread class with a stop() method.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialise the thread.""" + super().__init__(*args, **kwargs) + self._stop_event = Event() + + def stop(self) -> None: + """Set the stop event.""" + self._stop_event.set() + + def stopped(self) -> bool: + """Check if the thread is stopped.""" + return self._stop_event.is_set() + + +class TendermintParams: # pylint: disable=too-few-public-methods + """Tendermint node parameters.""" + + def __init__( # pylint: disable=too-many-arguments + self, + proxy_app: str, + rpc_laddr: str = DEFAULT_RPC_LISTEN_ADDRESS, + p2p_laddr: str = DEFAULT_P2P_LISTEN_ADDRESS, + p2p_seeds: Optional[List[str]] = None, + consensus_create_empty_blocks: bool = True, + home: Optional[str] = None, + use_grpc: bool = False, + ): + """ + Initialize the parameters to the Tendermint node. + + :param proxy_app: ABCI address. + :param rpc_laddr: RPC address. + :param p2p_laddr: P2P address. + :param p2p_seeds: P2P seeds. + :param consensus_create_empty_blocks: if true, Tendermint node creates empty blocks. + :param home: Tendermint's home directory. + :param use_grpc: Whether to use a gRPC server, or TCP + """ + + self.proxy_app = proxy_app + self.rpc_laddr = rpc_laddr + self.p2p_laddr = p2p_laddr + self.p2p_seeds = p2p_seeds + self.consensus_create_empty_blocks = consensus_create_empty_blocks + self.home = home + self.use_grpc = use_grpc + + def __str__(self) -> str: + """Get the string representation.""" + return ( + f"{self.__class__.__name__}(" + f" proxy_app={self.proxy_app},\n" + f" rpc_laddr={self.rpc_laddr},\n" + f" p2p_laddr={self.p2p_laddr},\n" + f" p2p_seeds={self.p2p_seeds},\n" + f" consensus_create_empty_blocks={self.consensus_create_empty_blocks},\n" + f" home={self.home},\n" + ")" + ) + + def build_node_command(self, debug: bool = False) -> List[str]: + """Build the 'node' command.""" + p2p_seeds = ",".join(self.p2p_seeds) if self.p2p_seeds else "" + cmd = [ + "tendermint", + "node", + f"--proxy_app={self.proxy_app}", + f"--rpc.laddr={self.rpc_laddr}", + f"--p2p.laddr={self.p2p_laddr}", + f"--p2p.seeds={p2p_seeds}", + f"--consensus.create_empty_blocks={str(self.consensus_create_empty_blocks).lower()}", + f"--abci={'grpc' if self.use_grpc else 'socket'}", + ] + if debug: + cmd.append("--log_level=debug") + if self.home is not None: # pragma: nocover + cmd += ["--home", self.home] + return cmd + + @staticmethod + def get_node_command_kwargs() -> Dict: + """Get the node command kwargs""" + kwargs = { + "bufsize": 1, + "universal_newlines": True, + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + } + if platform.system() == "Windows": # pragma: nocover + kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore + else: + kwargs["preexec_fn"] = os.setsid # type: ignore + return kwargs + + +class TendermintNode: + """A class to manage a Tendermint node.""" + + def __init__( + self, + params: TendermintParams, + logger: Optional[Logger] = None, + write_to_log: bool = False, + ): + """ + Initialize a Tendermint node. + + :param params: the parameters. + :param logger: the logger. + :param write_to_log: Write to log file. + """ + self.params = params + self._process: Optional[subprocess.Popen] = None + self._monitoring: Optional[StoppableThread] = None + self._stopping = False + self.logger = logger or logging.getLogger() + self.log_file = os.environ.get("LOG_FILE", DEFAULT_TENDERMINT_LOG_FILE) + self.write_to_log = write_to_log + + def _build_init_command(self) -> List[str]: + """Build the 'init' command.""" + cmd = [ + "tendermint", + "init", + ] + if self.params.home is not None: # pragma: nocover + cmd += ["--home", self.params.home] + return cmd + + def init(self) -> None: + """Initialize Tendermint node.""" + cmd = self._build_init_command() + subprocess.call(cmd) # nosec + + def _monitor_tendermint_process( + self, + ) -> None: + """Check server status.""" + if self._monitoring is None: + raise ValueError("Monitoring is not running") + self.log("Monitoring thread started\n") + while not self._monitoring.stopped(): + try: + if self._process is not None and self._process.stdout is not None: + line = self._process.stdout.readline() + self.log(line) + for trigger in [ + # this occurs when we lose connection from the tm side + "RPC HTTP server stopped", + # whenever the node is stopped because of a closed connection + # from on any of the tendermint modules (abci, p2p, rpc, etc) + # we restart the node + "Stopping abci.socketClient for error: read message: EOF", + ]: + if self._monitoring.stopped(): + break + if line.find(trigger) >= 0: + self._stop_tm_process() + # we can only reach this step if monitoring was activated + # so we make sure that after reset the monitoring continues + self._start_tm_process() + self.log( + f"Restarted the HTTP RPC server, as a connection was dropped with message:\n\t\t {line}\n" + ) + except Exception as e: # pylint: disable=broad-except + self.log(f"Error!: {str(e)}") + self.log("Monitoring thread terminated\n") + + def _start_tm_process(self, debug: bool = False) -> None: + """Start a Tendermint node process.""" + if self._process is not None or self._stopping: # pragma: nocover + return + cmd = self.params.build_node_command(debug) + kwargs = self.params.get_node_command_kwargs() + self.log(f"Starting Tendermint: {cmd}\n") + self._process = ( + subprocess.Popen( # nosec # pylint: disable=consider-using-with,W1509 + cmd, **kwargs + ) + ) + self.log("Tendermint process started\n") + + def _start_monitoring_thread(self) -> None: + """Start a monitoring thread.""" + self._monitoring = StoppableThread(target=self._monitor_tendermint_process) + self._monitoring.start() + + def start(self, debug: bool = False) -> None: + """Start a Tendermint node process.""" + self._start_tm_process(debug) + self._start_monitoring_thread() + + def _stop_tm_process(self) -> None: + """Stop a Tendermint node process.""" + if self._process is None or self._stopping: + return + + self._stopping = True + if platform.system() == "Windows": + self._win_stop_tm() + else: + # this will raise an exception if the process + # is not terminated within the specified timeout + self._unix_stop_tm() + + self._stopping = False + self._process = None + self.log("Tendermint process stopped\n") + + def _win_stop_tm(self) -> None: + """Stop a Tendermint node process on Windows.""" + os.kill(self._process.pid, signal.CTRL_C_EVENT) # type: ignore # pylint: disable=no-member + try: + self._process.wait(timeout=5) # type: ignore + except subprocess.TimeoutExpired: # nosec + os.kill(self._process.pid, signal.CTRL_BREAK_EVENT) # type: ignore # pylint: disable=no-member + + def _unix_stop_tm(self) -> None: + """Stop a Tendermint node process on Unix.""" + self._process.send_signal(signal.SIGTERM) # type: ignore + try: + self._process.wait(timeout=5) # type: ignore + except subprocess.TimeoutExpired: # nosec + self.log("Tendermint process did not stop gracefully\n") + + # if the process is still running poll will return None + poll = self._process.poll() # type: ignore + if poll is not None: + return + + self._process.terminate() # type: ignore + self._process.wait(3) # type: ignore + + def _stop_monitoring_thread(self) -> None: + """Stop a monitoring process.""" + if self._monitoring is not None: + self._monitoring.stop() # set stop event + self._monitoring.join() + + def stop(self) -> None: + """Stop a Tendermint node process.""" + self._stop_tm_process() + self._stop_monitoring_thread() + + @staticmethod + def _write_to_console(line: str) -> None: + """Write line to console.""" + sys.stdout.write(str(line)) + sys.stdout.flush() + + def _write_to_file(self, line: str) -> None: + """Write line to console.""" + with open(self.log_file, "a", encoding=ENCODING) as file: + file.write(line) + + def log(self, line: str) -> None: + """Open and write a line to the log file.""" + self._write_to_console(line=line) + if self.write_to_log: + self._write_to_file(line=line) + + def prune_blocks(self) -> int: + """Prune blocks from the Tendermint state""" + return subprocess.call( # nosec: + ["tendermint", "--home", str(self.params.home), "unsafe-reset-all"] + ) + + def reset_genesis_file( + self, genesis_time: str, initial_height: str, period_count: str + ) -> None: + """Reset genesis file.""" + + genesis_file = Path(str(self.params.home), "config", "genesis.json") + genesis_config = json.loads(genesis_file.read_text(encoding=ENCODING)) + genesis_config["genesis_time"] = genesis_time + genesis_config["initial_height"] = initial_height + # chain id should be max 50 chars. + # this means that the app would theoretically break when a 40-digit period is reached + genesis_config["chain_id"] = f"autonolas-{period_count}" + genesis_file.write_text(json.dumps(genesis_config, indent=2), encoding=ENCODING) diff --git a/autonomy/deploy/generators/localhost/utils.py b/autonomy/deploy/generators/localhost/utils.py new file mode 100644 index 0000000000..d11c29cd69 --- /dev/null +++ b/autonomy/deploy/generators/localhost/utils.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2024 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Localhost Deployment utilities.""" +import json +import os +import platform +import shutil +import subprocess # nosec +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +from aea.configurations.constants import AGENT + +from autonomy.chain.config import ChainType +from autonomy.deploy.constants import ( + BENCHMARKS_DIR, + TENDERMINT_BIN_UNIX, + TENDERMINT_BIN_WINDOWS, +) + + +LOCAL_TENDERMINT_VERSION = "0.34.19" + + +def check_tendermint_version() -> Path: + """Check tendermint version.""" + tendermint_executable = Path(str(shutil.which(TENDERMINT_BIN_UNIX))) + if platform.system() == "Windows": + tendermint_executable = ( + Path(os.path.dirname(sys.executable)) / TENDERMINT_BIN_WINDOWS + ) + + if ( # check tendermint version + tendermint_executable is None + or ( + current_version := subprocess.check_output( # nosec + [tendermint_executable, "version"] + ) + .strip() + .decode() + ) + != LOCAL_TENDERMINT_VERSION + ): + raise FileNotFoundError( + f"Please install tendermint version {LOCAL_TENDERMINT_VERSION} " + f"or build and run via docker by using the --docker flag." + + f"\nYour tendermint version is: {current_version}" + if current_version + else "" + ) + + return tendermint_executable + + +def _run_aea_cmd( + args: List[str], + cwd: Optional[Path] = None, + stdout: Optional[int] = None, + stderr: Optional[int] = subprocess.PIPE, + ignore_error: Optional[str] = None, + **kwargs: Any, +) -> None: + """Run an aea command in a subprocess.""" + result = subprocess.run( # pylint: disable=subprocess-run-check # nosec + args=[sys.executable, "-m", "aea.cli", *args], + cwd=cwd, + stdout=stdout, + stderr=stderr, + **kwargs, + ) + if result.returncode != 0: + result_error = result.stderr.decode() + if ignore_error and ignore_error not in result_error: + raise RuntimeError(f"Error running: {args} @ {cwd}\n{result_error}") + + +def _prepare_agent_env(working_dir: Path) -> Dict[str, Any]: + """Prepare agent env, add keys, run aea commands.""" + env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8")) + + # TODO: Dynamic port allocation, backport to service builder + env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost" + env["CONNECTION_ABCI_CONFIG_PORT"] = "26658" + + for var in env: + # Fix tendermint connection params + if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"): + env[var] = "http://localhost:8080" + + if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"): + env[var] = "http://localhost:26657" + + if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"): + env[var] = "localhost:26656" + + if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"): + benchmarks_dir = working_dir / BENCHMARKS_DIR + benchmarks_dir.mkdir(exist_ok=True, parents=True) + env[var] = str(benchmarks_dir.resolve()) + + (working_dir / "agent.json").write_text( + json.dumps(env, indent=2), + encoding="utf-8", + ) + + return env + + +def setup_agent(working_dir: Path, agent_dir: Path, keys_file: Path) -> None: + """Setup locally deployed agent.""" + env = _prepare_agent_env(working_dir) + + _run_aea_cmd( + ["fetch", env["AEA_AGENT"], "--alias", AGENT], + cwd=working_dir, + ) + + # add private keys + shutil.copy(keys_file, agent_dir) + _run_aea_cmd( + ["add-key", ChainType.ETHEREUM.value], + cwd=agent_dir, + ignore_error="already present", + ) + _run_aea_cmd(["issue-certificates"], cwd=agent_dir) diff --git a/docs/api/cli/deploy.md b/docs/api/cli/deploy.md index 5635726c15..4adbc1453c 100644 --- a/docs/api/cli/deploy.md +++ b/docs/api/cli/deploy.md @@ -53,12 +53,18 @@ Deploy an agent service. default=1, help="Number of services.", ) +@click.option( + "--localhost", + "deployment_type", + flag_value=HostDeploymentGenerator.deployment_type, + help="Use localhost as a backend.", +) @click.option( "--docker", "deployment_type", flag_value=DockerComposeGenerator.deployment_type, default=True, - help="Use docker as a backend.", + help="Use docker as a backend. (default)", ) @click.option( "--kubernetes", @@ -140,6 +146,13 @@ Deploy an agent service. help="Set agent memory usage limit.", default=DEFAULT_AGENT_MEMORY_LIMIT, ) +@click.option( + "--mkdir", + type=str, + help="Directory names to create in the build directory.", + default=[], + multiple=True, +) @registry_flag() @password_option(confirmation_prompt=True) @image_author_option @@ -151,6 +164,7 @@ def build_deployment_command( output_dir: Optional[Path], dev_mode: bool, registry: str, + mkdir: List[str], number_of_agents: Optional[int] = None, number_of_services: int = 1, password: Optional[str] = None, @@ -200,10 +214,21 @@ Build deployment setup for n agents. default=False, help="Run service in the background.", ) -def run(build_dir: Path, - no_recreate: bool, - remove_orphans: bool, - detach: bool = False) -> None +@click.option( + "--localhost", + "deployment_type", + flag_value="localhost", + help="Use localhost as a backend.", +) +@click.option( + "--docker", + "deployment_type", + flag_value="docker", + help="Use docker as a backend. (default)", + default=True, +) +def run(build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool, + deployment_type: str) -> None ``` Run deployment. diff --git a/docs/api/cli/helpers/deployment.md b/docs/api/cli/helpers/deployment.md index 116d2c0164..8ee8eebf8b 100644 --- a/docs/api/cli/helpers/deployment.md +++ b/docs/api/cli/helpers/deployment.md @@ -17,6 +17,16 @@ def run_deployment(build_dir: Path, Run deployment. + + +#### run`_`host`_`deployment + +```python +def run_host_deployment(build_dir: Path, detach: bool = False) -> None +``` + +Run host deployment. + #### stop`_`deployment @@ -51,7 +61,8 @@ def build_deployment(keys_file: Path, image_author: Optional[str] = None, resources: Optional[Resources] = None, service_hash_id: Optional[str] = None, - service_offset: int = 0) -> None + service_offset: int = 0, + mkdir: Optional[List[str]] = None) -> None ``` Build deployment. diff --git a/docs/api/deploy/generators/localhost/base.md b/docs/api/deploy/generators/localhost/base.md new file mode 100644 index 0000000000..6adfa8c3c2 --- /dev/null +++ b/docs/api/deploy/generators/localhost/base.md @@ -0,0 +1,69 @@ + + +# autonomy.deploy.generators.localhost.base + +Localhost Deployment Generator. + + + +## HostDeploymentGenerator Objects + +```python +class HostDeploymentGenerator(BaseDeploymentGenerator) +``` + +Localhost deployment. + + + +#### agent`_`dir + +```python +@property +def agent_dir() -> Path +``` + +Path to the agent directory. + + + +#### generate`_`config`_`tendermint + +```python +def generate_config_tendermint() -> "HostDeploymentGenerator" +``` + +Generate tendermint configuration. + + + +#### generate + +```python +def generate(image_version: t.Optional[str] = None, + use_hardhat: bool = False, + use_acn: bool = False) -> "HostDeploymentGenerator" +``` + +Generate agent and tendermint configurations + + + +#### populate`_`private`_`keys + +```python +def populate_private_keys() -> "HostDeploymentGenerator" +``` + +Populate the private keys to the build directory for host mapping. + + + +#### write`_`config + +```python +def write_config() -> "BaseDeploymentGenerator" +``` + +Write output to build dir + diff --git a/docs/api/deploy/generators/localhost/tendermint/app.md b/docs/api/deploy/generators/localhost/tendermint/app.md new file mode 100644 index 0000000000..44e16adaad --- /dev/null +++ b/docs/api/deploy/generators/localhost/tendermint/app.md @@ -0,0 +1,128 @@ + + +# autonomy.deploy.generators.localhost.tendermint.app + +HTTP server to control the tendermint execution environment. + + + +#### load`_`genesis + +```python +def load_genesis() -> Any +``` + +Load genesis file. + + + +#### get`_`defaults + +```python +def get_defaults() -> Dict[str, str] +``` + +Get defaults from genesis file. + + + +#### override`_`config`_`toml + +```python +def override_config_toml() -> None +``` + +Update sync method. + + + +#### update`_`peers + +```python +def update_peers(validators: List[Dict], config_path: Path) -> None +``` + +Fix peers. + + + +#### update`_`external`_`address + +```python +def update_external_address(external_address: str, config_path: Path) -> None +``` + +Update the external address. + + + +#### update`_`genesis`_`config + +```python +def update_genesis_config(data: Dict) -> None +``` + +Update genesis.json file for the tendermint node. + + + +## PeriodDumper Objects + +```python +class PeriodDumper() +``` + +Dumper for tendermint data. + + + +#### `__`init`__` + +```python +def __init__(logger: logging.Logger, dump_dir: Optional[Path] = None) -> None +``` + +Initialize object. + + + +#### readonly`_`handler + +```python +@staticmethod +def readonly_handler(func: Callable, path: str, execinfo: Any) -> None +``` + +If permission is readonly, we change and retry. + + + +#### dump`_`period + +```python +def dump_period() -> None +``` + +Dump tendermint run data for replay + + + +#### create`_`app + +```python +def create_app(dump_dir: Optional[Path] = None, + debug: bool = False) -> Tuple[Flask, TendermintNode] +``` + +Create the Tendermint server app + + + +#### create`_`server + +```python +def create_server() -> Any +``` + +Function to retrieve just the app to be used by flask entry point. + diff --git a/docs/api/deploy/generators/localhost/tendermint/tendermint.md b/docs/api/deploy/generators/localhost/tendermint/tendermint.md new file mode 100644 index 0000000000..739c25d315 --- /dev/null +++ b/docs/api/deploy/generators/localhost/tendermint/tendermint.md @@ -0,0 +1,202 @@ + + +# autonomy.deploy.generators.localhost.tendermint.tendermint + +Tendermint manager. + + + +## StoppableThread Objects + +```python +class StoppableThread(Thread) +``` + +Thread class with a stop() method. + + + +#### `__`init`__` + +```python +def __init__(*args: Any, **kwargs: Any) -> None +``` + +Initialise the thread. + + + +#### stop + +```python +def stop() -> None +``` + +Set the stop event. + + + +#### stopped + +```python +def stopped() -> bool +``` + +Check if the thread is stopped. + + + +## TendermintParams Objects + +```python +class TendermintParams() +``` + +Tendermint node parameters. + + + +#### `__`init`__` + +```python +def __init__(proxy_app: str, + rpc_laddr: str = DEFAULT_RPC_LISTEN_ADDRESS, + p2p_laddr: str = DEFAULT_P2P_LISTEN_ADDRESS, + p2p_seeds: Optional[List[str]] = None, + consensus_create_empty_blocks: bool = True, + home: Optional[str] = None, + use_grpc: bool = False) +``` + +Initialize the parameters to the Tendermint node. + +**Arguments**: + +- `proxy_app`: ABCI address. +- `rpc_laddr`: RPC address. +- `p2p_laddr`: P2P address. +- `p2p_seeds`: P2P seeds. +- `consensus_create_empty_blocks`: if true, Tendermint node creates empty blocks. +- `home`: Tendermint's home directory. +- `use_grpc`: Whether to use a gRPC server, or TCP + + + +#### `__`str`__` + +```python +def __str__() -> str +``` + +Get the string representation. + + + +#### build`_`node`_`command + +```python +def build_node_command(debug: bool = False) -> List[str] +``` + +Build the 'node' command. + + + +#### get`_`node`_`command`_`kwargs + +```python +@staticmethod +def get_node_command_kwargs() -> Dict +``` + +Get the node command kwargs + + + +## TendermintNode Objects + +```python +class TendermintNode() +``` + +A class to manage a Tendermint node. + + + +#### `__`init`__` + +```python +def __init__(params: TendermintParams, + logger: Optional[Logger] = None, + write_to_log: bool = False) +``` + +Initialize a Tendermint node. + +**Arguments**: + +- `params`: the parameters. +- `logger`: the logger. +- `write_to_log`: Write to log file. + + + +#### init + +```python +def init() -> None +``` + +Initialize Tendermint node. + + + +#### start + +```python +def start(debug: bool = False) -> None +``` + +Start a Tendermint node process. + + + +#### stop + +```python +def stop() -> None +``` + +Stop a Tendermint node process. + + + +#### log + +```python +def log(line: str) -> None +``` + +Open and write a line to the log file. + + + +#### prune`_`blocks + +```python +def prune_blocks() -> int +``` + +Prune blocks from the Tendermint state + + + +#### reset`_`genesis`_`file + +```python +def reset_genesis_file(genesis_time: str, initial_height: str, + period_count: str) -> None +``` + +Reset genesis file. + diff --git a/docs/api/deploy/generators/localhost/utils.md b/docs/api/deploy/generators/localhost/utils.md new file mode 100644 index 0000000000..97a0e8b1a7 --- /dev/null +++ b/docs/api/deploy/generators/localhost/utils.md @@ -0,0 +1,26 @@ + + +# autonomy.deploy.generators.localhost.utils + +Localhost Deployment utilities. + + + +#### check`_`tendermint`_`version + +```python +def check_tendermint_version() -> Path +``` + +Check tendermint version. + + + +#### setup`_`agent + +```python +def setup_agent(working_dir: Path, agent_dir: Path, keys_file: Path) -> None +``` + +Setup locally deployed agent. + diff --git a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py index 3420891da0..5a2c71c0fa 100644 --- a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py +++ b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py @@ -28,8 +28,18 @@ from unittest import mock import yaml +from aea.cli.registry.settings import REGISTRY_LOCAL from aea.cli.utils.config import get_default_author_from_cli_config -from aea.configurations.constants import DEFAULT_ENV_DOTFILE, PACKAGES +from aea.cli.utils.constants import CLI_CONFIG_PATH, DEFAULT_CLI_CONFIG +from aea.configurations.constants import ( + CONNECTION, + CONTRACT, + DEFAULT_AEA_CONFIG_FILE, + DEFAULT_ENV_DOTFILE, + PACKAGES, + PROTOCOL, + SKILL, +) from aea_test_autonomy.configurations import ( ETHEREUM_ENCRYPTED_KEYS, ETHEREUM_ENCRYPTION_PASSWORD, @@ -39,6 +49,7 @@ DEFAULT_BUILD_FOLDER, DEFAULT_DOCKER_IMAGE_AUTHOR, DOCKER_COMPOSE_YAML, + VALORY, ) from autonomy.deploy.base import ( DEFAULT_AGENT_CPU_LIMIT, @@ -49,12 +60,26 @@ build_hash_id, ) from autonomy.deploy.constants import ( + AGENT_VARS_CONFIG_FILE, DEBUG, DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA, DEPLOYMENT_KEY_DIRECTORY, + INFO, KUBERNETES_AGENT_KEY_NAME, + PERSISTENT_DATA_DIR, + TENDERMINT_VARS_CONFIG_FILE, + TM_ENV_CREATE_EMPTY_BLOCKS, + TM_ENV_P2P_LADDR, + TM_ENV_PROXY_APP, + TM_ENV_RPC_LADDR, + TM_ENV_TMHOME, + TM_ENV_TMSTATE, + TM_ENV_USE_GRPC, + TM_STATE_DIR, ) from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator +from autonomy.deploy.generators.localhost.utils import _run_aea_cmd +from autonomy.replay.agent import ETHEREUM_PRIVATE_KEY_FILE from tests.conftest import ROOT_DIR, skip_docker_tests from tests.test_autonomy.base import get_dummy_service_config @@ -145,6 +170,57 @@ def load_and_check_docker_compose_file( return docker_compose + @staticmethod + def check_localhost_build(build_dir: Path) -> None: + """Check localhost build directory.""" + build_tree = list(map(lambda x: x.name, build_dir.iterdir())) + assert all( + [ + child in build_tree + for child in { + ".certs", + DEFAULT_AEA_CONFIG_FILE, + DEPLOYMENT_KEY_DIRECTORY, + ETHEREUM_PRIVATE_KEY_FILE, + PERSISTENT_DATA_DIR, + AGENT_VARS_CONFIG_FILE, + "data", + "node", + TENDERMINT_VARS_CONFIG_FILE, + } + ] + ) + + def load_and_check_localhost_build(self, path: Path) -> None: + """Load localhost build config.""" + with open(path / TENDERMINT_VARS_CONFIG_FILE, "r", encoding="utf-8") as fp: + assert json.load(fp) == { + TM_ENV_TMHOME: ( + self.t / "register_reset" / DEFAULT_BUILD_FOLDER / "node" + ).as_posix(), + TM_ENV_TMSTATE: ( + self.t / "register_reset" / DEFAULT_BUILD_FOLDER / TM_STATE_DIR + ).as_posix(), + TM_ENV_P2P_LADDR: "tcp://localhost:26656", + TM_ENV_RPC_LADDR: "tcp://localhost:26657", + TM_ENV_PROXY_APP: "tcp://localhost:26658", + TM_ENV_CREATE_EMPTY_BLOCKS: "true", + TM_ENV_USE_GRPC: "false", + } + with open(path / AGENT_VARS_CONFIG_FILE, "r", encoding="utf-8") as fp: + assert json.load(fp) == { + "ID": "0", + "AEA_AGENT": "valory/register_reset:0.1.0:bafybeia4pxlphcvco3ttlv3tytklriwvuwxxxr5m2tdry32yc5vogxtm7u", + "LOG_LEVEL": INFO, + "AEA_PASSWORD": "", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_ADDRESS": "http://host.docker.internal:8545", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_CHAIN_ID": "31337", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_POA_CHAIN": "False", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_DEFAULT_GAS_PRICE_STRATEGY": "eip1559", + "CONNECTION_ABCI_CONFIG_HOST": "localhost", + "CONNECTION_ABCI_CONFIG_PORT": "26658", + } + @staticmethod def check_docker_compose_build( build_dir: Path, @@ -163,6 +239,73 @@ def check_docker_compose_build( ) +class TestLocalhostBuilds(BaseDeployBuildTest): + """Test localhost builds.""" + + def setup(self) -> None: + """Setup test for localhost deployment.""" + super().setup() + shutil.copy( + ROOT_DIR + / PACKAGES + / VALORY + / "agents" + / "register_reset" + / DEFAULT_AEA_CONFIG_FILE, + self.t / "register_reset", + ) + aea_cli_config = DEFAULT_CLI_CONFIG + aea_cli_config["registry_config"]["settings"][REGISTRY_LOCAL][ + "default_packages_path" + ] = (ROOT_DIR / PACKAGES).as_posix() + Path(CLI_CONFIG_PATH).write_text(yaml.dump(aea_cli_config)) + + with open(self.t / "register_reset" / DEFAULT_AEA_CONFIG_FILE, "r") as fp: + agent_config = next(yaml.safe_load_all(fp)) + agent_config["private_key_paths"]["ethereum"] = ETHEREUM_PRIVATE_KEY_FILE + with open(self.t / "register_reset" / DEFAULT_AEA_CONFIG_FILE, "w") as fp: + yaml.dump(agent_config, fp) + + # add all the components + for component_type in (CONNECTION, CONTRACT, SKILL, PROTOCOL): + for component_name in agent_config[component_type + "s"]: + _run_aea_cmd( + [ + "--skip-consistency-check", + "add", + component_type, + component_name, + "--mixed", + ], + cwd=self.t / "register_reset", + ignore_error="already exists", + ) + + # prepare ethereum private key + with open(self.t / "register_reset" / ETHEREUM_PRIVATE_KEY_FILE, "w") as fp: + fp.write( # mock private key + "0x0000000000000000000000000000000000000000000000000000000000000001" + ) + + def test_localhost_build( + self, + ) -> None: + """Test that the build command works.""" + + build_dir = self.t / "register_reset" / DEFAULT_BUILD_FOLDER + with mock.patch("os.chown"), OS_ENV_PATCH: + result = self.run_cli( + (str(self.keys_file), "--localhost", "--mkdir", "data") + ) + + assert result.exit_code == 0, result.output + assert build_dir.exists() + assert (build_dir / "data").exists() + + self.check_localhost_build(build_dir=build_dir) + self.load_and_check_localhost_build(path=build_dir) + + class TestDockerComposeBuilds(BaseDeployBuildTest): """Test docker-compose build.""" diff --git a/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py index e1b57e7a66..1679db0738 100644 --- a/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py +++ b/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py @@ -20,12 +20,25 @@ """Test `run` command.""" +import json import os +import shutil from unittest import mock -from autonomy.constants import DOCKER_COMPOSE_YAML +from aea.configurations.constants import DEFAULT_AEA_CONFIG_FILE, PACKAGES +from autonomy.constants import DEFAULT_BUILD_FOLDER, DOCKER_COMPOSE_YAML, VALORY +from autonomy.deploy.base import ServiceBuilder +from autonomy.deploy.constants import ( + AGENT_VARS_CONFIG_FILE, + TENDERMINT_VARS_CONFIG_FILE, +) + +from tests.conftest import ROOT_DIR from tests.test_autonomy.test_cli.base import BaseCliTest +from tests.test_autonomy.test_cli.test_deploy.test_build.test_deployment import ( + OS_ENV_PATCH, +) class TestRun(BaseCliTest): @@ -51,6 +64,51 @@ def test_run( assert result.exit_code == 0, result.output assert "Running build @" in result.output + def test_run_local(self) -> None: + """Test that `deploy run` works on localhost.""" + super().setup() + + # setup the service keys and packages + self.keys_file = self.t / "keys.json" + shutil.copytree(ROOT_DIR / PACKAGES, self.t / PACKAGES) + shutil.copy( + ROOT_DIR / "deployments" / "keys" / "hardhat_keys.json", self.keys_file + ) + shutil.copytree( + self.t / PACKAGES / "valory" / "services" / "register_reset", + self.t / "register_reset", + ) + with OS_ENV_PATCH: + self.spec = ServiceBuilder.from_dir( + self.t / "register_reset", + self.keys_file, + ) + os.chdir(self.t / "register_reset") + + # setup aea-config.yaml + shutil.copy( + ROOT_DIR + / PACKAGES + / VALORY + / "agents" + / "register_reset" + / DEFAULT_AEA_CONFIG_FILE, + self.t / "register_reset", + ) + + # setup agent.json and tendermint.json + os.mkdir(build_path := self.t / "register_reset" / DEFAULT_BUILD_FOLDER) + with open(build_path / TENDERMINT_VARS_CONFIG_FILE, "w") as fp: + json.dump({}, fp) + with open(build_path / AGENT_VARS_CONFIG_FILE, "w") as fp: + json.dump({}, fp) + with mock.patch("autonomy.cli.helpers.deployment.subprocess.run"), mock.patch( + "autonomy.cli.helpers.deployment.subprocess.Popen" + ), mock.patch("autonomy.cli.helpers.deployment.check_tendermint_version"): + result = self.run_cli(("--localhost", "--build-dir", build_path.as_posix())) + assert result.exit_code == 0, result.output + assert "Running build @" in result.output + def test_missing_config_file( self, ) -> None: diff --git a/tests/test_autonomy/test_deploy/test_deployment_generators.py b/tests/test_autonomy/test_deploy/test_deployment_generators.py index 5b2de7fccb..ab5f0ea036 100644 --- a/tests/test_autonomy/test_deploy/test_deployment_generators.py +++ b/tests/test_autonomy/test_deploy/test_deployment_generators.py @@ -33,6 +33,7 @@ from autonomy.deploy.base import BaseDeploymentGenerator, ServiceBuilder from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator from tests.conftest import ROOT_DIR @@ -52,7 +53,10 @@ def get_dummy_service() -> Service: @skip_docker_tests -@pytest.mark.parametrize("generator_cls", (DockerComposeGenerator, KubernetesGenerator)) +@pytest.mark.parametrize( + "generator_cls", + (DockerComposeGenerator, KubernetesGenerator, HostDeploymentGenerator), +) @pytest.mark.parametrize("image_version", [None, "0.1.0"]) @pytest.mark.parametrize("use_hardhat", [False, True]) @pytest.mark.parametrize("use_acn", [False, True]) @@ -84,5 +88,9 @@ def test_versioning( ) deployment_generator.generate(**generate_kwargs) - expected = f"valory/oar-oracle:{image_version or AGENT.hash}" + oar_image = "oar-" + if generator_cls == HostDeploymentGenerator: + oar_image = "" + image_version = f"latest:{AGENT.hash}" + expected = f"valory/{oar_image}oracle:{image_version or AGENT.hash}" assert expected in deployment_generator.output diff --git a/tests/test_docs/test_commands.py b/tests/test_docs/test_commands.py index cef4d4a03e..196c9c6e3c 100644 --- a/tests/test_docs/test_commands.py +++ b/tests/test_docs/test_commands.py @@ -168,6 +168,7 @@ def test_validate_doc_commands() -> None: if cmd in skips: continue + print(cmd) assert validator.validate(cmd, str(file_)), cmd