From 1389e71208815a5c853f1729adf7882bbdcf54ae Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:49:21 +0100 Subject: [PATCH 01/23] feat(proto): grpc server and client support --- proto-compiler/Cargo.toml | 12 ++++++++++++ proto-compiler/src/lib.rs | 8 ++++++++ proto/Cargo.toml | 11 +++++++++++ proto/src/lib.rs | 2 +- 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/proto-compiler/Cargo.toml b/proto-compiler/Cargo.toml index 9980ade..efc4ab9 100644 --- a/proto-compiler/Cargo.toml +++ b/proto-compiler/Cargo.toml @@ -17,3 +17,15 @@ regex = { "version" = "1.7.1" } ureq = { "version" = "2.6.2" } zip = { version = "0.6.4", default-features = false, features = ["deflate"] } fs_extra = { version = "1.3.0" } +tonic-build = { version = "0.11.0", optional = true } + + +[features] +default = [] +# Enable gRPC support; needed by server and client features. +# Conflicts with no_std +grpc = ["dep:tonic-build"] +# Build the gRPC server. Requires tenderdash-proto/std feature. +server = ["grpc"] +# Build the gRPC client. Requires tenderdash-proto/std feature. +client = ["grpc"] diff --git a/proto-compiler/src/lib.rs b/proto-compiler/src/lib.rs index 0dcb721..1aa26e8 100644 --- a/proto-compiler/src/lib.rs +++ b/proto-compiler/src/lib.rs @@ -97,6 +97,14 @@ pub fn proto_compile() { let tenderdash_ver = tenderdash_version(tenderdash_dir); println!("[info] => Creating structs."); + + #[cfg(feature = "grpc")] + tonic_build::configure() + .generate_default_stubs(true) + .compile_with_config(pb, &protos, &proto_includes_paths) + .unwrap(); + + #[cfg(not(feature = "grpc"))] pb.compile_protos(&protos, &proto_includes_paths).unwrap(); println!("[info] => Removing old structs and copying new structs."); diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 430b464..06a480d 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -15,10 +15,21 @@ description = """ [package.metadata.docs.rs] all-features = true +[features] +default = ["grpc-server"] + +# Enable standard library support +std = ["prost/std", "prost-types/std"] +# Build gRPC server +grpc-server = ["tenderdash-proto-compiler/server", "std"] +grpc-client = ["tenderdash-proto-compiler/client", "std"] + [dependencies] prost = { version = "0.12", default-features = false, features = [ "prost-derive", ] } +prost-types = { version = "0.12", default-features = false } +tonic = { version = "0.11" } bytes = { version = "1.0", default-features = false, features = ["serde"] } serde = { version = "1.0", default-features = false, features = ["derive"] } subtle-encoding = { version = "0.5", default-features = false, features = [ diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 667be8e..5c2c887 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -1,7 +1,7 @@ //! tenderdash-proto library gives the developer access to the Tenderdash //! proto-defined structs. -#![no_std] +#![cfg_attr(not(feature = "std"), no_std)] #![deny(warnings, trivial_casts, trivial_numeric_casts, unused_import_braces)] #![allow(clippy::large_enum_variant)] #![forbid(unsafe_code)] From 75bff9d69237f4f2e9fa1c0e75958aa570431071 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 10:20:00 +0100 Subject: [PATCH 02/23] chore: make ServerRuntime public --- abci/src/lib.rs | 2 +- abci/src/server.rs | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/abci/src/lib.rs b/abci/src/lib.rs index 53e983d..fc86059 100644 --- a/abci/src/lib.rs +++ b/abci/src/lib.rs @@ -22,7 +22,7 @@ pub use application::{check_version, Application, RequestDispatcher}; use prost::{DecodeError, EncodeError}; #[allow(deprecated)] #[cfg(feature = "server")] -pub use server::{start_server, CancellationToken, Server, ServerBuilder}; +pub use server::{start_server, CancellationToken, Server, ServerBuilder, ServerRuntime}; pub use tenderdash_proto as proto; #[cfg(feature = "crypto")] diff --git a/abci/src/server.rs b/abci/src/server.rs index 630379a..f9cd02f 100644 --- a/abci/src/server.rs +++ b/abci/src/server.rs @@ -8,11 +8,15 @@ use std::{ str::FromStr, }; +use futures::Future; #[cfg(feature = "tcp")] use tokio::net::TcpListener; #[cfg(feature = "unix")] use tokio::net::UnixListener; -use tokio::runtime::{Handle, Runtime}; +use tokio::{ + runtime::{Handle, Runtime}, + task::JoinHandle, +}; pub use tokio_util::sync::CancellationToken; use self::generic::GenericServer; @@ -208,10 +212,24 @@ impl<'a, App: RequestDispatcher + 'a> ServerBuilder { } /// Server runtime that must be alive for the whole lifespan of the server -pub(crate) struct ServerRuntime { +pub struct ServerRuntime { /// Runtime stored here to ensure it is never dropped _runtime: Option, - handle: Handle, + pub handle: Handle, +} + +impl ServerRuntime { + pub fn block_on(&self, future: F) -> F::Output { + self.handle.block_on(future) + } + + pub fn spawn(&self, future: F) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.handle.spawn(future) + } } impl Default for ServerRuntime { From 500fd903cf632c5a589b606333034216c8ccdef0 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 10:20:27 +0100 Subject: [PATCH 03/23] test(proto): test grpc server with tenderdash in docker --- abci/Cargo.toml | 2 + abci/tests/common/docker.rs | 38 +++++++--- abci/tests/grpc.rs | 146 ++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 abci/tests/grpc.rs diff --git a/abci/Cargo.toml b/abci/Cargo.toml index 152126a..66bfbc0 100644 --- a/abci/Cargo.toml +++ b/abci/Cargo.toml @@ -66,3 +66,5 @@ futures = { version = "0.3.26" } tokio = { version = "1", features = ["macros", "signal", "time", "io-std"] } hex = { version = "0.4" } lazy_static = { version = "1.4" } +# For tests of gRPC server +tonic = { version = "0.11" } diff --git a/abci/tests/common/docker.rs b/abci/tests/common/docker.rs index 9d084ca..1645dc4 100644 --- a/abci/tests/common/docker.rs +++ b/abci/tests/common/docker.rs @@ -6,7 +6,8 @@ use bollard::{ Docker, API_DEFAULT_VERSION, }; use futures::StreamExt; -use tokio::{io::AsyncWriteExt, runtime::Runtime, time::timeout}; +use tenderdash_abci::ServerRuntime; +use tokio::{io::AsyncWriteExt, time::timeout}; use tracing::{debug, error, info}; use url::Url; @@ -16,7 +17,7 @@ pub struct TenderdashDocker { name: String, docker: Docker, image: String, - runtime: Runtime, + runtime: ServerRuntime, } impl TenderdashDocker { /// new() creates and starts new Tenderdash docker container for provided @@ -31,8 +32,8 @@ impl TenderdashDocker { /// /// * `tag` - Docker tag to use; provide empty string to use default /// * `app_address` - address of ABCI app server; for example, - /// `tcp://172.17.0.1:4567`, `tcp://[::ffff:ac11:1]:5678` or - /// `unix:///path/to/file` + /// `tcp://172.17.0.1:4567`, `tcp://[::ffff:ac11:1]:5678`, + /// `grpc://172.17.01:5678` or `unix:///path/to/file` pub(crate) fn new( container_name: &str, tag: Option<&str>, @@ -45,14 +46,14 @@ impl TenderdashDocker { }; let app_address = url::Url::parse(app_address).expect("invalid app address"); - if app_address.scheme() != "tcp" && app_address.scheme() != "unix" { - panic!("app_address must be either tcp:// or unix://"); + if app_address.scheme() != "tcp" + && app_address.scheme() != "unix" + && app_address.scheme() != "grpc" + { + panic!("app_address must be either grpc://, tcp:// or unix://"); } - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("cannot initialize tokio runtime"); + let runtime = tenderdash_abci::ServerRuntime::default(); info!("Starting Tenderdash docker container"); @@ -152,12 +153,24 @@ impl TenderdashDocker { None }; - let app_address = app_address.to_string().replace('/', "\\/"); + let (abci, app_address) = match app_address.scheme() { + "grpc" => { + let address = app_address + .to_string() + .replace("grpc://", "") + .replace('/', "\\/"); + ("grpc", address) + }, + _ => ("socket", app_address.to_string().replace('/', "\\/")), + }; debug!("Tenderdash will connect to ABCI address: {}", app_address); let container_config = Config { image: Some(self.image.clone()), - env: Some(vec![format!("PROXY_APP={}", app_address)]), + env: Some(vec![ + format!("PROXY_APP={}", app_address), + format!("ABCI={}", abci), + ]), host_config: Some(HostConfig { binds, ..Default::default() @@ -263,6 +276,7 @@ impl Drop for TenderdashDocker { } } } + /// Use custom panic handler to dump logs on panic #[allow(dead_code)] pub fn setup_td_logs_panic(td_docker: &Arc) { diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs new file mode 100644 index 0000000..1bf736b --- /dev/null +++ b/abci/tests/grpc.rs @@ -0,0 +1,146 @@ +//! Test gRPC server for ABCI protocol. +//! +//! This test verifies that the gRPC server generated with tonic as part of the +//! tenderdash-proto crate can successfully connect to Tenderdash instance. +//! +//! This test should be implemented in the tenderdash-proto crate; however, it +//! is implemented here to use already existing docker container testing +//! logic. +use std::sync::Arc; + +use proto::abci::{ + abci_application_server::AbciApplication, RequestEcho, RequestInfo, ResponseInfo, +}; +use tenderdash_abci::CancellationToken; +mod common; +use tenderdash_abci::proto; +use tonic::{async_trait, Response, Status}; + +#[cfg(feature = "docker-tests")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +/// Test server listening on ipv4 address. +/// +/// See [tcp_server_test()]. +async fn test_ipv4_server() { + // we assume the host uses default Docker network configuration, with the host + // using 172.17.0.1 + let bind_address = "172.17.0.1:1234".to_string(); + + grpc_server_test("v4", bind_address.as_str()).await; +} + +#[cfg(feature = "docker-tests")] +#[tokio::test] +/// Test server listening on ipv6 address. +/// +/// See [tcp_server_test()]. +async fn test_ipv6_server() { + // we assume the host uses default Docker network configuration, with the host + // using 172.17.0.1. This is IPv6 notation of the IPv4 address. + let bind_address = "[::ffff:ac11:1]:5678".to_string(); + + grpc_server_test("v6", bind_address.as_str()).await; +} + +#[cfg(feature = "docker-tests")] +/// Feature: ABCI App TCO server +/// +/// * Given that we have Tenderdash instance using TCP connection to communicate +/// with ABCI APP +/// * When we estabilish connection with Tenderdash +/// * Then Tenderdash sends Info request +async fn grpc_server_test(test_name: &str, bind_address: &str) { + use core::panic; + + use proto::abci::abci_application_server::AbciApplicationServer; + use tonic::transport::Server; + + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::new("debug")) + .with_ansi(true) + .try_init() + .ok(); + + let cancel = CancellationToken::new(); + let app = TestApp { + cancel: cancel.clone(), + }; + + let addr = bind_address.parse().expect("address must be valid"); + let server_cancel = cancel.clone(); + let server_handle = tokio::spawn(async move { + tracing::debug!("starting gRPC server"); + Server::builder() + .add_service(AbciApplicationServer::new(app)) + .serve_with_shutdown(addr, server_cancel.cancelled()) + .await + .expect("server failed"); + tracing::debug!("gRPC server stopped"); + }); + + let socket_uri = format!("grpc://{}", bind_address); + let container_name = format!("tenderdash_{}", test_name); + + let td = tokio::task::spawn_blocking(move || { + tracing::debug!("starting Tenderdash in Docker container"); + let td = Arc::new(common::docker::TenderdashDocker::new( + &container_name, + Some("feat-ABCI-protocol-env-var"), + &socket_uri, + )); + common::docker::setup_td_logs_panic(&td); + tracing::debug!("started Tenderdash in Docker container"); + + td + }) + .await + .expect("start tenderdash"); + + // tracing::debug!(?result, "connection handled"); + // assert!(matches!(result, Ok(()))); + tokio::select! { + _ = tokio::time::sleep(tokio::time::Duration::from_secs(60)) => { + panic!("Test timed out"); + } + _ = cancel.cancelled() => { + tracing::debug!("CancellationToken cancelled"); + } + ret = server_handle => { + ret.expect("gRPC server failed"); + } + } + + tokio::task::spawn_blocking(move || drop(td)) + .await + .expect("tenderdash cleanup"); + + tracing::info!("Test finished successfully"); +} + +pub struct TestApp { + // when test succeeds, we cancel this token to finish it + cancel: CancellationToken, +} +#[async_trait] +impl AbciApplication for TestApp { + async fn echo( + &self, + request: tonic::Request, + ) -> Result, Status> { + tracing::info!(?request, "Echo called"); + Ok(Response::new(proto::abci::ResponseEcho { + message: request.into_inner().message, + })) + } + async fn info( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + tracing::info!("Info called, test successful"); + let resp = ResponseInfo { + ..Default::default() + }; + self.cancel.cancel(); + Ok(Response::new(resp)) + } +} From 3bf8d72576218c347d81da7340fe5852c8b62518 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 10:28:10 +0100 Subject: [PATCH 04/23] chore(proto): fix imports --- proto/src/error.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proto/src/error.rs b/proto/src/error.rs index 1495ff2..0e917d9 100644 --- a/proto/src/error.rs +++ b/proto/src/error.rs @@ -1,7 +1,9 @@ //! This module defines the various errors that be raised during Protobuf //! conversions. - +#[cfg(no_std)] use core::{convert::TryFrom, fmt::Display, num::TryFromIntError}; +#[cfg(not(no_std))] +use std::{fmt::Display, num::TryFromIntError}; use flex_error::{define_error, DisplayOnly}; use prost::{DecodeError, EncodeError}; From 99d1cab35b7309d9482a3d5a004ebfe2bd32b1b3 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:04:16 +0100 Subject: [PATCH 05/23] chore: build issues --- abci/tests/grpc.rs | 2 ++ proto/build.rs | 3 +++ proto/src/error.rs | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs index 1bf736b..c94ce3b 100644 --- a/abci/tests/grpc.rs +++ b/abci/tests/grpc.rs @@ -6,6 +6,8 @@ //! This test should be implemented in the tenderdash-proto crate; however, it //! is implemented here to use already existing docker container testing //! logic. +#![cfg(feature = "grpc-server")] + use std::sync::Arc; use proto::abci::{ diff --git a/proto/build.rs b/proto/build.rs index 80894aa..d29a2f8 100644 --- a/proto/build.rs +++ b/proto/build.rs @@ -16,4 +16,7 @@ fn main() { println!("cargo:rerun-if-changed=Cargo.toml"); println!("cargo:rerun-if-env-changed=CARGO_PKG_VERSION"); println!("cargo:rerun-if-env-changed=TENDERDASH_COMMITISH"); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_STD"); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_GRPC_SERVER"); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_GRPC_CLIENT"); } diff --git a/proto/src/error.rs b/proto/src/error.rs index 0e917d9..cfa6eb4 100644 --- a/proto/src/error.rs +++ b/proto/src/error.rs @@ -1,8 +1,8 @@ //! This module defines the various errors that be raised during Protobuf //! conversions. -#[cfg(no_std)] +#[cfg(not(feature = "std"))] use core::{convert::TryFrom, fmt::Display, num::TryFromIntError}; -#[cfg(not(no_std))] +#[cfg(feature = "std")] use std::{fmt::Display, num::TryFromIntError}; use flex_error::{define_error, DisplayOnly}; From eb5d1afcd1eaa77ac6e5a84ca1ed0556c3acaee1 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:12:18 +0100 Subject: [PATCH 06/23] chore: abci grpc-server feature --- abci/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/abci/Cargo.toml b/abci/Cargo.toml index 66bfbc0..3064284 100644 --- a/abci/Cargo.toml +++ b/abci/Cargo.toml @@ -20,6 +20,8 @@ server = [ "dep:tokio-util", "dep:futures", ] + +grpc-server = ["tenderdash-proto/grpc-server"] crypto = ["dep:lhash"] tcp = ["server"] unix = ["server"] From 01481bf0a5a7d43cbcfa6943f81ab4fce258e7f2 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:12:35 +0100 Subject: [PATCH 07/23] chore: imports and features --- proto/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 5c2c887..6500554 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -23,10 +23,16 @@ mod error; #[allow(warnings)] mod tenderdash; +#[cfg(not(feature = "std"))] use core::{ convert::{TryFrom, TryInto}, fmt::Display, }; +#[cfg(feature = "std")] +use std::{ + convert::{TryFrom, TryInto}, + fmt::Display, +}; use bytes::{Buf, BufMut}; pub use error::Error; From a708dc118d4e62a72389a8edc275f8dcdb0cdae3 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:19:22 +0100 Subject: [PATCH 08/23] chore: grpc test --- abci/tests/grpc.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs index c94ce3b..b2969ba 100644 --- a/abci/tests/grpc.rs +++ b/abci/tests/grpc.rs @@ -10,10 +10,12 @@ use std::sync::Arc; -use proto::abci::{ - abci_application_server::AbciApplication, RequestEcho, RequestInfo, ResponseInfo, +use tenderdash_abci::{ + proto::abci::{ + abci_application_server::AbciApplication, RequestEcho, RequestInfo, ResponseInfo, + }, + CancellationToken, }; -use tenderdash_abci::CancellationToken; mod common; use tenderdash_abci::proto; use tonic::{async_trait, Response, Status}; From 271dc51ea60625b3503a09e58882e680ade9da8c Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 21 Feb 2024 17:48:34 +0700 Subject: [PATCH 09/23] chore: re-export tonic --- proto/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 6500554..9725257 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -44,6 +44,9 @@ pub mod serializers; use prelude::*; pub use tenderdash::meta::ABCI_VERSION; +#[cfg(any(feature = "grpc-server", feature = "grpc-client"))] +pub use tonic; + /// Allows for easy Google Protocol Buffers encoding and decoding of domain /// types with validation. /// From 191d0729ba05ea94b7002c20770b01e56f43111e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:28:00 +0100 Subject: [PATCH 10/23] chore: fix missing serde serialization --- proto-compiler/src/constants.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index a328e4a..057e9a3 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -54,7 +54,6 @@ const DERIVE_FROM_STR: &str = r#"#[derive(derive_more::FromStr)]"#; /// here: pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.libs.bits.BitArray", SERIALIZED), - (".tendermint.types.EvidenceParams", SERIALIZED), (".tendermint.types.BlockIDFlag", PRIMITIVE_ENUM), (".tendermint.types.Block", SERIALIZED), (".tendermint.types.Data", SERIALIZED), @@ -64,6 +63,14 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.types.DuplicateVoteEvidence", SERIALIZED), (".tendermint.types.Vote", SERIALIZED), (".tendermint.types.BlockID", SERIALIZED), + (".tendermint.types.ConsensusParams", SERIALIZED), + (".tendermint.types.ABCIParams", SERIALIZED), + (".tendermint.types.BlockParams", SERIALIZED), + (".tendermint.types.EvidenceParams", SERIALIZED), + (".tendermint.types.ValidatorParams", SERIALIZED), + (".tendermint.types.VersionParams", SERIALIZED), + (".tendermint.types.SynchronyParams", SERIALIZED), + (".tendermint.types.TimeoutParams", SERIALIZED), (".tendermint.types.PartSetHeader", SERIALIZED), (".tendermint.types.LightClientAttackEvidence", SERIALIZED), (".tendermint.types.LightBlock", SERIALIZED), From c57971b1b76fc091523513d85dee1886942f9d4f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:38:23 +0100 Subject: [PATCH 11/23] chore(proto): fix feature flags --- proto/src/lib.rs | 6 +----- proto/src/serializers/part_set_header_total.rs | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 9725257..047ee86 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -29,10 +29,7 @@ use core::{ fmt::Display, }; #[cfg(feature = "std")] -use std::{ - convert::{TryFrom, TryInto}, - fmt::Display, -}; +use std::fmt::Display; use bytes::{Buf, BufMut}; pub use error::Error; @@ -43,7 +40,6 @@ pub mod serializers; use prelude::*; pub use tenderdash::meta::ABCI_VERSION; - #[cfg(any(feature = "grpc-server", feature = "grpc-client"))] pub use tonic; diff --git a/proto/src/serializers/part_set_header_total.rs b/proto/src/serializers/part_set_header_total.rs index cecfc52..17c9222 100644 --- a/proto/src/serializers/part_set_header_total.rs +++ b/proto/src/serializers/part_set_header_total.rs @@ -5,7 +5,11 @@ //! from a string-quoted integer value into an integer value without quotes in //! Tendermint Core v0.34.0. This deserializer allows backwards-compatibility by //! deserializing both ways. See also: + +#[cfg(not(feature = "std"))] use core::{convert::TryFrom, fmt::Formatter}; +#[cfg(feature = "std")] +use std::fmt::Formatter; use serde::{ de::{Error, Visitor}, From 81d28aa0b15fc0844dfa7f7251f6949f6c6c405a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:44:39 +0100 Subject: [PATCH 12/23] chore(proto): fix warn --- proto/tests/unit.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/tests/unit.rs b/proto/tests/unit.rs index f820160..763f97e 100644 --- a/proto/tests/unit.rs +++ b/proto/tests/unit.rs @@ -1,3 +1,4 @@ +#[cfg(not(feature = "std"))] use core::convert::TryFrom; use tenderdash_proto::{ From 47d4105a49a4a332d5578a91371333686c7c5b1a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:10:46 +0100 Subject: [PATCH 13/23] fix: wrong json serialization of ConsensusParams --- proto-compiler/src/constants.rs | 39 ++++++++++----- proto/src/protobuf.rs | 66 +++++++++++++++++++++++--- proto/src/serializers/from_str.rs | 12 +++-- proto/src/serializers/time_duration.rs | 3 ++ proto/src/serializers/timestamp.rs | 8 ++-- proto/tests/unit.rs | 43 ++++++++++++++++- 6 files changed, 145 insertions(+), 26 deletions(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index 057e9a3..25f27f3 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -63,14 +63,6 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.types.DuplicateVoteEvidence", SERIALIZED), (".tendermint.types.Vote", SERIALIZED), (".tendermint.types.BlockID", SERIALIZED), - (".tendermint.types.ConsensusParams", SERIALIZED), - (".tendermint.types.ABCIParams", SERIALIZED), - (".tendermint.types.BlockParams", SERIALIZED), - (".tendermint.types.EvidenceParams", SERIALIZED), - (".tendermint.types.ValidatorParams", SERIALIZED), - (".tendermint.types.VersionParams", SERIALIZED), - (".tendermint.types.SynchronyParams", SERIALIZED), - (".tendermint.types.TimeoutParams", SERIALIZED), (".tendermint.types.PartSetHeader", SERIALIZED), (".tendermint.types.LightClientAttackEvidence", SERIALIZED), (".tendermint.types.LightBlock", SERIALIZED), @@ -98,6 +90,15 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.crypto.Proof", SERIALIZED), (".tendermint.abci.Response.value", DERIVE_FROM), (".tendermint.abci.Request.value", DERIVE_FROM), + // Consensus params + (".tendermint.types.ConsensusParams", SERIALIZED), + (".tendermint.types.ABCIParams", SERIALIZED), + (".tendermint.types.BlockParams", SERIALIZED), + (".tendermint.types.EvidenceParams", SERIALIZED), + (".tendermint.types.ValidatorParams", SERIALIZED), + (".tendermint.types.VersionParams", SERIALIZED), + (".tendermint.types.SynchronyParams", SERIALIZED), + (".tendermint.types.TimeoutParams", SERIALIZED), ]; /// Custom field attributes applied on top of protobuf fields in (a) struct(s) @@ -106,10 +107,6 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ /// The first item is a path as defined in the prost_build::Config::btree_map /// here: pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[ - ( - ".tendermint.types.EvidenceParams.max_bytes", - QUOTED_WITH_DEFAULT, - ), (".tendermint.version.Consensus.block", QUOTED), (".tendermint.version.Consensus.app", QUOTED_WITH_DEFAULT), (".tendermint.abci.ResponseInfo.data", DEFAULT), @@ -207,4 +204,22 @@ pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.crypto.Proof.total", QUOTED), (".tendermint.crypto.Proof.aunts", VEC_BASE64STRING), (".tendermint.crypto.Proof.leaf_hash", BASE64STRING), + // Consensus params + ( + ".tendermint.types.BlockParams.max_bytes", + QUOTED_WITH_DEFAULT, + ), + (".tendermint.types.BlockParams.max_gas", QUOTED_WITH_DEFAULT), + ( + ".tendermint.types.EvidenceParams.max_age_num_blocks", + QUOTED_WITH_DEFAULT, + ), + ( + ".tendermint.types.EvidenceParams.max_bytes", + QUOTED_WITH_DEFAULT, + ), + ( + ".tendermint.types.VersionParams.app_version", + QUOTED_WITH_DEFAULT, + ), ]; diff --git a/proto/src/protobuf.rs b/proto/src/protobuf.rs index 8cececa..b21e789 100644 --- a/proto/src/protobuf.rs +++ b/proto/src/protobuf.rs @@ -1,7 +1,12 @@ -// Google protobuf Timestamp and Duration types reimplemented because their comments are turned -// into invalid documentation texts and doctest chokes on them. See https://github.com/danburkert/prost/issues/374 -// Prost does not seem to have a way yet to remove documentations defined in protobuf files. -// These structs are defined in gogoproto v1.3.1 at https://github.com/gogo/protobuf/tree/v1.3.1/protobuf/google/protobuf +// Google protobuf Timestamp and Duration types reimplemented because their +// comments are turned into invalid documentation texts and doctest chokes on them. See https://github.com/danburkert/prost/issues/374 +// Prost does not seem to have a way yet to remove documentations defined in +// protobuf files. These structs are defined in gogoproto v1.3.1 at https://github.com/gogo/protobuf/tree/v1.3.1/protobuf/google/protobuf + +#[cfg(not(feature = "std"))] +use core::fmt; +#[cfg(feature = "std")] +use std::fmt; /// A Timestamp represents a point in time independent of any time zone or local /// calendar, encoded as a count of seconds and fractions of seconds at @@ -17,7 +22,10 @@ /// restricting to that range, we ensure that we can convert to and from [RFC /// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. #[derive(Clone, PartialEq, ::prost::Message, ::serde::Deserialize, ::serde::Serialize)] -#[serde(from = "crate::serializers::timestamp::Rfc3339", into = "crate::serializers::timestamp::Rfc3339")] +#[serde( + from = "crate::serializers::timestamp::Rfc3339", + into = "crate::serializers::timestamp::Rfc3339" +)] pub struct Timestamp { /// Represents seconds of UTC time since Unix epoch /// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to @@ -38,7 +46,7 @@ pub struct Timestamp { /// or "month". It is related to Timestamp in that the difference between /// two Timestamp values is a Duration and it can be added or subtracted /// from a Timestamp. Range is approximately +-10,000 years. -#[derive(Clone, PartialEq, ::prost::Message, ::serde::Deserialize, ::serde::Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Duration { /// Signed seconds of the span of time. Must be from -315,576,000,000 /// to +315,576,000,000 inclusive. Note: these bounds are computed from: @@ -54,3 +62,49 @@ pub struct Duration { #[prost(int32, tag = "2")] pub nanos: i32, } + +impl serde::Serialize for Duration { + fn serialize(&self, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + let total_nanos = self.seconds * 1_000_000_000 + self.nanos as i64; + serializer.serialize_i64(total_nanos) + } +} + +struct DurationVisitor; + +impl<'de> serde::de::Visitor<'de> for DurationVisitor { + type Value = Duration; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a nanosecond representation of a duration") + } + + fn visit_i128(self, value: i128) -> Result + where + E: serde::de::Error, + { + let seconds = (value / 1_000_000_000) as i64; + let nanos = (value % 1_000_000_000) as i32; + Ok(Duration { seconds, nanos }) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + let value = value.parse::().map_err(E::custom)?; + self.visit_i128(value) + } +} + +impl<'de> serde::Deserialize<'de> for Duration { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + deserializer.deserialize_str(DurationVisitor) + } +} diff --git a/proto/src/serializers/from_str.rs b/proto/src/serializers/from_str.rs index c176252..0544cc7 100644 --- a/proto/src/serializers/from_str.rs +++ b/proto/src/serializers/from_str.rs @@ -1,16 +1,20 @@ //! Serialize and deserialize any `T` that implements [[core::str::FromStr]] //! and [[core::fmt::Display]] from or into string. Note this can be used for //! all primitive data types. +#[cfg(not(feature = "std"))] +use core::{fmt::Display, str::FromStr}; +#[cfg(feature = "std")] +use std::{fmt::Display, str::FromStr}; + use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer}; use crate::prelude::*; - /// Deserialize string into T pub fn deserialize<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, - T: core::str::FromStr, - ::Err: core::fmt::Display, + T: FromStr, + ::Err: Display, { String::deserialize(deserializer)? .parse::() @@ -21,7 +25,7 @@ where pub fn serialize(value: &T, serializer: S) -> Result where S: Serializer, - T: core::fmt::Display, + T: Display, { format!("{value}").serialize(serializer) } diff --git a/proto/src/serializers/time_duration.rs b/proto/src/serializers/time_duration.rs index b8af330..f30e943 100644 --- a/proto/src/serializers/time_duration.rs +++ b/proto/src/serializers/time_duration.rs @@ -1,5 +1,8 @@ //! Serialize/deserialize core::time::Duration type from and into string: +#[cfg(not(feature = "std"))] use core::time::Duration; +#[cfg(feature = "std")] +use std::time::Duration; use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer}; diff --git a/proto/src/serializers/timestamp.rs b/proto/src/serializers/timestamp.rs index fc0a5f3..945f881 100644 --- a/proto/src/serializers/timestamp.rs +++ b/proto/src/serializers/timestamp.rs @@ -1,6 +1,8 @@ //! Serialize/deserialize Timestamp type from and into string: - +#[cfg(not(feature = "std"))] use core::fmt::{self, Debug}; +#[cfg(feature = "std")] +use std::fmt::{self, Debug}; use serde::{de::Error as _, ser::Error, Deserialize, Deserializer, Serialize, Serializer}; use time::{ @@ -147,8 +149,8 @@ pub fn to_rfc3339_nanos(t: OffsetDateTime) -> String { /// ie. a RFC3339 date-time with left-padded subsecond digits without /// trailing zeros and no trailing dot. /// -/// [`Display`]: core::fmt::Display -/// [`Debug`]: core::fmt::Debug +/// [`Display`]: fmt::Display +/// [`Debug`]: fmt::Debug pub fn fmt_as_rfc3339_nanos(t: OffsetDateTime, f: &mut impl fmt::Write) -> fmt::Result { let t = t.to_offset(offset!(UTC)); let nanos = t.nanosecond(); diff --git a/proto/tests/unit.rs b/proto/tests/unit.rs index 763f97e..e0fdfeb 100644 --- a/proto/tests/unit.rs +++ b/proto/tests/unit.rs @@ -3,7 +3,7 @@ use core::convert::TryFrom; use tenderdash_proto::{ abci::ResponseException, - types::{BlockId as RawBlockId, PartSetHeader as RawPartSetHeader}, + types::{BlockId as RawBlockId, ConsensusParams, PartSetHeader as RawPartSetHeader}, Protobuf, }; @@ -134,3 +134,44 @@ pub fn test_response_exception_from() { "string" ); } + +#[test] +pub fn test_consensus_params_serde() { + let json = r#" + { + "block": { + "max_bytes": "2097152", + "max_gas": "500000000" + }, + "evidence": { + "max_age_num_blocks": "10000", + "max_age_duration": "172800000000000", + "max_bytes": "0" + }, + "validator": { + "pub_key_types": [ + "bls12381" + ] + }, + "version": { + "app_version": "1" + }, + "synchrony": { + "precision": "500000000", + "message_delay": "60000000000" + }, + "timeout": { + "propose": "40000000000", + "propose_delta": "5000000000", + "vote": "40000000000", + "vote_delta": "5000000000", + "bypass_commit_timeout": true + }, + "abci": { + "recheck_tx": true + } + } + "#; + + let _new_params: ConsensusParams = serde_json::from_str(json).unwrap(); +} From 2b9dcec93713a6b204b451442a54c7c635de0618 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:10:46 +0100 Subject: [PATCH 14/23] fix: wrong json serialization of ConsensusParams --- proto-compiler/src/constants.rs | 31 ++++++++++-- proto/src/protobuf.rs | 66 +++++++++++++++++++++++--- proto/src/serializers/from_str.rs | 12 +++-- proto/src/serializers/time_duration.rs | 3 ++ proto/src/serializers/timestamp.rs | 8 ++-- proto/tests/unit.rs | 43 ++++++++++++++++- 6 files changed, 145 insertions(+), 18 deletions(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index a328e4a..924e142 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -91,6 +91,15 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.crypto.Proof", SERIALIZED), (".tendermint.abci.Response.value", DERIVE_FROM), (".tendermint.abci.Request.value", DERIVE_FROM), + // Consensus params + (".tendermint.types.ConsensusParams", SERIALIZED), + (".tendermint.types.ABCIParams", SERIALIZED), + (".tendermint.types.BlockParams", SERIALIZED), + (".tendermint.types.EvidenceParams", SERIALIZED), + (".tendermint.types.ValidatorParams", SERIALIZED), + (".tendermint.types.VersionParams", SERIALIZED), + (".tendermint.types.SynchronyParams", SERIALIZED), + (".tendermint.types.TimeoutParams", SERIALIZED), ]; /// Custom field attributes applied on top of protobuf fields in (a) struct(s) @@ -99,10 +108,6 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ /// The first item is a path as defined in the prost_build::Config::btree_map /// here: pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[ - ( - ".tendermint.types.EvidenceParams.max_bytes", - QUOTED_WITH_DEFAULT, - ), (".tendermint.version.Consensus.block", QUOTED), (".tendermint.version.Consensus.app", QUOTED_WITH_DEFAULT), (".tendermint.abci.ResponseInfo.data", DEFAULT), @@ -200,4 +205,22 @@ pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.crypto.Proof.total", QUOTED), (".tendermint.crypto.Proof.aunts", VEC_BASE64STRING), (".tendermint.crypto.Proof.leaf_hash", BASE64STRING), + // Consensus params + ( + ".tendermint.types.BlockParams.max_bytes", + QUOTED_WITH_DEFAULT, + ), + (".tendermint.types.BlockParams.max_gas", QUOTED_WITH_DEFAULT), + ( + ".tendermint.types.EvidenceParams.max_age_num_blocks", + QUOTED_WITH_DEFAULT, + ), + ( + ".tendermint.types.EvidenceParams.max_bytes", + QUOTED_WITH_DEFAULT, + ), + ( + ".tendermint.types.VersionParams.app_version", + QUOTED_WITH_DEFAULT, + ), ]; diff --git a/proto/src/protobuf.rs b/proto/src/protobuf.rs index 8cececa..b21e789 100644 --- a/proto/src/protobuf.rs +++ b/proto/src/protobuf.rs @@ -1,7 +1,12 @@ -// Google protobuf Timestamp and Duration types reimplemented because their comments are turned -// into invalid documentation texts and doctest chokes on them. See https://github.com/danburkert/prost/issues/374 -// Prost does not seem to have a way yet to remove documentations defined in protobuf files. -// These structs are defined in gogoproto v1.3.1 at https://github.com/gogo/protobuf/tree/v1.3.1/protobuf/google/protobuf +// Google protobuf Timestamp and Duration types reimplemented because their +// comments are turned into invalid documentation texts and doctest chokes on them. See https://github.com/danburkert/prost/issues/374 +// Prost does not seem to have a way yet to remove documentations defined in +// protobuf files. These structs are defined in gogoproto v1.3.1 at https://github.com/gogo/protobuf/tree/v1.3.1/protobuf/google/protobuf + +#[cfg(not(feature = "std"))] +use core::fmt; +#[cfg(feature = "std")] +use std::fmt; /// A Timestamp represents a point in time independent of any time zone or local /// calendar, encoded as a count of seconds and fractions of seconds at @@ -17,7 +22,10 @@ /// restricting to that range, we ensure that we can convert to and from [RFC /// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. #[derive(Clone, PartialEq, ::prost::Message, ::serde::Deserialize, ::serde::Serialize)] -#[serde(from = "crate::serializers::timestamp::Rfc3339", into = "crate::serializers::timestamp::Rfc3339")] +#[serde( + from = "crate::serializers::timestamp::Rfc3339", + into = "crate::serializers::timestamp::Rfc3339" +)] pub struct Timestamp { /// Represents seconds of UTC time since Unix epoch /// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to @@ -38,7 +46,7 @@ pub struct Timestamp { /// or "month". It is related to Timestamp in that the difference between /// two Timestamp values is a Duration and it can be added or subtracted /// from a Timestamp. Range is approximately +-10,000 years. -#[derive(Clone, PartialEq, ::prost::Message, ::serde::Deserialize, ::serde::Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Duration { /// Signed seconds of the span of time. Must be from -315,576,000,000 /// to +315,576,000,000 inclusive. Note: these bounds are computed from: @@ -54,3 +62,49 @@ pub struct Duration { #[prost(int32, tag = "2")] pub nanos: i32, } + +impl serde::Serialize for Duration { + fn serialize(&self, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + let total_nanos = self.seconds * 1_000_000_000 + self.nanos as i64; + serializer.serialize_i64(total_nanos) + } +} + +struct DurationVisitor; + +impl<'de> serde::de::Visitor<'de> for DurationVisitor { + type Value = Duration; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a nanosecond representation of a duration") + } + + fn visit_i128(self, value: i128) -> Result + where + E: serde::de::Error, + { + let seconds = (value / 1_000_000_000) as i64; + let nanos = (value % 1_000_000_000) as i32; + Ok(Duration { seconds, nanos }) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + let value = value.parse::().map_err(E::custom)?; + self.visit_i128(value) + } +} + +impl<'de> serde::Deserialize<'de> for Duration { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + deserializer.deserialize_str(DurationVisitor) + } +} diff --git a/proto/src/serializers/from_str.rs b/proto/src/serializers/from_str.rs index c176252..0544cc7 100644 --- a/proto/src/serializers/from_str.rs +++ b/proto/src/serializers/from_str.rs @@ -1,16 +1,20 @@ //! Serialize and deserialize any `T` that implements [[core::str::FromStr]] //! and [[core::fmt::Display]] from or into string. Note this can be used for //! all primitive data types. +#[cfg(not(feature = "std"))] +use core::{fmt::Display, str::FromStr}; +#[cfg(feature = "std")] +use std::{fmt::Display, str::FromStr}; + use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer}; use crate::prelude::*; - /// Deserialize string into T pub fn deserialize<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, - T: core::str::FromStr, - ::Err: core::fmt::Display, + T: FromStr, + ::Err: Display, { String::deserialize(deserializer)? .parse::() @@ -21,7 +25,7 @@ where pub fn serialize(value: &T, serializer: S) -> Result where S: Serializer, - T: core::fmt::Display, + T: Display, { format!("{value}").serialize(serializer) } diff --git a/proto/src/serializers/time_duration.rs b/proto/src/serializers/time_duration.rs index b8af330..f30e943 100644 --- a/proto/src/serializers/time_duration.rs +++ b/proto/src/serializers/time_duration.rs @@ -1,5 +1,8 @@ //! Serialize/deserialize core::time::Duration type from and into string: +#[cfg(not(feature = "std"))] use core::time::Duration; +#[cfg(feature = "std")] +use std::time::Duration; use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer}; diff --git a/proto/src/serializers/timestamp.rs b/proto/src/serializers/timestamp.rs index fc0a5f3..945f881 100644 --- a/proto/src/serializers/timestamp.rs +++ b/proto/src/serializers/timestamp.rs @@ -1,6 +1,8 @@ //! Serialize/deserialize Timestamp type from and into string: - +#[cfg(not(feature = "std"))] use core::fmt::{self, Debug}; +#[cfg(feature = "std")] +use std::fmt::{self, Debug}; use serde::{de::Error as _, ser::Error, Deserialize, Deserializer, Serialize, Serializer}; use time::{ @@ -147,8 +149,8 @@ pub fn to_rfc3339_nanos(t: OffsetDateTime) -> String { /// ie. a RFC3339 date-time with left-padded subsecond digits without /// trailing zeros and no trailing dot. /// -/// [`Display`]: core::fmt::Display -/// [`Debug`]: core::fmt::Debug +/// [`Display`]: fmt::Display +/// [`Debug`]: fmt::Debug pub fn fmt_as_rfc3339_nanos(t: OffsetDateTime, f: &mut impl fmt::Write) -> fmt::Result { let t = t.to_offset(offset!(UTC)); let nanos = t.nanosecond(); diff --git a/proto/tests/unit.rs b/proto/tests/unit.rs index f820160..bd44a9b 100644 --- a/proto/tests/unit.rs +++ b/proto/tests/unit.rs @@ -2,7 +2,7 @@ use core::convert::TryFrom; use tenderdash_proto::{ abci::ResponseException, - types::{BlockId as RawBlockId, PartSetHeader as RawPartSetHeader}, + types::{BlockId as RawBlockId, ConsensusParams, PartSetHeader as RawPartSetHeader}, Protobuf, }; @@ -133,3 +133,44 @@ pub fn test_response_exception_from() { "string" ); } + +#[test] +pub fn test_consensus_params_serde() { + let json = r#" + { + "block": { + "max_bytes": "2097152", + "max_gas": "500000000" + }, + "evidence": { + "max_age_num_blocks": "10000", + "max_age_duration": "172800000000000", + "max_bytes": "0" + }, + "validator": { + "pub_key_types": [ + "bls12381" + ] + }, + "version": { + "app_version": "1" + }, + "synchrony": { + "precision": "500000000", + "message_delay": "60000000000" + }, + "timeout": { + "propose": "40000000000", + "propose_delta": "5000000000", + "vote": "40000000000", + "vote_delta": "5000000000", + "bypass_commit_timeout": true + }, + "abci": { + "recheck_tx": true + } + } + "#; + + let _new_params: ConsensusParams = serde_json::from_str(json).unwrap(); +} From 3c4ecad26e88a88508544fef43251ee2053edcfe Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:13:53 +0100 Subject: [PATCH 15/23] chore: fix build --- proto-compiler/src/constants.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index 924e142..25f27f3 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -54,7 +54,6 @@ const DERIVE_FROM_STR: &str = r#"#[derive(derive_more::FromStr)]"#; /// here: pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.libs.bits.BitArray", SERIALIZED), - (".tendermint.types.EvidenceParams", SERIALIZED), (".tendermint.types.BlockIDFlag", PRIMITIVE_ENUM), (".tendermint.types.Block", SERIALIZED), (".tendermint.types.Data", SERIALIZED), From 91919920dcb04488c90e43f346228ad214e6967d Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:49:04 +0100 Subject: [PATCH 16/23] chore: simplify features --- abci/Cargo.toml | 2 +- abci/tests/grpc.rs | 2 +- proto/Cargo.toml | 9 ++++++--- proto/src/lib.rs | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/abci/Cargo.toml b/abci/Cargo.toml index 3064284..8fd7b4f 100644 --- a/abci/Cargo.toml +++ b/abci/Cargo.toml @@ -21,7 +21,7 @@ server = [ "dep:futures", ] -grpc-server = ["tenderdash-proto/grpc-server"] +grpc = ["tenderdash-proto/grpc"] crypto = ["dep:lhash"] tcp = ["server"] unix = ["server"] diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs index b2969ba..9d4a1ce 100644 --- a/abci/tests/grpc.rs +++ b/abci/tests/grpc.rs @@ -6,7 +6,7 @@ //! This test should be implemented in the tenderdash-proto crate; however, it //! is implemented here to use already existing docker container testing //! logic. -#![cfg(feature = "grpc-server")] +#![cfg(feature = "grpc")] use std::sync::Arc; diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 06a480d..a9851d2 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -16,13 +16,16 @@ description = """ all-features = true [features] -default = ["grpc-server"] +default = ["grpc"] # Enable standard library support std = ["prost/std", "prost-types/std"] # Build gRPC server -grpc-server = ["tenderdash-proto-compiler/server", "std"] -grpc-client = ["tenderdash-proto-compiler/client", "std"] +grpc = [ + "std", + "tenderdash-proto-compiler/server", + "tenderdash-proto-compiler/client", +] [dependencies] prost = { version = "0.12", default-features = false, features = [ diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 047ee86..ca3ad64 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -40,7 +40,7 @@ pub mod serializers; use prelude::*; pub use tenderdash::meta::ABCI_VERSION; -#[cfg(any(feature = "grpc-server", feature = "grpc-client"))] +#[cfg(feature = "grpc")] pub use tonic; /// Allows for easy Google Protocol Buffers encoding and decoding of domain From 1e9689df2c0dafb65ed636a67bac460de22c2b26 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:36:22 +0100 Subject: [PATCH 17/23] chore: self review, simplify features --- abci/Cargo.toml | 10 +++++++++- abci/tests/grpc.rs | 2 +- proto/Cargo.toml | 11 ++++++++++- proto/build.rs | 3 --- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/abci/Cargo.toml b/abci/Cargo.toml index 8fd7b4f..6b9a7b1 100644 --- a/abci/Cargo.toml +++ b/abci/Cargo.toml @@ -11,7 +11,15 @@ description = """tenderdash-abci provides a simple framework with which to build low-level applications on top of Tenderdash.""" [features] -default = ["server", "docker-tests", "crypto", "tcp", "unix", "tracing-span"] +default = [ + "server", + "docker-tests", + "crypto", + "tcp", + "unix", + "grpc", + "tracing-span", +] # docker-tests includes integration tests that require docker to be available docker-tests = ["server"] server = [ diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs index 9d4a1ce..f0ce962 100644 --- a/abci/tests/grpc.rs +++ b/abci/tests/grpc.rs @@ -21,7 +21,7 @@ use tenderdash_abci::proto; use tonic::{async_trait, Response, Status}; #[cfg(feature = "docker-tests")] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[tokio::test] /// Test server listening on ipv4 address. /// /// See [tcp_server_test()]. diff --git a/proto/Cargo.toml b/proto/Cargo.toml index a9851d2..edc7a12 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -16,6 +16,14 @@ description = """ all-features = true [features] +# Features configuration. +# +# Note that, due to the way build.rs scripts work, change of features does not trigger +# regeneration of protobuf files. This means you need to be extra careful when changing +# features, as you might end up with outdated and/or conflicting generated files. +# +# Sometimes cleaning the build cache with `cargo clean` might be necessary to solve +# issues related to outdated generated files. default = ["grpc"] # Enable standard library support @@ -25,6 +33,7 @@ grpc = [ "std", "tenderdash-proto-compiler/server", "tenderdash-proto-compiler/client", + "dep:tonic", ] [dependencies] @@ -32,7 +41,7 @@ prost = { version = "0.12", default-features = false, features = [ "prost-derive", ] } prost-types = { version = "0.12", default-features = false } -tonic = { version = "0.11" } +tonic = { version = "0.11", optional = true } bytes = { version = "1.0", default-features = false, features = ["serde"] } serde = { version = "1.0", default-features = false, features = ["derive"] } subtle-encoding = { version = "0.5", default-features = false, features = [ diff --git a/proto/build.rs b/proto/build.rs index d29a2f8..80894aa 100644 --- a/proto/build.rs +++ b/proto/build.rs @@ -16,7 +16,4 @@ fn main() { println!("cargo:rerun-if-changed=Cargo.toml"); println!("cargo:rerun-if-env-changed=CARGO_PKG_VERSION"); println!("cargo:rerun-if-env-changed=TENDERDASH_COMMITISH"); - println!("cargo:rerun-if-env-changed=CARGO_FEATURE_STD"); - println!("cargo:rerun-if-env-changed=CARGO_FEATURE_GRPC_SERVER"); - println!("cargo:rerun-if-env-changed=CARGO_FEATURE_GRPC_CLIENT"); } From 85a09f1a26146a575e6dae2302e2268f7aa39c70 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:44:31 +0100 Subject: [PATCH 18/23] chore: comments --- abci/tests/grpc.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs index f0ce962..46ad8bf 100644 --- a/abci/tests/grpc.rs +++ b/abci/tests/grpc.rs @@ -89,6 +89,7 @@ async fn grpc_server_test(test_name: &str, bind_address: &str) { tracing::debug!("starting Tenderdash in Docker container"); let td = Arc::new(common::docker::TenderdashDocker::new( &container_name, + // TODO: replace with None once tenderdash dev is released Some("feat-ABCI-protocol-env-var"), &socket_uri, )); @@ -100,8 +101,6 @@ async fn grpc_server_test(test_name: &str, bind_address: &str) { .await .expect("start tenderdash"); - // tracing::debug!(?result, "connection handled"); - // assert!(matches!(result, Ok(()))); tokio::select! { _ = tokio::time::sleep(tokio::time::Duration::from_secs(60)) => { panic!("Test timed out"); From 448a2ce4178d035701ef606b3538feb0089e90e6 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:50:33 +0100 Subject: [PATCH 19/23] test(abci): grpc test uses latest tenderdash tag --- abci/tests/grpc.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/abci/tests/grpc.rs b/abci/tests/grpc.rs index 46ad8bf..b75516c 100644 --- a/abci/tests/grpc.rs +++ b/abci/tests/grpc.rs @@ -89,8 +89,7 @@ async fn grpc_server_test(test_name: &str, bind_address: &str) { tracing::debug!("starting Tenderdash in Docker container"); let td = Arc::new(common::docker::TenderdashDocker::new( &container_name, - // TODO: replace with None once tenderdash dev is released - Some("feat-ABCI-protocol-env-var"), + None, &socket_uri, )); common::docker::setup_td_logs_panic(&td); From 9c8d18a049606ad907d6710f3427409c4190be2f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:57:51 +0100 Subject: [PATCH 20/23] chore: bump version to 0.14.0-dev.7 for tenderdash v0.14.0-dev.3, abci 0.26.0 --- abci/Cargo.toml | 2 +- proto-compiler/Cargo.toml | 2 +- proto/Cargo.toml | 2 +- proto/build.rs | 3 ++- scripts/release.sh | 14 ++++++++------ 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/abci/Cargo.toml b/abci/Cargo.toml index 8c235ee..ce90065 100644 --- a/abci/Cargo.toml +++ b/abci/Cargo.toml @@ -1,5 +1,5 @@ [package] -version = "0.14.0-dev.6" +version = "0.14.0-dev.7" name = "tenderdash-abci" edition = "2021" license = "Apache-2.0" diff --git a/proto-compiler/Cargo.toml b/proto-compiler/Cargo.toml index efc4ab9..2033d63 100644 --- a/proto-compiler/Cargo.toml +++ b/proto-compiler/Cargo.toml @@ -1,5 +1,5 @@ [package] -version = "0.14.0-dev.6" +version = "0.14.0-dev.7" name = "tenderdash-proto-compiler" authors = ["Informal Systems ", "Dash Core Group"] edition = "2021" diff --git a/proto/Cargo.toml b/proto/Cargo.toml index edc7a12..0008338 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -1,5 +1,5 @@ [package] -version = "0.14.0-dev.6" +version = "0.14.0-dev.7" name = "tenderdash-proto" edition = "2021" license = "Apache-2.0" diff --git a/proto/build.rs b/proto/build.rs index 80894aa..a3e2fa4 100644 --- a/proto/build.rs +++ b/proto/build.rs @@ -1,7 +1,8 @@ use std::env; fn main() { - const DEFAULT_VERSION: &str = "v0.14.0-dev.2"; + // default Tenderdash version to use if TENDERDASH_COMMITISH is not set + const DEFAULT_VERSION: &str = "v0.14.0-dev.3"; // check if TENDERDASH_COMMITISH is already set; if not, set it to the current // version diff --git a/scripts/release.sh b/scripts/release.sh index fe5198e..0670b8d 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,6 +1,6 @@ #! /bin/bash -PLATFORM_DIR="$(realpath "$(dirname $0)/../../platform")" +PLATFORM_DIR="$(realpath "$(dirname "$0")/../../platform")" function help() { cat </dev/null if [ -d "$PLATFORM_DIR" ]; then rs_tenderdash="git = \"https:\/\/github.com\/dashpay\/rs-tenderdash-abci\", version = \"$rs_tenderdash_abci_version\" " echo "INFO: Updating references to tenderdash-abci / tenderdash-proto in $PLATFORM_DIR" - sed -i "s/^tenderdash-abci = { git = .* }/tenderdash-abci = { $rs_tenderdash }/" ${PLATFORM_DIR}/packages/*/Cargo.toml - sed -i "s/^tenderdash-proto = { git = .* }/tenderdash-proto = { $rs_tenderdash }/" ${PLATFORM_DIR}/packages/*/Cargo.toml + sed -i "s/^tenderdash-abci = { git = .* }/tenderdash-abci = { $rs_tenderdash }/" "${PLATFORM_DIR}"/packages/*/Cargo.toml + sed -i "s/^tenderdash-proto = { git = .* }/tenderdash-proto = { $rs_tenderdash }/" "${PLATFORM_DIR}"/packages/*/Cargo.toml else - echo WARN: Dash Platform not found in $PLATFORM_DIR, skipping + echo "WARN: Dash Platform not found in $PLATFORM_DIR, skipping" fi From bc3b05f23a7b4ae9b51f8730ce1f69e7b792cd8b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:28:00 +0100 Subject: [PATCH 21/23] chore: fix missing serde serialization --- proto-compiler/src/constants.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index 25f27f3..369dddf 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -63,6 +63,14 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.types.DuplicateVoteEvidence", SERIALIZED), (".tendermint.types.Vote", SERIALIZED), (".tendermint.types.BlockID", SERIALIZED), + (".tendermint.types.ConsensusParams", SERIALIZED), + (".tendermint.types.ABCIParams", SERIALIZED), + (".tendermint.types.BlockParams", SERIALIZED), + (".tendermint.types.EvidenceParams", SERIALIZED), + (".tendermint.types.ValidatorParams", SERIALIZED), + (".tendermint.types.VersionParams", SERIALIZED), + (".tendermint.types.SynchronyParams", SERIALIZED), + (".tendermint.types.TimeoutParams", SERIALIZED), (".tendermint.types.PartSetHeader", SERIALIZED), (".tendermint.types.LightClientAttackEvidence", SERIALIZED), (".tendermint.types.LightBlock", SERIALIZED), From b043ae7ee2789363a647316053d95a27909c9cba Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:10:46 +0100 Subject: [PATCH 22/23] fix: wrong json serialization of ConsensusParams --- proto-compiler/src/constants.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index 369dddf..25f27f3 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -63,14 +63,6 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.types.DuplicateVoteEvidence", SERIALIZED), (".tendermint.types.Vote", SERIALIZED), (".tendermint.types.BlockID", SERIALIZED), - (".tendermint.types.ConsensusParams", SERIALIZED), - (".tendermint.types.ABCIParams", SERIALIZED), - (".tendermint.types.BlockParams", SERIALIZED), - (".tendermint.types.EvidenceParams", SERIALIZED), - (".tendermint.types.ValidatorParams", SERIALIZED), - (".tendermint.types.VersionParams", SERIALIZED), - (".tendermint.types.SynchronyParams", SERIALIZED), - (".tendermint.types.TimeoutParams", SERIALIZED), (".tendermint.types.PartSetHeader", SERIALIZED), (".tendermint.types.LightClientAttackEvidence", SERIALIZED), (".tendermint.types.LightBlock", SERIALIZED), From 94eb1d5ec55f8874ea71e00ccfded5ef1be96a6b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:31:51 +0100 Subject: [PATCH 23/23] build(deps): update chrono and replace deprecated fn calls --- proto/Cargo.toml | 2 +- proto/src/serializers/timestamp.rs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 430b464..3a7f060 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -33,7 +33,7 @@ time = { version = "0.3", default-features = false, features = [ "parsing", ] } flex-error = { version = "0.4.4", default-features = false } -chrono = { version = "0.4.24", default-features = false } +chrono = { version = "0.4.35", default-features = false } derive_more = { version = "0.99.17" } diff --git a/proto/src/serializers/timestamp.rs b/proto/src/serializers/timestamp.rs index 945f881..41bf1f0 100644 --- a/proto/src/serializers/timestamp.rs +++ b/proto/src/serializers/timestamp.rs @@ -47,8 +47,9 @@ pub trait ToMilis { impl ToMilis for Timestamp { /// Convert protobuf timestamp into miliseconds since epoch fn to_milis(&self) -> u64 { - chrono::NaiveDateTime::from_timestamp_opt(self.seconds, self.nanos as u32) + chrono::DateTime::from_timestamp(self.seconds, self.nanos as u32) .unwrap() + .to_utc() .timestamp_millis() .try_into() .expect("timestamp value out of u64 range") @@ -75,12 +76,13 @@ impl FromMilis for Timestamp { /// /// Panics when `millis` don't fit `i64` type fn from_milis(millis: u64) -> Self { - let dt = chrono::NaiveDateTime::from_timestamp_millis( + let dt = chrono::DateTime::from_timestamp_millis( millis .try_into() .expect("milliseconds timestamp out of i64 range"), ) - .expect("cannot parse timestamp"); + .expect("cannot parse timestamp") + .to_utc(); Self { nanos: dt.timestamp_subsec_nanos() as i32,