Internal Overhaul: RatchetManager, HeaderObfuscator, Refactoring

.github/workflows/validate.yml

@@ -152,7 +152,7 @@ jobs:
       - run: cargo clippy --tests --examples -- -D warnings
       - run: cargo fmt --check
       - run: RUSTDOCFLAGS="-D warnings" cargo make docs
-      - run: cargo test --package citadel_sdk --doc
+      - run: cargo test --doc
       #      - name: cargo build pq_kems
   #        run: cargo build --bin pq_kems_test
   #      # Run with valgrind

Makefile.toml

@@ -10,6 +10,10 @@ args = ["clean"]
 command = "cargo"
 args = ["clippy", "--all"]
 
+[tasks.check]
+command = "cargo"
+args = ["clippy", "--tests", "--examples", "--all", "--", "-D", "warnings"]
+
 [tasks.clippy-tests]
 command = "cargo"
 args = ["clippy", "--all", "--tests", "--features=localhost-testing,multi-threaded"]
@@ -47,13 +51,13 @@ dependencies = ["install-binstall"]
 command = "cargo"
 description = "Tests all available unit/integration tests locally without using SQL/redis backends and appropriate localhost network settings"
 env = { "SKIP_EXT_BACKENDS" = "true" }
-args = ["nextest", "run", "--features", "localhost-testing"]
+args = ["nextest", "run", "--features", "localhost-testing,multi-threaded"]
 dependencies = ["install-nextest"]
 
 [tasks.test]
 command = "cargo"
 description = "Tests all available unit/integration tests locally using SQL/redis backends and appropriate localhost network settings"
-condition = { env_set = [ "TESTING_SQL_SERVER_ADDR_CLIENT", "TESTING_SQL_SERVER_ADDR_SERVER" ], env_not_set = ["SKIP_EXT_BACKENDS"] }
+condition = { env_set = ["TESTING_SQL_SERVER_ADDR_CLIENT", "TESTING_SQL_SERVER_ADDR_SERVER"], env_not_set = ["SKIP_EXT_BACKENDS"] }
 args = ["nextest", "run", "--features", "localhost-testing"]
 dependencies = ["install-nextest"]
 
@@ -250,41 +254,41 @@ run_task = [
 install_crate = { crate_name = "cargo-workspaces", binary = "cargo", test_arg = ["workspaces", "--help"] }
 
 [tasks.publish-bump-version-patch]
-condition = { env_set = [ "CARGO_REGISTRY_TOKEN" ] }
+condition = { env_set = ["CARGO_REGISTRY_TOKEN"] }
 command = "cargo"
 args = ["workspaces", "version", "-y", "minor"]
 
 [tasks.publish-bump-version-minor]
-condition = { env_set = [ "CARGO_REGISTRY_TOKEN" ] }
+condition = { env_set = ["CARGO_REGISTRY_TOKEN"] }
 command = "cargo"
 args = ["workspaces", "version", "-y", "minor"]
 
 [tasks.publish-bump-version-major]
-condition = { env_set = [ "CARGO_REGISTRY_TOKEN" ] }
+condition = { env_set = ["CARGO_REGISTRY_TOKEN"] }
 command = "cargo"
 args = ["workspaces", "version", "-y", "major"]
 
 [tasks.publish-patch]
 command = "cargo"
-condition = { env_set = [ "CARGO_REGISTRY_TOKEN" ] }
+condition = { env_set = ["CARGO_REGISTRY_TOKEN"] }
 args = ["workspaces", "publish", "-y", "--token", "${CARGO_REGISTRY_TOKEN}", "--allow-dirty", "patch", "${@}"]
 dependencies = ["publish-install-deps"]
 
 [tasks.publish-minor]
 command = "cargo"
-condition = { env_set = [ "CARGO_REGISTRY_TOKEN" ] }
+condition = { env_set = ["CARGO_REGISTRY_TOKEN"] }
 args = ["workspaces", "publish", "-y", "--token", "${CARGO_REGISTRY_TOKEN}", "--allow-dirty", "minor", "${@}"]
 dependencies = ["publish-install-deps"]
 
 [tasks.publish-major]
 command = "cargo"
-condition = { env_set = [ "CARGO_REGISTRY_TOKEN" ] }
+condition = { env_set = ["CARGO_REGISTRY_TOKEN"] }
 args = ["workspaces", "publish", "-y", "--token", "${CARGO_REGISTRY_TOKEN}", "--allow-dirty", "major", "${@}"]
 dependencies = ["publish-install-deps"]
 
 [tasks.check-docs]
 command = "cargo"
-args = ["test", "--package", "citadel_sdk", "--doc"]
+args = ["test", "--package", "citadel_sdk", "--doc", "${@}"]
 
 [tasks.install-cargo-wasix]
 command = "cargo"

README.md

@@ -56,7 +56,7 @@ Multiple Key Encapsulation Mechanism (KEM) families:
 - [NTRU](https://ntru.org/) (Sntrup761)
 
 Novel Multi-layered Security Architecture:
-- [Patent-pending](#-patent-and-open-source-commitment) multi-layered ratcheting algorithm
+- [Patent-pending (allowed)](#-patent-and-open-source-commitment) 3D matrix ratcheting algorithm
 - Per-message re-keying mechanism
 - Multi-layered key exchange protocol
 - Multi-layered encryption with customizable algorithms
@@ -166,7 +166,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 The Citadel Protocol implements a novel multi-layered security approach that goes beyond traditional encryption methods:
 
 ### 🔄 Multi-layered Ratcheting
-- Implements an advanced patent-pending ratcheting algorithm that operates across multiple security layers
+- Implements an advanced patent-pending (allowed as of Dec 2024) ratcheting algorithm that operates across multiple security layers
 - Each layer maintains its own independent key schedule
 - Provides enhanced forward secrecy by frequently rotating keys at different intervals
 - Resistant against quantum attacks through post-quantum primitives
@@ -308,6 +308,8 @@ The Citadel Protocol offers flexible data persistence options to suit various de
 
 ## 📜 Patent and Open Source Commitment
 
+Status: Allowed as of December 2024
+
 The Citadel Protocol's core technology is a [patent-pending innovative security architecture](https://image-ppubs.uspto.gov/dirsearch-public/print/downloadPdf/20230403261) that combines multiple novel features into a unique, highly secure communication system. Despite the patent protection, we remain committed to keeping this technology free and open source for the benefit of the entire community. This approach ensures that:
 
 - The protocol remains freely available for everyone to use

async_ip/src/lib.rs

@@ -1,4 +1,64 @@
-//! An asynchronous client used to obtain one's global Ipv6 or Ipv4 address
+//! # Async IP Resolution
+//!
+//! A lightweight, asynchronous client for obtaining global IPv4 and IPv6 addresses.
+//! This crate provides reliable IP address resolution using multiple fallback services
+//! and concurrent requests for improved reliability.
+//!
+//! ## Features
+//!
+//! - Async IP address resolution
+//! - Support for both IPv4 and IPv6
+//! - Multiple fallback services
+//! - Concurrent resolution for improved reliability
+//! - Internal IP address detection
+//! - WebAssembly support
+//! - Custom HTTP client support
+//!
+//! ## Usage
+//!
+//! ```rust,no_run
+//! use async_ip::get_all;
+//! use citadel_io::tokio;
+//!
+//! #[tokio::main(flavor = "current_thread")]
+//! async fn main() -> Result<(), async_ip::IpRetrieveError> {
+//!     // Get both internal and external IP addresses
+//!     use reqwest::Client;
+//! let ip_info = get_all::<Client>(None).await?;
+//!     println!("External IPv6: {:?}", ip_info.external_ipv6);
+//!     println!("Internal IPv4: {:?}", ip_info.internal_ip);
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Advanced Usage
+//!
+//! ```rust,no_run
+//! use async_ip::{get_all_multi_concurrent, get_default_client};
+//! use citadel_io::tokio;
+//!
+//! #[tokio::main(flavor = "current_thread")]
+//! async fn main() -> Result<(), async_ip::IpRetrieveError> {
+//!     // Use multiple services concurrently with a custom client
+//!     let client = get_default_client();
+//!     let ip_info = get_all_multi_concurrent(Some(client)).await?;
+//!     println!("External IPs: {:?}", ip_info);
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## WebAssembly Support
+//!
+//! When compiled with the `wasm` target, this crate uses a lightweight HTTP client
+//! suitable for WebAssembly environments. The functionality remains the same, but
+//! some features (like internal IP detection) may be limited.
+//!
+//! ## Error Handling
+//!
+//! The crate uses a custom `IpRetrieveError` type that wraps various error
+//! conditions that may occur during IP resolution, including network errors
+//! and parsing failures.
+
 #![deny(
     missing_docs,
     trivial_numeric_casts,
@@ -176,10 +236,12 @@ fn addr(addr: &str) -> Option<SocketAddr> {
 }
 
 #[cfg(not(target_family = "wasm"))]
-fn get_default_client() -> Client {
+/// Returns a default client
+pub fn get_default_client() -> Client {
     Client::builder().tcp_nodelay(true).build().unwrap()
 }
 #[cfg(target_family = "wasm")]
+/// Returns a default client
 fn get_default_client() -> UreqClient {
     UreqClient
 }

citadel_crypt/Cargo.toml

@@ -36,6 +36,7 @@ fcm = []
 async-trait = { workspace = true }
 bincode = { workspace = true }
 serde = { workspace = true, features=["rc", "derive"] }
+serde-big-array = { workspace = true }
 futures = { workspace = true }
 log = { workspace = true }
 byteorder = { workspace = true }
@@ -52,6 +53,8 @@ auto_impl = { workspace = true }
 zeroize = { workspace = true, features = ["zeroize_derive", "alloc", "serde"] }
 citadel_types = { workspace = true }
 uuid = { version = "1.8.0", features = ["v4"] }
+atomic = { workspace = true }
+bytemuck = { workspace = true, features = ["derive"] }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 rayon = { workspace = true }
@@ -62,6 +65,7 @@ num_cpus = { workspace = true }
 citadel_logging = { workspace = true }
 rstest = { workspace = true }
 lazy_static = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
 
 [lib]
 doctest = false

citadel_crypt/src/argon/argon_container.rs

@@ -1,3 +1,59 @@
+//! # Argon2 Password Hashing Container
+//!
+//! This module provides a secure, asynchronous wrapper around the Argon2 password hashing
+//! algorithm. It implements both client-side and server-side password handling with
+//! configurable parameters and secure memory management.
+//!
+//! ## Features
+//! - Asynchronous password hashing and verification
+//! - Configurable Argon2id parameters (memory, time, parallelism)
+//! - Secure memory handling with SecBuffer
+//! - Automatic salt generation
+//! - Support for associated data and secret keys
+//! - Client and server container types
+//!
+//! ## Usage Example
+//! ```rust
+//! use citadel_crypt::argon::argon_container::{ServerArgonContainer, ArgonSettings, ClientArgonContainer, AsyncArgon};
+//! use citadel_types::crypto::SecBuffer;
+//!
+//! async fn hash_password() {
+//!     // Create settings with default parameters
+//!     let settings = ArgonSettings::new_defaults(vec![1, 2, 3]); // Associated data
+//!     
+//!     // Create client container
+//!     let client = ClientArgonContainer::from(settings);
+//!     
+//!     // Hash a password
+//!     let password = SecBuffer::from("my_secure_password");
+//!     let hashed = client.hash_insecure_input(password).await.unwrap();
+//!     
+//!     // Verify a password (server-side)
+//!     let server_container = ServerArgonContainer::new(
+//!         client.settings.clone(),
+//!         hashed
+//!     );
+//!     
+//!     let verify_result = AsyncArgon::verify(
+//!         SecBuffer::from("my_secure_password"),
+//!         server_container
+//!     ).await.unwrap();
+//! }
+//! ```
+//!
+//! ## Important Notes
+//! - Uses Argon2id variant for optimal security
+//! - Memory-hard algorithm with configurable cost parameters
+//! - Handles password hashing on blocking threads
+//! - Provides secure memory zeroing through SecBuffer
+//! - Supports custom associated data for domain separation
+//!
+//! ## Related Components
+//! - [`SecBuffer`](citadel_types::crypto::SecBuffer): Secure memory management
+//! - [`AsyncArgon`]: Asynchronous hashing interface
+//! - [`ArgonSettings`]: Configuration parameters
+//! - Argon2 password hashing algorithm
+
 use argon2::Config;
 use citadel_io::tokio;
 use citadel_types::crypto::SecBuffer;

citadel_crypt/src/argon/autotuner.rs

@@ -1,3 +1,56 @@
+//! # Argon2 Parameter Auto-tuner
+//!
+//! Automatically determines optimal Argon2 parameters for password hashing based on system capabilities
+//! and performance requirements. This module implements the recommendations from ORY's Argon2 parameter
+//! selection guidelines.
+//!
+//! ## Features
+//!
+//! * Dynamic parameter tuning based on available system memory
+//! * Multi-threaded optimization using available CPU cores
+//! * Memory-first tuning strategy for optimal security
+//! * Iterative time-cost adjustment
+//! * Configurable minimum execution time
+//! * Support for custom hash lengths and secret keys
+//!
+//! ## Usage Example
+//!
+//! ```rust
+//! use citadel_crypt::argon::autotuner::calculate_optimal_argon_params;
+//! use citadel_crypt::misc::CryptError;
+//!
+//! async fn configure_argon() -> Result<(), CryptError> {
+//!     // Configure Argon2 to take at least 500ms
+//!     let optimal_params = calculate_optimal_argon_params(
+//!         500,                    // minimum milliseconds
+//!         None,                   // use default hash length
+//!         Some(b"secret".to_vec()) // optional secret key
+//!     ).await?;
+//!     
+//!     println!("Optimal parameters:");
+//!     println!("Memory cost: {} KB", optimal_params.mem_cost);
+//!     println!("Time cost: {}", optimal_params.time_cost);
+//!     println!("Parallelism: {}", optimal_params.lanes);
+//!     
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Important Notes
+//!
+//! * Should be run in release mode for accurate results
+//! * Memory cost is automatically capped at available system memory
+//! * Parameters are tuned for the specific CPU running the autotuner
+//! * Memory-cost is prioritized over time-cost for better security
+//! * Results may vary between runs due to system load
+//!
+//! ## Related Components
+//!
+//! * `argon_container`: Core Argon2 implementation and settings
+//! * `SecBuffer`: Secure memory management for passwords
+//! * `CryptError`: Error handling for cryptographic operations
+//!
+
 use crate::argon::argon_container::{
     ArgonDefaultServerSettings, ArgonSettings, ArgonStatus, AsyncArgon, DEFAULT_HASH_LENGTH,
 };