From 9876fd3245a818f33f2e39f2792b7aed8377475b Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 14 Oct 2024 18:04:13 +1100 Subject: [PATCH 1/6] feat(filemanager): add file change schema definition and generation --- .../stateless/stacks/filemanager/Cargo.lock | 50 +++++++++- .../stateless/stacks/filemanager/Makefile | 2 +- .../filemanager-api-server/src/main.rs | 9 +- .../filemanager/filemanager-build/Cargo.toml | 7 +- .../filemanager-build/src/error.rs | 2 + .../filemanager/filemanager-build/src/lib.rs | 99 +++++++++++++------ .../filemanager/filemanager-build/src/main.rs | 24 +++-- .../filemanager-build/src/schema.rs | 99 +++++++++++++++++++ .../stacks/filemanager/filemanager/Cargo.toml | 8 +- .../filemanager/src/events/aws/mod.rs | 10 +- 10 files changed, 251 insertions(+), 59 deletions(-) create mode 100644 lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs diff --git a/lib/workload/stateless/stacks/filemanager/Cargo.lock b/lib/workload/stateless/stacks/filemanager/Cargo.lock index a08e5d20d..e38bbc58f 100644 --- a/lib/workload/stateless/stacks/filemanager/Cargo.lock +++ b/lib/workload/stateless/stacks/filemanager/Cargo.lock @@ -1992,6 +1992,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + [[package]] name = "ecdsa" version = "0.14.8" @@ -2176,10 +2182,8 @@ dependencies = [ "bytes", "chrono", "csv", - "dotenvy", "envy", "filemanager", - "filemanager-build", "flate2", "futures", "hex", @@ -2187,7 +2191,6 @@ dependencies = [ "json-patch", "lazy_static", "md5", - "miette", "mockall", "mockall_double", "orc-rust", @@ -2243,6 +2246,7 @@ dependencies = [ name = "filemanager-build" version = "0.1.0" dependencies = [ + "chrono", "clap", "clap_builder", "dotenvy", @@ -2250,11 +2254,14 @@ dependencies = [ "miette", "prettyplease", "quote", + "schemars", "sea-orm-cli", "serde", + "serde_json", "syn 2.0.77", "thiserror", "tokio", + "uuid", ] [[package]] @@ -4449,6 +4456,32 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "schemars" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" +dependencies = [ + "chrono", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "schemars_derive" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.77", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4704,6 +4737,17 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "serde_dynamo" version = "4.2.14" diff --git a/lib/workload/stateless/stacks/filemanager/Makefile b/lib/workload/stateless/stacks/filemanager/Makefile index 664e71225..c70b0e8d2 100644 --- a/lib/workload/stateless/stacks/filemanager/Makefile +++ b/lib/workload/stateless/stacks/filemanager/Makefile @@ -44,7 +44,7 @@ docker-api: docker-postgres ## Build related commands entity: docker-postgres - @cargo run -p filemanager-build -- --out-dir filemanager/src/database/entities + @cargo run -p filemanager-build -- entities --out-dir filemanager/src/database/entities build: docker-postgres entity @cargo build --all-features --all-targets watch: docker-postgres entity diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs index 6412b9e88..9e0d4c46b 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs @@ -36,15 +36,14 @@ struct Args { /// Apply migrations before starting the server. #[arg(short, long, default_value_t = false, env)] migrate: bool, - /// Mock testing data for the API to use. Records are generated incrementally - /// with integers as buckets and keys. + /// Subcommands for the filemanager CLI. #[command(subcommand)] - mock_data: Option, + mock_data: Option, } /// Mock data into the filemanager database. #[derive(Subcommand, Debug)] -pub enum MockData { +pub enum SubCommands { /// Mock data into the filemanager database. Note that this should only be run once on the same /// postgres database to avoid duplicate key errors. Mock { @@ -100,7 +99,7 @@ async fn main() -> Result<()> { .await?; } - if let Some(MockData::Mock { + if let Some(SubCommands::Mock { n_records, bucket_divisor, key_divisor, diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml index dacd89f79..e16572edc 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml @@ -15,7 +15,12 @@ sea-orm-cli = { version = "1.1.0-rc.1", default-features = false, features = ["c tokio = { version = "1", features = ["macros", "rt-multi-thread", "process"] } miette = { version = "7", features = ["fancy"] } serde = { version = "1", features = ["derive"] } +serde_json = "1" + quote = "1" syn = { version = "2", features = ["full", "extra-traits", "parsing", "visit-mut"] } prettyplease = "0.2" -heck = "0.5" \ No newline at end of file +heck = "0.5" +schemars = { version = "0.8", features = ["chrono", "uuid1"] } +uuid = { version = "1", features = ["serde"] } +chrono = { version = "0.4", features = ["serde"] } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/error.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/error.rs index 36a202ef3..ddfa14498 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/error.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/error.rs @@ -21,6 +21,8 @@ pub enum ErrorKind { EntityGeneration(String), #[error("Error generating OpenAPI definitions: {0}")] OpenAPIGeneration(String), + #[error("Error generating JSON schemas: {0}")] + SchemaGeneration(String), #[error("Missing or incorrect environment variables: {0}")] LoadingEnvironment(String), #[error("io error: {0}")] diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs index 3b8ffd37d..274c8522d 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, Subcommand}; use std::env::var; use std::path::{Path, PathBuf}; @@ -10,23 +10,53 @@ use crate::error::ErrorKind::LoadingEnvironment; pub mod error; pub mod gen_entities; pub mod gen_openapi; +pub mod schema; -/// Run the filemanager-build tool to generate sea-orm entities. This always generates entities +/// Run the filemanager-build tool to generate sea-orm entities or generate JSON schemas. This always generates entities /// if a database url is defined, otherwise it skips generating entities if `--skip-if-no-database` /// is used, or errors if it is not. #[derive(Parser, Debug)] #[command(version, about, long_about = None)] pub struct Config { - /// Skip generating if no database URL is set. This allows assuming that any - /// checked-in code is up-to-date with the database and proceeding with the build. - #[arg(short, long, default_value_t = false, env)] - pub(crate) skip_if_no_database: bool, - /// The database URL to use for generating entities. - #[arg(short, long, default_value = "", env)] - pub(crate) database_url: String, - /// The output directory. - #[arg(short, long, env)] - pub(crate) out_dir: PathBuf, + /// The filemanager-build subcommands. + #[command(subcommand)] + pub(crate) sub_commands: SubCommands, +} + +impl Config { + /// Get the subcommands. + pub fn into_inner(self) -> SubCommands { + self.sub_commands + } +} + +/// The filemanager-build subcommands. +#[derive(Subcommand, Debug, Clone)] +pub enum SubCommands { + /// Generate sea-orm entities. This always generates entities if a database url is defined, + /// otherwise it skips generating entities if `--skip-if-no-database` is used, or errors if it + /// is not. + Entities { + /// Skip generating if no database URL is set. This allows assuming that any + /// checked-in code is up-to-date with the database and proceeding with the build. + #[arg(short, long, default_value_t = false, env)] + skip_if_no_database: bool, + /// The database URL to use for generating entities. + #[arg(short, long, default_value = "", env)] + database_url: String, + /// The output directory. + #[arg(short, long, env)] + out_dir: PathBuf, + }, + /// Generate the JSON schemas that filemanager can consume. + Schemas { + /// The output file. + #[arg(short, long, env)] + out_file: PathBuf, + /// Whether to generate schema examples. + #[arg(short, long, default_value_t = true, env)] + examples: bool, + }, } impl Config { @@ -35,30 +65,37 @@ impl Config { pub fn load() -> Result { let args = Config::parse(); - if !args.skip_if_no_database && args.database_url.is_empty() { - return Err(LoadingEnvironment( - "Missing database URL and not skipping entity generation".to_string(), - ) - .into()); + if let SubCommands::Entities { + skip_if_no_database, + database_url, + .. + } = &args.sub_commands + { + if !skip_if_no_database && database_url.is_empty() { + return Err(LoadingEnvironment( + "Missing database URL and not skipping entity generation".to_string(), + ) + .into()); + } } Ok(args) } - /// Get whether to skip the generation if the database URL is empty. - pub fn skip_if_no_database(&self) -> bool { - self.skip_if_no_database && self.database_url.is_empty() - } - - /// Get the database url. - pub fn database_url(&self) -> &str { - &self.database_url - } - - /// Get the out dir. - pub fn out_dir(&self) -> &Path { - &self.out_dir - } + // /// Get whether to skip the generation if the database URL is empty. + // pub fn skip_if_no_database(&self) -> bool { + // self.skip_if_no_database && self.database_url.is_empty() + // } + // + // /// Get the database url. + // pub fn database_url(&self) -> &str { + // &self.database_url + // } + // + // /// Get the out dir. + // pub fn out_dir(&self) -> &Path { + // &self.out_dir + // } } /// Get the path of the workspace. diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs index 82a0faf03..97e186187 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs @@ -1,6 +1,7 @@ use filemanager_build::gen_entities::generate_entities; use filemanager_build::gen_openapi::generate_openapi; -use filemanager_build::Config; +use filemanager_build::schema::generate_schemas; +use filemanager_build::{Config, SubCommands}; use miette::Result; #[tokio::main] @@ -8,11 +9,22 @@ async fn main() -> Result<()> { let _ = dotenvy::dotenv(); let config = Config::load()?; - if !config.skip_if_no_database() { - generate_entities(config.out_dir(), config.database_url(), false).await?; - generate_openapi(config.out_dir()).await?; - } else { - println!("Skipping entity generation as no database url is defined, nothing to do.") + match config.into_inner() { + SubCommands::Entities { + skip_if_no_database, + database_url, + out_dir, + } => { + if !skip_if_no_database { + generate_entities(&out_dir, &database_url, false).await?; + generate_openapi(&out_dir).await?; + } else { + println!("Skipping entity generation as no database url is defined, nothing to do.") + } + } + SubCommands::Schemas { out_file, .. } => { + generate_schemas(&out_file).await?; + } } Ok(()) diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs new file mode 100644 index 000000000..2c3722e4a --- /dev/null +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs @@ -0,0 +1,99 @@ +//! Logic for AWS EventBridge schema for filemanager. +//! + +use crate::error::ErrorKind::SchemaGeneration; +use crate::error::Result; +use chrono::{DateTime, Utc}; +use schemars::{schema_for, JsonSchema}; +use serde::{Deserialize, Serialize}; +use serde_json::to_string_pretty; +use std::path::Path; +use tokio::fs; +use uuid::Uuid; + +/// A filemanager object state change event. +#[derive(Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct FileStateChange { + /// The type of the event. + detail_type: FileStateChangeType, + /// The detail of the event. + detail: Detail, + /// The source of the event. + source: String, + /// The version of the event. + version: Option, + /// The ID of the event. + id: Option, + /// The account ID of the event. + account: Option, + /// The time the event was generated. + time: Option>, + /// The region the event was generated in. + region: Option, + /// The resources of the event. + resources: Option>, +} + +/// The type of S3 object state change event. +#[derive(Deserialize, Serialize, JsonSchema)] +#[serde(rename = "FileStateChange")] +pub enum FileStateChangeType { + FileStateChange, +} + +/// The detail of an S3 object state change event. +#[derive(Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct Detail { + /// The key name prefix to apply this transition to. Applies to all objects by default. + prefix: Option, + /// A set of transitions to apply. Performs no transition by default. + transitions: Option>, + /// The number of days to wait before expiring the object. Performs no expiry by default. + expiration: Option, + /// Apply the transition to objects that are less than the specified size. Applies to objects + /// of any size by default. + #[schemars(range(min = 1))] + object_size_less_than: Option, + /// Apply the transition to objects that are greater than the specified size. Applies to objects + /// of any size by default. + object_size_greater_than: Option, + /// Whether to keep the expiry rule when the object is moved. By default, the rule is removed + /// when an object is moved. + keep_on_move: bool, +} + +/// An S3 storage class transition. +#[derive(Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct Transition { + /// The storage class to transition to. + storage_class: StorageClass, + /// The number of days to wait before transitioning the object. + days: String, +} + +/// AWS storage types. +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum StorageClass { + StandardIa, + IntelligentTiering, + OnezoneIa, + Glacier, + GlacierIr, + DeepArchive, +} + +/// Generate the JSON schemas. +pub async fn generate_schemas(out_file: &Path) -> Result<()> { + let schema = to_string_pretty(&schema_for!(FileStateChange)) + .map_err(|err| SchemaGeneration(err.to_string()))?; + + fs::write(out_file, schema) + .await + .map_err(|err| SchemaGeneration(err.to_string()))?; + + Ok(()) +} diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml index 1e9f54be3..7fe142956 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml @@ -49,7 +49,7 @@ json-patch = "2" # General chrono = { version = "0.4", features = ["serde"] } thiserror = "1" -uuid = { version = "1", features = ["v7"] } +uuid = { version = "1", features = ["serde", "v7"] } mockall = "0.13" mockall_double = "0.3" itertools = "0.13" @@ -87,9 +87,3 @@ aws-sdk-s3 = { version = "1", features = ["test-util"] } # The migrate feature is required to run sqlx tests filemanager = { path = ".", features = ["migrate"] } - -[build-dependencies] -filemanager-build = { path = "../filemanager-build" } -miette = { version = "7", features = ["fancy"] } -tokio = { version = "1", features = ["macros"] } -dotenvy = "0.15" diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/events/aws/mod.rs index 4a6595cad..7e590db3b 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/events/aws/mod.rs @@ -19,21 +19,21 @@ pub mod collecter; pub mod inventory; pub mod message; -/// A wrapper around AWS storage types with sqlx support. +/// AWS storage types. #[derive(Debug, Eq, PartialEq, PartialOrd, Ord, Clone, sqlx::Type, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] #[sqlx(type_name = "storage_class")] pub enum StorageClass { - DeepArchive, - Glacier, - GlacierIr, + StandardIa, IntelligentTiering, OnezoneIa, + Glacier, + GlacierIr, + DeepArchive, Outposts, ReducedRedundancy, Snow, Standard, - StandardIa, } impl StorageClass { From 5c86dd805408c864fd27652563489b68823ac1da Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 17 Oct 2024 10:19:14 +1100 Subject: [PATCH 2/6] feat(filemanager): add file state change schema generation --- .../stateless/stacks/filemanager/Cargo.lock | 422 +++++++++++++++++- .../filemanager/filemanager-build/Cargo.toml | 7 +- .../filemanager-build/src/gen_schema.rs | 105 +++++ .../filemanager/filemanager-build/src/lib.rs | 7 +- .../filemanager/filemanager-build/src/main.rs | 6 +- .../src/{schema.rs => types.rs} | 72 ++- 6 files changed, 559 insertions(+), 60 deletions(-) create mode 100644 lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs rename lib/workload/stateless/stacks/filemanager/filemanager-build/src/{schema.rs => types.rs} (59%) diff --git a/lib/workload/stateless/stacks/filemanager/Cargo.lock b/lib/workload/stateless/stacks/filemanager/Cargo.lock index e38bbc58f..bacae0913 100644 --- a/lib/workload/stateless/stacks/filemanager/Cargo.lock +++ b/lib/workload/stateless/stacks/filemanager/Cargo.lock @@ -33,6 +33,7 @@ dependencies = [ "const-random", "getrandom", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -378,7 +379,7 @@ dependencies = [ "memchr", "num", "regex", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -1093,6 +1094,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -1139,6 +1155,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borrow-or-share" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" + [[package]] name = "brotli" version = "6.0.0" @@ -1175,6 +1197,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + [[package]] name = "byteorder" version = "1.5.0" @@ -1806,7 +1834,7 @@ dependencies = [ "indexmap 2.5.0", "itertools 0.12.1", "log", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -2051,6 +2079,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "encoding_rs" version = "0.8.34" @@ -2145,6 +2182,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata 0.4.7", + "regex-syntax 0.8.5", +] + [[package]] name = "fastrand" version = "2.1.1" @@ -2251,6 +2299,7 @@ dependencies = [ "clap_builder", "dotenvy", "heck 0.5.0", + "jsonschema", "miette", "prettyplease", "quote", @@ -2258,6 +2307,7 @@ dependencies = [ "sea-orm-cli", "serde", "serde_json", + "serde_with", "syn 2.0.77", "thiserror", "tokio", @@ -2337,6 +2387,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "flume" version = "0.11.0" @@ -2363,6 +2424,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fragile" version = "2.0.0" @@ -2830,6 +2901,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2846,6 +3035,18 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd69211b9b519e98303c015e21a007e293db403b6c85b9b124e133d25e242cdd" +dependencies = [ + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2984,9 +3185,36 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" dependencies = [ - "fluent-uri", + "fluent-uri 0.1.4", + "serde", + "serde_json", +] + +[[package]] +name = "jsonschema" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea85509e7320309cc8be62d8badb46f9525157bdc748bf2ec089cd4083a3f7e" +dependencies = [ + "ahash", + "anyhow", + "base64 0.22.1", + "bytecount", + "email_address", + "fancy-regex", + "fraction", + "idna 1.0.2", + "itoa", + "num-cmp", + "once_cell", + "percent-encoding", + "referencing", + "regex-syntax 0.8.5", + "reqwest", "serde", "serde_json", + "url", + "uuid-simd", ] [[package]] @@ -3167,6 +3395,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -3441,6 +3675,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.6" @@ -3560,9 +3800,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -4115,6 +4355,39 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "ref-cast" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "referencing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bdf02a06a0820fcb9ce064715b424f1cdd79e24f991990a92425afe11eaf4a" +dependencies = [ + "ahash", + "fluent-uri 0.3.2", + "once_cell", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.10.6" @@ -4124,7 +4397,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -4144,7 +4417,7 @@ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -4161,9 +4434,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -5235,6 +5508,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -5344,6 +5623,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "tempfile" version = "3.12.0" @@ -5465,6 +5755,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -5785,7 +6085,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", "serde", ] @@ -5796,6 +6096,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -5857,6 +6169,17 @@ dependencies = [ "serde", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "uuid", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.0" @@ -6235,6 +6558,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xmlparser" version = "0.13.6" @@ -6256,6 +6591,30 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -6277,12 +6636,55 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "zip" version = "1.1.4" diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml index e16572edc..95d75d4b1 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml @@ -14,13 +14,18 @@ dotenvy = "0.15" sea-orm-cli = { version = "1.1.0-rc.1", default-features = false, features = ["cli", "codegen", "runtime-tokio-rustls"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "process"] } miette = { version = "7", features = ["fancy"] } + serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_with = "3" quote = "1" syn = { version = "2", features = ["full", "extra-traits", "parsing", "visit-mut"] } prettyplease = "0.2" heck = "0.5" schemars = { version = "0.8", features = ["chrono", "uuid1"] } -uuid = { version = "1", features = ["serde"] } +uuid = { version = "1", features = ["v7", "serde"] } chrono = { version = "0.4", features = ["serde"] } + +[dev-dependencies] +jsonschema = "0.23" \ No newline at end of file diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs new file mode 100644 index 000000000..a5c66af7d --- /dev/null +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs @@ -0,0 +1,105 @@ +//! Logic for AWS EventBridge schema for filemanager. +//! + +use crate::error::ErrorKind::SchemaGeneration; +use crate::error::Result; +use crate::types::FileStateChange; +use crate::types::{Detail, FileStateChangeType, StorageClass, Transition}; +use chrono::Utc; +use schemars::schema_for; +use serde::Serialize; +use serde_json::to_string_pretty; +use std::path::Path; +use tokio::fs::write; +use uuid::Uuid; + +/// Write the schema to the out directory. +pub async fn write_schema(out_dir: &Path) -> Result<()> { + let schema = generate_schema().await?; + + Ok(write(out_dir.join("schema.json"), schema).await?) +} + +/// Generate the JSON schemas. +pub async fn generate_schema() -> Result { + to_json_string(&schema_for!(FileStateChange)).await +} + +/// Generate an example schema for an expiration rule. +pub async fn generate_example_one() -> Result { + let example = FileStateChange { + detail_type: FileStateChangeType::FileStateChange, + detail: Detail { + prefix: None, + transitions: None, + expiration: Some(30), + object_size_less_than: None, + object_size_greater_than: Some(1024), + keep_on_move: true, + }, + source: "orcabus.filemanager".to_string(), + version: Some("0".to_string()), + id: Some(Uuid::now_v7()), + account: Some("123456789012".to_string()), + time: Some(Utc::now()), + region: Some("ap-southeast-2".to_string()), + resources: Some(vec![]), + }; + + to_json_string(&example).await +} + +/// Generate an example schema for transition rules and an expiration rule. +pub async fn generate_example_two() -> Result { + let example = FileStateChange { + detail_type: FileStateChangeType::FileStateChange, + detail: Detail { + prefix: Some("key_prefix/".to_string()), + transitions: Some(vec![ + Transition { + storage_class: StorageClass::StandardIa, + days: 30, + }, + Transition { + storage_class: StorageClass::Glacier, + days: 90, + }, + ]), + expiration: Some(365), + object_size_less_than: Some(1024), + object_size_greater_than: None, + keep_on_move: false, + }, + source: "orcabus.filemanager".to_string(), + version: Some("0".to_string()), + id: Some(Uuid::now_v7()), + account: Some("123456789012".to_string()), + time: Some(Utc::now()), + region: Some("ap-southeast-2".to_string()), + resources: Some(vec![]), + }; + + to_json_string(&example).await +} + +async fn to_json_string(value: &T) -> Result { + Ok(to_string_pretty(value).map_err(|err| SchemaGeneration(err.to_string()))?) +} + +#[cfg(test)] +mod tests { + use super::*; + use jsonschema::is_valid; + use serde_json::from_str; + + #[tokio::test] + async fn test_schemas() { + let schema = from_str(&generate_schema().await.unwrap()).unwrap(); + + let example_one = from_str(&generate_example_one().await.unwrap()).unwrap(); + assert!(is_valid(&schema, &example_one)); + + let example_two = from_str(&generate_example_two().await.unwrap()).unwrap(); + assert!(is_valid(&schema, &example_two)); + } +} diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs index 274c8522d..593e7402d 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs @@ -10,7 +10,8 @@ use crate::error::ErrorKind::LoadingEnvironment; pub mod error; pub mod gen_entities; pub mod gen_openapi; -pub mod schema; +pub mod gen_schema; +pub mod types; /// Run the filemanager-build tool to generate sea-orm entities or generate JSON schemas. This always generates entities /// if a database url is defined, otherwise it skips generating entities if `--skip-if-no-database` @@ -50,9 +51,9 @@ pub enum SubCommands { }, /// Generate the JSON schemas that filemanager can consume. Schemas { - /// The output file. + /// The output directory. #[arg(short, long, env)] - out_file: PathBuf, + out_dir: PathBuf, /// Whether to generate schema examples. #[arg(short, long, default_value_t = true, env)] examples: bool, diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs index 97e186187..83efdf1f9 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs @@ -1,6 +1,6 @@ use filemanager_build::gen_entities::generate_entities; use filemanager_build::gen_openapi::generate_openapi; -use filemanager_build::schema::generate_schemas; +use filemanager_build::gen_schema::write_schema; use filemanager_build::{Config, SubCommands}; use miette::Result; @@ -22,8 +22,8 @@ async fn main() -> Result<()> { println!("Skipping entity generation as no database url is defined, nothing to do.") } } - SubCommands::Schemas { out_file, .. } => { - generate_schemas(&out_file).await?; + SubCommands::Schemas { out_dir, .. } => { + write_schema(&out_dir).await?; } } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs similarity index 59% rename from lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs rename to lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs index 2c3722e4a..4e2527843 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/schema.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs @@ -1,67 +1,65 @@ -//! Logic for AWS EventBridge schema for filemanager. -//! - -use crate::error::ErrorKind::SchemaGeneration; -use crate::error::Result; use chrono::{DateTime, Utc}; -use schemars::{schema_for, JsonSchema}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use serde_json::to_string_pretty; -use std::path::Path; -use tokio::fs; +use serde_with::skip_serializing_none; +use std::ops::Not; use uuid::Uuid; /// A filemanager object state change event. -#[derive(Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "kebab-case")] +#[skip_serializing_none] +#[derive(Deserialize, Serialize, JsonSchema, Default)] +#[serde(rename_all = "kebab-case", default)] pub struct FileStateChange { /// The type of the event. - detail_type: FileStateChangeType, + pub(crate) detail_type: FileStateChangeType, /// The detail of the event. - detail: Detail, + pub(crate) detail: Detail, /// The source of the event. - source: String, + pub(crate) source: String, /// The version of the event. - version: Option, + pub(crate) version: Option, /// The ID of the event. - id: Option, + pub(crate) id: Option, /// The account ID of the event. - account: Option, + pub(crate) account: Option, /// The time the event was generated. - time: Option>, + pub(crate) time: Option>, /// The region the event was generated in. - region: Option, + pub(crate) region: Option, /// The resources of the event. - resources: Option>, + pub(crate) resources: Option>, } /// The type of S3 object state change event. -#[derive(Deserialize, Serialize, JsonSchema)] +#[derive(Deserialize, Serialize, JsonSchema, Default)] #[serde(rename = "FileStateChange")] pub enum FileStateChangeType { + #[default] FileStateChange, } /// The detail of an S3 object state change event. -#[derive(Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "kebab-case")] +#[skip_serializing_none] +#[derive(Deserialize, Serialize, JsonSchema, Default)] +#[serde(rename_all = "kebab-case", default)] pub struct Detail { /// The key name prefix to apply this transition to. Applies to all objects by default. - prefix: Option, + pub(crate) prefix: Option, /// A set of transitions to apply. Performs no transition by default. - transitions: Option>, + pub(crate) transitions: Option>, /// The number of days to wait before expiring the object. Performs no expiry by default. - expiration: Option, + pub(crate) expiration: Option, /// Apply the transition to objects that are less than the specified size. Applies to objects /// of any size by default. #[schemars(range(min = 1))] - object_size_less_than: Option, + pub(crate) object_size_less_than: Option, /// Apply the transition to objects that are greater than the specified size. Applies to objects /// of any size by default. - object_size_greater_than: Option, + pub(crate) object_size_greater_than: Option, /// Whether to keep the expiry rule when the object is moved. By default, the rule is removed /// when an object is moved. - keep_on_move: bool, + #[serde(skip_serializing_if = "Not::not")] + pub(crate) keep_on_move: bool, } /// An S3 storage class transition. @@ -69,9 +67,9 @@ pub struct Detail { #[serde(rename_all = "kebab-case")] pub struct Transition { /// The storage class to transition to. - storage_class: StorageClass, + pub(crate) storage_class: StorageClass, /// The number of days to wait before transitioning the object. - days: String, + pub(crate) days: u64, } /// AWS storage types. @@ -85,15 +83,3 @@ pub enum StorageClass { GlacierIr, DeepArchive, } - -/// Generate the JSON schemas. -pub async fn generate_schemas(out_file: &Path) -> Result<()> { - let schema = to_string_pretty(&schema_for!(FileStateChange)) - .map_err(|err| SchemaGeneration(err.to_string()))?; - - fs::write(out_file, schema) - .await - .map_err(|err| SchemaGeneration(err.to_string()))?; - - Ok(()) -} From b685a324e4a1147b49156f2a6d9eb35e06923135 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 17 Oct 2024 12:55:32 +1100 Subject: [PATCH 3/6] feat(filemanager): generate schemas from Makefile --- .../filemanager/FileStateChange.schema.json | 179 ++++++++++++++++++ docs/schemas/events/filemanager/Makefile | 3 + .../filemanager/example/FSC__example1.json | 15 ++ .../filemanager/example/FSC__example2.json | 25 +++ .../stateless/stacks/filemanager/Makefile | 2 + .../src/{gen_schema.rs => gen_schemas.rs} | 25 ++- .../filemanager/filemanager-build/src/lib.rs | 2 +- .../filemanager/filemanager-build/src/main.rs | 4 +- .../filemanager-build/src/types.rs | 6 +- 9 files changed, 248 insertions(+), 13 deletions(-) create mode 100644 docs/schemas/events/filemanager/FileStateChange.schema.json create mode 100644 docs/schemas/events/filemanager/Makefile create mode 100644 docs/schemas/events/filemanager/example/FSC__example1.json create mode 100644 docs/schemas/events/filemanager/example/FSC__example2.json rename lib/workload/stateless/stacks/filemanager/filemanager-build/src/{gen_schema.rs => gen_schemas.rs} (78%) diff --git a/docs/schemas/events/filemanager/FileStateChange.schema.json b/docs/schemas/events/filemanager/FileStateChange.schema.json new file mode 100644 index 000000000..d80f1eaed --- /dev/null +++ b/docs/schemas/events/filemanager/FileStateChange.schema.json @@ -0,0 +1,179 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "FileStateChange", + "description": "A filemanager object state change event.", + "type": "object", + "required": [ + "detail", + "detail-type", + "source" + ], + "properties": { + "account": { + "description": "The account ID of the event.", + "type": [ + "string", + "null" + ] + }, + "detail": { + "description": "The detail of the event.", + "allOf": [ + { + "$ref": "#/definitions/Detail" + } + ] + }, + "detail-type": { + "description": "The type of the event.", + "allOf": [ + { + "$ref": "#/definitions/FileStateChange" + } + ] + }, + "id": { + "description": "The ID of the event.", + "type": [ + "string", + "null" + ], + "format": "uuid" + }, + "region": { + "description": "The region the event was generated in.", + "type": [ + "string", + "null" + ] + }, + "resources": { + "description": "The resources of the event.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "source": { + "description": "The source of the event.", + "type": "string" + }, + "time": { + "description": "The time the event was generated.", + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "version": { + "description": "The version of the event.", + "type": [ + "string", + "null" + ] + } + }, + "definitions": { + "Detail": { + "description": "The detail of an S3 object state change event.", + "type": "object", + "properties": { + "expiration": { + "description": "The number of days to wait before expiring the object. Performs no expiry by default.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "keep-on-move": { + "description": "Whether to keep the expiry rule when the object is moved. By default, the rule is removed when an object is moved.", + "type": "boolean" + }, + "object-size-greater-than": { + "description": "Apply the transition to objects that are greater than the specified size. Applies to objects of any size by default.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "object-size-less-than": { + "description": "Apply the transition to objects that are less than the specified size. Applies to objects of any size by default.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 1.0 + }, + "prefix": { + "description": "The key name prefix to apply this transition to. Applies to all objects by default.", + "type": [ + "string", + "null" + ] + }, + "transitions": { + "description": "A set of transitions to apply. Performs no transition by default.", + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/Transition" + } + } + } + }, + "FileStateChange": { + "description": "The type of S3 object state change event.", + "type": "string", + "enum": [ + "FileStateChange" + ] + }, + "StorageClass": { + "description": "AWS storage types.", + "type": "string", + "enum": [ + "STANDARD_IA", + "INTELLIGENT_TIERING", + "ONEZONE_IA", + "GLACIER", + "GLACIER_IR", + "DEEP_ARCHIVE" + ] + }, + "Transition": { + "description": "An S3 storage class transition.", + "type": "object", + "required": [ + "days", + "storage-class" + ], + "properties": { + "days": { + "description": "The number of days to wait before transitioning the object.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "storage-class": { + "description": "The storage class to transition to.", + "allOf": [ + { + "$ref": "#/definitions/StorageClass" + } + ] + } + } + } + } +} diff --git a/docs/schemas/events/filemanager/Makefile b/docs/schemas/events/filemanager/Makefile new file mode 100644 index 000000000..cbbdd454c --- /dev/null +++ b/docs/schemas/events/filemanager/Makefile @@ -0,0 +1,3 @@ +test: + @check-jsonschema --schemafile FileStateChange.schema.json example/FSC__example1.json + @check-jsonschema --schemafile FileStateChange.schema.json example/FSC__example2.json diff --git a/docs/schemas/events/filemanager/example/FSC__example1.json b/docs/schemas/events/filemanager/example/FSC__example1.json new file mode 100644 index 000000000..9ae985c35 --- /dev/null +++ b/docs/schemas/events/filemanager/example/FSC__example1.json @@ -0,0 +1,15 @@ +{ + "detail-type": "FileStateChange", + "source": "orcabus.filemanager", + "version": "0", + "id": "019297c6-0547-7391-a24c-207a529aa1a2", + "account": "123456789012", + "time": "2024-10-16T23:59:50.087257150Z", + "region": "ap-southeast-2", + "resources": [], + "detail": { + "expiration": 30, + "object-size-greater-than": 1024, + "keep-on-move": true + } +} diff --git a/docs/schemas/events/filemanager/example/FSC__example2.json b/docs/schemas/events/filemanager/example/FSC__example2.json new file mode 100644 index 000000000..ae6a34082 --- /dev/null +++ b/docs/schemas/events/filemanager/example/FSC__example2.json @@ -0,0 +1,25 @@ +{ + "detail-type": "FileStateChange", + "source": "orcabus.filemanager", + "version": "0", + "id": "019297c6-0547-7391-a24c-2084299a77f1", + "account": "123456789012", + "time": "2024-10-16T23:59:50.087370946Z", + "region": "ap-southeast-2", + "resources": [], + "detail": { + "prefix": "key_prefix/", + "transitions": [ + { + "storage-class": "STANDARD_IA", + "days": 30 + }, + { + "storage-class": "GLACIER", + "days": 90 + } + ], + "expiration": 365, + "object-size-less-than": 1024 + } +} diff --git a/lib/workload/stateless/stacks/filemanager/Makefile b/lib/workload/stateless/stacks/filemanager/Makefile index c70b0e8d2..7c289e187 100644 --- a/lib/workload/stateless/stacks/filemanager/Makefile +++ b/lib/workload/stateless/stacks/filemanager/Makefile @@ -45,6 +45,8 @@ docker-api: docker-postgres ## Build related commands entity: docker-postgres @cargo run -p filemanager-build -- entities --out-dir filemanager/src/database/entities +schema: + @cargo run -p filemanager-build -- schemas --out-dir $(shell git rev-parse --show-toplevel)/docs/schemas/events/filemanager build: docker-postgres entity @cargo build --all-features --all-targets watch: docker-postgres entity diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs similarity index 78% rename from lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs rename to lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs index a5c66af7d..4ce687d98 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schema.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs @@ -9,19 +9,30 @@ use chrono::Utc; use schemars::schema_for; use serde::Serialize; use serde_json::to_string_pretty; +use std::fs::File; +use std::io::Write; use std::path::Path; -use tokio::fs::write; +use tokio::fs::create_dir_all; use uuid::Uuid; -/// Write the schema to the out directory. -pub async fn write_schema(out_dir: &Path) -> Result<()> { - let schema = generate_schema().await?; +/// Write the schema and examples to the out directory. +pub async fn write_schemas(out_dir: &Path) -> Result<()> { + create_dir_all(out_dir.join("example")).await?; - Ok(write(out_dir.join("schema.json"), schema).await?) + let mut example_one = File::create(out_dir.join("example/FSC__example1.json"))?; + writeln!(example_one, "{}", generate_example_one().await?)?; + + let mut example_two = File::create(out_dir.join("example/FSC__example2.json"))?; + writeln!(example_two, "{}", generate_example_two().await?)?; + + let mut schema = File::create(out_dir.join("FileStateChange.schema.json"))?; + writeln!(schema, "{}", generate_file_schema().await?)?; + + Ok(()) } /// Generate the JSON schemas. -pub async fn generate_schema() -> Result { +pub async fn generate_file_schema() -> Result { to_json_string(&schema_for!(FileStateChange)).await } @@ -94,7 +105,7 @@ mod tests { #[tokio::test] async fn test_schemas() { - let schema = from_str(&generate_schema().await.unwrap()).unwrap(); + let schema = from_str(&generate_file_schema().await.unwrap()).unwrap(); let example_one = from_str(&generate_example_one().await.unwrap()).unwrap(); assert!(is_valid(&schema, &example_one)); diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs index 593e7402d..1de608813 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs @@ -10,7 +10,7 @@ use crate::error::ErrorKind::LoadingEnvironment; pub mod error; pub mod gen_entities; pub mod gen_openapi; -pub mod gen_schema; +pub mod gen_schemas; pub mod types; /// Run the filemanager-build tool to generate sea-orm entities or generate JSON schemas. This always generates entities diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs index 83efdf1f9..1afeda38e 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/main.rs @@ -1,6 +1,6 @@ use filemanager_build::gen_entities::generate_entities; use filemanager_build::gen_openapi::generate_openapi; -use filemanager_build::gen_schema::write_schema; +use filemanager_build::gen_schemas::write_schemas; use filemanager_build::{Config, SubCommands}; use miette::Result; @@ -23,7 +23,7 @@ async fn main() -> Result<()> { } } SubCommands::Schemas { out_dir, .. } => { - write_schema(&out_dir).await?; + write_schemas(&out_dir).await?; } } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs index 4e2527843..3002fd136 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/types.rs @@ -8,12 +8,10 @@ use uuid::Uuid; /// A filemanager object state change event. #[skip_serializing_none] #[derive(Deserialize, Serialize, JsonSchema, Default)] -#[serde(rename_all = "kebab-case", default)] +#[serde(rename_all = "kebab-case")] pub struct FileStateChange { /// The type of the event. pub(crate) detail_type: FileStateChangeType, - /// The detail of the event. - pub(crate) detail: Detail, /// The source of the event. pub(crate) source: String, /// The version of the event. @@ -28,6 +26,8 @@ pub struct FileStateChange { pub(crate) region: Option, /// The resources of the event. pub(crate) resources: Option>, + /// The detail of the event. + pub(crate) detail: Detail, } /// The type of S3 object state change event. From 1540b3c7b19366495a1c39b2d3b966398fbf712f Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 17 Oct 2024 13:04:48 +1100 Subject: [PATCH 4/6] feat(platform): add filemanager schemas to registry --- config/stacks/schema/events.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/stacks/schema/events.ts b/config/stacks/schema/events.ts index 7daf1a6d1..6d23bdc0e 100644 --- a/config/stacks/schema/events.ts +++ b/config/stacks/schema/events.ts @@ -49,6 +49,12 @@ export const getEventSchemaStackProps = (): SchemaStackProps => { docBase + '/metadatamanager/MetadataStateChange.schema.json' ), }, + { + schemaType: 'JSONSchemaDraft7', + schemaName: 'orcabus.filemanager@FileStateChange', + schemaDescription: 'Change of state for an object in the file manager', + schemaLocation: path.join(__dirname, docBase + '/filemanager/FileStateChange.schema.json'), + }, ], }; }; From f45ed2e94e6048d62185add3c2411b639633888e Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 17 Oct 2024 14:02:16 +1100 Subject: [PATCH 5/6] fix(platform): event registry only support draft 4 of JSON schema --- config/stacks/schema/events.ts | 2 +- .../events/filemanager/FileStateChange.schema.json | 2 +- .../events/filemanager/example/FSC__example1.json | 4 ++-- .../events/filemanager/example/FSC__example2.json | 4 ++-- .../stacks/filemanager/filemanager-build/Cargo.toml | 2 +- .../filemanager/filemanager-build/src/gen_schemas.rs | 10 +++++++--- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/config/stacks/schema/events.ts b/config/stacks/schema/events.ts index 6d23bdc0e..b66865b93 100644 --- a/config/stacks/schema/events.ts +++ b/config/stacks/schema/events.ts @@ -50,7 +50,7 @@ export const getEventSchemaStackProps = (): SchemaStackProps => { ), }, { - schemaType: 'JSONSchemaDraft7', + ...defaultProps, schemaName: 'orcabus.filemanager@FileStateChange', schemaDescription: 'Change of state for an object in the file manager', schemaLocation: path.join(__dirname, docBase + '/filemanager/FileStateChange.schema.json'), diff --git a/docs/schemas/events/filemanager/FileStateChange.schema.json b/docs/schemas/events/filemanager/FileStateChange.schema.json index d80f1eaed..063f49d1f 100644 --- a/docs/schemas/events/filemanager/FileStateChange.schema.json +++ b/docs/schemas/events/filemanager/FileStateChange.schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "http://json-schema.org/draft-04/schema#", "title": "FileStateChange", "description": "A filemanager object state change event.", "type": "object", diff --git a/docs/schemas/events/filemanager/example/FSC__example1.json b/docs/schemas/events/filemanager/example/FSC__example1.json index 9ae985c35..46026ee96 100644 --- a/docs/schemas/events/filemanager/example/FSC__example1.json +++ b/docs/schemas/events/filemanager/example/FSC__example1.json @@ -2,9 +2,9 @@ "detail-type": "FileStateChange", "source": "orcabus.filemanager", "version": "0", - "id": "019297c6-0547-7391-a24c-207a529aa1a2", + "id": "01929850-275f-7511-9fe2-3825cd786a76", "account": "123456789012", - "time": "2024-10-16T23:59:50.087257150Z", + "time": "2024-10-17T02:30:42.783745070Z", "region": "ap-southeast-2", "resources": [], "detail": { diff --git a/docs/schemas/events/filemanager/example/FSC__example2.json b/docs/schemas/events/filemanager/example/FSC__example2.json index ae6a34082..d2e343055 100644 --- a/docs/schemas/events/filemanager/example/FSC__example2.json +++ b/docs/schemas/events/filemanager/example/FSC__example2.json @@ -2,9 +2,9 @@ "detail-type": "FileStateChange", "source": "orcabus.filemanager", "version": "0", - "id": "019297c6-0547-7391-a24c-2084299a77f1", + "id": "01929850-275f-7511-9fe2-383de94eca70", "account": "123456789012", - "time": "2024-10-16T23:59:50.087370946Z", + "time": "2024-10-17T02:30:42.783850460Z", "region": "ap-southeast-2", "resources": [], "detail": { diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml index 95d75d4b1..c9cc380fa 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/Cargo.toml @@ -28,4 +28,4 @@ uuid = { version = "1", features = ["v7", "serde"] } chrono = { version = "0.4", features = ["serde"] } [dev-dependencies] -jsonschema = "0.23" \ No newline at end of file +jsonschema = "0.23" diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs index 4ce687d98..d39f5fdcd 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_schemas.rs @@ -6,7 +6,7 @@ use crate::error::Result; use crate::types::FileStateChange; use crate::types::{Detail, FileStateChangeType, StorageClass, Transition}; use chrono::Utc; -use schemars::schema_for; +use schemars::gen::{SchemaGenerator, SchemaSettings}; use serde::Serialize; use serde_json::to_string_pretty; use std::fs::File; @@ -33,7 +33,11 @@ pub async fn write_schemas(out_dir: &Path) -> Result<()> { /// Generate the JSON schemas. pub async fn generate_file_schema() -> Result { - to_json_string(&schema_for!(FileStateChange)).await + let mut settings = SchemaSettings::default(); + settings.meta_schema = Some("http://json-schema.org/draft-04/schema#".to_string()); + let schema = SchemaGenerator::new(settings).into_root_schema_for::(); + + to_json_string(&schema).await } /// Generate an example schema for an expiration rule. @@ -100,7 +104,7 @@ async fn to_json_string(value: &T) -> Result { #[cfg(test)] mod tests { use super::*; - use jsonschema::is_valid; + use jsonschema::draft4::is_valid; use serde_json::from_str; #[tokio::test] From 104ffc5a59cf4fab37e7325b27c7b51b95c3f912 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 17 Oct 2024 14:05:30 +1100 Subject: [PATCH 6/6] style(filemanager): remove commented-out unused code --- .../filemanager/filemanager-build/src/lib.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs index 1de608813..239530fb9 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/lib.rs @@ -82,21 +82,6 @@ impl Config { Ok(args) } - - // /// Get whether to skip the generation if the database URL is empty. - // pub fn skip_if_no_database(&self) -> bool { - // self.skip_if_no_database && self.database_url.is_empty() - // } - // - // /// Get the database url. - // pub fn database_url(&self) -> &str { - // &self.database_url - // } - // - // /// Get the out dir. - // pub fn out_dir(&self) -> &Path { - // &self.out_dir - // } } /// Get the path of the workspace.