diff --git a/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md b/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md index 0ca02f89a..bafd58f16 100644 --- a/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md +++ b/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md @@ -199,6 +199,25 @@ curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/js "https://file.dev.umccr.org/api/v1/s3?key=*202405212aecb782*" | jq ``` +In addition to updating attributes, the PATCH request can also be used to update the `ingestId` for records with a null +`ingestId`. For example, update the `ingestId` on a single record: + +```sh +curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ +--data '{ "ingestId": [ { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" } ] }' \ +"https://file.dev.umccr.org/api/v1/s3/0190465f-68fa-76e4-9c36-12bdf1a1571d" | jq +``` + +Or, update the `ingestId` for multiple records: + +```sh +curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ +--data '{ "ingestId": [ { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" } ] }' \ +"https://file.dev.umccr.org/api/v1/s3?key=*202405212aecb782*" | jq +``` + +Note the extra `ingestId` key in the JSON body. The operation must be `add`, and the path must be `/`. + ## Count objects There is an API route which counts the total number of records in the database, which supports diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs index e3c3faa8d..b66e02b4a 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs @@ -8,10 +8,11 @@ use sea_orm::sea_query::{ WithQuery, }; use sea_orm::{ - ColumnTrait, ConnectionTrait, EntityTrait, FromQueryResult, Iterable, ModelTrait, QueryFilter, - QueryTrait, StatementBuilder, Value, + ColumnTrait, ConnectionTrait, EntityTrait, FromQueryResult, Iden, Iterable, ModelTrait, + QueryFilter, QueryTrait, StatementBuilder, Value, }; use serde_json::json; +use std::str::FromStr; use uuid::Uuid; use crate::database::entities::s3_object; @@ -74,12 +75,13 @@ where /// Update the attributes on an s3_object using the attribute patch. pub async fn update_s3_attributes(self, patch: PatchBody) -> Result { - self.update_attributes( - patch, - s3_object::Column::S3ObjectId, - s3_object::Column::Attributes, - ) - .await + let col = match patch { + PatchBody::NestedIngestId { .. } => s3_object::Column::IngestId, + _ => s3_object::Column::Attributes, + }; + + self.update_attributes(patch, s3_object::Column::S3ObjectId, col) + .await } } @@ -161,6 +163,79 @@ where .collect::>>() } + /// Create an update for the attributes column. + fn patch_for_attributes( + patch_body: Vec, + update_col: <::Entity as EntityTrait>::Column, + model: M, + ) -> Result { + let mut current = if let Value::Json(json) = model.get(update_col) { + let mut json = json.unwrap_or_else(|| Box::new(json!({}))); + if let &Json::Null = json.as_ref() { + json = Box::new(json!({})); + } + json + } else { + return Err(QueryError("expected JSON attribute column".to_string())); + }; + + // Only append-style patching is supported. + let operations = Self::verify_patch(patch_body, ¤t)?; + + // Patch it based on JSON patch. + patch(&mut current, operations.as_slice()).map_err(|err| { + InvalidQuery(format!( + "JSON patch {} operation for {} path failed: {}", + err.operation, err.path, err.kind + )) + })?; + + Ok(Value::Json(Some(current))) + } + + /// Create an update for the ingestId column. + fn patch_for_ingest_id( + patch_body: Vec, + update_col: <::Entity as EntityTrait>::Column, + model: M, + ) -> Result { + if let Value::Uuid(None) = model.get(update_col) { + } else { + return Err(QueryError( + "cannot update `ingestId` unless it is null".to_string(), + )); + }; + + if patch_body.len() != 1 { + return Err(QueryError( + "expected one patch operation for `ingestId` update".to_string(), + )); + } + if patch_body[0].path() != "/" { + return Err(QueryError( + "expected `/` path for `ingestId` update".to_string(), + )); + } + + let uuid = if let PatchOperation::Add(add) = &patch_body[0] { + Uuid::from_str(add.value.as_str().ok_or_else(|| { + QueryError("expected string value for `ingestId` update".to_string()) + })?) + .map_err(|err| { + QueryError(format!( + "failed to parse UUID for `ingestId` update: {}", + err + )) + })? + } else { + return Err(QueryError( + "expected `add` operation for `ingestId` update".to_string(), + )); + }; + + Ok(Value::Uuid(Some(Box::new(uuid)))) + } + /// Update the attributes on an object using the attribute patch. This first queries the /// required records to update using a previously specified select query in functions like /// `Self::for_id` and `Self::filter_all`. It then applies a JSON patch to the attributes of @@ -180,7 +255,7 @@ where self, patch_body: PatchBody, id_col: ::Column, - attribute_col: ::Column, + update_col: ::Column, ) -> Result { let (conn, select_to_update, mut with_query) = self.into_inner(); let select = select_to_update.cloned(); @@ -201,33 +276,22 @@ where return Err(QueryError("expected uuid id column".to_string())); }; - let mut current = if let Value::Json(json) = model.get(attribute_col) { - let mut json = json.unwrap_or_else(|| Box::new(json!({}))); - if let &Json::Null = json.as_ref() { - json = Box::new(json!({})); + let update = match patch_body.clone() { + PatchBody::NestedIngestId { ingest_id } => { + Self::patch_for_ingest_id(ingest_id.into_inner().0, update_col, model)? + } + PatchBody::UnnestedAttributes(attributes) + | PatchBody::NestedAttributes { attributes } => { + Self::patch_for_attributes(attributes.into_inner().0, update_col, model)? } - json - } else { - return Err(QueryError("expected JSON attribute column".to_string())); }; - // Only append-style patching is supported. - let operations = Self::verify_patch(patch_body.clone().into_inner().0, ¤t)?; - - // Patch it based on JSON patch. - patch(&mut current, operations.as_slice()).map_err(|err| { - InvalidQuery(format!( - "JSON patch {} operation for {} path failed: {}", - err.operation, err.path, err.kind - )) - })?; - - Ok((Value::Uuid(Some(id)), Value::Json(Some(current)))) + Ok((Value::Uuid(Some(id)), update)) }) .collect::>>()?; let cte_id = Alias::new("id"); - let cte_attributes = Alias::new("attributes"); + let cte_attributes = Alias::new(update_col.to_string()); let cte_name = Alias::new("update_with"); // select * from (values ((, ), ...) @@ -264,7 +328,7 @@ where let update = E::update_many() .into_query() .value( - attribute_col, + update_col, SimpleExpr::SubQuery(None, Box::new(select_update.into_sub_query_statement())), ) .and_where(id_col.in_subquery(select_in)) @@ -321,10 +385,10 @@ pub(crate) mod tests { ]); let results = test_s3_builder_result( &client, - patch, Some(json!({ "attributeId": "1" })), + PatchBody::new(from_value(patch).unwrap()), ) .await; assert!(matches!(results, Err(InvalidQuery(_)))); @@ -335,10 +399,10 @@ pub(crate) mod tests { ]); let results = test_s3_builder_result( &client, - patch, Some(json!({ "attributeId": "1" })), + PatchBody::new(from_value(patch).unwrap()), ) .await; assert!(matches!(results, Err(InvalidQuery(_)))); @@ -349,10 +413,10 @@ pub(crate) mod tests { ]); let results = test_s3_builder_result( &client, - patch, Some(json!({ "attributeId": "1" })), + PatchBody::new(from_value(patch).unwrap()), ) .await; assert!(matches!(results, Err(InvalidQuery(_)))); @@ -390,6 +454,99 @@ pub(crate) mod tests { assert_correct_records(&client, entries).await; } + #[sqlx::test(migrator = "MIGRATOR")] + async fn update_ingest_id(pool: PgPool) { + let client = Client::from_pool(pool); + let mut entries = EntriesBuilder::default().build(&client).await.unwrap(); + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + + change_many( + &client, + &entries, + &[0, 1], + Some(json!({"attributeId": "1"})), + ) + .await; + update_ingest_ids(&client, &mut entries).await; + + let results = test_s3_builder_result( + &client, + Some(json!({ + "attributeId": "1" + })), + from_value(patch).unwrap(), + ) + .await + .unwrap() + .all() + .await + .unwrap(); + + entries_many(&mut entries, &[0, 1], json!({"attributeId": "1"})); + entries.s3_objects[0].ingest_id = Some(Uuid::default()); + entries.s3_objects[1].ingest_id = Some(Uuid::default()); + + assert_contains(&results, &entries, 0..2); + assert_correct_records(&client, entries).await; + } + + #[sqlx::test(migrator = "MIGRATOR")] + async fn update_ingest_id_error(pool: PgPool) { + let client = Client::from_pool(pool); + let mut entries = EntriesBuilder::default().build(&client).await.unwrap(); + + change_many( + &client, + &entries, + &[0, 1], + Some(json!({"attributeId": "1"})), + ) + .await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + assert_ingest_id_error(&client, patch).await; + + update_ingest_ids(&client, &mut entries).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + ] + }); + assert_ingest_id_error(&client, patch).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/ingestId", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + assert_ingest_id_error(&client, patch).await; + + let patch = json!({ + "ingestId": [ + { "op": "replace", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + ] + }); + assert_ingest_id_error(&client, patch).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + ] + }); + assert_ingest_id_error(&client, patch).await; + } + #[sqlx::test(migrator = "MIGRATOR")] async fn update_attributes_add_wildcard(pool: PgPool) { let client = Client::from_pool(pool); @@ -596,10 +753,10 @@ pub(crate) mod tests { let s3_objects = test_s3_builder_result( &client, - patch, Some(json!({ "attributeId": "1" })), + PatchBody::new(from_value(patch).unwrap()), ) .await; @@ -642,10 +799,22 @@ pub(crate) mod tests { assert_correct_records(&client, entries).await; } + async fn assert_ingest_id_error(client: &Client, patch: Value) { + assert!(test_s3_builder_result( + client, + Some(json!({ + "attributeId": "1" + })), + from_value(patch).unwrap() + ) + .await + .is_err()); + } + async fn test_s3_builder_result( client: &Client, - patch: Value, attributes: Option, + patch_body: PatchBody, ) -> Result> { UpdateQueryBuilder::<_, s3_object::Entity>::new(client.connection_ref()) .filter_all( @@ -656,7 +825,7 @@ pub(crate) mod tests { true, false, )? - .update_s3_attributes(PatchBody::new(from_value(patch)?)) + .update_s3_attributes(patch_body) .await } @@ -665,12 +834,16 @@ pub(crate) mod tests { patch: Value, attributes: Option, ) -> Vec { - test_s3_builder_result(client, patch, attributes) - .await - .unwrap() - .all() - .await - .unwrap() + test_s3_builder_result( + client, + attributes, + PatchBody::new(from_value(patch).unwrap()), + ) + .await + .unwrap() + .all() + .await + .unwrap() } async fn test_update_attributes_for_id( @@ -709,6 +882,15 @@ pub(crate) mod tests { } } + pub(crate) async fn update_ingest_ids(client: &Client, entries: &mut Entries) { + for i in [0, 1] { + let mut model: s3_object::ActiveModel = + entries.s3_objects[i].clone().into_active_model(); + model.ingest_id = Set(None); + model.update(client.connection_ref()).await.unwrap(); + } + } + /// Make attributes null for an entry. pub(crate) async fn null_attributes(client: &Client, entries: &Entries, entry: usize) { change_attributes(client, entries, entry, None).await; diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/update.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/update.rs index 382cbde98..9179618a4 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/update.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/update.rs @@ -23,20 +23,38 @@ use crate::routes::AppState; /// In order to apply the patch, JSON body must contain an array with patch operations. The patch operations /// are append-only, which means that only "add" and "test" is supported. If a "test" check fails, /// a patch operations that isn't "add" or "test" is used, or if a key already exists, a `BAD_REQUEST` -/// is returned and no records are updated. +/// is returned and no records are updated. Use `attributes` to update attributes and `ingestId` to +/// update the ingest id. #[derive(Debug, Deserialize, Clone, ToSchema)] #[serde(untagged)] #[schema( - example = json!([ - { "op": "add", "path": "/attributeId", "value": "attributeId" } - ]) + examples( + json!([ + { "op": "add", "path": "/attributeId", "value": "attributeId" } + ]), + json!({ + "attributes": [ + { "op": "add", "path": "/attributeId", "value": "attributeId" } + ] + }), + json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" } + ] + }) + ) )] pub enum PatchBody { - Nested { + NestedAttributes { /// The JSON patch for a record's attributes. attributes: Patch, }, - Unnested(Patch), + NestedIngestId { + /// The JSON patch for a record's ingest_id. Only `add` with a `/` path is supported. + #[serde(rename = "ingestId")] + ingest_id: Patch, + }, + UnnestedAttributes(Patch), } /// The JSON patch for attributes. @@ -45,25 +63,39 @@ pub enum PatchBody { #[schema(value_type = Value)] pub struct Patch(json_patch::Patch); +impl Patch { + /// Create a new patch. + pub fn new(patch: json_patch::Patch) -> Self { + Self(patch) + } + + /// Get the inner patch. + pub fn into_inner(self) -> json_patch::Patch { + self.0 + } +} + impl PatchBody { /// Create a new attribute body. pub fn new(attributes: Patch) -> Self { - Self::Unnested(attributes) + Self::UnnestedAttributes(attributes) } /// Get the inner map. pub fn into_inner(self) -> json_patch::Patch { match self { - PatchBody::Nested { attributes } => attributes.0, - PatchBody::Unnested(attributes) => attributes.0, + PatchBody::NestedAttributes { attributes } => attributes.0, + PatchBody::NestedIngestId { ingest_id } => ingest_id.0, + PatchBody::UnnestedAttributes(attributes) => attributes.0, } } /// Get the inner map as a reference pub fn get_ref(&self) -> &json_patch::Patch { match self { - PatchBody::Nested { attributes } => &attributes.0, - PatchBody::Unnested(attributes) => &attributes.0, + PatchBody::NestedAttributes { attributes } => &attributes.0, + PatchBody::NestedIngestId { ingest_id } => &ingest_id.0, + PatchBody::UnnestedAttributes(attributes) => &attributes.0, } } } @@ -163,7 +195,7 @@ mod tests { use crate::queries::update::tests::{assert_contains, entries_many}; use crate::queries::update::tests::{ assert_correct_records, assert_model_contains, assert_wildcard_update, - change_attribute_entries, change_attributes, change_many, + change_attribute_entries, change_attributes, change_many, update_ingest_ids, }; use crate::queries::EntriesBuilder; use crate::routes::list::tests::response_from; @@ -387,6 +419,126 @@ mod tests { assert_correct_records(state.database_client(), entries).await; } + #[sqlx::test(migrator = "MIGRATOR")] + async fn update_ingest_id(pool: PgPool) { + let state = AppState::from_pool(pool).await; + let client = state.database_client(); + let mut entries = EntriesBuilder::default().build(client).await.unwrap(); + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + + change_many(client, &entries, &[0, 1], Some(json!({"attributeId": "1"}))).await; + update_ingest_ids(client, &mut entries).await; + + let (_, s3_objects) = response_from::>( + state.clone(), + "/s3?attributes[attributeId]=1¤tState=false", + Method::PATCH, + Body::new(patch.to_string()), + ) + .await; + + entries_many(&mut entries, &[0, 1], json!({"attributeId": "1"})); + entries.s3_objects[0].ingest_id = Some(Uuid::default()); + entries.s3_objects[1].ingest_id = Some(Uuid::default()); + + assert_contains(&s3_objects, &entries, 0..2); + assert_correct_records(client, entries).await; + } + + #[sqlx::test(migrator = "MIGRATOR")] + async fn update_ingest_id_single(pool: PgPool) { + let state = AppState::from_pool(pool).await; + let client = state.database_client(); + let mut entries = EntriesBuilder::default().build(client).await.unwrap(); + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + + change_many(client, &entries, &[0, 1], Some(json!({"attributeId": "1"}))).await; + update_ingest_ids(client, &mut entries).await; + + let (_, s3_objects) = response_from::( + state.clone(), + &format!("/s3/{}", entries.s3_objects[0].s3_object_id), + Method::PATCH, + Body::new(patch.to_string()), + ) + .await; + + entries_many(&mut entries, &[0, 1], json!({"attributeId": "1"})); + entries.s3_objects[0].ingest_id = Some(Uuid::default()); + entries.s3_objects[1].ingest_id = None; + + assert_contains(&vec![s3_objects], &entries, 0..1); + assert_correct_records(client, entries).await; + } + + #[sqlx::test(migrator = "MIGRATOR")] + async fn update_ingest_id_error(pool: PgPool) { + let state = AppState::from_pool(pool).await; + let client = state.database_client(); + let mut entries = EntriesBuilder::default().build(client).await.unwrap(); + + change_many(client, &entries, &[0, 1], Some(json!({"attributeId": "1"}))).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + assert_ingest_id_error(state.clone(), patch).await; + + update_ingest_ids(client, &mut entries).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + ] + }); + assert_ingest_id_error(state.clone(), patch).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/ingestId", "value": "00000000-0000-0000-0000-000000000000" }, + ] + }); + assert_ingest_id_error(state.clone(), patch).await; + + let patch = json!({ + "ingestId": [ + { "op": "replace", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + ] + }); + assert_ingest_id_error(state.clone(), patch).await; + + let patch = json!({ + "ingestId": [ + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + { "op": "add", "path": "/", "value": "00000000-0000-0000-0000-00000000000" }, + ] + }); + assert_ingest_id_error(state, patch).await; + } + + async fn assert_ingest_id_error(state: AppState, patch: Value) { + let (code, _) = response_from::( + state, + "/s3?attributes[attributeId]=1¤tState=false", + Method::PATCH, + Body::new(patch.to_string()), + ) + .await; + assert!(code.is_client_error() || code.is_server_error()); + } + #[sqlx::test(migrator = "MIGRATOR")] async fn update_collection_attributes_api_no_op(pool: PgPool) { let state = AppState::from_pool(pool).await;