Skip to content

fix(gateway): report isolation backend mismatches in sandbox status #4738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shiju-nv wants to merge 2 commits into
base: main
Choose a base branch
from
+198 −12
Open

fix(gateway): report isolation backend mismatches in sandbox status #4738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f69370
fix(gateway): report isolation backend mismatches in sandbox status
shiju-nv Oct 20, 2026
c609dc2
docs: drop backend mismatch paragraph from sandbox overview
shiju-nv Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 178 additions & 2 deletions crates/openshell-server/src/compute/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4420,8 +4420,42 @@ impl ComputeRuntime {
}

let sandbox_name = current.object_name().to_string();
self.validate_isolation_backend_binding(&current)
.map_err(|error| error.to_string())?;
if let Err(error) = self.validate_isolation_backend_binding(&current) {
// Only an explicit stop and start can relaunch the sandbox once the
// gateway selection matches again, so leave the restart scan.
let message = error.message().to_string();
match self
.store
.update_message_cas::<Sandbox, _>(
sandbox_id,
sandbox_resource_version(&current),
|sandbox| {
let status = sandbox.status.get_or_insert_with(Default::default);
set_next_restart_at_ms(status, 0);
sandbox.set_phase(SandboxPhase::Error as i32);
upsert_ready_condition(
&mut sandbox.status,
SandboxCondition {
r#type: "Ready".to_string(),
status: "False".to_string(),
reason: "IsolationBackendMismatch".to_string(),
message: message.clone(),
transition_time: None,
},
);
},
)
.await
{
Ok(updated) => {
self.sandbox_index.update_from_sandbox(&updated);
self.sandbox_watch_bus.notify(sandbox_id);
}
Err(crate::persistence::PersistenceError::Conflict { .. }) => return Ok(()),
Err(err) => return Err(err.to_string()),
}
return Err(error.to_string());
}
let already_claimed = current
.status
.as_ref()
Expand Down Expand Up @@ -10526,6 +10560,64 @@ mod tests {
assert!(next_restart_at_ms(&stored.status.unwrap()) > openshell_core::time::now_ms());
}

#[tokio::test]
async fn restart_scan_reports_isolation_backend_mismatch_once() {
let driver = ControlledDriver::new();
let runtime = test_runtime(driver.clone()).await;
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting);
sandbox
.metadata
.as_mut()
.unwrap()
.annotations
.insert(ISOLATION_BACKEND_ANNOTATION.into(), "other-backend".into());
sandbox.status = Some(SandboxStatus {
phase: SandboxPhase::Starting as i32,
exit_code: Some(9),
restart_count: 1,
next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1),
..Default::default()
});
runtime.store.put_message(&sandbox).await.unwrap();

runtime.restart_due_sandboxes().await.unwrap();
let failed = runtime
.store
.get_message::<Sandbox>("sb-1")
.await
.unwrap()
.unwrap();
runtime.restart_due_sandboxes().await.unwrap();

assert_eq!(driver.stop_calls(), 0);
assert_eq!(driver.start_calls(), 0);
assert_eq!(failed.phase(), SandboxPhase::Error as i32);
assert!(
failed
.status
.as_ref()
.unwrap()
.conditions
.iter()
.any(|condition| {
condition.r#type == "Ready"
&& condition.reason == "IsolationBackendMismatch"
&& condition.message.contains("restore the gateway selection")
})
);
let rescanned = runtime
.store
.get_message::<Sandbox>("sb-1")
.await
.unwrap()
.unwrap();
assert_eq!(
sandbox_resource_version(&rescanned),
sandbox_resource_version(&failed),
"the next scan must leave a mismatched sandbox alone"
);
}

#[tokio::test]
async fn concurrent_stop_transition_prevents_replacement_start() {
let driver = ControlledDriver::new();
Expand Down Expand Up @@ -17277,6 +17369,17 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock"
driver.calls().is_empty(),
"another replica must not launch a differently bound resource"
);
assert_eq!(
runtime
.store
.get_message::<Sandbox>("external-start")
.await
.unwrap()
.unwrap()
.phase(),
SandboxPhase::Starting as i32,
"another replica must not fail a launch it cannot run"
);
}
let mut capabilities = driver
.get_capabilities(Request::new(GetCapabilitiesRequest::default()))
Expand Down Expand Up @@ -19436,6 +19539,79 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock"
assert_eq!(driver.stop_calls(), 1);
}

#[tokio::test]
async fn preparation_timeout_reports_isolation_backend_mismatch() {
let driver = ControlledDriver::new();
let runtime = test_runtime(driver.clone()).await;
let mut sandbox = sandbox_record("sb-prepare", "prepare", SandboxPhase::Starting);
sandbox
.metadata
.as_mut()
.unwrap()
.annotations
.insert(ISOLATION_BACKEND_ANNOTATION.into(), "other-backend".into());
sandbox.status.as_mut().unwrap().provisioning =
Some(provisioning_deadline::new_preparation_record(0, 1800));
runtime.store.put_message(&sandbox).await.unwrap();
let sandbox = runtime
.store
.get_message::<Sandbox>("sb-prepare")
.await
.unwrap()
.unwrap();
let gate = runtime.lifecycle_gates.lock_for("sb-prepare").await;
let _sandbox_guard = runtime.lock_sandbox_for_lifecycle(&gate).await;
let expired = runtime
.claim_provisioning_timeout(&sandbox, 1_800_000)
.await
.unwrap()
.unwrap();
assert_eq!(expired.phase(), i32::from(SandboxPhase::Error));
assert!(
expired
.status
.as_ref()
.unwrap()
.conditions
.iter()
.any(|condition| {
condition.reason == "IsolationBackendMismatch"
&& condition.message.contains("restore the gateway selection")
})
);

// A registered supervisor keeps the admission diagnostic.
let mut admitted = sandbox_record("sb-admission", "admission", SandboxPhase::Starting);
admitted
.metadata
.as_mut()
.unwrap()
.annotations
.insert(ISOLATION_BACKEND_ANNOTATION.into(), "other-backend".into());
admitted.status.as_mut().unwrap().provisioning = Some(provisioning_deadline::new_record(0));
runtime.store.put_message(&admitted).await.unwrap();
let admitted = runtime
.store
.get_message::<Sandbox>("sb-admission")
.await
.unwrap()
.unwrap();
let expired = runtime
.claim_provisioning_timeout(&admitted, 300_000)
.await
.unwrap()
.unwrap();
assert!(
expired
.status
.as_ref()
.unwrap()
.conditions
.iter()
.any(|condition| condition.reason == "ProvisioningTimedOut")
);
}

#[tokio::test]
async fn preparation_timeout_retains_reason_and_uses_existing_cleanup() {
let driver = ControlledDriver::new();
Expand Down
30 changes: 20 additions & 10 deletions crates/openshell-server/src/compute/provisioning_deadline.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,12 @@ impl super::ComputeRuntime {
};
let mut deadline = ProvisioningDeadline::from_record(record)?;
let preparation_expired = deadline.is_preparing();
// A supervisor that never registered may be waiting on a launch bound
// to another isolation backend; report that instead of a timeout.
let backend_mismatch = preparation_expired
.then(|| self.validate_isolation_backend_binding(current).err())
.flatten()
.map(|error| error.message().to_string());
if !deadline.expire(&record.attempt_id, &record.configuration_change_id, now_ms) {
return Ok(None);
}
Expand All @@ -573,22 +579,26 @@ impl super::ComputeRuntime {
.configuration_admission
.as_ref()
.map_or("", |admission| admission.error.as_str());
let message = if preparation_expired {
"Image preparation or initial supervisor startup exceeded its absolute deadline".to_string()
} else if diagnostic.is_empty() {
"Provisioning repair window expired after 300 seconds".to_string()
} else {
format!(
"Provisioning repair window expired after 300 seconds: {diagnostic}"
)
};
let message = backend_mismatch.clone().unwrap_or_else(|| {
if preparation_expired {
"Image preparation or initial supervisor startup exceeded its absolute deadline".to_string()
} else if diagnostic.is_empty() {
"Provisioning repair window expired after 300 seconds".to_string()
} else {
format!(
"Provisioning repair window expired after 300 seconds: {diagnostic}"
)
}
});
status
.conditions
.retain(|condition| condition.r#type != "Ready");
status.conditions.push(SandboxCondition {
r#type: "Ready".into(),
status: "False".into(),
reason: if preparation_expired {
reason: if backend_mismatch.is_some() {
"IsolationBackendMismatch"
} else if preparation_expired {
"ImagePreparationTimedOut"
} else {
"ProvisioningTimedOut"
Expand Down
Loading