1mod approval;
12mod collaboration;
13mod environment;
14mod persistence;
15mod recovery;
16
17use crate::pipeline::sync_task_reflection_outcomes;
18use crate::tasks::{TaskBackgroundJob, TaskBackgroundStatus};
19use crate::tools::PermissionManager;
20use crate::{
21 AgentPipeline, AgentRequest, AppConfig, CancellationToken, PausedExecutionState, RequestSource,
22 SessionWorkspace, StreamChunk, ToolCallRecord,
23};
24use crate::{MemoryBankEntry, MemoryScope, MemoryType};
25use crate::{TaskManager, TaskStatus};
26use chrono::{DateTime, Utc};
27use gestura_core_a2a::{
28 A2AClient, A2AMessage, A2ATask, Artifact as RemoteArtifact, ArtifactManifestEntry,
29 CreateTaskRequest, MessagePart, RemoteTaskContract, RemoteTaskLease, RemoteTaskLeaseRequest,
30 RemoteTaskProgress as A2ARemoteTaskProgress, TaskProvenance, TaskStatus as A2ATaskStatus,
31};
32use gestura_core_foundation::{OutcomeSignal, OutcomeSignalKind};
33use serde::{Deserialize, Serialize};
34use serde_json::json;
35use std::collections::{HashMap, HashSet};
36use std::path::{Path, PathBuf};
37use std::sync::{Arc, OnceLock};
38use tokio::sync::{Mutex, RwLock, mpsc};
39use tokio::time::{Duration as TokioDuration, sleep};
40use tracing::Instrument;
41use uuid::Uuid;
42
43use self::persistence::{
44 load_persisted_checkpoints, load_persisted_environments, load_persisted_runs,
45 persist_checkpoint_to_disk, persist_checkpoint_to_disk_async, persist_environment_to_disk,
46 persist_run_to_disk, persist_run_to_disk_async,
47};
48
49pub use self::approval::{
51 ApprovalActor, ApprovalActorKind, ApprovalDecision, ApprovalDecisionKind, ApprovalPolicy,
52 ApprovalRequest, ApprovalRequirement, ApprovalScope, ApprovalState, TaskApprovalRecord,
53 actor_kind_for_agent_role, default_actor_kind_for_scope,
54};
55pub use self::collaboration::{
56 CollaborationActionStatus, CollaborationEscalationLevel, CollaborationRequestKind,
57 CollaborationThreadStatus, DEFAULT_RESOLVED_THREAD_RETENTION_DAYS, TeamActionRequest,
58 TeamActionRequestDraft, TeamArtifactReference, TeamEscalation, TeamEscalationDraft,
59 TeamMessage, TeamMessageDraft, TeamMessageKind, TeamResultReference, TeamThread,
60 archive_resolved_threads, build_team_threads, build_team_threads_with_options,
61};
62pub use crate::agents::{
63 AgentExecutionMode, AgentInfo, AgentRole, AgentSpawnRequest, AgentSpawner, DelegatedTask,
64 DelegationBrief, OrchestratorToolCall, RemoteAgentTarget, TaskArtifactRecord, TaskResult,
65 TaskTerminalStateHint,
66};
67
68#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
70#[serde(rename_all = "snake_case")]
71pub enum SupervisorTaskState {
72 Queued,
74 Blocked,
76 PendingApproval,
78 Running,
80 ReviewPending,
82 TestPending,
84 Completed,
86 Failed,
88 Cancelled,
90}
91
92#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
94#[serde(rename_all = "snake_case")]
95pub enum DelegatedCheckpointStage {
96 Queued,
98 Dispatched,
100 Running,
102 Completed,
104 Failed,
106 Cancelled,
108 Blocked,
110}
111
112#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
114#[serde(rename_all = "snake_case")]
115pub enum DelegatedReplaySafety {
116 PureReadonly,
118 IdempotentWrite,
120 CheckpointResumable,
122 OperatorGated,
124 NonReplayableSideEffect,
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
130#[serde(rename_all = "snake_case")]
131pub enum DelegatedResumeDisposition {
132 ResumeFromCheckpoint,
134 RestartFromBoundary,
136 OperatorInterventionRequired,
138 NotApplicable,
140}
141
142#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
144#[serde(rename_all = "snake_case")]
145pub enum DelegatedCheckpointAction {
146 ResumeFromCheckpoint,
148 RestartFromScratch,
150 AcknowledgeBlocked,
152}
153
154#[derive(Debug, Clone, Serialize, Deserialize)]
156pub struct DelegatedTaskCheckpoint {
157 pub id: String,
159 pub task_id: String,
161 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub run_id: Option<String>,
164 #[serde(default, skip_serializing_if = "Option::is_none")]
166 pub session_id: Option<String>,
167 pub agent_id: String,
169 #[serde(default, skip_serializing_if = "Option::is_none")]
171 pub environment_id: Option<String>,
172 pub execution_mode: AgentExecutionMode,
174 pub stage: DelegatedCheckpointStage,
176 pub replay_safety: DelegatedReplaySafety,
178 pub resume_disposition: DelegatedResumeDisposition,
180 pub safe_boundary_label: String,
182 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub workspace_dir: Option<PathBuf>,
185 #[serde(default, skip_serializing_if = "Vec::is_empty")]
187 pub completed_tool_calls: Vec<OrchestratorToolCall>,
188 #[serde(default)]
190 pub result_published: bool,
191 #[serde(default, skip_serializing_if = "Option::is_none")]
193 pub note: Option<String>,
194 #[serde(default, skip_serializing_if = "Option::is_none")]
196 pub resume_state: Option<PausedExecutionState>,
197 pub created_at: DateTime<Utc>,
199 pub updated_at: DateTime<Utc>,
201}
202
203#[derive(Debug, Clone, Serialize, Deserialize)]
205pub struct DelegatedCheckpointSummary {
206 pub stage: DelegatedCheckpointStage,
208 pub replay_safety: DelegatedReplaySafety,
210 pub resume_disposition: DelegatedResumeDisposition,
212 pub safe_boundary_label: String,
214 #[serde(default, skip_serializing_if = "Vec::is_empty")]
216 pub available_actions: Vec<DelegatedCheckpointAction>,
217 #[serde(default, skip_serializing_if = "Option::is_none")]
219 pub note: Option<String>,
220 #[serde(default)]
222 pub completed_tool_call_count: usize,
223 #[serde(default)]
225 pub has_resume_state: bool,
226 #[serde(default)]
228 pub result_published: bool,
229 pub updated_at: DateTime<Utc>,
231}
232
233#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
235#[serde(rename_all = "snake_case")]
236pub enum SupervisorRunStatus {
237 Draft,
239 Running,
241 Waiting,
243 Completed,
245 Failed,
247 Cancelled,
249}
250
251pub const MAX_CHILD_SUPERVISOR_DEPTH: u8 = 1;
253pub const SHARED_COGNITION_CATEGORY: &str = "shared_cognition";
255pub const SHARED_COGNITION_TAG: &str = "shared-cognition";
257
258fn default_max_child_supervisor_depth() -> u8 {
259 MAX_CHILD_SUPERVISOR_DEPTH
260}
261
262fn workflow_run_memory_tag(run_id: &str) -> String {
263 format!("workflow-run:{run_id}")
264}
265
266fn push_unique_tag(tags: &mut Vec<String>, tag: impl Into<String>) {
267 let tag = tag.into();
268 if !tags.iter().any(|existing| existing == &tag) {
269 tags.push(tag);
270 }
271}
272
273fn summarize_shared_cognition(content: &str) -> String {
274 let collapsed = content.split_whitespace().collect::<Vec<_>>().join(" ");
275 if collapsed.chars().count() <= 96 {
276 return collapsed;
277 }
278
279 format!(
280 "{}…",
281 collapsed.chars().take(95).collect::<String>().trim_end()
282 )
283}
284
285fn shared_cognition_kind_tag(kind: SharedCognitionKind) -> &'static str {
286 match kind {
287 SharedCognitionKind::Discovery => "shared-cognition:discovery",
288 SharedCognitionKind::Blocker => "shared-cognition:blocker",
289 SharedCognitionKind::Hypothesis => "shared-cognition:hypothesis",
290 SharedCognitionKind::Steering => "shared-cognition:steering",
291 SharedCognitionKind::Decision => "shared-cognition:decision",
292 SharedCognitionKind::Handoff => "shared-cognition:handoff",
293 }
294}
295
296fn shared_cognition_memory_type(kind: SharedCognitionKind) -> MemoryType {
297 match kind {
298 SharedCognitionKind::Blocker => MemoryType::Blocker,
299 SharedCognitionKind::Decision => MemoryType::Decision,
300 SharedCognitionKind::Handoff => MemoryType::Handoff,
301 SharedCognitionKind::Discovery
302 | SharedCognitionKind::Hypothesis
303 | SharedCognitionKind::Steering => MemoryType::Procedural,
304 }
305}
306
307fn shared_cognition_confidence(kind: SharedCognitionKind) -> f32 {
308 match kind {
309 SharedCognitionKind::Blocker => 0.92,
310 SharedCognitionKind::Decision => 0.88,
311 SharedCognitionKind::Handoff => 0.86,
312 SharedCognitionKind::Steering => 0.82,
313 SharedCognitionKind::Discovery => 0.78,
314 SharedCognitionKind::Hypothesis => 0.74,
315 }
316}
317
318fn common_directive_id_for_run(run: &SupervisorRun) -> Option<String> {
319 let mut directives = run
320 .tasks
321 .iter()
322 .filter_map(|record| record.task.directive_id.as_deref())
323 .collect::<Vec<_>>();
324 directives.sort_unstable();
325 directives.dedup();
326 if directives.len() == 1 {
327 Some(directives[0].to_string())
328 } else {
329 None
330 }
331}
332
333fn shared_cognition_kind_for_message(
334 run: &SupervisorRun,
335 message: &TeamMessage,
336) -> Option<SharedCognitionKind> {
337 match message.kind {
338 TeamMessageKind::Blocker => Some(SharedCognitionKind::Blocker),
339 TeamMessageKind::Handoff => Some(SharedCognitionKind::Handoff),
340 TeamMessageKind::ReviewFeedback | TeamMessageKind::ApprovalDecision => {
341 Some(SharedCognitionKind::Decision)
342 }
343 TeamMessageKind::StatusUpdate => {
344 if message.sender_agent_id.as_deref() == run.lead_agent_id.as_deref() {
345 Some(SharedCognitionKind::Steering)
346 } else {
347 Some(SharedCognitionKind::Discovery)
348 }
349 }
350 TeamMessageKind::Clarification => {
351 if message.sender_agent_id.as_deref() == run.lead_agent_id.as_deref() {
352 Some(SharedCognitionKind::Steering)
353 } else {
354 Some(SharedCognitionKind::Hypothesis)
355 }
356 }
357 TeamMessageKind::ReviewRequest
358 | TeamMessageKind::ApprovalRequest
359 | TeamMessageKind::TestValidationRequest => None,
360 }
361}
362
363fn ensure_shared_memory_tags(task: &mut DelegatedTask, run_id: &str) {
364 push_unique_tag(&mut task.memory_tags, SHARED_COGNITION_TAG);
365 push_unique_tag(&mut task.memory_tags, workflow_run_memory_tag(run_id));
366}
367
368#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
370#[serde(rename_all = "snake_case")]
371pub enum SharedCognitionKind {
372 Discovery,
374 Blocker,
376 Hypothesis,
378 Steering,
380 Decision,
382 Handoff,
384}
385
386#[derive(Debug, Clone, Serialize, Deserialize)]
388pub struct SharedCognitionNote {
389 pub id: String,
391 pub run_id: String,
393 #[serde(default, skip_serializing_if = "Option::is_none")]
395 pub task_id: Option<String>,
396 #[serde(default, skip_serializing_if = "Option::is_none")]
398 pub directive_id: Option<String>,
399 pub kind: SharedCognitionKind,
401 pub message_kind: TeamMessageKind,
403 pub summary: String,
405 pub detail: String,
407 #[serde(default, skip_serializing_if = "Option::is_none")]
409 pub sender_agent_id: Option<String>,
410 #[serde(default, skip_serializing_if = "Option::is_none")]
412 pub recipient_agent_id: Option<String>,
413 #[serde(default, skip_serializing_if = "Vec::is_empty")]
415 pub tags: Vec<String>,
416 pub confidence: f32,
418 pub source_message_id: String,
420 pub created_at: DateTime<Utc>,
422}
423
424#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
426pub struct SupervisorRunTaskSummary {
427 pub total: usize,
429 pub queued: usize,
431 pub blocked: usize,
433 pub pending_approval: usize,
435 pub running: usize,
437 pub review_pending: usize,
439 pub test_pending: usize,
441 pub completed: usize,
443 pub failed: usize,
445 pub cancelled: usize,
447}
448
449#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
451pub struct SupervisorInheritancePolicy {
452 #[serde(default)]
454 pub approval_required: bool,
455 #[serde(default)]
457 pub reviewer_required: bool,
458 #[serde(default)]
460 pub test_required: bool,
461 #[serde(default, skip_serializing_if = "Option::is_none")]
463 pub execution_mode: Option<AgentExecutionMode>,
464 #[serde(default, skip_serializing_if = "Option::is_none")]
466 pub workspace_dir: Option<PathBuf>,
467 #[serde(default, skip_serializing_if = "Vec::is_empty")]
469 pub memory_tags: Vec<String>,
470 #[serde(default, skip_serializing_if = "Vec::is_empty")]
472 pub constraint_notes: Vec<String>,
473}
474
475impl SupervisorInheritancePolicy {
476 pub fn apply_to_task(&self, task: &mut DelegatedTask) {
478 task.approval_required |= self.approval_required;
479 task.reviewer_required |= self.reviewer_required;
480 task.test_required |= self.test_required;
481 if let Some(execution_mode) = self.execution_mode.clone() {
482 task.execution_mode = execution_mode;
483 }
484 if task.workspace_dir.is_none() {
485 task.workspace_dir = self.workspace_dir.clone();
486 }
487 for tag in &self.memory_tags {
488 if !task.memory_tags.contains(tag) {
489 task.memory_tags.push(tag.clone());
490 }
491 }
492 }
493}
494
495#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
497pub struct SupervisorParentRunRef {
498 pub parent_run_id: String,
500 #[serde(default, skip_serializing_if = "Option::is_none")]
502 pub parent_task_id: Option<String>,
503 #[serde(default, skip_serializing_if = "Option::is_none")]
505 pub delegated_by_agent_id: Option<String>,
506 pub objective: String,
508 pub created_at: DateTime<Utc>,
510}
511
512#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
514pub struct ChildSupervisorRunSummary {
515 pub run_id: String,
517 #[serde(default, skip_serializing_if = "Option::is_none")]
519 pub name: Option<String>,
520 pub objective: String,
522 #[serde(default, skip_serializing_if = "Option::is_none")]
524 pub lead_agent_id: Option<String>,
525 pub status: SupervisorRunStatus,
527 #[serde(default)]
529 pub task_summary: SupervisorRunTaskSummary,
530 #[serde(default)]
532 pub requires_attention: bool,
533 #[serde(default, skip_serializing_if = "Vec::is_empty")]
535 pub blocked_reasons: Vec<String>,
536 pub created_at: DateTime<Utc>,
538 pub updated_at: DateTime<Utc>,
540 #[serde(default, skip_serializing_if = "Option::is_none")]
542 pub completed_at: Option<DateTime<Utc>>,
543}
544
545#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
547pub struct SupervisorHierarchySummary {
548 #[serde(default)]
550 pub depth: u8,
551 #[serde(default = "default_max_child_supervisor_depth")]
553 pub max_depth: u8,
554 #[serde(default)]
556 pub child_run_count: usize,
557 #[serde(default)]
559 pub descendant_task_count: usize,
560 #[serde(default)]
562 pub action_required_child_count: usize,
563 pub rollup_status: SupervisorRunStatus,
565 #[serde(default)]
567 pub requires_attention: bool,
568 #[serde(default, skip_serializing_if = "Vec::is_empty")]
570 pub blocked_reasons: Vec<String>,
571}
572
573#[derive(Debug, Clone, Serialize, Deserialize)]
575pub struct ChildSupervisorRunRequest {
576 pub parent_run_id: String,
578 #[serde(default, skip_serializing_if = "Option::is_none")]
580 pub run_id: Option<String>,
581 pub lead_agent_id: String,
583 pub objective: String,
585 #[serde(default, skip_serializing_if = "Option::is_none")]
587 pub name: Option<String>,
588 #[serde(default, skip_serializing_if = "Option::is_none")]
590 pub parent_task_id: Option<String>,
591 #[serde(default, skip_serializing_if = "Option::is_none")]
593 pub session_id: Option<String>,
594 #[serde(default, skip_serializing_if = "Option::is_none")]
596 pub workspace_dir: Option<PathBuf>,
597 #[serde(default)]
599 pub approval_required: bool,
600 #[serde(default)]
602 pub reviewer_required: bool,
603 #[serde(default)]
605 pub test_required: bool,
606 #[serde(default)]
608 pub execution_mode: AgentExecutionMode,
609 #[serde(default, skip_serializing_if = "Vec::is_empty")]
611 pub memory_tags: Vec<String>,
612 #[serde(default, skip_serializing_if = "Vec::is_empty")]
614 pub constraint_notes: Vec<String>,
615}
616
617#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
619#[serde(rename_all = "snake_case")]
620pub enum EnvironmentState {
621 #[default]
623 Requested,
624 Provisioning,
626 Ready,
628 InUse,
630 CleanupQueued,
632 Cleaning,
634 Archived,
636 Removed,
638 Recovering,
640 Failed,
642}
643
644#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
646#[serde(rename_all = "snake_case")]
647pub enum EnvironmentHealth {
648 Clean,
650 Dirty,
652 Missing,
654 Drifted,
656 Orphaned,
658 #[default]
660 Unknown,
661}
662
663#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
665#[serde(rename_all = "snake_case")]
666pub enum CleanupPolicy {
667 #[default]
669 KeepAlways,
670 RemoveOnSuccess,
672 ArchiveOnFailure,
674 ArchiveAlways,
676 RemoveWhenCleanOtherwiseArchive,
678}
679
680#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
682#[serde(rename_all = "snake_case")]
683pub enum CleanupDisposition {
684 Kept,
686 Archived,
688 Removed,
690}
691
692#[derive(Debug, Clone, Serialize, Deserialize)]
694pub struct CleanupResult {
695 pub disposition: CleanupDisposition,
697 pub completed_at: DateTime<Utc>,
699 #[serde(default, skip_serializing_if = "Option::is_none")]
701 pub retained_path: Option<PathBuf>,
702 pub summary: String,
704}
705
706#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
708#[serde(rename_all = "snake_case")]
709pub enum RecoveryStatus {
710 #[default]
712 NotRequired,
713 Pending,
715 Reconciled,
717 NeedsOperatorAction,
719 Failed,
721}
722
723#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
725#[serde(rename_all = "snake_case")]
726pub enum RecoveryAction {
727 Noop,
729 RecreateMissingEnvironment,
731 ReleaseStaleLease,
733 ArchiveDirtyEnvironment,
735 QueueCleanup,
737 MarkTaskBlocked,
739}
740
741#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
743#[serde(rename_all = "snake_case")]
744pub enum EnvironmentFailureKind {
745 WorkspaceNotFound,
746 PathOutsideWorkspace,
747 NotGitRepository,
748 GitCommandFailed,
749 WorktreeAlreadyExists,
750 WorktreeCreationFailed,
751 WorktreeInvalid,
752 WorktreeDirty,
753 CleanupDenied,
754 LeaseConflict,
755 PersistenceError,
756 RecoveryError,
757}
758
759#[derive(Debug, Clone, Serialize, Deserialize)]
761pub struct EnvironmentFailure {
762 pub kind: EnvironmentFailureKind,
764 pub message: String,
766 #[serde(default, skip_serializing_if = "Option::is_none")]
768 pub command: Option<String>,
769 #[serde(default, skip_serializing_if = "Option::is_none")]
771 pub stderr: Option<String>,
772 pub occurred_at: DateTime<Utc>,
774}
775
776#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
778#[serde(rename_all = "snake_case")]
779pub enum EnvironmentLeaseKind {
780 Execution,
782 Recovery,
784 Cleanup,
786}
787
788#[derive(Debug, Clone, Serialize, Deserialize)]
790pub struct EnvironmentLease {
791 pub task_id: String,
793 pub agent_id: String,
795 pub lease_kind: EnvironmentLeaseKind,
797 pub acquired_at: DateTime<Utc>,
799 #[serde(default, skip_serializing_if = "Option::is_none")]
801 pub released_at: Option<DateTime<Utc>>,
802}
803
804#[derive(Debug, Clone, Serialize, Deserialize)]
806pub struct GitWorktreeSpec {
807 pub repo_root: PathBuf,
809 pub base_branch: String,
811 pub worktree_branch: String,
813 pub worktree_path: PathBuf,
815 pub create_branch_if_missing: bool,
817}
818
819#[derive(Debug, Clone, Serialize, Deserialize)]
821pub struct EnvironmentSpec {
822 pub id: String,
824 pub execution_mode: AgentExecutionMode,
826 pub workspace_root: PathBuf,
828 pub prepared_path: PathBuf,
830 #[serde(default, skip_serializing_if = "Option::is_none")]
832 pub session_id: Option<String>,
833 pub run_id: String,
835 pub task_id: String,
837 pub agent_id: String,
839 pub cleanup_policy: CleanupPolicy,
841 pub write_access: bool,
843 #[serde(default, skip_serializing_if = "Option::is_none")]
845 pub git_worktree: Option<GitWorktreeSpec>,
846 #[serde(default, skip_serializing_if = "Option::is_none")]
848 pub remote_url: Option<String>,
849}
850
851#[derive(Debug, Clone, Serialize, Deserialize)]
853pub struct ExecutionEnvironment {
854 pub id: String,
856 pub execution_mode: AgentExecutionMode,
858 pub root_dir: PathBuf,
860 pub write_access: bool,
862 #[serde(default, skip_serializing_if = "Option::is_none")]
864 pub branch_name: Option<String>,
865 #[serde(default, skip_serializing_if = "Option::is_none")]
867 pub worktree_path: Option<PathBuf>,
868 #[serde(default, skip_serializing_if = "Option::is_none")]
870 pub remote_url: Option<String>,
871 #[serde(default)]
873 pub state: EnvironmentState,
874 #[serde(default)]
876 pub health: EnvironmentHealth,
877 #[serde(default)]
879 pub cleanup_policy: CleanupPolicy,
880 #[serde(default)]
882 pub recovery_status: RecoveryStatus,
883 #[serde(default, skip_serializing_if = "Option::is_none")]
885 pub recovery_action: Option<RecoveryAction>,
886 #[serde(default, skip_serializing_if = "Option::is_none")]
888 pub failure: Option<EnvironmentFailure>,
889 #[serde(default, skip_serializing_if = "Option::is_none")]
891 pub cleanup_result: Option<CleanupResult>,
892}
893
894impl ExecutionEnvironment {
895 fn from_record(record: &EnvironmentRecord) -> Self {
896 let git_worktree = record.spec.git_worktree.as_ref();
897 Self {
898 id: record.id.clone(),
899 execution_mode: record.spec.execution_mode.clone(),
900 root_dir: record.prepared_path.clone(),
901 write_access: record.spec.write_access,
902 branch_name: git_worktree.map(|spec| spec.worktree_branch.clone()),
903 worktree_path: git_worktree.map(|spec| spec.worktree_path.clone()),
904 remote_url: record.spec.remote_url.clone(),
905 state: record.state,
906 health: record.health,
907 cleanup_policy: record.spec.cleanup_policy,
908 recovery_status: record.recovery_status,
909 recovery_action: record.recovery_action,
910 failure: record.failure.clone(),
911 cleanup_result: record.cleanup_result.clone(),
912 }
913 }
914}
915
916#[derive(Debug, Clone, Serialize, Deserialize)]
918pub struct EnvironmentRecord {
919 pub id: String,
921 pub spec: EnvironmentSpec,
923 pub state: EnvironmentState,
925 pub health: EnvironmentHealth,
927 pub prepared_path: PathBuf,
929 #[serde(default, skip_serializing_if = "Option::is_none")]
931 pub lease: Option<EnvironmentLease>,
932 #[serde(default, skip_serializing_if = "Option::is_none")]
934 pub cleanup_result: Option<CleanupResult>,
935 #[serde(default)]
937 pub recovery_status: RecoveryStatus,
938 #[serde(default, skip_serializing_if = "Option::is_none")]
940 pub recovery_action: Option<RecoveryAction>,
941 #[serde(default, skip_serializing_if = "Option::is_none")]
943 pub failure: Option<EnvironmentFailure>,
944 pub created_at: DateTime<Utc>,
946 pub updated_at: DateTime<Utc>,
948 #[serde(default, skip_serializing_if = "Option::is_none")]
950 pub last_verified_at: Option<DateTime<Utc>>,
951 #[serde(default, skip_serializing_if = "Option::is_none")]
953 pub metadata: Option<serde_json::Value>,
954}
955
956impl EnvironmentRecord {
957 fn summary(&self) -> ExecutionEnvironment {
958 ExecutionEnvironment::from_record(self)
959 }
960}
961
962#[derive(Debug, Clone, Serialize, Deserialize)]
964pub struct RemoteExecutionProgress {
965 #[serde(default, skip_serializing_if = "Option::is_none")]
967 pub stage: Option<String>,
968 #[serde(default, skip_serializing_if = "Option::is_none")]
970 pub message: Option<String>,
971 #[serde(default, skip_serializing_if = "Option::is_none")]
973 pub percent: Option<u8>,
974 pub updated_at: DateTime<Utc>,
976}
977
978#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
980#[serde(rename_all = "snake_case")]
981pub enum LocalExecutionPhase {
982 Queued,
984 Running,
986 Waiting,
988 Blocked,
990 Completed,
992 Failed,
994 Cancelled,
996}
997
998#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1000#[serde(rename_all = "snake_case")]
1001pub enum LocalExecutionWaitingReason {
1002 ShellProcess,
1004 Reflection,
1006 ToolConfirmation,
1008 EnvironmentTransition,
1010}
1011
1012#[derive(Debug, Clone, Serialize, Deserialize)]
1014pub struct LocalExecutionTokenUsageSnapshot {
1015 #[serde(default, skip_serializing_if = "Option::is_none")]
1017 pub estimated_tokens: Option<usize>,
1018 #[serde(default, skip_serializing_if = "Option::is_none")]
1020 pub limit: Option<usize>,
1021 #[serde(default, skip_serializing_if = "Option::is_none")]
1023 pub percentage: Option<u8>,
1024 #[serde(default, skip_serializing_if = "Option::is_none")]
1026 pub status: Option<String>,
1027 #[serde(default, skip_serializing_if = "Option::is_none")]
1029 pub estimated_cost_usd: Option<f64>,
1030 #[serde(default, skip_serializing_if = "Option::is_none")]
1032 pub input_tokens: Option<u32>,
1033 #[serde(default, skip_serializing_if = "Option::is_none")]
1035 pub output_tokens: Option<u32>,
1036 #[serde(default, skip_serializing_if = "Option::is_none")]
1038 pub total_tokens: Option<u32>,
1039 #[serde(default, skip_serializing_if = "Option::is_none")]
1041 pub model: Option<String>,
1042 #[serde(default, skip_serializing_if = "Option::is_none")]
1044 pub provider: Option<String>,
1045}
1046
1047#[derive(Debug, Clone, Serialize, Deserialize)]
1049pub struct LocalExecutionEnvironmentSnapshot {
1050 pub state: EnvironmentState,
1052 pub health: EnvironmentHealth,
1054 pub recovery_status: RecoveryStatus,
1056 pub updated_at: DateTime<Utc>,
1058}
1059
1060#[derive(Debug, Clone, Serialize, Deserialize)]
1062pub struct LocalExecutionProgress {
1063 pub phase: LocalExecutionPhase,
1065 #[serde(default, skip_serializing_if = "Option::is_none")]
1067 pub waiting_reason: Option<LocalExecutionWaitingReason>,
1068 #[serde(default, skip_serializing_if = "Option::is_none")]
1070 pub stage: Option<String>,
1071 #[serde(default, skip_serializing_if = "Option::is_none")]
1073 pub message: Option<String>,
1074 #[serde(default, skip_serializing_if = "Option::is_none")]
1076 pub percent: Option<u8>,
1077 #[serde(default)]
1079 pub iteration: u32,
1080 #[serde(default, skip_serializing_if = "Option::is_none")]
1082 pub current_tool_name: Option<String>,
1083 #[serde(default, skip_serializing_if = "Option::is_none")]
1085 pub last_completed_tool_name: Option<String>,
1086 #[serde(default, skip_serializing_if = "Option::is_none")]
1088 pub last_completed_tool_duration_ms: Option<u64>,
1089 #[serde(default)]
1091 pub completed_tool_call_count: usize,
1092 #[serde(default)]
1094 pub has_partial_content: bool,
1095 #[serde(default)]
1097 pub partial_content_chars: usize,
1098 #[serde(default)]
1100 pub has_partial_thinking: bool,
1101 #[serde(default)]
1103 pub partial_thinking_chars: usize,
1104 #[serde(default, skip_serializing_if = "Option::is_none")]
1106 pub token_usage: Option<LocalExecutionTokenUsageSnapshot>,
1107 #[serde(default, skip_serializing_if = "Option::is_none")]
1109 pub environment: Option<LocalExecutionEnvironmentSnapshot>,
1110 pub updated_at: DateTime<Utc>,
1112}
1113
1114#[derive(Debug, Clone, Serialize, Deserialize)]
1116pub struct LocalExecutionRecord {
1117 pub status: String,
1119 #[serde(default, skip_serializing_if = "Option::is_none")]
1121 pub status_reason: Option<String>,
1122 #[serde(default, skip_serializing_if = "Option::is_none")]
1124 pub progress: Option<LocalExecutionProgress>,
1125 pub last_synced_at: DateTime<Utc>,
1127}
1128
1129#[derive(Debug, Clone, Serialize, Deserialize)]
1131pub struct ActiveTaskSnapshot {
1132 pub task: DelegatedTask,
1134 pub state: SupervisorTaskState,
1136 #[serde(default, skip_serializing_if = "Option::is_none")]
1138 pub remote_execution: Option<RemoteExecutionRecord>,
1139 #[serde(default, skip_serializing_if = "Option::is_none")]
1141 pub local_execution: Option<LocalExecutionRecord>,
1142 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1144 pub blocked_reasons: Vec<String>,
1145 #[serde(default, skip_serializing_if = "Option::is_none")]
1147 pub checkpoint: Option<DelegatedCheckpointSummary>,
1148}
1149
1150#[derive(Debug, Clone, Serialize, Deserialize)]
1152pub struct RemoteExecutionArtifact {
1153 pub name: String,
1155 #[serde(default)]
1157 pub part_count: usize,
1158 #[serde(default, skip_serializing_if = "HashMap::is_empty")]
1160 pub metadata: HashMap<String, serde_json::Value>,
1161}
1162
1163#[derive(Debug, Clone, Serialize, Deserialize, Default)]
1165pub struct RemoteExecutionCompatibility {
1166 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1168 pub supported_features: Vec<String>,
1169 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1171 pub warnings: Vec<String>,
1172 #[serde(default, skip_serializing_if = "Option::is_none")]
1174 pub protocol_version: Option<String>,
1175}
1176
1177#[derive(Debug, Clone, Serialize, Deserialize)]
1179pub struct RemoteExecutionRecord {
1180 pub target: RemoteAgentTarget,
1182 pub remote_task_id: String,
1184 pub status: String,
1186 #[serde(default, skip_serializing_if = "Option::is_none")]
1188 pub status_reason: Option<String>,
1189 #[serde(default, skip_serializing_if = "Option::is_none")]
1191 pub lease: Option<RemoteTaskLease>,
1192 #[serde(default, skip_serializing_if = "Option::is_none")]
1194 pub progress: Option<RemoteExecutionProgress>,
1195 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1197 pub artifacts: Vec<RemoteExecutionArtifact>,
1198 #[serde(default, skip_serializing_if = "Option::is_none")]
1200 pub provenance: Option<gestura_core_a2a::TaskProvenance>,
1201 #[serde(default)]
1203 pub compatibility: RemoteExecutionCompatibility,
1204 pub last_synced_at: DateTime<Utc>,
1206}
1207
1208#[derive(Debug, Clone, Serialize, Deserialize)]
1210pub struct SupervisorTaskRecord {
1211 pub task: DelegatedTask,
1213 pub state: SupervisorTaskState,
1215 pub approval: TaskApprovalRecord,
1217 #[serde(default)]
1219 pub environment_id: String,
1220 pub environment: ExecutionEnvironment,
1222 #[serde(default, skip_serializing_if = "Option::is_none")]
1224 pub claimed_by: Option<String>,
1225 #[serde(default)]
1227 pub attempts: u32,
1228 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1230 pub blocked_reasons: Vec<String>,
1231 #[serde(default, skip_serializing_if = "Option::is_none")]
1233 pub result: Option<TaskResult>,
1234 #[serde(default, skip_serializing_if = "Option::is_none")]
1236 pub remote_execution: Option<RemoteExecutionRecord>,
1237 #[serde(default, skip_serializing_if = "Option::is_none")]
1239 pub local_execution: Option<LocalExecutionRecord>,
1240 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1242 pub messages: Vec<TeamMessage>,
1243 #[serde(default, skip_serializing_if = "Option::is_none")]
1245 pub checkpoint: Option<DelegatedCheckpointSummary>,
1246 pub created_at: DateTime<Utc>,
1248 pub updated_at: DateTime<Utc>,
1250 #[serde(default, skip_serializing_if = "Option::is_none")]
1252 pub started_at: Option<DateTime<Utc>>,
1253 #[serde(default, skip_serializing_if = "Option::is_none")]
1255 pub completed_at: Option<DateTime<Utc>>,
1256}
1257
1258#[derive(Debug, Clone, Serialize, Deserialize)]
1260pub struct SupervisorRun {
1261 pub id: String,
1263 #[serde(default, skip_serializing_if = "Option::is_none")]
1265 pub name: Option<String>,
1266 #[serde(default, skip_serializing_if = "Option::is_none")]
1268 pub session_id: Option<String>,
1269 #[serde(default, skip_serializing_if = "Option::is_none")]
1271 pub workspace_dir: Option<PathBuf>,
1272 #[serde(default, skip_serializing_if = "Option::is_none")]
1274 pub lead_agent_id: Option<String>,
1275 #[serde(default, skip_serializing_if = "Option::is_none")]
1277 pub parent_run: Option<SupervisorParentRunRef>,
1278 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1280 pub child_runs: Vec<ChildSupervisorRunSummary>,
1281 #[serde(default)]
1283 pub hierarchy_depth: u8,
1284 #[serde(default = "default_max_child_supervisor_depth")]
1286 pub max_hierarchy_depth: u8,
1287 #[serde(default, skip_serializing_if = "Option::is_none")]
1289 pub inherited_policy: Option<SupervisorInheritancePolicy>,
1290 pub status: SupervisorRunStatus,
1292 #[serde(default)]
1294 pub task_summary: SupervisorRunTaskSummary,
1295 #[serde(default, skip_serializing_if = "Option::is_none")]
1297 pub hierarchy_summary: Option<SupervisorHierarchySummary>,
1298 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1300 pub tasks: Vec<SupervisorTaskRecord>,
1301 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1303 pub messages: Vec<TeamMessage>,
1304 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1306 pub shared_cognition: Vec<SharedCognitionNote>,
1307 pub created_at: DateTime<Utc>,
1309 pub updated_at: DateTime<Utc>,
1311 #[serde(default, skip_serializing_if = "Option::is_none")]
1313 pub completed_at: Option<DateTime<Utc>>,
1314 #[serde(default, skip_serializing_if = "Option::is_none")]
1316 pub metadata: Option<serde_json::Value>,
1317}
1318
1319#[async_trait::async_trait]
1324pub trait OrchestratorAgentManager: AgentSpawner + Clone + Send + Sync + 'static {
1325 async fn get_agent_status(&self, id: &str) -> Option<AgentInfo>;
1327
1328 async fn list_agents(&self) -> Vec<AgentInfo>;
1330
1331 async fn update_activity(&self, id: &str);
1333}
1334
1335#[async_trait::async_trait]
1340pub trait OrchestratorObserver: Send + Sync {
1341 async fn on_task_started(&self, task: DelegatedTask);
1343
1344 async fn on_task_completed(&self, task: DelegatedTask, result: TaskResult);
1346
1347 async fn on_run_updated(&self, _run: SupervisorRun) {}
1349
1350 async fn on_team_message(&self, _message: TeamMessage) {}
1352
1353 async fn on_team_thread_updated(&self, _thread: TeamThread) {}
1355
1356 async fn on_environment_updated(&self, _environment: EnvironmentRecord) {}
1358
1359 async fn on_environment_recovery(
1361 &self,
1362 _environment_id: String,
1363 _action: RecoveryAction,
1364 _summary: String,
1365 ) {
1366 }
1367
1368 async fn on_environment_cleanup(&self, _environment_id: String, _result: CleanupResult) {}
1370
1371 #[cfg(feature = "ring-integration")]
1373 async fn on_haptic_feedback(
1374 &self,
1375 _pattern: gestura_core_haptics::HapticPattern,
1376 _intensity: f32,
1377 _duration_ms: u32,
1378 ) {
1379 }
1380}
1381
1382#[derive(Clone, Debug)]
1383struct ActiveTaskControl {
1384 task: DelegatedTask,
1385 local_cancel_token: Option<CancellationToken>,
1386 attempt: u32,
1387}
1388
1389#[derive(Debug)]
1390struct LocalDelegatedExecutionOutcome {
1391 result: Result<String, String>,
1392 tool_calls: Vec<OrchestratorToolCall>,
1393 terminal_state_hint: TaskTerminalStateHint,
1394 preserve_existing_checkpoint: bool,
1395}
1396
1397impl LocalDelegatedExecutionOutcome {
1398 fn into_task_result(self, task: &DelegatedTask, duration_ms: u64) -> TaskResult {
1399 TaskResult {
1400 task_id: task.id.clone(),
1401 agent_id: task.agent_id.clone(),
1402 success: self.result.is_ok(),
1403 run_id: task.run_id.clone(),
1404 tracking_task_id: task.tracking_task_id.clone(),
1405 output: self.result.unwrap_or_else(|error| error),
1406 summary: task.name.clone(),
1407 tool_calls: self.tool_calls,
1408 artifacts: Vec::new(),
1409 terminal_state_hint: Some(self.terminal_state_hint),
1410 duration_ms,
1411 }
1412 }
1413}
1414
1415#[derive(Clone)]
1420pub struct AgentOrchestrator<M: OrchestratorAgentManager> {
1421 agent_manager: M,
1422 permission_manager: Arc<PermissionManager>,
1423 active_tasks: Arc<Mutex<HashMap<String, ActiveTaskControl>>>,
1424 supervisor_runs: Arc<Mutex<HashMap<String, SupervisorRun>>>,
1425 environments: Arc<Mutex<HashMap<String, EnvironmentRecord>>>,
1426 task_run_index: Arc<Mutex<HashMap<String, String>>>,
1427 result_tx: mpsc::Sender<TaskResult>,
1428 result_rx: Arc<Mutex<mpsc::Receiver<TaskResult>>>,
1429 config: AppConfig,
1430 observer: Arc<RwLock<Option<Arc<dyn OrchestratorObserver>>>>,
1431 default_workspace_dir: Option<PathBuf>,
1432}
1433
1434impl<M: OrchestratorAgentManager> AgentOrchestrator<M> {
1435 pub fn new(agent_manager: M, config: AppConfig) -> Self {
1437 Self::new_with_workspace_root(agent_manager, config, std::env::current_dir().ok())
1438 }
1439
1440 pub fn new_with_workspace_root(
1442 agent_manager: M,
1443 config: AppConfig,
1444 default_workspace_dir: Option<PathBuf>,
1445 ) -> Self {
1446 let (result_tx, result_rx) = mpsc::channel(100);
1447 let orchestrator = Self {
1448 agent_manager,
1449 permission_manager: Arc::new(PermissionManager::new()),
1450 active_tasks: Arc::new(Mutex::new(HashMap::new())),
1451 supervisor_runs: Arc::new(Mutex::new(HashMap::new())),
1452 environments: Arc::new(Mutex::new(HashMap::new())),
1453 task_run_index: Arc::new(Mutex::new(HashMap::new())),
1454 result_tx,
1455 result_rx: Arc::new(Mutex::new(result_rx)),
1456 config,
1457 observer: Arc::new(RwLock::new(None)),
1458 default_workspace_dir,
1459 };
1460 orchestrator.bootstrap_persisted_state();
1461 orchestrator
1462 }
1463
1464 pub async fn set_observer(&self, observer: Arc<dyn OrchestratorObserver>) {
1470 *self.observer.write().await = Some(observer);
1471 }
1472
1473 pub async fn clear_observer(&self) {
1475 *self.observer.write().await = None;
1476 }
1477
1478 pub async fn spawn_subagent(&self, id: &str, name: &str) -> Result<(), String> {
1480 self.spawn_subagent_with_request(AgentSpawnRequest::new(
1481 id.to_string(),
1482 name.to_string(),
1483 AgentRole::Implementer,
1484 ))
1485 .await
1486 }
1487
1488 pub async fn spawn_subagent_with_request(
1490 &self,
1491 request: AgentSpawnRequest,
1492 ) -> Result<(), String> {
1493 tracing::info!(agent_id = %request.id, agent_name = %request.name, role = ?request.role, "Spawning subagent");
1494 self.agent_manager.spawn_agent_with_request(request).await;
1495 Ok(())
1496 }
1497
1498 pub async fn delegate_task(&self, mut task: DelegatedTask) -> Result<String, String> {
1504 let task_id = task.id.clone();
1505 let agent_id = task.agent_id.clone();
1506
1507 if task.run_id.is_none() {
1508 task.run_id = Some(Uuid::new_v4().to_string());
1509 }
1510 if task.role.is_none() {
1511 task.role = Some(AgentRole::Implementer);
1512 }
1513 if task.name.is_none() {
1514 task.name = Some(format!(
1515 "{} task",
1516 task.role.clone().unwrap_or_default().label()
1517 ));
1518 }
1519
1520 if let Some(run_id) = task.run_id.as_deref() {
1521 let runs = self.supervisor_runs.lock().await;
1522 if let Some(run) = runs.get(run_id) {
1523 if task.session_id.is_none() {
1524 task.session_id = run.session_id.clone();
1525 }
1526 if task.workspace_dir.is_none() {
1527 task.workspace_dir = run
1528 .workspace_dir
1529 .clone()
1530 .or_else(|| self.default_workspace_dir.clone());
1531 }
1532 if let Some(policy) = run.inherited_policy.as_ref() {
1533 policy.apply_to_task(&mut task);
1534 }
1535 }
1536 }
1537 if task.workspace_dir.is_none() {
1538 task.workspace_dir = self.default_workspace_dir.clone();
1539 }
1540
1541 let session_id = task.session_id.clone();
1542
1543 tracing::info!(
1544 task_id = %task_id,
1545 agent_id = %agent_id,
1546 session_id = ?session_id,
1547 run_id = ?task.run_id,
1548 role = ?task.role,
1549 priority = task.priority,
1550 "Delegating task to subagent"
1551 );
1552
1553 self.ensure_agent_exists(&task).await?;
1554
1555 if task.execution_mode != AgentExecutionMode::Remote {
1558 for tool in &task.required_tools {
1559 let check = self
1560 .permission_manager
1561 .check(tool, "execute", None)
1562 .map_err(|e| format!("Permission check error: {}", e))?;
1563 if !check.allowed {
1564 tracing::warn!(tool = %tool, task_id = %task_id, reason = %check.reason, "Tool not permitted for task");
1565 return Err(format!("Tool '{}' not permitted: {}", tool, check.reason));
1566 }
1567 }
1568 }
1569
1570 let environment_record = self.prepare_environment(&task).await?;
1571 let environment = environment_record.summary();
1572 task.environment_id = Some(environment_record.id.clone());
1573 self.ensure_tracking_task(&mut task).await;
1574
1575 let run_id = task
1576 .run_id
1577 .clone()
1578 .ok_or_else(|| "Delegated task is missing run_id".to_string())?;
1579
1580 let (run_snapshot, task_snapshot, should_start, initial_message) = {
1581 let mut runs = self.supervisor_runs.lock().await;
1582 let run = runs.entry(run_id.clone()).or_insert_with(|| SupervisorRun {
1583 id: run_id.clone(),
1584 name: task.name.clone(),
1585 session_id: task.session_id.clone(),
1586 workspace_dir: task
1587 .workspace_dir
1588 .clone()
1589 .or_else(|| self.default_workspace_dir.clone()),
1590 lead_agent_id: Some("supervisor".to_string()),
1591 parent_run: None,
1592 child_runs: Vec::new(),
1593 hierarchy_depth: 0,
1594 max_hierarchy_depth: MAX_CHILD_SUPERVISOR_DEPTH,
1595 inherited_policy: None,
1596 status: SupervisorRunStatus::Draft,
1597 task_summary: SupervisorRunTaskSummary::default(),
1598 hierarchy_summary: None,
1599 tasks: Vec::new(),
1600 messages: Vec::new(),
1601 shared_cognition: Vec::new(),
1602 created_at: Utc::now(),
1603 updated_at: Utc::now(),
1604 completed_at: None,
1605 metadata: None,
1606 });
1607
1608 if run.session_id.is_none() {
1609 run.session_id = task.session_id.clone();
1610 }
1611 if run.workspace_dir.is_none() {
1612 run.workspace_dir = task
1613 .workspace_dir
1614 .clone()
1615 .or_else(|| self.default_workspace_dir.clone());
1616 }
1617 if run.name.is_none() {
1618 run.name = task.name.clone();
1619 }
1620 if task.session_id.is_none() {
1621 task.session_id = run.session_id.clone();
1622 }
1623 if task.workspace_dir.is_none() {
1624 task.workspace_dir = run
1625 .workspace_dir
1626 .clone()
1627 .or_else(|| self.default_workspace_dir.clone());
1628 }
1629 if let Some(policy) = run.inherited_policy.as_ref() {
1630 policy.apply_to_task(&mut task);
1631 }
1632 ensure_shared_memory_tags(&mut task, &run.id);
1633
1634 let blocked_reasons = unresolved_dependency_reasons(run, &task);
1635 let approval = if task.approval_required {
1636 TaskApprovalRecord::pending(
1637 &task,
1638 ApprovalScope::PreExecution,
1639 ApprovalActor::system("orchestrator"),
1640 Some("Task submitted. Awaiting explicit pre-execution approval.".to_string()),
1641 )
1642 } else {
1643 TaskApprovalRecord::not_required(&task)
1644 };
1645 let mut blocked_reasons = blocked_reasons;
1646 if matches!(environment.state, EnvironmentState::Failed)
1647 && let Some(failure) = &environment.failure
1648 {
1649 blocked_reasons.push(failure.message.clone());
1650 }
1651 let state = if matches!(environment.state, EnvironmentState::Failed) {
1652 SupervisorTaskState::Blocked
1653 } else if task.approval_required {
1654 SupervisorTaskState::PendingApproval
1655 } else if blocked_reasons.is_empty() {
1656 SupervisorTaskState::Queued
1657 } else {
1658 SupervisorTaskState::Blocked
1659 };
1660
1661 let now = Utc::now();
1662 let mut new_record = SupervisorTaskRecord {
1663 task: task.clone(),
1664 state,
1665 approval,
1666 environment_id: environment_record.id.clone(),
1667 environment,
1668 claimed_by: Some(task.agent_id.clone()),
1669 attempts: 0,
1670 blocked_reasons,
1671 result: None,
1672 remote_execution: None,
1673 local_execution: None,
1674 messages: Vec::new(),
1675 checkpoint: None,
1676 created_at: now,
1677 updated_at: now,
1678 started_at: None,
1679 completed_at: None,
1680 };
1681
1682 let initial_message = if matches!(state, SupervisorTaskState::PendingApproval) {
1683 let message =
1684 build_gate_request_message(&run.id, &new_record, ApprovalScope::PreExecution);
1685 new_record.messages.push(message.clone());
1686 Some(message)
1687 } else {
1688 None
1689 };
1690
1691 if let Some(existing) = run
1692 .tasks
1693 .iter_mut()
1694 .find(|record| record.task.id == task_id)
1695 {
1696 new_record.attempts = existing.attempts;
1697 new_record.created_at = existing.created_at;
1698 *existing = new_record.clone();
1699 } else {
1700 run.tasks.push(new_record.clone());
1701 }
1702
1703 run.updated_at = Utc::now();
1704 run.status = recalculate_run_status(run);
1705 run.task_summary = summarize_run_tasks(run);
1706 run.hierarchy_summary = Some(build_hierarchy_summary(run));
1707
1708 (
1709 run.clone(),
1710 new_record.clone(),
1711 state == SupervisorTaskState::Queued,
1712 initial_message,
1713 )
1714 };
1715
1716 self.task_run_index
1717 .lock()
1718 .await
1719 .insert(task_id.clone(), run_id.clone());
1720
1721 record_task_dispatch(&task, &task_snapshot, &run_snapshot);
1722 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
1723 if let Some(message) = initial_message {
1724 self.notify_team_message(message).await;
1725 }
1726
1727 if should_start {
1728 self.start_task_execution(task.clone()).await?;
1729 }
1730
1731 Ok(task_id)
1732 }
1733
1734 pub async fn create_child_supervisor_run(
1736 &self,
1737 request: ChildSupervisorRunRequest,
1738 ) -> Result<SupervisorRun, String> {
1739 let objective = request.objective.trim();
1740 if objective.is_empty() {
1741 return Err("Child supervisor objective cannot be empty".to_string());
1742 }
1743 if request.lead_agent_id.trim().is_empty() {
1744 return Err("Child supervisor lead_agent_id cannot be empty".to_string());
1745 }
1746
1747 let child_run_id = request
1748 .run_id
1749 .clone()
1750 .filter(|value| !value.trim().is_empty())
1751 .unwrap_or_else(|| format!("run-child-{}", Uuid::new_v4()));
1752 if self.get_supervisor_run(&child_run_id).await.is_some() {
1753 return Err(format!("Supervisor run '{}' already exists", child_run_id));
1754 }
1755
1756 let parent_snapshot = self
1757 .get_supervisor_run(&request.parent_run_id)
1758 .await
1759 .ok_or_else(|| {
1760 format!(
1761 "Parent supervisor run '{}' not found",
1762 request.parent_run_id
1763 )
1764 })?;
1765 ensure_parent_run_accepts_child(&parent_snapshot)?;
1766
1767 if self
1768 .agent_manager
1769 .get_agent_status(&request.lead_agent_id)
1770 .await
1771 .is_none()
1772 {
1773 let mut spawn_request = AgentSpawnRequest::new(
1774 request.lead_agent_id.clone(),
1775 request
1776 .name
1777 .clone()
1778 .unwrap_or_else(|| format!("Child supervisor {}", request.lead_agent_id)),
1779 AgentRole::Supervisor,
1780 );
1781 spawn_request.workspace_dir = request
1782 .workspace_dir
1783 .clone()
1784 .or_else(|| parent_snapshot.workspace_dir.clone());
1785 spawn_request.execution_mode = request.execution_mode.clone();
1786 self.spawn_subagent_with_request(spawn_request).await?;
1787 }
1788
1789 let now = Utc::now();
1790 let (child_run, parent_run, parent_message, child_message) = {
1791 let mut runs = self.supervisor_runs.lock().await;
1792 let parent = runs.get_mut(&request.parent_run_id).ok_or_else(|| {
1793 format!(
1794 "Parent supervisor run '{}' no longer exists",
1795 request.parent_run_id
1796 )
1797 })?;
1798 ensure_parent_run_accepts_child(parent)?;
1799
1800 let inherited_policy = build_child_inherited_policy(parent, &request);
1801 let child_run = SupervisorRun {
1802 id: child_run_id.clone(),
1803 name: request.name.clone().or_else(|| Some(objective.to_string())),
1804 session_id: request
1805 .session_id
1806 .clone()
1807 .or_else(|| parent.session_id.clone()),
1808 workspace_dir: request
1809 .workspace_dir
1810 .clone()
1811 .or_else(|| parent.workspace_dir.clone()),
1812 lead_agent_id: Some(request.lead_agent_id.clone()),
1813 parent_run: Some(SupervisorParentRunRef {
1814 parent_run_id: parent.id.clone(),
1815 parent_task_id: request.parent_task_id.clone(),
1816 delegated_by_agent_id: parent.lead_agent_id.clone(),
1817 objective: objective.to_string(),
1818 created_at: now,
1819 }),
1820 child_runs: Vec::new(),
1821 hierarchy_depth: parent.hierarchy_depth.saturating_add(1),
1822 max_hierarchy_depth: parent.max_hierarchy_depth,
1823 inherited_policy: Some(inherited_policy),
1824 status: SupervisorRunStatus::Draft,
1825 task_summary: SupervisorRunTaskSummary::default(),
1826 hierarchy_summary: Some(SupervisorHierarchySummary {
1827 depth: parent.hierarchy_depth.saturating_add(1),
1828 max_depth: parent.max_hierarchy_depth,
1829 child_run_count: 0,
1830 descendant_task_count: 0,
1831 action_required_child_count: 0,
1832 rollup_status: SupervisorRunStatus::Draft,
1833 requires_attention: false,
1834 blocked_reasons: Vec::new(),
1835 }),
1836 tasks: Vec::new(),
1837 messages: Vec::new(),
1838 shared_cognition: Vec::new(),
1839 created_at: now,
1840 updated_at: now,
1841 completed_at: None,
1842 metadata: None,
1843 };
1844
1845 let parent_message = TeamMessage::new(
1846 parent.id.clone(),
1847 request.parent_task_id.clone(),
1848 TeamMessageKind::Handoff,
1849 parent.lead_agent_id.clone(),
1850 Some(request.lead_agent_id.clone()),
1851 format!(
1852 "Delegated child supervisor run {} for objective: {}",
1853 child_run_id, objective
1854 ),
1855 );
1856 let child_message = TeamMessage::new(
1857 child_run_id.clone(),
1858 None,
1859 TeamMessageKind::StatusUpdate,
1860 parent.lead_agent_id.clone(),
1861 Some(request.lead_agent_id.clone()),
1862 format!(
1863 "Child supervisor run created under {} with objective: {}",
1864 parent.id, objective
1865 ),
1866 );
1867
1868 parent.messages.push(parent_message.clone());
1869 parent.updated_at = now;
1870 parent.completed_at = None;
1871 parent.child_runs.push(build_child_run_summary(&child_run));
1872 parent.task_summary = summarize_run_tasks(parent);
1873 parent.status = recalculate_run_status(parent);
1874 parent.hierarchy_summary = Some(build_hierarchy_summary(parent));
1875 let parent_run = parent.clone();
1876
1877 let mut child_run = child_run;
1878 child_run.messages.push(child_message.clone());
1879 runs.insert(child_run.id.clone(), child_run.clone());
1880 (child_run, parent_run, parent_message, child_message)
1881 };
1882
1883 self.persist_run_async(&child_run).await?;
1884 self.persist_run_async(&parent_run).await?;
1885 self.notify_run_updated(child_run.clone()).await;
1886 self.notify_run_updated(parent_run).await;
1887 self.notify_team_message(parent_message).await;
1888 self.notify_team_message(child_message).await;
1889 Ok(child_run)
1890 }
1891
1892 pub async fn poll_result(&self) -> Option<TaskResult> {
1894 let mut rx = self.result_rx.lock().await;
1895 rx.try_recv().ok()
1896 }
1897
1898 pub async fn list_active_tasks(&self) -> Vec<DelegatedTask> {
1900 let active = self.active_tasks.lock().await;
1901 active.values().map(|entry| entry.task.clone()).collect()
1902 }
1903
1904 pub async fn list_active_task_snapshots(&self) -> Vec<ActiveTaskSnapshot> {
1906 let active = self
1907 .active_tasks
1908 .lock()
1909 .await
1910 .values()
1911 .cloned()
1912 .collect::<Vec<_>>();
1913 if active.is_empty() {
1914 return Vec::new();
1915 }
1916
1917 let runs = self.list_supervisor_runs().await;
1918 let task_run_index = self.task_run_index.lock().await.clone();
1919
1920 let mut snapshots = active
1921 .into_iter()
1922 .map(|entry| {
1923 let record = task_run_index
1924 .get(&entry.task.id)
1925 .and_then(|run_id| runs.iter().find(|run| run.id == *run_id))
1926 .and_then(|run| {
1927 run.tasks
1928 .iter()
1929 .find(|record| record.task.id == entry.task.id)
1930 });
1931 active_task_snapshot(entry.task, record)
1932 })
1933 .collect::<Vec<_>>();
1934 snapshots.sort_by(|left, right| {
1935 right
1936 .task
1937 .priority
1938 .cmp(&left.task.priority)
1939 .then_with(|| left.task.id.cmp(&right.task.id))
1940 });
1941 snapshots
1942 }
1943
1944 pub async fn list_supervisor_runs(&self) -> Vec<SupervisorRun> {
1946 let mut runs = self
1947 .supervisor_runs
1948 .lock()
1949 .await
1950 .values()
1951 .cloned()
1952 .collect::<Vec<_>>();
1953 if let Some(root) = self.default_workspace_dir.as_deref() {
1954 for run in load_persisted_runs(root) {
1955 if !runs.iter().any(|existing| existing.id == run.id) {
1956 runs.push(run);
1957 }
1958 }
1959 }
1960 let checkpoints = load_latest_checkpoints_by_task(checkpoint_roots_for_runs(
1961 &runs,
1962 self.default_workspace_dir.as_deref(),
1963 ));
1964 attach_checkpoint_summaries(&mut runs, checkpoints);
1965 synchronize_run_hierarchy_snapshots(&mut runs);
1966 runs.sort_by(|left, right| right.updated_at.cmp(&left.updated_at));
1967 runs
1968 }
1969
1970 pub async fn list_root_supervisor_runs(&self) -> Vec<SupervisorRun> {
1972 self.list_supervisor_runs()
1973 .await
1974 .into_iter()
1975 .filter(|run| run.parent_run.is_none())
1976 .collect()
1977 }
1978
1979 pub async fn list_child_supervisor_runs(&self, parent_run_id: &str) -> Vec<SupervisorRun> {
1981 self.list_supervisor_runs()
1982 .await
1983 .into_iter()
1984 .filter(|run| {
1985 run.parent_run
1986 .as_ref()
1987 .is_some_and(|parent| parent.parent_run_id == parent_run_id)
1988 })
1989 .collect()
1990 }
1991
1992 pub async fn get_supervisor_run_ancestry(&self, run_id: &str) -> Vec<SupervisorRun> {
1994 let runs = self.list_supervisor_runs().await;
1995 let index = runs
1996 .iter()
1997 .cloned()
1998 .map(|run| (run.id.clone(), run))
1999 .collect::<HashMap<_, _>>();
2000 let mut ancestors = Vec::new();
2001 let mut current_parent = index.get(run_id).and_then(|run| {
2002 run.parent_run
2003 .as_ref()
2004 .map(|parent| parent.parent_run_id.clone())
2005 });
2006 while let Some(parent_id) = current_parent {
2007 let Some(parent) = index.get(&parent_id).cloned() else {
2008 break;
2009 };
2010 current_parent = parent
2011 .parent_run
2012 .as_ref()
2013 .map(|parent| parent.parent_run_id.clone());
2014 ancestors.push(parent);
2015 }
2016 ancestors.reverse();
2017 ancestors
2018 }
2019
2020 pub async fn get_supervisor_run_descendants(&self, run_id: &str) -> Vec<SupervisorRun> {
2022 self.list_supervisor_runs()
2023 .await
2024 .into_iter()
2025 .filter(|run| {
2026 run.parent_run
2027 .as_ref()
2028 .is_some_and(|parent| parent.parent_run_id == run_id)
2029 })
2030 .collect()
2031 }
2032
2033 pub async fn list_supervisor_leaf_tasks(&self, run_id: &str) -> Vec<SupervisorTaskRecord> {
2035 let runs = self.list_supervisor_runs().await;
2036 let mut leaf_tasks = Vec::new();
2037 if let Some(run) = runs.iter().find(|run| run.id == run_id) {
2038 leaf_tasks.extend(run.tasks.clone());
2039 }
2040 for child in runs.iter().filter(|run| {
2041 run.parent_run
2042 .as_ref()
2043 .is_some_and(|parent| parent.parent_run_id == run_id)
2044 }) {
2045 leaf_tasks.extend(child.tasks.clone());
2046 }
2047 leaf_tasks
2048 }
2049
2050 pub async fn get_supervisor_run(&self, run_id: &str) -> Option<SupervisorRun> {
2052 let mut in_memory_runs = self
2053 .supervisor_runs
2054 .lock()
2055 .await
2056 .values()
2057 .cloned()
2058 .collect::<Vec<_>>();
2059 if !in_memory_runs.is_empty() {
2060 let checkpoints = load_latest_checkpoints_by_task(checkpoint_roots_for_runs(
2061 &in_memory_runs,
2062 self.default_workspace_dir.as_deref(),
2063 ));
2064 attach_checkpoint_summaries(&mut in_memory_runs, checkpoints);
2065 synchronize_run_hierarchy_snapshots(&mut in_memory_runs);
2066 if let Some(run) = in_memory_runs.into_iter().find(|run| run.id == run_id) {
2067 return Some(run);
2068 }
2069 }
2070
2071 self.default_workspace_dir.as_deref().and_then(|root| {
2072 let mut runs = load_persisted_runs(root);
2073 let checkpoints = load_latest_checkpoints_by_task(checkpoint_roots_for_runs(
2074 &runs,
2075 self.default_workspace_dir.as_deref(),
2076 ));
2077 attach_checkpoint_summaries(&mut runs, checkpoints);
2078 synchronize_run_hierarchy_snapshots(&mut runs);
2079 runs.into_iter().find(|run| run.id == run_id)
2080 })
2081 }
2082
2083 pub async fn list_team_messages(&self, run_id: &str) -> Vec<TeamMessage> {
2085 self.get_supervisor_run(run_id)
2086 .await
2087 .map(|run| {
2088 let mut messages = run.messages;
2089 for task in run.tasks {
2090 messages.extend(task.messages);
2091 }
2092 messages.sort_by(|left, right| left.created_at.cmp(&right.created_at));
2093 messages
2094 })
2095 .unwrap_or_default()
2096 }
2097
2098 pub async fn list_team_threads(&self, run_id: &str) -> Vec<TeamThread> {
2100 self.list_team_threads_with_options(run_id, false).await
2101 }
2102
2103 pub async fn list_team_threads_with_options(
2105 &self,
2106 run_id: &str,
2107 include_archived: bool,
2108 ) -> Vec<TeamThread> {
2109 build_team_threads_with_options(&self.list_team_messages(run_id).await, include_archived)
2110 }
2111
2112 pub async fn list_subagents(&self) -> Vec<AgentInfo> {
2114 self.agent_manager.list_agents().await
2115 }
2116
2117 pub async fn approve_task(
2119 &self,
2120 task_id: &str,
2121 actor: ApprovalActor,
2122 note: Option<String>,
2123 ) -> Result<(), String> {
2124 let run_id = self
2125 .task_run_index
2126 .lock()
2127 .await
2128 .get(task_id)
2129 .cloned()
2130 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2131
2132 let mut queued_to_start = None;
2133 let mut completion_environment_id = None;
2134 let (mut run_snapshot, collaboration_messages, reflection_sync) = {
2135 let mut runs = self.supervisor_runs.lock().await;
2136 let run = runs
2137 .get_mut(&run_id)
2138 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2139 let dependency_states = run
2140 .tasks
2141 .iter()
2142 .map(|record| (record.task.id.clone(), record.state))
2143 .collect::<HashMap<_, _>>();
2144 let record = run
2145 .tasks
2146 .iter_mut()
2147 .find(|record| record.task.id == task_id)
2148 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
2149
2150 let scope = approval_scope_for_state(record.state).ok_or_else(|| {
2151 format!(
2152 "Task '{}' is not waiting for approval or gate completion",
2153 task_id
2154 )
2155 })?;
2156
2157 let decision = record.approval.record_decision(
2158 scope,
2159 ApprovalDecisionKind::Approved,
2160 actor,
2161 note,
2162 )?;
2163 let thread_context = resolve_open_gate_request(
2164 record,
2165 scope,
2166 &decision.actor.id,
2167 CollaborationActionStatus::Resolved,
2168 decision.note.clone(),
2169 );
2170 let mut approval_message = TeamMessage::new(
2171 run.id.clone(),
2172 Some(record.task.id.clone()),
2173 TeamMessageKind::ApprovalDecision,
2174 Some(decision.actor.id.clone()),
2175 Some(record.task.agent_id.clone()),
2176 format_approval_decision_message(&decision),
2177 );
2178 if let Some((thread_id, reply_to_message_id)) = thread_context {
2179 approval_message =
2180 approval_message.with_thread(thread_id, Some(reply_to_message_id));
2181 }
2182 if let Some(result) = record.result.as_ref() {
2183 approval_message = approval_message
2184 .with_result_reference(TeamResultReference::from_task_result(result));
2185 approval_message = approval_message.with_artifact_references(
2186 result
2187 .artifacts
2188 .iter()
2189 .map(|artifact| {
2190 TeamArtifactReference::from_task_artifact(
2191 Some(record.task.id.clone()),
2192 artifact,
2193 )
2194 })
2195 .collect(),
2196 );
2197 }
2198 let mut collaboration_messages = vec![approval_message.clone()];
2199
2200 record.updated_at = Utc::now();
2201
2202 match record.state {
2203 SupervisorTaskState::PendingApproval => {
2204 let blocked = dependency_reasons_from_states(&dependency_states, &record.task);
2205 if blocked.is_empty() {
2206 record.state = SupervisorTaskState::Queued;
2207 record.blocked_reasons.clear();
2208 queued_to_start = Some(record.task.clone());
2209 } else {
2210 record.state = SupervisorTaskState::Blocked;
2211 record.blocked_reasons = blocked;
2212 }
2213 }
2214 SupervisorTaskState::ReviewPending => {
2215 if record.task.test_required {
2216 record.state = SupervisorTaskState::TestPending;
2217 record.approval.request(
2218 ApprovalScope::TestValidation,
2219 ApprovalActor::system("orchestrator"),
2220 Some("Review approved. Awaiting explicit test validation.".to_string()),
2221 );
2222 let message = build_gate_request_message(
2223 &run.id,
2224 record,
2225 ApprovalScope::TestValidation,
2226 );
2227 record.messages.push(message.clone());
2228 collaboration_messages.push(message);
2229 } else {
2230 record.state = SupervisorTaskState::Completed;
2231 record.completed_at = Some(Utc::now());
2232 completion_environment_id = Some(record.environment_id.clone());
2233 }
2234 }
2235 SupervisorTaskState::TestPending => {
2236 record.state = SupervisorTaskState::Completed;
2237 record.completed_at = Some(Utc::now());
2238 completion_environment_id = Some(record.environment_id.clone());
2239 }
2240 _ => {
2241 return Err(format!(
2242 "Task '{}' is not waiting for approval or gate completion",
2243 task_id
2244 ));
2245 }
2246 }
2247
2248 let reflection_sync = {
2249 record.messages.push(approval_message.clone());
2250 task_reflection_sync_context(&record.task).map(
2251 |(workspace_dir, session_id, tracking_task_id)| {
2252 (
2253 workspace_dir,
2254 session_id,
2255 tracking_task_id,
2256 task_record_outcome_signals(record),
2257 )
2258 },
2259 )
2260 };
2261
2262 run.messages.push(approval_message.clone());
2263 run.updated_at = Utc::now();
2264 run.status = recalculate_run_status(run);
2265 (run.clone(), collaboration_messages, reflection_sync)
2266 };
2267
2268 if let Some(environment_id) = completion_environment_id
2269 && let Some(environment) = self
2270 .finalize_environment_for_task(&environment_id, true, false)
2271 .await?
2272 {
2273 self.update_environment_in_runs(&environment).await?;
2274 if let Some(run) = self.supervisor_runs.lock().await.get(&run_id).cloned() {
2275 run_snapshot = run;
2276 }
2277 }
2278
2279 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2280 for message in collaboration_messages {
2281 self.notify_team_message(message).await;
2282 }
2283 if let Some((workspace_dir, session_id, tracking_task_id, signals)) = reflection_sync {
2284 let _ = sync_task_reflection_outcomes(
2285 &workspace_dir,
2286 &session_id,
2287 &tracking_task_id,
2288 &signals,
2289 )
2290 .await;
2291 }
2292
2293 if let Some(task) = queued_to_start {
2294 self.start_task_execution(task).await?;
2295 }
2296
2297 Ok(())
2298 }
2299
2300 pub async fn reject_task(
2302 &self,
2303 task_id: &str,
2304 actor: ApprovalActor,
2305 note: Option<String>,
2306 ) -> Result<(), String> {
2307 let run_id = self
2308 .task_run_index
2309 .lock()
2310 .await
2311 .get(task_id)
2312 .cloned()
2313 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2314
2315 let (mut run_snapshot, approval_message, reflection_sync) = {
2316 let mut runs = self.supervisor_runs.lock().await;
2317 let run = runs
2318 .get_mut(&run_id)
2319 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2320 let record = run
2321 .tasks
2322 .iter_mut()
2323 .find(|record| record.task.id == task_id)
2324 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
2325
2326 let scope = approval_scope_for_state(record.state).ok_or_else(|| {
2327 format!(
2328 "Task '{}' is not waiting for approval or gate completion",
2329 task_id
2330 )
2331 })?;
2332 let decision = record.approval.record_decision(
2333 scope,
2334 ApprovalDecisionKind::NeedsRevision,
2335 actor,
2336 note,
2337 )?;
2338 let thread_context = resolve_open_gate_request(
2339 record,
2340 scope,
2341 &decision.actor.id,
2342 CollaborationActionStatus::NeedsRevision,
2343 decision.note.clone(),
2344 );
2345 let mut approval_message = TeamMessage::new(
2346 run.id.clone(),
2347 Some(record.task.id.clone()),
2348 TeamMessageKind::ApprovalDecision,
2349 Some(decision.actor.id.clone()),
2350 Some(record.task.agent_id.clone()),
2351 format_approval_decision_message(&decision),
2352 );
2353 if let Some((thread_id, reply_to_message_id)) = thread_context {
2354 approval_message =
2355 approval_message.with_thread(thread_id, Some(reply_to_message_id));
2356 }
2357 if let Some(result) = record.result.as_ref() {
2358 approval_message = approval_message
2359 .with_result_reference(TeamResultReference::from_task_result(result));
2360 approval_message = approval_message.with_artifact_references(
2361 result
2362 .artifacts
2363 .iter()
2364 .map(|artifact| {
2365 TeamArtifactReference::from_task_artifact(
2366 Some(record.task.id.clone()),
2367 artifact,
2368 )
2369 })
2370 .collect(),
2371 );
2372 }
2373 record.state = SupervisorTaskState::Failed;
2374 record.updated_at = Utc::now();
2375 let reflection_sync = {
2376 record.messages.push(approval_message.clone());
2377 task_reflection_sync_context(&record.task).map(
2378 |(workspace_dir, session_id, tracking_task_id)| {
2379 (
2380 workspace_dir,
2381 session_id,
2382 tracking_task_id,
2383 task_record_outcome_signals(record),
2384 )
2385 },
2386 )
2387 };
2388 run.messages.push(approval_message.clone());
2389 run.updated_at = Utc::now();
2390 run.status = recalculate_run_status(run);
2391 (run.clone(), Some(approval_message), reflection_sync)
2392 };
2393
2394 if let Some(environment_id) = run_snapshot
2395 .tasks
2396 .iter()
2397 .find(|record| record.task.id == task_id)
2398 .map(|record| record.environment_id.clone())
2399 && let Some(environment) = self
2400 .finalize_environment_for_task(&environment_id, false, true)
2401 .await?
2402 {
2403 self.update_environment_in_runs(&environment).await?;
2404 if let Some(run) = self.supervisor_runs.lock().await.get(&run_id).cloned() {
2405 run_snapshot = run;
2406 }
2407 }
2408
2409 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2410 if let Some(message) = approval_message {
2411 self.notify_team_message(message).await;
2412 }
2413 if let Some((workspace_dir, session_id, tracking_task_id, signals)) = reflection_sync {
2414 let _ = sync_task_reflection_outcomes(
2415 &workspace_dir,
2416 &session_id,
2417 &tracking_task_id,
2418 &signals,
2419 )
2420 .await;
2421 }
2422 Ok(())
2423 }
2424
2425 pub async fn retry_task(&self, task_id: &str) -> Result<(), String> {
2427 let run_id = self
2428 .task_run_index
2429 .lock()
2430 .await
2431 .get(task_id)
2432 .cloned()
2433 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2434
2435 let mut queued_to_start = None;
2436 let (mut run_snapshot, environment_id, retry_message) = {
2437 let mut runs = self.supervisor_runs.lock().await;
2438 let run = runs
2439 .get_mut(&run_id)
2440 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2441 let dependency_states = run
2442 .tasks
2443 .iter()
2444 .map(|record| (record.task.id.clone(), record.state))
2445 .collect::<HashMap<_, _>>();
2446 let record = run
2447 .tasks
2448 .iter_mut()
2449 .find(|record| record.task.id == task_id)
2450 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
2451
2452 if !matches!(
2453 record.state,
2454 SupervisorTaskState::Failed
2455 | SupervisorTaskState::Cancelled
2456 | SupervisorTaskState::Blocked
2457 ) {
2458 return Err(format!(
2459 "Task '{}' cannot be retried from its current state",
2460 task_id
2461 ));
2462 }
2463
2464 record.attempts += 1;
2465 record.result = None;
2466 record.remote_execution = None;
2467 record.local_execution = None;
2468 record.completed_at = None;
2469 record.started_at = None;
2470 record.updated_at = Utc::now();
2471 record.blocked_reasons.clear();
2472 let environment_id = record.environment_id.clone();
2473 let mut retry_message = None;
2474
2475 if record.task.approval_required {
2476 record.state = SupervisorTaskState::PendingApproval;
2477 record.approval.reset_for_task(&record.task);
2478 record.approval.request(
2479 ApprovalScope::PreExecution,
2480 ApprovalActor::system("orchestrator"),
2481 Some("Task retried. Awaiting explicit pre-execution approval.".to_string()),
2482 );
2483 let message =
2484 build_gate_request_message(&run.id, record, ApprovalScope::PreExecution);
2485 record.messages.push(message.clone());
2486 retry_message = Some(message);
2487 } else {
2488 record.approval.reset_for_task(&record.task);
2489 let blocked = dependency_reasons_from_states(&dependency_states, &record.task);
2490 if blocked.is_empty() {
2491 record.state = SupervisorTaskState::Queued;
2492 record.blocked_reasons.clear();
2493 queued_to_start = Some(record.task.clone());
2494 } else {
2495 record.state = SupervisorTaskState::Blocked;
2496 record.blocked_reasons = blocked;
2497 }
2498 }
2499
2500 run.updated_at = Utc::now();
2501 run.status = recalculate_run_status(run);
2502 (run.clone(), environment_id, retry_message)
2503 };
2504
2505 let environment = self.retry_environment_preparation(&environment_id).await?;
2506 self.update_environment_in_runs(&environment).await?;
2507 if matches!(environment.state, EnvironmentState::Failed) {
2508 let mut runs = self.supervisor_runs.lock().await;
2509 if let Some(run) = runs.get_mut(&run_id)
2510 && let Some(record) = run
2511 .tasks
2512 .iter_mut()
2513 .find(|record| record.task.id == task_id)
2514 {
2515 record.state = SupervisorTaskState::Blocked;
2516 record.environment = environment.summary();
2517 record.blocked_reasons = environment
2518 .failure
2519 .as_ref()
2520 .map(|failure| vec![failure.message.clone()])
2521 .unwrap_or_default();
2522 run.updated_at = Utc::now();
2523 run.status = recalculate_run_status(run);
2524 run_snapshot = run.clone();
2525 queued_to_start = None;
2526 }
2527 } else if let Some(run) = self.supervisor_runs.lock().await.get(&run_id).cloned() {
2528 run_snapshot = run;
2529 }
2530
2531 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2532 if let Some(message) = retry_message {
2533 self.notify_team_message(message).await;
2534 }
2535
2536 if let Some(task) = queued_to_start {
2537 self.start_task_execution(task).await?;
2538 }
2539
2540 Ok(())
2541 }
2542
2543 pub async fn resume_task_from_checkpoint(&self, task_id: &str) -> Result<(), String> {
2545 let run_id = self
2546 .task_run_index
2547 .lock()
2548 .await
2549 .get(task_id)
2550 .cloned()
2551 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2552 let task = self
2553 .supervisor_task_record(task_id)
2554 .await
2555 .map(|record| record.task)
2556 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2557 let checkpoint = self
2558 .load_delegated_checkpoint_async(&task)
2559 .await
2560 .ok_or_else(|| format!("Task '{}' has no delegated checkpoint", task_id))?;
2561
2562 if checkpoint.result_published {
2563 return Err(format!(
2564 "Task '{}' already published a terminal result and cannot be resumed",
2565 task_id
2566 ));
2567 }
2568 if checkpoint.resume_disposition != DelegatedResumeDisposition::ResumeFromCheckpoint
2569 || checkpoint.resume_state.is_none()
2570 {
2571 return Err(format!(
2572 "Task '{}' is not currently resumable from checkpoint",
2573 task_id
2574 ));
2575 }
2576 if self.active_tasks.lock().await.contains_key(task_id) {
2577 return Err(format!("Task '{}' is already active", task_id));
2578 }
2579
2580 let blocked_reason = restart_blocked_reason_for_checkpoint(&checkpoint);
2581 let (run_snapshot, queued_to_start) = {
2582 let mut runs = self.supervisor_runs.lock().await;
2583 let run = runs
2584 .get_mut(&run_id)
2585 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2586 let record = run
2587 .tasks
2588 .iter_mut()
2589 .find(|record| record.task.id == task_id)
2590 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
2591
2592 record
2593 .blocked_reasons
2594 .retain(|reason| reason != &blocked_reason);
2595 if !record.blocked_reasons.is_empty() {
2596 return Err(format!(
2597 "Task '{}' still has unresolved blockers: {}",
2598 task_id,
2599 record.blocked_reasons.join("; ")
2600 ));
2601 }
2602
2603 record.state = SupervisorTaskState::Queued;
2604 record.result = None;
2605 record.local_execution = None;
2606 record.completed_at = None;
2607 record.started_at = None;
2608 record.updated_at = Utc::now();
2609 let queued_to_start = Some(record.task.clone());
2610 run.updated_at = Utc::now();
2611 refresh_run_rollups(run);
2612 (run.clone(), queued_to_start)
2613 };
2614
2615 let mut updated_checkpoint = checkpoint.clone();
2616 updated_checkpoint.stage = DelegatedCheckpointStage::Queued;
2617 updated_checkpoint.note = Some("manual resume requested from checkpoint".to_string());
2618 updated_checkpoint.updated_at = Utc::now();
2619 self.persist_delegated_checkpoint_async(&updated_checkpoint)
2620 .await?;
2621 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2622 if let Some(task) = queued_to_start {
2623 self.schedule_task_execution(task);
2624 }
2625 Ok(())
2626 }
2627
2628 pub async fn restart_task_from_scratch(&self, task_id: &str) -> Result<(), String> {
2630 if let Some(task) = self
2631 .supervisor_task_record(task_id)
2632 .await
2633 .map(|record| record.task)
2634 && let Some(mut checkpoint) = self.load_delegated_checkpoint_async(&task).await
2635 {
2636 checkpoint.stage = DelegatedCheckpointStage::Queued;
2637 checkpoint.resume_state = None;
2638 checkpoint.completed_tool_calls.clear();
2639 checkpoint.resume_disposition = DelegatedResumeDisposition::RestartFromBoundary;
2640 checkpoint.safe_boundary_label = "manual restart from scratch".to_string();
2641 checkpoint.result_published = false;
2642 checkpoint.note = Some("operator cleared checkpoint resume state".to_string());
2643 checkpoint.updated_at = Utc::now();
2644 self.persist_delegated_checkpoint_async(&checkpoint).await?;
2645 }
2646
2647 self.retry_task(task_id).await
2648 }
2649
2650 pub async fn acknowledge_blocked_task(
2652 &self,
2653 task_id: &str,
2654 note: Option<String>,
2655 ) -> Result<(), String> {
2656 let run_id = self
2657 .task_run_index
2658 .lock()
2659 .await
2660 .get(task_id)
2661 .cloned()
2662 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2663 let task = self
2664 .supervisor_task_record(task_id)
2665 .await
2666 .map(|record| record.task)
2667 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2668
2669 let run_snapshot = {
2670 let mut runs = self.supervisor_runs.lock().await;
2671 let run = runs
2672 .get_mut(&run_id)
2673 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2674 let record = run
2675 .tasks
2676 .iter_mut()
2677 .find(|record| record.task.id == task_id)
2678 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
2679
2680 if record.state != SupervisorTaskState::Blocked {
2681 return Err(format!(
2682 "Task '{}' is not blocked and does not require acknowledgement",
2683 task_id
2684 ));
2685 }
2686
2687 record.updated_at = Utc::now();
2688 run.updated_at = Utc::now();
2689 refresh_run_rollups(run);
2690 run.clone()
2691 };
2692
2693 if let Some(mut checkpoint) = self.load_delegated_checkpoint_async(&task).await {
2694 let message = note
2695 .filter(|value| !value.trim().is_empty())
2696 .unwrap_or_else(|| "blocked state acknowledged by operator".to_string());
2697 checkpoint.stage = DelegatedCheckpointStage::Blocked;
2698 checkpoint.note = Some(message);
2699 checkpoint.updated_at = Utc::now();
2700 self.persist_delegated_checkpoint_async(&checkpoint).await?;
2701 }
2702
2703 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2704 Ok(())
2705 }
2706
2707 pub async fn claim_task(&self, task_id: &str, agent_id: &str) -> Result<(), String> {
2709 let run_id = self
2710 .task_run_index
2711 .lock()
2712 .await
2713 .get(task_id)
2714 .cloned()
2715 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
2716
2717 let run_snapshot = {
2718 let mut runs = self.supervisor_runs.lock().await;
2719 let run = runs
2720 .get_mut(&run_id)
2721 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2722 let record = run
2723 .tasks
2724 .iter_mut()
2725 .find(|record| record.task.id == task_id)
2726 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
2727
2728 record.claimed_by = Some(agent_id.to_string());
2729 record.task.agent_id = agent_id.to_string();
2730 record.updated_at = Utc::now();
2731 run.updated_at = Utc::now();
2732 run.status = recalculate_run_status(run);
2733 run.clone()
2734 };
2735
2736 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2737 Ok(())
2738 }
2739
2740 async fn supervisor_task_record(&self, task_id: &str) -> Option<SupervisorTaskRecord> {
2741 let run_id = self.task_run_index.lock().await.get(task_id).cloned()?;
2742 let runs = self.supervisor_runs.lock().await;
2743 runs.get(&run_id).and_then(|run| {
2744 run.tasks
2745 .iter()
2746 .find(|record| record.task.id == task_id)
2747 .cloned()
2748 })
2749 }
2750
2751 async fn remote_execution_for_task(&self, task_id: &str) -> Option<RemoteExecutionRecord> {
2752 self.supervisor_task_record(task_id)
2753 .await
2754 .and_then(|record| record.remote_execution)
2755 }
2756
2757 async fn sync_remote_task_snapshot(
2758 &self,
2759 task: &DelegatedTask,
2760 remote_target: &RemoteAgentTarget,
2761 remote_task: &A2ATask,
2762 manifest: &[ArtifactManifestEntry],
2763 compatibility: RemoteExecutionCompatibility,
2764 ) -> Result<(), String> {
2765 let run_id = task
2766 .run_id
2767 .clone()
2768 .ok_or_else(|| format!("Task '{}' missing run_id", task.id))?;
2769 let run_snapshot = {
2770 let mut runs = self.supervisor_runs.lock().await;
2771 let run = runs
2772 .get_mut(&run_id)
2773 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2774 let record = run
2775 .tasks
2776 .iter_mut()
2777 .find(|record| record.task.id == task.id)
2778 .ok_or_else(|| format!("Task '{}' not found in run", task.id))?;
2779 record.remote_execution = Some(RemoteExecutionRecord {
2780 target: remote_target.clone(),
2781 remote_task_id: remote_task.id.clone(),
2782 status: a2a_status_label(remote_task.status),
2783 status_reason: remote_task.status_reason.clone(),
2784 lease: remote_task.lease.clone(),
2785 progress: progress_from_remote(remote_task.progress.as_ref()),
2786 artifacts: artifacts_from_manifest(manifest),
2787 provenance: remote_task
2788 .provenance
2789 .clone()
2790 .or_else(|| provenance_from_metadata(&remote_task.metadata)),
2791 compatibility,
2792 last_synced_at: Utc::now(),
2793 });
2794 if matches!(remote_task.status, A2ATaskStatus::Blocked) {
2795 record.state = SupervisorTaskState::Blocked;
2796 record.blocked_reasons = vec![
2797 remote_task
2798 .status_reason
2799 .clone()
2800 .unwrap_or_else(|| "Remote execution blocked".to_string()),
2801 ];
2802 }
2803 record.updated_at = Utc::now();
2804 run.updated_at = Utc::now();
2805 run.status = recalculate_run_status(run);
2806 run.clone()
2807 };
2808 self.persist_run_with_hierarchy_sync(run_snapshot.clone())
2809 .await?;
2810 let observer = { self.observer.read().await.clone() };
2811 if let Some(observer) = observer.as_ref() {
2812 observer.on_run_updated(run_snapshot).await;
2813 }
2814 Ok(())
2815 }
2816
2817 async fn sync_local_execution_progress(
2818 &self,
2819 task: &DelegatedTask,
2820 mut progress: LocalExecutionProgress,
2821 ) -> Result<(), String> {
2822 let run_id = task
2823 .run_id
2824 .clone()
2825 .ok_or_else(|| format!("Task '{}' missing run_id", task.id))?;
2826 let (run_snapshot, task_snapshot) = {
2827 let mut runs = self.supervisor_runs.lock().await;
2828 let run = runs
2829 .get_mut(&run_id)
2830 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2831 let task_index = run
2832 .tasks
2833 .iter()
2834 .position(|record| record.task.id == task.id)
2835 .ok_or_else(|| format!("Task '{}' not found in run", task.id))?;
2836 let now = Utc::now();
2837 {
2838 let record = &mut run.tasks[task_index];
2839 progress.environment =
2840 Some(environment_snapshot_from_execution(&record.environment));
2841 let local_execution = record
2842 .local_execution
2843 .get_or_insert_with(local_execution_record_for_start);
2844 local_execution.status = local_execution_status_label(record.state).to_string();
2845 local_execution.status_reason = None;
2846 local_execution.progress = Some(progress);
2847 local_execution.last_synced_at = now;
2848 record.updated_at = now;
2849 }
2850 run.updated_at = now;
2851 run.status = recalculate_run_status(run);
2852 (run.clone(), run.tasks[task_index].clone())
2853 };
2854
2855 record_task_progress(task, &task_snapshot, &run_snapshot);
2856 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2857 Ok(())
2858 }
2859
2860 pub async fn send_team_message(
2862 &self,
2863 run_id: &str,
2864 task_id: Option<String>,
2865 kind: TeamMessageKind,
2866 sender_agent_id: Option<String>,
2867 recipient_agent_id: Option<String>,
2868 content: impl Into<String>,
2869 ) -> Result<TeamMessage, String> {
2870 self.send_team_message_draft(
2871 run_id,
2872 TeamMessageDraft {
2873 task_id,
2874 kind,
2875 sender_agent_id,
2876 recipient_agent_id,
2877 content: content.into(),
2878 thread_id: None,
2879 reply_to_message_id: None,
2880 action_request: None,
2881 escalation: None,
2882 unread_by_agent_ids: Vec::new(),
2883 },
2884 )
2885 .await
2886 }
2887
2888 pub async fn send_team_message_draft(
2890 &self,
2891 run_id: &str,
2892 draft: TeamMessageDraft,
2893 ) -> Result<TeamMessage, String> {
2894 let message = draft.into_message(run_id.to_string());
2895 let thread_id = message.effective_thread_id().to_string();
2896
2897 let run_snapshot = {
2898 let mut runs = self.supervisor_runs.lock().await;
2899 let run = runs
2900 .get_mut(run_id)
2901 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
2902 insert_team_message(run, message.clone())?;
2903 apply_collaboration_retention(run);
2904 run.clone()
2905 };
2906
2907 build_team_threads_with_options(&collect_team_messages(&run_snapshot), true)
2908 .into_iter()
2909 .find(|thread| thread.id == thread_id)
2910 .ok_or_else(|| format!("Thread '{}' not found after recording message", thread_id))?;
2911
2912 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
2913 self.capture_shared_cognition_from_message(run_id, &message)
2914 .await;
2915 self.notify_team_message(message.clone()).await;
2916 Ok(message)
2917 }
2918
2919 async fn capture_shared_cognition_from_message(&self, run_id: &str, message: &TeamMessage) {
2920 let run_snapshot = {
2921 let runs = self.supervisor_runs.lock().await;
2922 runs.get(run_id).cloned()
2923 };
2924
2925 let Some(run_snapshot) = run_snapshot else {
2926 return;
2927 };
2928 let Some(kind) = shared_cognition_kind_for_message(&run_snapshot, message) else {
2929 return;
2930 };
2931
2932 let task_record = message.task_id.as_deref().and_then(|task_id| {
2933 run_snapshot
2934 .tasks
2935 .iter()
2936 .find(|record| record.task.id == task_id)
2937 });
2938 let directive_id = task_record
2939 .and_then(|record| record.task.directive_id.clone())
2940 .or_else(|| common_directive_id_for_run(&run_snapshot));
2941 let mut tags = task_record
2942 .map(|record| record.task.memory_tags.clone())
2943 .unwrap_or_default();
2944 push_unique_tag(&mut tags, SHARED_COGNITION_TAG);
2945 push_unique_tag(&mut tags, workflow_run_memory_tag(run_id));
2946 push_unique_tag(&mut tags, shared_cognition_kind_tag(kind));
2947
2948 let summary = summarize_shared_cognition(&message.content);
2949 let detail = message.content.trim().to_string();
2950 let confidence = shared_cognition_confidence(kind);
2951
2952 let note = SharedCognitionNote {
2953 id: Uuid::new_v4().to_string(),
2954 run_id: run_id.to_string(),
2955 task_id: message.task_id.clone(),
2956 directive_id: directive_id.clone(),
2957 kind,
2958 message_kind: message.kind,
2959 summary: summary.clone(),
2960 detail: detail.clone(),
2961 sender_agent_id: message.sender_agent_id.clone(),
2962 recipient_agent_id: message.recipient_agent_id.clone(),
2963 tags: tags.clone(),
2964 confidence,
2965 source_message_id: message.id.clone(),
2966 created_at: message.created_at,
2967 };
2968
2969 let memory_path = if let Some(workspace_dir) = run_snapshot.workspace_dir.as_deref() {
2970 let scope = if note.task_id.is_some() {
2971 MemoryScope::Task
2972 } else if note.directive_id.is_some() {
2973 MemoryScope::Directive
2974 } else if run_snapshot.session_id.is_some() {
2975 MemoryScope::Session
2976 } else {
2977 MemoryScope::Workspace
2978 };
2979
2980 let mut content_lines = vec![
2981 format!(
2982 "Shared cognition ({:?}) for workflow run {}",
2983 note.kind, run_snapshot.id
2984 ),
2985 format!("Source message kind: {:?}", note.message_kind),
2986 ];
2987 if let Some(task_id) = note.task_id.as_deref() {
2988 content_lines.push(format!("Task: {task_id}"));
2989 }
2990 if let Some(directive_id) = note.directive_id.as_deref() {
2991 content_lines.push(format!("Directive: {directive_id}"));
2992 }
2993 if let Some(sender) = note.sender_agent_id.as_deref() {
2994 content_lines.push(format!("Sender: {sender}"));
2995 }
2996 if let Some(recipient) = note.recipient_agent_id.as_deref() {
2997 content_lines.push(format!("Recipient: {recipient}"));
2998 }
2999 content_lines.push(String::new());
3000 content_lines.push(detail.clone());
3001
3002 let mut entry = MemoryBankEntry::new(
3003 run_snapshot
3004 .session_id
3005 .clone()
3006 .unwrap_or_else(|| format!("workflow-run-{}", run_snapshot.id)),
3007 summary,
3008 content_lines.join("\n"),
3009 )
3010 .with_memory_type(shared_cognition_memory_type(kind))
3011 .with_scope(scope)
3012 .with_confidence(confidence)
3013 .with_provenance(
3014 note.task_id.clone(),
3015 note.directive_id.clone(),
3016 note.sender_agent_id.clone(),
3017 )
3018 .with_tags(tags);
3019 entry.category = Some(SHARED_COGNITION_CATEGORY.to_string());
3020
3021 match crate::save_to_memory_bank(workspace_dir, &entry).await {
3022 Ok(path) => Some(path),
3023 Err(error) => {
3024 tracing::warn!(
3025 run_id = %run_snapshot.id,
3026 message_id = %message.id,
3027 error = %error,
3028 "Failed to persist shared cognition memory"
3029 );
3030 None
3031 }
3032 }
3033 } else {
3034 None
3035 };
3036
3037 if let Some(task_record) = task_record
3038 && let (Some(session_id), Some(tracking_task_id)) = (
3039 task_record.task.session_id.as_deref(),
3040 task_record.task.tracking_task_id.as_deref(),
3041 )
3042 {
3043 let phase = match kind {
3044 SharedCognitionKind::Blocker => crate::tasks::TaskMemoryPhase::Blocked,
3045 SharedCognitionKind::Handoff => crate::tasks::TaskMemoryPhase::Handoff,
3046 SharedCognitionKind::Discovery
3047 | SharedCognitionKind::Hypothesis
3048 | SharedCognitionKind::Steering
3049 | SharedCognitionKind::Decision => crate::tasks::TaskMemoryPhase::Promoted,
3050 };
3051 let manager = crate::get_global_task_manager();
3052 let _ = manager.record_memory_event(
3053 session_id,
3054 tracking_task_id,
3055 crate::tasks::TaskMemoryEvent::new(
3056 phase,
3057 format!("Shared cognition: {}", note.summary),
3058 Some(format!("{:?}", kind).to_lowercase()),
3059 Some(format!("{:?}", shared_cognition_memory_type(kind)).to_lowercase()),
3060 memory_path.as_ref().map(|path| path.display().to_string()),
3061 ),
3062 );
3063 }
3064
3065 let updated_run = {
3066 let mut runs = self.supervisor_runs.lock().await;
3067 let Some(run) = runs.get_mut(run_id) else {
3068 return;
3069 };
3070 if run
3071 .shared_cognition
3072 .iter()
3073 .any(|existing| existing.source_message_id == note.source_message_id)
3074 {
3075 return;
3076 }
3077 run.shared_cognition.push(note);
3078 run.updated_at = Utc::now();
3079 run.clone()
3080 };
3081
3082 if let Err(error) = self.persist_run_with_hierarchy_sync(updated_run).await {
3083 tracing::warn!(
3084 run_id = %run_id,
3085 message_id = %message.id,
3086 error = %error,
3087 "Failed to persist shared cognition note on supervisor run"
3088 );
3089 }
3090 }
3091
3092 pub async fn update_team_thread_action(
3094 &self,
3095 run_id: &str,
3096 thread_id: &str,
3097 status: CollaborationActionStatus,
3098 actor_id: Option<String>,
3099 note: Option<String>,
3100 ) -> Result<TeamThread, String> {
3101 let (run_snapshot, thread, action_reply) = {
3102 let mut runs = self.supervisor_runs.lock().await;
3103 let run = runs
3104 .get_mut(run_id)
3105 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
3106
3107 let (task_id, reply_to_message_id) = resolve_team_thread_action_request(
3108 run,
3109 thread_id,
3110 status,
3111 actor_id.clone(),
3112 note.clone(),
3113 )?;
3114
3115 let reply_kind = match status {
3116 CollaborationActionStatus::NeedsRevision => TeamMessageKind::ReviewFeedback,
3117 _ => TeamMessageKind::StatusUpdate,
3118 };
3119 let reply_content = collaboration_status_message(status, note.clone());
3120 let action_reply = TeamMessage::new(
3121 run_id.to_string(),
3122 task_id,
3123 reply_kind,
3124 actor_id.clone(),
3125 None,
3126 reply_content,
3127 )
3128 .with_thread(thread_id.to_string(), Some(reply_to_message_id));
3129 insert_team_message(run, action_reply.clone())?;
3130 apply_collaboration_retention(run);
3131 let run_snapshot = run.clone();
3132 let thread =
3133 build_team_threads_with_options(&collect_team_messages(&run_snapshot), true)
3134 .into_iter()
3135 .find(|thread| thread.id == thread_id)
3136 .ok_or_else(|| format!("Thread '{}' not found", thread_id))?;
3137 (run_snapshot, thread, action_reply)
3138 };
3139
3140 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
3141 self.notify_team_message(action_reply).await;
3142 Ok(thread)
3143 }
3144
3145 pub async fn archive_team_thread(
3147 &self,
3148 run_id: &str,
3149 thread_id: &str,
3150 actor_id: Option<String>,
3151 note: Option<String>,
3152 ) -> Result<TeamThread, String> {
3153 let (run_snapshot, thread) = {
3154 let mut runs = self.supervisor_runs.lock().await;
3155 let run = runs
3156 .get_mut(run_id)
3157 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
3158
3159 archive_team_thread_messages(run, thread_id, actor_id, note)?;
3160 apply_collaboration_retention(run);
3161 let run_snapshot = run.clone();
3162 let thread =
3163 build_team_threads_with_options(&collect_team_messages(&run_snapshot), true)
3164 .into_iter()
3165 .find(|thread| thread.id == thread_id)
3166 .ok_or_else(|| format!("Thread '{}' not found", thread_id))?;
3167 (run_snapshot, thread)
3168 };
3169
3170 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
3171 self.notify_team_thread_updated(thread.clone()).await;
3172 Ok(thread)
3173 }
3174
3175 pub async fn cancel_task(&self, task_id: &str) -> Result<(), String> {
3177 let task_control = { self.active_tasks.lock().await.get(task_id).cloned() };
3178
3179 if let Some(task_control) = task_control {
3180 let task = task_control.task.clone();
3181 tracing::info!(task_id = %task_id, agent_id = %task.agent_id, "Cancelling task");
3182 if matches!(task.execution_mode, AgentExecutionMode::Remote) {
3183 self.active_tasks.lock().await.remove(task_id);
3184 if let (Some(remote_target), Some(remote_execution)) = (
3185 task.remote_target.clone(),
3186 self.remote_execution_for_task(task_id).await,
3187 ) {
3188 let client = match remote_target.auth_token {
3189 Some(token) => A2AClient::with_auth(token),
3190 None => A2AClient::new(),
3191 };
3192 if let Err(error) = client
3193 .cancel_task(&remote_target.url, &remote_execution.remote_task_id)
3194 .await
3195 {
3196 tracing::warn!(task_id = %task_id, error = %error, "Failed to cancel remote A2A task");
3197 }
3198 }
3199 } else if let Some(cancel_token) = task_control.local_cancel_token {
3200 cancel_token.cancel();
3201 self.agent_manager
3202 .send_event(&task.agent_id, format!("cancel:{}", task_id))
3203 .await;
3204 } else {
3205 return Err(format!(
3206 "Task '{}' is active locally but missing a cancellation handle",
3207 task_id
3208 ));
3209 }
3210
3211 if let Some(run_id) = self.task_run_index.lock().await.get(task_id).cloned() {
3212 let mut run_snapshot = {
3213 let mut runs = self.supervisor_runs.lock().await;
3214 let run = runs
3215 .get_mut(&run_id)
3216 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
3217 if let Some(record) = run
3218 .tasks
3219 .iter_mut()
3220 .find(|record| record.task.id == task_id)
3221 {
3222 record.state = SupervisorTaskState::Cancelled;
3223 record.completed_at = Some(Utc::now());
3224 record.updated_at = Utc::now();
3225 }
3226 run.updated_at = Utc::now();
3227 run.status = recalculate_run_status(run);
3228 run.clone()
3229 };
3230 if let Some(environment_id) = run_snapshot
3231 .tasks
3232 .iter()
3233 .find(|record| record.task.id == task_id)
3234 .map(|record| record.environment_id.clone())
3235 && let Some(environment) = self
3236 .finalize_environment_for_task(&environment_id, false, true)
3237 .await?
3238 {
3239 self.update_environment_in_runs(&environment).await?;
3240 if let Some(run) = self.supervisor_runs.lock().await.get(&run_id).cloned() {
3241 run_snapshot = run;
3242 }
3243 }
3244 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
3245 }
3246 Ok(())
3247 } else {
3248 Err(format!("Task '{}' not found", task_id))
3249 }
3250 }
3251
3252 pub async fn pause_task(&self, task_id: &str) -> Result<(), String> {
3254 let task_control = self
3255 .active_tasks
3256 .lock()
3257 .await
3258 .get(task_id)
3259 .cloned()
3260 .ok_or_else(|| format!("Task '{}' not found", task_id))?;
3261
3262 if matches!(task_control.task.execution_mode, AgentExecutionMode::Remote) {
3263 return Err(format!(
3264 "Task '{}' is running remotely and cannot be paused locally",
3265 task_id
3266 ));
3267 }
3268
3269 let cancel_token = task_control.local_cancel_token.ok_or_else(|| {
3270 format!(
3271 "Task '{}' is active locally but missing a pause control handle",
3272 task_id
3273 )
3274 })?;
3275
3276 tracing::info!(
3277 task_id = %task_id,
3278 agent_id = %task_control.task.agent_id,
3279 "Pausing local delegated task"
3280 );
3281 cancel_token.pause();
3282 self.agent_manager
3283 .send_event(&task_control.task.agent_id, format!("pause:{}", task_id))
3284 .await;
3285 Ok(())
3286 }
3287
3288 pub async fn shutdown_all(&self, grace_secs: u64) {
3290 tracing::info!(grace_secs = grace_secs, "Shutting down all subagents");
3291 self.agent_manager.shutdown_all(grace_secs).await;
3292 }
3293
3294 async fn ensure_agent_exists(&self, task: &DelegatedTask) -> Result<(), String> {
3295 if self
3296 .agent_manager
3297 .get_agent_status(&task.agent_id)
3298 .await
3299 .is_none()
3300 {
3301 let role = task.role.clone().unwrap_or_default();
3302 let mut request = AgentSpawnRequest::new(
3303 task.agent_id.clone(),
3304 format!("{}-{}", role.label().replace(' ', "-"), task.agent_id),
3305 role.clone(),
3306 );
3307 request.workspace_dir = task.workspace_dir.clone();
3308 request.execution_mode = task.execution_mode.clone();
3309 self.spawn_subagent_with_request(request).await?;
3310 }
3311 Ok(())
3312 }
3313
3314 async fn ensure_tracking_task(&self, task: &mut DelegatedTask) {
3315 let Some(session_id) = task.session_id.as_deref() else {
3316 return;
3317 };
3318 if task.tracking_task_id.is_some() {
3319 return;
3320 }
3321
3322 let manager = crate::get_global_task_manager();
3323 let name = task
3324 .name
3325 .clone()
3326 .unwrap_or_else(|| format!("Delegated: {}", task.agent_id));
3327 let description = task
3328 .delegation_brief
3329 .as_ref()
3330 .map(|brief| brief.objective.clone())
3331 .unwrap_or_else(|| task.prompt.clone());
3332 if let Ok(created) = manager.create_orchestrator_task(
3333 session_id,
3334 task.id.clone(),
3335 task.agent_id.clone(),
3336 name,
3337 description,
3338 task.context.clone(),
3339 ) {
3340 task.tracking_task_id = Some(created.id);
3341 }
3342 }
3343
3344 async fn start_task_execution(&self, task: DelegatedTask) -> Result<(), String> {
3345 let run_id = task
3346 .run_id
3347 .clone()
3348 .ok_or_else(|| "Task is missing run_id".to_string())?;
3349 let environment_id = task
3350 .environment_id
3351 .clone()
3352 .ok_or_else(|| format!("Task '{}' is missing environment_id", task.id))?;
3353 let leased_environment = self
3354 .acquire_environment_lease(&environment_id, &task.id, &task.agent_id)
3355 .await?;
3356
3357 let local_cancel_token = (!matches!(task.execution_mode, AgentExecutionMode::Remote))
3358 .then(CancellationToken::new);
3359
3360 let (run_snapshot, task_snapshot, attempt) = {
3361 let mut active = self.active_tasks.lock().await;
3362 let mut runs = self.supervisor_runs.lock().await;
3363 let run = runs
3364 .get_mut(&run_id)
3365 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
3366 let task_snapshot = {
3367 let record = run
3368 .tasks
3369 .iter_mut()
3370 .find(|record| record.task.id == task.id)
3371 .ok_or_else(|| format!("Task '{}' not found in run", task.id))?;
3372
3373 record.state = SupervisorTaskState::Running;
3374 record.started_at = Some(Utc::now());
3375 record.updated_at = Utc::now();
3376 record.blocked_reasons.clear();
3377 record.environment = leased_environment.summary();
3378 record.local_execution =
3379 if matches!(task.execution_mode, AgentExecutionMode::Remote) {
3380 None
3381 } else {
3382 Some(local_execution_record_for_start())
3383 };
3384 record.clone()
3385 };
3386 let attempt = task_snapshot.attempts;
3387
3388 active.insert(
3389 task.id.clone(),
3390 ActiveTaskControl {
3391 task: task.clone(),
3392 local_cancel_token: local_cancel_token.clone(),
3393 attempt,
3394 },
3395 );
3396
3397 run.updated_at = Utc::now();
3398 run.status = recalculate_run_status(run);
3399
3400 (run.clone(), task_snapshot, attempt)
3401 };
3402
3403 record_task_dispatch(&task, &task_snapshot, &run_snapshot);
3404 self.persist_run_with_hierarchy_sync(run_snapshot.clone())
3405 .await?;
3406 if !matches!(task.execution_mode, AgentExecutionMode::Remote) {
3407 self.persist_delegated_checkpoint_async(&delegated_start_checkpoint(&task))
3408 .await?;
3409 }
3410
3411 let observer_for_start = { self.observer.read().await.clone() };
3412 if let Some(obs) = observer_for_start.as_ref() {
3413 obs.on_task_started(task.clone()).await;
3414 }
3415
3416 let orchestrator = self.clone();
3417 let execution_span = tracing::info_span!(
3418 "delegated_task_execution",
3419 run_id = %run_id,
3420 task_id = %task.id,
3421 agent_id = %task.agent_id,
3422 execution_mode = ?task.execution_mode,
3423 tracking_task_id = %task.tracking_task_id.as_deref().unwrap_or("n/a"),
3424 );
3425 tokio::spawn(
3426 async move {
3427 let start = std::time::Instant::now();
3428 let (task_result, preserve_existing_checkpoint) =
3429 if matches!(task.execution_mode, AgentExecutionMode::Remote) {
3430 (orchestrator.execute_remote_task(&task, start).await, false)
3431 } else {
3432 let execution = execute_delegated_task(
3433 &orchestrator,
3434 &task,
3435 local_cancel_token
3436 .clone()
3437 .expect("local delegated task missing cancellation token"),
3438 )
3439 .await;
3440 let duration_ms = start.elapsed().as_millis() as u64;
3441 let preserve_existing_checkpoint = execution.preserve_existing_checkpoint;
3442 (
3443 execution.into_task_result(&task, duration_ms),
3444 preserve_existing_checkpoint,
3445 )
3446 };
3447
3448 if let Err(error) = orchestrator
3449 .complete_task_execution(
3450 task,
3451 task_result,
3452 preserve_existing_checkpoint,
3453 attempt,
3454 )
3455 .await
3456 {
3457 tracing::error!(error = %error, "Failed to finalize delegated task execution");
3458 }
3459 }
3460 .instrument(execution_span),
3461 );
3462
3463 Ok(())
3464 }
3465
3466 async fn execute_remote_task(
3467 &self,
3468 task: &DelegatedTask,
3469 start: std::time::Instant,
3470 ) -> TaskResult {
3471 let duration_ms = || start.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
3472 let Some(remote_target) = task.remote_target.clone() else {
3473 return TaskResult {
3474 task_id: task.id.clone(),
3475 agent_id: task.agent_id.clone(),
3476 success: false,
3477 run_id: task.run_id.clone(),
3478 tracking_task_id: task.tracking_task_id.clone(),
3479 output: "Task is marked remote without a remote target".to_string(),
3480 summary: task.name.clone(),
3481 tool_calls: vec![],
3482 artifacts: vec![],
3483 terminal_state_hint: Some(TaskTerminalStateHint::Failed),
3484 duration_ms: duration_ms(),
3485 };
3486 };
3487
3488 let client = match remote_target.auth_token.as_ref() {
3489 Some(token) => A2AClient::with_auth(token.clone()),
3490 None => A2AClient::new(),
3491 };
3492 let remote_card = match client.discover(&remote_target.url).await {
3493 Ok(card) => card,
3494 Err(error) => {
3495 return TaskResult {
3496 task_id: task.id.clone(),
3497 agent_id: task.agent_id.clone(),
3498 success: false,
3499 run_id: task.run_id.clone(),
3500 tracking_task_id: task.tracking_task_id.clone(),
3501 output: format!(
3502 "Failed to discover remote agent at {}: {error}",
3503 remote_target.url
3504 ),
3505 summary: task.name.clone(),
3506 tool_calls: vec![],
3507 artifacts: vec![],
3508 terminal_state_hint: Some(TaskTerminalStateHint::Blocked),
3509 duration_ms: duration_ms(),
3510 };
3511 }
3512 };
3513
3514 let compatibility = compatibility_from_card(&remote_card, &remote_target);
3515 if remote_card.authentication.is_some() && remote_target.auth_token.is_none() {
3516 return TaskResult {
3517 task_id: task.id.clone(),
3518 agent_id: task.agent_id.clone(),
3519 success: false,
3520 run_id: task.run_id.clone(),
3521 tracking_task_id: task.tracking_task_id.clone(),
3522 output: format!(
3523 "Remote agent at {} requires authentication, but no auth token is configured",
3524 remote_target.url
3525 ),
3526 summary: task.name.clone(),
3527 tool_calls: vec![],
3528 artifacts: vec![],
3529 terminal_state_hint: Some(TaskTerminalStateHint::Blocked),
3530 duration_ms: duration_ms(),
3531 };
3532 }
3533 let request = match self.supervisor_task_record(&task.id).await {
3534 Some(record) => build_remote_task_request(task, &record, &compatibility),
3535 None => {
3536 return TaskResult {
3537 task_id: task.id.clone(),
3538 agent_id: task.agent_id.clone(),
3539 success: false,
3540 run_id: task.run_id.clone(),
3541 tracking_task_id: task.tracking_task_id.clone(),
3542 output: format!("Missing supervisor record for remote task {}", task.id),
3543 summary: task.name.clone(),
3544 tool_calls: vec![],
3545 artifacts: vec![],
3546 terminal_state_hint: Some(TaskTerminalStateHint::Failed),
3547 duration_ms: duration_ms(),
3548 };
3549 }
3550 };
3551
3552 let remote_task = match client
3553 .create_task_with_request(&remote_target.url, request)
3554 .await
3555 {
3556 Ok(task_state) => task_state,
3557 Err(error) => {
3558 return TaskResult {
3559 task_id: task.id.clone(),
3560 agent_id: task.agent_id.clone(),
3561 success: false,
3562 run_id: task.run_id.clone(),
3563 tracking_task_id: task.tracking_task_id.clone(),
3564 output: format!(
3565 "Failed to create remote task on {}: {error}",
3566 remote_target.url
3567 ),
3568 summary: task.name.clone(),
3569 tool_calls: vec![],
3570 artifacts: vec![],
3571 terminal_state_hint: Some(TaskTerminalStateHint::Blocked),
3572 duration_ms: duration_ms(),
3573 };
3574 }
3575 };
3576
3577 let mut manifest = if remote_card
3578 .supported_rpc_methods
3579 .iter()
3580 .any(|method| method == "task/artifacts")
3581 {
3582 client
3583 .list_task_artifacts(&remote_target.url, &remote_task.id)
3584 .await
3585 .unwrap_or_default()
3586 } else {
3587 Vec::new()
3588 };
3589 if let Err(error) = self
3590 .sync_remote_task_snapshot(
3591 task,
3592 &remote_target,
3593 &remote_task,
3594 &manifest,
3595 compatibility.clone(),
3596 )
3597 .await
3598 {
3599 tracing::warn!(task_id = %task.id, error = %error, "Failed to persist initial remote task snapshot");
3600 }
3601
3602 let mut current = remote_task;
3603 loop {
3604 match current.status {
3605 A2ATaskStatus::Completed => {
3606 return TaskResult {
3607 task_id: task.id.clone(),
3608 agent_id: task.agent_id.clone(),
3609 success: true,
3610 run_id: task.run_id.clone(),
3611 tracking_task_id: task.tracking_task_id.clone(),
3612 output: summarize_remote_task_output(¤t),
3613 summary: task.name.clone(),
3614 tool_calls: vec![],
3615 artifacts: task_artifacts_from_remote_payload(¤t.artifacts),
3616 terminal_state_hint: task_terminal_hint_for_a2a_status(current.status),
3617 duration_ms: duration_ms(),
3618 };
3619 }
3620 A2ATaskStatus::Failed | A2ATaskStatus::Cancelled | A2ATaskStatus::Blocked => {
3621 return TaskResult {
3622 task_id: task.id.clone(),
3623 agent_id: task.agent_id.clone(),
3624 success: false,
3625 run_id: task.run_id.clone(),
3626 tracking_task_id: task.tracking_task_id.clone(),
3627 output: summarize_remote_task_output(¤t),
3628 summary: task.name.clone(),
3629 tool_calls: vec![],
3630 artifacts: task_artifacts_from_remote_payload(¤t.artifacts),
3631 terminal_state_hint: task_terminal_hint_for_a2a_status(current.status),
3632 duration_ms: duration_ms(),
3633 };
3634 }
3635 _ => {}
3636 }
3637
3638 sleep(TokioDuration::from_secs(2)).await;
3639 current = match client
3640 .get_task_status(&remote_target.url, ¤t.id)
3641 .await
3642 {
3643 Ok(task_state) => task_state,
3644 Err(error) => {
3645 return TaskResult {
3646 task_id: task.id.clone(),
3647 agent_id: task.agent_id.clone(),
3648 success: false,
3649 run_id: task.run_id.clone(),
3650 tracking_task_id: task.tracking_task_id.clone(),
3651 output: format!("Failed to refresh remote task {}: {error}", current.id),
3652 summary: task.name.clone(),
3653 tool_calls: vec![],
3654 artifacts: vec![],
3655 terminal_state_hint: Some(TaskTerminalStateHint::Blocked),
3656 duration_ms: duration_ms(),
3657 };
3658 }
3659 };
3660 if remote_card
3661 .supported_rpc_methods
3662 .iter()
3663 .any(|method| method == "task/artifacts")
3664 {
3665 manifest = client
3666 .list_task_artifacts(&remote_target.url, ¤t.id)
3667 .await
3668 .unwrap_or_default();
3669 }
3670 if let Err(error) = self
3671 .sync_remote_task_snapshot(
3672 task,
3673 &remote_target,
3674 ¤t,
3675 &manifest,
3676 compatibility.clone(),
3677 )
3678 .await
3679 {
3680 tracing::warn!(task_id = %task.id, error = %error, "Failed to sync remote task snapshot");
3681 }
3682 }
3683 }
3684
3685 async fn complete_task_execution(
3686 &self,
3687 task: DelegatedTask,
3688 task_result: TaskResult,
3689 preserve_existing_checkpoint: bool,
3690 expected_attempt: u32,
3691 ) -> Result<(), String> {
3692 let run_id = task_result
3693 .run_id
3694 .clone()
3695 .or_else(|| task.run_id.clone())
3696 .ok_or_else(|| "Completed task result is missing run_id".to_string())?;
3697
3698 let (
3699 mut run_snapshot,
3700 task_snapshot,
3701 tasks_to_start,
3702 environment_id,
3703 finalized_state,
3704 gate_message,
3705 ) = {
3706 let mut active = self.active_tasks.lock().await;
3707
3708 let mut runs = self.supervisor_runs.lock().await;
3709 let run = runs
3710 .get_mut(&run_id)
3711 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
3712 let record = run
3713 .tasks
3714 .iter_mut()
3715 .find(|record| record.task.id == task.id)
3716 .ok_or_else(|| format!("Task '{}' not found in run", task.id))?;
3717
3718 if record.attempts != expected_attempt {
3719 tracing::debug!(
3720 task_id = %task.id,
3721 run_id = %run_id,
3722 expected_attempt,
3723 actual_attempt = record.attempts,
3724 "Ignoring stale delegated task completion after retry"
3725 );
3726 return Ok(());
3727 }
3728
3729 if let Some(control) = active.get(&task.id)
3730 && control.attempt != expected_attempt
3731 {
3732 tracing::debug!(
3733 task_id = %task.id,
3734 run_id = %run_id,
3735 expected_attempt,
3736 active_attempt = control.attempt,
3737 "Ignoring stale delegated task completion for superseded active attempt"
3738 );
3739 return Ok(());
3740 }
3741
3742 active.remove(&task.id);
3743
3744 if record.result.is_some()
3745 && record.completed_at.is_some()
3746 && matches!(
3747 record.state,
3748 SupervisorTaskState::Completed
3749 | SupervisorTaskState::Failed
3750 | SupervisorTaskState::Cancelled
3751 | SupervisorTaskState::Blocked
3752 | SupervisorTaskState::ReviewPending
3753 | SupervisorTaskState::TestPending
3754 )
3755 {
3756 tracing::debug!(
3757 task_id = %task.id,
3758 run_id = %run_id,
3759 state = ?record.state,
3760 "Ignoring duplicate terminal task completion"
3761 );
3762 return Ok(());
3763 }
3764
3765 record.result = Some(task_result.clone());
3766 record.completed_at = Some(Utc::now());
3767 record.updated_at = Utc::now();
3768 let previous_local_execution = record.local_execution.clone();
3769 let mut gate_message = None;
3770 let hinted_terminal_state = task_result.terminal_state_hint.map(|hint| match hint {
3771 TaskTerminalStateHint::Completed => SupervisorTaskState::Completed,
3772 TaskTerminalStateHint::Failed => SupervisorTaskState::Failed,
3773 TaskTerminalStateHint::Cancelled => SupervisorTaskState::Cancelled,
3774 TaskTerminalStateHint::Blocked => SupervisorTaskState::Blocked,
3775 });
3776
3777 if task_result.success {
3778 if record.task.reviewer_required {
3779 record.state = SupervisorTaskState::ReviewPending;
3780 record.approval.request(
3781 ApprovalScope::Review,
3782 ApprovalActor::system("orchestrator"),
3783 Some("Execution finished. Awaiting explicit review approval.".to_string()),
3784 );
3785 let message =
3786 build_gate_request_message(&run.id, record, ApprovalScope::Review);
3787 record.messages.push(message.clone());
3788 gate_message = Some(message);
3789 } else if record.task.test_required {
3790 record.state = SupervisorTaskState::TestPending;
3791 record.approval.request(
3792 ApprovalScope::TestValidation,
3793 ApprovalActor::system("orchestrator"),
3794 Some("Execution finished. Awaiting explicit test validation.".to_string()),
3795 );
3796 let message =
3797 build_gate_request_message(&run.id, record, ApprovalScope::TestValidation);
3798 record.messages.push(message.clone());
3799 gate_message = Some(message);
3800 } else {
3801 record.state = SupervisorTaskState::Completed;
3802 if matches!(record.approval.state, ApprovalState::Pending) {
3803 record.approval.reset_for_task(&record.task);
3804 record.approval.note = Some(
3805 "Execution completed after clearing a stale pending approval state."
3806 .into(),
3807 );
3808 }
3809 }
3810 } else {
3811 record.state = hinted_terminal_state.unwrap_or(SupervisorTaskState::Failed);
3812 record.blocked_reasons = if matches!(record.state, SupervisorTaskState::Blocked) {
3813 vec![task_result.output.clone()]
3814 } else {
3815 Vec::new()
3816 };
3817 if matches!(record.approval.state, ApprovalState::Pending) {
3818 record.approval.reset_for_task(&record.task);
3819 record.approval.note = Some(
3820 "Execution failed after clearing a stale pending approval state.".into(),
3821 );
3822 }
3823 }
3824
3825 if matches!(task.execution_mode, AgentExecutionMode::Remote) {
3826 record.local_execution = None;
3827 } else {
3828 record.local_execution = Some(local_execution_record_for_terminal(
3829 &task_result,
3830 record.state,
3831 previous_local_execution.as_ref(),
3832 ));
3833 }
3834
3835 let (task_snapshot, environment_id, finalized_state) =
3836 (record.clone(), record.environment_id.clone(), record.state);
3837 let ready_to_start = if preserve_existing_checkpoint
3838 && matches!(record.state, SupervisorTaskState::Blocked)
3839 {
3840 collect_ready_tasks_except(run, Some(&task.id))
3841 } else {
3842 collect_ready_tasks(run)
3843 };
3844 run.updated_at = Utc::now();
3845 run.status = recalculate_run_status(run);
3846 (
3847 run.clone(),
3848 task_snapshot,
3849 ready_to_start,
3850 environment_id,
3851 finalized_state,
3852 gate_message,
3853 )
3854 };
3855 let memory_file_path = persist_delegated_task_memory(&task_snapshot).await;
3856
3857 let updated_environment = match finalized_state {
3858 SupervisorTaskState::Completed => {
3859 self.finalize_environment_for_task(&environment_id, true, false)
3860 .await?
3861 }
3862 SupervisorTaskState::Failed | SupervisorTaskState::Cancelled => {
3863 self.finalize_environment_for_task(&environment_id, false, true)
3864 .await?
3865 }
3866 _ => self.release_environment_lease(&environment_id).await?,
3867 };
3868 if let Some(environment) = updated_environment {
3869 self.update_environment_in_runs(&environment).await?;
3870 if let Some(run) = self.supervisor_runs.lock().await.get(&run_id).cloned() {
3871 run_snapshot = run;
3872 }
3873 }
3874
3875 record_task_completion(
3876 &task,
3877 &task_result,
3878 memory_file_path.as_deref(),
3879 &task_snapshot,
3880 &run_snapshot,
3881 );
3882 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
3883 if let Some((workspace_dir, session_id, tracking_task_id)) =
3884 task_reflection_sync_context(&task_snapshot.task)
3885 {
3886 let signals = task_record_outcome_signals(&task_snapshot);
3887 let _ = sync_task_reflection_outcomes(
3888 &workspace_dir,
3889 &session_id,
3890 &tracking_task_id,
3891 &signals,
3892 )
3893 .await;
3894 }
3895 self.publish_delegated_task_memory_handoff(
3896 &run_id,
3897 &task_snapshot,
3898 &task_result,
3899 memory_file_path.as_deref(),
3900 )
3901 .await?;
3902 let should_preserve_blocked_checkpoint =
3903 preserve_existing_checkpoint && matches!(finalized_state, SupervisorTaskState::Blocked);
3904 if !matches!(task.execution_mode, AgentExecutionMode::Remote)
3905 && !should_preserve_blocked_checkpoint
3906 {
3907 self.persist_delegated_checkpoint_async(&delegated_terminal_checkpoint(
3908 &task,
3909 &task_result,
3910 ))
3911 .await?;
3912 }
3913 if let Some(message) = gate_message {
3914 self.notify_team_message(message).await;
3915 }
3916
3917 let observer_for_complete = { self.observer.read().await.clone() };
3918 if let Some(obs) = observer_for_complete.as_ref() {
3919 obs.on_task_completed(task.clone(), task_result.clone())
3920 .await;
3921 }
3922
3923 let _ = self.result_tx.send(task_result).await;
3924
3925 for next_task in tasks_to_start {
3926 self.schedule_task_execution(next_task);
3927 }
3928
3929 Ok(())
3930 }
3931
3932 async fn publish_delegated_task_memory_handoff(
3933 &self,
3934 run_id: &str,
3935 task_record: &SupervisorTaskRecord,
3936 task_result: &TaskResult,
3937 memory_file_path: Option<&Path>,
3938 ) -> Result<(), String> {
3939 let kind = match task_record.state {
3940 SupervisorTaskState::Completed
3941 | SupervisorTaskState::ReviewPending
3942 | SupervisorTaskState::TestPending => TeamMessageKind::Handoff,
3943 SupervisorTaskState::Failed
3944 | SupervisorTaskState::Cancelled
3945 | SupervisorTaskState::Blocked => TeamMessageKind::Blocker,
3946 _ => return Ok(()),
3947 };
3948
3949 let recipient_agent_id = {
3950 let runs = self.supervisor_runs.lock().await;
3951 runs.get(run_id).and_then(|run| run.lead_agent_id.clone())
3952 };
3953 let mut message = TeamMessage::new(
3954 run_id.to_string(),
3955 Some(task_record.task.id.clone()),
3956 kind,
3957 Some(task_record.task.agent_id.clone()),
3958 recipient_agent_id.clone(),
3959 build_delegated_task_memory_handoff_content(task_record, task_result, memory_file_path),
3960 )
3961 .with_result_reference(TeamResultReference::from_task_result(task_result))
3962 .with_artifact_references(
3963 task_result
3964 .artifacts
3965 .iter()
3966 .map(|artifact| {
3967 TeamArtifactReference::from_task_artifact(
3968 Some(task_record.task.id.clone()),
3969 artifact,
3970 )
3971 })
3972 .collect(),
3973 );
3974
3975 let unread_by_agent_ids = recipient_agent_id
3976 .into_iter()
3977 .filter(|agent_id| agent_id != &task_record.task.agent_id)
3978 .collect::<Vec<_>>();
3979 if !unread_by_agent_ids.is_empty() {
3980 message = message.with_unread_by_agent_ids(unread_by_agent_ids);
3981 }
3982
3983 let run_snapshot = {
3984 let mut runs = self.supervisor_runs.lock().await;
3985 let run = runs
3986 .get_mut(run_id)
3987 .ok_or_else(|| format!("Run '{}' not found", run_id))?;
3988 insert_team_message(run, message.clone())?;
3989 apply_collaboration_retention(run);
3990 run.clone()
3991 };
3992
3993 self.persist_run_with_hierarchy_sync(run_snapshot).await?;
3994 self.capture_shared_cognition_from_message(run_id, &message)
3995 .await;
3996
3997 let memory_file_path_display = memory_file_path.map(|path| path.display().to_string());
3998 tracing::info!(
3999 run_id = %run_id,
4000 task_id = %task_record.task.id,
4001 agent_id = %task_record.task.agent_id,
4002 message_id = %message.id,
4003 message_kind = ?kind,
4004 memory_file_path = %memory_file_path_display.as_deref().unwrap_or("n/a"),
4005 "Published delegated task memory handoff"
4006 );
4007
4008 self.notify_team_message(message).await;
4009 Ok(())
4010 }
4011
4012 fn schedule_task_execution(&self, task: DelegatedTask) {
4013 let orchestrator = self.clone();
4014 let run_id = task.run_id.clone().unwrap_or_else(|| "n/a".to_string());
4015 let task_id = task.id.clone();
4016 let agent_id = task.agent_id.clone();
4017 let schedule_span = tracing::info_span!(
4018 "schedule_ready_task",
4019 run_id = %run_id,
4020 task_id = %task_id,
4021 agent_id = %agent_id,
4022 );
4023 tokio::spawn(
4024 async move {
4025 if let Err(error) = orchestrator.start_task_execution(task).await {
4026 tracing::error!(error = %error, "Failed to start ready task");
4027 }
4028 }
4029 .instrument(schedule_span),
4030 );
4031 }
4032
4033 fn persist_run(&self, run: &SupervisorRun) -> Result<(), String> {
4034 let Some(root) = run
4035 .workspace_dir
4036 .as_deref()
4037 .or(self.default_workspace_dir.as_deref())
4038 else {
4039 return Ok(());
4040 };
4041
4042 persist_run_to_disk(root, run)
4043 }
4044
4045 async fn persist_run_async(&self, run: &SupervisorRun) -> Result<(), String> {
4046 let Some(root) = run
4047 .workspace_dir
4048 .as_deref()
4049 .or(self.default_workspace_dir.as_deref())
4050 else {
4051 return Ok(());
4052 };
4053
4054 persist_run_to_disk_async(root, run).await
4055 }
4056
4057 fn persist_delegated_checkpoint(
4058 &self,
4059 checkpoint: &DelegatedTaskCheckpoint,
4060 ) -> Result<(), String> {
4061 let Some(root) = self
4062 .default_workspace_dir
4063 .as_deref()
4064 .or(checkpoint.workspace_dir.as_deref())
4065 else {
4066 return Ok(());
4067 };
4068
4069 persist_checkpoint_to_disk(root, checkpoint)
4070 }
4071
4072 async fn persist_delegated_checkpoint_async(
4073 &self,
4074 checkpoint: &DelegatedTaskCheckpoint,
4075 ) -> Result<(), String> {
4076 let Some(root) = self
4077 .default_workspace_dir
4078 .as_deref()
4079 .or(checkpoint.workspace_dir.as_deref())
4080 else {
4081 return Ok(());
4082 };
4083
4084 persist_checkpoint_to_disk_async(root, checkpoint).await
4085 }
4086
4087 #[cfg_attr(not(test), allow(dead_code))]
4088 fn load_delegated_checkpoint(&self, task: &DelegatedTask) -> Option<DelegatedTaskCheckpoint> {
4089 let mut checkpoints = load_latest_checkpoints_by_task(checkpoint_roots_for_task(
4090 task,
4091 self.default_workspace_dir.as_deref(),
4092 ));
4093 checkpoints.remove(&task.id)
4094 }
4095
4096 async fn load_delegated_checkpoint_async(
4097 &self,
4098 task: &DelegatedTask,
4099 ) -> Option<DelegatedTaskCheckpoint> {
4100 let roots = checkpoint_roots_for_task(task, self.default_workspace_dir.as_deref());
4101 let task_id = task.id.clone();
4102 match tokio::task::spawn_blocking(move || {
4103 let mut checkpoints = load_latest_checkpoints_by_task(roots);
4104 checkpoints.remove(&task_id)
4105 })
4106 .await
4107 {
4108 Ok(checkpoint) => checkpoint,
4109 Err(error) => {
4110 tracing::warn!(task_id = %task.id, "Failed to join delegated checkpoint load task: {error}");
4111 None
4112 }
4113 }
4114 }
4115
4116 async fn load_checkpoint_resume_state_async(
4117 &self,
4118 task: &DelegatedTask,
4119 ) -> Option<PausedExecutionState> {
4120 self.load_delegated_checkpoint_async(task)
4121 .await
4122 .and_then(|checkpoint| checkpoint.resume_state)
4123 }
4124
4125 async fn persist_run_with_hierarchy_sync(
4126 &self,
4127 run: SupervisorRun,
4128 ) -> Result<Vec<SupervisorRun>, String> {
4129 self.persist_run_async(&run).await?;
4130 let mut updated_runs = vec![run.clone()];
4131 if let Some(parent_run) = self.sync_parent_run_from_child(&run.id).await? {
4132 self.persist_run_async(&parent_run).await?;
4133 updated_runs.push(parent_run);
4134 }
4135 for snapshot in &updated_runs {
4136 self.notify_run_updated(snapshot.clone()).await;
4137 }
4138 Ok(updated_runs)
4139 }
4140
4141 async fn sync_parent_run_from_child(
4142 &self,
4143 child_run_id: &str,
4144 ) -> Result<Option<SupervisorRun>, String> {
4145 let mut runs = self.supervisor_runs.lock().await;
4146 let Some(child_run) = runs.get(child_run_id).cloned() else {
4147 return Ok(None);
4148 };
4149 let Some(parent_ref) = child_run.parent_run.as_ref() else {
4150 return Ok(None);
4151 };
4152 let parent = runs.get_mut(&parent_ref.parent_run_id).ok_or_else(|| {
4153 format!(
4154 "Parent supervisor run '{}' not found for child '{}'",
4155 parent_ref.parent_run_id, child_run_id
4156 )
4157 })?;
4158 upsert_child_run_summary(parent, &child_run);
4159 parent.updated_at = child_run.updated_at.max(parent.updated_at);
4160 refresh_run_rollups(parent);
4161 if matches!(
4162 parent.status,
4163 SupervisorRunStatus::Completed | SupervisorRunStatus::Cancelled
4164 ) {
4165 parent.completed_at.get_or_insert(Utc::now());
4166 } else {
4167 parent.completed_at = None;
4168 }
4169 Ok(Some(parent.clone()))
4170 }
4171
4172 async fn notify_run_updated(&self, run: SupervisorRun) {
4173 let observer_for_run = { self.observer.read().await.clone() };
4174 if let Some(obs) = observer_for_run.as_ref() {
4175 obs.on_run_updated(run).await;
4176 }
4177 }
4178
4179 async fn notify_team_message(&self, message: TeamMessage) {
4180 let observer_for_message = { self.observer.read().await.clone() };
4181 if let Some(obs) = observer_for_message.as_ref() {
4182 obs.on_team_message(message.clone()).await;
4183 }
4184
4185 if let Some(thread) = self
4186 .list_team_threads_with_options(&message.run_id, true)
4187 .await
4188 .into_iter()
4189 .find(|thread| thread.id == message.effective_thread_id())
4190 {
4191 self.notify_team_thread_updated(thread).await;
4192 }
4193 }
4194
4195 async fn notify_team_thread_updated(&self, thread: TeamThread) {
4196 let observer_for_thread = { self.observer.read().await.clone() };
4197 if let Some(obs) = observer_for_thread.as_ref() {
4198 obs.on_team_thread_updated(thread).await;
4199 }
4200 }
4201}
4202
4203#[derive(Debug, Clone, Copy)]
4204enum TeamMessageLocation {
4205 Run(usize),
4206 Task {
4207 task_index: usize,
4208 message_index: usize,
4209 },
4210}
4211
4212fn collect_team_messages(run: &SupervisorRun) -> Vec<TeamMessage> {
4213 let mut messages = Vec::with_capacity(
4214 run.messages.len()
4215 + run
4216 .tasks
4217 .iter()
4218 .map(|task| task.messages.len())
4219 .sum::<usize>(),
4220 );
4221 messages.extend(run.messages.clone());
4222 for task in &run.tasks {
4223 messages.extend(task.messages.clone());
4224 }
4225 messages.sort_by(|left, right| left.created_at.cmp(&right.created_at));
4226 messages
4227}
4228
4229fn insert_team_message(run: &mut SupervisorRun, message: TeamMessage) -> Result<(), String> {
4230 if let Some(task_id) = message.task_id.as_deref() {
4231 let record = run
4232 .tasks
4233 .iter_mut()
4234 .find(|record| record.task.id == task_id)
4235 .ok_or_else(|| format!("Task '{}' not found in run", task_id))?;
4236 if matches!(message.kind, TeamMessageKind::Blocker)
4237 && !record
4238 .blocked_reasons
4239 .iter()
4240 .any(|reason| reason == &message.content)
4241 {
4242 record.blocked_reasons.push(message.content.clone());
4243 if !matches!(
4244 record.state,
4245 SupervisorTaskState::Completed
4246 | SupervisorTaskState::Failed
4247 | SupervisorTaskState::Cancelled
4248 ) {
4249 record.state = SupervisorTaskState::Blocked;
4250 }
4251 }
4252 record.messages.push(message);
4253 record.updated_at = Utc::now();
4254 } else {
4255 run.messages.push(message);
4256 }
4257
4258 run.updated_at = Utc::now();
4259 run.status = recalculate_run_status(run);
4260 Ok(())
4261}
4262
4263fn find_latest_thread_action_location(
4264 run: &SupervisorRun,
4265 thread_id: &str,
4266) -> Option<(TeamMessageLocation, DateTime<Utc>)> {
4267 let mut latest = None;
4268 for (message_index, message) in run.messages.iter().enumerate() {
4269 if message.effective_thread_id() == thread_id && message.action_request.is_some() {
4270 latest = Some((TeamMessageLocation::Run(message_index), message.created_at));
4271 }
4272 }
4273 for (task_index, task) in run.tasks.iter().enumerate() {
4274 for (message_index, message) in task.messages.iter().enumerate() {
4275 if message.effective_thread_id() == thread_id && message.action_request.is_some() {
4276 match latest {
4277 Some((_, current_time)) if current_time >= message.created_at => {}
4278 _ => {
4279 latest = Some((
4280 TeamMessageLocation::Task {
4281 task_index,
4282 message_index,
4283 },
4284 message.created_at,
4285 ));
4286 }
4287 }
4288 }
4289 }
4290 }
4291 latest
4292}
4293
4294fn resolve_team_thread_action_request(
4295 run: &mut SupervisorRun,
4296 thread_id: &str,
4297 status: CollaborationActionStatus,
4298 actor_id: Option<String>,
4299 note: Option<String>,
4300) -> Result<(Option<String>, String), String> {
4301 let (location, _) = find_latest_thread_action_location(run, thread_id)
4302 .ok_or_else(|| format!("Thread '{}' has no actionable request", thread_id))?;
4303 let blocker_contents = collect_team_messages(run)
4304 .into_iter()
4305 .filter(|message| {
4306 message.effective_thread_id() == thread_id
4307 && matches!(message.kind, TeamMessageKind::Blocker)
4308 })
4309 .map(|message| message.content)
4310 .collect::<Vec<_>>();
4311
4312 let (task_id, message_id) = match location {
4313 TeamMessageLocation::Run(message_index) => {
4314 let message = run
4315 .messages
4316 .get_mut(message_index)
4317 .ok_or_else(|| format!("Thread '{}' message not found", thread_id))?;
4318 let action_request = message
4319 .action_request
4320 .as_mut()
4321 .ok_or_else(|| format!("Thread '{}' has no actionable request", thread_id))?;
4322 action_request.resolve(status, actor_id.clone(), note.clone());
4323 message.unread_by_agent_ids.clear();
4324 (message.task_id.clone(), message.id.clone())
4325 }
4326 TeamMessageLocation::Task {
4327 task_index,
4328 message_index,
4329 } => {
4330 let record = run
4331 .tasks
4332 .get_mut(task_index)
4333 .ok_or_else(|| format!("Thread '{}' task not found", thread_id))?;
4334 let message = record
4335 .messages
4336 .get_mut(message_index)
4337 .ok_or_else(|| format!("Thread '{}' message not found", thread_id))?;
4338 let action_request = message
4339 .action_request
4340 .as_mut()
4341 .ok_or_else(|| format!("Thread '{}' has no actionable request", thread_id))?;
4342 action_request.resolve(status, actor_id.clone(), note.clone());
4343 message.unread_by_agent_ids.clear();
4344 record.updated_at = Utc::now();
4345 (Some(record.task.id.clone()), message.id.clone())
4346 }
4347 };
4348
4349 if matches!(
4350 status,
4351 CollaborationActionStatus::Resolved | CollaborationActionStatus::Cancelled
4352 ) && !blocker_contents.is_empty()
4353 && let Some(task_id) = task_id.as_deref()
4354 && let Some(record) = run
4355 .tasks
4356 .iter_mut()
4357 .find(|record| record.task.id == task_id)
4358 {
4359 record
4360 .blocked_reasons
4361 .retain(|reason| !blocker_contents.iter().any(|content| content == reason));
4362 if record.blocked_reasons.is_empty() && matches!(record.state, SupervisorTaskState::Blocked)
4363 {
4364 record.state = SupervisorTaskState::Queued;
4365 }
4366 record.updated_at = Utc::now();
4367 }
4368
4369 run.updated_at = Utc::now();
4370 run.status = recalculate_run_status(run);
4371 Ok((task_id, message_id))
4372}
4373
4374fn archive_team_thread_messages(
4375 run: &mut SupervisorRun,
4376 thread_id: &str,
4377 actor_id: Option<String>,
4378 note: Option<String>,
4379) -> Result<(), String> {
4380 let mut found = false;
4381 for message in &mut run.messages {
4382 if message.effective_thread_id() == thread_id {
4383 message.archive(actor_id.clone(), note.clone());
4384 found = true;
4385 }
4386 }
4387 for task in &mut run.tasks {
4388 for message in &mut task.messages {
4389 if message.effective_thread_id() == thread_id {
4390 message.archive(actor_id.clone(), note.clone());
4391 found = true;
4392 }
4393 }
4394 if found {
4395 task.updated_at = Utc::now();
4396 }
4397 }
4398 if !found {
4399 return Err(format!("Thread '{}' not found", thread_id));
4400 }
4401 run.updated_at = Utc::now();
4402 run.status = recalculate_run_status(run);
4403 Ok(())
4404}
4405
4406fn apply_collaboration_retention(run: &mut SupervisorRun) {
4407 let cutoff = Utc::now() - chrono::Duration::days(DEFAULT_RESOLVED_THREAD_RETENTION_DAYS.max(0));
4408 let thread_ids = build_team_threads_with_options(&collect_team_messages(run), true)
4409 .into_iter()
4410 .filter(|thread| {
4411 !thread.archived
4412 && matches!(thread.status, CollaborationThreadStatus::Resolved)
4413 && thread.updated_at <= cutoff
4414 })
4415 .map(|thread| thread.id)
4416 .collect::<Vec<_>>();
4417
4418 if thread_ids.is_empty() {
4419 return;
4420 }
4421
4422 for message in &mut run.messages {
4423 if thread_ids
4424 .iter()
4425 .any(|candidate| candidate == message.effective_thread_id())
4426 {
4427 message.archive(
4428 Some("orchestrator".to_string()),
4429 Some("Auto-archived after retention period".to_string()),
4430 );
4431 }
4432 }
4433 for task in &mut run.tasks {
4434 let mut touched = false;
4435 for message in &mut task.messages {
4436 if thread_ids
4437 .iter()
4438 .any(|candidate| candidate == message.effective_thread_id())
4439 {
4440 message.archive(
4441 Some("orchestrator".to_string()),
4442 Some("Auto-archived after retention period".to_string()),
4443 );
4444 touched = true;
4445 }
4446 }
4447 if touched {
4448 task.updated_at = Utc::now();
4449 }
4450 }
4451}
4452
4453fn collaboration_status_message(status: CollaborationActionStatus, note: Option<String>) -> String {
4454 note.unwrap_or_else(|| match status {
4455 CollaborationActionStatus::Open => "Action request reopened.".to_string(),
4456 CollaborationActionStatus::Acknowledged => "Action request acknowledged.".to_string(),
4457 CollaborationActionStatus::Resolved => "Action request resolved.".to_string(),
4458 CollaborationActionStatus::NeedsRevision => {
4459 "Changes requested before proceeding.".to_string()
4460 }
4461 CollaborationActionStatus::Cancelled => "Action request cancelled.".to_string(),
4462 })
4463}
4464
4465async fn execute_delegated_task<M: OrchestratorAgentManager>(
4469 orchestrator: &AgentOrchestrator<M>,
4470 task: &DelegatedTask,
4471 cancel_token: CancellationToken,
4472) -> LocalDelegatedExecutionOutcome {
4473 let role = task.role.clone().unwrap_or_default();
4474 let mut prompt_sections = vec![role.prompt_preamble().to_string()];
4475
4476 if let Some(brief) = &task.delegation_brief {
4477 prompt_sections.push(format!("Delegation Brief:\n{}", brief.as_prompt_section()));
4478 }
4479 if let Some(ctx) = &task.context {
4480 prompt_sections.push(format!(
4481 "Context:\n{}",
4482 serde_json::to_string_pretty(ctx).unwrap_or_default()
4483 ));
4484 }
4485 prompt_sections.push(format!("Task:\n{}", task.prompt));
4486 if task.planning_only {
4487 prompt_sections.push(
4488 "Execution Mode:\nPlanning only. Produce a structured plan and do not claim implementation was performed."
4489 .to_string(),
4490 );
4491 }
4492
4493 let full_prompt = prompt_sections.join("\n\n");
4494
4495 tracing::debug!(
4496 agent_id = %task.agent_id,
4497 task_id = %task.id,
4498 prompt_len = full_prompt.len(),
4499 required_tools = ?task.required_tools,
4500 "Executing delegated task via AgentPipeline"
4501 );
4502
4503 orchestrator
4505 .agent_manager
4506 .update_activity(&task.agent_id)
4507 .await;
4508
4509 let mut request = AgentRequest::new(&full_prompt)
4511 .with_streaming(true)
4512 .with_source(RequestSource::Orchestrator)
4513 .with_allowed_tools(task.required_tools.clone())
4514 .with_agent(task.agent_id.clone())
4515 .with_memory_tags(task.memory_tags.clone());
4516
4517 if !task.prompt.trim().is_empty() {
4518 request.metadata.hints.insert(
4519 "requirement_detection_input".to_string(),
4520 task.prompt.clone(),
4521 );
4522 }
4523
4524 if let Some(session_id) = task.session_id.as_deref() {
4525 request = request.with_session(session_id.to_string());
4526 }
4527 if let Some(directive_id) = task.directive_id.as_deref() {
4528 request = request.with_directive(directive_id.to_string());
4529 }
4530 if let Some(tracking_task_id) = task.tracking_task_id.as_deref() {
4531 request = request.with_task(tracking_task_id.to_string());
4532 }
4533 if let Some(workspace_dir) = task.workspace_dir.as_ref() {
4534 request = request.with_workspace(workspace_dir.clone());
4535 }
4536
4537 if let Some(resume_state) = orchestrator.load_checkpoint_resume_state_async(task).await {
4538 request = request.with_resume_state(resume_state);
4539 }
4540
4541 let pipeline = AgentPipeline::with_provider_optimized_config(orchestrator.config.clone())
4542 .with_knowledge(
4543 orchestrator_knowledge_store(),
4544 orchestrator_knowledge_settings(),
4545 );
4546 let (tx, mut rx) = mpsc::channel(256);
4547 let pipeline_handle =
4548 tokio::spawn(async move { pipeline.process_streaming(request, tx, cancel_token).await });
4549
4550 #[derive(Default)]
4551 struct PendingDelegatedToolCall {
4552 id: String,
4553 name: String,
4554 arguments: String,
4555 }
4556
4557 let mut partial_content = String::new();
4558 let mut partial_thinking = String::new();
4559 let mut current_iteration = 0u32;
4560 let mut pending_tool_call: Option<PendingDelegatedToolCall> = None;
4561 let mut completed_resume_tool_calls: Vec<ToolCallRecord> = Vec::new();
4562 let mut completed_orchestrator_tool_calls: Vec<OrchestratorToolCall> = Vec::new();
4563 let mut telemetry_context = LocalExecutionTelemetryContext {
4564 iteration: current_iteration,
4565 completed_tool_call_count: 0,
4566 ..LocalExecutionTelemetryContext::default()
4567 };
4568
4569 while let Some(chunk) = rx.recv().await {
4570 match &chunk {
4571 StreamChunk::AgentLoopIteration { iteration } => {
4572 current_iteration = *iteration;
4573 telemetry_context.iteration = current_iteration;
4574 if let Some(progress) =
4575 local_execution_progress_from_chunk(&chunk, &telemetry_context)
4576 {
4577 let _ = orchestrator
4578 .sync_local_execution_progress(task, progress)
4579 .await;
4580 }
4581 }
4582 StreamChunk::Text(text) => {
4583 partial_content.push_str(text);
4584 telemetry_context.has_partial_content = !partial_content.is_empty();
4585 telemetry_context.partial_content_chars = partial_content.chars().count();
4586 }
4587 StreamChunk::Thinking(thinking) => {
4588 partial_thinking.push_str(thinking);
4589 telemetry_context.has_partial_thinking = !partial_thinking.is_empty();
4590 telemetry_context.partial_thinking_chars = partial_thinking.chars().count();
4591 }
4592 StreamChunk::ToolCallStart { id, name } => {
4593 pending_tool_call = Some(PendingDelegatedToolCall {
4594 id: id.clone(),
4595 name: name.clone(),
4596 arguments: String::new(),
4597 });
4598 telemetry_context.current_tool_name = Some(name.clone());
4599 if let Some(progress) =
4600 local_execution_progress_from_chunk(&chunk, &telemetry_context)
4601 {
4602 let _ = orchestrator
4603 .sync_local_execution_progress(task, progress)
4604 .await;
4605 }
4606 }
4607 StreamChunk::ToolCallArgs(args) => {
4608 if let Some(pending) = pending_tool_call.as_mut() {
4609 pending.arguments.push_str(args);
4610 }
4611 }
4612 StreamChunk::ToolCallEnd => {
4613 if let Some(pending) = pending_tool_call.as_ref() {
4614 let pending_call =
4615 pending_orchestrator_tool_call(&pending.name, &pending.arguments);
4616 let replay_safety = replay_safety_for_tool_call(&pending_call);
4617 let checkpoint = delegated_running_checkpoint(
4618 task,
4619 completed_orchestrator_tool_calls.clone(),
4620 build_delegated_resume_state(
4621 task,
4622 &full_prompt,
4623 &partial_content,
4624 &partial_thinking,
4625 &completed_resume_tool_calls,
4626 current_iteration,
4627 ),
4628 replay_safety,
4629 restart_resume_disposition(replay_safety),
4630 format!("before tool '{}' execution", pending.name),
4631 Some(format!(
4632 "Waiting for completion of tool '{}' during delegated execution.",
4633 pending.name
4634 )),
4635 );
4636 let _ = orchestrator
4637 .persist_delegated_checkpoint_async(&checkpoint)
4638 .await;
4639 }
4640 }
4641 StreamChunk::ToolCallResult {
4642 name,
4643 success,
4644 output,
4645 duration_ms,
4646 } => {
4647 let pending = pending_tool_call
4648 .take()
4649 .unwrap_or(PendingDelegatedToolCall {
4650 id: format!("delegated-tool-{}", completed_resume_tool_calls.len() + 1),
4651 name: name.clone(),
4652 arguments: String::new(),
4653 });
4654 let record = ToolCallRecord {
4655 id: pending.id.clone(),
4656 name: pending.name.clone(),
4657 arguments: pending.arguments.clone(),
4658 result: if *success {
4659 crate::ToolResult::Success(output.clone())
4660 } else {
4661 crate::ToolResult::Error(output.clone())
4662 },
4663 duration_ms: *duration_ms,
4664 };
4665 completed_orchestrator_tool_calls.push(orchestrator_tool_call_from_record(&record));
4666 completed_resume_tool_calls.push(record);
4667 telemetry_context.current_tool_name = None;
4668 telemetry_context.last_completed_tool_name = Some(name.clone());
4669 telemetry_context.last_completed_tool_duration_ms = Some(*duration_ms);
4670 telemetry_context.completed_tool_call_count = completed_resume_tool_calls.len();
4671 let checkpoint = delegated_running_checkpoint(
4672 task,
4673 completed_orchestrator_tool_calls.clone(),
4674 build_delegated_resume_state(
4675 task,
4676 &full_prompt,
4677 &partial_content,
4678 &partial_thinking,
4679 &completed_resume_tool_calls,
4680 current_iteration,
4681 ),
4682 DelegatedReplaySafety::CheckpointResumable,
4683 DelegatedResumeDisposition::ResumeFromCheckpoint,
4684 format!("after tool '{}' result", pending.name),
4685 Some(format!(
4686 "Persisted tool result for '{}' and captured resumable state.",
4687 pending.name
4688 )),
4689 );
4690 let _ = orchestrator
4691 .persist_delegated_checkpoint_async(&checkpoint)
4692 .await;
4693 if let Some(progress) =
4694 local_execution_progress_from_chunk(&chunk, &telemetry_context)
4695 {
4696 let _ = orchestrator
4697 .sync_local_execution_progress(task, progress)
4698 .await;
4699 }
4700 }
4701 StreamChunk::Paused => {
4702 telemetry_context.current_tool_name = None;
4703 if let Some(progress) =
4704 local_execution_progress_from_chunk(&chunk, &telemetry_context)
4705 {
4706 let _ = orchestrator
4707 .sync_local_execution_progress(task, progress)
4708 .await;
4709 }
4710
4711 let safe_boundary_label = pending_tool_call
4712 .as_ref()
4713 .map(|pending| format!("before tool '{}' execution", pending.name))
4714 .unwrap_or_else(|| {
4715 format!("operator pause during iteration {}", current_iteration)
4716 });
4717 let pause_note = pending_tool_call
4718 .as_ref()
4719 .map(|pending| {
4720 format!(
4721 "Paused by operator before tool '{}' execution completed; resumable checkpoint preserved.",
4722 pending.name
4723 )
4724 })
4725 .unwrap_or_else(|| {
4726 "Paused by operator; resumable checkpoint preserved for local delegated task."
4727 .to_string()
4728 });
4729 let checkpoint = DelegatedTaskCheckpoint {
4730 id: delegated_checkpoint_id(&task.id),
4731 task_id: task.id.clone(),
4732 run_id: task.run_id.clone(),
4733 session_id: task.session_id.clone(),
4734 agent_id: task.agent_id.clone(),
4735 environment_id: task.environment_id.clone(),
4736 execution_mode: task.execution_mode.clone(),
4737 stage: DelegatedCheckpointStage::Blocked,
4738 replay_safety: DelegatedReplaySafety::CheckpointResumable,
4739 resume_disposition: DelegatedResumeDisposition::ResumeFromCheckpoint,
4740 safe_boundary_label,
4741 workspace_dir: task.workspace_dir.clone(),
4742 completed_tool_calls: completed_orchestrator_tool_calls.clone(),
4743 result_published: false,
4744 note: Some(pause_note.clone()),
4745 resume_state: Some(build_delegated_resume_state(
4746 task,
4747 &full_prompt,
4748 &partial_content,
4749 &partial_thinking,
4750 &completed_resume_tool_calls,
4751 current_iteration,
4752 )),
4753 created_at: Utc::now(),
4754 updated_at: Utc::now(),
4755 };
4756 let _ = orchestrator
4757 .persist_delegated_checkpoint_async(&checkpoint)
4758 .await;
4759 let _ = pipeline_handle.await;
4760
4761 return LocalDelegatedExecutionOutcome {
4762 result: Err(pause_note),
4763 tool_calls: completed_orchestrator_tool_calls,
4764 terminal_state_hint: TaskTerminalStateHint::Blocked,
4765 preserve_existing_checkpoint: true,
4766 };
4767 }
4768 StreamChunk::Cancelled => {
4769 telemetry_context.current_tool_name = None;
4770 if let Some(progress) =
4771 local_execution_progress_from_chunk(&chunk, &telemetry_context)
4772 {
4773 let _ = orchestrator
4774 .sync_local_execution_progress(task, progress)
4775 .await;
4776 }
4777 let _ = pipeline_handle.await;
4778
4779 return LocalDelegatedExecutionOutcome {
4780 result: Err("Local delegated task cancelled by operator.".to_string()),
4781 tool_calls: completed_orchestrator_tool_calls,
4782 terminal_state_hint: TaskTerminalStateHint::Cancelled,
4783 preserve_existing_checkpoint: false,
4784 };
4785 }
4786 other => {
4787 if let StreamChunk::TokenUsageUpdate {
4788 estimated,
4789 limit,
4790 percentage,
4791 status,
4792 estimated_cost,
4793 } = other
4794 {
4795 telemetry_context.token_usage = Some(LocalExecutionTokenUsageSnapshot {
4796 estimated_tokens: Some(*estimated),
4797 limit: Some(*limit),
4798 percentage: Some(*percentage),
4799 status: Some(token_usage_status_label(*status).to_string()),
4800 estimated_cost_usd: Some(*estimated_cost),
4801 input_tokens: telemetry_context
4802 .token_usage
4803 .as_ref()
4804 .and_then(|usage| usage.input_tokens),
4805 output_tokens: telemetry_context
4806 .token_usage
4807 .as_ref()
4808 .and_then(|usage| usage.output_tokens),
4809 total_tokens: telemetry_context
4810 .token_usage
4811 .as_ref()
4812 .and_then(|usage| usage.total_tokens),
4813 model: telemetry_context
4814 .token_usage
4815 .as_ref()
4816 .and_then(|usage| usage.model.clone()),
4817 provider: telemetry_context
4818 .token_usage
4819 .as_ref()
4820 .and_then(|usage| usage.provider.clone()),
4821 });
4822 }
4823 if let StreamChunk::Done(usage) = other {
4824 telemetry_context.token_usage = Some(token_usage_snapshot_from_done(
4825 usage.as_ref(),
4826 telemetry_context.token_usage.as_ref(),
4827 ));
4828 telemetry_context.current_tool_name = None;
4829 }
4830 if let StreamChunk::ToolConfirmationRequired { tool_name, .. } = other {
4831 telemetry_context.current_tool_name = Some(tool_name.clone());
4832 }
4833 if let StreamChunk::ToolBlocked { tool_name, .. } = other {
4834 telemetry_context.current_tool_name = Some(tool_name.clone());
4835 }
4836 if let Some(progress) =
4837 local_execution_progress_from_chunk(other, &telemetry_context)
4838 {
4839 let _ = orchestrator
4840 .sync_local_execution_progress(task, progress)
4841 .await;
4842 }
4843 }
4844 }
4845 }
4846
4847 let result = match pipeline_handle.await {
4848 Ok(result) => result,
4849 Err(error) => {
4850 tracing::error!(
4851 agent_id = %task.agent_id,
4852 task_id = %task.id,
4853 error = %error,
4854 "Delegated task pipeline worker panicked"
4855 );
4856 return LocalDelegatedExecutionOutcome {
4857 result: Err(format!("Delegated pipeline task failed: {error}")),
4858 tool_calls: completed_orchestrator_tool_calls,
4859 terminal_state_hint: TaskTerminalStateHint::Failed,
4860 preserve_existing_checkpoint: false,
4861 };
4862 }
4863 };
4864
4865 match result {
4866 Ok(response) => {
4867 let tool_calls = if completed_orchestrator_tool_calls.is_empty() {
4868 response
4869 .tool_calls
4870 .iter()
4871 .map(orchestrator_tool_call_from_record)
4872 .collect()
4873 } else {
4874 completed_orchestrator_tool_calls
4875 };
4876
4877 tracing::info!(
4878 agent_id = %task.agent_id,
4879 task_id = %task.id,
4880 response_len = response.content.len(),
4881 tool_calls_count = tool_calls.len(),
4882 "Task completed via AgentPipeline"
4883 );
4884
4885 LocalDelegatedExecutionOutcome {
4886 result: Ok(response.content),
4887 tool_calls,
4888 terminal_state_hint: TaskTerminalStateHint::Completed,
4889 preserve_existing_checkpoint: false,
4890 }
4891 }
4892 Err(e) => {
4893 tracing::error!(
4894 agent_id = %task.agent_id,
4895 task_id = %task.id,
4896 error = %e,
4897 tool_calls_count = completed_orchestrator_tool_calls.len(),
4898 "Task failed"
4899 );
4900 LocalDelegatedExecutionOutcome {
4901 result: Err(e.to_string()),
4902 tool_calls: completed_orchestrator_tool_calls,
4903 terminal_state_hint: TaskTerminalStateHint::Failed,
4904 preserve_existing_checkpoint: false,
4905 }
4906 }
4907 }
4908}
4909
4910async fn persist_delegated_task_memory(record: &SupervisorTaskRecord) -> Option<PathBuf> {
4911 let task = &record.task;
4912 let task_result = record.result.as_ref()?;
4913 let workspace_dir = task.workspace_dir.as_deref()?;
4914 let session_id = task.session_id.as_deref()?;
4915
4916 let outcome_signals = task_record_outcome_signals(record);
4917 let mut tags = task.memory_tags.clone();
4918 tags.extend(["delegation".to_string(), "subagent".to_string()]);
4919 tags.extend(
4920 outcome_signal_labels(&outcome_signals)
4921 .into_iter()
4922 .map(|label| format!("outcome:{label}")),
4923 );
4924 tags.sort();
4925 tags.dedup();
4926
4927 let summary = if task_result.success {
4928 task.name
4929 .clone()
4930 .unwrap_or_else(|| format!("Delegated task completed by {}", task.agent_id))
4931 } else {
4932 task.name
4933 .clone()
4934 .unwrap_or_else(|| format!("Delegated task blocked on {}", task.agent_id))
4935 };
4936
4937 let tool_calls = if task_result.tool_calls.is_empty() {
4938 "- No tool calls recorded".to_string()
4939 } else {
4940 task_result
4941 .tool_calls
4942 .iter()
4943 .map(|call| {
4944 format!(
4945 "- {} (success: {}, {} ms)",
4946 call.tool_name, call.success, call.duration_ms
4947 )
4948 })
4949 .collect::<Vec<_>>()
4950 .join("\n")
4951 };
4952 let outcome_lines = if outcome_signals.is_empty() {
4953 "- No durable outcome signals recorded yet".to_string()
4954 } else {
4955 outcome_signals
4956 .iter()
4957 .map(|signal| {
4958 let detail = signal
4959 .summary
4960 .as_deref()
4961 .map(|summary| format!(": {summary}"))
4962 .unwrap_or_default();
4963 format!("- {}{}", signal.kind.label(), detail)
4964 })
4965 .collect::<Vec<_>>()
4966 .join("\n")
4967 };
4968
4969 let content = format!(
4970 "## Delegated Task\n- Orchestrator Task ID: {}\n- Tracking Task ID: {}\n- Agent ID: {}\n- Directive ID: {}\n\n## Prompt\n{}\n\n## Result\n{}\n\n## Tool Calls\n{}\n\n## Outcome Signals\n{}\n",
4971 task.id,
4972 task.tracking_task_id.as_deref().unwrap_or("n/a"),
4973 task.agent_id,
4974 task.directive_id.as_deref().unwrap_or("n/a"),
4975 task.prompt,
4976 task_result.output,
4977 tool_calls,
4978 outcome_lines,
4979 );
4980
4981 let entry = MemoryBankEntry::new(session_id.to_string(), summary, content)
4982 .with_memory_type(if task_result.success {
4983 MemoryType::Handoff
4984 } else {
4985 MemoryType::Blocker
4986 })
4987 .with_scope(if task.directive_id.is_some() {
4988 MemoryScope::Directive
4989 } else {
4990 MemoryScope::Session
4991 })
4992 .with_category("delegation")
4993 .with_provenance(
4994 Some(
4995 task.tracking_task_id
4996 .clone()
4997 .unwrap_or_else(|| task.id.clone()),
4998 ),
4999 task.directive_id.clone(),
5000 Some(task.agent_id.clone()),
5001 )
5002 .with_outcome_provenance(
5003 outcome_signal_summary(&outcome_signals),
5004 outcome_signal_labels(&outcome_signals),
5005 )
5006 .with_tags(tags)
5007 .with_promotion(
5008 session_id.to_string(),
5009 "Delegated subagent result promoted for supervisor retrieval",
5010 )
5011 .with_confidence(match record.state {
5012 SupervisorTaskState::Completed => 0.90,
5013 SupervisorTaskState::ReviewPending | SupervisorTaskState::TestPending => 0.82,
5014 SupervisorTaskState::Cancelled => 0.55,
5015 SupervisorTaskState::Failed | SupervisorTaskState::Blocked => 0.65,
5016 _ => {
5017 if task_result.success {
5018 0.80
5019 } else {
5020 0.65
5021 }
5022 }
5023 });
5024
5025 match crate::save_to_memory_bank(workspace_dir, &entry).await {
5026 Ok(path) => {
5027 tracing::info!(
5028 task_id = %task.id,
5029 agent_id = %task.agent_id,
5030 session_id = %session_id,
5031 memory_file_path = %path.display(),
5032 "Persisted delegated task memory"
5033 );
5034 Some(path)
5035 }
5036 Err(error) => {
5037 tracing::warn!(
5038 task_id = %task.id,
5039 agent_id = %task.agent_id,
5040 error = %error,
5041 "Failed to persist delegated task memory"
5042 );
5043 None
5044 }
5045 }
5046}
5047
5048fn task_record_outcome_signals(record: &SupervisorTaskRecord) -> Vec<OutcomeSignal> {
5049 let mut signals = Vec::new();
5050
5051 match record.state {
5052 SupervisorTaskState::ReviewPending => signals.push(
5053 OutcomeSignal::new(OutcomeSignalKind::ExecutionAwaitingReview)
5054 .with_summary("Execution finished and is waiting for review approval."),
5055 ),
5056 SupervisorTaskState::TestPending => signals.push(
5057 OutcomeSignal::new(OutcomeSignalKind::ExecutionAwaitingTestValidation)
5058 .with_summary("Execution finished and is waiting for explicit test validation."),
5059 ),
5060 SupervisorTaskState::Completed => {
5061 signals.push(OutcomeSignal::new(OutcomeSignalKind::TaskCompleted));
5062 }
5063 SupervisorTaskState::Failed => {
5064 let summary = record
5065 .result
5066 .as_ref()
5067 .map(|result| result.output.clone())
5068 .filter(|value| !value.trim().is_empty());
5069 signals.push(summary.map_or_else(
5070 || OutcomeSignal::new(OutcomeSignalKind::TaskFailed),
5071 |summary| OutcomeSignal::new(OutcomeSignalKind::TaskFailed).with_summary(summary),
5072 ));
5073 }
5074 SupervisorTaskState::Blocked => {
5075 let summary = record.blocked_reasons.first().cloned();
5076 signals.push(summary.map_or_else(
5077 || OutcomeSignal::new(OutcomeSignalKind::TaskBlocked),
5078 |summary| OutcomeSignal::new(OutcomeSignalKind::TaskBlocked).with_summary(summary),
5079 ));
5080 }
5081 SupervisorTaskState::Cancelled => {
5082 signals.push(OutcomeSignal::new(OutcomeSignalKind::TaskCancelled));
5083 }
5084 SupervisorTaskState::Queued
5085 | SupervisorTaskState::PendingApproval
5086 | SupervisorTaskState::Running => {}
5087 }
5088
5089 for scope in [
5090 ApprovalScope::PreExecution,
5091 ApprovalScope::Review,
5092 ApprovalScope::TestValidation,
5093 ] {
5094 if let Some(decision) = latest_approval_decision_for_scope(&record.approval, scope) {
5095 let kind = match (scope, decision.decision) {
5096 (ApprovalScope::PreExecution, ApprovalDecisionKind::Approved) => {
5097 OutcomeSignalKind::PreExecutionApproved
5098 }
5099 (ApprovalScope::PreExecution, ApprovalDecisionKind::Rejected) => {
5100 OutcomeSignalKind::PreExecutionRejected
5101 }
5102 (ApprovalScope::PreExecution, ApprovalDecisionKind::NeedsRevision) => {
5103 OutcomeSignalKind::PreExecutionNeedsRevision
5104 }
5105 (ApprovalScope::Review, ApprovalDecisionKind::Approved) => {
5106 OutcomeSignalKind::ReviewApproved
5107 }
5108 (ApprovalScope::Review, ApprovalDecisionKind::Rejected) => {
5109 OutcomeSignalKind::ReviewRejected
5110 }
5111 (ApprovalScope::Review, ApprovalDecisionKind::NeedsRevision) => {
5112 OutcomeSignalKind::ReviewNeedsRevision
5113 }
5114 (ApprovalScope::TestValidation, ApprovalDecisionKind::Approved) => {
5115 OutcomeSignalKind::TestValidationApproved
5116 }
5117 (ApprovalScope::TestValidation, ApprovalDecisionKind::Rejected) => {
5118 OutcomeSignalKind::TestValidationRejected
5119 }
5120 (ApprovalScope::TestValidation, ApprovalDecisionKind::NeedsRevision) => {
5121 OutcomeSignalKind::TestValidationNeedsRevision
5122 }
5123 };
5124 signals.push(decision.note.clone().map_or_else(
5125 || OutcomeSignal::new(kind),
5126 |note| OutcomeSignal::new(kind).with_summary(note),
5127 ));
5128 }
5129 }
5130
5131 signals
5132}
5133
5134fn latest_approval_decision_for_scope(
5135 approval: &TaskApprovalRecord,
5136 scope: ApprovalScope,
5137) -> Option<&ApprovalDecision> {
5138 approval
5139 .decisions
5140 .iter()
5141 .rev()
5142 .find(|decision| decision.scope == scope)
5143}
5144
5145fn outcome_signal_summary(signals: &[OutcomeSignal]) -> Option<String> {
5146 if signals.is_empty() {
5147 return None;
5148 }
5149
5150 Some(
5151 signals
5152 .iter()
5153 .map(|signal| {
5154 signal
5155 .summary
5156 .clone()
5157 .unwrap_or_else(|| signal.kind.label().to_string())
5158 })
5159 .collect::<Vec<_>>()
5160 .join("; "),
5161 )
5162}
5163
5164fn outcome_signal_labels(signals: &[OutcomeSignal]) -> Vec<String> {
5165 signals
5166 .iter()
5167 .map(|signal| signal.durable_label().to_string())
5168 .collect()
5169}
5170
5171fn delegated_checkpoint_id(task_id: &str) -> String {
5172 format!("checkpoint-{task_id}")
5173}
5174
5175fn delegated_start_checkpoint(task: &DelegatedTask) -> DelegatedTaskCheckpoint {
5176 let now = Utc::now();
5177 DelegatedTaskCheckpoint {
5178 id: delegated_checkpoint_id(&task.id),
5179 task_id: task.id.clone(),
5180 run_id: task.run_id.clone(),
5181 session_id: task.session_id.clone(),
5182 agent_id: task.agent_id.clone(),
5183 environment_id: task.environment_id.clone(),
5184 execution_mode: task.execution_mode.clone(),
5185 stage: DelegatedCheckpointStage::Running,
5186 replay_safety: DelegatedReplaySafety::CheckpointResumable,
5187 resume_disposition: DelegatedResumeDisposition::ResumeFromCheckpoint,
5188 safe_boundary_label: "delegated task dispatch boundary".to_string(),
5189 workspace_dir: task.workspace_dir.clone(),
5190 completed_tool_calls: Vec::new(),
5191 result_published: false,
5192 note: Some("Task dispatched and awaiting local execution progress.".to_string()),
5193 resume_state: None,
5194 created_at: now,
5195 updated_at: now,
5196 }
5197}
5198
5199fn delegated_terminal_checkpoint(
5200 task: &DelegatedTask,
5201 task_result: &TaskResult,
5202) -> DelegatedTaskCheckpoint {
5203 let now = Utc::now();
5204 DelegatedTaskCheckpoint {
5205 id: delegated_checkpoint_id(&task.id),
5206 task_id: task.id.clone(),
5207 run_id: task_result.run_id.clone().or_else(|| task.run_id.clone()),
5208 session_id: task.session_id.clone(),
5209 agent_id: task.agent_id.clone(),
5210 environment_id: task.environment_id.clone(),
5211 execution_mode: task.execution_mode.clone(),
5212 stage: delegated_terminal_stage(task_result),
5213 replay_safety: replay_safety_for_task_result(task_result),
5214 resume_disposition: DelegatedResumeDisposition::NotApplicable,
5215 safe_boundary_label: format!(
5216 "result persisted after {} tool call(s)",
5217 task_result.tool_calls.len()
5218 ),
5219 workspace_dir: task.workspace_dir.clone(),
5220 completed_tool_calls: task_result.tool_calls.clone(),
5221 result_published: true,
5222 note: Some(if task_result.success {
5223 "Task completed and terminal result was published.".to_string()
5224 } else {
5225 format!("Task finished unsuccessfully: {}", task_result.output)
5226 }),
5227 resume_state: None,
5228 created_at: now,
5229 updated_at: now,
5230 }
5231}
5232
5233fn delegated_running_checkpoint(
5234 task: &DelegatedTask,
5235 completed_tool_calls: Vec<OrchestratorToolCall>,
5236 resume_state: PausedExecutionState,
5237 replay_safety: DelegatedReplaySafety,
5238 resume_disposition: DelegatedResumeDisposition,
5239 safe_boundary_label: String,
5240 note: Option<String>,
5241) -> DelegatedTaskCheckpoint {
5242 let now = Utc::now();
5243 DelegatedTaskCheckpoint {
5244 id: delegated_checkpoint_id(&task.id),
5245 task_id: task.id.clone(),
5246 run_id: task.run_id.clone(),
5247 session_id: task.session_id.clone(),
5248 agent_id: task.agent_id.clone(),
5249 environment_id: task.environment_id.clone(),
5250 execution_mode: task.execution_mode.clone(),
5251 stage: DelegatedCheckpointStage::Running,
5252 replay_safety,
5253 resume_disposition,
5254 safe_boundary_label,
5255 workspace_dir: task.workspace_dir.clone(),
5256 completed_tool_calls,
5257 result_published: false,
5258 note,
5259 resume_state: Some(resume_state),
5260 created_at: now,
5261 updated_at: now,
5262 }
5263}
5264
5265fn delegated_terminal_stage(task_result: &TaskResult) -> DelegatedCheckpointStage {
5266 if task_result.success {
5267 return DelegatedCheckpointStage::Completed;
5268 }
5269
5270 match task_result.terminal_state_hint {
5271 Some(TaskTerminalStateHint::Cancelled) => DelegatedCheckpointStage::Cancelled,
5272 Some(TaskTerminalStateHint::Blocked) => DelegatedCheckpointStage::Blocked,
5273 _ => DelegatedCheckpointStage::Failed,
5274 }
5275}
5276
5277fn replay_safety_for_task_result(task_result: &TaskResult) -> DelegatedReplaySafety {
5278 if task_result.tool_calls.is_empty() {
5279 return DelegatedReplaySafety::CheckpointResumable;
5280 }
5281
5282 task_result
5283 .tool_calls
5284 .iter()
5285 .map(replay_safety_for_tool_call)
5286 .max_by_key(|safety| replay_safety_rank(*safety))
5287 .unwrap_or(DelegatedReplaySafety::CheckpointResumable)
5288}
5289
5290fn replay_safety_for_tool_call(tool_call: &OrchestratorToolCall) -> DelegatedReplaySafety {
5291 match tool_call.tool_name.as_str() {
5292 "web" | "web_search" | "code" => DelegatedReplaySafety::PureReadonly,
5293 "shell" | "screen_record" => DelegatedReplaySafety::NonReplayableSideEffect,
5294 "a2a" | "mcp" | "permissions" | "task" | "screenshot" => {
5295 DelegatedReplaySafety::OperatorGated
5296 }
5297 "file" => match tool_call
5298 .input
5299 .get("operation")
5300 .and_then(|value| value.as_str())
5301 {
5302 Some("read" | "list" | "search" | "tree" | "stat") => {
5303 DelegatedReplaySafety::PureReadonly
5304 }
5305 Some("write" | "append" | "copy" | "move" | "mkdir") => {
5306 DelegatedReplaySafety::OperatorGated
5307 }
5308 Some("delete" | "remove") => DelegatedReplaySafety::NonReplayableSideEffect,
5309 _ => DelegatedReplaySafety::OperatorGated,
5310 },
5311 "git" => match tool_call
5312 .input
5313 .get("operation")
5314 .or_else(|| tool_call.input.get("action"))
5315 .and_then(|value| value.as_str())
5316 {
5317 Some("status" | "diff" | "log" | "show" | "blame" | "branch_list") => {
5318 DelegatedReplaySafety::PureReadonly
5319 }
5320 Some("checkout" | "restore" | "stash_apply" | "worktree_add") => {
5321 DelegatedReplaySafety::OperatorGated
5322 }
5323 Some("commit" | "push" | "merge" | "rebase" | "reset" | "stash") => {
5324 DelegatedReplaySafety::NonReplayableSideEffect
5325 }
5326 _ => DelegatedReplaySafety::OperatorGated,
5327 },
5328 _ => DelegatedReplaySafety::OperatorGated,
5329 }
5330}
5331
5332fn replay_safety_rank(safety: DelegatedReplaySafety) -> u8 {
5333 match safety {
5334 DelegatedReplaySafety::PureReadonly => 0,
5335 DelegatedReplaySafety::IdempotentWrite => 1,
5336 DelegatedReplaySafety::CheckpointResumable => 2,
5337 DelegatedReplaySafety::OperatorGated => 3,
5338 DelegatedReplaySafety::NonReplayableSideEffect => 4,
5339 }
5340}
5341
5342fn restart_resume_disposition(safety: DelegatedReplaySafety) -> DelegatedResumeDisposition {
5343 match safety {
5344 DelegatedReplaySafety::PureReadonly | DelegatedReplaySafety::IdempotentWrite => {
5345 DelegatedResumeDisposition::RestartFromBoundary
5346 }
5347 DelegatedReplaySafety::CheckpointResumable => {
5348 DelegatedResumeDisposition::ResumeFromCheckpoint
5349 }
5350 DelegatedReplaySafety::OperatorGated | DelegatedReplaySafety::NonReplayableSideEffect => {
5351 DelegatedResumeDisposition::OperatorInterventionRequired
5352 }
5353 }
5354}
5355
5356fn checkpoint_for_restart_recovery(
5357 checkpoint: &DelegatedTaskCheckpoint,
5358) -> DelegatedTaskCheckpoint {
5359 let mut updated = checkpoint.clone();
5360 updated.stage = DelegatedCheckpointStage::Blocked;
5361 updated.resume_disposition = restart_resume_disposition(updated.replay_safety);
5362 updated.note = Some("execution interrupted during restart".to_string());
5363 updated.updated_at = Utc::now();
5364 updated
5365}
5366
5367fn build_delegated_resume_state(
5368 task: &DelegatedTask,
5369 original_input: &str,
5370 partial_content: &str,
5371 partial_thinking: &str,
5372 completed_tool_calls: &[ToolCallRecord],
5373 iteration: u32,
5374) -> PausedExecutionState {
5375 PausedExecutionState {
5376 original_input: original_input.to_string(),
5377 system_prompt: None,
5378 history: Vec::new(),
5379 partial_content: partial_content.to_string(),
5380 partial_thinking: if partial_thinking.is_empty() {
5381 None
5382 } else {
5383 Some(partial_thinking.to_string())
5384 },
5385 completed_tool_calls: completed_tool_calls.to_vec(),
5386 iteration,
5387 source: RequestSource::Orchestrator,
5388 session_id: task.session_id.clone(),
5389 workspace_dir: task.workspace_dir.clone(),
5390 model_snapshot: None,
5391 paused_at: Utc::now(),
5392 }
5393}
5394
5395fn orchestrator_tool_call_from_record(record: &ToolCallRecord) -> OrchestratorToolCall {
5396 let input = serde_json::from_str(&record.arguments).unwrap_or_else(|_| json!({}));
5397 let (output, success) = match &record.result {
5398 crate::ToolResult::Success(value) => (
5399 serde_json::from_str(value).unwrap_or_else(|_| json!({ "result": value })),
5400 true,
5401 ),
5402 crate::ToolResult::Error(error) => (json!({ "error": error }), false),
5403 crate::ToolResult::Skipped(reason) => (json!({ "skipped": reason }), false),
5404 };
5405
5406 OrchestratorToolCall {
5407 tool_name: record.name.clone(),
5408 input,
5409 output,
5410 success,
5411 duration_ms: record.duration_ms,
5412 }
5413}
5414
5415fn pending_orchestrator_tool_call(name: &str, arguments: &str) -> OrchestratorToolCall {
5416 OrchestratorToolCall {
5417 tool_name: name.to_string(),
5418 input: serde_json::from_str(arguments).unwrap_or_else(|_| json!({})),
5419 output: json!({}),
5420 success: false,
5421 duration_ms: 0,
5422 }
5423}
5424
5425fn restart_blocked_reason_for_checkpoint(checkpoint: &DelegatedTaskCheckpoint) -> String {
5426 match checkpoint.resume_disposition {
5427 DelegatedResumeDisposition::ResumeFromCheckpoint => format!(
5428 "execution interrupted during restart; task can resume from checkpoint '{}'",
5429 checkpoint.safe_boundary_label
5430 ),
5431 DelegatedResumeDisposition::RestartFromBoundary => format!(
5432 "execution interrupted during restart; task can safely restart from boundary '{}'",
5433 checkpoint.safe_boundary_label
5434 ),
5435 DelegatedResumeDisposition::OperatorInterventionRequired => format!(
5436 "execution interrupted during restart; operator action required because boundary '{}' is {:?}",
5437 checkpoint.safe_boundary_label, checkpoint.replay_safety
5438 ),
5439 DelegatedResumeDisposition::NotApplicable => {
5440 "execution interrupted during restart".to_string()
5441 }
5442 }
5443}
5444
5445fn record_task_dispatch(task: &DelegatedTask, record: &SupervisorTaskRecord, run: &SupervisorRun) {
5446 let Some(session_id) = task.session_id.as_deref() else {
5447 return;
5448 };
5449 let Some(tracking_task_id) = task.tracking_task_id.as_deref() else {
5450 return;
5451 };
5452
5453 let manager = crate::get_global_task_manager();
5454 let task_status = match record.state {
5455 SupervisorTaskState::Blocked
5456 | SupervisorTaskState::PendingApproval
5457 | SupervisorTaskState::ReviewPending
5458 | SupervisorTaskState::TestPending => TaskStatus::Blocked,
5459 SupervisorTaskState::Running => TaskStatus::InProgress,
5460 SupervisorTaskState::Completed => TaskStatus::Completed,
5461 SupervisorTaskState::Cancelled | SupervisorTaskState::Failed => TaskStatus::Cancelled,
5462 _ => TaskStatus::NotStarted,
5463 };
5464 let _ = manager.update_task_status(session_id, tracking_task_id, task_status);
5465 let _ = manager.set_task_background_job(
5466 session_id,
5467 tracking_task_id,
5468 Some(TaskBackgroundJob::new(
5469 background_status_for_state(record.state),
5470 Some(task.id.clone()),
5471 Some(background_message_for_record(record)),
5472 )),
5473 );
5474 let _ = manager.record_memory_event(
5475 session_id,
5476 tracking_task_id,
5477 crate::tasks::TaskMemoryEvent::new(
5478 crate::tasks::TaskMemoryPhase::Delegated,
5479 format!("Delegated to agent {} as {:?}", task.agent_id, task.role),
5480 task.directive_id.as_ref().map(|_| "directive".to_string()),
5481 Some("handoff".to_string()),
5482 None,
5483 ),
5484 );
5485 merge_task_metadata(
5486 manager,
5487 session_id,
5488 tracking_task_id,
5489 json!({
5490 "delegation": {
5491 "run_id": task.run_id,
5492 "orchestrator_task_id": task.id,
5493 "agent_id": task.agent_id,
5494 "directive_id": task.directive_id,
5495 "role": task.role,
5496 "state": format!("{:?}", record.state).to_lowercase(),
5497 "approval_state": format!("{:?}", record.approval.state).to_lowercase(),
5498 "approval": {
5499 "scope": record.approval.scope.map(|value| format!("{:?}", value).to_lowercase()),
5500 "requested_at": record.approval.requested_at,
5501 "decided_at": record.approval.decided_at,
5502 "decided_by": record.approval.decided_by,
5503 "note": record.approval.note,
5504 "active_request": record.approval.active_request.as_ref(),
5505 "latest_decision": record.approval.latest_decision(),
5506 "policy": {
5507 "pre_execution": {
5508 "required": record.approval.policy.pre_execution.required,
5509 "allowed_deciders": record.approval.policy.pre_execution.allowed_deciders,
5510 },
5511 "review": {
5512 "required": record.approval.policy.review.required,
5513 "allowed_deciders": record.approval.policy.review.allowed_deciders,
5514 },
5515 "test_validation": {
5516 "required": record.approval.policy.test_validation.required,
5517 "allowed_deciders": record.approval.policy.test_validation.allowed_deciders,
5518 }
5519 }
5520 },
5521 "dependencies": task.depends_on,
5522 "environment": {
5523 "id": record.environment.id,
5524 "mode": format!("{:?}", record.environment.execution_mode).to_lowercase(),
5525 "root_dir": record.environment.root_dir,
5526 "write_access": record.environment.write_access,
5527 "state": format!("{:?}", record.environment.state).to_lowercase(),
5528 "health": format!("{:?}", record.environment.health).to_lowercase(),
5529 "cleanup_policy": format!("{:?}", record.environment.cleanup_policy).to_lowercase(),
5530 "recovery_status": format!("{:?}", record.environment.recovery_status).to_lowercase(),
5531 "recovery_action": record.environment.recovery_action.map(|value| format!("{:?}", value).to_lowercase()),
5532 "failure": record.environment.failure.as_ref(),
5533 "branch_name": record.environment.branch_name,
5534 "worktree_path": record.environment.worktree_path,
5535 "remote_url": record.environment.remote_url,
5536 },
5537 "planning_only": task.planning_only,
5538 "reviewer_required": task.reviewer_required,
5539 "test_required": task.test_required,
5540 "memory_tags": task.memory_tags,
5541 "run_status": format!("{:?}", run.status).to_lowercase(),
5542 }
5543 }),
5544 );
5545}
5546
5547fn record_task_progress(task: &DelegatedTask, record: &SupervisorTaskRecord, run: &SupervisorRun) {
5548 let Some(session_id) = task.session_id.as_deref() else {
5549 return;
5550 };
5551 let Some(tracking_task_id) = task.tracking_task_id.as_deref() else {
5552 return;
5553 };
5554
5555 let manager = crate::get_global_task_manager();
5556 let task_status = match record.state {
5557 SupervisorTaskState::Blocked
5558 | SupervisorTaskState::PendingApproval
5559 | SupervisorTaskState::ReviewPending
5560 | SupervisorTaskState::TestPending => TaskStatus::Blocked,
5561 SupervisorTaskState::Running => TaskStatus::InProgress,
5562 SupervisorTaskState::Completed => TaskStatus::Completed,
5563 SupervisorTaskState::Cancelled | SupervisorTaskState::Failed => TaskStatus::Cancelled,
5564 _ => TaskStatus::NotStarted,
5565 };
5566 let _ = manager.update_task_status(session_id, tracking_task_id, task_status);
5567 let _ = manager.set_task_background_job(
5568 session_id,
5569 tracking_task_id,
5570 Some(TaskBackgroundJob::new(
5571 background_status_for_state(record.state),
5572 Some(task.id.clone()),
5573 Some(background_message_for_record(record)),
5574 )),
5575 );
5576 merge_task_metadata(
5577 manager,
5578 session_id,
5579 tracking_task_id,
5580 json!({
5581 "delegation": {
5582 "state": format!("{:?}", record.state).to_lowercase(),
5583 "run_status": format!("{:?}", run.status).to_lowercase(),
5584 "local_execution": record.local_execution.as_ref(),
5585 "remote_execution": record.remote_execution.as_ref(),
5586 }
5587 }),
5588 );
5589}
5590
5591fn record_task_completion(
5592 task: &DelegatedTask,
5593 task_result: &TaskResult,
5594 memory_file_path: Option<&Path>,
5595 record: &SupervisorTaskRecord,
5596 run: &SupervisorRun,
5597) {
5598 let Some(session_id) = task.session_id.as_deref() else {
5599 return;
5600 };
5601 let Some(tracking_task_id) = task.tracking_task_id.as_deref() else {
5602 return;
5603 };
5604
5605 let manager = crate::get_global_task_manager();
5606 let task_status = match record.state {
5607 SupervisorTaskState::Blocked
5608 | SupervisorTaskState::PendingApproval
5609 | SupervisorTaskState::ReviewPending
5610 | SupervisorTaskState::TestPending => TaskStatus::Blocked,
5611 SupervisorTaskState::Completed => TaskStatus::Completed,
5612 SupervisorTaskState::Cancelled | SupervisorTaskState::Failed => TaskStatus::Cancelled,
5613 SupervisorTaskState::Running => TaskStatus::InProgress,
5614 _ => TaskStatus::NotStarted,
5615 };
5616 let _ = manager.update_task_status(session_id, tracking_task_id, task_status);
5617 let _ = manager.set_task_background_job(
5618 session_id,
5619 tracking_task_id,
5620 Some(TaskBackgroundJob::new(
5621 background_status_for_state(record.state),
5622 Some(task.id.clone()),
5623 Some(background_message_for_record(record)),
5624 )),
5625 );
5626 let _ = manager.record_memory_event(
5627 session_id,
5628 tracking_task_id,
5629 crate::tasks::TaskMemoryEvent::new(
5630 if matches!(record.state, SupervisorTaskState::Completed) {
5631 crate::tasks::TaskMemoryPhase::Promoted
5632 } else {
5633 crate::tasks::TaskMemoryPhase::Blocked
5634 },
5635 if matches!(record.state, SupervisorTaskState::Completed) {
5636 format!("Delegated work completed by {}", task.agent_id)
5637 } else {
5638 format!(
5639 "Delegated work waiting on {:?} for {}",
5640 record.state, task.agent_id
5641 )
5642 },
5643 Some(
5644 if task.directive_id.is_some() {
5645 "directive"
5646 } else {
5647 "session"
5648 }
5649 .to_string(),
5650 ),
5651 Some(
5652 if matches!(record.state, SupervisorTaskState::Completed) {
5653 "handoff"
5654 } else {
5655 "blocker"
5656 }
5657 .to_string(),
5658 ),
5659 memory_file_path.map(|path| path.display().to_string()),
5660 ),
5661 );
5662 merge_task_metadata(
5663 manager,
5664 session_id,
5665 tracking_task_id,
5666 json!({
5667 "delegation": {
5668 "run_id": task.run_id,
5669 "orchestrator_task_id": task.id,
5670 "agent_id": task.agent_id,
5671 "directive_id": task.directive_id,
5672 "role": task.role,
5673 "state": format!("{:?}", record.state).to_lowercase(),
5674 "approval_state": format!("{:?}", record.approval.state).to_lowercase(),
5675 "approval": {
5676 "scope": record.approval.scope.map(|value| format!("{:?}", value).to_lowercase()),
5677 "requested_at": record.approval.requested_at,
5678 "decided_at": record.approval.decided_at,
5679 "decided_by": record.approval.decided_by,
5680 "note": record.approval.note,
5681 "active_request": record.approval.active_request.as_ref(),
5682 "latest_decision": record.approval.latest_decision(),
5683 "policy": {
5684 "pre_execution": {
5685 "required": record.approval.policy.pre_execution.required,
5686 "allowed_deciders": record.approval.policy.pre_execution.allowed_deciders,
5687 },
5688 "review": {
5689 "required": record.approval.policy.review.required,
5690 "allowed_deciders": record.approval.policy.review.allowed_deciders,
5691 },
5692 "test_validation": {
5693 "required": record.approval.policy.test_validation.required,
5694 "allowed_deciders": record.approval.policy.test_validation.allowed_deciders,
5695 }
5696 }
5697 },
5698 "last_output": task_result.output,
5699 "summary": task_result.summary,
5700 "attempts": record.attempts,
5701 "memory_file_path": memory_file_path.map(|path| path.display().to_string()),
5702 "tool_calls": task_result.tool_calls,
5703 "artifacts": task_result.artifacts,
5704 "environment": {
5705 "id": record.environment.id,
5706 "mode": format!("{:?}", record.environment.execution_mode).to_lowercase(),
5707 "state": format!("{:?}", record.environment.state).to_lowercase(),
5708 "health": format!("{:?}", record.environment.health).to_lowercase(),
5709 "recovery_status": format!("{:?}", record.environment.recovery_status).to_lowercase(),
5710 "recovery_action": record.environment.recovery_action.map(|value| format!("{:?}", value).to_lowercase()),
5711 "failure": record.environment.failure.as_ref(),
5712 "cleanup_result": record.environment.cleanup_result.as_ref(),
5713 "branch_name": record.environment.branch_name,
5714 "worktree_path": record.environment.worktree_path,
5715 "remote_url": record.environment.remote_url,
5716 },
5717 "run_status": format!("{:?}", run.status).to_lowercase(),
5718 }
5719 }),
5720 );
5721}
5722
5723fn background_status_for_state(state: SupervisorTaskState) -> TaskBackgroundStatus {
5724 match state {
5725 SupervisorTaskState::Queued => TaskBackgroundStatus::Queued,
5726 SupervisorTaskState::Blocked => TaskBackgroundStatus::Blocked,
5727 SupervisorTaskState::PendingApproval
5728 | SupervisorTaskState::ReviewPending
5729 | SupervisorTaskState::TestPending => TaskBackgroundStatus::AwaitingApproval,
5730 SupervisorTaskState::Running => TaskBackgroundStatus::Running,
5731 SupervisorTaskState::Completed => TaskBackgroundStatus::Succeeded,
5732 SupervisorTaskState::Failed => TaskBackgroundStatus::Failed,
5733 SupervisorTaskState::Cancelled => TaskBackgroundStatus::Cancelled,
5734 }
5735}
5736
5737fn background_message_for_record(record: &SupervisorTaskRecord) -> String {
5738 if let Some(remote) = remote_background_message(record)
5739 && matches!(
5740 record.state,
5741 SupervisorTaskState::Running | SupervisorTaskState::Blocked
5742 )
5743 {
5744 return remote;
5745 }
5746 if let Some(local) = local_background_message(record)
5747 && matches!(
5748 record.state,
5749 SupervisorTaskState::Running | SupervisorTaskState::Blocked
5750 )
5751 {
5752 return local;
5753 }
5754 match record.state {
5755 SupervisorTaskState::Queued => "Queued for execution".to_string(),
5756 SupervisorTaskState::Blocked => {
5757 if record.blocked_reasons.is_empty() {
5758 "Blocked".to_string()
5759 } else {
5760 format!("Blocked: {}", record.blocked_reasons.join("; "))
5761 }
5762 }
5763 SupervisorTaskState::PendingApproval => "Awaiting supervisor approval".to_string(),
5764 SupervisorTaskState::Running => "Running".to_string(),
5765 SupervisorTaskState::ReviewPending => "Awaiting review approval".to_string(),
5766 SupervisorTaskState::TestPending => "Awaiting test validation".to_string(),
5767 SupervisorTaskState::Completed => "Completed".to_string(),
5768 SupervisorTaskState::Failed => "Failed".to_string(),
5769 SupervisorTaskState::Cancelled => "Cancelled".to_string(),
5770 }
5771}
5772
5773fn local_background_message(record: &SupervisorTaskRecord) -> Option<String> {
5774 let local = record.local_execution.as_ref()?;
5775 let mut parts = vec![format!("Local {}", local.status)];
5776 if let Some(reason) = local.status_reason.as_deref() {
5777 parts.push(reason.to_string());
5778 }
5779 if let Some(progress) = local.progress.as_ref() {
5780 parts.push(format!("phase {:?}", progress.phase).to_lowercase());
5781 if let Some(waiting_reason) = progress.waiting_reason {
5782 parts.push(format!("waiting {:?}", waiting_reason).to_lowercase());
5783 }
5784 if let Some(percent) = progress.percent {
5785 parts.push(format!("{percent}%"));
5786 }
5787 if let Some(stage) = progress.stage.as_deref() {
5788 parts.push(stage.to_string());
5789 }
5790 if let Some(tool) = progress.current_tool_name.as_deref() {
5791 parts.push(format!("tool {tool}"));
5792 }
5793 if progress.completed_tool_call_count > 0 {
5794 parts.push(format!(
5795 "{} tool call(s)",
5796 progress.completed_tool_call_count
5797 ));
5798 }
5799 if let Some(token_usage) = progress.token_usage.as_ref() {
5800 if let (Some(estimated_tokens), Some(limit), Some(percentage)) = (
5801 token_usage.estimated_tokens,
5802 token_usage.limit,
5803 token_usage.percentage,
5804 ) {
5805 parts.push(format!("tokens {estimated_tokens}/{limit} ({percentage}%)"));
5806 } else if let Some(total_tokens) = token_usage.total_tokens {
5807 parts.push(format!("tokens {total_tokens}"));
5808 }
5809 }
5810 if let Some(environment) = progress.environment.as_ref() {
5811 parts.push(format!("env {:?}", environment.state).to_lowercase());
5812 }
5813 if let Some(message) = progress.message.as_deref() {
5814 parts.push(message.to_string());
5815 }
5816 }
5817 Some(parts.join(" • "))
5818}
5819
5820fn remote_background_message(record: &SupervisorTaskRecord) -> Option<String> {
5821 let remote = record.remote_execution.as_ref()?;
5822 let mut parts = vec![format!("Remote {}", remote.status)];
5823 if let Some(reason) = remote.status_reason.as_deref() {
5824 parts.push(reason.to_string());
5825 }
5826 if let Some(progress) = remote.progress.as_ref() {
5827 if let Some(percent) = progress.percent {
5828 parts.push(format!("{percent}%"));
5829 }
5830 if let Some(stage) = progress.stage.as_deref() {
5831 parts.push(stage.to_string());
5832 }
5833 }
5834 Some(parts.join(" • "))
5835}
5836
5837fn a2a_status_label(status: A2ATaskStatus) -> String {
5838 match status {
5839 A2ATaskStatus::Pending => "pending",
5840 A2ATaskStatus::Blocked => "blocked",
5841 A2ATaskStatus::Running => "running",
5842 A2ATaskStatus::Completed => "completed",
5843 A2ATaskStatus::Cancelled => "cancelled",
5844 A2ATaskStatus::Failed => "failed",
5845 }
5846 .to_string()
5847}
5848
5849fn task_terminal_hint_for_a2a_status(status: A2ATaskStatus) -> Option<TaskTerminalStateHint> {
5850 match status {
5851 A2ATaskStatus::Completed => Some(TaskTerminalStateHint::Completed),
5852 A2ATaskStatus::Cancelled => Some(TaskTerminalStateHint::Cancelled),
5853 A2ATaskStatus::Failed => Some(TaskTerminalStateHint::Failed),
5854 A2ATaskStatus::Blocked => Some(TaskTerminalStateHint::Blocked),
5855 _ => None,
5856 }
5857}
5858
5859fn progress_from_remote(
5860 progress: Option<&A2ARemoteTaskProgress>,
5861) -> Option<RemoteExecutionProgress> {
5862 progress.map(|progress| RemoteExecutionProgress {
5863 stage: progress.stage.clone(),
5864 message: progress.message.clone(),
5865 percent: progress.percent,
5866 updated_at: progress.updated_at,
5867 })
5868}
5869
5870#[derive(Debug, Clone, Default)]
5871struct LocalExecutionTelemetryContext {
5872 iteration: u32,
5873 current_tool_name: Option<String>,
5874 last_completed_tool_name: Option<String>,
5875 last_completed_tool_duration_ms: Option<u64>,
5876 completed_tool_call_count: usize,
5877 partial_content_chars: usize,
5878 partial_thinking_chars: usize,
5879 has_partial_content: bool,
5880 has_partial_thinking: bool,
5881 token_usage: Option<LocalExecutionTokenUsageSnapshot>,
5882 environment: Option<LocalExecutionEnvironmentSnapshot>,
5883}
5884
5885fn local_execution_record_for_start() -> LocalExecutionRecord {
5886 let now = Utc::now();
5887 let context = LocalExecutionTelemetryContext::default();
5888 LocalExecutionRecord {
5889 status: "running".to_string(),
5890 status_reason: None,
5891 progress: Some(LocalExecutionProgress {
5892 phase: LocalExecutionPhase::Running,
5893 waiting_reason: None,
5894 stage: Some("starting".to_string()),
5895 message: Some("Delegated task dispatched to local agent".to_string()),
5896 percent: None,
5897 iteration: context.iteration,
5898 current_tool_name: None,
5899 last_completed_tool_name: None,
5900 last_completed_tool_duration_ms: None,
5901 completed_tool_call_count: context.completed_tool_call_count,
5902 has_partial_content: context.has_partial_content,
5903 partial_content_chars: context.partial_content_chars,
5904 has_partial_thinking: context.has_partial_thinking,
5905 partial_thinking_chars: context.partial_thinking_chars,
5906 token_usage: None,
5907 environment: None,
5908 updated_at: now,
5909 }),
5910 last_synced_at: now,
5911 }
5912}
5913
5914fn local_execution_record_for_terminal(
5915 task_result: &TaskResult,
5916 state: SupervisorTaskState,
5917 previous: Option<&LocalExecutionRecord>,
5918) -> LocalExecutionRecord {
5919 let now = Utc::now();
5920 let previous_progress = previous.and_then(|local| local.progress.as_ref());
5921 LocalExecutionRecord {
5922 status: local_execution_status_label(state).to_string(),
5923 status_reason: if task_result.success {
5924 None
5925 } else {
5926 Some(task_result.output.clone())
5927 },
5928 progress: Some(LocalExecutionProgress {
5929 phase: local_execution_phase_for_terminal_state(state),
5930 waiting_reason: None,
5931 stage: Some(local_execution_stage_for_terminal_state(state).to_string()),
5932 message: Some(if task_result.success {
5933 format!(
5934 "Local delegated task completed after {} tool call(s)",
5935 task_result.tool_calls.len()
5936 )
5937 } else {
5938 format!(
5939 "Local delegated task ended as {}: {}",
5940 local_execution_status_label(state),
5941 task_result.output
5942 )
5943 }),
5944 percent: Some(if matches!(state, SupervisorTaskState::Completed) {
5945 100
5946 } else {
5947 previous_progress
5948 .and_then(|progress| progress.percent)
5949 .unwrap_or(100)
5950 }),
5951 iteration: previous_progress
5952 .map(|progress| progress.iteration)
5953 .unwrap_or_default(),
5954 current_tool_name: None,
5955 last_completed_tool_name: previous_progress
5956 .and_then(|progress| progress.last_completed_tool_name.clone()),
5957 last_completed_tool_duration_ms: previous_progress
5958 .and_then(|progress| progress.last_completed_tool_duration_ms),
5959 completed_tool_call_count: task_result.tool_calls.len(),
5960 has_partial_content: previous_progress
5961 .map(|progress| progress.has_partial_content)
5962 .unwrap_or(false),
5963 partial_content_chars: previous_progress
5964 .map(|progress| progress.partial_content_chars)
5965 .unwrap_or_default(),
5966 has_partial_thinking: previous_progress
5967 .map(|progress| progress.has_partial_thinking)
5968 .unwrap_or(false),
5969 partial_thinking_chars: previous_progress
5970 .map(|progress| progress.partial_thinking_chars)
5971 .unwrap_or_default(),
5972 token_usage: previous_progress.and_then(|progress| progress.token_usage.clone()),
5973 environment: previous_progress.and_then(|progress| progress.environment.clone()),
5974 updated_at: now,
5975 }),
5976 last_synced_at: now,
5977 }
5978}
5979
5980fn local_execution_progress_from_chunk(
5981 chunk: &StreamChunk,
5982 context: &LocalExecutionTelemetryContext,
5983) -> Option<LocalExecutionProgress> {
5984 let now = Utc::now();
5985 match chunk {
5986 StreamChunk::Status { message } => Some(LocalExecutionProgress {
5987 phase: LocalExecutionPhase::Running,
5988 waiting_reason: None,
5989 stage: Some("status".to_string()),
5990 message: Some(message.clone()),
5991 percent: None,
5992 iteration: context.iteration,
5993 current_tool_name: context.current_tool_name.clone(),
5994 last_completed_tool_name: context.last_completed_tool_name.clone(),
5995 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
5996 completed_tool_call_count: context.completed_tool_call_count,
5997 has_partial_content: context.has_partial_content,
5998 partial_content_chars: context.partial_content_chars,
5999 has_partial_thinking: context.has_partial_thinking,
6000 partial_thinking_chars: context.partial_thinking_chars,
6001 token_usage: context.token_usage.clone(),
6002 environment: context.environment.clone(),
6003 updated_at: now,
6004 }),
6005 StreamChunk::AgentLoopIteration { iteration } => Some(LocalExecutionProgress {
6006 phase: LocalExecutionPhase::Running,
6007 waiting_reason: None,
6008 stage: Some("agent_loop".to_string()),
6009 message: Some(format!("Agent loop iteration {}", iteration + 1)),
6010 percent: None,
6011 iteration: *iteration,
6012 current_tool_name: None,
6013 last_completed_tool_name: context.last_completed_tool_name.clone(),
6014 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6015 completed_tool_call_count: context.completed_tool_call_count,
6016 has_partial_content: context.has_partial_content,
6017 partial_content_chars: context.partial_content_chars,
6018 has_partial_thinking: context.has_partial_thinking,
6019 partial_thinking_chars: context.partial_thinking_chars,
6020 token_usage: context.token_usage.clone(),
6021 environment: context.environment.clone(),
6022 updated_at: now,
6023 }),
6024 StreamChunk::ToolCallStart { name, .. } => Some(LocalExecutionProgress {
6025 phase: LocalExecutionPhase::Running,
6026 waiting_reason: None,
6027 stage: Some("executing_tools".to_string()),
6028 message: Some(format!("Running tool '{name}'")),
6029 percent: None,
6030 iteration: context.iteration,
6031 current_tool_name: Some(name.clone()),
6032 last_completed_tool_name: context.last_completed_tool_name.clone(),
6033 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6034 completed_tool_call_count: context.completed_tool_call_count,
6035 has_partial_content: context.has_partial_content,
6036 partial_content_chars: context.partial_content_chars,
6037 has_partial_thinking: context.has_partial_thinking,
6038 partial_thinking_chars: context.partial_thinking_chars,
6039 token_usage: context.token_usage.clone(),
6040 environment: context.environment.clone(),
6041 updated_at: now,
6042 }),
6043 StreamChunk::ToolCallResult {
6044 name,
6045 success,
6046 duration_ms,
6047 ..
6048 } => Some(LocalExecutionProgress {
6049 phase: LocalExecutionPhase::Running,
6050 waiting_reason: None,
6051 stage: Some("executing_tools".to_string()),
6052 message: Some(if *success {
6053 format!("Tool '{name}' completed successfully")
6054 } else {
6055 format!("Tool '{name}' failed")
6056 }),
6057 percent: None,
6058 iteration: context.iteration,
6059 current_tool_name: None,
6060 last_completed_tool_name: Some(name.clone()),
6061 last_completed_tool_duration_ms: Some(*duration_ms),
6062 completed_tool_call_count: context.completed_tool_call_count,
6063 has_partial_content: context.has_partial_content,
6064 partial_content_chars: context.partial_content_chars,
6065 has_partial_thinking: context.has_partial_thinking,
6066 partial_thinking_chars: context.partial_thinking_chars,
6067 token_usage: context.token_usage.clone(),
6068 environment: context.environment.clone(),
6069 updated_at: now,
6070 }),
6071 StreamChunk::ShellLifecycle { state, command, .. } => Some(LocalExecutionProgress {
6072 phase: if matches!(
6073 state,
6074 crate::streaming::ShellProcessState::Started
6075 | crate::streaming::ShellProcessState::Paused
6076 | crate::streaming::ShellProcessState::Resumed
6077 ) {
6078 LocalExecutionPhase::Waiting
6079 } else {
6080 LocalExecutionPhase::Running
6081 },
6082 waiting_reason: if matches!(
6083 state,
6084 crate::streaming::ShellProcessState::Started
6085 | crate::streaming::ShellProcessState::Paused
6086 | crate::streaming::ShellProcessState::Resumed
6087 ) {
6088 Some(LocalExecutionWaitingReason::ShellProcess)
6089 } else {
6090 None
6091 },
6092 stage: Some("shell".to_string()),
6093 message: Some(format!("Shell {:?}: {command}", state).to_lowercase()),
6094 percent: None,
6095 iteration: context.iteration,
6096 current_tool_name: Some("shell".to_string()),
6097 last_completed_tool_name: context.last_completed_tool_name.clone(),
6098 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6099 completed_tool_call_count: context.completed_tool_call_count,
6100 has_partial_content: context.has_partial_content,
6101 partial_content_chars: context.partial_content_chars,
6102 has_partial_thinking: context.has_partial_thinking,
6103 partial_thinking_chars: context.partial_thinking_chars,
6104 token_usage: context.token_usage.clone(),
6105 environment: context.environment.clone(),
6106 updated_at: now,
6107 }),
6108 StreamChunk::TokenUsageUpdate {
6109 estimated,
6110 limit,
6111 percentage,
6112 status,
6113 estimated_cost,
6114 } => Some(LocalExecutionProgress {
6115 phase: LocalExecutionPhase::Running,
6116 waiting_reason: None,
6117 stage: Some("token_usage".to_string()),
6118 message: Some(format!(
6119 "Estimated token usage {estimated}/{limit} ({percentage}%)"
6120 )),
6121 percent: Some(*percentage),
6122 iteration: context.iteration,
6123 current_tool_name: context.current_tool_name.clone(),
6124 last_completed_tool_name: context.last_completed_tool_name.clone(),
6125 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6126 completed_tool_call_count: context.completed_tool_call_count,
6127 has_partial_content: context.has_partial_content,
6128 partial_content_chars: context.partial_content_chars,
6129 has_partial_thinking: context.has_partial_thinking,
6130 partial_thinking_chars: context.partial_thinking_chars,
6131 token_usage: Some(LocalExecutionTokenUsageSnapshot {
6132 estimated_tokens: Some(*estimated),
6133 limit: Some(*limit),
6134 percentage: Some(*percentage),
6135 status: Some(token_usage_status_label(*status).to_string()),
6136 estimated_cost_usd: Some(*estimated_cost),
6137 input_tokens: context
6138 .token_usage
6139 .as_ref()
6140 .and_then(|usage| usage.input_tokens),
6141 output_tokens: context
6142 .token_usage
6143 .as_ref()
6144 .and_then(|usage| usage.output_tokens),
6145 total_tokens: context
6146 .token_usage
6147 .as_ref()
6148 .and_then(|usage| usage.total_tokens),
6149 model: context
6150 .token_usage
6151 .as_ref()
6152 .and_then(|usage| usage.model.clone()),
6153 provider: context
6154 .token_usage
6155 .as_ref()
6156 .and_then(|usage| usage.provider.clone()),
6157 }),
6158 environment: context.environment.clone(),
6159 updated_at: now,
6160 }),
6161 StreamChunk::ReflectionStarted { reason } => Some(LocalExecutionProgress {
6162 phase: LocalExecutionPhase::Waiting,
6163 waiting_reason: Some(LocalExecutionWaitingReason::Reflection),
6164 stage: Some("reflection".to_string()),
6165 message: Some(reason.clone()),
6166 percent: None,
6167 iteration: context.iteration,
6168 current_tool_name: context.current_tool_name.clone(),
6169 last_completed_tool_name: context.last_completed_tool_name.clone(),
6170 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6171 completed_tool_call_count: context.completed_tool_call_count,
6172 has_partial_content: context.has_partial_content,
6173 partial_content_chars: context.partial_content_chars,
6174 has_partial_thinking: context.has_partial_thinking,
6175 partial_thinking_chars: context.partial_thinking_chars,
6176 token_usage: context.token_usage.clone(),
6177 environment: context.environment.clone(),
6178 updated_at: now,
6179 }),
6180 StreamChunk::ReflectionComplete { summary, .. } => Some(LocalExecutionProgress {
6181 phase: LocalExecutionPhase::Running,
6182 waiting_reason: None,
6183 stage: Some("reflection".to_string()),
6184 message: Some(summary.clone()),
6185 percent: None,
6186 iteration: context.iteration,
6187 current_tool_name: context.current_tool_name.clone(),
6188 last_completed_tool_name: context.last_completed_tool_name.clone(),
6189 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6190 completed_tool_call_count: context.completed_tool_call_count,
6191 has_partial_content: context.has_partial_content,
6192 partial_content_chars: context.partial_content_chars,
6193 has_partial_thinking: context.has_partial_thinking,
6194 partial_thinking_chars: context.partial_thinking_chars,
6195 token_usage: context.token_usage.clone(),
6196 environment: context.environment.clone(),
6197 updated_at: now,
6198 }),
6199 StreamChunk::ToolConfirmationRequired { tool_name, .. } => Some(LocalExecutionProgress {
6200 phase: LocalExecutionPhase::Waiting,
6201 waiting_reason: Some(LocalExecutionWaitingReason::ToolConfirmation),
6202 stage: Some("tool_confirmation".to_string()),
6203 message: Some(format!(
6204 "Waiting for confirmation before running '{tool_name}'"
6205 )),
6206 percent: None,
6207 iteration: context.iteration,
6208 current_tool_name: Some(tool_name.clone()),
6209 last_completed_tool_name: context.last_completed_tool_name.clone(),
6210 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6211 completed_tool_call_count: context.completed_tool_call_count,
6212 has_partial_content: context.has_partial_content,
6213 partial_content_chars: context.partial_content_chars,
6214 has_partial_thinking: context.has_partial_thinking,
6215 partial_thinking_chars: context.partial_thinking_chars,
6216 token_usage: context.token_usage.clone(),
6217 environment: context.environment.clone(),
6218 updated_at: now,
6219 }),
6220 StreamChunk::ToolBlocked { tool_name, reason } => Some(LocalExecutionProgress {
6221 phase: LocalExecutionPhase::Blocked,
6222 waiting_reason: None,
6223 stage: Some("tool_blocked".to_string()),
6224 message: Some(format!("Tool '{tool_name}' blocked: {reason}")),
6225 percent: None,
6226 iteration: context.iteration,
6227 current_tool_name: Some(tool_name.clone()),
6228 last_completed_tool_name: context.last_completed_tool_name.clone(),
6229 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6230 completed_tool_call_count: context.completed_tool_call_count,
6231 has_partial_content: context.has_partial_content,
6232 partial_content_chars: context.partial_content_chars,
6233 has_partial_thinking: context.has_partial_thinking,
6234 partial_thinking_chars: context.partial_thinking_chars,
6235 token_usage: context.token_usage.clone(),
6236 environment: context.environment.clone(),
6237 updated_at: now,
6238 }),
6239 StreamChunk::Done(usage) => Some(LocalExecutionProgress {
6240 phase: LocalExecutionPhase::Running,
6241 waiting_reason: None,
6242 stage: Some("finishing".to_string()),
6243 message: Some("Local delegated task finished streaming output".to_string()),
6244 percent: Some(100),
6245 iteration: context.iteration,
6246 current_tool_name: None,
6247 last_completed_tool_name: context.last_completed_tool_name.clone(),
6248 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6249 completed_tool_call_count: context.completed_tool_call_count,
6250 has_partial_content: context.has_partial_content,
6251 partial_content_chars: context.partial_content_chars,
6252 has_partial_thinking: context.has_partial_thinking,
6253 partial_thinking_chars: context.partial_thinking_chars,
6254 token_usage: Some(token_usage_snapshot_from_done(
6255 usage.as_ref(),
6256 context.token_usage.as_ref(),
6257 )),
6258 environment: context.environment.clone(),
6259 updated_at: now,
6260 }),
6261 StreamChunk::Paused => Some(LocalExecutionProgress {
6262 phase: LocalExecutionPhase::Blocked,
6263 waiting_reason: None,
6264 stage: Some("paused".to_string()),
6265 message: Some(
6266 "Execution paused by operator; resumable checkpoint preserved".to_string(),
6267 ),
6268 percent: None,
6269 iteration: context.iteration,
6270 current_tool_name: None,
6271 last_completed_tool_name: context.last_completed_tool_name.clone(),
6272 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6273 completed_tool_call_count: context.completed_tool_call_count,
6274 has_partial_content: context.has_partial_content,
6275 partial_content_chars: context.partial_content_chars,
6276 has_partial_thinking: context.has_partial_thinking,
6277 partial_thinking_chars: context.partial_thinking_chars,
6278 token_usage: context.token_usage.clone(),
6279 environment: context.environment.clone(),
6280 updated_at: now,
6281 }),
6282 StreamChunk::Cancelled => Some(LocalExecutionProgress {
6283 phase: LocalExecutionPhase::Cancelled,
6284 waiting_reason: None,
6285 stage: Some("cancelled".to_string()),
6286 message: Some("Execution cancelled by operator".to_string()),
6287 percent: None,
6288 iteration: context.iteration,
6289 current_tool_name: None,
6290 last_completed_tool_name: context.last_completed_tool_name.clone(),
6291 last_completed_tool_duration_ms: context.last_completed_tool_duration_ms,
6292 completed_tool_call_count: context.completed_tool_call_count,
6293 has_partial_content: context.has_partial_content,
6294 partial_content_chars: context.partial_content_chars,
6295 has_partial_thinking: context.has_partial_thinking,
6296 partial_thinking_chars: context.partial_thinking_chars,
6297 token_usage: context.token_usage.clone(),
6298 environment: context.environment.clone(),
6299 updated_at: now,
6300 }),
6301 _ => None,
6302 }
6303}
6304
6305fn token_usage_snapshot_from_done(
6306 usage: Option<&crate::llm_provider::TokenUsage>,
6307 previous: Option<&LocalExecutionTokenUsageSnapshot>,
6308) -> LocalExecutionTokenUsageSnapshot {
6309 LocalExecutionTokenUsageSnapshot {
6310 estimated_tokens: previous.and_then(|usage| usage.estimated_tokens),
6311 limit: previous.and_then(|usage| usage.limit),
6312 percentage: previous.and_then(|usage| usage.percentage),
6313 status: previous.and_then(|usage| usage.status.clone()),
6314 estimated_cost_usd: usage
6315 .and_then(|usage| usage.estimated_cost_usd)
6316 .or_else(|| previous.and_then(|usage| usage.estimated_cost_usd)),
6317 input_tokens: usage.map(|usage| usage.input_tokens),
6318 output_tokens: usage.map(|usage| usage.output_tokens),
6319 total_tokens: usage.map(|usage| usage.total_tokens),
6320 model: usage
6321 .and_then(|usage| usage.model.clone())
6322 .or_else(|| previous.and_then(|usage| usage.model.clone())),
6323 provider: usage
6324 .and_then(|usage| usage.provider.clone())
6325 .or_else(|| previous.and_then(|usage| usage.provider.clone())),
6326 }
6327}
6328
6329fn token_usage_status_label(status: crate::streaming::TokenUsageStatus) -> &'static str {
6330 match status {
6331 crate::streaming::TokenUsageStatus::Green => "green",
6332 crate::streaming::TokenUsageStatus::Yellow => "yellow",
6333 crate::streaming::TokenUsageStatus::Red => "red",
6334 }
6335}
6336
6337fn environment_snapshot_from_execution(
6338 environment: &ExecutionEnvironment,
6339) -> LocalExecutionEnvironmentSnapshot {
6340 LocalExecutionEnvironmentSnapshot {
6341 state: environment.state,
6342 health: environment.health,
6343 recovery_status: environment.recovery_status,
6344 updated_at: Utc::now(),
6345 }
6346}
6347
6348fn local_execution_status_label(state: SupervisorTaskState) -> &'static str {
6349 match state {
6350 SupervisorTaskState::Queued => "queued",
6351 SupervisorTaskState::PendingApproval => "pending_approval",
6352 SupervisorTaskState::Running => "running",
6353 SupervisorTaskState::ReviewPending => "review_pending",
6354 SupervisorTaskState::TestPending => "test_pending",
6355 SupervisorTaskState::Completed => "completed",
6356 SupervisorTaskState::Failed => "failed",
6357 SupervisorTaskState::Blocked => "blocked",
6358 SupervisorTaskState::Cancelled => "cancelled",
6359 }
6360}
6361
6362fn local_execution_phase_for_terminal_state(state: SupervisorTaskState) -> LocalExecutionPhase {
6363 match state {
6364 SupervisorTaskState::Completed => LocalExecutionPhase::Completed,
6365 SupervisorTaskState::Cancelled => LocalExecutionPhase::Cancelled,
6366 SupervisorTaskState::Blocked => LocalExecutionPhase::Blocked,
6367 SupervisorTaskState::Failed => LocalExecutionPhase::Failed,
6368 SupervisorTaskState::Queued
6369 | SupervisorTaskState::PendingApproval
6370 | SupervisorTaskState::Running
6371 | SupervisorTaskState::ReviewPending
6372 | SupervisorTaskState::TestPending => LocalExecutionPhase::Running,
6373 }
6374}
6375
6376fn local_execution_stage_for_terminal_state(state: SupervisorTaskState) -> &'static str {
6377 match state {
6378 SupervisorTaskState::Completed => "completed",
6379 SupervisorTaskState::Cancelled => "cancelled",
6380 SupervisorTaskState::Blocked => "blocked",
6381 SupervisorTaskState::Failed => "failed",
6382 _ => "running",
6383 }
6384}
6385
6386fn artifacts_from_manifest(manifest: &[ArtifactManifestEntry]) -> Vec<RemoteExecutionArtifact> {
6387 manifest
6388 .iter()
6389 .map(|artifact| RemoteExecutionArtifact {
6390 name: artifact.name.clone(),
6391 part_count: artifact.part_count,
6392 metadata: artifact.metadata.clone(),
6393 })
6394 .collect()
6395}
6396
6397fn summarize_remote_task_output(task: &A2ATask) -> String {
6398 task.messages
6399 .iter()
6400 .rev()
6401 .flat_map(|message| message.parts.iter().rev())
6402 .find_map(|part| match part {
6403 MessagePart::Text { text } => Some(text.clone()),
6404 _ => None,
6405 })
6406 .or_else(|| task.status_reason.clone())
6407 .unwrap_or_else(|| {
6408 format!(
6409 "Remote task {} finished with status {}",
6410 task.id,
6411 a2a_status_label(task.status)
6412 )
6413 })
6414}
6415
6416fn task_artifacts_from_remote_payload(artifacts: &[RemoteArtifact]) -> Vec<TaskArtifactRecord> {
6417 artifacts
6418 .iter()
6419 .map(|artifact| TaskArtifactRecord {
6420 name: artifact.name.clone(),
6421 kind: "a2a_artifact".to_string(),
6422 uri: Some(format!("a2a://artifact/{}", artifact.name)),
6423 summary: Some(
6424 artifact
6425 .parts
6426 .iter()
6427 .find_map(|part| match part {
6428 MessagePart::Text { text } => {
6429 Some(text.chars().take(160).collect::<String>())
6430 }
6431 _ => None,
6432 })
6433 .unwrap_or_else(|| "Remote artifact".to_string()),
6434 ),
6435 })
6436 .collect()
6437}
6438
6439fn compatibility_from_card(
6440 card: &gestura_core_a2a::AgentCard,
6441 remote_target: &RemoteAgentTarget,
6442) -> RemoteExecutionCompatibility {
6443 let mut warnings = Vec::new();
6444 if card.authentication.is_some() && remote_target.auth_token.is_none() {
6445 warnings.push(
6446 "Remote peer advertises authentication, but no auth token is configured".to_string(),
6447 );
6448 }
6449 if !card
6450 .supported_task_features
6451 .iter()
6452 .any(|feature| feature == "authenticated-mutations")
6453 {
6454 warnings
6455 .push("Remote peer does not advertise authenticated mutation enforcement".to_string());
6456 }
6457 if !card
6458 .supported_task_features
6459 .iter()
6460 .any(|feature| feature == "provenance")
6461 {
6462 warnings.push("Remote peer does not advertise provenance support".to_string());
6463 }
6464 if !card
6465 .supported_task_features
6466 .iter()
6467 .any(|feature| feature == "leases")
6468 {
6469 warnings.push(
6470 "Remote peer does not advertise lease support; heartbeat-based tracking will be disabled"
6471 .to_string(),
6472 );
6473 }
6474 if !card
6475 .supported_task_features
6476 .iter()
6477 .any(|feature| feature == "idempotency")
6478 {
6479 warnings.push(
6480 "Remote peer does not advertise idempotency support; retries may duplicate work"
6481 .to_string(),
6482 );
6483 }
6484 if !card
6485 .supported_rpc_methods
6486 .iter()
6487 .any(|method| method == "task/artifacts")
6488 {
6489 warnings.push("Remote peer does not advertise artifact manifest support".to_string());
6490 }
6491 RemoteExecutionCompatibility {
6492 supported_features: card.supported_task_features.clone(),
6493 warnings,
6494 protocol_version: Some(card.protocol_version.clone()),
6495 }
6496}
6497
6498fn build_remote_task_request(
6499 task: &DelegatedTask,
6500 record: &SupervisorTaskRecord,
6501 compatibility: &RemoteExecutionCompatibility,
6502) -> CreateTaskRequest {
6503 let brief = task
6504 .delegation_brief
6505 .clone()
6506 .unwrap_or_else(|| DelegationBrief {
6507 objective: task.name.clone().unwrap_or_else(|| task.prompt.clone()),
6508 acceptance_criteria: Vec::new(),
6509 constraints: Vec::new(),
6510 deliverables: vec!["Provide a concise text result".to_string()],
6511 context_summary: None,
6512 });
6513 let mut metadata = HashMap::new();
6514 metadata.insert("gesturaRunId".to_string(), json!(task.run_id));
6515 metadata.insert("gesturaTaskId".to_string(), json!(task.id));
6516 metadata.insert("executionMode".to_string(), json!("remote"));
6517 metadata.insert("attempt".to_string(), json!(record.attempts));
6518 metadata.insert(
6519 "approvalRequired".to_string(),
6520 json!(task.approval_required),
6521 );
6522 metadata.insert(
6523 "reviewerRequired".to_string(),
6524 json!(task.reviewer_required),
6525 );
6526 metadata.insert("testRequired".to_string(), json!(task.test_required));
6527 if let Some(workspace_dir) = task.workspace_dir.as_ref() {
6528 metadata.insert(
6529 "workspaceDir".to_string(),
6530 json!(workspace_dir.to_string_lossy().to_string()),
6531 );
6532 }
6533 CreateTaskRequest {
6534 message: A2AMessage {
6535 role: "user".to_string(),
6536 parts: vec![MessagePart::Text {
6537 text: task.prompt.clone(),
6538 }],
6539 },
6540 run_id: task.run_id.clone(),
6541 parent_task_id: task.parent_task_id.clone(),
6542 role: task
6543 .role
6544 .clone()
6545 .map(|role| format!("{role:?}").to_lowercase()),
6546 requested_capabilities: task.required_tools.clone(),
6547 contract: Some(RemoteTaskContract {
6548 objective: brief.objective,
6549 acceptance_criteria: brief.acceptance_criteria,
6550 constraints: brief.constraints,
6551 deliverables: brief.deliverables,
6552 output_format: Some("text".to_string()),
6553 }),
6554 idempotency_key: compatibility
6555 .supported_features
6556 .iter()
6557 .any(|feature| feature == "idempotency")
6558 .then(|| {
6559 format!(
6560 "gestura:{}:{}:{}",
6561 task.run_id
6562 .clone()
6563 .unwrap_or_else(|| "standalone".to_string()),
6564 task.id,
6565 record.attempts
6566 )
6567 }),
6568 lease_request: compatibility
6569 .supported_features
6570 .iter()
6571 .any(|feature| feature == "leases")
6572 .then_some(RemoteTaskLeaseRequest {
6573 ttl_secs: 120,
6574 heartbeat_interval_secs: 15,
6575 }),
6576 metadata,
6577 }
6578}
6579
6580fn provenance_from_metadata(
6581 metadata: &HashMap<String, serde_json::Value>,
6582) -> Option<TaskProvenance> {
6583 let caller_agent_id = metadata
6584 .get("caller_agent_id")
6585 .and_then(|value| value.as_str())
6586 .map(ToOwned::to_owned);
6587 let caller_name = metadata
6588 .get("caller_name")
6589 .and_then(|value| value.as_str())
6590 .map(ToOwned::to_owned);
6591 let caller_version = metadata
6592 .get("caller_version")
6593 .and_then(|value| value.as_str())
6594 .map(ToOwned::to_owned);
6595 let caller_capabilities = metadata
6596 .get("caller_capabilities")
6597 .and_then(|value| value.as_array())
6598 .map(|values| {
6599 values
6600 .iter()
6601 .filter_map(|value| value.as_str().map(ToOwned::to_owned))
6602 .collect::<Vec<_>>()
6603 })
6604 .unwrap_or_default();
6605 let authenticated = metadata
6606 .get("caller_authenticated")
6607 .and_then(|value| value.as_bool())
6608 .unwrap_or(false);
6609 let auth_scheme = metadata
6610 .get("caller_auth_scheme")
6611 .and_then(|value| value.as_str())
6612 .map(ToOwned::to_owned);
6613
6614 (caller_agent_id.is_some()
6615 || caller_name.is_some()
6616 || caller_version.is_some()
6617 || !caller_capabilities.is_empty()
6618 || authenticated
6619 || auth_scheme.is_some())
6620 .then_some(TaskProvenance {
6621 caller_agent_id,
6622 caller_name,
6623 caller_version,
6624 caller_capabilities,
6625 authenticated,
6626 auth_scheme,
6627 })
6628}
6629
6630fn checkpoint_roots_for_runs(runs: &[SupervisorRun], default_root: Option<&Path>) -> Vec<PathBuf> {
6631 let mut roots = HashSet::new();
6632 if let Some(root) = default_root {
6633 roots.insert(root.to_path_buf());
6634 }
6635 for run in runs {
6636 if let Some(workspace_dir) = run.workspace_dir.as_ref() {
6637 roots.insert(workspace_dir.clone());
6638 }
6639 for record in &run.tasks {
6640 if let Some(workspace_dir) = record.task.workspace_dir.as_ref() {
6641 roots.insert(workspace_dir.clone());
6642 }
6643 }
6644 }
6645 roots.into_iter().collect()
6646}
6647
6648fn checkpoint_roots_for_task(task: &DelegatedTask, default_root: Option<&Path>) -> Vec<PathBuf> {
6649 let mut roots = Vec::new();
6650 if let Some(root) = default_root {
6651 roots.push(root.to_path_buf());
6652 }
6653 if let Some(workspace_dir) = task.workspace_dir.as_ref()
6654 && !roots.iter().any(|root| root == workspace_dir)
6655 {
6656 roots.push(workspace_dir.clone());
6657 }
6658 roots
6659}
6660
6661fn load_latest_checkpoints_by_task<I>(roots: I) -> HashMap<String, DelegatedTaskCheckpoint>
6662where
6663 I: IntoIterator<Item = PathBuf>,
6664{
6665 let mut checkpoints: HashMap<String, DelegatedTaskCheckpoint> = HashMap::new();
6666 for root in roots {
6667 for checkpoint in load_persisted_checkpoints(&root) {
6668 match checkpoints.get(&checkpoint.task_id) {
6669 Some(existing) if existing.updated_at >= checkpoint.updated_at => {}
6670 _ => {
6671 checkpoints.insert(checkpoint.task_id.clone(), checkpoint);
6672 }
6673 }
6674 }
6675 }
6676 checkpoints
6677}
6678
6679fn attach_checkpoint_summaries(
6680 runs: &mut [SupervisorRun],
6681 checkpoints: HashMap<String, DelegatedTaskCheckpoint>,
6682) {
6683 for run in runs {
6684 for record in &mut run.tasks {
6685 record.checkpoint = checkpoints
6686 .get(&record.task.id)
6687 .map(|checkpoint| checkpoint_summary_for_record(checkpoint, record.state));
6688 }
6689 }
6690}
6691
6692fn checkpoint_summary_for_record(
6693 checkpoint: &DelegatedTaskCheckpoint,
6694 task_state: SupervisorTaskState,
6695) -> DelegatedCheckpointSummary {
6696 DelegatedCheckpointSummary {
6697 stage: checkpoint.stage,
6698 replay_safety: checkpoint.replay_safety,
6699 resume_disposition: checkpoint.resume_disposition,
6700 safe_boundary_label: checkpoint.safe_boundary_label.clone(),
6701 available_actions: checkpoint_available_actions(checkpoint, task_state),
6702 note: checkpoint.note.clone(),
6703 completed_tool_call_count: checkpoint.completed_tool_calls.len(),
6704 has_resume_state: checkpoint.resume_state.is_some(),
6705 result_published: checkpoint.result_published,
6706 updated_at: checkpoint.updated_at,
6707 }
6708}
6709
6710fn active_task_snapshot(
6711 task: DelegatedTask,
6712 record: Option<&SupervisorTaskRecord>,
6713) -> ActiveTaskSnapshot {
6714 if let Some(record) = record {
6715 ActiveTaskSnapshot {
6716 task,
6717 state: record.state,
6718 remote_execution: record.remote_execution.clone(),
6719 local_execution: record.local_execution.clone(),
6720 blocked_reasons: record.blocked_reasons.clone(),
6721 checkpoint: record.checkpoint.clone(),
6722 }
6723 } else {
6724 ActiveTaskSnapshot {
6725 task,
6726 state: SupervisorTaskState::Running,
6727 remote_execution: None,
6728 local_execution: None,
6729 blocked_reasons: Vec::new(),
6730 checkpoint: None,
6731 }
6732 }
6733}
6734
6735fn checkpoint_available_actions(
6736 checkpoint: &DelegatedTaskCheckpoint,
6737 task_state: SupervisorTaskState,
6738) -> Vec<DelegatedCheckpointAction> {
6739 if task_state != SupervisorTaskState::Blocked || checkpoint.result_published {
6740 return Vec::new();
6741 }
6742
6743 let mut actions = Vec::new();
6744 if checkpoint.resume_disposition == DelegatedResumeDisposition::ResumeFromCheckpoint
6745 && checkpoint.resume_state.is_some()
6746 {
6747 actions.push(DelegatedCheckpointAction::ResumeFromCheckpoint);
6748 }
6749 actions.push(DelegatedCheckpointAction::RestartFromScratch);
6750 actions.push(DelegatedCheckpointAction::AcknowledgeBlocked);
6751 actions
6752}
6753
6754fn unresolved_dependency_reasons(run: &SupervisorRun, task: &DelegatedTask) -> Vec<String> {
6755 let dependency_states = run
6756 .tasks
6757 .iter()
6758 .map(|record| (record.task.id.clone(), record.state))
6759 .collect::<HashMap<_, _>>();
6760 dependency_reasons_from_states(&dependency_states, task)
6761}
6762
6763fn dependency_reasons_from_states(
6764 dependency_states: &HashMap<String, SupervisorTaskState>,
6765 task: &DelegatedTask,
6766) -> Vec<String> {
6767 task.depends_on
6768 .iter()
6769 .filter_map(|dependency_id| match dependency_states.get(dependency_id) {
6770 Some(SupervisorTaskState::Completed) => None,
6771 Some(state) => Some(format!("Waiting on task '{}' ({:?})", dependency_id, state)),
6772 None => Some(format!("Waiting on unknown dependency '{}'", dependency_id)),
6773 })
6774 .collect()
6775}
6776
6777fn recalculate_run_status(run: &SupervisorRun) -> SupervisorRunStatus {
6778 if run.tasks.is_empty() && run.child_runs.is_empty() {
6779 return SupervisorRunStatus::Draft;
6780 }
6781 let own_tasks = !run.tasks.is_empty();
6782 let own_cancelled = own_tasks
6783 && run
6784 .tasks
6785 .iter()
6786 .all(|record| matches!(record.state, SupervisorTaskState::Cancelled));
6787 let child_cancelled = !run.child_runs.is_empty()
6788 && run
6789 .child_runs
6790 .iter()
6791 .all(|child| matches!(child.status, SupervisorRunStatus::Cancelled));
6792 if (own_tasks || !run.child_runs.is_empty())
6793 && (if own_tasks { own_cancelled } else { true })
6794 && (if !run.child_runs.is_empty() {
6795 child_cancelled
6796 } else {
6797 true
6798 })
6799 {
6800 return SupervisorRunStatus::Cancelled;
6801 }
6802 if run.tasks.iter().any(|record| {
6803 matches!(
6804 record.state,
6805 SupervisorTaskState::Running | SupervisorTaskState::Queued
6806 )
6807 }) || run
6808 .child_runs
6809 .iter()
6810 .any(|child| matches!(child.status, SupervisorRunStatus::Running))
6811 {
6812 return SupervisorRunStatus::Running;
6813 }
6814 if run.tasks.iter().any(|record| {
6815 matches!(
6816 record.state,
6817 SupervisorTaskState::Blocked
6818 | SupervisorTaskState::PendingApproval
6819 | SupervisorTaskState::ReviewPending
6820 | SupervisorTaskState::TestPending
6821 )
6822 }) || run.child_runs.iter().any(|child| {
6823 child.requires_attention || matches!(child.status, SupervisorRunStatus::Waiting)
6824 }) {
6825 return SupervisorRunStatus::Waiting;
6826 }
6827 if run
6828 .tasks
6829 .iter()
6830 .any(|record| matches!(record.state, SupervisorTaskState::Failed))
6831 || run
6832 .child_runs
6833 .iter()
6834 .any(|child| matches!(child.status, SupervisorRunStatus::Failed))
6835 {
6836 return SupervisorRunStatus::Failed;
6837 }
6838 let own_completed = own_tasks
6839 && run
6840 .tasks
6841 .iter()
6842 .all(|record| matches!(record.state, SupervisorTaskState::Completed));
6843 let child_completed = !run.child_runs.is_empty()
6844 && run
6845 .child_runs
6846 .iter()
6847 .all(|child| matches!(child.status, SupervisorRunStatus::Completed));
6848 if (own_tasks || !run.child_runs.is_empty())
6849 && (if own_tasks { own_completed } else { true })
6850 && (if !run.child_runs.is_empty() {
6851 child_completed
6852 } else {
6853 true
6854 })
6855 {
6856 return SupervisorRunStatus::Completed;
6857 }
6858 if run.tasks.iter().all(|record| {
6859 matches!(
6860 record.state,
6861 SupervisorTaskState::Completed | SupervisorTaskState::Cancelled
6862 )
6863 }) && run.child_runs.iter().all(|child| {
6864 matches!(
6865 child.status,
6866 SupervisorRunStatus::Completed | SupervisorRunStatus::Cancelled
6867 )
6868 }) {
6869 return SupervisorRunStatus::Completed;
6870 }
6871 SupervisorRunStatus::Draft
6872}
6873
6874fn summarize_run_tasks(run: &SupervisorRun) -> SupervisorRunTaskSummary {
6875 let mut summary = SupervisorRunTaskSummary {
6876 total: run.tasks.len(),
6877 ..SupervisorRunTaskSummary::default()
6878 };
6879 for record in &run.tasks {
6880 match record.state {
6881 SupervisorTaskState::Queued => summary.queued += 1,
6882 SupervisorTaskState::Blocked => summary.blocked += 1,
6883 SupervisorTaskState::PendingApproval => summary.pending_approval += 1,
6884 SupervisorTaskState::Running => summary.running += 1,
6885 SupervisorTaskState::ReviewPending => summary.review_pending += 1,
6886 SupervisorTaskState::TestPending => summary.test_pending += 1,
6887 SupervisorTaskState::Completed => summary.completed += 1,
6888 SupervisorTaskState::Failed => summary.failed += 1,
6889 SupervisorTaskState::Cancelled => summary.cancelled += 1,
6890 }
6891 }
6892 summary
6893}
6894
6895fn run_requires_attention(run: &SupervisorRun) -> bool {
6896 run.tasks.iter().any(|record| {
6897 matches!(
6898 record.state,
6899 SupervisorTaskState::Blocked
6900 | SupervisorTaskState::PendingApproval
6901 | SupervisorTaskState::ReviewPending
6902 | SupervisorTaskState::TestPending
6903 | SupervisorTaskState::Failed
6904 )
6905 }) || run.child_runs.iter().any(|child| child.requires_attention)
6906}
6907
6908fn build_child_inherited_policy(
6909 parent: &SupervisorRun,
6910 request: &ChildSupervisorRunRequest,
6911) -> SupervisorInheritancePolicy {
6912 let mut policy = parent.inherited_policy.clone().unwrap_or_default();
6913 policy.approval_required |= request.approval_required;
6914 policy.reviewer_required |= request.reviewer_required;
6915 policy.test_required |= request.test_required;
6916 policy.execution_mode = Some(request.execution_mode.clone());
6917 policy.workspace_dir = request
6918 .workspace_dir
6919 .clone()
6920 .or_else(|| parent.workspace_dir.clone());
6921 for tag in &request.memory_tags {
6922 if !policy.memory_tags.contains(tag) {
6923 policy.memory_tags.push(tag.clone());
6924 }
6925 }
6926 for note in &request.constraint_notes {
6927 if !policy.constraint_notes.contains(note) {
6928 policy.constraint_notes.push(note.clone());
6929 }
6930 }
6931 policy
6932}
6933
6934fn build_child_run_summary(run: &SupervisorRun) -> ChildSupervisorRunSummary {
6935 ChildSupervisorRunSummary {
6936 run_id: run.id.clone(),
6937 name: run.name.clone(),
6938 objective: run
6939 .parent_run
6940 .as_ref()
6941 .map(|parent| parent.objective.clone())
6942 .unwrap_or_else(|| run.name.clone().unwrap_or_else(|| run.id.clone())),
6943 lead_agent_id: run.lead_agent_id.clone(),
6944 status: recalculate_run_status(run),
6945 task_summary: summarize_run_tasks(run),
6946 requires_attention: run_requires_attention(run),
6947 blocked_reasons: run
6948 .tasks
6949 .iter()
6950 .flat_map(|record| record.blocked_reasons.clone())
6951 .collect(),
6952 created_at: run.created_at,
6953 updated_at: run.updated_at,
6954 completed_at: run.completed_at,
6955 }
6956}
6957
6958fn upsert_child_run_summary(parent: &mut SupervisorRun, child: &SupervisorRun) {
6959 let summary = build_child_run_summary(child);
6960 if let Some(existing) = parent
6961 .child_runs
6962 .iter_mut()
6963 .find(|entry| entry.run_id == child.id)
6964 {
6965 *existing = summary;
6966 } else {
6967 parent.child_runs.push(summary);
6968 }
6969}
6970
6971fn build_hierarchy_summary(run: &SupervisorRun) -> SupervisorHierarchySummary {
6972 let descendant_task_count = run
6973 .child_runs
6974 .iter()
6975 .map(|child| child.task_summary.total)
6976 .sum();
6977 let action_required_child_count = run
6978 .child_runs
6979 .iter()
6980 .filter(|child| child.requires_attention)
6981 .count();
6982 let blocked_reasons = run
6983 .child_runs
6984 .iter()
6985 .flat_map(|child| child.blocked_reasons.clone())
6986 .collect();
6987 SupervisorHierarchySummary {
6988 depth: run.hierarchy_depth,
6989 max_depth: run.max_hierarchy_depth,
6990 child_run_count: run.child_runs.len(),
6991 descendant_task_count,
6992 action_required_child_count,
6993 rollup_status: recalculate_run_status(run),
6994 requires_attention: action_required_child_count > 0,
6995 blocked_reasons,
6996 }
6997}
6998
6999fn refresh_run_rollups(run: &mut SupervisorRun) {
7000 if run.name.is_none() {
7001 run.name = run
7002 .tasks
7003 .first()
7004 .and_then(|record| record.task.name.clone());
7005 }
7006 if run.max_hierarchy_depth == 0 {
7007 run.max_hierarchy_depth = MAX_CHILD_SUPERVISOR_DEPTH;
7008 }
7009 run.task_summary = summarize_run_tasks(run);
7010 run.status = recalculate_run_status(run);
7011 run.hierarchy_summary = Some(build_hierarchy_summary(run));
7012}
7013
7014fn synchronize_run_hierarchy_snapshots(runs: &mut [SupervisorRun]) {
7015 for run in runs.iter_mut() {
7016 run.task_summary = summarize_run_tasks(run);
7017 }
7018
7019 let mut child_map = std::collections::HashMap::<String, Vec<ChildSupervisorRunSummary>>::new();
7020 for run in runs.iter() {
7021 if let Some(parent) = run.parent_run.as_ref() {
7022 child_map
7023 .entry(parent.parent_run_id.clone())
7024 .or_default()
7025 .push(build_child_run_summary(run));
7026 }
7027 }
7028
7029 for run in runs.iter_mut() {
7030 run.child_runs = child_map.remove(&run.id).unwrap_or_default();
7031 refresh_run_rollups(run);
7032 }
7033}
7034
7035fn ensure_parent_run_accepts_child(parent: &SupervisorRun) -> Result<(), String> {
7036 if parent.hierarchy_depth >= parent.max_hierarchy_depth {
7037 return Err(format!(
7038 "Supervisor run '{}' is already at the maximum hierarchy depth of {}",
7039 parent.id, parent.max_hierarchy_depth
7040 ));
7041 }
7042 if parent.parent_run.is_some() {
7043 return Err(format!(
7044 "Supervisor run '{}' is already a child run and cannot delegate another child supervisor",
7045 parent.id
7046 ));
7047 }
7048 if matches!(
7049 parent.status,
7050 SupervisorRunStatus::Completed | SupervisorRunStatus::Cancelled
7051 ) {
7052 return Err(format!(
7053 "Supervisor run '{}' is terminal and cannot accept a child supervisor",
7054 parent.id
7055 ));
7056 }
7057 Ok(())
7058}
7059
7060fn collect_ready_tasks(run: &mut SupervisorRun) -> Vec<DelegatedTask> {
7061 collect_ready_tasks_except(run, None)
7062}
7063
7064fn collect_ready_tasks_except(
7065 run: &mut SupervisorRun,
7066 skip_task_id: Option<&str>,
7067) -> Vec<DelegatedTask> {
7068 let completed_ids = run
7069 .tasks
7070 .iter()
7071 .filter(|record| matches!(record.state, SupervisorTaskState::Completed))
7072 .map(|record| record.task.id.clone())
7073 .collect::<std::collections::HashSet<_>>();
7074 let now = Utc::now();
7075
7076 let mut ready = Vec::new();
7077 for record in &mut run.tasks {
7078 if skip_task_id.is_some_and(|task_id| record.task.id == task_id) {
7079 continue;
7080 }
7081 if !matches!(record.state, SupervisorTaskState::Blocked) {
7082 continue;
7083 }
7084 if matches!(record.approval.state, ApprovalState::Pending) {
7085 continue;
7086 }
7087 if record
7088 .task
7089 .depends_on
7090 .iter()
7091 .all(|dependency_id| completed_ids.contains(dependency_id))
7092 {
7093 record.state = SupervisorTaskState::Queued;
7094 record.blocked_reasons.clear();
7095 record.updated_at = now;
7096 ready.push(record.task.clone());
7097 }
7098 }
7099
7100 ready
7101}
7102
7103fn merge_task_metadata(
7104 manager: &TaskManager,
7105 session_id: &str,
7106 task_id: &str,
7107 patch: serde_json::Value,
7108) {
7109 let existing = manager
7110 .get_task(session_id, task_id)
7111 .ok()
7112 .flatten()
7113 .and_then(|task| task.metadata)
7114 .unwrap_or_else(|| json!({}));
7115
7116 let Some(mut existing_map) = existing.as_object().cloned() else {
7117 return;
7118 };
7119 let Some(patch_map) = patch.as_object() else {
7120 return;
7121 };
7122
7123 for (key, value) in patch_map {
7124 existing_map.insert(key.clone(), value.clone());
7125 }
7126
7127 let _ =
7128 manager.update_task_metadata(session_id, task_id, serde_json::Value::Object(existing_map));
7129}
7130
7131fn task_reflection_sync_context(task: &DelegatedTask) -> Option<(PathBuf, String, String)> {
7132 Some((
7133 task.workspace_dir.clone()?,
7134 task.session_id.clone()?,
7135 task.tracking_task_id.clone()?,
7136 ))
7137}
7138
7139fn approval_scope_for_state(state: SupervisorTaskState) -> Option<ApprovalScope> {
7140 match state {
7141 SupervisorTaskState::PendingApproval => Some(ApprovalScope::PreExecution),
7142 SupervisorTaskState::ReviewPending => Some(ApprovalScope::Review),
7143 SupervisorTaskState::TestPending => Some(ApprovalScope::TestValidation),
7144 _ => None,
7145 }
7146}
7147
7148fn collaboration_request_kind_for_scope(scope: ApprovalScope) -> CollaborationRequestKind {
7149 match scope {
7150 ApprovalScope::PreExecution => CollaborationRequestKind::ApprovalRequest,
7151 ApprovalScope::Review => CollaborationRequestKind::ReviewRequest,
7152 ApprovalScope::TestValidation => CollaborationRequestKind::TestValidationRequest,
7153 }
7154}
7155
7156fn team_message_kind_for_scope(scope: ApprovalScope) -> TeamMessageKind {
7157 match scope {
7158 ApprovalScope::PreExecution => TeamMessageKind::ApprovalRequest,
7159 ApprovalScope::Review => TeamMessageKind::ReviewRequest,
7160 ApprovalScope::TestValidation => TeamMessageKind::TestValidationRequest,
7161 }
7162}
7163
7164fn gate_request_message_content(record: &SupervisorTaskRecord, scope: ApprovalScope) -> String {
7165 let task_summary = record
7166 .task
7167 .name
7168 .as_deref()
7169 .unwrap_or(record.task.prompt.as_str());
7170 match scope {
7171 ApprovalScope::PreExecution => {
7172 format!("Pre-execution approval requested for: {task_summary}")
7173 }
7174 ApprovalScope::Review => format!("Review requested for: {task_summary}"),
7175 ApprovalScope::TestValidation => format!("Test validation requested for: {task_summary}"),
7176 }
7177}
7178
7179fn build_gate_request_message(
7180 run_id: &str,
7181 record: &SupervisorTaskRecord,
7182 scope: ApprovalScope,
7183) -> TeamMessage {
7184 let note = record.approval.note.clone();
7185 let mut action_request = TeamActionRequest::new(
7186 collaboration_request_kind_for_scope(scope),
7187 Some("orchestrator".to_string()),
7188 note.clone(),
7189 );
7190 action_request.approval_scope = Some(scope);
7191 action_request.requested_for_actor_kinds = record.approval.allowed_actor_kinds(scope).to_vec();
7192
7193 let mut message = TeamMessage::new(
7194 run_id.to_string(),
7195 Some(record.task.id.clone()),
7196 team_message_kind_for_scope(scope),
7197 Some("orchestrator".to_string()),
7198 None,
7199 gate_request_message_content(record, scope),
7200 )
7201 .with_action_request(action_request);
7202
7203 if let Some(result) = record.result.as_ref() {
7204 message = message.with_result_reference(TeamResultReference::from_task_result(result));
7205 message = message.with_artifact_references(
7206 result
7207 .artifacts
7208 .iter()
7209 .map(|artifact| {
7210 TeamArtifactReference::from_task_artifact(
7211 Some(record.task.id.clone()),
7212 artifact,
7213 )
7214 })
7215 .collect(),
7216 );
7217 }
7218
7219 message
7220}
7221
7222fn build_delegated_task_memory_handoff_content(
7223 record: &SupervisorTaskRecord,
7224 task_result: &TaskResult,
7225 memory_file_path: Option<&Path>,
7226) -> String {
7227 let task_name = record
7228 .task
7229 .name
7230 .as_deref()
7231 .or(task_result.summary.as_deref())
7232 .unwrap_or(record.task.id.as_str());
7233 let outcome = format!("{:?}", record.state).to_ascii_lowercase();
7234 let mut lines = vec![format!("Delegated task handoff for {task_name}")];
7235 lines.push(format!("Outcome: {outcome}"));
7236 lines.push(format!("Duration: {} ms", task_result.duration_ms));
7237
7238 if let Some(summary) = task_result
7239 .summary
7240 .as_deref()
7241 .filter(|summary| !summary.trim().is_empty())
7242 {
7243 lines.push(format!(
7244 "Summary: {}",
7245 truncate_delegated_task_message(summary, 240)
7246 ));
7247 }
7248
7249 if !task_result.output.trim().is_empty() {
7250 lines.push(format!(
7251 "Result: {}",
7252 truncate_delegated_task_message(&task_result.output, 400)
7253 ));
7254 }
7255
7256 if !task_result.artifacts.is_empty() {
7257 let artifact_names = task_result
7258 .artifacts
7259 .iter()
7260 .map(|artifact| artifact.name.as_str())
7261 .collect::<Vec<_>>()
7262 .join(", ");
7263 lines.push(format!("Artifacts: {artifact_names}"));
7264 }
7265
7266 if let Some(path) = memory_file_path {
7267 lines.push(format!("Memory: {}", path.display()));
7268 }
7269
7270 lines.join("\n")
7271}
7272
7273fn truncate_delegated_task_message(text: &str, max_chars: usize) -> String {
7274 let trimmed = text.trim();
7275 let truncated = trimmed.chars().take(max_chars).collect::<String>();
7276 if trimmed.chars().count() > max_chars {
7277 format!("{truncated}…")
7278 } else {
7279 truncated
7280 }
7281}
7282
7283fn resolve_open_gate_request(
7284 record: &mut SupervisorTaskRecord,
7285 scope: ApprovalScope,
7286 actor_id: &str,
7287 status: CollaborationActionStatus,
7288 note: Option<String>,
7289) -> Option<(String, String)> {
7290 let message = record.messages.iter_mut().rev().find(|message| {
7291 message.action_request.as_ref().is_some_and(|request| {
7292 request.approval_scope == Some(scope) && request.requires_attention()
7293 })
7294 })?;
7295
7296 if let Some(request) = message.action_request.as_mut() {
7297 request.resolve(status, Some(actor_id.to_string()), note);
7298 }
7299 message.unread_by_agent_ids.clear();
7300 Some((
7301 message.effective_thread_id().to_string(),
7302 message.id.clone(),
7303 ))
7304}
7305
7306fn format_approval_decision_message(decision: &ApprovalDecision) -> String {
7307 format!(
7308 "{:?} gate {:?} by {} ({:?}){}",
7309 decision.scope,
7310 decision.decision,
7311 decision.actor.id,
7312 decision.actor.kind,
7313 decision
7314 .note
7315 .as_ref()
7316 .map(|note| format!(": {note}"))
7317 .unwrap_or_default()
7318 )
7319}
7320
7321#[async_trait::async_trait]
7322impl OrchestratorAgentManager for crate::agents::AgentManager {
7323 async fn get_agent_status(&self, id: &str) -> Option<AgentInfo> {
7324 crate::agents::AgentManager::get_agent_status(self, id).await
7325 }
7326
7327 async fn list_agents(&self) -> Vec<AgentInfo> {
7328 crate::agents::AgentManager::list_agents(self).await
7329 }
7330
7331 async fn update_activity(&self, id: &str) {
7332 crate::agents::AgentManager::update_activity(self, id).await;
7333 }
7334}
7335
7336static ORCHESTRATOR_KNOWLEDGE_STORE: OnceLock<crate::KnowledgeStore> = OnceLock::new();
7342
7343static ORCHESTRATOR_KNOWLEDGE_SETTINGS: OnceLock<crate::KnowledgeSettingsManager> = OnceLock::new();
7345
7346fn orchestrator_knowledge_store() -> &'static crate::KnowledgeStore {
7347 ORCHESTRATOR_KNOWLEDGE_STORE.get_or_init(|| {
7348 let store = crate::KnowledgeStore::with_default_dir();
7349 crate::register_builtin_knowledge(&store);
7350 if let Err(e) = store.load_user_items() {
7351 tracing::warn!(error = %e, "Failed to load persisted user knowledge (continuing)");
7352 }
7353 store
7354 })
7355}
7356
7357fn orchestrator_knowledge_settings() -> &'static crate::KnowledgeSettingsManager {
7358 ORCHESTRATOR_KNOWLEDGE_SETTINGS.get_or_init(|| {
7359 let base_dir = dirs::home_dir().unwrap_or_else(|| std::path::PathBuf::from("."));
7360 crate::KnowledgeSettingsManager::new(base_dir)
7361 })
7362}
7363
7364#[cfg(test)]
7365mod tests {
7366 use super::*;
7367 use crate::create_gestura_agent_card;
7368 use std::path::PathBuf;
7369 use std::sync::Arc;
7370 use tempfile::tempdir;
7371
7372 fn test_environment(id: &str, root_dir: PathBuf) -> ExecutionEnvironment {
7373 ExecutionEnvironment {
7374 id: id.to_string(),
7375 execution_mode: AgentExecutionMode::SharedWorkspace,
7376 root_dir,
7377 write_access: true,
7378 branch_name: None,
7379 worktree_path: None,
7380 remote_url: None,
7381 state: EnvironmentState::Ready,
7382 health: EnvironmentHealth::Clean,
7383 cleanup_policy: CleanupPolicy::KeepAlways,
7384 recovery_status: RecoveryStatus::NotRequired,
7385 recovery_action: None,
7386 failure: None,
7387 cleanup_result: None,
7388 }
7389 }
7390
7391 fn empty_supervisor_run(id: &str, workspace_dir: PathBuf) -> SupervisorRun {
7392 SupervisorRun {
7393 id: id.to_string(),
7394 name: Some(id.to_string()),
7395 session_id: None,
7396 workspace_dir: Some(workspace_dir),
7397 lead_agent_id: Some("supervisor-root".to_string()),
7398 parent_run: None,
7399 child_runs: Vec::new(),
7400 hierarchy_depth: 0,
7401 max_hierarchy_depth: MAX_CHILD_SUPERVISOR_DEPTH,
7402 inherited_policy: None,
7403 status: SupervisorRunStatus::Draft,
7404 task_summary: SupervisorRunTaskSummary::default(),
7405 hierarchy_summary: None,
7406 tasks: Vec::new(),
7407 messages: Vec::new(),
7408 shared_cognition: Vec::new(),
7409 created_at: Utc::now(),
7410 updated_at: Utc::now(),
7411 completed_at: None,
7412 metadata: None,
7413 }
7414 }
7415
7416 #[derive(Default)]
7417 struct RecordingObserver {
7418 runs: tokio::sync::Mutex<Vec<SupervisorRun>>,
7419 }
7420
7421 #[async_trait::async_trait]
7422 impl OrchestratorObserver for RecordingObserver {
7423 async fn on_task_started(&self, _task: DelegatedTask) {}
7424
7425 async fn on_task_completed(&self, _task: DelegatedTask, _result: TaskResult) {}
7426
7427 async fn on_run_updated(&self, run: SupervisorRun) {
7428 self.runs.lock().await.push(run);
7429 }
7430 }
7431
7432 async fn seed_review_pending_task(
7433 orchestrator: &AgentOrchestrator<crate::agents::AgentManager>,
7434 workspace_dir: PathBuf,
7435 test_required: bool,
7436 ) {
7437 let task = DelegatedTask {
7438 id: "task-approval".into(),
7439 agent_id: "agent-reviewer".into(),
7440 prompt: "Review the patch".into(),
7441 context: None,
7442 required_tools: vec![],
7443 priority: 1,
7444 session_id: None,
7445 directive_id: None,
7446 tracking_task_id: None,
7447 run_id: Some("run-approval".into()),
7448 parent_task_id: None,
7449 depends_on: vec![],
7450 role: Some(AgentRole::Reviewer),
7451 delegation_brief: None,
7452 planning_only: false,
7453 approval_required: false,
7454 reviewer_required: true,
7455 test_required,
7456 workspace_dir: Some(workspace_dir.clone()),
7457 execution_mode: AgentExecutionMode::SharedWorkspace,
7458 environment_id: Some("env-approval".into()),
7459 remote_target: None,
7460 memory_tags: vec![],
7461 name: Some("Review the patch".into()),
7462 };
7463
7464 let record = SupervisorTaskRecord {
7465 task: task.clone(),
7466 state: SupervisorTaskState::ReviewPending,
7467 approval: TaskApprovalRecord::pending(
7468 &task,
7469 ApprovalScope::Review,
7470 ApprovalActor::system("orchestrator"),
7471 Some("Execution finished. Awaiting explicit review approval.".into()),
7472 ),
7473 environment_id: "env-approval".into(),
7474 environment: test_environment("env-approval", workspace_dir.clone()),
7475 claimed_by: Some(task.agent_id.clone()),
7476 attempts: 0,
7477 blocked_reasons: vec![],
7478 result: None,
7479 remote_execution: None,
7480 local_execution: None,
7481 messages: vec![],
7482 checkpoint: None,
7483 created_at: Utc::now(),
7484 updated_at: Utc::now(),
7485 started_at: None,
7486 completed_at: None,
7487 };
7488
7489 let run = SupervisorRun {
7490 id: "run-approval".into(),
7491 name: Some("approval-run".into()),
7492 session_id: None,
7493 workspace_dir: Some(workspace_dir),
7494 lead_agent_id: None,
7495 parent_run: None,
7496 child_runs: Vec::new(),
7497 hierarchy_depth: 0,
7498 max_hierarchy_depth: MAX_CHILD_SUPERVISOR_DEPTH,
7499 inherited_policy: None,
7500 task_summary: SupervisorRunTaskSummary::default(),
7501 hierarchy_summary: None,
7502 metadata: None,
7503 status: SupervisorRunStatus::Waiting,
7504 tasks: vec![record],
7505 messages: vec![],
7506 shared_cognition: vec![],
7507 created_at: Utc::now(),
7508 updated_at: Utc::now(),
7509 completed_at: None,
7510 };
7511
7512 orchestrator
7513 .supervisor_runs
7514 .lock()
7515 .await
7516 .insert(run.id.clone(), run);
7517 orchestrator
7518 .task_run_index
7519 .lock()
7520 .await
7521 .insert(task.id.clone(), "run-approval".into());
7522 }
7523
7524 fn sample_task_result(success: bool, output: &str) -> TaskResult {
7525 TaskResult {
7526 task_id: "task-approval".into(),
7527 agent_id: "agent-reviewer".into(),
7528 success,
7529 run_id: Some("run-approval".into()),
7530 tracking_task_id: Some("tracking-approval".into()),
7531 output: output.into(),
7532 summary: None,
7533 tool_calls: vec![],
7534 artifacts: vec![],
7535 terminal_state_hint: None,
7536 duration_ms: 125,
7537 }
7538 }
7539
7540 fn sample_task_record(
7541 workspace_dir: PathBuf,
7542 state: SupervisorTaskState,
7543 reviewer_required: bool,
7544 test_required: bool,
7545 ) -> SupervisorTaskRecord {
7546 let task = DelegatedTask {
7547 id: "task-approval".into(),
7548 agent_id: "agent-reviewer".into(),
7549 prompt: "Review the patch".into(),
7550 context: None,
7551 required_tools: vec![],
7552 priority: 1,
7553 session_id: Some("session-approval".into()),
7554 directive_id: Some("directive-approval".into()),
7555 tracking_task_id: Some("tracking-approval".into()),
7556 run_id: Some("run-approval".into()),
7557 parent_task_id: None,
7558 depends_on: vec![],
7559 role: Some(AgentRole::Reviewer),
7560 delegation_brief: None,
7561 planning_only: false,
7562 approval_required: false,
7563 reviewer_required,
7564 test_required,
7565 workspace_dir: Some(workspace_dir.clone()),
7566 execution_mode: AgentExecutionMode::SharedWorkspace,
7567 environment_id: Some("env-approval".into()),
7568 remote_target: None,
7569 memory_tags: vec!["quality".into()],
7570 name: Some("Review the patch".into()),
7571 };
7572
7573 SupervisorTaskRecord {
7574 task: task.clone(),
7575 state,
7576 approval: TaskApprovalRecord::not_required(&task),
7577 environment_id: "env-approval".into(),
7578 environment: test_environment("env-approval", workspace_dir),
7579 claimed_by: Some(task.agent_id.clone()),
7580 attempts: 1,
7581 blocked_reasons: vec![],
7582 result: Some(sample_task_result(true, "Patch applied and checks passed.")),
7583 remote_execution: None,
7584 local_execution: None,
7585 messages: vec![],
7586 checkpoint: None,
7587 created_at: Utc::now(),
7588 updated_at: Utc::now(),
7589 started_at: Some(Utc::now()),
7590 completed_at: Some(Utc::now()),
7591 }
7592 }
7593
7594 #[tokio::test]
7595 async fn test_orchestrator_creation_and_spawn() {
7596 let manager = crate::agents::AgentManager::new(PathBuf::from("/tmp/test.db"));
7597 let config = AppConfig::default();
7598
7599 let orchestrator = AgentOrchestrator::new(manager, config);
7600 assert!(orchestrator.list_subagents().await.is_empty());
7601
7602 orchestrator
7603 .spawn_subagent("test-1", "Test Agent")
7604 .await
7605 .unwrap();
7606
7607 let agents = orchestrator.list_subagents().await;
7608 assert_eq!(agents.len(), 1);
7609 assert_eq!(agents[0].id, "test-1");
7610 }
7611
7612 #[tokio::test]
7613 async fn test_delegate_task_submission() {
7614 let tmp = tempdir().unwrap();
7615 let manager = crate::agents::AgentManager::new(tmp.path().join("test.db"));
7616 let config = AppConfig::default();
7617
7618 let orchestrator = AgentOrchestrator::new(manager, config);
7619
7620 let task = DelegatedTask {
7621 id: "task-1".into(),
7622 agent_id: "agent-1".into(),
7623 prompt: "Hello".into(),
7624 context: None,
7625 required_tools: vec![],
7626 priority: 1,
7627 session_id: None,
7628 directive_id: None,
7629 tracking_task_id: None,
7630 run_id: Some("run-test".into()),
7631 parent_task_id: None,
7632 depends_on: vec![],
7633 role: Some(AgentRole::Implementer),
7634 delegation_brief: None,
7635 planning_only: false,
7636 approval_required: false,
7637 reviewer_required: false,
7638 test_required: false,
7639 workspace_dir: None,
7640 execution_mode: AgentExecutionMode::SharedWorkspace,
7641 environment_id: None,
7642 remote_target: None,
7643 memory_tags: vec![],
7644 name: None,
7645 };
7646
7647 let id = orchestrator.delegate_task(task).await.unwrap();
7649 assert_eq!(id, "task-1");
7650 }
7651
7652 #[tokio::test]
7653 async fn test_review_gate_rejects_unauthorized_actor() {
7654 let tmp = tempdir().unwrap();
7655 let manager = crate::agents::AgentManager::new(tmp.path().join("approval.db"));
7656 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
7657
7658 seed_review_pending_task(&orchestrator, tmp.path().to_path_buf(), false).await;
7659
7660 let error = orchestrator
7661 .approve_task(
7662 "task-approval",
7663 ApprovalActor::new(ApprovalActorKind::Tester, "tester-1"),
7664 Some("Looks good".into()),
7665 )
7666 .await
7667 .unwrap_err();
7668
7669 assert!(error.contains("not authorized"));
7670 }
7671
7672 #[tokio::test]
7673 async fn test_review_gate_records_authorized_decision() {
7674 let tmp = tempdir().unwrap();
7675 let manager = crate::agents::AgentManager::new(tmp.path().join("approval-success.db"));
7676 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
7677
7678 seed_review_pending_task(&orchestrator, tmp.path().to_path_buf(), false).await;
7679
7680 orchestrator
7681 .approve_task(
7682 "task-approval",
7683 ApprovalActor::new(ApprovalActorKind::Reviewer, "reviewer-1"),
7684 Some("Approved after review".into()),
7685 )
7686 .await
7687 .unwrap();
7688
7689 let run = orchestrator
7690 .supervisor_runs
7691 .lock()
7692 .await
7693 .get("run-approval")
7694 .cloned()
7695 .unwrap();
7696 let record = run
7697 .tasks
7698 .iter()
7699 .find(|record| record.task.id == "task-approval")
7700 .unwrap();
7701
7702 assert_eq!(record.state, SupervisorTaskState::Completed);
7703 assert_eq!(record.approval.state, ApprovalState::Approved);
7704 assert_eq!(record.approval.decisions.len(), 1);
7705 assert_eq!(
7706 record.approval.decisions[0].actor.kind,
7707 ApprovalActorKind::Reviewer
7708 );
7709 assert_eq!(
7710 run.messages.last().map(|message| message.kind),
7711 Some(TeamMessageKind::ApprovalDecision)
7712 );
7713 }
7714
7715 #[tokio::test]
7716 async fn test_review_then_test_gates_require_separate_authorized_decisions() {
7717 let tmp = tempdir().unwrap();
7718 let manager = crate::agents::AgentManager::new(tmp.path().join("approval-multistep.db"));
7719 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
7720
7721 seed_review_pending_task(&orchestrator, tmp.path().to_path_buf(), true).await;
7722
7723 orchestrator
7724 .approve_task(
7725 "task-approval",
7726 ApprovalActor::new(ApprovalActorKind::Reviewer, "reviewer-1"),
7727 Some("Review passed".into()),
7728 )
7729 .await
7730 .unwrap();
7731
7732 {
7733 let run = orchestrator
7734 .supervisor_runs
7735 .lock()
7736 .await
7737 .get("run-approval")
7738 .cloned()
7739 .unwrap();
7740 let record = run
7741 .tasks
7742 .iter()
7743 .find(|record| record.task.id == "task-approval")
7744 .unwrap();
7745 assert_eq!(record.state, SupervisorTaskState::TestPending);
7746 assert_eq!(record.approval.state, ApprovalState::Pending);
7747 assert_eq!(record.approval.scope, Some(ApprovalScope::TestValidation));
7748 assert_eq!(record.approval.requests.len(), 2);
7749 assert_eq!(record.approval.decisions.len(), 1);
7750 }
7751
7752 let error = orchestrator
7753 .approve_task(
7754 "task-approval",
7755 ApprovalActor::new(ApprovalActorKind::Reviewer, "reviewer-1"),
7756 Some("Trying to approve test gate".into()),
7757 )
7758 .await
7759 .unwrap_err();
7760 assert!(error.contains("not authorized"));
7761
7762 orchestrator
7763 .approve_task(
7764 "task-approval",
7765 ApprovalActor::new(ApprovalActorKind::Tester, "tester-1"),
7766 Some("Tests passed".into()),
7767 )
7768 .await
7769 .unwrap();
7770
7771 let run = orchestrator
7772 .supervisor_runs
7773 .lock()
7774 .await
7775 .get("run-approval")
7776 .cloned()
7777 .unwrap();
7778 let record = run
7779 .tasks
7780 .iter()
7781 .find(|record| record.task.id == "task-approval")
7782 .unwrap();
7783
7784 assert_eq!(record.state, SupervisorTaskState::Completed);
7785 assert_eq!(record.approval.state, ApprovalState::Approved);
7786 assert_eq!(record.approval.decisions.len(), 2);
7787 assert_eq!(
7788 record.approval.decisions[1].scope,
7789 ApprovalScope::TestValidation
7790 );
7791 assert_eq!(
7792 record.approval.decisions[1].actor.kind,
7793 ApprovalActorKind::Tester
7794 );
7795 }
7796
7797 #[test]
7798 fn task_record_outcome_signals_include_gate_and_terminal_decisions() {
7799 let workspace = tempdir().unwrap();
7800 let mut record = sample_task_record(
7801 workspace.path().to_path_buf(),
7802 SupervisorTaskState::Completed,
7803 true,
7804 true,
7805 );
7806 record.approval.decisions.push(ApprovalDecision {
7807 id: "decision-review".into(),
7808 request_id: "request-review".into(),
7809 scope: ApprovalScope::Review,
7810 actor: ApprovalActor::new(ApprovalActorKind::Reviewer, "reviewer-1"),
7811 decision: ApprovalDecisionKind::Approved,
7812 decided_at: Utc::now(),
7813 note: Some("Review sign-off recorded.".into()),
7814 });
7815 record.approval.decisions.push(ApprovalDecision {
7816 id: "decision-test".into(),
7817 request_id: "request-test".into(),
7818 scope: ApprovalScope::TestValidation,
7819 actor: ApprovalActor::new(ApprovalActorKind::Tester, "tester-1"),
7820 decision: ApprovalDecisionKind::Approved,
7821 decided_at: Utc::now(),
7822 note: Some("Targeted tests passed.".into()),
7823 });
7824
7825 let signals = task_record_outcome_signals(&record);
7826 let labels = outcome_signal_labels(&signals);
7827 let summary = outcome_signal_summary(&signals).unwrap();
7828
7829 assert!(labels.contains(&"task_completed".to_string()));
7830 assert!(labels.contains(&"review_approved".to_string()));
7831 assert!(labels.contains(&"test_validation_approved".to_string()));
7832 assert!(summary.contains("Review sign-off recorded."));
7833 assert!(summary.contains("Targeted tests passed."));
7834 }
7835
7836 #[tokio::test]
7837 async fn delegated_task_memory_persists_outcome_provenance() {
7838 let workspace = tempdir().unwrap();
7839 let mut record = sample_task_record(
7840 workspace.path().to_path_buf(),
7841 SupervisorTaskState::Completed,
7842 true,
7843 true,
7844 );
7845 record.approval.decisions.push(ApprovalDecision {
7846 id: "decision-review".into(),
7847 request_id: "request-review".into(),
7848 scope: ApprovalScope::Review,
7849 actor: ApprovalActor::new(ApprovalActorKind::Reviewer, "reviewer-1"),
7850 decision: ApprovalDecisionKind::Approved,
7851 decided_at: Utc::now(),
7852 note: Some("Review sign-off recorded.".into()),
7853 });
7854 record.approval.decisions.push(ApprovalDecision {
7855 id: "decision-test".into(),
7856 request_id: "request-test".into(),
7857 scope: ApprovalScope::TestValidation,
7858 actor: ApprovalActor::new(ApprovalActorKind::Tester, "tester-1"),
7859 decision: ApprovalDecisionKind::Approved,
7860 decided_at: Utc::now(),
7861 note: Some("Targeted tests passed.".into()),
7862 });
7863
7864 let path = persist_delegated_task_memory(&record).await.unwrap();
7865 let entry = crate::memory_bank::load_from_memory_bank(&path)
7866 .await
7867 .unwrap();
7868
7869 assert_eq!(
7870 entry.outcome_labels,
7871 vec![
7872 "task_completed",
7873 "review_approved",
7874 "test_validation_approved"
7875 ]
7876 );
7877 assert!(
7878 entry
7879 .outcome_summary
7880 .unwrap_or_default()
7881 .contains("Review sign-off recorded.")
7882 );
7883 assert!(entry.content.contains("## Outcome Signals"));
7884 }
7885
7886 #[tokio::test]
7887 async fn test_delegate_task_creates_pre_execution_collaboration_request() {
7888 let tmp = tempdir().unwrap();
7889 let manager = crate::agents::AgentManager::new(tmp.path().join("collaboration-request.db"));
7890 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
7891
7892 orchestrator
7893 .delegate_task(DelegatedTask {
7894 id: "task-collab".into(),
7895 agent_id: "agent-impl".into(),
7896 prompt: "Implement the feature".into(),
7897 context: None,
7898 required_tools: vec![],
7899 priority: 1,
7900 session_id: None,
7901 directive_id: None,
7902 tracking_task_id: None,
7903 run_id: Some("run-collab".into()),
7904 parent_task_id: None,
7905 depends_on: vec![],
7906 role: Some(AgentRole::Implementer),
7907 delegation_brief: None,
7908 planning_only: false,
7909 approval_required: true,
7910 reviewer_required: false,
7911 test_required: false,
7912 workspace_dir: Some(tmp.path().to_path_buf()),
7913 execution_mode: AgentExecutionMode::SharedWorkspace,
7914 environment_id: Some("env-collab".into()),
7915 remote_target: None,
7916 memory_tags: vec![],
7917 name: Some("Implement the feature".into()),
7918 })
7919 .await
7920 .unwrap();
7921
7922 let run = orchestrator.get_supervisor_run("run-collab").await.unwrap();
7923 let record = run.tasks.first().unwrap();
7924 let message = record.messages.first().unwrap();
7925
7926 assert_eq!(record.state, SupervisorTaskState::PendingApproval);
7927 assert_eq!(message.kind, TeamMessageKind::ApprovalRequest);
7928 assert_eq!(
7929 message
7930 .action_request
7931 .as_ref()
7932 .and_then(|request| request.approval_scope),
7933 Some(ApprovalScope::PreExecution)
7934 );
7935 assert_eq!(
7936 message
7937 .action_request
7938 .as_ref()
7939 .map(|request| request.status),
7940 Some(CollaborationActionStatus::Open)
7941 );
7942
7943 let threads = orchestrator.list_team_threads("run-collab").await;
7944 assert_eq!(threads.len(), 1);
7945 assert_eq!(threads[0].status, CollaborationThreadStatus::ActionRequired);
7946 assert!(threads[0].requires_attention);
7947 }
7948
7949 #[tokio::test]
7950 async fn test_thread_actions_can_be_acknowledged_resolved_and_archived() {
7951 let tmp = tempdir().unwrap();
7952 let manager = crate::agents::AgentManager::new(tmp.path().join("thread-actions.db"));
7953 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
7954
7955 orchestrator
7956 .delegate_task(DelegatedTask {
7957 id: "task-thread-actions".into(),
7958 agent_id: "agent-impl".into(),
7959 prompt: "Implement the feature".into(),
7960 context: None,
7961 required_tools: vec![],
7962 priority: 1,
7963 session_id: None,
7964 directive_id: None,
7965 tracking_task_id: None,
7966 run_id: Some("run-thread-actions".into()),
7967 parent_task_id: None,
7968 depends_on: vec![],
7969 role: Some(AgentRole::Implementer),
7970 delegation_brief: None,
7971 planning_only: false,
7972 approval_required: true,
7973 reviewer_required: false,
7974 test_required: false,
7975 workspace_dir: Some(tmp.path().to_path_buf()),
7976 execution_mode: AgentExecutionMode::SharedWorkspace,
7977 environment_id: Some("env-thread-actions".into()),
7978 remote_target: None,
7979 memory_tags: vec![],
7980 name: Some("Implement the feature".into()),
7981 })
7982 .await
7983 .unwrap();
7984
7985 let thread_id = orchestrator
7986 .list_team_threads("run-thread-actions")
7987 .await
7988 .first()
7989 .unwrap()
7990 .id
7991 .clone();
7992
7993 let acknowledged = orchestrator
7994 .update_team_thread_action(
7995 "run-thread-actions",
7996 &thread_id,
7997 CollaborationActionStatus::Acknowledged,
7998 Some("reviewer-1".to_string()),
7999 Some("Looking now".to_string()),
8000 )
8001 .await
8002 .unwrap();
8003 assert_eq!(
8004 acknowledged.status,
8005 CollaborationThreadStatus::ActionRequired
8006 );
8007 assert_eq!(
8008 acknowledged
8009 .latest_action_request
8010 .as_ref()
8011 .map(|request| request.status),
8012 Some(CollaborationActionStatus::Acknowledged)
8013 );
8014
8015 let resolved = orchestrator
8016 .update_team_thread_action(
8017 "run-thread-actions",
8018 &thread_id,
8019 CollaborationActionStatus::Resolved,
8020 Some("reviewer-1".to_string()),
8021 Some("Approved after review".to_string()),
8022 )
8023 .await
8024 .unwrap();
8025 assert_eq!(resolved.status, CollaborationThreadStatus::Resolved);
8026 assert!(!resolved.requires_attention);
8027
8028 let archived = orchestrator
8029 .archive_team_thread(
8030 "run-thread-actions",
8031 &thread_id,
8032 Some("reviewer-1".to_string()),
8033 Some("Archive resolved thread".to_string()),
8034 )
8035 .await
8036 .unwrap();
8037 assert!(archived.archived);
8038 assert!(
8039 orchestrator
8040 .list_team_threads("run-thread-actions")
8041 .await
8042 .is_empty()
8043 );
8044 assert_eq!(
8045 orchestrator
8046 .list_team_threads_with_options("run-thread-actions", true)
8047 .await
8048 .len(),
8049 1
8050 );
8051 }
8052
8053 #[tokio::test]
8054 async fn test_blocker_collaboration_marks_task_blocked_until_resolved() {
8055 let tmp = tempdir().unwrap();
8056 let manager = crate::agents::AgentManager::new(tmp.path().join("thread-blocker.db"));
8057 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8058
8059 orchestrator
8060 .delegate_task(DelegatedTask {
8061 id: "task-blocker".into(),
8062 agent_id: "agent-impl".into(),
8063 prompt: "Implement the feature".into(),
8064 context: None,
8065 required_tools: vec![],
8066 priority: 1,
8067 session_id: None,
8068 directive_id: None,
8069 tracking_task_id: None,
8070 run_id: Some("run-blocker".into()),
8071 parent_task_id: None,
8072 depends_on: vec![],
8073 role: Some(AgentRole::Implementer),
8074 delegation_brief: None,
8075 planning_only: false,
8076 approval_required: true,
8077 reviewer_required: false,
8078 test_required: false,
8079 workspace_dir: Some(tmp.path().to_path_buf()),
8080 execution_mode: AgentExecutionMode::SharedWorkspace,
8081 environment_id: Some("env-blocker".into()),
8082 remote_target: None,
8083 memory_tags: vec![],
8084 name: Some("Implement the feature".into()),
8085 })
8086 .await
8087 .unwrap();
8088
8089 let blocker = orchestrator
8090 .send_team_message_draft(
8091 "run-blocker",
8092 TeamMessageDraft {
8093 task_id: Some("task-blocker".to_string()),
8094 kind: TeamMessageKind::Blocker,
8095 sender_agent_id: Some("agent-impl".to_string()),
8096 recipient_agent_id: None,
8097 content: "Waiting on credentials".to_string(),
8098 thread_id: None,
8099 reply_to_message_id: None,
8100 action_request: Some(TeamActionRequestDraft {
8101 kind: CollaborationRequestKind::BlockerEscalation,
8102 requested_for_agent_ids: Vec::new(),
8103 requested_for_roles: vec![AgentRole::Supervisor],
8104 requested_for_actor_kinds: Vec::new(),
8105 approval_scope: None,
8106 note: Some("Need credentials".to_string()),
8107 }),
8108 escalation: Some(TeamEscalationDraft {
8109 level: CollaborationEscalationLevel::Warning,
8110 escalated_by_agent_id: Some("agent-impl".to_string()),
8111 target_role: Some(AgentRole::Supervisor),
8112 note: Some("Credentials missing".to_string()),
8113 }),
8114 unread_by_agent_ids: Vec::new(),
8115 },
8116 )
8117 .await
8118 .unwrap();
8119
8120 let run = orchestrator
8121 .get_supervisor_run("run-blocker")
8122 .await
8123 .unwrap();
8124 let record = run.tasks.first().unwrap();
8125 assert_eq!(record.state, SupervisorTaskState::Blocked);
8126 assert!(
8127 record
8128 .blocked_reasons
8129 .iter()
8130 .any(|reason| reason == "Waiting on credentials")
8131 );
8132
8133 orchestrator
8134 .update_team_thread_action(
8135 "run-blocker",
8136 blocker.effective_thread_id(),
8137 CollaborationActionStatus::Resolved,
8138 Some("supervisor-1".to_string()),
8139 Some("Credentials delivered".to_string()),
8140 )
8141 .await
8142 .unwrap();
8143
8144 let resolved_run = orchestrator
8145 .get_supervisor_run("run-blocker")
8146 .await
8147 .unwrap();
8148 let resolved_record = resolved_run.tasks.first().unwrap();
8149 assert!(resolved_record.blocked_reasons.is_empty());
8150 assert_eq!(resolved_record.state, SupervisorTaskState::Queued);
8151 }
8152
8153 #[tokio::test]
8154 async fn test_team_message_publishes_shared_cognition_to_run_and_memory_bank() {
8155 let tmp = tempdir().unwrap();
8156 let manager = crate::agents::AgentManager::new(tmp.path().join("shared-cognition.db"));
8157 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8158
8159 orchestrator
8160 .delegate_task(DelegatedTask {
8161 id: "task-shared".into(),
8162 agent_id: "agent-impl".into(),
8163 prompt: "Implement the feature".into(),
8164 context: None,
8165 required_tools: vec![],
8166 priority: 1,
8167 session_id: Some("session-shared".into()),
8168 directive_id: Some("directive-shared".into()),
8169 tracking_task_id: None,
8170 run_id: Some("run-shared".into()),
8171 parent_task_id: None,
8172 depends_on: vec![],
8173 role: Some(AgentRole::Implementer),
8174 delegation_brief: None,
8175 planning_only: false,
8176 approval_required: true,
8177 reviewer_required: false,
8178 test_required: false,
8179 workspace_dir: Some(tmp.path().to_path_buf()),
8180 execution_mode: AgentExecutionMode::SharedWorkspace,
8181 environment_id: None,
8182 remote_target: None,
8183 memory_tags: vec!["frontend".to_string()],
8184 name: Some("Implement the feature".into()),
8185 })
8186 .await
8187 .unwrap();
8188
8189 orchestrator
8190 .send_team_message(
8191 "run-shared",
8192 Some("task-shared".to_string()),
8193 TeamMessageKind::StatusUpdate,
8194 Some("supervisor".to_string()),
8195 Some("agent-impl".to_string()),
8196 "Use ripgrep first and keep the worktree clean.",
8197 )
8198 .await
8199 .unwrap();
8200
8201 let run = orchestrator.get_supervisor_run("run-shared").await.unwrap();
8202 assert_eq!(run.shared_cognition.len(), 1);
8203 let note = &run.shared_cognition[0];
8204 assert_eq!(note.kind, SharedCognitionKind::Steering);
8205 assert_eq!(note.task_id.as_deref(), Some("task-shared"));
8206 assert_eq!(note.directive_id.as_deref(), Some("directive-shared"));
8207 assert!(note.tags.contains(&SHARED_COGNITION_TAG.to_string()));
8208 assert!(note.tags.contains(&workflow_run_memory_tag("run-shared")));
8209
8210 let query = crate::memory_bank::MemoryBankQuery::default()
8211 .with_category(SHARED_COGNITION_CATEGORY)
8212 .with_task("task-shared")
8213 .with_directive("directive-shared")
8214 .with_tags(vec![workflow_run_memory_tag("run-shared")])
8215 .with_limit(5);
8216 let results = crate::memory_bank::search_memory_bank_with_query(tmp.path(), &query)
8217 .await
8218 .unwrap();
8219 assert_eq!(results.len(), 1);
8220 assert_eq!(
8221 results[0].entry.category.as_deref(),
8222 Some(SHARED_COGNITION_CATEGORY)
8223 );
8224 assert!(
8225 results[0]
8226 .entry
8227 .tags
8228 .contains(&workflow_run_memory_tag("run-shared"))
8229 );
8230 }
8231
8232 #[tokio::test]
8233 async fn test_team_message_publishes_partial_discovery_and_blocker_shared_cognition() {
8234 let tmp = tempdir().unwrap();
8235 let manager =
8236 crate::agents::AgentManager::new(tmp.path().join("shared-cognition-partial.db"));
8237 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8238
8239 orchestrator
8240 .delegate_task(DelegatedTask {
8241 id: "task-partial".into(),
8242 agent_id: "agent-impl".into(),
8243 prompt: "Investigate the flaky test".into(),
8244 context: None,
8245 required_tools: vec![],
8246 priority: 1,
8247 session_id: Some("session-partial".into()),
8248 directive_id: Some("directive-partial".into()),
8249 tracking_task_id: None,
8250 run_id: Some("run-partial".into()),
8251 parent_task_id: None,
8252 depends_on: vec![],
8253 role: Some(AgentRole::Implementer),
8254 delegation_brief: None,
8255 planning_only: false,
8256 approval_required: true,
8257 reviewer_required: false,
8258 test_required: false,
8259 workspace_dir: Some(tmp.path().to_path_buf()),
8260 execution_mode: AgentExecutionMode::SharedWorkspace,
8261 environment_id: None,
8262 remote_target: None,
8263 memory_tags: vec!["flaky-test".to_string()],
8264 name: Some("Investigate flaky test".into()),
8265 })
8266 .await
8267 .unwrap();
8268
8269 orchestrator
8270 .send_team_message(
8271 "run-partial",
8272 Some("task-partial".to_string()),
8273 TeamMessageKind::StatusUpdate,
8274 Some("agent-impl".to_string()),
8275 Some("supervisor".to_string()),
8276 "Partial finding: the failure appears after the fixture cache is cleared.",
8277 )
8278 .await
8279 .unwrap();
8280 orchestrator
8281 .send_team_message(
8282 "run-partial",
8283 Some("task-partial".to_string()),
8284 TeamMessageKind::Blocker,
8285 Some("agent-impl".to_string()),
8286 Some("supervisor".to_string()),
8287 "Blocked on reproducing the cleanup timing issue without the CI fixture bundle.",
8288 )
8289 .await
8290 .unwrap();
8291
8292 let run = orchestrator
8293 .get_supervisor_run("run-partial")
8294 .await
8295 .unwrap();
8296 assert_eq!(run.shared_cognition.len(), 2);
8297 assert_eq!(run.shared_cognition[0].kind, SharedCognitionKind::Discovery);
8298 assert_eq!(run.shared_cognition[1].kind, SharedCognitionKind::Blocker);
8299 assert_eq!(
8300 run.shared_cognition[0].sender_agent_id.as_deref(),
8301 Some("agent-impl")
8302 );
8303 assert_eq!(
8304 run.shared_cognition[1].sender_agent_id.as_deref(),
8305 Some("agent-impl")
8306 );
8307 assert!(run.shared_cognition[0].confidence < run.shared_cognition[1].confidence);
8308
8309 assert!(run.shared_cognition[0].summary.contains("Partial finding"));
8310 assert!(
8311 run.shared_cognition[1]
8312 .summary
8313 .contains("Blocked on reproducing")
8314 );
8315 }
8316
8317 #[tokio::test]
8318 async fn test_team_message_publishes_conflicting_hypotheses_from_multiple_agents() {
8319 let tmp = tempdir().unwrap();
8320 let manager =
8321 crate::agents::AgentManager::new(tmp.path().join("shared-cognition-hypotheses.db"));
8322 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8323
8324 let make_task = |id: &str, agent_id: &str| DelegatedTask {
8325 id: id.to_string(),
8326 agent_id: agent_id.to_string(),
8327 prompt: "Compare ownership approaches".into(),
8328 context: None,
8329 required_tools: vec![],
8330 priority: 1,
8331 session_id: Some("session-hypothesis".into()),
8332 directive_id: Some("directive-hypothesis".into()),
8333 tracking_task_id: None,
8334 run_id: Some("run-hypothesis".into()),
8335 parent_task_id: None,
8336 depends_on: vec![],
8337 role: Some(AgentRole::Implementer),
8338 delegation_brief: None,
8339 planning_only: false,
8340 approval_required: true,
8341 reviewer_required: false,
8342 test_required: false,
8343 workspace_dir: Some(tmp.path().to_path_buf()),
8344 execution_mode: AgentExecutionMode::SharedWorkspace,
8345 environment_id: None,
8346 remote_target: None,
8347 memory_tags: vec!["ownership".to_string()],
8348 name: Some(format!("Ownership check {agent_id}")),
8349 };
8350
8351 orchestrator
8352 .delegate_task(make_task("task-h1", "agent-a"))
8353 .await
8354 .unwrap();
8355 orchestrator
8356 .delegate_task(make_task("task-h2", "agent-b"))
8357 .await
8358 .unwrap();
8359 {
8360 let mut runs = orchestrator.supervisor_runs.lock().await;
8361 runs.get_mut("run-hypothesis").unwrap().lead_agent_id = Some("supervisor".to_string());
8362 }
8363
8364 orchestrator
8365 .send_team_message(
8366 "run-hypothesis",
8367 Some("task-h1".to_string()),
8368 TeamMessageKind::Clarification,
8369 Some("agent-a".to_string()),
8370 Some("supervisor".to_string()),
8371 "Hypothesis A: the bug is in ownership normalization before task routing.",
8372 )
8373 .await
8374 .unwrap();
8375 orchestrator
8376 .send_team_message(
8377 "run-hypothesis",
8378 Some("task-h2".to_string()),
8379 TeamMessageKind::Clarification,
8380 Some("agent-b".to_string()),
8381 Some("supervisor".to_string()),
8382 "Hypothesis B: the bug is downstream in the assignment cache after routing succeeds.",
8383 )
8384 .await
8385 .unwrap();
8386
8387 let run = orchestrator
8388 .get_supervisor_run("run-hypothesis")
8389 .await
8390 .unwrap();
8391 let hypothesis_notes = run
8392 .shared_cognition
8393 .iter()
8394 .filter(|note| note.kind == SharedCognitionKind::Hypothesis)
8395 .collect::<Vec<_>>();
8396 assert_eq!(hypothesis_notes.len(), 2);
8397 assert!(
8398 hypothesis_notes
8399 .iter()
8400 .any(|note| note.sender_agent_id.as_deref() == Some("agent-a"))
8401 );
8402 assert!(
8403 hypothesis_notes
8404 .iter()
8405 .any(|note| note.sender_agent_id.as_deref() == Some("agent-b"))
8406 );
8407
8408 assert!(
8409 hypothesis_notes
8410 .iter()
8411 .any(|note| note.summary.contains("Hypothesis A"))
8412 );
8413 assert!(
8414 hypothesis_notes
8415 .iter()
8416 .any(|note| note.summary.contains("Hypothesis B"))
8417 );
8418 }
8419
8420 #[tokio::test]
8421 #[cfg(not(target_os = "windows"))]
8422 async fn test_child_supervisor_run_inherits_policy_and_task_defaults() {
8423 use std::process::Command;
8424
8425 let tmp = tempdir().unwrap();
8426 assert!(
8427 Command::new("git")
8428 .args(["init", "-b", "main"])
8429 .current_dir(tmp.path())
8430 .status()
8431 .unwrap()
8432 .success()
8433 );
8434 std::fs::write(tmp.path().join("README.md"), "workspace root\n").unwrap();
8435 assert!(
8436 Command::new("git")
8437 .args(["add", "."])
8438 .current_dir(tmp.path())
8439 .status()
8440 .unwrap()
8441 .success()
8442 );
8443 assert!(
8444 Command::new("git")
8445 .args([
8446 "-c",
8447 "user.name=Gestura Tests",
8448 "-c",
8449 "user.email=tests@example.com",
8450 "commit",
8451 "-m",
8452 "Initial commit",
8453 ])
8454 .current_dir(tmp.path())
8455 .status()
8456 .unwrap()
8457 .success()
8458 );
8459
8460 let manager = crate::agents::AgentManager::new(tmp.path().join("child-hierarchy.db"));
8461 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8462
8463 let mut parent_run = empty_supervisor_run("run-parent", tmp.path().to_path_buf());
8464 parent_run.session_id = Some("session-child-hierarchy".to_string());
8465 parent_run.inherited_policy = Some(SupervisorInheritancePolicy {
8466 approval_required: true,
8467 reviewer_required: false,
8468 test_required: false,
8469 execution_mode: Some(AgentExecutionMode::GitWorktree),
8470 workspace_dir: Some(tmp.path().to_path_buf()),
8471 memory_tags: vec!["root-tag".to_string()],
8472 constraint_notes: vec!["Stay within app scope".to_string()],
8473 });
8474 orchestrator
8475 .supervisor_runs
8476 .lock()
8477 .await
8478 .insert(parent_run.id.clone(), parent_run);
8479
8480 let child_run = orchestrator
8481 .create_child_supervisor_run(ChildSupervisorRunRequest {
8482 parent_run_id: "run-parent".to_string(),
8483 run_id: Some("run-child".to_string()),
8484 lead_agent_id: "supervisor-frontend".to_string(),
8485 objective: "Own frontend delivery".to_string(),
8486 name: Some("Frontend pod".to_string()),
8487 parent_task_id: None,
8488 session_id: None,
8489 workspace_dir: None,
8490 approval_required: false,
8491 reviewer_required: true,
8492 test_required: false,
8493 execution_mode: AgentExecutionMode::GitWorktree,
8494 memory_tags: vec!["child-tag".to_string()],
8495 constraint_notes: vec!["Escalate API changes".to_string()],
8496 })
8497 .await
8498 .unwrap();
8499
8500 assert_eq!(child_run.hierarchy_depth, 1);
8501 assert_eq!(
8502 child_run.parent_run.as_ref().unwrap().parent_run_id,
8503 "run-parent"
8504 );
8505 let inherited = child_run.inherited_policy.as_ref().unwrap();
8506 assert!(inherited.approval_required);
8507 assert!(inherited.reviewer_required);
8508 assert!(inherited.memory_tags.contains(&"root-tag".to_string()));
8509 assert!(inherited.memory_tags.contains(&"child-tag".to_string()));
8510
8511 orchestrator
8512 .delegate_task(DelegatedTask {
8513 id: "task-child".into(),
8514 agent_id: "implementer-1".into(),
8515 prompt: "Implement frontend flow".into(),
8516 context: None,
8517 required_tools: vec![],
8518 priority: 2,
8519 session_id: None,
8520 directive_id: None,
8521 tracking_task_id: None,
8522 run_id: Some("run-child".into()),
8523 parent_task_id: None,
8524 depends_on: vec![],
8525 role: Some(AgentRole::Implementer),
8526 delegation_brief: None,
8527 planning_only: false,
8528 approval_required: false,
8529 reviewer_required: false,
8530 test_required: false,
8531 workspace_dir: None,
8532 execution_mode: AgentExecutionMode::SharedWorkspace,
8533 environment_id: None,
8534 remote_target: None,
8535 memory_tags: vec![],
8536 name: Some("Frontend implementation".into()),
8537 })
8538 .await
8539 .unwrap();
8540
8541 let child_run = orchestrator.get_supervisor_run("run-child").await.unwrap();
8542 let record = child_run.tasks.first().unwrap();
8543 assert_eq!(record.state, SupervisorTaskState::PendingApproval);
8544 assert_eq!(record.task.session_id, child_run.session_id);
8545 assert_eq!(record.task.workspace_dir, child_run.workspace_dir);
8546 assert!(record.task.tracking_task_id.is_some());
8547 assert_eq!(record.task.execution_mode, AgentExecutionMode::GitWorktree);
8548 assert!(record.task.memory_tags.contains(&"root-tag".to_string()));
8549 assert!(record.task.memory_tags.contains(&"child-tag".to_string()));
8550 assert!(
8551 record
8552 .task
8553 .memory_tags
8554 .contains(&SHARED_COGNITION_TAG.to_string())
8555 );
8556 assert!(
8557 record
8558 .task
8559 .memory_tags
8560 .contains(&workflow_run_memory_tag("run-child"))
8561 );
8562 }
8563
8564 #[test]
8565 fn local_execution_progress_from_chunk_tracks_iterations_and_tool_results() {
8566 let mut context = LocalExecutionTelemetryContext::default();
8567 let iteration_progress = local_execution_progress_from_chunk(
8568 &StreamChunk::AgentLoopIteration { iteration: 2 },
8569 &context,
8570 )
8571 .expect("iteration progress should be captured");
8572 assert_eq!(iteration_progress.stage.as_deref(), Some("agent_loop"));
8573 assert_eq!(iteration_progress.iteration, 2);
8574
8575 context.iteration = 2;
8576 let tool_start = local_execution_progress_from_chunk(
8577 &StreamChunk::ToolCallStart {
8578 id: "tool-1".to_string(),
8579 name: "file".to_string(),
8580 },
8581 &context,
8582 )
8583 .expect("tool start progress should be captured");
8584 assert_eq!(tool_start.stage.as_deref(), Some("executing_tools"));
8585 assert_eq!(tool_start.current_tool_name.as_deref(), Some("file"));
8586
8587 context.completed_tool_call_count = 1;
8588 context.last_completed_tool_duration_ms = Some(14);
8589 let tool_result = local_execution_progress_from_chunk(
8590 &StreamChunk::ToolCallResult {
8591 name: "file".to_string(),
8592 success: true,
8593 output: "{\"ok\":true}".to_string(),
8594 duration_ms: 14,
8595 },
8596 &context,
8597 )
8598 .expect("tool result progress should be captured");
8599 assert_eq!(tool_result.completed_tool_call_count, 1);
8600 assert_eq!(tool_result.last_completed_tool_duration_ms, Some(14));
8601 assert_eq!(
8602 tool_result.message.as_deref(),
8603 Some("Tool 'file' completed successfully")
8604 );
8605 }
8606
8607 #[test]
8608 fn local_execution_progress_from_chunk_captures_waiting_and_token_usage() {
8609 let mut context = LocalExecutionTelemetryContext {
8610 iteration: 3,
8611 current_tool_name: Some("shell".to_string()),
8612 has_partial_content: true,
8613 partial_content_chars: 18,
8614 ..LocalExecutionTelemetryContext::default()
8615 };
8616
8617 let shell_progress = local_execution_progress_from_chunk(
8618 &StreamChunk::ShellLifecycle {
8619 process_id: "cmd-1".to_string(),
8620 shell_session_id: None,
8621 duration_ms: None,
8622 command: "cargo test".to_string(),
8623 state: crate::streaming::ShellProcessState::Started,
8624 exit_code: None,
8625 cwd: None,
8626 },
8627 &context,
8628 )
8629 .expect("shell lifecycle progress should be captured");
8630 assert_eq!(shell_progress.phase, LocalExecutionPhase::Waiting);
8631 assert_eq!(
8632 shell_progress.waiting_reason,
8633 Some(LocalExecutionWaitingReason::ShellProcess)
8634 );
8635 assert!(shell_progress.has_partial_content);
8636
8637 context.token_usage = Some(LocalExecutionTokenUsageSnapshot {
8638 estimated_tokens: Some(256),
8639 limit: Some(4096),
8640 percentage: Some(6),
8641 status: Some("green".to_string()),
8642 estimated_cost_usd: Some(0.0001),
8643 input_tokens: None,
8644 output_tokens: None,
8645 total_tokens: None,
8646 model: None,
8647 provider: None,
8648 });
8649 let token_progress = local_execution_progress_from_chunk(
8650 &StreamChunk::TokenUsageUpdate {
8651 estimated: 512,
8652 limit: 4096,
8653 percentage: 12,
8654 status: crate::streaming::TokenUsageStatus::Green,
8655 estimated_cost: 0.0002,
8656 },
8657 &context,
8658 )
8659 .expect("token usage progress should be captured");
8660 let token_usage = token_progress
8661 .token_usage
8662 .as_ref()
8663 .expect("token usage snapshot should be stored");
8664 assert_eq!(token_usage.estimated_tokens, Some(512));
8665 assert_eq!(token_usage.limit, Some(4096));
8666 assert_eq!(token_usage.percentage, Some(12));
8667 }
8668
8669 #[test]
8670 fn background_message_prefers_local_execution_progress_when_available() {
8671 let now = Utc::now();
8672 let record = SupervisorTaskRecord {
8673 task: DelegatedTask {
8674 id: "task-local-progress".to_string(),
8675 agent_id: "agent-local".to_string(),
8676 prompt: "Implement local telemetry".to_string(),
8677 context: None,
8678 required_tools: vec![],
8679 priority: 1,
8680 session_id: None,
8681 directive_id: None,
8682 tracking_task_id: None,
8683 run_id: Some("run-local-progress".to_string()),
8684 parent_task_id: None,
8685 depends_on: vec![],
8686 role: Some(AgentRole::Implementer),
8687 delegation_brief: None,
8688 planning_only: false,
8689 approval_required: false,
8690 reviewer_required: false,
8691 test_required: false,
8692 workspace_dir: Some(std::env::temp_dir()),
8693 execution_mode: AgentExecutionMode::SharedWorkspace,
8694 environment_id: Some("env-local".to_string()),
8695 remote_target: None,
8696 memory_tags: vec![],
8697 name: Some("Local telemetry".to_string()),
8698 },
8699 state: SupervisorTaskState::Running,
8700 approval: TaskApprovalRecord::default(),
8701 environment_id: "env-local".to_string(),
8702 environment: test_environment("env-local", std::env::temp_dir()),
8703 claimed_by: Some("agent-local".to_string()),
8704 attempts: 1,
8705 blocked_reasons: vec![],
8706 result: None,
8707 remote_execution: None,
8708 local_execution: Some(LocalExecutionRecord {
8709 status: "running".to_string(),
8710 status_reason: None,
8711 progress: Some(LocalExecutionProgress {
8712 phase: LocalExecutionPhase::Running,
8713 waiting_reason: Some(LocalExecutionWaitingReason::ShellProcess),
8714 stage: Some("executing_tools".to_string()),
8715 message: Some("Running tool 'file'".to_string()),
8716 percent: Some(40),
8717 iteration: 2,
8718 current_tool_name: Some("file".to_string()),
8719 last_completed_tool_name: Some("read".to_string()),
8720 last_completed_tool_duration_ms: Some(9),
8721 completed_tool_call_count: 1,
8722 has_partial_content: true,
8723 partial_content_chars: 24,
8724 has_partial_thinking: false,
8725 partial_thinking_chars: 0,
8726 token_usage: Some(LocalExecutionTokenUsageSnapshot {
8727 estimated_tokens: Some(256),
8728 limit: Some(4096),
8729 percentage: Some(6),
8730 status: Some("green".to_string()),
8731 estimated_cost_usd: Some(0.0001),
8732 input_tokens: None,
8733 output_tokens: None,
8734 total_tokens: None,
8735 model: None,
8736 provider: None,
8737 }),
8738 environment: Some(environment_snapshot_from_execution(&test_environment(
8739 "env-local",
8740 std::env::temp_dir(),
8741 ))),
8742 updated_at: now,
8743 }),
8744 last_synced_at: now,
8745 }),
8746 messages: vec![],
8747 checkpoint: None,
8748 created_at: now,
8749 updated_at: now,
8750 started_at: Some(now),
8751 completed_at: None,
8752 };
8753
8754 let message = background_message_for_record(&record);
8755 assert!(message.contains("Local running"));
8756 assert!(message.contains("phase running"));
8757 assert!(message.contains("waiting shellprocess"));
8758 assert!(message.contains("executing_tools"));
8759 assert!(message.contains("tool file"));
8760 assert!(message.contains("1 tool call(s)"));
8761 assert!(message.contains("tokens 256/4096 (6%)"));
8762 assert!(message.contains("env ready"));
8763 }
8764
8765 #[test]
8766 fn background_message_surfaces_remote_progress_and_reason() {
8767 let now = Utc::now();
8768 let record = SupervisorTaskRecord {
8769 task: DelegatedTask {
8770 id: "task-remote-progress".to_string(),
8771 agent_id: "agent-remote".to_string(),
8772 prompt: "Track remote telemetry".to_string(),
8773 context: None,
8774 required_tools: vec![],
8775 priority: 1,
8776 session_id: None,
8777 directive_id: None,
8778 tracking_task_id: None,
8779 run_id: Some("run-remote-progress".to_string()),
8780 parent_task_id: None,
8781 depends_on: vec![],
8782 role: Some(AgentRole::Implementer),
8783 delegation_brief: None,
8784 planning_only: false,
8785 approval_required: false,
8786 reviewer_required: false,
8787 test_required: false,
8788 workspace_dir: None,
8789 execution_mode: AgentExecutionMode::Remote,
8790 environment_id: None,
8791 remote_target: Some(RemoteAgentTarget {
8792 url: "http://localhost:32145/a2a".to_string(),
8793 name: Some("remote-peer".to_string()),
8794 auth_token: None,
8795 capabilities: vec!["shell".to_string()],
8796 }),
8797 memory_tags: vec![],
8798 name: Some("Remote telemetry".to_string()),
8799 },
8800 state: SupervisorTaskState::Blocked,
8801 approval: TaskApprovalRecord::default(),
8802 environment_id: "env-remote".to_string(),
8803 environment: test_environment("env-remote", std::env::temp_dir()),
8804 claimed_by: Some("agent-remote".to_string()),
8805 attempts: 1,
8806 blocked_reasons: vec!["Awaiting remote shell completion".to_string()],
8807 result: None,
8808 remote_execution: Some(RemoteExecutionRecord {
8809 target: RemoteAgentTarget {
8810 url: "http://localhost:32145/a2a".to_string(),
8811 name: Some("remote-peer".to_string()),
8812 auth_token: None,
8813 capabilities: vec!["shell".to_string()],
8814 },
8815 remote_task_id: "remote-task-1".to_string(),
8816 status: "blocked".to_string(),
8817 status_reason: Some("Awaiting remote shell completion".to_string()),
8818 lease: None,
8819 progress: Some(RemoteExecutionProgress {
8820 stage: Some("shell_running".to_string()),
8821 message: Some("Remote shell still streaming".to_string()),
8822 percent: Some(60),
8823 updated_at: now,
8824 }),
8825 artifacts: vec![RemoteExecutionArtifact {
8826 name: "result.txt".to_string(),
8827 part_count: 1,
8828 metadata: HashMap::new(),
8829 }],
8830 provenance: None,
8831 compatibility: RemoteExecutionCompatibility {
8832 supported_features: vec!["artifacts".to_string()],
8833 warnings: vec![],
8834 protocol_version: Some("2025-11-25".to_string()),
8835 },
8836 last_synced_at: now,
8837 }),
8838 local_execution: None,
8839 messages: vec![],
8840 checkpoint: None,
8841 created_at: now,
8842 updated_at: now,
8843 started_at: Some(now),
8844 completed_at: None,
8845 };
8846
8847 let message = background_message_for_record(&record);
8848 assert!(message.contains("Remote blocked"));
8849 assert!(message.contains("Awaiting remote shell completion"));
8850 assert!(message.contains("60%"));
8851 assert!(message.contains("shell_running"));
8852 }
8853
8854 #[tokio::test]
8855 async fn sync_local_execution_progress_persists_and_notifies_observer() {
8856 let tmp = tempdir().unwrap();
8857 let manager = crate::agents::AgentManager::new(tmp.path().join("local-progress.db"));
8858 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8859 let observer = Arc::new(RecordingObserver::default());
8860 orchestrator.set_observer(observer.clone()).await;
8861
8862 let task = DelegatedTask {
8863 id: "task-local-sync".to_string(),
8864 agent_id: "agent-local".to_string(),
8865 prompt: "Implement telemetry".to_string(),
8866 context: None,
8867 required_tools: vec![],
8868 priority: 1,
8869 session_id: None,
8870 directive_id: None,
8871 tracking_task_id: None,
8872 run_id: Some("run-local-sync".to_string()),
8873 parent_task_id: None,
8874 depends_on: vec![],
8875 role: Some(AgentRole::Implementer),
8876 delegation_brief: None,
8877 planning_only: false,
8878 approval_required: false,
8879 reviewer_required: false,
8880 test_required: false,
8881 workspace_dir: Some(tmp.path().to_path_buf()),
8882 execution_mode: AgentExecutionMode::SharedWorkspace,
8883 environment_id: Some("env-local-sync".to_string()),
8884 remote_target: None,
8885 memory_tags: vec![],
8886 name: Some("Local sync".to_string()),
8887 };
8888 let environment = test_environment("env-local-sync", tmp.path().to_path_buf());
8889 orchestrator.supervisor_runs.lock().await.insert(
8890 "run-local-sync".to_string(),
8891 SupervisorRun {
8892 status: SupervisorRunStatus::Running,
8893 task_summary: SupervisorRunTaskSummary {
8894 total: 1,
8895 running: 1,
8896 ..SupervisorRunTaskSummary::default()
8897 },
8898 tasks: vec![SupervisorTaskRecord {
8899 task: task.clone(),
8900 state: SupervisorTaskState::Running,
8901 approval: TaskApprovalRecord::default(),
8902 environment_id: environment.id.clone(),
8903 environment,
8904 claimed_by: Some("agent-local".to_string()),
8905 attempts: 1,
8906 blocked_reasons: vec![],
8907 result: None,
8908 remote_execution: None,
8909 local_execution: None,
8910 messages: vec![],
8911 checkpoint: None,
8912 created_at: Utc::now(),
8913 updated_at: Utc::now(),
8914 started_at: Some(Utc::now()),
8915 completed_at: None,
8916 }],
8917 ..empty_supervisor_run("run-local-sync", tmp.path().to_path_buf())
8918 },
8919 );
8920
8921 let progress = LocalExecutionProgress {
8922 phase: LocalExecutionPhase::Running,
8923 waiting_reason: None,
8924 stage: Some("executing_tools".to_string()),
8925 message: Some("Running tool 'file'".to_string()),
8926 percent: Some(35),
8927 iteration: 1,
8928 current_tool_name: Some("file".to_string()),
8929 last_completed_tool_name: Some("read".to_string()),
8930 last_completed_tool_duration_ms: Some(11),
8931 completed_tool_call_count: 1,
8932 has_partial_content: true,
8933 partial_content_chars: 42,
8934 has_partial_thinking: true,
8935 partial_thinking_chars: 17,
8936 token_usage: Some(LocalExecutionTokenUsageSnapshot {
8937 estimated_tokens: Some(512),
8938 limit: Some(4096),
8939 percentage: Some(12),
8940 status: Some("green".to_string()),
8941 estimated_cost_usd: Some(0.0002),
8942 input_tokens: None,
8943 output_tokens: None,
8944 total_tokens: None,
8945 model: None,
8946 provider: None,
8947 }),
8948 environment: None,
8949 updated_at: Utc::now(),
8950 };
8951
8952 orchestrator
8953 .sync_local_execution_progress(&task, progress.clone())
8954 .await
8955 .expect("local progress sync should succeed");
8956
8957 let run = orchestrator
8958 .get_supervisor_run("run-local-sync")
8959 .await
8960 .expect("run should persist");
8961 let record = run.tasks.first().expect("task record should persist");
8962 let local_execution = record
8963 .local_execution
8964 .as_ref()
8965 .expect("local execution should be stored on the task");
8966 assert_eq!(local_execution.status, "running");
8967 assert_eq!(
8968 local_execution
8969 .progress
8970 .as_ref()
8971 .and_then(|progress| progress.current_tool_name.as_deref()),
8972 Some("file")
8973 );
8974
8975 let observed = observer.runs.lock().await;
8976 assert!(observed.iter().any(|run| {
8977 run.id == "run-local-sync"
8978 && run
8979 .tasks
8980 .first()
8981 .and_then(|record| record.local_execution.as_ref())
8982 .and_then(|local| local.progress.as_ref())
8983 .and_then(|progress| progress.stage.as_deref())
8984 == Some("executing_tools")
8985 }));
8986 }
8987
8988 #[tokio::test]
8989 async fn test_child_supervisor_run_updates_parent_rollup_status() {
8990 let tmp = tempdir().unwrap();
8991 let manager = crate::agents::AgentManager::new(tmp.path().join("child-rollup.db"));
8992 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
8993
8994 orchestrator.supervisor_runs.lock().await.insert(
8995 "run-parent".to_string(),
8996 empty_supervisor_run("run-parent", tmp.path().to_path_buf()),
8997 );
8998
8999 orchestrator
9000 .create_child_supervisor_run(ChildSupervisorRunRequest {
9001 parent_run_id: "run-parent".to_string(),
9002 run_id: Some("run-child".to_string()),
9003 lead_agent_id: "supervisor-child".to_string(),
9004 objective: "Handle the frontend pod".to_string(),
9005 name: Some("Frontend pod".to_string()),
9006 parent_task_id: None,
9007 session_id: None,
9008 workspace_dir: None,
9009 approval_required: true,
9010 reviewer_required: false,
9011 test_required: false,
9012 execution_mode: AgentExecutionMode::SharedWorkspace,
9013 memory_tags: vec![],
9014 constraint_notes: vec![],
9015 })
9016 .await
9017 .unwrap();
9018
9019 orchestrator
9020 .delegate_task(DelegatedTask {
9021 id: "task-child-rollup".into(),
9022 agent_id: "implementer-rollup".into(),
9023 prompt: "Build the assigned scope".into(),
9024 context: None,
9025 required_tools: vec![],
9026 priority: 1,
9027 session_id: None,
9028 directive_id: None,
9029 tracking_task_id: None,
9030 run_id: Some("run-child".into()),
9031 parent_task_id: None,
9032 depends_on: vec![],
9033 role: Some(AgentRole::Implementer),
9034 delegation_brief: None,
9035 planning_only: false,
9036 approval_required: false,
9037 reviewer_required: false,
9038 test_required: false,
9039 workspace_dir: None,
9040 execution_mode: AgentExecutionMode::SharedWorkspace,
9041 environment_id: None,
9042 remote_target: None,
9043 memory_tags: vec![],
9044 name: Some("Build child scope".into()),
9045 })
9046 .await
9047 .unwrap();
9048
9049 let parent_waiting = orchestrator.get_supervisor_run("run-parent").await.unwrap();
9050 assert_eq!(parent_waiting.status, SupervisorRunStatus::Waiting);
9051 assert_eq!(parent_waiting.child_runs.len(), 1);
9052 assert_eq!(
9053 parent_waiting.child_runs[0].status,
9054 SupervisorRunStatus::Waiting
9055 );
9056
9057 orchestrator
9058 .approve_task(
9059 "task-child-rollup",
9060 ApprovalActor::new(ApprovalActorKind::Supervisor, "supervisor-root"),
9061 Some("Proceed".into()),
9062 )
9063 .await
9064 .unwrap();
9065
9066 let child_task = orchestrator
9067 .get_supervisor_run("run-child")
9068 .await
9069 .unwrap()
9070 .tasks
9071 .first()
9072 .unwrap()
9073 .task
9074 .clone();
9075 orchestrator
9076 .complete_task_execution(
9077 child_task,
9078 TaskResult {
9079 task_id: "task-child-rollup".to_string(),
9080 agent_id: "implementer-rollup".to_string(),
9081 run_id: Some("run-child".to_string()),
9082 tracking_task_id: None,
9083 success: true,
9084 output: "Completed child work".to_string(),
9085 summary: Some("Completed child work".to_string()),
9086 tool_calls: Vec::new(),
9087 artifacts: Vec::new(),
9088 terminal_state_hint: Some(TaskTerminalStateHint::Completed),
9089 duration_ms: 10,
9090 },
9091 false,
9092 0,
9093 )
9094 .await
9095 .unwrap();
9096
9097 let parent_completed = orchestrator.get_supervisor_run("run-parent").await.unwrap();
9098 assert_eq!(parent_completed.status, SupervisorRunStatus::Completed);
9099 assert_eq!(
9100 parent_completed.child_runs[0].status,
9101 SupervisorRunStatus::Completed
9102 );
9103 }
9104
9105 #[tokio::test]
9106 async fn test_child_supervisor_run_cannot_create_grandchild() {
9107 let tmp = tempdir().unwrap();
9108 let manager = crate::agents::AgentManager::new(tmp.path().join("grandchild-block.db"));
9109 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
9110
9111 orchestrator.supervisor_runs.lock().await.insert(
9112 "run-parent".to_string(),
9113 empty_supervisor_run("run-parent", tmp.path().to_path_buf()),
9114 );
9115
9116 orchestrator
9117 .create_child_supervisor_run(ChildSupervisorRunRequest {
9118 parent_run_id: "run-parent".to_string(),
9119 run_id: Some("run-child".to_string()),
9120 lead_agent_id: "supervisor-child".to_string(),
9121 objective: "Manage sub-scope".to_string(),
9122 name: None,
9123 parent_task_id: None,
9124 session_id: None,
9125 workspace_dir: None,
9126 approval_required: false,
9127 reviewer_required: false,
9128 test_required: false,
9129 execution_mode: AgentExecutionMode::SharedWorkspace,
9130 memory_tags: vec![],
9131 constraint_notes: vec![],
9132 })
9133 .await
9134 .unwrap();
9135
9136 let error = orchestrator
9137 .create_child_supervisor_run(ChildSupervisorRunRequest {
9138 parent_run_id: "run-child".to_string(),
9139 run_id: Some("run-grandchild".to_string()),
9140 lead_agent_id: "supervisor-grandchild".to_string(),
9141 objective: "Forbidden depth".to_string(),
9142 name: None,
9143 parent_task_id: None,
9144 session_id: None,
9145 workspace_dir: None,
9146 approval_required: false,
9147 reviewer_required: false,
9148 test_required: false,
9149 execution_mode: AgentExecutionMode::SharedWorkspace,
9150 memory_tags: vec![],
9151 constraint_notes: vec![],
9152 })
9153 .await
9154 .unwrap_err();
9155
9156 assert!(error.contains("maximum hierarchy depth") || error.contains("child run"));
9157 }
9158
9159 #[tokio::test]
9160 async fn test_hierarchy_query_apis_return_ancestry_descendants_and_leaf_tasks() {
9161 let tmp = tempdir().unwrap();
9162 let manager = crate::agents::AgentManager::new(tmp.path().join("hierarchy-queries.db"));
9163 let orchestrator = AgentOrchestrator::new(manager, AppConfig::default());
9164
9165 orchestrator.supervisor_runs.lock().await.insert(
9166 "run-parent".to_string(),
9167 empty_supervisor_run("run-parent", tmp.path().to_path_buf()),
9168 );
9169 orchestrator
9170 .create_child_supervisor_run(ChildSupervisorRunRequest {
9171 parent_run_id: "run-parent".to_string(),
9172 run_id: Some("run-child".to_string()),
9173 lead_agent_id: "supervisor-child".to_string(),
9174 objective: "Own sub-scope".to_string(),
9175 name: None,
9176 parent_task_id: None,
9177 session_id: None,
9178 workspace_dir: None,
9179 approval_required: false,
9180 reviewer_required: false,
9181 test_required: false,
9182 execution_mode: AgentExecutionMode::SharedWorkspace,
9183 memory_tags: vec![],
9184 constraint_notes: vec![],
9185 })
9186 .await
9187 .unwrap();
9188 orchestrator
9189 .delegate_task(DelegatedTask {
9190 id: "task-parent".into(),
9191 agent_id: "agent-parent".into(),
9192 prompt: "Root task".into(),
9193 context: None,
9194 required_tools: vec![],
9195 priority: 1,
9196 session_id: None,
9197 directive_id: None,
9198 tracking_task_id: None,
9199 run_id: Some("run-parent".into()),
9200 parent_task_id: None,
9201 depends_on: vec![],
9202 role: Some(AgentRole::Implementer),
9203 delegation_brief: None,
9204 planning_only: false,
9205 approval_required: false,
9206 reviewer_required: false,
9207 test_required: false,
9208 workspace_dir: None,
9209 execution_mode: AgentExecutionMode::SharedWorkspace,
9210 environment_id: None,
9211 remote_target: None,
9212 memory_tags: vec![],
9213 name: Some("Root task".into()),
9214 })
9215 .await
9216 .unwrap();
9217 orchestrator
9218 .delegate_task(DelegatedTask {
9219 id: "task-child-query".into(),
9220 agent_id: "agent-child".into(),
9221 prompt: "Child task".into(),
9222 context: None,
9223 required_tools: vec![],
9224 priority: 1,
9225 session_id: None,
9226 directive_id: None,
9227 tracking_task_id: None,
9228 run_id: Some("run-child".into()),
9229 parent_task_id: None,
9230 depends_on: vec![],
9231 role: Some(AgentRole::Implementer),
9232 delegation_brief: None,
9233 planning_only: false,
9234 approval_required: false,
9235 reviewer_required: false,
9236 test_required: false,
9237 workspace_dir: None,
9238 execution_mode: AgentExecutionMode::SharedWorkspace,
9239 environment_id: None,
9240 remote_target: None,
9241 memory_tags: vec![],
9242 name: Some("Child task".into()),
9243 })
9244 .await
9245 .unwrap();
9246
9247 let ancestry = orchestrator.get_supervisor_run_ancestry("run-child").await;
9248 assert_eq!(ancestry.len(), 1);
9249 assert_eq!(ancestry[0].id, "run-parent");
9250
9251 let descendants = orchestrator
9252 .get_supervisor_run_descendants("run-parent")
9253 .await;
9254 assert_eq!(descendants.len(), 1);
9255 assert_eq!(descendants[0].id, "run-child");
9256
9257 let leaf_tasks = orchestrator.list_supervisor_leaf_tasks("run-parent").await;
9258 let leaf_ids = leaf_tasks
9259 .into_iter()
9260 .map(|record| record.task.id)
9261 .collect::<Vec<_>>();
9262 assert!(leaf_ids.contains(&"task-parent".to_string()));
9263 assert!(leaf_ids.contains(&"task-child-query".to_string()));
9264 }
9265
9266 #[tokio::test]
9267 async fn test_child_run_cancellation_and_retry_roll_up_to_parent() {
9268 let tmp = tempdir().unwrap();
9269 let manager = crate::agents::AgentManager::new(tmp.path().join("child-cancel-retry.db"));
9270 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9271 manager,
9272 AppConfig::default(),
9273 Some(tmp.path().to_path_buf()),
9274 );
9275
9276 orchestrator.supervisor_runs.lock().await.insert(
9277 "run-parent".to_string(),
9278 empty_supervisor_run("run-parent", tmp.path().to_path_buf()),
9279 );
9280 orchestrator
9281 .create_child_supervisor_run(ChildSupervisorRunRequest {
9282 parent_run_id: "run-parent".to_string(),
9283 run_id: Some("run-child".to_string()),
9284 lead_agent_id: "supervisor-child".to_string(),
9285 objective: "Handle retryable sub-scope".to_string(),
9286 name: None,
9287 parent_task_id: None,
9288 session_id: None,
9289 workspace_dir: Some(tmp.path().to_path_buf()),
9290 approval_required: false,
9291 reviewer_required: false,
9292 test_required: false,
9293 execution_mode: AgentExecutionMode::SharedWorkspace,
9294 memory_tags: vec![],
9295 constraint_notes: vec![],
9296 })
9297 .await
9298 .unwrap();
9299 orchestrator
9300 .delegate_task(DelegatedTask {
9301 id: "task-child-retry".into(),
9302 agent_id: "agent-child-retry".into(),
9303 prompt: "Retryable child task".into(),
9304 context: None,
9305 required_tools: vec![],
9306 priority: 1,
9307 session_id: None,
9308 directive_id: None,
9309 tracking_task_id: None,
9310 run_id: Some("run-child".into()),
9311 parent_task_id: None,
9312 depends_on: vec![],
9313 role: Some(AgentRole::Implementer),
9314 delegation_brief: None,
9315 planning_only: false,
9316 approval_required: false,
9317 reviewer_required: false,
9318 test_required: false,
9319 workspace_dir: Some(tmp.path().to_path_buf()),
9320 execution_mode: AgentExecutionMode::SharedWorkspace,
9321 environment_id: None,
9322 remote_target: None,
9323 memory_tags: vec![],
9324 name: Some("Retryable child task".into()),
9325 })
9326 .await
9327 .unwrap();
9328
9329 orchestrator.cancel_task("task-child-retry").await.unwrap();
9330 let parent_cancelled = orchestrator.get_supervisor_run("run-parent").await.unwrap();
9331 assert_eq!(parent_cancelled.status, SupervisorRunStatus::Cancelled);
9332
9333 orchestrator.retry_task("task-child-retry").await.unwrap();
9334 let parent_retried = orchestrator.get_supervisor_run("run-parent").await.unwrap();
9335 let child_retried = orchestrator.get_supervisor_run("run-child").await.unwrap();
9336 let child_record = child_retried
9337 .tasks
9338 .iter()
9339 .find(|record| record.task.id == "task-child-retry")
9340 .expect("retried child task should still exist");
9341 assert_eq!(child_record.attempts, 1);
9342 assert_ne!(child_record.state, SupervisorTaskState::Cancelled);
9343 assert_ne!(child_retried.status, SupervisorRunStatus::Cancelled);
9344 assert_ne!(parent_retried.status, SupervisorRunStatus::Cancelled);
9345 }
9346
9347 #[tokio::test]
9348 async fn test_supervisor_run_queries_attach_checkpoint_summary() {
9349 let tmp = tempdir().unwrap();
9350 let manager = crate::agents::AgentManager::new(tmp.path().join("checkpoint-summary.db"));
9351 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9352 manager,
9353 AppConfig::default(),
9354 Some(tmp.path().to_path_buf()),
9355 );
9356
9357 let task = DelegatedTask {
9358 id: "task-checkpoint-summary".to_string(),
9359 agent_id: "agent-checkpoint".to_string(),
9360 prompt: "Inspect resumability".to_string(),
9361 context: None,
9362 required_tools: vec![],
9363 priority: 1,
9364 session_id: Some("session-checkpoint".to_string()),
9365 directive_id: None,
9366 tracking_task_id: None,
9367 run_id: Some("run-checkpoint-summary".to_string()),
9368 parent_task_id: None,
9369 depends_on: vec![],
9370 role: Some(AgentRole::Implementer),
9371 delegation_brief: None,
9372 planning_only: false,
9373 approval_required: false,
9374 reviewer_required: false,
9375 test_required: false,
9376 workspace_dir: Some(tmp.path().to_path_buf()),
9377 execution_mode: AgentExecutionMode::SharedWorkspace,
9378 environment_id: None,
9379 remote_target: None,
9380 memory_tags: vec![],
9381 name: Some("Checkpoint task".to_string()),
9382 };
9383 let now = Utc::now();
9384 let record = SupervisorTaskRecord {
9385 task: task.clone(),
9386 state: SupervisorTaskState::Blocked,
9387 approval: TaskApprovalRecord::default(),
9388 environment_id: "env-checkpoint".to_string(),
9389 environment: test_environment("env-checkpoint", tmp.path().to_path_buf()),
9390 claimed_by: None,
9391 attempts: 1,
9392 blocked_reasons: vec!["Task was running when orchestrator restarted; resumable checkpoint available (before tool 'file').".to_string()],
9393 result: None,
9394 remote_execution: None,
9395 local_execution: None,
9396 messages: vec![],
9397 checkpoint: None,
9398 created_at: now,
9399 updated_at: now,
9400 started_at: None,
9401 completed_at: None,
9402 };
9403 let run = SupervisorRun {
9404 id: "run-checkpoint-summary".to_string(),
9405 session_id: Some("session-checkpoint".to_string()),
9406 workspace_dir: Some(tmp.path().to_path_buf()),
9407 lead_agent_id: Some("supervisor".to_string()),
9408 parent_run: None,
9409 child_runs: vec![],
9410 hierarchy_depth: 0,
9411 max_hierarchy_depth: MAX_CHILD_SUPERVISOR_DEPTH,
9412 inherited_policy: None,
9413 status: SupervisorRunStatus::Waiting,
9414 task_summary: SupervisorRunTaskSummary {
9415 total: 1,
9416 queued: 0,
9417 blocked: 1,
9418 pending_approval: 0,
9419 running: 0,
9420 review_pending: 0,
9421 test_pending: 0,
9422 completed: 0,
9423 failed: 0,
9424 cancelled: 0,
9425 },
9426 hierarchy_summary: None,
9427 tasks: vec![record],
9428 messages: vec![],
9429 shared_cognition: vec![],
9430 created_at: now,
9431 updated_at: now,
9432 completed_at: None,
9433 name: None,
9434 metadata: None,
9435 };
9436 orchestrator
9437 .supervisor_runs
9438 .lock()
9439 .await
9440 .insert(run.id.clone(), run.clone());
9441 orchestrator
9442 .task_run_index
9443 .lock()
9444 .await
9445 .insert(task.id.clone(), run.id.clone());
9446
9447 let completed_tool_call_records = vec![ToolCallRecord {
9448 id: "tool-file-1".to_string(),
9449 name: "file".to_string(),
9450 arguments: "{\"path\":\"README.md\"}".to_string(),
9451 result: crate::ToolResult::Success("{\"ok\":true}".to_string()),
9452 duration_ms: 12,
9453 }];
9454 let completed_tool_calls = vec![OrchestratorToolCall {
9455 tool_name: "file".to_string(),
9456 input: json!({ "path": "README.md" }),
9457 output: json!({ "ok": true }),
9458 success: true,
9459 duration_ms: 12,
9460 }];
9461 let checkpoint = DelegatedTaskCheckpoint {
9462 id: delegated_checkpoint_id(&task.id),
9463 task_id: task.id.clone(),
9464 run_id: task.run_id.clone(),
9465 session_id: task.session_id.clone(),
9466 agent_id: task.agent_id.clone(),
9467 environment_id: task.environment_id.clone(),
9468 execution_mode: task.execution_mode.clone(),
9469 stage: DelegatedCheckpointStage::Blocked,
9470 replay_safety: DelegatedReplaySafety::CheckpointResumable,
9471 resume_disposition: DelegatedResumeDisposition::ResumeFromCheckpoint,
9472 safe_boundary_label: "after tool 'file' result".to_string(),
9473 workspace_dir: task.workspace_dir.clone(),
9474 completed_tool_calls: completed_tool_calls.clone(),
9475 result_published: false,
9476 note: Some("resume available after restart".to_string()),
9477 resume_state: Some(build_delegated_resume_state(
9478 &task,
9479 "prompt",
9480 "partial",
9481 "",
9482 &completed_tool_call_records,
9483 1,
9484 )),
9485 created_at: now,
9486 updated_at: now,
9487 };
9488 persist_checkpoint_to_disk(tmp.path(), &checkpoint).unwrap();
9489
9490 let listed_run = orchestrator
9491 .get_supervisor_run("run-checkpoint-summary")
9492 .await
9493 .unwrap();
9494 let checkpoint_summary = listed_run.tasks[0].checkpoint.as_ref().unwrap();
9495 assert_eq!(checkpoint_summary.stage, DelegatedCheckpointStage::Blocked);
9496 assert_eq!(
9497 checkpoint_summary.resume_disposition,
9498 DelegatedResumeDisposition::ResumeFromCheckpoint
9499 );
9500 assert!(checkpoint_summary.has_resume_state);
9501 assert_eq!(checkpoint_summary.completed_tool_call_count, 1);
9502 assert!(
9503 checkpoint_summary
9504 .available_actions
9505 .contains(&DelegatedCheckpointAction::ResumeFromCheckpoint)
9506 );
9507 }
9508
9509 #[tokio::test]
9510 async fn list_active_task_snapshots_attach_local_execution_telemetry() {
9511 let tmp = tempdir().unwrap();
9512 let manager = crate::agents::AgentManager::new(tmp.path().join("active-snapshots.db"));
9513 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9514 manager,
9515 AppConfig::default(),
9516 Some(tmp.path().to_path_buf()),
9517 );
9518
9519 let now = Utc::now();
9520 let task = DelegatedTask {
9521 id: "task-active-snapshot".to_string(),
9522 agent_id: "agent-local".to_string(),
9523 prompt: "Surface live local telemetry".to_string(),
9524 context: None,
9525 required_tools: vec!["file".to_string()],
9526 priority: 2,
9527 session_id: Some("session-active-snapshot".to_string()),
9528 directive_id: None,
9529 tracking_task_id: None,
9530 run_id: Some("run-active-snapshot".to_string()),
9531 parent_task_id: None,
9532 depends_on: vec![],
9533 role: Some(AgentRole::Implementer),
9534 delegation_brief: None,
9535 planning_only: false,
9536 approval_required: false,
9537 reviewer_required: false,
9538 test_required: false,
9539 workspace_dir: Some(tmp.path().to_path_buf()),
9540 execution_mode: AgentExecutionMode::SharedWorkspace,
9541 environment_id: Some("env-active-snapshot".to_string()),
9542 remote_target: None,
9543 memory_tags: vec![],
9544 name: Some("Active telemetry task".to_string()),
9545 };
9546 let mut run = empty_supervisor_run("run-active-snapshot", tmp.path().to_path_buf());
9547 run.session_id = task.session_id.clone();
9548 run.status = SupervisorRunStatus::Running;
9549 run.task_summary = SupervisorRunTaskSummary {
9550 total: 1,
9551 running: 1,
9552 ..SupervisorRunTaskSummary::default()
9553 };
9554 run.tasks.push(SupervisorTaskRecord {
9555 task: task.clone(),
9556 state: SupervisorTaskState::Running,
9557 approval: TaskApprovalRecord::default(),
9558 environment_id: "env-active-snapshot".to_string(),
9559 environment: test_environment("env-active-snapshot", tmp.path().to_path_buf()),
9560 claimed_by: Some(task.agent_id.clone()),
9561 attempts: 1,
9562 blocked_reasons: vec![],
9563 result: None,
9564 remote_execution: None,
9565 local_execution: Some(LocalExecutionRecord {
9566 status: "running".to_string(),
9567 status_reason: None,
9568 progress: Some(LocalExecutionProgress {
9569 phase: LocalExecutionPhase::Waiting,
9570 waiting_reason: Some(LocalExecutionWaitingReason::ShellProcess),
9571 stage: Some("shell_running".to_string()),
9572 message: Some("Streaming shell output".to_string()),
9573 percent: Some(45),
9574 iteration: 2,
9575 current_tool_name: Some("shell".to_string()),
9576 last_completed_tool_name: Some("file".to_string()),
9577 last_completed_tool_duration_ms: Some(12),
9578 completed_tool_call_count: 1,
9579 has_partial_content: true,
9580 partial_content_chars: 48,
9581 has_partial_thinking: false,
9582 partial_thinking_chars: 0,
9583 token_usage: None,
9584 environment: None,
9585 updated_at: now,
9586 }),
9587 last_synced_at: now,
9588 }),
9589 messages: vec![],
9590 checkpoint: None,
9591 created_at: now,
9592 updated_at: now,
9593 started_at: Some(now),
9594 completed_at: None,
9595 });
9596 orchestrator
9597 .supervisor_runs
9598 .lock()
9599 .await
9600 .insert(run.id.clone(), run);
9601 orchestrator
9602 .task_run_index
9603 .lock()
9604 .await
9605 .insert(task.id.clone(), "run-active-snapshot".to_string());
9606 orchestrator.active_tasks.lock().await.insert(
9607 task.id.clone(),
9608 ActiveTaskControl {
9609 task: task.clone(),
9610 local_cancel_token: Some(CancellationToken::new()),
9611 attempt: 1,
9612 },
9613 );
9614 persist_checkpoint_to_disk(
9615 tmp.path(),
9616 &DelegatedTaskCheckpoint {
9617 id: delegated_checkpoint_id(&task.id),
9618 task_id: task.id.clone(),
9619 run_id: task.run_id.clone(),
9620 session_id: task.session_id.clone(),
9621 agent_id: task.agent_id.clone(),
9622 environment_id: task.environment_id.clone(),
9623 execution_mode: task.execution_mode.clone(),
9624 stage: DelegatedCheckpointStage::Running,
9625 replay_safety: DelegatedReplaySafety::CheckpointResumable,
9626 resume_disposition: DelegatedResumeDisposition::ResumeFromCheckpoint,
9627 safe_boundary_label: "after file".to_string(),
9628 workspace_dir: task.workspace_dir.clone(),
9629 completed_tool_calls: vec![],
9630 result_published: false,
9631 note: Some("live telemetry checkpoint".to_string()),
9632 resume_state: None,
9633 created_at: now,
9634 updated_at: now,
9635 },
9636 )
9637 .unwrap();
9638
9639 let snapshots = orchestrator.list_active_task_snapshots().await;
9640 let snapshot = snapshots
9641 .iter()
9642 .find(|snapshot| snapshot.task.id == task.id)
9643 .unwrap();
9644
9645 assert_eq!(snapshot.state, SupervisorTaskState::Running);
9646 assert_eq!(
9647 snapshot
9648 .local_execution
9649 .as_ref()
9650 .and_then(|local| local.progress.as_ref())
9651 .and_then(|progress| progress.current_tool_name.as_deref()),
9652 Some("shell")
9653 );
9654 assert_eq!(
9655 snapshot
9656 .local_execution
9657 .as_ref()
9658 .and_then(|local| local.progress.as_ref())
9659 .and_then(|progress| progress.waiting_reason),
9660 Some(LocalExecutionWaitingReason::ShellProcess)
9661 );
9662 assert_eq!(
9663 snapshot
9664 .checkpoint
9665 .as_ref()
9666 .map(|checkpoint| checkpoint.stage),
9667 Some(DelegatedCheckpointStage::Running)
9668 );
9669 }
9670
9671 #[tokio::test]
9672 async fn test_acknowledge_blocked_task_updates_checkpoint_note() {
9673 let tmp = tempdir().unwrap();
9674 let manager = crate::agents::AgentManager::new(tmp.path().join("checkpoint-ack.db"));
9675 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9676 manager,
9677 AppConfig::default(),
9678 Some(tmp.path().to_path_buf()),
9679 );
9680
9681 let task = DelegatedTask {
9682 id: "task-checkpoint-ack".to_string(),
9683 agent_id: "agent-checkpoint".to_string(),
9684 prompt: "Need operator acknowledgement".to_string(),
9685 context: None,
9686 required_tools: vec![],
9687 priority: 1,
9688 session_id: Some("session-checkpoint".to_string()),
9689 directive_id: None,
9690 tracking_task_id: None,
9691 run_id: Some("run-checkpoint-ack".to_string()),
9692 parent_task_id: None,
9693 depends_on: vec![],
9694 role: Some(AgentRole::Implementer),
9695 delegation_brief: None,
9696 planning_only: false,
9697 approval_required: false,
9698 reviewer_required: false,
9699 test_required: false,
9700 workspace_dir: Some(tmp.path().to_path_buf()),
9701 execution_mode: AgentExecutionMode::SharedWorkspace,
9702 environment_id: None,
9703 remote_target: None,
9704 memory_tags: vec![],
9705 name: Some("Blocked checkpoint task".to_string()),
9706 };
9707 let now = Utc::now();
9708 let run = SupervisorRun {
9709 id: "run-checkpoint-ack".to_string(),
9710 session_id: Some("session-checkpoint".to_string()),
9711 workspace_dir: Some(tmp.path().to_path_buf()),
9712 lead_agent_id: Some("supervisor".to_string()),
9713 parent_run: None,
9714 child_runs: vec![],
9715 hierarchy_depth: 0,
9716 max_hierarchy_depth: MAX_CHILD_SUPERVISOR_DEPTH,
9717 inherited_policy: None,
9718 status: SupervisorRunStatus::Waiting,
9719 task_summary: SupervisorRunTaskSummary {
9720 total: 1,
9721 queued: 0,
9722 blocked: 1,
9723 pending_approval: 0,
9724 running: 0,
9725 review_pending: 0,
9726 test_pending: 0,
9727 completed: 0,
9728 failed: 0,
9729 cancelled: 0,
9730 },
9731 hierarchy_summary: None,
9732 tasks: vec![SupervisorTaskRecord {
9733 task: task.clone(),
9734 state: SupervisorTaskState::Blocked,
9735 approval: TaskApprovalRecord::default(),
9736 environment_id: "env-checkpoint".to_string(),
9737 environment: test_environment("env-checkpoint", tmp.path().to_path_buf()),
9738 claimed_by: None,
9739 attempts: 1,
9740 blocked_reasons: vec!["manual review required".to_string()],
9741 result: None,
9742 remote_execution: None,
9743 local_execution: None,
9744 messages: vec![],
9745 checkpoint: None,
9746 created_at: now,
9747 updated_at: now,
9748 started_at: None,
9749 completed_at: None,
9750 }],
9751 messages: vec![],
9752 shared_cognition: vec![],
9753 created_at: now,
9754 updated_at: now,
9755 completed_at: None,
9756 name: None,
9757 metadata: None,
9758 };
9759 orchestrator
9760 .supervisor_runs
9761 .lock()
9762 .await
9763 .insert(run.id.clone(), run.clone());
9764 orchestrator
9765 .task_run_index
9766 .lock()
9767 .await
9768 .insert(task.id.clone(), run.id.clone());
9769
9770 persist_checkpoint_to_disk(
9771 tmp.path(),
9772 &DelegatedTaskCheckpoint {
9773 id: delegated_checkpoint_id(&task.id),
9774 task_id: task.id.clone(),
9775 run_id: task.run_id.clone(),
9776 session_id: task.session_id.clone(),
9777 agent_id: task.agent_id.clone(),
9778 environment_id: task.environment_id.clone(),
9779 execution_mode: task.execution_mode.clone(),
9780 stage: DelegatedCheckpointStage::Blocked,
9781 replay_safety: DelegatedReplaySafety::OperatorGated,
9782 resume_disposition: DelegatedResumeDisposition::OperatorInterventionRequired,
9783 safe_boundary_label: "before tool 'shell' execution".to_string(),
9784 workspace_dir: task.workspace_dir.clone(),
9785 completed_tool_calls: vec![],
9786 result_published: false,
9787 note: Some("waiting for operator".to_string()),
9788 resume_state: None,
9789 created_at: now,
9790 updated_at: now,
9791 },
9792 )
9793 .unwrap();
9794
9795 orchestrator
9796 .acknowledge_blocked_task(&task.id, Some("reviewed by operator".to_string()))
9797 .await
9798 .unwrap();
9799
9800 let listed = orchestrator
9801 .get_supervisor_run("run-checkpoint-ack")
9802 .await
9803 .unwrap();
9804 assert_eq!(listed.tasks[0].state, SupervisorTaskState::Blocked);
9805 assert_eq!(
9806 listed.tasks[0]
9807 .checkpoint
9808 .as_ref()
9809 .and_then(|summary| summary.note.as_deref()),
9810 Some("reviewed by operator")
9811 );
9812 }
9813
9814 #[tokio::test]
9815 async fn pause_task_requests_local_pause_intent() {
9816 let tmp = tempdir().unwrap();
9817 let manager = crate::agents::AgentManager::new(tmp.path().join("pause-local.db"));
9818 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9819 manager,
9820 AppConfig::default(),
9821 Some(tmp.path().to_path_buf()),
9822 );
9823
9824 let task = DelegatedTask {
9825 id: "task-local-pause".to_string(),
9826 agent_id: "agent-local".to_string(),
9827 prompt: "Pause this local task".to_string(),
9828 context: None,
9829 required_tools: vec![],
9830 priority: 1,
9831 session_id: None,
9832 directive_id: None,
9833 tracking_task_id: None,
9834 run_id: Some("run-local-pause".to_string()),
9835 parent_task_id: None,
9836 depends_on: vec![],
9837 role: Some(AgentRole::Implementer),
9838 delegation_brief: None,
9839 planning_only: false,
9840 approval_required: false,
9841 reviewer_required: false,
9842 test_required: false,
9843 workspace_dir: Some(tmp.path().to_path_buf()),
9844 execution_mode: AgentExecutionMode::SharedWorkspace,
9845 environment_id: None,
9846 remote_target: None,
9847 memory_tags: vec![],
9848 name: Some("Local pause test".to_string()),
9849 };
9850 let token = CancellationToken::new();
9851 orchestrator.active_tasks.lock().await.insert(
9852 task.id.clone(),
9853 ActiveTaskControl {
9854 task: task.clone(),
9855 local_cancel_token: Some(token.clone()),
9856 attempt: 1,
9857 },
9858 );
9859
9860 orchestrator.pause_task(&task.id).await.unwrap();
9861
9862 assert!(token.is_cancelled());
9863 assert!(token.is_pause_requested());
9864 assert!(
9865 orchestrator
9866 .active_tasks
9867 .lock()
9868 .await
9869 .contains_key(&task.id)
9870 );
9871 }
9872
9873 #[tokio::test]
9874 async fn pause_task_rejects_remote_execution() {
9875 let tmp = tempdir().unwrap();
9876 let manager = crate::agents::AgentManager::new(tmp.path().join("pause-remote.db"));
9877 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9878 manager,
9879 AppConfig::default(),
9880 Some(tmp.path().to_path_buf()),
9881 );
9882
9883 let task = DelegatedTask {
9884 id: "task-remote-pause".to_string(),
9885 agent_id: "agent-remote".to_string(),
9886 prompt: "Attempt remote pause".to_string(),
9887 context: None,
9888 required_tools: vec![],
9889 priority: 1,
9890 session_id: None,
9891 directive_id: None,
9892 tracking_task_id: None,
9893 run_id: Some("run-remote-pause".to_string()),
9894 parent_task_id: None,
9895 depends_on: vec![],
9896 role: Some(AgentRole::Implementer),
9897 delegation_brief: None,
9898 planning_only: false,
9899 approval_required: false,
9900 reviewer_required: false,
9901 test_required: false,
9902 workspace_dir: None,
9903 execution_mode: AgentExecutionMode::Remote,
9904 environment_id: None,
9905 remote_target: Some(RemoteAgentTarget {
9906 url: "http://localhost:32145/a2a".to_string(),
9907 name: Some("remote-peer".to_string()),
9908 auth_token: Some("token".to_string()),
9909 capabilities: vec!["shell".to_string()],
9910 }),
9911 memory_tags: vec![],
9912 name: Some("Remote pause test".to_string()),
9913 };
9914 orchestrator.active_tasks.lock().await.insert(
9915 task.id.clone(),
9916 ActiveTaskControl {
9917 task: task.clone(),
9918 local_cancel_token: None,
9919 attempt: 1,
9920 },
9921 );
9922
9923 let error = orchestrator.pause_task(&task.id).await.unwrap_err();
9924 assert!(error.contains("cannot be paused locally"));
9925 assert!(
9926 orchestrator
9927 .active_tasks
9928 .lock()
9929 .await
9930 .contains_key(&task.id)
9931 );
9932 }
9933
9934 #[tokio::test]
9935 async fn complete_task_execution_preserves_paused_checkpoint_resume_state() {
9936 let tmp = tempdir().unwrap();
9937 let manager = crate::agents::AgentManager::new(tmp.path().join("pause-checkpoint.db"));
9938 let orchestrator = AgentOrchestrator::new_with_workspace_root(
9939 manager,
9940 AppConfig::default(),
9941 Some(tmp.path().to_path_buf()),
9942 );
9943
9944 let task = DelegatedTask {
9945 id: "task-paused-checkpoint".to_string(),
9946 agent_id: "agent-local".to_string(),
9947 prompt: "Resume me later".to_string(),
9948 context: None,
9949 required_tools: vec![],
9950 priority: 1,
9951 session_id: Some("session-paused".to_string()),
9952 directive_id: None,
9953 tracking_task_id: None,
9954 run_id: Some("run-paused-checkpoint".to_string()),
9955 parent_task_id: None,
9956 depends_on: vec![],
9957 role: Some(AgentRole::Implementer),
9958 delegation_brief: None,
9959 planning_only: false,
9960 approval_required: false,
9961 reviewer_required: false,
9962 test_required: false,
9963 workspace_dir: Some(tmp.path().to_path_buf()),
9964 execution_mode: AgentExecutionMode::SharedWorkspace,
9965 environment_id: None,
9966 remote_target: None,
9967 memory_tags: vec![],
9968 name: Some("Paused checkpoint task".to_string()),
9969 };
9970 let now = Utc::now();
9971 let mut run = empty_supervisor_run("run-paused-checkpoint", tmp.path().to_path_buf());
9972 run.session_id = task.session_id.clone();
9973 run.status = SupervisorRunStatus::Running;
9974 run.task_summary = SupervisorRunTaskSummary {
9975 total: 1,
9976 queued: 0,
9977 blocked: 0,
9978 pending_approval: 0,
9979 running: 1,
9980 review_pending: 0,
9981 test_pending: 0,
9982 completed: 0,
9983 failed: 0,
9984 cancelled: 0,
9985 };
9986 run.tasks.push(SupervisorTaskRecord {
9987 task: task.clone(),
9988 state: SupervisorTaskState::Running,
9989 approval: TaskApprovalRecord::default(),
9990 environment_id: "env-paused-checkpoint".to_string(),
9991 environment: test_environment("env-paused-checkpoint", tmp.path().to_path_buf()),
9992 claimed_by: Some(task.agent_id.clone()),
9993 attempts: 1,
9994 blocked_reasons: vec![],
9995 result: None,
9996 remote_execution: None,
9997 local_execution: Some(local_execution_record_for_start()),
9998 messages: vec![],
9999 checkpoint: None,
10000 created_at: now,
10001 updated_at: now,
10002 started_at: Some(now),
10003 completed_at: None,
10004 });
10005 orchestrator
10006 .supervisor_runs
10007 .lock()
10008 .await
10009 .insert(run.id.clone(), run);
10010 orchestrator
10011 .task_run_index
10012 .lock()
10013 .await
10014 .insert(task.id.clone(), "run-paused-checkpoint".to_string());
10015
10016 let completed_tool_call_records = vec![ToolCallRecord {
10017 id: "tool-file-1".to_string(),
10018 name: "file".to_string(),
10019 arguments: "{\"path\":\"README.md\"}".to_string(),
10020 result: crate::ToolResult::Success("{\"ok\":true}".to_string()),
10021 duration_ms: 12,
10022 }];
10023 orchestrator
10024 .persist_delegated_checkpoint(&DelegatedTaskCheckpoint {
10025 id: delegated_checkpoint_id(&task.id),
10026 task_id: task.id.clone(),
10027 run_id: task.run_id.clone(),
10028 session_id: task.session_id.clone(),
10029 agent_id: task.agent_id.clone(),
10030 environment_id: task.environment_id.clone(),
10031 execution_mode: task.execution_mode.clone(),
10032 stage: DelegatedCheckpointStage::Blocked,
10033 replay_safety: DelegatedReplaySafety::CheckpointResumable,
10034 resume_disposition: DelegatedResumeDisposition::ResumeFromCheckpoint,
10035 safe_boundary_label: "operator pause during iteration 2".to_string(),
10036 workspace_dir: task.workspace_dir.clone(),
10037 completed_tool_calls: vec![OrchestratorToolCall {
10038 tool_name: "file".to_string(),
10039 input: serde_json::json!({ "path": "README.md" }),
10040 output: serde_json::json!({ "ok": true }),
10041 success: true,
10042 duration_ms: 12,
10043 }],
10044 result_published: false,
10045 note: Some("Paused by operator; resumable checkpoint preserved.".to_string()),
10046 resume_state: Some(build_delegated_resume_state(
10047 &task,
10048 "prompt",
10049 "partial",
10050 "thinking",
10051 &completed_tool_call_records,
10052 2,
10053 )),
10054 created_at: now,
10055 updated_at: now,
10056 })
10057 .unwrap();
10058
10059 orchestrator
10060 .complete_task_execution(
10061 task.clone(),
10062 TaskResult {
10063 task_id: task.id.clone(),
10064 agent_id: task.agent_id.clone(),
10065 success: false,
10066 run_id: task.run_id.clone(),
10067 tracking_task_id: None,
10068 output: "Paused by operator; resumable checkpoint preserved.".to_string(),
10069 summary: task.name.clone(),
10070 tool_calls: vec![],
10071 artifacts: vec![],
10072 terminal_state_hint: Some(TaskTerminalStateHint::Blocked),
10073 duration_ms: 10,
10074 },
10075 true,
10076 1,
10077 )
10078 .await
10079 .unwrap();
10080
10081 let listed = orchestrator
10082 .supervisor_runs
10083 .lock()
10084 .await
10085 .get("run-paused-checkpoint")
10086 .cloned()
10087 .unwrap();
10088 let record = &listed.tasks[0];
10089 assert_eq!(record.state, SupervisorTaskState::Blocked);
10090 assert!(
10091 record
10092 .blocked_reasons
10093 .iter()
10094 .any(|reason| reason.contains("Paused by operator"))
10095 );
10096
10097 let persisted_checkpoint = orchestrator.load_delegated_checkpoint(&task).unwrap();
10098 assert_eq!(
10099 persisted_checkpoint.stage,
10100 DelegatedCheckpointStage::Blocked
10101 );
10102 assert!(!persisted_checkpoint.result_published);
10103 assert!(persisted_checkpoint.resume_state.is_some());
10104 assert!(
10105 checkpoint_available_actions(&persisted_checkpoint, SupervisorTaskState::Blocked,)
10106 .contains(&DelegatedCheckpointAction::ResumeFromCheckpoint)
10107 );
10108 }
10109
10110 #[tokio::test]
10111 async fn complete_task_execution_publishes_memory_handoff_to_supervisor() {
10112 let tmp = tempdir().unwrap();
10113 let manager = crate::agents::AgentManager::new(tmp.path().join("memory-handoff.db"));
10114 let orchestrator = AgentOrchestrator::new_with_workspace_root(
10115 manager,
10116 AppConfig::default(),
10117 Some(tmp.path().to_path_buf()),
10118 );
10119
10120 let task = DelegatedTask {
10121 id: "task-memory-handoff".to_string(),
10122 agent_id: "agent-impl".to_string(),
10123 prompt: "Implement the requested fix".to_string(),
10124 context: None,
10125 required_tools: vec![],
10126 priority: 1,
10127 session_id: Some("session-memory-handoff".to_string()),
10128 directive_id: Some("directive-memory-handoff".to_string()),
10129 tracking_task_id: Some("tracking-memory-handoff".to_string()),
10130 run_id: Some("run-memory-handoff".to_string()),
10131 parent_task_id: None,
10132 depends_on: vec![],
10133 role: Some(AgentRole::Implementer),
10134 delegation_brief: None,
10135 planning_only: false,
10136 approval_required: false,
10137 reviewer_required: false,
10138 test_required: false,
10139 workspace_dir: Some(tmp.path().to_path_buf()),
10140 execution_mode: AgentExecutionMode::SharedWorkspace,
10141 environment_id: None,
10142 remote_target: None,
10143 memory_tags: vec!["delegation".to_string()],
10144 name: Some("Implement delegated fix".to_string()),
10145 };
10146 let now = Utc::now();
10147 let mut run = empty_supervisor_run("run-memory-handoff", tmp.path().to_path_buf());
10148 run.session_id = task.session_id.clone();
10149 run.lead_agent_id = Some("supervisor-root".to_string());
10150 run.status = SupervisorRunStatus::Running;
10151 run.task_summary = SupervisorRunTaskSummary {
10152 total: 1,
10153 queued: 0,
10154 blocked: 0,
10155 pending_approval: 0,
10156 running: 1,
10157 review_pending: 0,
10158 test_pending: 0,
10159 completed: 0,
10160 failed: 0,
10161 cancelled: 0,
10162 };
10163 run.tasks.push(SupervisorTaskRecord {
10164 task: task.clone(),
10165 state: SupervisorTaskState::Running,
10166 approval: TaskApprovalRecord::default(),
10167 environment_id: "env-memory-handoff".to_string(),
10168 environment: test_environment("env-memory-handoff", tmp.path().to_path_buf()),
10169 claimed_by: Some(task.agent_id.clone()),
10170 attempts: 1,
10171 blocked_reasons: vec![],
10172 result: None,
10173 remote_execution: None,
10174 local_execution: Some(local_execution_record_for_start()),
10175 messages: vec![],
10176 checkpoint: None,
10177 created_at: now,
10178 updated_at: now,
10179 started_at: Some(now),
10180 completed_at: None,
10181 });
10182 orchestrator
10183 .supervisor_runs
10184 .lock()
10185 .await
10186 .insert(run.id.clone(), run);
10187
10188 orchestrator
10189 .complete_task_execution(
10190 task.clone(),
10191 TaskResult {
10192 task_id: task.id.clone(),
10193 agent_id: task.agent_id.clone(),
10194 success: true,
10195 run_id: task.run_id.clone(),
10196 tracking_task_id: task.tracking_task_id.clone(),
10197 output: "Applied the fix and verified the targeted path.".to_string(),
10198 summary: Some("Delegated fix applied".to_string()),
10199 tool_calls: vec![],
10200 artifacts: vec![TaskArtifactRecord {
10201 name: "summary.md".to_string(),
10202 kind: "report".to_string(),
10203 uri: Some("memory://summary".to_string()),
10204 summary: Some("Delegated completion summary".to_string()),
10205 }],
10206 terminal_state_hint: Some(TaskTerminalStateHint::Completed),
10207 duration_ms: 25,
10208 },
10209 false,
10210 1,
10211 )
10212 .await
10213 .unwrap();
10214
10215 let run = orchestrator
10216 .supervisor_runs
10217 .lock()
10218 .await
10219 .get("run-memory-handoff")
10220 .cloned()
10221 .unwrap();
10222 let record = &run.tasks[0];
10223 assert!(record.messages.iter().any(|message| {
10224 message.kind == TeamMessageKind::Handoff
10225 && message.sender_agent_id.as_deref() == Some("agent-impl")
10226 && message
10227 .result_reference
10228 .as_ref()
10229 .is_some_and(|result| result.success)
10230 }));
10231 assert!(
10232 record
10233 .messages
10234 .iter()
10235 .any(|message| message.content.contains("Memory:"))
10236 );
10237 assert!(run.shared_cognition.iter().any(|note| {
10238 note.kind == SharedCognitionKind::Handoff
10239 && note.sender_agent_id.as_deref() == Some("agent-impl")
10240 }));
10241
10242 let shared_query = crate::memory_bank::MemoryBankQuery::default()
10243 .with_category(SHARED_COGNITION_CATEGORY)
10244 .with_task("task-memory-handoff")
10245 .with_tags(vec![workflow_run_memory_tag("run-memory-handoff")])
10246 .with_limit(5);
10247 let shared_results =
10248 crate::memory_bank::search_memory_bank_with_query(tmp.path(), &shared_query)
10249 .await
10250 .unwrap();
10251 assert_eq!(shared_results.len(), 1);
10252 }
10253
10254 #[test]
10255 fn test_approval_record_deserializes_from_legacy_shape() {
10256 let legacy = serde_json::json!({
10257 "state": "pending",
10258 "requested_at": "2026-03-10T00:00:00Z",
10259 "note": "Legacy approval"
10260 });
10261
10262 let record: TaskApprovalRecord = serde_json::from_value(legacy).unwrap();
10263
10264 assert_eq!(record.state, ApprovalState::Pending);
10265 assert_eq!(record.scope, None);
10266 assert!(record.requests.is_empty());
10267 assert!(record.decisions.is_empty());
10268 assert!(record.active_request.is_none());
10269 }
10270
10271 #[test]
10272 fn test_supervisor_run_deserializes_legacy_shape_without_shared_cognition() {
10273 let mut value = serde_json::to_value(empty_supervisor_run(
10274 "run-legacy",
10275 PathBuf::from("/tmp/gestura-legacy-run"),
10276 ))
10277 .unwrap();
10278 value.as_object_mut().unwrap().remove("shared_cognition");
10279
10280 let run: SupervisorRun = serde_json::from_value(value).unwrap();
10281
10282 assert!(run.shared_cognition.is_empty());
10283 assert_eq!(run.id, "run-legacy");
10284 }
10285
10286 #[test]
10287 fn test_compatibility_from_card_warns_for_missing_auth_and_remote_features() {
10288 let mut card = create_gestura_agent_card("http://localhost:32145");
10289 card.authentication = Some(crate::AuthenticationInfo {
10290 schemes: vec!["bearer".to_string()],
10291 oauth2: None,
10292 });
10293 card.supported_task_features = vec!["artifacts".to_string()];
10294 card.supported_rpc_methods
10295 .retain(|method| method != "task/artifacts");
10296 let remote_target = RemoteAgentTarget {
10297 url: card.url.clone(),
10298 name: Some("legacy-peer".to_string()),
10299 auth_token: None,
10300 capabilities: vec!["shell".to_string()],
10301 };
10302
10303 let compatibility = compatibility_from_card(&card, &remote_target);
10304
10305 assert!(
10306 compatibility
10307 .warnings
10308 .iter()
10309 .any(|warning| warning.contains("no auth token"))
10310 );
10311 assert!(
10312 compatibility
10313 .warnings
10314 .iter()
10315 .any(|warning| warning.contains("authenticated mutation enforcement"))
10316 );
10317 assert!(
10318 compatibility
10319 .warnings
10320 .iter()
10321 .any(|warning| warning.contains("provenance support"))
10322 );
10323 assert!(
10324 compatibility
10325 .warnings
10326 .iter()
10327 .any(|warning| warning.contains("lease support"))
10328 );
10329 assert!(
10330 compatibility
10331 .warnings
10332 .iter()
10333 .any(|warning| warning.contains("idempotency support"))
10334 );
10335 assert!(
10336 compatibility
10337 .warnings
10338 .iter()
10339 .any(|warning| warning.contains("artifact manifest support"))
10340 );
10341 }
10342
10343 #[test]
10344 fn test_build_remote_task_request_omits_unsupported_lease_and_idempotency() {
10345 let task = DelegatedTask {
10346 id: "task-remote-compat".to_string(),
10347 agent_id: "agent-remote".to_string(),
10348 prompt: "Inspect the codebase".to_string(),
10349 context: None,
10350 required_tools: vec!["shell".to_string()],
10351 priority: 1,
10352 session_id: None,
10353 directive_id: None,
10354 tracking_task_id: None,
10355 run_id: Some("run-remote-compat".to_string()),
10356 parent_task_id: None,
10357 depends_on: vec![],
10358 role: Some(AgentRole::Implementer),
10359 delegation_brief: None,
10360 planning_only: false,
10361 approval_required: true,
10362 reviewer_required: true,
10363 test_required: false,
10364 workspace_dir: None,
10365 execution_mode: AgentExecutionMode::Remote,
10366 environment_id: None,
10367 remote_target: Some(RemoteAgentTarget {
10368 url: "http://localhost:32145/a2a".to_string(),
10369 name: Some("legacy-peer".to_string()),
10370 auth_token: Some("token".to_string()),
10371 capabilities: vec!["shell".to_string()],
10372 }),
10373 memory_tags: vec!["integration".to_string()],
10374 name: Some("Compatibility task".to_string()),
10375 };
10376 let record = SupervisorTaskRecord {
10377 task: task.clone(),
10378 state: SupervisorTaskState::Queued,
10379 approval: TaskApprovalRecord::default(),
10380 environment_id: "env-compat".to_string(),
10381 environment: test_environment("env-compat", PathBuf::from("/tmp")),
10382 claimed_by: None,
10383 attempts: 2,
10384 blocked_reasons: vec![],
10385 result: None,
10386 remote_execution: None,
10387 local_execution: None,
10388 messages: vec![],
10389 checkpoint: None,
10390 created_at: Utc::now(),
10391 updated_at: Utc::now(),
10392 started_at: None,
10393 completed_at: None,
10394 };
10395 let compatibility = RemoteExecutionCompatibility {
10396 protocol_version: Some("0.2.0".to_string()),
10397 supported_features: vec!["artifacts".to_string(), "provenance".to_string()],
10398 warnings: vec![],
10399 };
10400
10401 let request = build_remote_task_request(&task, &record, &compatibility);
10402
10403 assert!(request.idempotency_key.is_none());
10404 assert!(request.lease_request.is_none());
10405 assert_eq!(
10406 request
10407 .metadata
10408 .get("approvalRequired")
10409 .and_then(|value| value.as_bool()),
10410 Some(true)
10411 );
10412 assert_eq!(
10413 request
10414 .metadata
10415 .get("reviewerRequired")
10416 .and_then(|value| value.as_bool()),
10417 Some(true)
10418 );
10419 }
10420
10421 #[test]
10422 fn test_provenance_from_metadata_restores_authenticated_caller_context() {
10423 let provenance = provenance_from_metadata(&HashMap::from([
10424 (
10425 "caller_agent_id".to_string(),
10426 serde_json::json!("remote-caller"),
10427 ),
10428 (
10429 "caller_name".to_string(),
10430 serde_json::json!("Remote Caller"),
10431 ),
10432 ("caller_version".to_string(), serde_json::json!("1.2.3")),
10433 (
10434 "caller_capabilities".to_string(),
10435 serde_json::json!(["shell", "file"]),
10436 ),
10437 ("caller_authenticated".to_string(), serde_json::json!(true)),
10438 (
10439 "caller_auth_scheme".to_string(),
10440 serde_json::json!("bearer"),
10441 ),
10442 ]))
10443 .expect("metadata should restore provenance");
10444
10445 assert_eq!(provenance.caller_agent_id.as_deref(), Some("remote-caller"));
10446 assert_eq!(provenance.caller_name.as_deref(), Some("Remote Caller"));
10447 assert_eq!(provenance.caller_version.as_deref(), Some("1.2.3"));
10448 assert_eq!(provenance.caller_capabilities, vec!["shell", "file"]);
10449 assert!(provenance.authenticated);
10450 assert_eq!(provenance.auth_scheme.as_deref(), Some("bearer"));
10451 }
10452}