perspt_agent/
probe.rs

1//! Behavioral provider probes (PSP-9 Gate U).
2//!
3//! A declared capability matrix is a vendor claim; a probe is an observation.
4//! Each probe runs a scripted two-tool round trip through the provider-neutral
5//! transport and records what the route actually did: whether it issued tool
6//! calls at all, whether it selected both distinct tools, whether it batched
7//! calls in one turn, whether its arguments satisfied the declared schema,
8//! and whether it consumed the results into a final answer. Evidence from a
9//! probe is labelled `behavioral`, never merged silently into the declared
10//! matrix.
11
12use std::time::Instant;
13
14use anyhow::Result;
15use perspt_sdk::{Conversation, ModelId, ModelTransport, ToolChoicePolicy, ToolSpec, TurnOutput};
16use serde::{Deserialize, Serialize};
17
18/// What one live probe observed about a route.
19#[derive(Debug, Clone, Serialize, Deserialize)]
20pub struct ProbeReport {
21    pub model: String,
22    /// The model issued at least one tool call and consumed its result.
23    pub tool_call_round_trip: bool,
24    /// The model selected both distinct probe tools across the run.
25    pub multi_tool_selection: bool,
26    /// The model batched more than one call into a single turn.
27    pub parallel_tool_calls: bool,
28    /// Every issued call carried arguments satisfying the declared schema.
29    pub schema_arguments_valid: bool,
30    pub turns: u32,
31    pub tool_calls: u32,
32    pub total_seconds: f64,
33    pub error: Option<String>,
34}
35
36fn probe_specs() -> Vec<ToolSpec> {
37    vec![
38        ToolSpec {
39            name: "read_file".into(),
40            description: "Read the contents of a workspace file".into(),
41            schema: serde_json::json!({
42                "type": "object",
43                "properties": {
44                    "path": {"type": "string", "description": "Workspace-relative path"}
45                },
46                "required": ["path"],
47            }),
48            strict: false,
49        },
50        ToolSpec {
51            name: "list_files".into(),
52            description: "List files in a workspace directory".into(),
53            schema: serde_json::json!({
54                "type": "object",
55                "properties": {
56                    "path": {"type": "string", "description": "Directory to list"}
57                },
58                "required": ["path"],
59            }),
60            strict: false,
61        },
62    ]
63}
64
65fn scripted_result(name: &str) -> &'static str {
66    match name {
67        "read_file" => "pub fn add(a: i32, b: i32) -> i32 { a + b }",
68        "list_files" => "src/lib.rs\nsrc/main.rs\nCargo.toml",
69        _ => "unknown tool",
70    }
71}
72
73/// Run one behavioral probe against a route. Never mutates anything: tool
74/// results are scripted, so only the transport contract is under test.
75pub async fn probe_route(transport: &dyn ModelTransport, model: &ModelId) -> ProbeReport {
76    let start = Instant::now();
77    let mut report = ProbeReport {
78        model: model.to_string(),
79        tool_call_round_trip: false,
80        multi_tool_selection: false,
81        parallel_tool_calls: false,
82        schema_arguments_valid: true,
83        turns: 0,
84        tool_calls: 0,
85        total_seconds: 0.0,
86        error: None,
87    };
88    if let Err(error) = drive_probe(transport, model, &mut report).await {
89        report.error = Some(format!("{error:#}"));
90    }
91    report.total_seconds = start.elapsed().as_secs_f64();
92    report
93}
94
95async fn drive_probe(
96    transport: &dyn ModelTransport,
97    model: &ModelId,
98    report: &mut ProbeReport,
99) -> Result<()> {
100    let specs = probe_specs();
101    let mut conversation = Conversation::with_system(
102        "You are verifying tool plumbing. First read src/lib.rs with read_file \
103         and list the src directory with list_files (both calls may share one \
104         turn), then summarize what you saw in one sentence.",
105    );
106    conversation.push_user("Inspect the project as instructed, then summarize.");
107
108    let mut names_seen = std::collections::BTreeSet::new();
109    let mut saw_text_after_tools = false;
110    for _ in 0..5 {
111        report.turns += 1;
112        // The probe runs pre-session (no recorder), but shares the actor
113        // turn discipline (PSP-10 system 27).
114        let mut runner = crate::turn::ActorTurnRunner {
115            transport,
116            model: model.clone(),
117            fallbacks: Vec::new(),
118            recorder: None,
119            actor: crate::turn::ActorKind::CapabilityProbe,
120            deadline_secs: crate::turn::DEFAULT_TURN_DEADLINE_SECS,
121            turn: report.turns,
122        };
123        let output = runner
124            .run_turn(&conversation, &specs, ToolChoicePolicy::Auto)
125            .await
126            .map_err(|e| anyhow::anyhow!("{e}"))?;
127        match output {
128            TurnOutput::ToolCalls(calls) => {
129                if calls.len() > 1 {
130                    report.parallel_tool_calls = true;
131                }
132                report.tool_calls += calls.len() as u32;
133                conversation.push_tool_calls(calls.clone());
134                for call in &calls {
135                    names_seen.insert(call.name.clone());
136                    let path_ok = call
137                        .arguments
138                        .get("path")
139                        .map(serde_json::Value::is_string)
140                        .unwrap_or(false);
141                    if !path_ok {
142                        report.schema_arguments_valid = false;
143                    }
144                    conversation
145                        .push_tool_response(call.call_id.clone(), scripted_result(&call.name));
146                }
147            }
148            TurnOutput::Text(_) => {
149                saw_text_after_tools = report.tool_calls > 0;
150                break;
151            }
152        }
153    }
154    report.tool_call_round_trip = saw_text_after_tools;
155    report.multi_tool_selection =
156        names_seen.contains("read_file") && names_seen.contains("list_files");
157    Ok(())
158}
perspt_agent/probe.rs

perspt_agent/
probe.rs
