perspt_sdk/prompt/
accountant.rs

1//! Versioned token accounting (Definition 4's size function; Definition 6's
2//! accountant `c`).
3//!
4//! v1 is a conservative upper bound, not an exact tokenizer: BPE English
5//! and code average roughly four bytes per token, so `ceil(bytes / 3)`
6//! overestimates by a comfortable margin, and Definition 6's guard reserve
7//! absorbs residual error. An underestimating accountant is not valid. The
8//! accountant's identity and mode enter every program digest, so a later
9//! exact tokenizer is a clean versioned swap with no silent replay
10//! divergence.
11
12use serde::{Deserialize, Serialize};
13
14/// Whether counts are exact or a declared upper bound.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum AccountingMode {
18    ExactTokenizer,
19    ConservativeUpperBound,
20}
21
22/// A named, versioned token accounting function.
23#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
24pub struct TokenAccountantRef {
25    /// e.g. `"approx_bytes_v1"`.
26    pub id: String,
27    pub mode: AccountingMode,
28}
29
30/// Fixed per-message framing overhead charged by v1 (role tags, separators,
31/// provider envelope slack).
32const MESSAGE_FRAMING_TOKENS: u64 = 8;
33
34impl TokenAccountantRef {
35    /// The v1 conservative accountant used until an exact tokenizer ships
36    /// for a route.
37    pub fn approx_bytes_v1() -> Self {
38        Self {
39            id: "approx_bytes_v1".into(),
40            mode: AccountingMode::ConservativeUpperBound,
41        }
42    }
43
44    /// Upper-bound token count of one text.
45    pub fn count_text(&self, text: &str) -> u64 {
46        (text.len() as u64).div_ceil(3)
47    }
48
49    /// Upper-bound token count of one message (content plus framing).
50    pub fn count_message(&self, content: &str) -> u64 {
51        self.count_text(content) + MESSAGE_FRAMING_TOKENS
52    }
53}
54
55#[cfg(test)]
56mod tests {
57    use super::*;
58
59    #[test]
60    fn counting_is_a_conservative_upper_bound() {
61        let accountant = TokenAccountantRef::approx_bytes_v1();
62        assert_eq!(accountant.count_text(""), 0);
63        assert_eq!(accountant.count_text("abc"), 1);
64        assert_eq!(accountant.count_text("abcd"), 2);
65        // ~4 bytes/token real-world average is always under bytes/3.
66        let text = "let value = compute(input); // typical code density";
67        assert!(accountant.count_text(text) * 4 > text.len() as u64);
68        assert_eq!(accountant.count_message("abc"), 1 + MESSAGE_FRAMING_TOKENS);
69    }
70}
perspt_sdk/prompt/accountant.rs

perspt_sdk/prompt/
accountant.rs
