M3: models.dev catalog and cost display wiring

Adds a bundled models.dev snapshot plus harness-providers::modelsdev to resolve model metadata, and wires live cost display into the TUI's input bar and message state, updating the render snapshots accordingly.
2026-07-10 16:19:59 +02:00
parent a99edc116a
commit ab0269df1a
16 changed files with 428 additions and 19 deletions
@@ -0,0 +1,67 @@
+{
+  "anthropic": {
+    "id": "anthropic",
+    "name": "Anthropic",
+    "models": {
+      "claude-sonnet-4-5": {
+        "id": "claude-sonnet-4-5",
+        "reasoning": true,
+        "tool_call": true,
+        "attachment": true,
+        "cost": { "input": 3, "output": 15, "cache_read": 0.3, "cache_write": 3.75 },
+        "limit": { "context": 200000, "output": 64000 }
+      },
+      "claude-opus-4-1": {
+        "id": "claude-opus-4-1",
+        "reasoning": true,
+        "tool_call": true,
+        "attachment": true,
+        "cost": { "input": 15, "output": 75, "cache_read": 1.5, "cache_write": 18.75 },
+        "limit": { "context": 200000, "output": 32000 }
+      },
+      "claude-haiku-4-5": {
+        "id": "claude-haiku-4-5",
+        "reasoning": true,
+        "tool_call": true,
+        "attachment": true,
+        "cost": { "input": 1, "output": 5, "cache_read": 0.1, "cache_write": 1.25 },
+        "limit": { "context": 200000, "output": 64000 }
+      }
+    }
+  },
+  "openai": {
+    "id": "openai",
+    "name": "OpenAI",
+    "models": {
+      "gpt-4o": {
+        "id": "gpt-4o",
+        "tool_call": true,
+        "attachment": true,
+        "cost": { "input": 2.5, "output": 10, "cache_read": 1.25 },
+        "limit": { "context": 128000, "output": 16384 }
+      },
+      "gpt-4o-mini": {
+        "id": "gpt-4o-mini",
+        "tool_call": true,
+        "attachment": true,
+        "cost": { "input": 0.15, "output": 0.6, "cache_read": 0.075 },
+        "limit": { "context": 128000, "output": 16384 }
+      },
+      "gpt-5": {
+        "id": "gpt-5",
+        "reasoning": true,
+        "tool_call": true,
+        "attachment": true,
+        "cost": { "input": 1.25, "output": 10, "cache_read": 0.125 },
+        "limit": { "context": 400000, "output": 128000 }
+      },
+      "o3": {
+        "id": "o3",
+        "reasoning": true,
+        "tool_call": true,
+        "cost": { "input": 2, "output": 8, "cache_read": 0.5 },
+        "limit": { "context": 200000, "output": 100000 }
+      }
+    }
+  }
+}
@@ -1,10 +1,12 @@
 pub mod anthropic;
 pub mod auth;
 pub mod codec;
+pub mod modelsdev;
 pub mod openai;
 pub mod registry;

 pub use anthropic::AnthropicProvider;
 pub use auth::{AuthRecord, AuthStorage};
+pub use modelsdev::ModelCatalog;
 pub use openai::OpenAiProvider;
 pub use registry::ProviderRegistry;
@@ -0,0 +1,296 @@
+//! models.dev metadata: per-model context/output limits, pricing, and capability flags.
+//!
+//! At runtime we prefer a cached copy of `https://models.dev/api.json` (refreshed every 24h);
+//! if the cache is missing/stale and the network is unavailable, we fall back to a baked
+//! snapshot so cost display and limits still work offline.
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::time::{Duration, SystemTime};
+
+use harness_core::types::{ModelCost, ModelInfo, ModelRef};
+use serde::Deserialize;
+
+const API_URL: &str = "https://models.dev/api.json";
+const CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+/// Baked fallback, refreshed manually. Keeps cost/limits working with no cache and no network.
+const SNAPSHOT: &str = include_str!("../assets/models-snapshot.json");
+
+/// models.dev top-level shape: `{ provider_id: { models: { model_id: {...} } } }`. Unknown
+/// keys (provider metadata, per-model extras) are ignored.
+#[derive(Deserialize)]
+struct ApiProvider {
+    #[serde(default)]
+    models: HashMap<String, ApiModel>,
+}
+
+#[derive(Deserialize)]
+struct ApiModel {
+    #[serde(default)]
+    reasoning: bool,
+    #[serde(default)]
+    tool_call: bool,
+    #[serde(default)]
+    attachment: bool,
+    #[serde(default)]
+    cost: ApiCost,
+    #[serde(default)]
+    limit: ApiLimit,
+}
+
+#[derive(Deserialize, Default)]
+struct ApiCost {
+    #[serde(default)]
+    input: f64,
+    #[serde(default)]
+    output: f64,
+    #[serde(default)]
+    cache_read: f64,
+    #[serde(default)]
+    cache_write: f64,
+}
+
+#[derive(Deserialize, Default)]
+struct ApiLimit {
+    #[serde(default)]
+    context: u64,
+    #[serde(default)]
+    output: u64,
+}
+
+/// Parsed metadata keyed by `(provider_id, model_id)`.
+#[derive(Debug, Clone, Default)]
+pub struct ModelCatalog {
+    models: HashMap<(String, String), ModelInfo>,
+}
+
+impl ModelCatalog {
+    /// Parses a models.dev `api.json` document.
+    pub fn from_api_json(bytes: &[u8]) -> Result<Self, serde_json::Error> {
+        let raw: HashMap<String, ApiProvider> = serde_json::from_slice(bytes)?;
+        let mut models = HashMap::new();
+        for (provider_id, provider) in raw {
+            for (model_id, m) in provider.models {
+                let info = ModelInfo {
+                    model: ModelRef::new(provider_id.clone(), model_id.clone()),
+                    context_limit: m.limit.context,
+                    output_limit: m.limit.output,
+                    cost: ModelCost {
+                        input: m.cost.input,
+                        output: m.cost.output,
+                        cache_read: m.cost.cache_read,
+                        cache_write: m.cost.cache_write,
+                    },
+                    reasoning: m.reasoning,
+                    tool_call: m.tool_call,
+                    attachment: m.attachment,
+                };
+                models.insert((provider_id.clone(), model_id), info);
+            }
+        }
+        Ok(Self { models })
+    }
+
+    /// The baked fallback snapshot — always available, never fails.
+    pub fn baked() -> Self {
+        Self::from_api_json(SNAPSHOT.as_bytes()).expect("baked models snapshot must parse")
+    }
+
+    pub fn get(&self, provider: &str, model: &str) -> Option<&ModelInfo> {
+        self.models.get(&(provider.to_string(), model.to_string()))
+    }
+
+    /// Pricing for a model, or zero-cost if unknown.
+    pub fn cost(&self, provider: &str, model: &str) -> ModelCost {
+        self.get(provider, model)
+            .map(|m| m.cost)
+            .unwrap_or_default()
+    }
+
+    pub fn len(&self) -> usize {
+        self.models.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.models.is_empty()
+    }
+
+    /// Default cache location: `~/.cache/ai-harness/models.json`.
+    pub fn default_cache_path() -> PathBuf {
+        dirs::cache_dir()
+            .unwrap_or_else(std::env::temp_dir)
+            .join("ai-harness")
+            .join("models.json")
+    }
+
+    /// Loads from the cache if present and younger than [`CACHE_TTL`]; otherwise returns the
+    /// baked snapshot. Never touches the network — call [`ModelCatalog::refresh`] for that.
+    pub fn load_cached_or_baked(cache_path: &Path) -> Self {
+        if cache_fresh(cache_path, CACHE_TTL, SystemTime::now()) {
+            if let Ok(bytes) = std::fs::read(cache_path) {
+                if let Ok(catalog) = Self::from_api_json(&bytes) {
+                    if !catalog.is_empty() {
+                        return catalog;
+                    }
+                }
+            }
+        }
+        Self::baked()
+    }
+
+    /// Refreshes the default cache location with a fresh HTTP client. Convenience wrapper for
+    /// callers (the app) that don't want to depend on `reqwest` directly.
+    pub async fn refresh_default_cache() -> Result<Self, RefreshError> {
+        let client = reqwest::Client::new();
+        Self::refresh(&client, &Self::default_cache_path()).await
+    }
+
+    /// Fetches the latest metadata and writes it to `cache_path` (best-effort). Returns the
+    /// freshly-parsed catalog. Intended to run in the background so the *next* launch is current.
+    pub async fn refresh(
+        client: &reqwest::Client,
+        cache_path: &Path,
+    ) -> Result<Self, RefreshError> {
+        let bytes = client
+            .get(API_URL)
+            .send()
+            .await
+            .map_err(|e| RefreshError::Network(e.to_string()))?
+            .error_for_status()
+            .map_err(|e| RefreshError::Network(e.to_string()))?
+            .bytes()
+            .await
+            .map_err(|e| RefreshError::Network(e.to_string()))?;
+
+        let catalog =
+            Self::from_api_json(&bytes).map_err(|e| RefreshError::Parse(e.to_string()))?;
+
+        if let Some(parent) = cache_path.parent() {
+            let _ = std::fs::create_dir_all(parent);
+        }
+        if let Err(e) = std::fs::write(cache_path, &bytes) {
+            tracing::warn!(error = %e, "failed to cache models.dev metadata");
+        }
+        Ok(catalog)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum RefreshError {
+    #[error("network: {0}")]
+    Network(String),
+    #[error("parse: {0}")]
+    Parse(String),
+}
+
+/// True when `path` exists and was modified within `ttl` of `now`.
+fn cache_fresh(path: &Path, ttl: Duration, now: SystemTime) -> bool {
+    let Ok(modified) = std::fs::metadata(path).and_then(|m| m.modified()) else {
+        return false;
+    };
+    match now.duration_since(modified) {
+        Ok(age) => age < ttl,
+        Err(_) => true, // modified in the future (clock skew) — treat as fresh
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FIXTURE: &str = r#"{
+        "anthropic": {
+            "id": "anthropic",
+            "name": "Anthropic",
+            "models": {
+                "claude-sonnet-4-5": {
+                    "id": "claude-sonnet-4-5",
+                    "reasoning": true,
+                    "tool_call": true,
+                    "attachment": true,
+                    "cost": {"input": 3, "output": 15, "cache_read": 0.3, "cache_write": 3.75},
+                    "limit": {"context": 200000, "output": 64000}
+                }
+            }
+        },
+        "openai": {
+            "id": "openai",
+            "models": {
+                "gpt-4o": {
+                    "id": "gpt-4o",
+                    "tool_call": true,
+                    "cost": {"input": 2.5, "output": 10},
+                    "limit": {"context": 128000, "output": 16384}
+                }
+            }
+        }
+    }"#;
+
+    #[test]
+    fn parses_providers_and_models() {
+        let catalog = ModelCatalog::from_api_json(FIXTURE.as_bytes()).unwrap();
+        assert_eq!(catalog.len(), 2);
+
+        let sonnet = catalog.get("anthropic", "claude-sonnet-4-5").unwrap();
+        assert_eq!(sonnet.context_limit, 200_000);
+        assert_eq!(sonnet.output_limit, 64_000);
+        assert_eq!(sonnet.cost.input, 3.0);
+        assert_eq!(sonnet.cost.cache_write, 3.75);
+        assert!(sonnet.reasoning && sonnet.tool_call && sonnet.attachment);
+    }
+
+    #[test]
+    fn missing_cost_fields_default_to_zero() {
+        let catalog = ModelCatalog::from_api_json(FIXTURE.as_bytes()).unwrap();
+        let gpt = catalog.get("openai", "gpt-4o").unwrap();
+        assert_eq!(gpt.cost.cache_read, 0.0);
+        assert_eq!(gpt.cost.cache_write, 0.0);
+        assert!(!gpt.reasoning); // absent → false
+        assert!(gpt.tool_call);
+    }
+
+    #[test]
+    fn cost_helper_is_zero_for_unknown_model() {
+        let catalog = ModelCatalog::from_api_json(FIXTURE.as_bytes()).unwrap();
+        assert_eq!(catalog.cost("openai", "nonexistent"), ModelCost::default());
+        assert_eq!(catalog.cost("anthropic", "claude-sonnet-4-5").input, 3.0);
+    }
+
+    #[test]
+    fn baked_snapshot_parses_and_is_nonempty() {
+        let catalog = ModelCatalog::baked();
+        assert!(!catalog.is_empty());
+    }
+
+    #[test]
+    fn cache_freshness_respects_ttl() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("models.json");
+        assert!(!cache_fresh(&path, CACHE_TTL, SystemTime::now())); // missing
+
+        std::fs::write(&path, "{}").unwrap();
+        let now = SystemTime::now();
+        assert!(cache_fresh(&path, CACHE_TTL, now));
+        // A "now" far in the future makes the file look stale.
+        let future = now + Duration::from_secs(48 * 60 * 60);
+        assert!(!cache_fresh(&path, CACHE_TTL, future));
+    }
+
+    #[test]
+    fn load_cached_or_baked_falls_back_when_cache_absent() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("models.json");
+        let catalog = ModelCatalog::load_cached_or_baked(&path);
+        assert!(!catalog.is_empty()); // baked fallback
+    }
+
+    #[test]
+    fn load_cached_or_baked_reads_fresh_cache() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("models.json");
+        std::fs::write(&path, FIXTURE).unwrap();
+        let catalog = ModelCatalog::load_cached_or_baked(&path);
+        assert_eq!(catalog.len(), 2);
+        assert!(catalog.get("anthropic", "claude-sonnet-4-5").is_some());
+    }
+}