Coverage for src/governed_harness/metrics/prices.py: 86%

55 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Estimated cost of the agent calls whose provider reported tokens without a cost (#58, 

2item 10). 

3 

4The price table maps a model id (or ``provider/model``) to US dollars per million input, 

5output and cached tokens. It comes from ``metrics.prices`` in ``project.yaml`` and from 

6``harness metrics --prices FILE`` (which takes precedence); the harness ships no prices, because 

7prices change and a number nobody configured would be invented. An estimate is always reported 

8apart from the reported cost and labelled ``estimated``; it is never written to a run or 

9counted against the budget.""" 

10 

11from __future__ import annotations 

12 

13from collections.abc import Mapping 

14from dataclasses import dataclass 

15from pathlib import Path 

16from typing import Any 

17 

18import yaml 

19from pydantic import ValidationError 

20 

21from governed_harness.configuration.friction import ModelPrice 

22from governed_harness.domain.errors import ConfigurationError 

23 

24PER_TOKENS = 1_000_000 

25 

26 

27@dataclass(frozen=True) 

28class CallCost: 

29 reported: float | None 

30 estimated: float | None 

31 source: str 

32 """``reported``, ``estimated``, ``unpriced`` (tokens but no price) or ``none`` (no usage).""" 

33 

34 

35def load_prices(path: Path) -> dict[str, ModelPrice]: 

36 """A price table file: a mapping of model ids to ``input``, ``output`` and optional 

37 ``cache`` prices (YAML or JSON), optionally under a top-level ``prices`` key.""" 

38 try: 

39 raw = yaml.safe_load(path.read_text(encoding="utf-8")) 

40 except (OSError, yaml.YAMLError) as error: 

41 raise ConfigurationError(f"cannot read the price table {path}: {error}") from error 

42 if isinstance(raw, dict) and isinstance(raw.get("prices"), dict): 

43 raw = raw["prices"] 

44 if not isinstance(raw, dict): 44 ↛ 45line 44 didn't jump to line 45 because the condition on line 44 was never true

45 raise ConfigurationError(f"the price table {path} must map model ids to prices") 

46 try: 

47 return {str(key): ModelPrice.model_validate(value) for key, value in raw.items()} 

48 except ValidationError as error: 

49 raise ConfigurationError(f"invalid price table {path}: {error}") from error 

50 

51 

52def price_for( 

53 prices: Mapping[str, ModelPrice], provider: str | None, model: str | None 

54) -> ModelPrice | None: 

55 """The price of a call: ``provider/model`` first, then the model id, then (for a call 

56 that recorded no model) the provider id.""" 

57 if model: 

58 if provider and f"{provider}/{model}" in prices: 58 ↛ 59line 58 didn't jump to line 59 because the condition on line 58 was never true

59 return prices[f"{provider}/{model}"] 

60 if model in prices: 

61 return prices[model] 

62 return None 

63 return prices.get(provider) if provider else None 

64 

65 

66def call_cost( 

67 usage: Mapping[str, Any] | None, 

68 prices: Mapping[str, ModelPrice], 

69 provider: str | None, 

70 model: str | None, 

71) -> CallCost: 

72 """The reported cost of a call, or its estimate from the price table.""" 

73 if not usage: 

74 return CallCost(None, None, "none") 

75 reported = usage.get("cost_usd") 

76 if isinstance(reported, int | float): 

77 return CallCost(float(reported), None, "reported") 

78 input_tokens = int(usage.get("input_tokens") or 0) 

79 output_tokens = int(usage.get("output_tokens") or 0) 

80 cache_tokens = min(int(usage.get("cache_tokens") or 0), input_tokens) 

81 if not (input_tokens or output_tokens): 81 ↛ 82line 81 didn't jump to line 82 because the condition on line 81 was never true

82 return CallCost(None, None, "none") 

83 price = price_for(prices, provider, model) 

84 if price is None: 

85 return CallCost(None, None, "unpriced") 

86 cache_price = price.cache if price.cache is not None else price.input 

87 estimate = ( 

88 (input_tokens - cache_tokens) * price.input 

89 + cache_tokens * cache_price 

90 + output_tokens * price.output 

91 ) / PER_TOKENS 

92 return CallCost(None, round(estimate, 6), "estimated") 

93 

94 

95__all__ = ["PER_TOKENS", "CallCost", "call_cost", "load_prices", "price_for"]