Coverage for src/governed_harness/agents/routing.py: 99%
343 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Model and effort of each agent call (issue #44).
3The policy is a cascade in the sense of FrugalGPT and AutoMix: a call starts on the cheapest
4rung that the task size justifies and climbs a ladder only after a quality failure. The scorer
5that decides whether to climb is not a model but the harness's own deterministic verifier
6(VERIFICATION and REVIEW findings), so the decision stays reproducible. On the ladder effort is
7raised before the model is changed, because a higher effort on the same model is usually the
8cheaper fix. Planning calls (clarify, plan, review) have their own rung so the planner can be
9stronger than the executor.
11Everything here is pure: the same signals, history and policy always give the same decision,
12and the decision carries the digest of the policy it came from so it can be audited later.
13"""
15from __future__ import annotations
17from collections.abc import Callable, Mapping, Sequence
18from dataclasses import dataclass, replace
19from pathlib import PurePosixPath, PureWindowsPath
20from typing import TYPE_CHECKING, Any, cast
22from governed_harness.configuration.agent_results import (
23 DEFAULT_ROUTING_TABLES,
24 DEFAULT_SIZE_THRESHOLDS,
25 DEFAULT_TRIVIAL_THRESHOLDS,
26 AgentCallConfig,
27 AgentRoutingConfig,
28 CallKind,
29 FamilyTable,
30 ProviderFamily,
31 Rung,
32 SizeClass,
33 SizeThresholds,
34)
35from governed_harness.evidence.hashing import sha256_json
37if TYPE_CHECKING:
38 from governed_harness.configuration.models import ProjectConfiguration
40__all__ = [
41 "ANCHORED",
42 "DEFAULT_MAX_ESCALATIONS",
43 "MODEL_TIERS",
44 "CalibrationRow",
45 "RoutingDecision",
46 "RoutingHistory",
47 "TaskSignals",
48 "anchored_ladder",
49 "calibrate",
50 "can_escalate",
51 "classify_size",
52 "flags_for",
53 "invoking_model",
54 "is_trivial",
55 "model_argument",
56 "model_rank",
57 "provider_family",
58 "select",
59 "select_reviewer",
60 "suggest_table",
61]
63DEFAULT_MAX_ESCALATIONS = 2
64"""Escalation steps allowed per run when ``agentRouting.maxEscalations`` is not set."""
66_SIZE_ORDER: dict[str, int] = {"S": 0, "M": 1, "L": 2}
67TRIVIAL = "trivial"
68_IMPLEMENT_ORDER: dict[str, int] = {TRIVIAL: -1, **_SIZE_ORDER}
69_SIGNALS: tuple[str, ...] = ("requirements", "files", "loc")
70_ESCALATING_KINDS: frozenset[str] = frozenset({"implement", "review"})
72ANCHORED = "anchored"
73MODEL_TIERS: dict[str, tuple[str, ...]] = {"claude-code": ("haiku", "sonnet", "opus")}
74"""Since #85: the tiers of a family's models, cheapest first, matched by name. A family without
75an entry ranks its models by their first rung in the family's ladder."""
78@dataclass(frozen=True)
79class TaskSignals:
80 """What is known about the size of a task before an agent touches it."""
82 requirements: int
83 files: int
84 loc: int
85 risk_flags: tuple[str, ...] = ()
86 planned_large: bool = False
89@dataclass(frozen=True)
90class RoutingHistory:
91 """Quality failures of this run that already led to an escalation."""
93 escalations: int = 0
96@dataclass(frozen=True)
97class RoutingDecision:
98 """The model and effort chosen for one call and why, recorded as evidence.
100 ``escalations`` is the number of ladder steps requested for this call (0 when the call
101 did not escalate); ``rung`` is the ladder index used, or ``None`` when no ladder applies.
102 """
104 call_kind: str
105 mode: str
106 family: str
107 size: str | None
108 rule: str
109 model: str | None
110 effort: str | None
111 rung: int | None
112 escalations: int
113 policy_digest: str
114 flags: tuple[str, ...]
115 warning: str | None = None
116 """Since #59: why the decision fell back (a call kind without its own routing entry)."""
117 anchor: str | None = None
118 """Since #85: the invoking model, the ceiling of an ``anchored`` decision (recorded only
119 under that mode)."""
121 def as_dict(self) -> dict[str, Any]:
122 value: dict[str, Any] = {
123 "callKind": self.call_kind,
124 "mode": self.mode,
125 "family": self.family,
126 "size": self.size,
127 "rule": self.rule,
128 "model": self.model,
129 "effort": self.effort,
130 "rung": self.rung,
131 "escalations": self.escalations,
132 "policyDigest": self.policy_digest,
133 "flags": list(self.flags),
134 }
135 if self.warning is not None:
136 value["warning"] = self.warning
137 if self.mode == ANCHORED:
138 value["anchor"] = self.anchor
139 return value
142def _bounds(thresholds: SizeThresholds | None, name: str) -> tuple[int, int]:
143 configured: tuple[int, int] | None = getattr(thresholds, name, None) if thresholds else None
144 return configured if configured is not None else DEFAULT_SIZE_THRESHOLDS[name]
147def classify_size(signals: TaskSignals, thresholds: SizeThresholds | None) -> tuple[SizeClass, str]:
148 """Classify a task as S, M or L and say which signal decided.
150 The largest class among the signals wins, so a task with few requirements but many files
151 is still treated as large. A risk flag lifts S to M because a risky small change deserves
152 the stronger rung; a plan that already judged the task large forces L.
153 """
154 size: SizeClass = "S"
155 rule = "all signals within S"
156 for name in _SIGNALS:
157 small, medium = _bounds(thresholds, name)
158 value: int = getattr(signals, name)
159 if value <= small:
160 continue
161 current: SizeClass = "M" if value <= medium else "L"
162 bound = small if current == "M" else medium
163 if _SIZE_ORDER[current] > _SIZE_ORDER[size]: 163 ↛ 156line 163 didn't jump to line 156 because the condition on line 163 was always true
164 size = current
165 rule = f"{name}={value}>{bound} -> {current}"
166 if size == "S" and signals.risk_flags:
167 size = "M"
168 rule = f"risk:{','.join(signals.risk_flags)} -> M"
169 if signals.planned_large and size != "L":
170 size = "L"
171 rule = "planned-large -> L"
172 return size, rule
175def is_trivial(signals: TaskSignals, thresholds: SizeThresholds | None) -> tuple[bool, str]:
176 """Whether an implement call may run on the ``trivial`` rung (#197), and why.
178 Deliberately conservative: the task is ``S``, has between one and the trivial bound of
179 requirements and of owned files (an unknown scope is not trivial), at most the trivial
180 bound of lines in them, no risk flag and no decomposition. Anything else keeps its size.
181 """
182 size, rule = classify_size(signals, thresholds)
183 if size != "S":
184 return False, rule
185 if signals.risk_flags: 185 ↛ 186line 185 didn't jump to line 186 because the condition on line 185 was never true
186 return False, f"risk:{','.join(signals.risk_flags)}"
187 configured = thresholds.trivial if thresholds is not None else None
188 for name in _SIGNALS:
189 bound: int | None = getattr(configured, name, None) if configured else None
190 limit = bound if bound is not None else DEFAULT_TRIVIAL_THRESHOLDS[name]
191 value: int = getattr(signals, name)
192 if value > limit:
193 return False, f"{name}={value}>{limit} -> not trivial"
194 if name != "loc" and value < 1:
195 return False, f"{name}={value}<1 -> not trivial"
196 return True, "all signals within trivial"
199def _basename(element: str) -> str:
200 return PureWindowsPath(PurePosixPath(element).name).name.lower()
203def provider_family(
204 provider_id: str,
205 command: Sequence[str],
206 families: Mapping[str, ProviderFamily] | None,
207) -> ProviderFamily:
208 """Tell which CLI family a provider belongs to, so the right flags are passed.
210 An explicit ``agentRouting.families`` entry wins; otherwise the provider id and the
211 executable names in its command are matched, and anything unknown is ``generic`` (which
212 receives no model or effort flags at all).
213 """
214 if families and provider_id in families:
215 return families[provider_id]
216 names = [provider_id.lower(), *(_basename(element) for element in command)]
217 if any("claude" in name for name in names):
218 return "claude-code"
219 if any("codex" in name for name in names):
220 return "codex"
221 return "generic"
224def flags_for(family: ProviderFamily, model: str | None, effort: str | None) -> tuple[str, ...]:
225 """The command-line flags that pass a model and an effort to a CLI of the family."""
226 flags: list[str] = []
227 if family == "claude-code":
228 if model:
229 flags += ["--model", model]
230 if effort:
231 flags += ["--effort", effort]
232 elif family == "codex":
233 if model:
234 flags += ["-m", model]
235 if effort:
236 flags += ["-c", f'model_reasoning_effort="{effort}"']
237 return tuple(flags)
240def _max_escalations(policy: AgentRoutingConfig | None) -> int:
241 if policy is None or policy.max_escalations is None:
242 return DEFAULT_MAX_ESCALATIONS
243 return policy.max_escalations
246def _tiered(policy: AgentRoutingConfig | None) -> bool:
247 """Whether the policy chooses from the tables (``tiered``, or ``anchored`` since #85)."""
248 return policy is not None and policy.mode in {"tiered", ANCHORED}
251def can_escalate(history: RoutingHistory, policy: AgentRoutingConfig | None) -> bool:
252 """Whether a further quality failure may still move the next call up the ladder."""
253 return _tiered(policy) and history.escalations < _max_escalations(policy)
256def _policy_digest(policy: AgentRoutingConfig | None, family: ProviderFamily) -> str:
257 dumped = policy.model_dump(mode="json", by_alias=True) if policy is not None else {}
258 return sha256_json({"policy": dumped, "family": family})
261def _table(policy: AgentRoutingConfig, family: ProviderFamily) -> FamilyTable | None:
262 if policy.tables and family in policy.tables:
263 return policy.tables[family]
264 default = DEFAULT_ROUTING_TABLES.get(family)
265 return FamilyTable.model_validate(default) if default is not None else None
268def _kind_rung(
269 table: FamilyTable, call_kind: CallKind, size: SizeClass
270) -> tuple[Rung | None, str | None]:
271 """The rung of a call kind and, when the kind has no entry of its own and the implement
272 rung of the size stands in for it (#59), the warning that says so."""
273 implement = table.implement.get(size) if table.implement else None
274 if call_kind == "implement":
275 return implement, None
276 rung: Rung | None
277 if call_kind == "locate":
278 # The cheapest adequate rung (#55): the family's own locate rung, else the bottom of
279 # its escalation ladder.
280 rung = table.locate or (table.ladder[0] if table.ladder else None)
281 elif call_kind == "architecture":
282 # One survey or advice per project (#56): its own rung, else the planning rung.
283 rung = table.architecture or table.plan
284 else:
285 rung = getattr(table, call_kind, None)
286 if rung is not None or implement is None:
287 return rung, None
288 return implement, (
289 f"no routing entry for the {call_kind} call: the implement rung of size {size} is used"
290 )
293def _ladder_index(ladder: tuple[Rung, ...], rung: Rung, *, by_model: bool) -> int | None:
294 for index, candidate in enumerate(ladder):
295 if candidate.model == rung.model and candidate.effort == rung.effort:
296 return index
297 if by_model:
298 for index, candidate in enumerate(ladder):
299 if candidate.model == rung.model:
300 return index
301 return None
304# ----- anchored routing (#85) ---------------------------------------------------------------
305def model_argument(argv: Sequence[str]) -> str | None:
306 """The value of a ``--model`` (or ``-m``) option of a command line, ``--model=X``
307 included; ``None`` without one."""
308 for index, item in enumerate(argv):
309 if item.startswith("--model="):
310 return item.split("=", 1)[1] or None
311 if item in {"--model", "-m"} and index + 1 < len(argv):
312 return argv[index + 1]
313 return None
316def invoking_model(project: ProjectConfiguration, provider_id: str) -> str | None:
317 """The invoking model, the ceiling of ``agentRouting.mode: anchored``, of a call that
318 ``provider_id`` answers: ``agentRouting.anchorModel`` when set, else the provider's
319 ``model``, else the ``--model``/``-m`` value of its command or ``args``. The embedded
320 ``session`` provider stands for the project's ``agentProvider``, which answers the read-only
321 calls of a session run. ``None`` when nothing says it."""
322 policy = project.agent_routing
323 if policy is not None and policy.anchor_model:
324 return policy.anchor_model
325 if provider_id == "session":
326 provider_id = project.agent_provider
327 configured = project.agent_providers.get(provider_id)
328 if configured is None:
329 return None
330 if configured.model:
331 return configured.model
332 return model_argument((*(configured.command or ()), *(configured.args or ())))
335def model_rank(model: str, family: ProviderFamily, ladder: Sequence[Rung]) -> int | None:
336 """The tier of ``model`` in its family, cheapest first: by name for a family listed in
337 ``MODEL_TIERS``, else by the first rung of the family's ladder that uses it; ``None`` when
338 it cannot be told."""
339 tiers = MODEL_TIERS.get(family)
340 if tiers:
341 lowered = model.lower()
342 return next((index for index, name in enumerate(tiers) if name in lowered), None)
343 models = list(dict.fromkeys(item.model for item in ladder))
344 return models.index(model) if model in models else None
347def anchored_ladder(table: FamilyTable, anchor: str, family: ProviderFamily) -> tuple[Rung, ...]:
348 """The rungs an ``anchored`` call may use, cheapest first: the ladder's rungs of a cheaper
349 tier than ``anchor``, its rungs of the anchor's tier with the anchor as their model, and at
350 least the anchor itself as the top rung. An anchor whose tier is unknown allows only
351 itself."""
352 ladder = table.ladder or ()
353 ceiling = model_rank(anchor, family, ladder)
354 rungs: list[Rung] = []
355 if ceiling is not None:
356 for rung in ladder:
357 rank = model_rank(rung.model, family, ladder)
358 if rank is None or rank > ceiling:
359 continue
360 rungs.append(rung if rank < ceiling else Rung(model=anchor, effort=rung.effort))
361 if not any(item.model == anchor for item in rungs):
362 rungs.append(Rung(model=anchor))
363 return tuple(dict.fromkeys(rungs))
366def _clamp(
367 rung: Rung, anchor: str, family: ProviderFamily, table: FamilyTable
368) -> tuple[Rung, bool]:
369 """``(rung, capped)``: a rung of a cheaper tier as it is, a rung of the anchor's tier on the
370 anchor with the rung's effort, anything above (or of an unknown tier) the top rung allowed."""
371 if rung.model == anchor:
372 return rung, False
373 ladder = table.ladder or ()
374 ceiling = model_rank(anchor, family, ladder)
375 rank = model_rank(rung.model, family, ladder)
376 if ceiling is None or rank is None or rank > ceiling:
377 return anchored_ladder(table, anchor, family)[-1], True
378 if rank == ceiling: 378 ↛ 379line 378 didn't jump to line 379 because the condition on line 378 was never true
379 return Rung(model=anchor, effort=rung.effort), False
380 return rung, False
383def _anchored_choice(
384 table: FamilyTable,
385 base: Rung,
386 call_kind: CallKind,
387 size: str,
388 steps: int,
389 anchor: str | None,
390 family: ProviderFamily,
391) -> tuple[str, Rung | None, int | None, int]:
392 """``(rule, rung, index in the allowed ladder, escalations)`` of an anchored call; without
393 an anchor the provider keeps its own model (rung ``None``)."""
394 if anchor is None:
395 return f"anchored:no-anchor:{call_kind}:{size}", None, None, 0
396 allowed = anchored_ladder(table, anchor, family)
397 chosen, capped = _clamp(base, anchor, family, table)
398 start = _ladder_index(allowed, chosen, by_model=True)
399 if call_kind in _ESCALATING_KINDS and steps > 0:
400 index = min((start or 0) + steps, len(allowed) - 1)
401 return f"anchored:escalation:{steps}:{call_kind}:{size}", allowed[index], index, steps
402 prefix = "anchored:ceiling" if capped else "anchored"
403 return f"{prefix}:{call_kind}:{size}", chosen, start, 0
406NO_EFFORT_MODELS: tuple[str, ...] = (
407 "claude-3",
408 "haiku-4",
409 "sonnet-4-0",
410 "sonnet-4-5",
411 "sonnet-4-2025",
412 "opus-4-0",
413 "opus-4-1",
414 "opus-4-2025",
415)
416"""Since 2.0.1: Claude models (matched by name) whose API rejects or ignores ``effort``; a
417read-only effort (``agentRouting.readOnlyEffort``) is never sent to them."""
420def takes_effort(model: str) -> bool:
421 lowered = model.lower()
422 return not any(item in lowered for item in NO_EFFORT_MODELS)
425def _read_only_effort(
426 decision: RoutingDecision, policy: AgentRoutingConfig | None
427) -> RoutingDecision:
428 """The decision with the read-only effort of its kind (``agentRouting.readOnlyEffort``)
429 for the first attempt of a routed call; unchanged without the key, for ``implement``, an
430 explicit per-call setting, an escalation, an unknown model or one that takes no effort."""
431 own = (policy.read_only_effort if policy else None) or {}
432 efforts = {str(kind): str(value) for kind, value in own.items()}
433 effort = efforts.get(decision.call_kind)
434 if (
435 effort is None
436 or decision.call_kind == "implement"
437 or decision.rule == "override"
438 or decision.escalations
439 or decision.model is None
440 or not takes_effort(decision.model)
441 or effort == decision.effort
442 ):
443 return decision
444 return replace(
445 decision,
446 effort=effort,
447 rule=f"{decision.rule}:read-only-effort",
448 flags=flags_for(cast("ProviderFamily", decision.family), decision.model, effort),
449 )
452def select(
453 call_kind: CallKind,
454 signals: TaskSignals,
455 history: RoutingHistory,
456 policy: AgentRoutingConfig | None,
457 *,
458 family: ProviderFamily,
459 override: AgentCallConfig | None = None,
460 anchor: str | None = None,
461) -> RoutingDecision:
462 """Choose the model and effort of one agent call.
464 An explicit per-call configuration wins over the tables. Without a ``tiered`` (or
465 ``anchored``) policy, or without a table for the family, the provider keeps its own model
466 (``fixed``). Only ``implement`` and ``review`` escalate, because those are the calls whose
467 output the deterministic verifier scores; the size is always classified so the record
468 shows it. Under ``anchored`` (#85) ``anchor``, the invoking model, is the ceiling.
469 """
470 thresholds = policy.thresholds if policy else None
471 size, _size_rule = classify_size(signals, thresholds)
472 mode = policy.mode if policy is not None and policy.mode is not None else "fixed"
473 digest = _policy_digest(policy, family)
475 warning: str | None = None
477 def decision(
478 rule: str,
479 model: str | None,
480 effort: str | None,
481 rung: int | None = None,
482 escalations: int = 0,
483 ) -> RoutingDecision:
484 return RoutingDecision(
485 call_kind=call_kind,
486 mode=mode,
487 family=family,
488 size=size,
489 rule=rule,
490 model=model,
491 effort=effort,
492 rung=rung,
493 escalations=escalations,
494 policy_digest=digest,
495 flags=flags_for(family, model, effort),
496 warning=warning,
497 anchor=anchor,
498 )
500 if override is not None and (override.model or override.effort):
501 return decision("override", override.model, override.effort)
502 if policy is None or not _tiered(policy):
503 return decision("fixed", None, None)
504 table = _table(policy, family)
505 if table is None:
506 return decision("fixed", None, None)
507 base, warning = _kind_rung(table, call_kind, size)
508 if base is None:
509 return decision("fixed", None, None)
510 steps = min(history.escalations, _max_escalations(policy))
511 if call_kind == "implement" and is_trivial(signals, thresholds)[0]:
512 return _trivial_decision(decision, table, base, steps, anchor, family, mode=mode)
513 if mode == ANCHORED:
514 chosen_decision = decision(
515 *_anchored_values(table, base, call_kind, size, steps, anchor, family)
516 )
517 elif warning is not None:
518 chosen_decision = decision(*_fallback_values(table, base, call_kind, size))
519 else:
520 rule, chosen, index, escalations = _tier_choice(table, base, call_kind, size, steps)
521 chosen_decision = decision(rule, chosen.model, chosen.effort, index, escalations)
522 return _read_only_effort(chosen_decision, policy)
525def _fallback_values(
526 table: FamilyTable, base: Rung, call_kind: CallKind, size: str
527) -> tuple[str, str | None, str | None, int | None]:
528 """The rule, model, effort and rung of a call kind that fell back to the implement rung."""
529 rung = _ladder_index(table.ladder, base, by_model=False) if table.ladder else None
530 return f"fallback:implement:{call_kind}:{size}", base.model, base.effort, rung
533def _trivial_decision(
534 decision: Callable[..., RoutingDecision],
535 table: FamilyTable,
536 s_rung: Rung,
537 steps: int,
538 anchor: str | None,
539 family: ProviderFamily,
540 *,
541 mode: str,
542) -> RoutingDecision:
543 """The decision of a trivial implement call (#197): the family's ``trivial`` rung (else its
544 ``S`` rung). Its first escalation runs on the ``S`` rung and further ones climb the ladder
545 from there, so a trivial call that fails moves to the S model (Sonnet in the starting
546 claude-code table) before anything else."""
547 trivial = (table.implement or {}).get("trivial") or s_rung
548 base, remaining = trivial, steps
549 if steps > 0 and trivial != s_rung:
550 base, remaining = s_rung, steps - 1
551 if mode == ANCHORED:
552 rule, model, effort, index, _ = _anchored_values(
553 table, base, "implement", TRIVIAL, remaining, anchor, family
554 )
555 prefix = "anchored:"
556 else:
557 rule, rung, index, _ = _tier_choice(table, base, "implement", TRIVIAL, remaining)
558 model, effort = rung.model, rung.effort
559 prefix = ""
560 if steps > 0 and model is not None:
561 rule = f"{prefix}escalation:{steps}:implement:{TRIVIAL}"
562 result: RoutingDecision = decision(rule, model, effort, index, steps if model else 0)
563 return RoutingDecision(
564 call_kind=result.call_kind,
565 mode=result.mode,
566 family=result.family,
567 size=TRIVIAL,
568 rule=result.rule,
569 model=result.model,
570 effort=result.effort,
571 rung=result.rung,
572 escalations=result.escalations,
573 policy_digest=result.policy_digest,
574 flags=result.flags,
575 warning=result.warning,
576 anchor=result.anchor,
577 )
580def _anchored_values(
581 table: FamilyTable,
582 base: Rung,
583 call_kind: CallKind,
584 size: str,
585 steps: int,
586 anchor: str | None,
587 family: ProviderFamily,
588) -> tuple[str, str | None, str | None, int | None, int]:
589 """``(rule, model, effort, rung, escalations)`` of an anchored call."""
590 rule, rung, index, escalations = _anchored_choice(
591 table, base, call_kind, size, steps, anchor, family
592 )
593 if rung is None:
594 return rule, None, None, None, 0
595 return rule, rung.model, rung.effort, index, escalations
598def _tier_choice(
599 table: FamilyTable, base: Rung, call_kind: CallKind, size: str, steps: int
600) -> tuple[str, Rung, int | None, int]:
601 """``(rule, rung, ladder index, escalations)`` of a tiered call: ``steps`` rungs up the
602 family's ladder from ``base`` for a call that escalates, else ``base`` itself."""
603 ladder = table.ladder or ()
604 if call_kind in _ESCALATING_KINDS and steps > 0 and ladder:
605 start = _ladder_index(ladder, base, by_model=True) or 0
606 index = min(start + steps, len(ladder) - 1)
607 return f"escalation:{steps}:{call_kind}:{size}", ladder[index], index, steps
608 index_used = _ladder_index(ladder, base, by_model=False) if ladder else None
609 return f"tier:{call_kind}:{size}", base, index_used, 0
612def select_reviewer(
613 reviewer_id: str,
614 policy: AgentRoutingConfig | None,
615 *,
616 family: ProviderFamily,
617 anchor: str | None = None,
618) -> RoutingDecision:
619 """The model and effort of a reviewer of the review panel (#57): its own entry of the
620 family table (``reviewers``), else the ``review`` rung, else the implement rung of size M
621 with a warning (#59). Without a ``tiered`` policy the provider keeps its own model; under
622 ``anchored`` (#85) the reviewer's rung is capped at ``anchor``."""
623 signals = TaskSignals(requirements=0, files=0, loc=0)
624 decision = select("review", signals, RoutingHistory(), policy, family=family, anchor=anchor)
625 if policy is None or not _tiered(policy):
626 return decision
627 table = _table(policy, family)
628 rung = (table.reviewers or {}).get(reviewer_id) if table is not None else None
629 if table is None or rung is None:
630 return decision
631 rule = f"tier:review:{reviewer_id}"
632 if policy.mode == ANCHORED:
633 if anchor is None: 633 ↛ 634line 633 didn't jump to line 634 because the condition on line 633 was never true
634 return decision
635 rung, capped = _clamp(rung, anchor, family, table)
636 rule = f"anchored:{'ceiling:' if capped else ''}review:{reviewer_id}"
637 index = _ladder_index(anchored_ladder(table, anchor, family), rung, by_model=False)
638 else:
639 index = _ladder_index(table.ladder, rung, by_model=False) if table.ladder else None
640 chosen = RoutingDecision(
641 call_kind="review",
642 mode=decision.mode,
643 family=family,
644 size=decision.size,
645 rule=rule,
646 model=rung.model,
647 effort=rung.effort,
648 rung=index,
649 escalations=0,
650 policy_digest=decision.policy_digest,
651 flags=flags_for(family, rung.model, rung.effort),
652 anchor=decision.anchor,
653 )
654 return _read_only_effort(chosen, policy)
657# ----- Calibration (N14: cost per approved task) ------------------------------------------
658@dataclass(frozen=True)
659class CalibrationRow:
660 """One agent call of a finished run, with its cost and whether the run was approved."""
662 family: str
663 call_kind: str
664 size: str | None
665 model: str | None
666 effort: str | None
667 cost_usd: float | None
668 approved: bool
669 run_id: str
672_GroupKey = tuple[str, str, str | None, str | None, str | None]
675def calibrate(rows: Sequence[CalibrationRow]) -> list[dict[str, Any]]:
676 """Summarise cost per approved task for each family, call kind, size, model and effort.
678 Cost per approved task (not cost per call) is the measure because a cheap rung that needs
679 many retries or never gets approved is not cheap.
680 """
681 groups: dict[_GroupKey, list[CalibrationRow]] = {}
682 for row in rows:
683 key: _GroupKey = (row.family, row.call_kind, row.size, row.model, row.effort)
684 groups.setdefault(key, []).append(row)
686 summary: list[dict[str, Any]] = []
687 for (family, call_kind, size, model, effort), members in groups.items():
688 runs = {row.run_id for row in members}
689 approved = {row.run_id for row in members if row.approved}
690 known = [row.cost_usd for row in members if row.cost_usd is not None]
691 cost = round(sum(known), 6) if known else None
692 per_task = round(cost / len(approved), 6) if cost is not None and approved else None
693 summary.append(
694 {
695 "family": family,
696 "callKind": call_kind,
697 "size": size,
698 "model": model,
699 "effort": effort,
700 "runs": len(runs),
701 "approvedRuns": len(approved),
702 "costUsd": cost,
703 "costPerApprovedTask": per_task,
704 "approvalRate": round(len(approved) / len(runs), 6),
705 }
706 )
707 summary.sort(
708 key=lambda item: (
709 item["family"],
710 item["callKind"],
711 item["size"] or "",
712 item["costPerApprovedTask"] is None,
713 item["costPerApprovedTask"] or 0.0,
714 item["model"] or "",
715 item["effort"] or "",
716 )
717 )
718 return summary
721def suggest_table(summary: list[dict[str, Any]]) -> dict[str, dict[str, dict[str, str | None]]]:
722 """Suggest the cheapest approved (model, effort) per family and implement size.
724 Groups with fewer than two approved runs are ignored so a single lucky run does not set
725 the table. The result is only a suggestion for the maintainer; nothing is applied.
726 """
727 best: dict[str, dict[str, dict[str, Any]]] = {}
728 for item in summary:
729 if item["callKind"] != "implement" or item["size"] not in _IMPLEMENT_ORDER:
730 continue
731 per_task = item["costPerApprovedTask"]
732 if per_task is None or item["approvedRuns"] < 2:
733 continue
734 sizes = best.setdefault(item["family"], {})
735 current = sizes.get(item["size"])
736 candidate_key = (per_task, item["model"] or "", item["effort"] or "")
737 if current is None or candidate_key < (
738 current["costPerApprovedTask"],
739 current["model"] or "",
740 current["effort"] or "",
741 ):
742 sizes[item["size"]] = item
743 return {
744 family: {
745 size: {"model": sizes[size]["model"], "effort": sizes[size]["effort"]}
746 for size in sorted(sizes, key=lambda name: _IMPLEMENT_ORDER[name])
747 }
748 for family, sizes in sorted(best.items())
749 }