Coverage for src/governed_harness/agents/self_report.py: 78%
111 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The agent's structured self-report (``provenance.selfReport``, since 2.0).
3A command provider answers with an optional ``selfReport`` object next to ``status``; a built-in
4adapter asks the agent to end its answer with a fenced JSON block holding
5``{"harnessSelfReport": {...}}``. Either way the report is read leniently: what can be read is
6kept, what cannot is listed in ``problems``. It is never a protocol error and never a check: the
7run does not depend on it."""
9from __future__ import annotations
11import json
12import re
13from dataclasses import dataclass
14from typing import Any
16from governed_harness.domain.models import DisputedAcceptanceTest, SelfReportItem
18SELF_REPORT_FIELDS = (
19 "assumptions",
20 "alternativesDiscarded",
21 "lowConfidenceAreas",
22 "unrequestedChanges",
23)
24DISPUTED_FIELD = "disputedAcceptanceTests"
25"""Optional fifth field (#94): ``[{test, reason}]``, asked for only when the frozen
26acceptance tests can be disputed (``verification.acceptanceTests.disputes``)."""
27MAX_ITEMS = 50
28MAX_TEXT = 1000
30REQUEST_BLOCK: dict[str, Any] = {
31 "requested": True,
32 "fields": list(SELF_REPORT_FIELDS),
33 "note": (
34 "Optional. Answer with selfReport: {assumptions: [text], alternativesDiscarded: [text], "
35 "lowConfidenceAreas: [{path, description}], unrequestedChanges: [{path, description}]}. "
36 "It is recorded as reported by the agent, not as verification."
37 ),
38}
39"""What the request of a command provider carries under ``selfReport``."""
41PROMPT_INSTRUCTIONS = (
42 "When you finish, end your answer with a fenced JSON code block (```json) containing one "
43 'object {"harnessSelfReport": {"assumptions": [...], "alternativesDiscarded": [...], '
44 '"lowConfidenceAreas": [{"path": "...", "description": "..."}], '
45 '"unrequestedChanges": [{"path": "...", "description": "..."}]}}. List the assumptions you '
46 "made, the alternatives you discarded, the places where you are not confident, and every "
47 "change you made that the task did not ask for. Use empty lists when there is nothing to "
48 "report. The harness records it as your report; it does not replace verification."
49)
51_FENCED_JSON = re.compile(r"```(?:json)?\s*\n(?P<body>\{(?:(?!\}\s*\n```).)*\})\s*\n```", re.DOTALL)
54@dataclass(frozen=True)
55class ParsedSelfReport:
56 assumptions: tuple[str, ...]
57 alternatives_discarded: tuple[str, ...]
58 low_confidence_areas: tuple[SelfReportItem, ...]
59 unrequested_changes: tuple[SelfReportItem, ...]
60 problems: tuple[str, ...]
61 disputed_acceptance_tests: tuple[DisputedAcceptanceTest, ...] = ()
63 def paths(self) -> tuple[str, ...]:
64 return tuple(
65 dict.fromkeys(
66 item.path
67 for item in (*self.low_confidence_areas, *self.unrequested_changes)
68 if item.path
69 )
70 )
73def _text(value: Any) -> str | None:
74 if not isinstance(value, str):
75 return None
76 text = " ".join(value.split())
77 return text[:MAX_TEXT] or None
80def _texts(raw: Any, name: str, problems: list[str]) -> tuple[str, ...]:
81 if raw is None:
82 return ()
83 if not isinstance(raw, list): 83 ↛ 84line 83 didn't jump to line 84 because the condition on line 83 was never true
84 problems.append(f"{name} is not a list")
85 return ()
86 values = []
87 for item in raw[:MAX_ITEMS]:
88 text = _text(item)
89 if text is None: 89 ↛ 90line 89 didn't jump to line 90 because the condition on line 89 was never true
90 problems.append(f"{name} has an entry that is not text")
91 continue
92 values.append(text)
93 if len(raw) > MAX_ITEMS: 93 ↛ 94line 93 didn't jump to line 94 because the condition on line 93 was never true
94 problems.append(f"{name}: {len(raw) - MAX_ITEMS} entries beyond {MAX_ITEMS} were dropped")
95 return tuple(values)
98def _items(raw: Any, name: str, problems: list[str]) -> tuple[SelfReportItem, ...]:
99 if raw is None:
100 return ()
101 if not isinstance(raw, list): 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true
102 problems.append(f"{name} is not a list")
103 return ()
104 items = []
105 for entry in raw[:MAX_ITEMS]:
106 item, valid = _entry_item(entry)
107 if item is not None: 107 ↛ 109line 107 didn't jump to line 109 because the condition on line 107 was always true
108 items.append(item)
109 elif not valid:
110 problems.append(f"{name} has an entry without a description")
111 if len(raw) > MAX_ITEMS: 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true
112 problems.append(f"{name}: {len(raw) - MAX_ITEMS} entries beyond {MAX_ITEMS} were dropped")
113 return tuple(items)
116def _entry_item(entry: Any) -> tuple[SelfReportItem | None, bool]:
117 """The item of one entry and whether the entry is valid: an empty text is valid but gives
118 no item; an entry without a description or a path is not valid."""
119 if isinstance(entry, str): 119 ↛ 120line 119 didn't jump to line 120 because the condition on line 119 was never true
120 text = _text(entry)
121 return (SelfReportItem(description=text) if text else None), True
122 if isinstance(entry, dict): 122 ↛ 130line 122 didn't jump to line 130 because the condition on line 122 was always true
123 description = _text(entry.get("description"))
124 path = entry.get("path")
125 path_text = path.strip()[:MAX_TEXT] if isinstance(path, str) and path.strip() else None
126 if description is None and path_text is not None: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true
127 description = "(no description)"
128 if description is not None: 128 ↛ 130line 128 didn't jump to line 130 because the condition on line 128 was always true
129 return SelfReportItem(path=path_text, description=description), True
130 return None, False
133def _disputed(raw: Any, problems: list[str]) -> tuple[DisputedAcceptanceTest, ...]:
134 """``[{test, reason}]`` (``path`` and ``description`` are read too); an entry without a
135 test or without a reason is a problem, not a dispute."""
136 if raw is None:
137 return ()
138 if not isinstance(raw, list): 138 ↛ 139line 138 didn't jump to line 139 because the condition on line 138 was never true
139 problems.append(f"{DISPUTED_FIELD} is not a list")
140 return ()
141 found = []
142 for entry in raw[:MAX_ITEMS]:
143 item = entry if isinstance(entry, dict) else {}
144 test = _text(item.get("test")) or _text(item.get("path"))
145 reason = _text(item.get("reason")) or _text(item.get("description"))
146 if test is None or reason is None:
147 problems.append(f"{DISPUTED_FIELD} has an entry without a test or a reason")
148 continue
149 found.append(DisputedAcceptanceTest(test=test, reason=reason))
150 return tuple(found)
153def parse_self_report(raw: Any) -> ParsedSelfReport | None:
154 """The report in ``raw`` (an object with the four fields), ``None`` when there is none."""
155 if raw is None:
156 return None
157 problems: list[str] = []
158 if not isinstance(raw, dict): 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true
159 return ParsedSelfReport((), (), (), (), ("the self-report is not an object",))
160 unknown = sorted(set(raw) - set(SELF_REPORT_FIELDS) - {DISPUTED_FIELD})
161 if unknown: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 problems.append(f"unknown field(s) ignored: {', '.join(unknown)}")
163 return ParsedSelfReport(
164 assumptions=_texts(raw.get("assumptions"), "assumptions", problems),
165 alternatives_discarded=_texts(
166 raw.get("alternativesDiscarded"), "alternativesDiscarded", problems
167 ),
168 low_confidence_areas=_items(raw.get("lowConfidenceAreas"), "lowConfidenceAreas", problems),
169 unrequested_changes=_items(raw.get("unrequestedChanges"), "unrequestedChanges", problems),
170 disputed_acceptance_tests=_disputed(raw.get(DISPUTED_FIELD), problems),
171 problems=tuple(problems),
172 )
175def extract_from_text(text: str) -> Any:
176 """The ``harnessSelfReport`` object of the last fenced JSON block of an agent's answer that
177 has one, or ``None``."""
178 found: Any = None
179 for match in _FENCED_JSON.finditer(text):
180 try:
181 value = json.loads(match.group("body"))
182 except json.JSONDecodeError:
183 continue
184 if isinstance(value, dict) and "harnessSelfReport" in value: 184 ↛ 179line 184 didn't jump to line 179 because the condition on line 184 was always true
185 found = value["harnessSelfReport"]
186 return found