Coverage for src/governed_harness/agents/self_report.py: 78%

111 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The agent's structured self-report (``provenance.selfReport``, since 2.0). 

2 

3A command provider answers with an optional ``selfReport`` object next to ``status``; a built-in 

4adapter asks the agent to end its answer with a fenced JSON block holding 

5``{"harnessSelfReport": {...}}``. Either way the report is read leniently: what can be read is 

6kept, what cannot is listed in ``problems``. It is never a protocol error and never a check: the 

7run does not depend on it.""" 

8 

9from __future__ import annotations 

10 

11import json 

12import re 

13from dataclasses import dataclass 

14from typing import Any 

15 

16from governed_harness.domain.models import DisputedAcceptanceTest, SelfReportItem 

17 

18SELF_REPORT_FIELDS = ( 

19 "assumptions", 

20 "alternativesDiscarded", 

21 "lowConfidenceAreas", 

22 "unrequestedChanges", 

23) 

24DISPUTED_FIELD = "disputedAcceptanceTests" 

25"""Optional fifth field (#94): ``[{test, reason}]``, asked for only when the frozen 

26acceptance tests can be disputed (``verification.acceptanceTests.disputes``).""" 

27MAX_ITEMS = 50 

28MAX_TEXT = 1000 

29 

30REQUEST_BLOCK: dict[str, Any] = { 

31 "requested": True, 

32 "fields": list(SELF_REPORT_FIELDS), 

33 "note": ( 

34 "Optional. Answer with selfReport: {assumptions: [text], alternativesDiscarded: [text], " 

35 "lowConfidenceAreas: [{path, description}], unrequestedChanges: [{path, description}]}. " 

36 "It is recorded as reported by the agent, not as verification." 

37 ), 

38} 

39"""What the request of a command provider carries under ``selfReport``.""" 

40 

41PROMPT_INSTRUCTIONS = ( 

42 "When you finish, end your answer with a fenced JSON code block (```json) containing one " 

43 'object {"harnessSelfReport": {"assumptions": [...], "alternativesDiscarded": [...], ' 

44 '"lowConfidenceAreas": [{"path": "...", "description": "..."}], ' 

45 '"unrequestedChanges": [{"path": "...", "description": "..."}]}}. List the assumptions you ' 

46 "made, the alternatives you discarded, the places where you are not confident, and every " 

47 "change you made that the task did not ask for. Use empty lists when there is nothing to " 

48 "report. The harness records it as your report; it does not replace verification." 

49) 

50 

51_FENCED_JSON = re.compile(r"```(?:json)?\s*\n(?P<body>\{(?:(?!\}\s*\n```).)*\})\s*\n```", re.DOTALL) 

52 

53 

54@dataclass(frozen=True) 

55class ParsedSelfReport: 

56 assumptions: tuple[str, ...] 

57 alternatives_discarded: tuple[str, ...] 

58 low_confidence_areas: tuple[SelfReportItem, ...] 

59 unrequested_changes: tuple[SelfReportItem, ...] 

60 problems: tuple[str, ...] 

61 disputed_acceptance_tests: tuple[DisputedAcceptanceTest, ...] = () 

62 

63 def paths(self) -> tuple[str, ...]: 

64 return tuple( 

65 dict.fromkeys( 

66 item.path 

67 for item in (*self.low_confidence_areas, *self.unrequested_changes) 

68 if item.path 

69 ) 

70 ) 

71 

72 

73def _text(value: Any) -> str | None: 

74 if not isinstance(value, str): 

75 return None 

76 text = " ".join(value.split()) 

77 return text[:MAX_TEXT] or None 

78 

79 

80def _texts(raw: Any, name: str, problems: list[str]) -> tuple[str, ...]: 

81 if raw is None: 

82 return () 

83 if not isinstance(raw, list): 83 ↛ 84line 83 didn't jump to line 84 because the condition on line 83 was never true

84 problems.append(f"{name} is not a list") 

85 return () 

86 values = [] 

87 for item in raw[:MAX_ITEMS]: 

88 text = _text(item) 

89 if text is None: 89 ↛ 90line 89 didn't jump to line 90 because the condition on line 89 was never true

90 problems.append(f"{name} has an entry that is not text") 

91 continue 

92 values.append(text) 

93 if len(raw) > MAX_ITEMS: 93 ↛ 94line 93 didn't jump to line 94 because the condition on line 93 was never true

94 problems.append(f"{name}: {len(raw) - MAX_ITEMS} entries beyond {MAX_ITEMS} were dropped") 

95 return tuple(values) 

96 

97 

98def _items(raw: Any, name: str, problems: list[str]) -> tuple[SelfReportItem, ...]: 

99 if raw is None: 

100 return () 

101 if not isinstance(raw, list): 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true

102 problems.append(f"{name} is not a list") 

103 return () 

104 items = [] 

105 for entry in raw[:MAX_ITEMS]: 

106 item, valid = _entry_item(entry) 

107 if item is not None: 107 ↛ 109line 107 didn't jump to line 109 because the condition on line 107 was always true

108 items.append(item) 

109 elif not valid: 

110 problems.append(f"{name} has an entry without a description") 

111 if len(raw) > MAX_ITEMS: 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true

112 problems.append(f"{name}: {len(raw) - MAX_ITEMS} entries beyond {MAX_ITEMS} were dropped") 

113 return tuple(items) 

114 

115 

116def _entry_item(entry: Any) -> tuple[SelfReportItem | None, bool]: 

117 """The item of one entry and whether the entry is valid: an empty text is valid but gives 

118 no item; an entry without a description or a path is not valid.""" 

119 if isinstance(entry, str): 119 ↛ 120line 119 didn't jump to line 120 because the condition on line 119 was never true

120 text = _text(entry) 

121 return (SelfReportItem(description=text) if text else None), True 

122 if isinstance(entry, dict): 122 ↛ 130line 122 didn't jump to line 130 because the condition on line 122 was always true

123 description = _text(entry.get("description")) 

124 path = entry.get("path") 

125 path_text = path.strip()[:MAX_TEXT] if isinstance(path, str) and path.strip() else None 

126 if description is None and path_text is not None: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 description = "(no description)" 

128 if description is not None: 128 ↛ 130line 128 didn't jump to line 130 because the condition on line 128 was always true

129 return SelfReportItem(path=path_text, description=description), True 

130 return None, False 

131 

132 

133def _disputed(raw: Any, problems: list[str]) -> tuple[DisputedAcceptanceTest, ...]: 

134 """``[{test, reason}]`` (``path`` and ``description`` are read too); an entry without a 

135 test or without a reason is a problem, not a dispute.""" 

136 if raw is None: 

137 return () 

138 if not isinstance(raw, list): 138 ↛ 139line 138 didn't jump to line 139 because the condition on line 138 was never true

139 problems.append(f"{DISPUTED_FIELD} is not a list") 

140 return () 

141 found = [] 

142 for entry in raw[:MAX_ITEMS]: 

143 item = entry if isinstance(entry, dict) else {} 

144 test = _text(item.get("test")) or _text(item.get("path")) 

145 reason = _text(item.get("reason")) or _text(item.get("description")) 

146 if test is None or reason is None: 

147 problems.append(f"{DISPUTED_FIELD} has an entry without a test or a reason") 

148 continue 

149 found.append(DisputedAcceptanceTest(test=test, reason=reason)) 

150 return tuple(found) 

151 

152 

153def parse_self_report(raw: Any) -> ParsedSelfReport | None: 

154 """The report in ``raw`` (an object with the four fields), ``None`` when there is none.""" 

155 if raw is None: 

156 return None 

157 problems: list[str] = [] 

158 if not isinstance(raw, dict): 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true

159 return ParsedSelfReport((), (), (), (), ("the self-report is not an object",)) 

160 unknown = sorted(set(raw) - set(SELF_REPORT_FIELDS) - {DISPUTED_FIELD}) 

161 if unknown: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true

162 problems.append(f"unknown field(s) ignored: {', '.join(unknown)}") 

163 return ParsedSelfReport( 

164 assumptions=_texts(raw.get("assumptions"), "assumptions", problems), 

165 alternatives_discarded=_texts( 

166 raw.get("alternativesDiscarded"), "alternativesDiscarded", problems 

167 ), 

168 low_confidence_areas=_items(raw.get("lowConfidenceAreas"), "lowConfidenceAreas", problems), 

169 unrequested_changes=_items(raw.get("unrequestedChanges"), "unrequestedChanges", problems), 

170 disputed_acceptance_tests=_disputed(raw.get(DISPUTED_FIELD), problems), 

171 problems=tuple(problems), 

172 ) 

173 

174 

175def extract_from_text(text: str) -> Any: 

176 """The ``harnessSelfReport`` object of the last fenced JSON block of an agent's answer that 

177 has one, or ``None``.""" 

178 found: Any = None 

179 for match in _FENCED_JSON.finditer(text): 

180 try: 

181 value = json.loads(match.group("body")) 

182 except json.JSONDecodeError: 

183 continue 

184 if isinstance(value, dict) and "harnessSelfReport" in value: 184 ↛ 179line 184 didn't jump to line 179 because the condition on line 184 was always true

185 found = value["harnessSelfReport"] 

186 return found