Coverage for src/governed_harness/orchestration/intent_review.py: 96%
310 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Agent review of a task in INTENT (#37) and the check of a person's answers (#52, N9).
3The deterministic rules (C0-C3, T1) run first. Under ``intake.ambiguityReview: agent`` the
4harness then asks a provider (call kind ``clarify``) for ambiguity and completeness questions,
5grouped by category. The review runs once per task revision, keyed by the task digest: resuming
6a run or answering nothing new reuses the stored questions instead of calling again. An answer
7of no questions is recorded as evidence. A malformed answer is a finding and blocks INTENT, as
8does a provider failure: the questions may be what makes the task workable.
10Under ``intake.validateAnswers`` the answers that produced the current revision are checked for
11references the task and the workspace cannot resolve (a requirement id such as ``A1`` that
12appears nowhere, a document such as ``SPEC.md`` that is not in the workspace); each one becomes
13a question asking the person to attach the document or transcribe what it says. The agent review
14of the revised task also receives the answers and checks them against the documents they cite.
16Under the object form of ``intake.ambiguityReview`` (#79) the review converges and is bounded;
17see :mod:`governed_harness.orchestration.intent_convergence`."""
19from __future__ import annotations
21import re
22from dataclasses import dataclass
23from pathlib import Path
24from typing import TYPE_CHECKING, Any
26from governed_harness.agents.requests import CLARIFY_CATEGORIES
27from governed_harness.domain.enums import FindingSeverity, ResultStatus
28from governed_harness.domain.models import (
29 ClarificationQuestion,
30 ClarificationRecord,
31 Execution,
32 PhaseExecution,
33 Task,
34)
35from governed_harness.intake import resolve_language, task_digest
36from governed_harness.intake.depth import TOPIC_LABELS, DepthPlan, depth_plan, topic_language
37from governed_harness.orchestration.intent_convergence import (
38 Convergence,
39 Item,
40 convergence,
41 with_assumptions,
42)
43from governed_harness.runtime.workspace import DEFAULT_EXCLUDES
45if TYPE_CHECKING:
46 from governed_harness.orchestration.hosts import ResultsHost
48MALFORMED_RULE = "intake.agent-review-malformed"
49MAX_AGENT_QUESTIONS = 40
50_TEXT_LIMIT = 1000
51_DOC_SUFFIXES = (".md", ".txt", ".rst", ".pdf", ".doc", ".docx", ".yaml", ".yml", ".json")
52_PATH_RUN = re.compile(r"[\w/.-]+", re.I)
53_DOC_EXTENSION = re.compile(r"(?:md|txt|rst|pdf|docx?|ya?ml|json)(?!\w)", re.I)
54_ID_REFERENCE = re.compile(
55 r"\b(?:per|see|as in|according to|following|by|from|under|in)\s+"
56 r"(?:requirements?|rules?|sections?|criteri(?:on|a)|items?\s+)?\s*"
57 r"([A-Z]{1,3}-?\d{1,3}(?:\.\d+)?)\b"
58)
59_LIST_REFERENCE = re.compile(r"\brequirements?\s+([A-Z]{1,3}\d{0,3})\s*(?:to|-|through)\s*", re.I)
60_SEARCH_SUFFIXES = (".md", ".txt", ".rst", ".yaml", ".yml", ".json", ".py", ".ts", ".js")
61_MAX_SEARCH_BYTES = 2_000_000
64CONVERGE_KEY = ":converge"
65"""Suffix of the stored review of a revision under the converging review (#79)."""
66LAST_ROUND_KEY = "agentclarifylast"
67"""Since 2.0.1 (``skipSettled``): the digest and question count of the last review of a task
68in a run."""
69STOP_EVENT = "stop.condition"
70AMBIGUITY_STOP = "unresolvable-ambiguity"
72DEPTH_SUFFIX = (
73 " The request is {level} (depth.level, measured from its words, criteria, requirements and "
74 "entities): this review may take up to {maxRounds} rounds of at most {maxQuestions} "
75 "questions. {topics}"
76)
77DEPTH_TOPICS = (
78 "In this round cover first, in this order, the fundamental topics in depth.topics that the "
79 "task and the previous answers leave open ({labels}), with concrete questions a product "
80 "owner can answer, before any detail."
81)
82DEPTH_DETAILS = (
83 "The fundamental topics were covered in earlier rounds: ask about the details that remain "
84 "open (business rules with numbers, edge cases, errors, observable acceptance results)."
85)
88@dataclass(frozen=True)
89class ReviewOutcome:
90 questions: tuple[ClarificationQuestion, ...]
91 evidence_refs: tuple[str, ...]
92 blocked: str | None = None
93 revised: Task | None = None
94 """Since #79: the task revision that records the open points as assumptions."""
97@dataclass(frozen=True)
98class _Items:
99 """The agent's questions about one revision, the evidence and why it blocked."""
101 items: list[Item]
102 refs: tuple[str, ...]
103 blocked: str | None = None
106def _questions_from(result: dict[str, Any], task: Task) -> list[tuple[str, str, str]]:
107 """``(category, target, text)`` of every question in a ``clarify`` result; raises
108 ``ValueError`` for a malformed result."""
109 raw = result.get("questions")
110 if not isinstance(raw, list):
111 raise ValueError("the clarify result needs a 'questions' list")
112 criteria = {item.criterion_id for item in task.acceptance_criteria}
113 items: list[tuple[str, str, str]] = []
114 for index, entry in enumerate(raw, start=1):
115 if not isinstance(entry, dict): 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true
116 raise ValueError(f"question {index} is not an object")
117 text = entry.get("text")
118 if not isinstance(text, str) or not text.strip(): 118 ↛ 119line 118 didn't jump to line 119 because the condition on line 118 was never true
119 raise ValueError(f"question {index} has no text")
120 category = str(entry.get("category") or "ambiguity").strip().lower()
121 if category not in CLARIFY_CATEGORIES: 121 ↛ 122line 121 didn't jump to line 122 because the condition on line 121 was never true
122 category = "ambiguity"
123 target = str(entry.get("target") or "task").strip()
124 if target not in criteria:
125 target = "task"
126 items.append((category, target, " ".join(text.split())[:_TEXT_LIMIT]))
127 # Grouped by category, in the order of CLARIFY_CATEGORIES, then in the agent's order.
128 order = {name: position for position, name in enumerate(CLARIFY_CATEGORIES)}
129 items = sorted(items, key=lambda item: order[item[0]])
130 return items[:MAX_AGENT_QUESTIONS]
133def _numbered(
134 start: int, rule: str, items: list[tuple[str, str, str]]
135) -> tuple[ClarificationQuestion, ...]:
136 return tuple(
137 ClarificationQuestion.model_validate(
138 {
139 "questionId": f"Q-{number}",
140 "ruleId": rule,
141 "target": target,
142 "text": text,
143 "category": category,
144 }
145 )
146 for number, (category, target, text) in enumerate(items, start=start)
147 )
150class IntentReview:
151 def __init__(self, results: ResultsHost) -> None:
152 self.results = results
154 @property
155 def enabled(self) -> bool:
156 intake = self.results.project.intake
157 return bool(intake and intake.agent_review_enabled)
159 @property
160 def validates_answers(self) -> bool:
161 intake = self.results.project.intake
162 return bool(intake and intake.validate_answers)
164 def questions(
165 self,
166 execution: Execution,
167 phase: PhaseExecution,
168 task: Task,
169 deterministic: tuple[ClarificationQuestion, ...],
170 *,
171 agent: bool = True,
172 ) -> ReviewOutcome:
173 """The questions the agent-results settings add after the deterministic ones.
174 ``agent`` false (the fast lane of #58) leaves out the agent's ambiguity review. Under
175 the converging review (#79) an exhausted review may revise the task instead
176 (``revised``: its assumptions)."""
177 refs: list[str] = []
178 added: list[ClarificationQuestion] = []
179 start = len(deterministic) + 1
180 answers = self._current_answers(execution, task)
181 if self.validates_answers and answers:
182 dangling = self.dangling_references(task, answers)
183 added.extend(_numbered(start, "A2", dangling))
184 start += len(dangling)
185 # intake.projectSetup (#56): architecture, testing strategy and standards, once per
186 # project, for what neither the configuration nor the repository establishes.
187 setup = self.results.project_setup.questions(execution, phase, start)
188 added.extend(setup)
189 start += len(setup)
190 revised: Task | None = None
191 if self.enabled and agent:
192 outcome = self._agent_questions(execution, phase, task, deterministic, answers, start)
193 refs.extend(outcome.evidence_refs)
194 if outcome.blocked is not None:
195 return ReviewOutcome(tuple(added), tuple(refs), outcome.blocked)
196 added.extend(outcome.questions)
197 revised = outcome.revised
198 return ReviewOutcome(tuple(added), tuple(refs), revised=revised)
200 # ----- agent review ----------------------------------------------------------------------
201 def _agent_questions(
202 self,
203 execution: Execution,
204 phase: PhaseExecution,
205 task: Task,
206 deterministic: tuple[ClarificationQuestion, ...],
207 answers: list[dict[str, str]],
208 start: int,
209 ) -> ReviewOutcome:
210 intake = self.results.project.intake
211 settings = intake.ambiguity_settings if intake else None
212 plan = self._plan(task)
213 if settings is not None and plan is not None:
214 # depth: adaptive: the rounds and the questions of this task's plan.
215 settings = settings.model_copy(
216 update={"max_rounds": plan.rounds, "max_questions": plan.questions}
217 )
218 context = convergence(self.results, execution, task, settings) if settings else None
219 fetched = self._review_items(execution, phase, task, deterministic, answers, context)
220 if fetched.blocked is not None:
221 return ReviewOutcome((), fetched.refs, fetched.blocked)
222 items = fetched.items
223 if context is None or not items or not context.exhausted:
224 return ReviewOutcome(_numbered(start, "A1", items), fetched.refs)
225 return self._exhausted(execution, phase, task, items, fetched.refs, context, start)
227 def _review_items(
228 self,
229 execution: Execution,
230 phase: PhaseExecution,
231 task: Task,
232 deterministic: tuple[ClarificationQuestion, ...],
233 answers: list[dict[str, str]],
234 context: Convergence | None,
235 ) -> _Items:
236 """The agent's questions about this revision: stored once per task revision (and per
237 review form), so a resumed run or a second run of the revision makes no call."""
238 results = self.results
239 digest = task_digest(task)
240 key = f"agentclarify:{execution.project_id}:{digest}"
241 if context is not None:
242 key += CONVERGE_KEY
243 cached = results.flag_json(key)
244 if isinstance(cached, dict) and isinstance(cached.get("items"), list):
245 items = [(str(a), str(b), str(c)) for a, b, c in cached["items"]]
246 results.s.events.append(
247 execution.execution_id,
248 "intent.agent-review.reused",
249 {"taskDigest": digest, "questions": len(items), "evidenceRef": cached["ref"]},
250 )
251 self._remember_round(execution, task, context, len(items))
252 return _Items(items, (cached["ref"],))
253 reason = self._settled(execution, task, context)
254 if reason is not None and context is not None:
255 ref = self._record_skip(execution, phase, task, reason, context)
256 results.set_flag_json(key, {"items": [], "ref": ref})
257 return _Items([], (ref,))
258 fetched = self._call_review(execution, phase, task, deterministic, answers, context)
259 if fetched.blocked is None:
260 results.set_flag_json(
261 key, {"items": [list(item) for item in fetched.items], "ref": fetched.refs[-1]}
262 )
263 self._remember_round(execution, task, context, len(fetched.items))
264 return fetched
266 # ----- a settled review (2.0.1, ``skipSettled``) -------------------------------------------
267 @staticmethod
268 def _skips(context: Convergence | None) -> bool:
269 return context is not None and context.settings.skips_settled
271 def _remember_round(
272 self, execution: Execution, task: Task, context: Convergence | None, questions: int
273 ) -> None:
274 if not self._skips(context):
275 return
276 self.results.set_flag_json(
277 f"{LAST_ROUND_KEY}:{execution.execution_id}:{task.task_id}",
278 {"taskDigest": task_digest(task), "questions": questions},
279 )
281 def _settled(self, execution: Execution, task: Task, context: Convergence | None) -> str | None:
282 """Why the review of this new revision is not called (``skipSettled``): the previous
283 round of this run left no agent question, or the ``unresolvable-ambiguity`` stop
284 condition was recorded in this run; ``None`` otherwise."""
285 if not self._skips(context):
286 return None
287 stopped = any(
288 item.event_type == STOP_EVENT and item.payload.get("condition") == AMBIGUITY_STOP
289 for item in self.results.s.events.list(execution.execution_id)
290 )
291 if stopped:
292 return AMBIGUITY_STOP
293 last = self.results.flag_json(f"{LAST_ROUND_KEY}:{execution.execution_id}:{task.task_id}")
294 if (
295 isinstance(last, dict)
296 and last.get("questions") == 0
297 and last.get("taskDigest") != task_digest(task)
298 ):
299 return "no-new-questions"
300 return None
302 def _record_skip(
303 self,
304 execution: Execution,
305 phase: PhaseExecution,
306 task: Task,
307 reason: str,
308 context: Convergence,
309 ) -> str:
310 """Evidence and event of a review that was not called: the run goes on as after an
311 exhausted review with no open point (``onExhausted`` has nothing to ask or assume)."""
312 results = self.results
313 record = {
314 "taskDigest": task_digest(task),
315 "reason": reason,
316 "rounds": context.rounds,
317 "maxRounds": context.settings.rounds,
318 "onExhausted": context.settings.exhausted,
319 "questions": [],
320 }
321 ref = results.record_json(
322 execution,
323 phase.phase_id,
324 record,
325 kind="agent-clarify-skipped",
326 summary=f"Agent review of the task not called again ({reason})",
327 )
328 results.s.events.append(
329 execution.execution_id,
330 "intent.agent-review.skipped",
331 {**record, "evidenceRef": ref},
332 phase_execution_id=phase.phase_execution_id,
333 )
334 return ref
336 def _call_review(
337 self,
338 execution: Execution,
339 phase: PhaseExecution,
340 task: Task,
341 deterministic: tuple[ClarificationQuestion, ...],
342 answers: list[dict[str, str]],
343 context: Convergence | None,
344 ) -> _Items:
345 results = self.results
346 payload: dict[str, Any] = {
347 "deterministicQuestions": [
348 item.model_dump(mode="json", by_alias=True) for item in deterministic
349 ],
350 "categories": list(CLARIFY_CATEGORIES),
351 }
352 if answers:
353 payload["clarifications"] = answers
354 suffix = ""
355 if context is not None:
356 payload.update(context.payload())
357 suffix = context.suffix()
358 plan = self._plan(task)
359 if plan is not None:
360 depth, depth_suffix = _depth_request(plan, context, self._language(task))
361 payload["depth"] = depth
362 suffix += depth_suffix
363 outcome = results.call_agent(
364 execution,
365 phase,
366 "clarify",
367 payload,
368 task=task,
369 instructions_suffix=suffix,
370 validate=lambda result: _questions_from(result, task),
371 )
372 if outcome.status is not ResultStatus.PASSED or outcome.result is None:
373 return _Items(
374 [],
375 outcome.evidence_refs,
376 f"Agent review of the task did not answer ({outcome.status}): {outcome.summary}",
377 )
378 try:
379 items = _questions_from(outcome.result, task)
380 except ValueError as error:
381 results.record_finding(
382 execution,
383 validator_id="intake.agent-review",
384 rule_id=MALFORMED_RULE,
385 category="intent-clarification",
386 severity=FindingSeverity.HIGH,
387 message=f"The agent review of the task returned a malformed result: {error}",
388 evidence_refs=outcome.evidence_refs,
389 recommendation="Fix the provider's clarify answer and continue the run.",
390 )
391 return _Items(
392 [], outcome.evidence_refs, f"Agent review of the task was malformed: {error}"
393 )
394 dropped = 0
395 if context is not None:
396 items, dropped = context.converge(items)
397 ref = self._record_review(execution, phase, task, outcome.invocation_id, items, dropped)
398 return _Items(items, (*outcome.evidence_refs, ref))
400 def _record_review(
401 self,
402 execution: Execution,
403 phase: PhaseExecution,
404 task: Task,
405 invocation_id: str | None,
406 items: list[Item],
407 dropped: int,
408 ) -> str:
409 results = self.results
410 digest = task_digest(task)
411 record: dict[str, Any] = {
412 "taskDigest": digest,
413 "invocationId": invocation_id,
414 "questions": [
415 {"category": category, "target": target, "text": text}
416 for category, target, text in items
417 ],
418 "byCategory": {
419 name: sum(1 for item in items if item[0] == name)
420 for name in CLARIFY_CATEGORIES
421 if any(item[0] == name for item in items)
422 },
423 }
424 if dropped:
425 # #79: questions already asked, or over the round's limit.
426 record["dropped"] = dropped
427 ref = results.record_json(
428 execution,
429 phase.phase_id,
430 record,
431 kind="agent-clarify-review",
432 summary=f"Agent review of the task: {len(items)} question(s)",
433 )
434 results.s.events.append(
435 execution.execution_id,
436 "intent.agent-review.completed",
437 {"taskDigest": digest, "questions": len(items), "evidenceRef": ref},
438 )
439 return ref
441 # ----- the converging review (#79) -------------------------------------------------------
442 def _exhausted(
443 self,
444 execution: Execution,
445 phase: PhaseExecution,
446 task: Task,
447 items: list[Item],
448 refs: tuple[str, ...],
449 context: Convergence,
450 start: int,
451 ) -> ReviewOutcome:
452 """The agent still asks after the last round: under ``assume`` its points become the
453 assumptions of a new task revision, under ``block`` they are asked again."""
454 settings = context.settings
455 self.results.s.events.append(
456 execution.execution_id,
457 "intent.ambiguity.exhausted",
458 {
459 "taskDigest": task_digest(task),
460 "rounds": context.rounds,
461 "maxRounds": settings.rounds,
462 "open": len(items),
463 "onExhausted": settings.exhausted,
464 },
465 phase_execution_id=phase.phase_execution_id,
466 )
467 if settings.exhausted == "block":
468 return ReviewOutcome(_numbered(start, "A1", items), refs)
469 revised, added = with_assumptions(task, items, context.rounds)
470 if not added: 470 ↛ 471line 470 didn't jump to line 471 because the condition on line 470 was never true
471 return ReviewOutcome((), refs)
472 ref = self._store_revision(execution, phase, task, revised, added, context)
473 return ReviewOutcome((), (*refs, ref), revised=revised)
475 def _store_revision(
476 self,
477 execution: Execution,
478 phase: PhaseExecution,
479 task: Task,
480 revised: Task,
481 added: list[dict[str, Any]],
482 context: Convergence,
483 ) -> str:
484 """Store the revision with the assumptions (pinned to the run under
485 ``governance.pinTaskRevision``), as evidence and on the chain."""
486 results = self.results
487 services = results.s
488 services.state.put("task", revised.task_id, revised, project_id=revised.project_id)
489 revision_ref = services.artifacts.put_json(
490 revised.model_dump(mode="json"), metadata={"kind": "task-revision"}
491 )
492 current = task_digest(revised)
493 if results.project.governance_settings.pin_task_revision: 493 ↛ 495line 493 didn't jump to line 495 because the condition on line 493 was always true
494 services.pin_task_revision(execution.execution_id, revision_ref.uri, current)
495 record = {
496 "previousTaskDigest": task_digest(task),
497 "taskDigest": current,
498 "taskRef": revision_ref.uri,
499 "rounds": context.rounds,
500 "maxRounds": context.settings.rounds,
501 "assumptions": added,
502 }
503 ref = results.record_json(
504 execution,
505 phase.phase_id,
506 record,
507 kind="ambiguity-assumptions",
508 summary=(
509 f"{len(added)} open point(s) of the agent review recorded as assumptions after "
510 f"{context.rounds} round(s)"
511 ),
512 supports=(task.task_id,),
513 )
514 # The revision's review is settled: its assumptions are its answer.
515 results.set_flag_json(
516 f"agentclarify:{execution.project_id}:{current}{CONVERGE_KEY}",
517 {"items": [], "ref": ref},
518 )
519 services.events.append(
520 execution.execution_id,
521 "intent.assumptions.recorded",
522 {**record, "evidenceRef": ref},
523 phase_execution_id=phase.phase_execution_id,
524 )
525 return ref
527 def _plan(self, task: Task) -> DepthPlan | None:
528 """The plan of ``depth: adaptive`` from the configured rounds and questions (never
529 from the settings the plan already adjusted); ``None`` without the key."""
530 intake = self.results.project.intake
531 settings = intake.ambiguity_settings if intake else None
532 if settings is None or not settings.adaptive:
533 return None
534 return depth_plan(task, settings.rounds, settings.questions)
536 def _language(self, task: Task) -> str:
537 intake = self.results.project.intake
538 return topic_language(resolve_language(task, intake.language if intake else None))
540 # ----- answers ---------------------------------------------------------------------------
541 def _current_answers(self, execution: Execution, task: Task) -> list[dict[str, str]]:
542 """The answers that produced the current revision of the task (if it came from
543 ``task clarify``)."""
544 digest = task_digest(task)
545 records = [
546 item
547 for item in self.results.s.state.list(
548 "clarification", ClarificationRecord, project_id=execution.project_id
549 )
550 if item.task_id == task.task_id and item.task_digest == digest
551 ]
552 if not records:
553 return []
554 latest = max(records, key=lambda item: item.recorded_at)
555 return [
556 {"questionId": item.question_id, "question": item.question, "answer": item.answer}
557 for item in latest.answers
558 ]
560 def dangling_references(
561 self, task: Task, answers: list[dict[str, str]]
562 ) -> list[tuple[str, str, str]]:
563 """Questions about answers that cite a requirement id or a document that neither the
564 task nor the workspace contains."""
565 workspace = self.results.s.paths.workspace
566 task_text = " ".join(
567 [
568 task.title,
569 task.intent,
570 *task.constraints,
571 *(item.text for item in task.requirements),
572 *(item.requirement_id for item in task.requirements),
573 *(item.text for item in task.acceptance_criteria),
574 *(item.criterion_id for item in task.acceptance_criteria),
575 ]
576 )
577 corpus: str | None = None
578 items: list[tuple[str, str, str]] = []
579 seen: set[str] = set()
580 for answer in answers:
581 text = answer["answer"]
582 _dangling_documents(workspace, answer, text, seen, items)
583 corpus = _dangling_identifiers(workspace, answer, text, task_text, corpus, seen, items)
584 return items
587def _depth_request(
588 plan: DepthPlan, context: Convergence, language: str
589) -> tuple[dict[str, Any], str]:
590 """The ``depth`` part of a ``clarify`` request under ``depth: adaptive`` and the line it adds
591 to the instructions: the level, the plan and the fundamental topics of this round."""
592 topics = plan.round_topics(context.round)
593 labels = TOPIC_LABELS[language]
594 request = {
595 **plan.as_dict(),
596 "round": context.round,
597 "topics": [{"topic": topic, "label": labels[topic]} for topic in topics],
598 }
599 focus = (
600 DEPTH_TOPICS.format(labels="; ".join(labels[topic] for topic in topics))
601 if topics
602 else DEPTH_DETAILS
603 )
604 return request, DEPTH_SUFFIX.format(
605 level=plan.level,
606 maxRounds=plan.rounds,
607 maxQuestions=plan.questions,
608 topics=focus,
609 )
612def _doc_references(text: str) -> list[str]:
613 """The document paths ``text`` names, such as ``docs/spec.md``, in order.
615 A reference starts a run of path characters (word characters, ``/``, ``.``, ``-``): its
616 stem, the part before the run's first dot, is names of word characters and ``-`` joined by
617 single slashes, and a known extension follows the dot, not followed by a word character.
618 Each run is read once, so the time is linear in the length of ``text``.
619 """
620 found: list[str] = []
621 for run in _PATH_RUN.finditer(text):
622 stem, dot, rest = run.group().partition(".")
623 if not dot or not stem or stem[0] == "/" or stem[-1] == "/" or "//" in stem:
624 continue
625 extension = _DOC_EXTENSION.match(rest)
626 if extension is not None:
627 found.append(f"{stem}.{extension.group()}")
628 return found
631def _dangling_documents(
632 workspace: Path,
633 answer: dict[str, str],
634 text: str,
635 seen: set[str],
636 items: list[tuple[str, str, str]],
637) -> None:
638 """A question for every document the answer refers to that is not in the workspace."""
639 for document in _doc_references(text):
640 if document in seen or _document_exists(workspace, document):
641 continue
642 seen.add(document)
643 items.append(
644 (
645 "consistency",
646 "task",
647 f"The answer to {answer['questionId']} refers to {document}, which is "
648 "not in the workspace: attach the document or transcribe the part "
649 "the answer relies on.",
650 )
651 )
654def _dangling_identifiers(
655 workspace: Path,
656 answer: dict[str, str],
657 text: str,
658 task_text: str,
659 corpus: str | None,
660 seen: set[str],
661 items: list[tuple[str, str, str]],
662) -> str | None:
663 """A question for every identifier the answer cites that appears neither in the task nor
664 in the workspace; returns the workspace text once it was read, so it is read only once."""
665 identifiers = set(_ID_REFERENCE.findall(text)) | set(_LIST_REFERENCE.findall(text))
666 for identifier in sorted(identifiers):
667 if identifier in seen or re.search(rf"(?<!\w){re.escape(identifier)}(?!\w)", task_text): 667 ↛ 668line 667 didn't jump to line 668 because the condition on line 667 was never true
668 continue
669 if corpus is None: 669 ↛ 671line 669 didn't jump to line 671 because the condition on line 669 was always true
670 corpus = _workspace_text(workspace)
671 if re.search(rf"(?<!\w){re.escape(identifier)}(?!\w)", corpus):
672 continue
673 seen.add(identifier)
674 items.append(
675 (
676 "consistency",
677 "task",
678 f"The answer to {answer['questionId']} cites {identifier!r}, which "
679 "appears neither in the task nor in the workspace: transcribe what "
680 f"{identifier} says, or attach the document that defines it.",
681 )
682 )
683 return corpus
686def _document_exists(workspace: Path, document: str) -> bool:
687 candidate = (workspace / document).resolve()
688 try:
689 candidate.relative_to(workspace.resolve())
690 except ValueError:
691 return False
692 if candidate.is_file():
693 return True
694 name = Path(document).name.lower()
695 return any(path.name.lower() == name for path in _text_files(workspace, _DOC_SUFFIXES))
698def _text_files(workspace: Path, suffixes: tuple[str, ...]) -> list[Path]:
699 found: list[Path] = []
700 for path in sorted(workspace.rglob("*")):
701 relative = path.relative_to(workspace)
702 if any(part in DEFAULT_EXCLUDES for part in relative.parts):
703 continue
704 if path.is_file() and not path.is_symlink() and path.suffix.lower() in suffixes:
705 found.append(path)
706 return found
709def _workspace_text(workspace: Path) -> str:
710 """The text of the workspace's documents and sources, bounded, to resolve a cited id."""
711 parts: list[str] = []
712 total = 0
713 for path in _text_files(workspace, _SEARCH_SUFFIXES):
714 try:
715 data = path.read_bytes()[: _MAX_SEARCH_BYTES - total]
716 except OSError:
717 continue
718 total += len(data)
719 parts.append(data.decode("utf-8", "replace"))
720 if total >= _MAX_SEARCH_BYTES: 720 ↛ 721line 720 didn't jump to line 721 because the condition on line 720 was never true
721 break
722 return "\n".join(parts)