Coverage for src/governed_harness/intake/language.py: 100%
39 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The language of a task at intake: detection, tokenization and the Spanish vocabularies.
3``intake.language`` (``auto``, ``en`` or ``es``) chooses the vocabularies and the question
4texts of the deterministic INTENT rules. Without the key the rules keep their original form:
5ASCII tokens, English vocabularies and English questions.
7With the key, words are Unicode tokens (``contraseñas`` is one word, not ``contrase`` and
8``as``) and are compared without diacritics, so ``rapido`` and ``rápido`` are the same word.
10Detection (``auto``) is a fixed count, no language model: each word of the task (title,
11intent, criteria and their hints) that is a Spanish function word, a common word of short
12Spanish requests or a word of the Spanish vocabularies below counts for Spanish, each English function word counts for English, and
13each ``ñ``, ``¿``, ``¡`` or accented vowel counts for Spanish. Words that are also English
14(``a``, ``general``, ``once``, ...) count for neither. The task is Spanish when Spanish counts
15more, otherwise English (a tie is English).
16"""
18from __future__ import annotations
20import re
21import unicodedata
22from typing import Literal
24Language = Literal["en", "es"]
25"""A resolved intake language."""
27LANGUAGE_SETTINGS = ("auto", "en", "es")
28"""The values of ``intake.language``."""
30# A word starts with a letter or digit (any script) and goes on with letters, digits, "_",
31# "'" and "-": for ASCII text this is the original pattern.
32UNICODE_WORD = re.compile(r"[^\W_][\w'-]*")
34_SPANISH_MARKS = frozenset("ñ¿¡áéíóú")
35_SPANISH_FUNCTION_WORDS = frozenset(
36 {
37 "el",
38 "la",
39 "los",
40 "las",
41 "del",
42 "al",
43 "de",
44 "en",
45 "y",
46 "que",
47 "por",
48 "para",
49 "con",
50 "un",
51 "una",
52 "unos",
53 "unas",
54 "es",
55 "son",
56 "se",
57 "su",
58 "sus",
59 "lo",
60 "como",
61 "pero",
62 "cuando",
63 "debe",
64 "deben",
65 "esta",
66 "este",
67 "esto",
68 "estos",
69 "estas",
70 "cada",
71 "si",
72 "sin",
73 "muy",
74 "ya",
75 "hay",
76 "todo",
77 "todos",
78 }
79)
80_ENGLISH_FUNCTION_WORDS = frozenset(
81 {
82 "the",
83 "an",
84 "and",
85 "of",
86 "to",
87 "is",
88 "are",
89 "it",
90 "that",
91 "for",
92 "with",
93 "on",
94 "be",
95 "should",
96 "must",
97 "when",
98 "this",
99 "by",
100 "from",
101 "at",
102 "or",
103 "not",
104 "if",
105 "each",
106 "will",
107 "has",
108 "have",
109 "which",
110 "its",
111 "all",
112 "any",
113 "only",
114 "into",
115 "than",
116 }
117)
120def fold(word: str) -> str:
121 """A lower-case word without diacritics (``Contraseñas`` -> ``contrasenas``)."""
122 decomposed = unicodedata.normalize("NFD", word.lower())
123 return "".join(char for char in decomposed if not unicodedata.combining(char))
126def unicode_words(text: str) -> list[str]:
127 """The folded Unicode words of a text."""
128 return [fold(word) for word in UNICODE_WORD.findall(text.lower())]
131def detect_language(text: str) -> Language:
132 """``es`` when the text has more Spanish than English signals, otherwise ``en``."""
133 lowered = text.lower()
134 words = unicode_words(lowered)
135 spanish = sum(1 for word in words if word in _SPANISH_SIGNALS)
136 spanish += sum(1 for char in lowered if char in _SPANISH_MARKS)
137 english = sum(1 for word in words if word in _ENGLISH_FUNCTION_WORDS)
138 return "es" if spanish > english else "en"
141# ----- Spanish vocabularies (folded: no diacritics) ---------------------------------------
142# Applied together with the English ones, because Spanish tasks keep English code words.
144VAGUE_PHRASES_ES = (
145 "como se espera",
146 "como se esperaba",
147 "como es debido",
148 "facil de usar",
149 "amigable",
150)
151VAGUE_WORDS_ES = frozenset(
152 {
153 "funciona",
154 "funcionan",
155 "funcione",
156 "funcionen",
157 "funcionar",
158 "funcionando",
159 "funciono",
160 "correctamente",
161 "correcto",
162 "correcta",
163 "adecuadamente",
164 "apropiadamente",
165 "perfectamente",
166 "bien",
167 "bueno",
168 "buena",
169 "buenos",
170 "buenas",
171 "limpio",
172 "limpia",
173 "robusto",
174 "robusta",
175 }
176)
177FILLER_WORDS_ES = frozenset(
178 {
179 "el",
180 "la",
181 "los",
182 "las",
183 "lo",
184 "un",
185 "una",
186 "unos",
187 "unas",
188 "esto",
189 "eso",
190 "este",
191 "esta",
192 "estos",
193 "estas",
194 "ese",
195 "esa",
196 "todo",
197 "toda",
198 "todos",
199 "todas",
200 "debe",
201 "deben",
202 "deberia",
203 "deberian",
204 "tiene",
205 "tienen",
206 "que",
207 "ser",
208 "es",
209 "son",
210 "estan",
211 "estar",
212 "sea",
213 "sean",
214 "fue",
215 "y",
216 "e",
217 "o",
218 "u",
219 "pero",
220 "tambien",
221 "solo",
222 "muy",
223 "realmente",
224 "asi",
225 "siempre",
226 "ya",
227 "ahora",
228 "de",
229 "del",
230 "en",
231 "a",
232 "al",
233 "para",
234 "por",
235 "con",
236 "se",
237 "vale",
238 "funcion",
239 "funcionalidad",
240 "codigo",
241 "sistema",
242 "aplicacion",
243 "solucion",
244 "implementacion",
245 "programa",
246 "cambio",
247 "cambios",
248 "cosa",
249 "cosas",
250 "comportamiento",
251 "resultado",
252 "resultados",
253 "salida",
254 "general",
255 }
256)
257RESULT_VERBS_ES = frozenset(
258 {
259 "devuelve",
260 "devuelven",
261 "devolver",
262 "devuelto",
263 "devuelta",
264 "retorna",
265 "retornan",
266 "retornar",
267 "lanza",
268 "lanzan",
269 "lanzar",
270 "lanzado",
271 "rechaza",
272 "rechazan",
273 "rechazar",
274 "rechazado",
275 "rechazada",
276 "acepta",
277 "aceptan",
278 "aceptar",
279 "aceptado",
280 "aceptada",
281 "igual",
282 "iguales",
283 "contiene",
284 "contienen",
285 "contener",
286 "lista",
287 "listan",
288 "listar",
289 "listado",
290 "guarda",
291 "guardan",
292 "guardar",
293 "guardado",
294 "almacena",
295 "almacenan",
296 "almacenar",
297 "almacenado",
298 "imprime",
299 "imprimen",
300 "imprimir",
301 "impreso",
302 "muestra",
303 "muestran",
304 "mostrar",
305 "termina",
306 "terminan",
307 "terminar",
308 "responde",
309 "responden",
310 "responder",
311 "crea",
312 "crean",
313 "crear",
314 "creado",
315 "creada",
316 "elimina",
317 "eliminan",
318 "eliminar",
319 "eliminado",
320 "eliminada",
321 "borra",
322 "borran",
323 "borrar",
324 "borrado",
325 "coincide",
326 "coinciden",
327 "coincidir",
328 "pasa",
329 "pasan",
330 "pasar",
331 "falla",
332 "fallan",
333 "fallar",
334 "dentro",
335 "antes",
336 "despues",
337 }
338)
339RESULT_PHRASES_ES = ("como maximo", "como minimo", "a lo sumo", "al menos", "como mucho")
340_RAPIDO_ES = "rápido"
341QUALITY_WORDS_ES = {
342 "rapido": _RAPIDO_ES,
343 "rapida": _RAPIDO_ES,
344 "rapidos": _RAPIDO_ES,
345 "rapidas": _RAPIDO_ES,
346 "rapidamente": _RAPIDO_ES,
347 "veloz": "veloz",
348 "eficiente": "eficiente",
349 "eficientes": "eficiente",
350 "eficientemente": "eficiente",
351 "escalable": "escalable",
352 "escalables": "escalable",
353 "seguro": "seguro",
354 "segura": "seguro",
355 "seguros": "seguro",
356 "seguras": "seguro",
357 "fiable": "fiable",
358 "fiables": "fiable",
359 "confiable": "confiable",
360 "confiables": "confiable",
361 "responsivo": "responsivo",
362 "responsiva": "responsivo",
363 "performante": "performante",
364}
365# "un" and "una" are left out: in Spanish they are mostly articles.
366NUMBER_WORDS_ES = frozenset(
367 {
368 "cero",
369 "uno",
370 "dos",
371 "tres",
372 "cuatro",
373 "cinco",
374 "seis",
375 "siete",
376 "ocho",
377 "nueve",
378 "diez",
379 "once",
380 "doce",
381 "veinte",
382 "treinta",
383 "cincuenta",
384 "cien",
385 "ciento",
386 "mil",
387 "millon",
388 "millones",
389 "medio",
390 "mitad",
391 "doble",
392 }
393)
395QUESTION_TEMPLATES_ES: dict[str, str] = {
396 "C0a": (
397 "La tarea '{title}' no tiene criterios de aceptación. ¿Qué resultados observables "
398 "muestran que está terminada y cómo se comprueba cada uno (entrada o acción -> "
399 "resultado exacto esperado)? Un resultado por línea."
400 ),
401 "C0b": (
402 "¿Cuáles son las entradas y salidas principales de la tarea '{title}' y en qué formatos "
403 "(por ejemplo campos JSON, una cabecera CSV o la firma de una función)?"
404 ),
405 "C0c": (
406 "¿Qué límites debe respetar la tarea '{title}' (números, tamaños, tiempos) y qué pasa "
407 "exactamente en cada límite?"
408 ),
409 "C0d": (
410 "¿Qué errores o rechazos debe producir la tarea '{title}' ante una entrada no válida y "
411 "qué devuelve, lanza o imprime exactamente cada uno?"
412 ),
413 "C0e": "¿Qué comportamientos entran en el alcance de la tarea '{title}'? Uno por línea.",
414 "C0f": "¿Qué queda explícitamente fuera del alcance de la tarea '{title}'? Uno por línea.",
415 "C0g": (
416 "¿Qué restricciones no funcionales se aplican a la tarea '{title}' (rendimiento, "
417 "seguridad, persistencia) y con qué medida se comprueba cada una?"
418 ),
419 "C1": (
420 "El criterio {target} ('{text}') no dice qué se puede observar. ¿Para qué entrada o "
421 "acción, y qué resultado exacto muestra que se cumple?"
422 ),
423 "C2": (
424 "El criterio {target} ('{text}') pide una cualidad ('{term}') sin una medida. ¿Qué "
425 "número o umbral decide si se cumple (por ejemplo un tiempo, un tamaño o una tasa)?"
426 ),
427 "C3": (
428 "El criterio {target} ('{text}') repite el criterio {other}. ¿Qué resultado distinto "
429 "debe comprobar, o qué criterio debe sustituirlo?"
430 ),
431 "T1": (
432 "La intención ('{text}') es corta, no tiene requisitos y tiene un solo criterio de "
433 "aceptación sin nada concreto que comprobar. ¿Qué comportamientos separados debe "
434 "entregar la tarea y qué queda fuera del alcance?"
435 ),
436}
437"""Spanish question templates, with the same keys and placeholders as the English ones."""
439RECOMMENDATION_TEMPLATES: dict[Language, str] = {
440 "en": (
441 "Answer {question_id} with harness task clarify, or set intake.criteriaPolicy to "
442 "enforce to block INTENT until it is answered."
443 ),
444 "es": (
445 "Responde {question_id} con harness task clarify, o pon intake.criteriaPolicy en "
446 "enforce para que INTENT se bloquee hasta que se responda."
447 ),
448}
449"""The next step a ``warn`` finding of a clarification question recommends."""
451# Words of short Spanish requests ("agrega pruebas unitarias") that carry no function word.
452_SPANISH_TASK_WORDS = frozenset(
453 {
454 "agrega",
455 "agregar",
456 "anade",
457 "anadir",
458 "implementa",
459 "implementar",
460 "corrige",
461 "corregir",
462 "arregla",
463 "arreglar",
464 "haz",
465 "hacer",
466 "escribe",
467 "escribir",
468 "cambia",
469 "cambiar",
470 "quiero",
471 "necesito",
472 "prueba",
473 "pruebas",
474 "unitarias",
475 "archivo",
476 "datos",
477 "usuario",
478 "usuarios",
479 }
480)
481# Spanish words that are also common English words count for neither language.
482_SHARED_WORDS = frozenset({"a", "e", "o", "u", "general", "once", "mil", "no"})
483_SPANISH_SIGNALS = (
484 (
485 _SPANISH_FUNCTION_WORDS
486 | _SPANISH_TASK_WORDS
487 | VAGUE_WORDS_ES
488 | FILLER_WORDS_ES
489 | RESULT_VERBS_ES
490 | frozenset(QUALITY_WORDS_ES)
491 | NUMBER_WORDS_ES
492 )
493 - _SHARED_WORDS
494 - _ENGLISH_FUNCTION_WORDS
495)