Criteria improvement

backend/api/criteria/context.py

@@ -0,0 +1,104 @@
+"""Canonical citation and imported-answer context helpers.
+
+This module deliberately has no database or web dependencies so prompt builders,
+CSV/RIS importers, and tests all use the same matching and formatting rules.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def resolve_source_value(row: dict[str, Any], header: str | None) -> Any:
+    """Resolve a configured source header without silently guessing."""
+    if not header:
+        return None
+    if header in row:
+        return row[header]
+    normalized = _normalize_key(header)
+    matches = [
+        value for key, value in row.items(
+        ) if _normalize_key(str(key)) == normalized
+    ]
+    return matches[0] if len(matches) == 1 else None
+
+
+def _normalize_key(value: str) -> str:
+    return re.sub(r'[^a-z0-9]+', '_', value.strip().casefold()).strip('_')
+
+
+def normalize_answer_text(value: Any) -> str:
+    """Normalize formatting only; semantic synonyms are intentionally excluded."""
+    return re.sub(r'\s+', ' ', str(value or '').strip()).casefold()
+
+
+def match_answer_label(raw_value: Any, options: list[dict[str, Any]]) -> dict[str, Any]:
+    """Match a raw value to an option label and retain the raw audit value."""
+    raw = '' if raw_value is None else str(raw_value)
+    if not raw.strip():
+        return {'status': 'blank', 'raw': raw, 'label': None, 'option_id': None}
+    normalized = normalize_answer_text(raw)
+    for option in options:
+        label = str(option.get('label') or '')
+        if normalized == normalize_answer_text(label):
+            return {'status': 'matched', 'raw': raw, 'label': label, 'option_id': option.get('id')}
+    return {'status': 'unmatched', 'raw': raw, 'label': None, 'option_id': None}
+
+
+def resolve_existing_human_value(
+    row: dict[str, Any], item: dict[str, Any],
+) -> dict[str, Any]:
+    """Resolve and classify an explicitly mapped human answer/parameter value."""
+    header = item.get('answer_column')
+    raw = resolve_source_value(row, header)
+    options = [*(item.get('answers') or []), *(item.get('options') or [])]
+    if options:
+        return match_answer_label(raw, options)
+    if raw is None or not str(raw).strip():
+        return {'status': 'blank', 'raw': '' if raw is None else str(raw), 'value': None}
+    return {'status': 'matched', 'raw': str(raw), 'value': raw}
+
+
+def format_title_abstract_context(
+    row: dict[str, Any], citation_fields: dict[str, Any],
+) -> str:
+    title = resolve_source_value(row, citation_fields.get('title'))
+    abstract = resolve_source_value(row, citation_fields.get('abstract'))
+    excluded = {citation_fields.get('title'), citation_fields.get('abstract')}
+    pairs = []
+    for header in citation_fields.get('l1_include') or []:
+        if header in excluded:
+            continue
+        pairs.append(f"{header}: {resolve_source_value(row, header) or ''}")
+    other = '; '.join(pairs) or '(none configured)'
+    return f"Title: {title or ''}\nAbstract: {abstract or ''}\nOther fields: {other}"
+
+
+def format_item_context(item: dict[str, Any]) -> str:
+    parts = [str(item.get('question') or item.get('name') or '').strip()]
+    if item.get('description'):
+        parts.append(f"Description: {item['description']}")
+    if item.get('context'):
+        parts.append(f"Context: {item['context']}")
+    if item.get('critical_prompt_additions'):
+        parts.append(
+            f"Critical prompt additions: {item['critical_prompt_additions']}",
+        )
+    if item.get('unit_instructions'):
+        parts.append(f"Units: {item['unit_instructions']}")
+    if item.get('calculation'):
+        parts.append(f"Calculation: {item['calculation']}")
+    if item.get('selection_mode'):
+        parts.append(f"Answer cardinality: {item['selection_mode']}")
+    options = [*(item.get('answers') or []), *(item.get('options') or [])]
+    if options:
+        parts.append(
+            'Answer-specific guidance:\n' + '\n'.join(
+                f"- {option.get('label')}: {option.get('context') or '(none)'}" for option in options
+            ),
+        )
+    if item.get('trigger', {}).get('all'):
+        parts.append(
+            f"Visible only when trigger conditions are met: {item['trigger']['all']}",
+        )
+    return '\n'.join(part for part in parts if part)

backend/api/criteria/models.py

@@ -52,6 +52,9 @@ def conditions_are_unique(self) -> Trigger:
 
 
 class CitationFields(StrictModel):
+    # Nullable while loading legacy criteria; save/use preflight requires both.
+    title: NonEmptyText | None = None
+    abstract: NonEmptyText | None = None
     l1_include: list[NonEmptyText] = Field(
         default_factory=list, max_length=100,
     )
@@ -77,7 +80,9 @@ class ScreeningQuestion(StrictModel):
     id: CriteriaId
     question: NonEmptyText
     answers: list[ScreeningAnswer] = Field(min_length=2, max_length=50)
+    context: str | None = Field(default=None, max_length=10000)
     trigger: Trigger = Field(default_factory=Trigger)
+    answer_column: NonEmptyText | None = None
 
     @model_validator(mode='after')
     def answer_ids_are_unique(self) -> ScreeningQuestion:
@@ -97,24 +102,28 @@ class TextParameter(StrictModel):
     id: CriteriaId
     name: NonEmptyText
     description: NonEmptyText
+    context: str | None = Field(default=None, max_length=10000)
     type: Literal['text']
     unit_instructions: str | None = Field(default=None, max_length=10000)
     calculation: str | None = Field(default=None, max_length=10000)
     trigger: Trigger = Field(default_factory=Trigger)
     legacy_category: str | None = Field(default=None, max_length=1000)
+    answer_column: NonEmptyText | None = None
 
 
 class SelectionParameter(StrictModel):
     id: CriteriaId
     name: NonEmptyText
     description: NonEmptyText
+    context: str | None = Field(default=None, max_length=10000)
     type: Literal['selection']
     selection_mode: Literal['single', 'multiple']
     options: list[ParameterOption] = Field(min_length=1, max_length=100)
     unit_instructions: str | None = Field(default=None, max_length=10000)
     calculation: str | None = Field(default=None, max_length=10000)
     trigger: Trigger = Field(default_factory=Trigger)
     legacy_category: str | None = Field(default=None, max_length=1000)
+    answer_column: NonEmptyText | None = None
 
     @model_validator(mode='after')
     def option_ids_are_unique(self) -> SelectionParameter:

backend/api/criteria/service.py

@@ -19,6 +19,12 @@
 LEGACY_KEYS = {'include', 'criteria', 'l2_criteria', 'parameters'}
 LEGACY_ONLY_KEYS = {'include', 'criteria', 'l2_criteria'}
 MAX_YAML_BYTES = 1_000_000
+DEFAULT_CITATION_FIELDS = ('Title', 'Abstract')
+
+
+def _with_default_citation_fields(values: list[str]) -> list[str]:
+    """Keep the fixed Title and Abstract fields in every criteria document."""
+    return [value for value in dict.fromkeys(values) if value not in DEFAULT_CITATION_FIELDS]
 
 
 class Diagnostic(BaseModel):
@@ -94,6 +100,19 @@ def normalize(self, raw: dict[str, Any], *, source_kind: str = 'backend_load') -
                     f'v2 criteria cannot contain legacy keys: {sorted(mixed)}',
                 )
             normalized = deepcopy(raw)
+            citation_fields = normalized.get('citation_fields')
+            if citation_fields is None:
+                normalized['citation_fields'] = {
+                    'title': 'Title', 'abstract': 'Abstract', 'l1_include': [],
+                }
+            elif isinstance(citation_fields, dict):
+                configured = citation_fields.get('l1_include', [])
+                if isinstance(configured, list) and all(isinstance(value, str) for value in configured):
+                    citation_fields['l1_include'] = _with_default_citation_fields(
+                        configured,
+                    )
+                citation_fields.setdefault('title', 'Title')
+                citation_fields.setdefault('abstract', 'Abstract')
             diagnostics = self._move_question_context_to_answers(normalized)
             return CriteriaLoadResult(
                 criteria=CriteriaConfigV2.model_validate(normalized),
@@ -110,39 +129,14 @@ def normalize(self, raw: dict[str, Any], *, source_kind: str = 'backend_load') -
 
     @staticmethod
     def _move_question_context_to_answers(raw: dict[str, Any]) -> list[Diagnostic]:
-        """Preserve obsolete v2 question guidance by applying it to every answer."""
-        diagnostics: list[Diagnostic] = []
-        for stage in ('l1', 'l2'):
-            items = raw.get(stage)
-            if not isinstance(items, list):
-                continue
-            for question_index, item in enumerate(items):
-                if not isinstance(item, dict) or 'context' not in item:
-                    continue
-                context = item.pop('context')
-                if not isinstance(context, str) or not context.strip():
-                    continue
-                answers = item.get('answers')
-                if not isinstance(answers, list):
-                    continue
-                for answer in answers:
-                    if not isinstance(answer, dict):
-                        continue
-                    answer_context = answer.get('context')
-                    parts = [context.strip()]
-                    if isinstance(answer_context, str) and answer_context.strip():
-                        parts.append(answer_context.strip())
-                    answer['context'] = '\n\n'.join(parts)
-                diagnostics.append(
-                    Diagnostic(
-                        severity='warning',
-                        code='question_context_moved_to_answers',
-                        source_path=[stage, question_index, 'context'],
-                        target_path=[stage, question_index, 'answers'],
-                        message='Question-level context was copied to each answer because screening guidance is answer-specific.',
-                    ),
-                )
-        return diagnostics
+        """Keep item context separate from answer-specific guidance.
+
+        Older code copied question context into every answer. That loses the
+        distinction required by the current prompt contract and duplicates
+        guidance. Reading is intentionally non-mutating and therefore safe for
+        both legacy and current v2 configurations.
+        """
+        return []
 
     def export_yaml(self, criteria: CriteriaConfigV2 | dict[str, Any]) -> str:
         model = criteria if isinstance(
@@ -151,6 +145,30 @@ def export_yaml(self, criteria: CriteriaConfigV2 | dict[str, Any]) -> str:
         payload = model.model_dump(mode='json', exclude_none=True)
         return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True, width=100)
 
+    @staticmethod
+    def required_citation_field_errors(
+        criteria: CriteriaConfigV2,
+        available_fields: set[str] | None = None,
+    ) -> list[dict[str, Any]]:
+        """Return save/use errors while allowing legacy criteria to be loaded."""
+        errors: list[dict[str, Any]] = []
+        fields = criteria.citation_fields
+        for key, label in (('title', 'Title'), ('abstract', 'Abstract')):
+            value = getattr(fields, key)
+            if not value:
+                errors.append({
+                    'code': f'missing_required_{key}_field', 'path': [
+                        'citation_fields', key,
+                    ], 'message': f'{label} source column is required.',
+                })
+            elif available_fields is not None and value not in available_fields:
+                errors.append({
+                    'code': 'unavailable_required_citation_field', 'path': [
+                        'citation_fields', key,
+                    ], 'message': f'{label} source column {value!r} is not available.',
+                })
+        return errors
+
     def build_compatibility_projection(
         self, criteria: CriteriaConfigV2 | dict[str, Any],
     ) -> dict[str, Any]:
@@ -188,6 +206,10 @@ def stage(items):
             grouped[category].append(item)
         return {
             'schema_version': 2,
+            # Keep the canonical mappings/items alongside the legacy-shaped
+            # projection so batch pipelines can use the same source columns
+            # and guidance as the interactive endpoints.
+            'citation_fields': model.citation_fields.model_dump(mode='json', exclude_none=True),
             'l1': {'include': model.citation_fields.l1_include, **stage(model.l1)},
             'l2': stage(model.l2),
             'parameters': {
@@ -217,6 +239,7 @@ def _migrate_legacy(self, legacy: dict[str, Any], *, source_kind: str) -> Criter
                 value.strip() for value in include
             ),
         )
+        normalized_include = _with_default_citation_fields(normalized_include)
         if len(normalized_include) != len(include):
             diagnostics.append(
                 Diagnostic(
@@ -323,7 +346,7 @@ def questions(key: str, stage: str) -> list[dict[str, Any]]:
         l2 = questions('l2_criteria', 'l2')
         criteria = CriteriaConfigV2.model_validate({
             'schema_version': 2,
-            'citation_fields': {'l1_include': normalized_include, 'doi': None},
+            'citation_fields': {'title': 'Title', 'abstract': 'Abstract', 'l1_include': normalized_include, 'doi': None},
             'l1': l1, 'l2': l2, 'parameters': converted_parameters,
         })
         canonical_source = json.dumps(

backend/api/extract/prompts.py

@@ -12,17 +12,18 @@
 - "found": a boolean (true/false) indicating whether the parameter was located or could be confidently derived.
 - "value": the extracted value as a string (or null if not found).
 - "explanation": a concise explanation (1-4 sentences) describing why this value was chosen or how it was derived.
-- "evidence_sentences": an array of integers indicating the sentence indices you used as evidence (e.g. [2, 5]). If there are no supporting sentences, return an empty array.
-- "evidence_tables": an array of integers indicating table numbers used (e.g. [1, 2]) or [].
-- "evidence_figures": an array of integers indicating figure numbers used (e.g. [3]) or [].
+- "evidence_sentences": an array of integers indicating the sentence indices you used as evidence (e.g. [2, 5]). Use this for both positive and negative conclusions.
+- "evidence_tables": an array of integers indicating table numbers used (e.g. [1, 2]) or []. Use this for both positive and negative conclusions.
+- "evidence_figures": an array of integers indicating figure numbers used (e.g. [3]) or []. Use this for both positive and negative conclusions.
 
 Requirements:
 - If the parameter is explicitly present, return the value exactly as found (preserve units/format) and list the sentence indices.
 - If the parameter must be computed or approximated, include the computed value and explain the computation in "explanation", and list the sentences used for calculation.
-- If the parameter is not present and cannot be deduced, set "found": false, "value": null, "explanation": briefly state why not found, and "evidence_sentences": [].
+- If the parameter is not present and cannot be deduced, set "found": false and "value": null, explain what the article does and does not report, and list the sentence indices that support that conclusion. Do not return an empty evidence_sentences array when relevant sentences support the explanation. An empty array is appropriate only when the full text contains no relevant supporting sentence.
+- Every factual claim in "explanation" must be grounded in the supplied full text, tables, or figures. In particular, a negative result still requires evidence for the reported quantities, definitions, or limitations that justify why the parameter cannot be extracted.
 - If a calculation is defined for the parameter, with a description of variables to be computed, find those variables and walk through the computation in the explanation.
 - Do NOT include any extra keys, XML, or human commentary. The output must be parseable by json.loads.
-- If a table or figure is referenced, ensure the explanation references the table/figure number and what was extracted from it.
+- If a sentence, table, or figure is referenced, ensure the explanation references the sentence/table/figure number and what was extracted from it.
 
 Inputs available for formatting:
 - {parameter_name}  (a short name for the parameter)

backend/api/extract/router.py

@@ -460,26 +460,6 @@ def _norm_int_list(v: Any) -> list[int]:
             detail='Citation not found to update',
         )
 
-    # Auto-fill human_param_* from llm_param_* if missing (never overwrite)
-    try:
-        col_human = col_name.replace('llm_param_', 'human_param_', 1)
-        human_payload = {
-            **stored,
-            'autofilled': True,
-            'source': 'llm',
-            'timestamp': datetime.utcnow().isoformat() + 'Z',
-        }
-        await run_in_threadpool(
-            cits_dp_service.copy_jsonb_if_empty,
-            citation_id,
-            col_name,
-            col_human,
-            human_payload,
-            table_name,
-        )
-    except Exception:
-        pass
-
     return {'status': 'success', 'sr_id': sr_id, 'citation_id': citation_id, 'column': col_name, 'extraction': stored}