diff --git a/backend/.pre-commit-config.yaml b/backend/.pre-commit-config.yaml index dd04f7c1..78b77155 100644 --- a/backend/.pre-commit-config.yaml +++ b/backend/.pre-commit-config.yaml @@ -1,3 +1,7 @@ +# SQL migrations are immutable after release because their exact contents are +# recorded in review_schema_migrations checksums. +exclude: ^migrations/.*\.sql$ + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 @@ -46,4 +50,4 @@ repos: - --explicit-package-bases - --ignore-missing-imports additional_dependencies: [types-pyyaml] - exclude: ^testing/resources/ \ No newline at end of file + exclude: ^testing/resources/ diff --git a/backend/api/citations/router.py b/backend/api/citations/router.py index 528c5e8f..1c86aa03 100644 --- a/backend/api/citations/router.py +++ b/backend/api/citations/router.py @@ -16,6 +16,7 @@ import csv import io +import json import os import re import time @@ -30,7 +31,7 @@ except Exception: # pragma: no cover rispy = None -from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, status +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, status from fastapi.responses import Response, StreamingResponse from fastapi.concurrency import run_in_threadpool from pydantic import BaseModel @@ -41,6 +42,14 @@ from ..core.config import settings from ..services.cit_db_service import cits_dp_service, snake_case, snake_case_column, parse_dsn from ..services.citation_export_service import citation_export_service, ExportValidationError +from ..services.citation_import_preview_service import CitationImportPreviewService +from ..services.citation_import_service import citation_import_service +from ..services.citation_workspace_preferences_service import citation_workspace_preferences_service +from ..services.citation_deduplication_preferences_service import citation_deduplication_preferences_service +from ..services.citation_deduplication_service import recompute_affected_duplicate_groups +from ..services.citation_duplicate_review_service import citation_duplicate_review_service +from ..services.citation_duplicate_run_service import citation_duplicate_run_service +from ..services.storage import storage_service from ..criteria.context import resolve_source_value from .export_models import CitationExportRequest, CitationExportSchema from ..core.cit_utils import load_sr_and_check @@ -92,6 +101,558 @@ class UploadResult(BaseModel): created_at: str +class CitationImportPreviewResponse(BaseModel): + id: str + source_format: str + source_sha256: str + schema_fingerprint: str + proposed_mapping: dict[str, str | None] + validation_report: dict[str, Any] + staging_expires_at: str + + +class CitationImportPreviewCancelResponse(BaseModel): + id: str + cancelled: bool + staging_cleanup_pending: bool + + +class CitationImportPreviewInspectionResponse(BaseModel): + id: str + citation_table_name: str | None + source_sha256: str + schema_fingerprint: str + proposed_mapping: dict[str, str | None] + validation_report: dict[str, Any] + staging_expires_at: str + created_at: str + + +class CitationImportPreviewCommitRequest(BaseModel): + approved_mapping: dict[str, str | None] + + +class CitationImportPreviewMappingUpdateRequest(BaseModel): + approved_mapping: dict[str, str | None] + excluded_source_columns: list[str] + excluded_ambiguous_row_numbers: list[int] = [] + + +class CitationImportPreviewMappingUpdateResponse(BaseModel): + id: str + proposed_mapping: dict[str, str | None] + mapping_decision: dict[str, Any] + validation_report: dict[str, Any] + + +class CitationImportPreviewCommitResponse(BaseModel): + id: str + batch_id: str + idempotent: bool + inserted_count: int + existing_exact_match_count: int + invalid_count: int + + +class CitationImportResponse(BaseModel): + sr_id: str + table_name: str + batch_id: str + idempotent: bool + rows_inserted: int + invalid_count: int + warnings: list[str] = [] + duplicates_skipped: int = 0 + rows_merged: int = 0 + + +class CitationWorkspaceColumnsRequest(BaseModel): + columns: list[str] + + +class SetAnswerRequest(BaseModel): + item_type: str + item_id: str + source_column: str + + +def _criteria_config_items(sr: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + criteria = sr.get('criteria') or sr.get('criteria_parsed') or {} + if not isinstance(criteria, dict): + return [], [] + + def items(stage: str) -> list[dict[str, Any]]: + value = criteria.get(stage) or [] + if isinstance(value, dict): + value = value.get('items') or [] + return [item for item in value if isinstance(item, dict)] + + return items('l1'), items('l2') + + +def _parameter_items(sr: dict[str, Any]) -> list[dict[str, Any]]: + criteria = sr.get('criteria') or sr.get('criteria_parsed') or {} + parameters = criteria.get('parameters') if isinstance( + criteria, dict, + ) else [] + if isinstance(parameters, dict): + parameters = parameters.get('items') or [] + return [item for item in (parameters or []) if isinstance(item, dict)] + + +def _set_answer_payload(value: Any, item_type: str, screening_step: str) -> dict[str, Any]: + value = str(value).strip() + provenance = 'retrospective_validation' + timestamp = datetime.utcnow().isoformat() + 'Z' + if item_type == 'parameter': + return { + 'found': True, + 'value': value, + 'explanation': '', + 'evidence_sentences': [], + 'human': True, + 'reviewer': None, + 'source': provenance, + 'timestamp': timestamp, + } + return { + 'selected': value, + 'explanation': '', + 'confidence': None, + 'human': True, + 'reviewer': None, + 'source': provenance, + 'timestamp': timestamp, + 'screening_step': screening_step, + 'pipeline': 'fulltext' if screening_step == 'l2' else 'title_abstract', + } + + +def _set_answer_destinations(item_type: str, core: str) -> list[str]: + if item_type == 'parameter': + return [f'human_param_{core}'] + if item_type == 'l1': + return [f'human_l1_{core}', f'human_l2_{core}'] + return [f'human_l2_{core}'] + + +@router.post('/{sr_id}/set-answer') +async def set_answer( + sr_id: str, + payload: SetAnswerRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Copy a configured retrospective citation field into human answer fields.""" + item_type = payload.item_type.strip().lower() + if item_type not in {'l1', 'l2', 'parameter'}: + raise HTTPException( + status_code=400, detail='item_type must be l1, l2, or parameter', + ) + source_column = payload.source_column.strip() + if not source_column: + raise HTTPException( + status_code=400, detail='source_column is required', + ) + + sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + l1_items, l2_items = _criteria_config_items(sr) + parameter_items = _parameter_items(sr) + if item_type == 'parameter': + item = next( + ( + i for i in parameter_items if str( + i.get('id'), + ) == payload.item_id + ), None, + ) + stage = 'parameter' + else: + items = l1_items if item_type == 'l1' else l2_items + item = next( + ( + i for i in items if str( + i.get('id'), + ) == payload.item_id + ), None, + ) + stage = item_type + if not item: + raise HTTPException( + status_code=404, detail='Configured item not found', + ) + if item.get('answer_column') != source_column: + raise HTTPException( + status_code=409, detail='source_column does not match the configured answer column', + ) + + item_name = item.get( + 'name', + ) if item_type == 'parameter' else item.get('question') + core = snake_case( + str(item_name or ''), + max_len=52 if item_type == 'parameter' else 56, + ) + if not core: + raise HTTPException( + status_code=400, detail='Configured item has no usable name', + ) + destinations = _set_answer_destinations(item_type, core) + + table_name = (screening or {}).get('table_name') or 'citations' + try: + columns = await run_in_threadpool(cits_dp_service.get_table_columns, table_name) + available_columns = { + str(column.get('column_name')) + for column in columns + } + if source_column not in available_columns: + raise HTTPException( + status_code=400, detail=f"Source citation field '{source_column}' was not found", + ) + citation_ids = await run_in_threadpool(cits_dp_service.list_citation_ids, None, table_name) + rows = await run_in_threadpool( + cits_dp_service.get_citations_by_ids, + citation_ids, + table_name, + [ + 'id', source_column, + ], + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) + except Exception as exc: + raise HTTPException( + status_code=500, detail=f'Failed to query screening DB: {exc}', + ) + + processed = copied = blank = 0 + for row in rows: + processed += 1 + value = row.get(source_column) + if value is None or (isinstance(value, str) and not value.strip()): + blank += 1 + continue + answer = _set_answer_payload(value, item_type, stage) + for destination in destinations: + await run_in_threadpool( + cits_dp_service.update_jsonb_column, int( + row['id'], + ), destination, answer, table_name, + ) + copied += 1 + + return { + 'status': 'success', 'item_type': item_type, 'item_id': payload.item_id, + 'processed': processed, 'copied': copied, 'blank': blank, + 'destinations': destinations, + } + + +@router.post('/{sr_id}/import-previews', response_model=CitationImportPreviewResponse) +async def create_citation_import_preview( + sr_id: str, + file: UploadFile = File(...), + commit_key: str = Form(...), + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Parse and encrypt-stage an import; this endpoint never mutates citations.""" + sr, _screening = await load_sr_and_check( + sr_id, current_user, srdb_service, require_screening=False, + ) + if not file or not file.filename: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail='File is required', + ) + try: + raw_bytes = await file.read() + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f'Failed to read upload: {exc}', + ) + if len(raw_bytes) > settings.MAX_FILE_SIZE: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail='Uploaded file is too large', + ) + citation_table_name = (sr.get('screening_db') or {}).get('table_name') + try: + service = CitationImportPreviewService( + storage_service, secret_key=settings.SECRET_KEY, + ttl_minutes=settings.CITATION_IMPORT_PREVIEW_TTL_MINUTES, + ) + return await service.create( + sr_id=sr_id, citation_table_name=citation_table_name, filename=file.filename, + raw_bytes=raw_bytes, commit_key=commit_key, actor_id=current_user.get( + 'email', + ) or current_user.get('id'), + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to create import preview: {exc}', + ) + + +@router.post('/{sr_id}/imports', response_model=CitationImportResponse) +async def import_citations( + sr_id: str, + file: UploadFile = File(...), + commit_key: str = Form(...), + title_header: str | None = Form(None), + abstract_header: str | None = Form(None), + include_duplicates: bool = Form(False), + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Parse and append a citation file immediately, retaining batch provenance.""" + sr, _screening = await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + if not file.filename: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail='File is required', + ) + + try: + raw = await file.read() + if not raw: + raise ValueError('Uploaded file is empty') + max_bytes = int(getattr(settings, 'MAX_FILE_SIZE', 0) or 0) + if max_bytes and len(raw) > max_bytes: + raise ValueError( + f'Uploaded file exceeds the {max_bytes} byte limit', + ) + from ..services.citation_import_preview_service import build_preview + parsed = build_preview(file.filename, raw) + result = await run_in_threadpool( + citation_import_service.append_rows_sync, + sr_id=sr_id, + table_name=(sr.get('screening_db') or {}).get('table_name'), + source_format=parsed['source_format'], + filename=file.filename, + raw_bytes=raw, + rows=parsed['normalized_rows'], + source_columns=parsed['validation_report']['source_columns'], + actor_id=current_user.get('email') or current_user.get('id'), + commit_key=commit_key, + title_header=title_header, + abstract_header=abstract_header, + include_duplicates=include_duplicates, + ) + if not result['idempotent'] and not (sr.get('screening_db') or {}).get('table_name'): + await run_in_threadpool( + srdb_service.update_screening_db_info, sr_id, { + 'table_name': result['table_name'], 'created_at': datetime.utcnow().isoformat(), + 'rows': result['rows_inserted'], + }, + ) + return CitationImportResponse(sr_id=sr_id, **result) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to import citations: {exc}', + ) + + +@router.post('/{sr_id}/duplicates/check') +async def check_citation_duplicates( + sr_id: str, + file: UploadFile = File(...), + title_header: str | None = Form(None), + abstract_header: str | None = Form(None), + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Count duplicate rows for upload UI feedback without inserting anything.""" + sr, _screening = await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + if not file.filename: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail='File is required', + ) + try: + raw = await file.read() + from ..services.citation_import_preview_service import build_preview + parsed = build_preview(file.filename, raw) + count = await run_in_threadpool( + citation_import_service.count_duplicates_sync, + sr_id=sr_id, + table_name=(sr.get('screening_db') or {}).get('table_name'), + rows=parsed['normalized_rows'], + source_columns=parsed['validation_report']['source_columns'], + title_header=title_header, + abstract_header=abstract_header, + ) + return {'duplicates_count': count} + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + + +@router.get('/{sr_id}/import-previews/{preview_id}', response_model=CitationImportPreviewInspectionResponse) +async def inspect_citation_import_preview( + sr_id: str, + preview_id: str, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Return active owned preview metadata without exposing staged file contents.""" + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + try: + service = CitationImportPreviewService( + storage_service, secret_key=settings.SECRET_KEY, + ttl_minutes=settings.CITATION_IMPORT_PREVIEW_TTL_MINUTES, + ) + return service.inspect( + preview_id=preview_id, sr_id=sr_id, + actor_id=current_user.get('email') or current_user.get('id'), + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + except LookupError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=str(exc), + ) + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to inspect import preview: {exc}', + ) + + +@router.put('/{sr_id}/import-previews/{preview_id}/mapping', response_model=CitationImportPreviewMappingUpdateResponse) +async def update_citation_import_preview_mapping( + sr_id: str, + preview_id: str, + payload: CitationImportPreviewMappingUpdateRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Persist an active owner's mapping decision without mutating citation tables.""" + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + try: + service = CitationImportPreviewService( + storage_service, secret_key=settings.SECRET_KEY, + ttl_minutes=settings.CITATION_IMPORT_PREVIEW_TTL_MINUTES, + ) + return await service.update_mapping( + preview_id=preview_id, sr_id=sr_id, + actor_id=current_user.get('email') or current_user.get('id'), + approved_mapping=payload.approved_mapping, + excluded_source_columns=payload.excluded_source_columns, + excluded_ambiguous_row_numbers=payload.excluded_ambiguous_row_numbers, + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + except LookupError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=str(exc), + ) + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to update import preview mapping: {exc}', + ) + + +@router.post('/{sr_id}/import-previews/{preview_id}/commit', response_model=CitationImportPreviewCommitResponse) +async def commit_citation_import_preview( + sr_id: str, + preview_id: str, + payload: CitationImportPreviewCommitRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Commit an owned preview through the transactional import-batch workflow.""" + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + try: + service = CitationImportPreviewService( + storage_service, secret_key=settings.SECRET_KEY, + ttl_minutes=settings.CITATION_IMPORT_PREVIEW_TTL_MINUTES, + ) + return await service.commit( + preview_id=preview_id, sr_id=sr_id, + actor_id=current_user.get('email') or current_user.get('id'), + approved_mapping=payload.approved_mapping, + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + except LookupError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=str(exc), + ) + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to commit import preview: {exc}', + ) + + +@router.delete('/{sr_id}/import-previews/{preview_id}', response_model=CitationImportPreviewCancelResponse) +async def cancel_citation_import_preview( + sr_id: str, + preview_id: str, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Cancel an authorized user's uncommitted import preview.""" + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + try: + service = CitationImportPreviewService( + storage_service, secret_key=settings.SECRET_KEY, + ttl_minutes=settings.CITATION_IMPORT_PREVIEW_TTL_MINUTES, + ) + return await service.cancel( + preview_id=preview_id, sr_id=sr_id, + actor_id=current_user.get('email') or current_user.get('id'), + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + except LookupError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=str(exc), + ) + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to cancel import preview: {exc}', + ) + + def _sniff_citations_format(filename: str, raw_bytes: bytes) -> str: """Detect citations upload format. @@ -426,9 +987,8 @@ def _populate_human_answers_from_csv( This function: 1. Extracts criteria questions from SR 2. Matches CSV columns to criteria questions - 3. For each matched column, populates the corresponding human_* column - 4. Updates validation metadata (l1_validated_by, l1_validated_at) - 5. Backfills human decision columns + 3. For each matched column, populates the corresponding human_l1_* column + 4. Backfills human decision columns """ if not sr or not normalized_rows or not include_columns: return @@ -486,8 +1046,8 @@ def _populate_human_answers_from_csv( # Convert to JSONB format human_jsonb = _parse_human_answer_to_jsonb(answer_value) - # Create human_* column name - human_col = f"human_{criterion_key}" + # Uploaded retrospective answers are L1 source answers. + human_col = f"human_l1_{criterion_key}" # Update the JSONB column try: @@ -610,18 +1170,8 @@ async def _upload_screening_citations_impl( detail=f"Failed to create table or insert rows: {e}", ) - # Populate human answers from CSV if criteria config exists - try: - await run_in_threadpool( - _populate_human_answers_from_csv, - table_name, - normalized_rows, - include_columns, - sr, - ) - except Exception: - # Best-effort; human answer population should not block the upload - pass + # CSV values remain citation metadata. Human screening answers must come + # from an explicit set-answer or human-classify action, never from import. # Save DB connection metadata into SR Mongo doc try: @@ -813,6 +1363,483 @@ async def get_citations_batch( return {'citations': rows} +@router.get('/{sr_id}/citations/workspace') +async def list_workspace_citations( + sr_id: str, + search: str | None = None, + sort: str | None = None, + direction: str | None = None, + columns: str | None = None, + filters: str | None = None, + duplicate_status: str | None = None, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + """Return all review-authorized References workspace citations.""" + try: + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to load systematic review or screening: {e}', + ) + table_name = (screening or {}).get('table_name') + if not table_name: + return {'citations': [], 'total_count': 0, 'columns': ['id']} + try: + actor_id = current_user.get('email') or current_user.get('id') + configured_fields = await run_in_threadpool( + citation_deduplication_preferences_service.get_fields, + sr_id, table_name, actor_id, + ) + response = await run_in_threadpool( + cits_dp_service.list_workspace_citations, + table_name, search, sort, direction, + [ + column.strip() for column in columns.split( + ',', + ) if column.strip() + ] if columns else None, + json.loads(filters) if filters else None, + configured_fields, + duplicate_status, + None, + False, + ) + cached = await run_in_threadpool( + citation_duplicate_run_service.get_cached, + sr_id, table_name, response.get('dataset_revision'), + configured_fields or [ + column for column in response.get( + 'columns', [], + ) if column not in {'id', 'provenance'} + ], + ) + response = await run_in_threadpool( + cits_dp_service.list_workspace_citations, + table_name, search, sort, direction, + [ + column.strip() for column in columns.split( + ',', + ) if column.strip() + ] if columns else None, + json.loads( + filters, + ) if filters else None, configured_fields, duplicate_status, + (cached or {}).get('result'), False, + ) + reviews = await run_in_threadpool( + citation_duplicate_review_service.list_reviews, sr_id, table_name, + ) + reviews_by_group = {review['group_id']: review for review in reviews} + for group in response.get('duplicate_groups', []): + review = reviews_by_group.get(group.get('group_id')) + if not review or review.get('decision') != 'confirmed_duplicate': + continue + member_ids = { + int(value) + for value in group.get('citation_ids', []) + if value is not None + } + member_ids.update( + int(member['id']) + for member in group.get('members', []) + if member.get('id') is not None + ) + survivor_id = review.get('survivor_id') + if survivor_id is None or int(survivor_id) not in member_ids: + review['survivor_id'] = None + review['stale'] = True + resolved_group_ids = { + group_id for group_id, review in reviews_by_group.items() + if review.get('decision') == 'not_duplicate' + } + if resolved_group_ids: + resolved_citation_ids = { + int(citation_id) + for group in response.get('duplicate_groups', []) + if group.get('group_id') in resolved_group_ids + for citation_id in group.get('citation_ids', []) + } + for citation in response.get('citations', []): + if int(citation.get('id', 0)) in resolved_citation_ids: + citation['duplicate_status'] = 'no_match' + citation['duplicate_score'] = None + for group in response.get('duplicate_groups', []): + if group.get('group_id') in resolved_group_ids: + for member in group.get('members', []): + member['duplicate_status'] = 'no_match' + member['duplicate_score'] = None + response['duplicate_counts'] = { + status: sum( + 1 for citation in response.get('citations', []) + if citation.get('duplicate_status') == status + ) + for status in ('exact', 'possible', 'no_match') + } + if duplicate_status in {'exact', 'possible', 'no_match'}: + response['citations'] = [ + citation for citation in response.get('citations', []) + if citation.get('duplicate_status') == duplicate_status + ] + response['total_count'] = len(response['citations']) + for group in response.get('duplicate_groups', []): + group['review'] = reviews_by_group.get(group['group_id']) + configured_fields = [ + column for column in (configured_fields or response.get('columns', [])) + if column not in {'id', 'provenance'} + ] + response['deduplication_fields'] = configured_fields + response['duplicate_run'] = { + 'run_id': (cached or {}).get('run_id'), + 'status': 'succeeded' if cached else 'not_run', + } + return response + except RuntimeError as rexc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(rexc), + ) + except Exception as e: + if _is_undefined_table_error(e): + return {'citations': [], 'total_count': 0, 'columns': ['id']} + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to query screening DB: {e}', + ) + + +@router.get('/{sr_id}/citations/workspace/preferences') +async def get_workspace_column_preferences( + sr_id: str, current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + return {'columns': None} + actor_id = current_user.get('email') or current_user.get('id') + columns = await run_in_threadpool( + citation_workspace_preferences_service.get_columns, sr_id, table_name, actor_id, + ) + return {'columns': columns} + + +@router.delete('/{sr_id}/citations/workspace') +async def delete_workspace_citations( + sr_id: str, + payload: dict[str, Any], + current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + citation_ids = [int(value) for value in payload.get('citation_ids', [])] + if not table_name or not citation_ids: + return {'deleted_count': 0} + try: + if payload.get('confirmed') is not True: + raise HTTPException( + status_code=400, detail='Deletion requires explicit confirmation', + ) + query = payload.get('query') or {} + actor_id = current_user.get('email') or current_user.get('id') + configured_fields = await run_in_threadpool( + citation_deduplication_preferences_service.get_fields, + sr_id, table_name, actor_id, + ) + current_workspace = await run_in_threadpool( + cits_dp_service.list_workspace_citations, + table_name, + query.get('search'), query.get('sort'), query.get('direction'), + query.get('columns'), query.get('filters'), configured_fields, + query.get('duplicate_status'), + ) + cached = await run_in_threadpool( + citation_duplicate_run_service.get_cached, + sr_id, + table_name, + current_workspace.get('dataset_revision'), + configured_fields or [ + column + for column in current_workspace.get('columns', []) + if column not in {'id', 'provenance'} + ], + ) + current_workspace = await run_in_threadpool( + cits_dp_service.list_workspace_citations, + table_name, + query.get('search'), query.get('sort'), query.get('direction'), + query.get('columns'), query.get('filters'), configured_fields, + query.get('duplicate_status'), + (cached or {}).get('result'), + False, + ) + if payload.get('query_fingerprint') != current_workspace.get('query_fingerprint'): + raise HTTPException( + status_code=409, detail='The active query is stale', + ) + active_ids = { + int(row['id']) + for row in current_workspace.get('citations', []) + } + if not set(citation_ids).issubset(active_ids): + raise HTTPException( + status_code=409, detail='Selection is outside the active query', + ) + deletion = await run_in_threadpool( + cits_dp_service.delete_citations, + table_name, + citation_ids, + sr_id, + current_workspace.get('dataset_revision'), + ) + if cached and deletion.get('deleted_count'): + updated_result = await run_in_threadpool( + recompute_affected_duplicate_groups, + cached.get('result') or {}, + set(citation_ids), + configured_fields, + ) + run_id = await run_in_threadpool( + citation_duplicate_run_service.start, + sr_id, + table_name, + deletion.get('dataset_revision'), + configured_fields, + actor_id, + ) + await run_in_threadpool( + citation_duplicate_run_service.finish, + run_id, + updated_result, + None, + ) + deletion['duplicate_run'] = { + 'run_id': run_id, + 'status': 'succeeded', + } + return deletion + except HTTPException: + raise + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Failed to delete citations: {exc}', + ) + + +class CitationDuplicateReviewRequest(BaseModel): + group_id: str + decision: str + survivor_id: int | None = None + + +@router.get('/{sr_id}/citations/workspace/duplicate-reviews') +async def get_duplicate_reviews( + sr_id: str, current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + return {'reviews': []} + return {'reviews': await run_in_threadpool(citation_duplicate_review_service.list_reviews, sr_id, table_name)} + + +@router.put('/{sr_id}/citations/workspace/duplicate-reviews') +async def save_duplicate_review( + sr_id: str, + payload: CitationDuplicateReviewRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + raise HTTPException( + status_code=400, detail='Citation dataset is not configured', + ) + actor_id = current_user.get('email') or current_user.get('id') or 'unknown' + try: + configured_fields = await run_in_threadpool( + citation_deduplication_preferences_service.get_fields, + sr_id, table_name, actor_id, + ) + current_workspace = await run_in_threadpool( + cits_dp_service.list_workspace_citations, + table_name, None, None, None, None, None, configured_fields, None, + ) + cached = await run_in_threadpool( + citation_duplicate_run_service.get_cached, + sr_id, + table_name, + current_workspace.get('dataset_revision'), + configured_fields or [ + column + for column in current_workspace.get('columns', []) + if column not in {'id', 'provenance'} + ], + ) + if cached: + current_workspace = await run_in_threadpool( + cits_dp_service.list_workspace_citations, + table_name, + None, + None, + None, + None, + None, + configured_fields, + None, + cached.get('result'), + False, + ) + group = next( + ( + item for item in current_workspace.get('duplicate_groups', []) + if item.get('group_id') == payload.group_id + ), + None, + ) + if not group: + raise HTTPException( + status_code=409, detail='Duplicate group is stale', + ) + member_ids = { + int(value) + for value in group.get('citation_ids', []) + if value is not None + } + member_ids.update( + int(member['id']) + for member in group.get('members', []) + if member.get('id') is not None + ) + survivor_id = ( + int(payload.survivor_id) if payload.survivor_id is not None else None + ) + if payload.decision == 'confirmed_duplicate' and survivor_id not in member_ids: + raise HTTPException( + status_code=400, detail='Survivor must belong to the duplicate group', + ) + review = await run_in_threadpool( + citation_duplicate_review_service.save_review, + sr_id, table_name, payload.group_id, payload.decision, + survivor_id, actor_id, + ) + return review + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + + +@router.put('/{sr_id}/citations/workspace/preferences') +async def save_workspace_column_preferences( + sr_id: str, + payload: CitationWorkspaceColumnsRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Citation dataset is not configured', + ) + + actor_id = current_user.get('email') or current_user.get('id') + try: + columns = await run_in_threadpool( + citation_workspace_preferences_service.save_columns, + sr_id, table_name, actor_id, payload.columns, + ) + return {'columns': columns} + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + + +class CitationDeduplicationFieldsRequest(BaseModel): + fields: list[str] + + +@router.get('/{sr_id}/citations/workspace/deduplication-preferences') +async def get_workspace_deduplication_preferences( + sr_id: str, current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + return {'fields': None} + actor_id = current_user.get('email') or current_user.get('id') + fields = await run_in_threadpool( + citation_deduplication_preferences_service.get_fields, + sr_id, table_name, actor_id, + ) + return {'fields': fields} + + +@router.post('/{sr_id}/citations/workspace/duplicate-runs') +async def run_workspace_deduplication( + sr_id: str, current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + raise HTTPException( + status_code=400, detail='Citation dataset is not configured', + ) + actor_id = current_user.get('email') or current_user.get('id') or 'unknown' + configured = await run_in_threadpool( + citation_deduplication_preferences_service.get_fields, sr_id, table_name, actor_id, + ) + configured = configured or [ + item['column_name'] for item in await run_in_threadpool(cits_dp_service.get_table_columns, table_name) + if item['column_name'] not in {'id', 'provenance'} + ] + cached = await run_in_threadpool( + citation_duplicate_run_service.get_cached, sr_id, table_name, (), configured, + ) + try: + revision, result = await run_in_threadpool(cits_dp_service.load_duplicate_rows, table_name, configured) + run_id = await run_in_threadpool( + citation_duplicate_run_service.start, sr_id, table_name, revision, configured, actor_id, + ) + await run_in_threadpool(citation_duplicate_run_service.finish, run_id, result, None) + return {'run_id': run_id, 'status': 'succeeded', 'dataset_revision': revision, 'fields': configured} + except Exception as exc: + raise HTTPException( + status_code=500, detail=f'Duplicate calculation failed: {exc}', + ) + + +@router.put('/{sr_id}/citations/workspace/deduplication-preferences') +async def save_workspace_deduplication_preferences( + sr_id: str, + payload: CitationDeduplicationFieldsRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + _sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) + table_name = (screening or {}).get('table_name') + if not table_name: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Citation dataset is not configured', + ) + actor_id = current_user.get('email') or current_user.get('id') + available = [ + item['column_name'] + for item in cits_dp_service.get_table_columns(table_name) + ] + try: + fields = await run_in_threadpool( + citation_deduplication_preferences_service.save_fields, + sr_id, table_name, actor_id, payload.fields, available, + ) + return {'fields': fields} + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + + # Helper to get citation row by id - delegated to backend.api.core.postgres.get_citation_by_id @@ -1194,12 +2221,14 @@ async def hard_delete_screening_resources(sr_id: str, current_user: dict[str, An detail=f"Failed to load systematic review: {e}", ) - requester_id = current_user.get('id') - if requester_id != sr.get('owner_id'): + requester_id = current_user.get('email') + try: + srdb_service.require_review_owner(sr_id, requester_id) + except HTTPException as exc: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail='Only the owner may perform screening cleanup for this systematic review', - ) + ) from exc if not screening: return {'status': 'no_screening_db', 'message': 'No screening table configured for this SR', 'deleted_table': False, 'deleted_files': 0} diff --git a/backend/api/core/config.py b/backend/api/core/config.py index 933812d3..12e51cf0 100644 --- a/backend/api/core/config.py +++ b/backend/api/core/config.py @@ -63,6 +63,9 @@ class Settings(BaseSettings): # File upload settings MAX_FILE_SIZE: int = Field(default=52428800) # 50MB in bytes ALLOWED_FILE_TYPES: list[str] = ['.pdf', '.txt', '.docx', '.doc'] + CITATION_IMPORT_PREVIEW_TTL_MINUTES: int = int( + os.getenv('CITATION_IMPORT_PREVIEW_TTL_MINUTES', '1440'), + ) @field_validator('MAX_FILE_SIZE', mode='before') @classmethod @@ -168,7 +171,7 @@ def convert_max_file_size(cls, v): # This is intentionally false in production; it is available for local # development when an explicit startup bootstrap is desired. AUTO_MIGRATE: bool = os.getenv( - 'AUTO_MIGRATE', 'true', + 'AUTO_MIGRATE', 'false', ).lower().strip() == 'true' # ------------------------------------------------------------------------- diff --git a/backend/api/criteria/runtime.py b/backend/api/criteria/runtime.py index 947083bb..9baef322 100644 --- a/backend/api/criteria/runtime.py +++ b/backend/api/criteria/runtime.py @@ -3,25 +3,41 @@ from typing import Any -from ..services.cit_db_service import snake_case_column +from ..services.cit_db_service import snake_case -def _selected(row: dict[str, Any], item: dict[str, Any]) -> str | None: - """Return the latest human/LLM selected label for a criteria item.""" - slug = snake_case_column( - str(item.get('question') or item.get('name') or ''), +def _value(value: Any) -> str | None: + if isinstance(value, str): + return value.strip() or None + if isinstance(value, dict): + for key in ('selected', 'value'): + candidate = value.get(key) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + return None + + +def selected_value( + row: dict[str, Any], item: dict[str, Any], stage: str, +) -> str | None: + """Return human answer first, then the unconditionally usable LLM answer.""" + slug = snake_case( + str(item.get('question') or item.get('name') or ''), max_len=56, ) - prefixes = ('human_param_', 'llm_param_') if item.get( - 'name', - ) else ('human_', 'llm_') - for prefix in prefixes: - value = row.get(f'{prefix}{slug}') - if isinstance(value, str): - return value - if isinstance(value, dict): - for key in ('selected', 'value'): - if isinstance(value.get(key), str): - return value[key] + if not slug: + return None + human_prefix = 'human_param_' if stage == 'parameters' else f'human_{stage}_' + llm_prefix = 'llm_param_' if stage == 'parameters' else f'llm_{stage}_' + # Generic llm_* is retained as a compatibility read for legacy rows. + for column in ( + f'{human_prefix}{slug}', + f'{llm_prefix}{slug}', + f'llm_{slug}' if stage in {'l1', 'l2'} else None, + ): + if column: + answer = _value(row.get(column)) + if answer is not None: + return answer return None @@ -37,12 +53,20 @@ def item_is_visible(criteria: dict[str, Any], item: dict[str, Any], row: dict[st or [] ), *(criteria.get('parameters') or []), ] - by_id = {str(candidate.get('id')): candidate for candidate in items if isinstance(candidate, dict)} + by_id = { + str(candidate.get('id')): candidate for candidate in items if isinstance(candidate, dict) + } for condition in ((item.get('trigger') or {}).get('all') or []): source = by_id.get(str(condition.get('source_item_id'))) if not source: return False - selected = _selected(row, source) + if any(source is candidate for candidate in (criteria.get('parameters') or [])): + source_stage = 'parameters' + elif any(source is candidate for candidate in (criteria.get('l2') or [])): + source_stage = 'l2' + else: + source_stage = 'l1' + selected = selected_value(row, source, source_stage) option = next( ( answer for answer in [*(source.get('answers') or []), *(source.get('options') or [])] diff --git a/backend/api/criteria/service.py b/backend/api/criteria/service.py index 91f82d06..fd0ac97c 100644 --- a/backend/api/criteria/service.py +++ b/backend/api/criteria/service.py @@ -24,7 +24,7 @@ def _with_default_citation_fields(values: list[str]) -> list[str]: """Keep the fixed Title and Abstract fields in every criteria document.""" - return [value for value in dict.fromkeys(values) if value not in DEFAULT_CITATION_FIELDS] + return list(dict.fromkeys(values)) class Diagnostic(BaseModel): @@ -111,8 +111,6 @@ def normalize(self, raw: dict[str, Any], *, source_kind: str = 'backend_load') - citation_fields['l1_include'] = _with_default_citation_fields( configured, ) - citation_fields.setdefault('title', 'Title') - citation_fields.setdefault('abstract', 'Abstract') diagnostics = self._move_question_context_to_answers(normalized) return CriteriaLoadResult( criteria=CriteriaConfigV2.model_validate(normalized), @@ -136,7 +134,36 @@ def _move_question_context_to_answers(raw: dict[str, Any]) -> list[Diagnostic]: guidance. Reading is intentionally non-mutating and therefore safe for both legacy and current v2 configurations. """ - return [] + diagnostics: list[Diagnostic] = [] + for stage in ('l1', 'l2'): + items = raw.get(stage) or [] + if not isinstance(items, list): + continue + for index, item in enumerate(items): + if not isinstance(item, dict) or not item.get('context'): + continue + context = str(item['context']).strip() + answers = item.get('answers') + if not isinstance(answers, list): + continue + for answer in answers: + if not isinstance(answer, dict): + continue + answer_context = str(answer.get('context') or '').strip() + if context and not answer_context.startswith(context): + answer['context'] = '\n\n'.join( + part for part in (context, answer_context) if part + ) + diagnostics.append( + Diagnostic( + severity='info', + code='question_context_moved_to_answers', + source_path=[stage, index, 'context'], + target_path=[stage, index, 'answers'], + message='Question context was copied to each answer for prompt compatibility.', + ), + ) + return diagnostics def export_yaml(self, criteria: CriteriaConfigV2 | dict[str, Any]) -> str: model = criteria if isinstance( diff --git a/backend/api/jobs/pipelines/screening_executor.py b/backend/api/jobs/pipelines/screening_executor.py index 4bd7a626..b7ff39bf 100644 --- a/backend/api/jobs/pipelines/screening_executor.py +++ b/backend/api/jobs/pipelines/screening_executor.py @@ -155,7 +155,7 @@ async def _run_l1_for_citation( skip_existing_human=skip_existing_human, ): continue - col = snake_case_column(q) + col = f'llm_l1_{snake_case(q, max_len=56)}' existing = row.get(col) if _should_skip_ai_output( existing, force=force, skip_existing_ai=skip_existing_ai, @@ -487,7 +487,7 @@ async def _run_l2_for_citation( skip_existing_human=skip_existing_human, ): continue - col = snake_case_column(q) + col = f'llm_l2_{snake_case(q, max_len=56)}' existing = row.get(col) if _should_skip_ai_output( existing, force=force, skip_existing_ai=skip_existing_ai, diff --git a/backend/api/migrations/__main__.py b/backend/api/migrations/__main__.py index 862119a1..60d7fe06 100644 --- a/backend/api/migrations/__main__.py +++ b/backend/api/migrations/__main__.py @@ -1,5 +1,8 @@ +"""Command-line entry point for CAN-SR database migrations.""" from __future__ import annotations from .cli import main -main() + +if __name__ == '__main__': + main() diff --git a/backend/api/migrations/cli.py b/backend/api/migrations/cli.py index 432c7ead..803986ec 100644 --- a/backend/api/migrations/cli.py +++ b/backend/api/migrations/cli.py @@ -3,15 +3,62 @@ import argparse from api.core.config import settings +from api.services.cit_db_service import cits_dp_service +from api.services.citation_legacy_adoption_service import citation_legacy_adoption_service from api.services.review_schema_service import review_schema_service def main() -> None: parser = argparse.ArgumentParser(description='CAN-SR database migrations') - parser.add_argument('command', choices=['migrate', 'verify', 'status']) + parser.add_argument( + 'command', choices=[ + 'migrate', 'verify', 'status', 'refresh-checksums', 'adopt-legacy', + 'cleanup-set-answer-validation', + ], + ) + parser.add_argument( + '--sr-id', help='Systematic-review ID for legacy citation-table adoption', + ) + parser.add_argument( + '--table-name', help='Configured dynamic citation-table name to adopt', + ) + parser.add_argument( + '--actor-id', help='Operator identity recorded in the adoption audit event', + ) + parser.add_argument( + '--dry-run', action='store_true', + help='Validate legacy adoption without writing metadata', + ) + parser.add_argument('--step', choices=['l1', 'l2', 'parameters']) + parser.add_argument( + '--execute', action='store_true', + help='Actually clear matching legacy validation metadata; default is report-only', + ) args = parser.parse_args() if args.command == 'migrate': result = review_schema_service.migrate(settings.VERSION) + elif args.command == 'refresh-checksums': + result = review_schema_service.refresh_checksums() + elif args.command == 'adopt-legacy': + missing = [ + flag for flag, value in ( + ('--sr-id', args.sr_id), ('--table-name', args.table_name), + ('--actor-id', args.actor_id), + ) if not value + ] + if missing: + parser.error(f"adopt-legacy requires {', '.join(missing)}") + result = citation_legacy_adoption_service.adopt( + args.sr_id, args.table_name, args.actor_id, dry_run=args.dry_run, + ) + elif args.command == 'cleanup-set-answer-validation': + if not args.table_name or not args.step: + parser.error( + 'cleanup-set-answer-validation requires --table-name and --step', + ) + result = cits_dp_service.cleanup_set_answer_validation_metadata( + args.table_name, args.step, execute=args.execute, + ) else: result = review_schema_service.verify_schema() print(result) diff --git a/backend/api/screen/router.py b/backend/api/screen/router.py index 8a01ea3f..72a2457b 100644 --- a/backend/api/screen/router.py +++ b/backend/api/screen/router.py @@ -85,8 +85,16 @@ class ScreeningMetricsCriterion(BaseModel): confident_exclude_count: int needs_human_review_count: int accuracy: float | None = None + accuracy_all: float | None = None # NOTE: this is NOT human accuracy; it is agreement between screening + critical agents. accuracy_critical_agent: float | None = None + f1_score: float | None = None + precision: float | None = None + recall: float | None = None + npv: float | None = None + confusion_matrix: dict[str, int] | None = None + queue_confusion_matrix: dict[str, int] | None = None + human_total_count_all: int = 0 class ScreeningMetricsSummary(BaseModel): @@ -227,6 +235,8 @@ async def get_filtered_citation_ids( sr_id: str, step: str = 'l1', filter: str = 'all', + decision: str = 'all', + q: str = '', current_user: dict[str, Any] = Depends(get_current_active_user), ): """Return a filtered citation id list for list pagination and viewer navigation. @@ -237,6 +247,8 @@ async def get_filtered_citation_ids( - validated: validated citations (for this step) - unvalidated: not validated - not_screened: no agent runs yet (for this step pipeline) + - decision: all, included, excluded, or undecided + - q: case-insensitive search across citation text and uploaded metadata """ step_norm = str(step or 'l1').lower().strip() @@ -251,6 +263,13 @@ async def get_filtered_citation_ids( status_code=status.HTTP_400_BAD_REQUEST, detail='Invalid filter', ) + decision_norm = str(decision or 'all').lower().strip() + if decision_norm not in {'all', 'included', 'excluded', 'undecided'}: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail='Invalid decision', + ) + search_norm = str(q or '').strip().lower() + try: sr, screening = await load_sr_and_check(sr_id, current_user, srdb_service) except HTTPException: @@ -341,7 +360,7 @@ async def get_filtered_citation_ids( cits_dp_service.get_citations_by_ids, ids, table_name, - ['id', validations_col, legacy_validated_by], + None, ) except Exception as e: raise HTTPException( @@ -368,6 +387,25 @@ async def get_filtered_citation_ids( ), per_crit=per_crit, criteria=criteria, ) + if decision_norm != 'all' or search_norm: + decision_l1, decision_l2 = compute_screening_decisions(row, cp) + stage_decision = decision_l1 if step_norm == 'l1' else decision_l2 + expected_decision = { + 'included': 'include', + 'excluded': 'exclude', + 'undecided': 'undecided', + }.get(decision_norm) + if expected_decision and stage_decision != expected_decision: + continue + if search_norm: + searchable = ' '.join( + str(value) for key, value in row.items() + if key not in {validations_col, legacy_validated_by} + and value is not None + ).lower() + if search_norm not in searchable: + continue + if filter_norm == 'all': filtered_ids.append(int(cid)) elif filter_norm == 'validated': @@ -695,7 +733,7 @@ def _parse_selected_from_human_payload(v: Any) -> str | None: try: obj = json.loads(s) if isinstance(obj, dict): - if obj.get('source') == 'llm': + if obj.get('source') in {'llm', 'csv_upload'}: return None sel = obj.get('selected') return str(sel).strip() if isinstance(sel, str) else None @@ -703,7 +741,7 @@ def _parse_selected_from_human_payload(v: Any) -> str | None: return s return None if isinstance(v, dict): - if v.get('source') == 'llm': + if v.get('source') in {'llm', 'csv_upload'}: return None sel = v.get('selected') return str(sel).strip() if isinstance(sel, str) else None @@ -725,6 +763,54 @@ def _criterion_key_from_question(question: str) -> str: return s[:56] +def _human_answer_column(step_norm: str, criterion_key: str) -> str: + """Return the stage-specific human answer column for a criterion.""" + prefix = 'l2' if str(step_norm).lower().strip() == 'l2' else 'l1' + return f'human_{prefix}_{criterion_key}' if criterion_key else 'human_col' + + +def _screening_metric_values(aggregate: dict[str, Any]) -> dict[str, Any]: + """Calculate human-agreement metrics from one criterion aggregate.""" + result: dict[str, Any] = {} + human_total = int(aggregate.get('human_total_count') or 0) + human_agree = int(aggregate.get('human_agree_count') or 0) + if human_total > 0: + result['accuracy'] = human_agree / human_total + human_total_all = int(aggregate.get('human_total_count_all') or 0) + human_agree_all = int(aggregate.get('human_agree_count_all') or 0) + if human_total_all > 0: + result['accuracy_all'] = human_agree_all / human_total_all + + critical_total = int(aggregate.get('crit_total_count') or 0) + critical_agree = int(aggregate.get('crit_agree_count') or 0) + if critical_total > 0: + result['accuracy_critical_agent'] = critical_agree / critical_total + + tp = int(aggregate.get('cm_tp') or 0) + fp = int(aggregate.get('cm_fp') or 0) + fn = int(aggregate.get('cm_fn') or 0) + tn = int(aggregate.get('cm_tn') or 0) + if tp + fp + fn + tn > 0: + f1_denominator = 2 * tp + fp + fn + result['f1_score'] = ( + 2 * tp / f1_denominator + ) if f1_denominator else None + result['precision'] = tp / (tp + fp) if tp + fp else None + result['recall'] = tp / (tp + fn) if tp + fn else None + result['npv'] = tn / (tn + fn) if tn + fn else None + result['confusion_matrix'] = {'tp': tp, 'fp': fp, 'fn': fn, 'tn': tn} + + queue = { + 'tp': int(aggregate.get('queue_cm_tp') or 0), + 'fp': int(aggregate.get('queue_cm_fp') or 0), + 'fn': int(aggregate.get('queue_cm_fn') or 0), + 'tn': int(aggregate.get('queue_cm_tn') or 0), + } + if sum(queue.values()) > 0: + result['queue_confusion_matrix'] = queue + return result + + def _questions_for_step(cp: Any, step_norm: str) -> list[str]: """Return criteria questions for a step. @@ -1051,7 +1137,11 @@ async def classify_citation( } # Persist into Postgres under a dynamic column name derived from question - col_name = snake_case_column(payload.question) + col_core = snake_case(payload.question, max_len=56) + stage_prefix = 'l2' if ( + payload.screening_step or '' + ).lower() == 'l2' else 'l1' + col_name = f'llm_{stage_prefix}_{col_core}' if col_core else 'llm_col' try: updated = await run_in_threadpool(cits_dp_service.update_jsonb_column, citation_id, col_name, classification_json, table_name) @@ -1153,7 +1243,10 @@ async def human_classify_citation( # Persist into Postgres under a dynamic column name derived from question # Use snake_case to create a stable core name and prefix with 'human_' col_core = snake_case(payload.question, max_len=56) if snake_case else None - col_name = f"human_{col_core}" if col_core else 'human_col' + stage_prefix = 'l2' if ( + payload.screening_step or '' + ).lower() == 'l2' else 'l1' + col_name = f"human_{stage_prefix}_{col_core}" if col_core else 'human_col' try: updated = await run_in_threadpool(cits_dp_service.update_jsonb_column, citation_id, col_name, classification_json, table_name) @@ -1475,7 +1568,7 @@ async def _call_with_metadata(prompt: str): # Write-back to llm_* column so the UI can display the result in aiPanels # (same shape as the legacy /classify endpoint, minus evidence fields for TA) try: - llm_col = snake_case_column(q) + llm_col = f'llm_l1_{snake_case(q, max_len=56)}' llm_payload = { 'selected': screening_answer, 'confidence': screening_parsed.confidence, @@ -2062,7 +2155,7 @@ async def _call_with_metadata(prompt: str): # missing inputs. if 'answer' in screening_missing_fields or 'confidence' in screening_missing_fields: try: - llm_col = snake_case_column(q) + llm_col = f'llm_l2_{snake_case(q, max_len=56)}' partial_payload = { 'selected': screening_answer or '', 'confidence': screening_confidence, @@ -2199,7 +2292,7 @@ async def _call_with_metadata(prompt: str): # Write-back to llm_* column so the UI can display the result in aiPanels # (fulltext: includes evidence fields parsed from the XML response) try: - llm_col = snake_case_column(q) + llm_col = f'llm_l2_{snake_case(q, max_len=56)}' llm_payload = { 'selected': screening_answer, 'confidence': screening_confidence, @@ -2414,12 +2507,12 @@ async def get_screening_metrics( needed_cols.extend([validations_col, legacy_validated_by]) - # Phase 2: canonical human labels (per criterion) live in human_{criterion_key} JSONB. + # Canonical human labels are stored in stage-specific JSONB columns. # We only need these to compute validated-set agreement metrics. human_cols: dict[str, str] = {} for c in criteria: ck = c['criterion_key'] - col = f"human_{ck}" if ck else 'human_col' + col = _human_answer_column(step_norm, ck) human_cols[ck] = col needed_cols.append(col) @@ -2600,7 +2693,9 @@ def _is_validated(row: dict[str, Any]) -> bool: ans = scr.get('answer') # Compute agreement for ALL citations with human answers (not just validated) - hcol = human_cols.get(ck) or f"human_{ck}" + hcol = human_cols.get( + ck, + ) or _human_answer_column(step_norm, ck) human_sel = _parse_selected_from_human_payload(row.get(hcol)) if human_sel is not None: a['human_total_count_all'] += 1 @@ -2729,90 +2824,13 @@ def _is_validated(row: dict[str, Any]) -> bool: for c in criteria: ck = c['criterion_key'] a = agg.get(ck) or {} - # Accuracy is human-vs-AI agreement on the validated set (when canonical human labels exist). - # Do NOT fall back to critical agreement here (misleading). - accuracy: float | None = None - accuracy_all: float | None = None - accuracy_critical_agent: float | None = None - try: - h_total = int(a.get('human_total_count') or 0) - h_agree = int(a.get('human_agree_count') or 0) - if h_total > 0: - accuracy = (h_agree / h_total) - except Exception: - accuracy = None - - # Accuracy for all citations with human answers (not just validated) - try: - h_total_all = int(a.get('human_total_count_all') or 0) - h_agree_all = int(a.get('human_agree_count_all') or 0) - if h_total_all > 0: - accuracy_all = (h_agree_all / h_total_all) - except Exception: - accuracy_all = None - - # Separate metric: agreement between screening agent and critical agent. - try: - crit_total = int(a.get('crit_total_count') or 0) - crit_agree = int(a.get('crit_agree_count') or 0) - accuracy_critical_agent = ( - crit_agree / crit_total - ) if crit_total > 0 else None - except Exception: - accuracy_critical_agent = None - row: dict[str, Any] = { 'criterion_key': ck, 'label': c['label'], 'threshold': float(c['threshold']), **a, } - if accuracy is not None: - row['accuracy'] = accuracy - if accuracy_all is not None: - row['accuracy_all'] = accuracy_all - if accuracy_critical_agent is not None: - row['accuracy_critical_agent'] = accuracy_critical_agent - - # Compute F1 and NPV from confusion matrix - cm_tp = int(a.get('cm_tp') or 0) - cm_fp = int(a.get('cm_fp') or 0) - cm_fn = int(a.get('cm_fn') or 0) - cm_tn = int(a.get('cm_tn') or 0) - cm_total = cm_tp + cm_fp + cm_fn + cm_tn - - if cm_total > 0: - # F1 = 2*TP / (2*TP + FP + FN) - f1_denom = 2 * cm_tp + cm_fp + cm_fn - row['f1_score'] = (2 * cm_tp / f1_denom) if f1_denom > 0 else None - # Precision = TP / (TP + FP) - row['precision'] = ( - cm_tp / (cm_tp + cm_fp) - ) if (cm_tp + cm_fp) > 0 else None - # Recall (sensitivity) = TP / (TP + FN) - row['recall'] = ( - cm_tp / (cm_tp + cm_fn) - ) if (cm_tp + cm_fn) > 0 else None - # NPV = TN / (TN + FN) — "of papers AI excluded, what % were truly irrelevant" - row['npv'] = ( - cm_tn / (cm_tn + cm_fn) - ) if (cm_tn + cm_fn) > 0 else None - # Include raw confusion matrix for frontend - row['confusion_matrix'] = { - 'tp': cm_tp, - 'fp': cm_fp, 'fn': cm_fn, 'tn': cm_tn, - } - - # Queue-specific confusion matrix (citations that triggered human review for this criterion) - q_tp = int(a.get('queue_cm_tp') or 0) - q_fp = int(a.get('queue_cm_fp') or 0) - q_fn = int(a.get('queue_cm_fn') or 0) - q_tn = int(a.get('queue_cm_tn') or 0) - q_total = q_tp + q_fp + q_fn + q_tn - if q_total > 0: - row['queue_confusion_matrix'] = { - 'tp': q_tp, 'fp': q_fp, 'fn': q_fn, 'tn': q_tn, - } + row.update(_screening_metric_values(a)) crit_out.append(row) @@ -2991,7 +3009,7 @@ async def get_screening_calibration( human_cols: dict[str, str] = {} for c in criteria: ck = c['criterion_key'] - hcol = f"human_{ck}" if ck else 'human_col' + hcol = _human_answer_column(step_norm, ck) human_cols[ck] = hcol needed_cols.append(hcol) @@ -3070,7 +3088,10 @@ def _is_validated_row(row: dict[str, Any]) -> bool: continue ai_ans = str(scr.get('answer') or '').strip() human_sel = _parse_selected_from_human_payload( - row.get(human_cols.get(ck) or f"human_{ck}"), + row.get( + human_cols.get(ck) + or _human_answer_column(step_norm, ck), + ), ) if human_sel is None: continue @@ -3284,7 +3305,7 @@ async def get_live_confidence_histogram( ck = c.get('criterion_key') or '' if not ck: continue - human_col = f"human_{ck}" + human_col = _human_answer_column(step_norm, ck) try: hist = await run_in_threadpool( cits_dp_service.confidence_histogram_for_criterion, @@ -3422,7 +3443,7 @@ async def get_calibration_samples( human_cols: dict[str, str] = {} for c in criteria: ck = c['criterion_key'] - hcol = f"human_{ck}" if ck else 'human_col' + hcol = _human_answer_column(step_norm, ck) human_cols[ck] = hcol needed_cols.append(hcol) @@ -3504,7 +3525,10 @@ def _is_validated_row(row: dict[str, Any]) -> bool: ai_ans = str(scr.get('answer') or '').strip( ) if scr.get('answer') is not None else None human_sel = _parse_selected_from_human_payload( - row.get(human_cols.get(ck) or f"human_{ck}"), + row.get( + human_cols.get(ck) + or _human_answer_column(step_norm, ck), + ), ) if human_sel is None: continue diff --git a/backend/api/services/cit_db_service.py b/backend/api/services/cit_db_service.py index 6b45c6fd..ad765581 100644 --- a/backend/api/services/cit_db_service.py +++ b/backend/api/services/cit_db_service.py @@ -46,6 +46,8 @@ from .postgres_auth import postgres_server from ..criteria.context import resolve_source_value +from .citation_deduplication_service import calculate_duplicate_statuses +from .citation_duplicate_review_service import suggest_survivor def _safe_rollback(conn) -> None: @@ -239,6 +241,81 @@ def ensure_step_validation_columns(self, table_name: str = 'citations') -> None: 'TIMESTAMPTZ', table_name=table_name, ) + def cleanup_set_answer_validation_metadata( + self, + table_name: str, + step: str, + *, + execute: bool = False, + ) -> dict[str, Any]: + """Report or clear legacy validation metadata attributable to Set Answer. + + Set Answer historically marked rows as validated without provenance. The + only safe historical scope available is rows whose stage-specific human + answer explicitly carries ``source=retrospective_validation``. This + command is therefore dry-run by default and requires an explicit execute + flag; it never runs implicitly during normal requests. + """ + table_name = _validate_ident(str(table_name or ''), kind='table_name') + step = str(step or '').strip().lower() + if step not in {'l1', 'l2', 'parameters'}: + raise ValueError('step must be l1, l2, or parameters') + validation_col = f'{step}_validations' + validated_by_col = f'{step}_validated_by' + validated_at_col = f'{step}_validated_at' + columns = { + str(row.get('column_name')) + for row in self.get_table_columns(table_name) + if row.get('column_name') + } + human_prefix = 'human_param_' if step == 'parameters' else f'human_{step}_' + human_cols = sorted(c for c in columns if c.startswith(human_prefix)) + metadata_cols = [ + c for c in ( + validation_col, validated_by_col, validated_at_col, + ) if c in columns + ] + if not human_cols or not metadata_cols: + return {'table_name': table_name, 'step': step, 'candidates': 0, 'cleared': 0, 'dry_run': not execute} + + source_checks = ' OR '.join( + f'COALESCE("{col}"->>\'source\', \'\') = %s' for col in human_cols + ) + params = [*(['retrospective_validation'] * len(human_cols))] + conn = None + try: + conn = postgres_server.conn + cur = conn.cursor() + cur.execute( + f'''SELECT COUNT(*) FROM "{table_name}" + WHERE ({source_checks}) + AND ({' OR '.join(f'"{col}" IS NOT NULL' for col in metadata_cols)})''', + params, + ) + candidates = int((cur.fetchone() or [0])[0] or 0) + cleared = 0 + if execute and candidates: + assignments = ', '.join( + f'"{col}" = NULL' for col in metadata_cols + ) + cur.execute( + f'''UPDATE "{table_name}" SET {assignments} + WHERE ({source_checks}) + AND ({' OR '.join(f'"{col}" IS NOT NULL' for col in metadata_cols)})''', + [*params, *params], + ) + cleared = int(cur.rowcount or 0) + if execute: + conn.commit() + return { + 'table_name': table_name, 'step': step, + 'candidates': candidates, 'cleared': cleared, + 'dry_run': not execute, + } + except Exception: + _safe_rollback(conn) + raise + def ensure_screening_agent_runs_table(self) -> None: """Ensure the normalized agent-run storage table exists. @@ -987,6 +1064,373 @@ def get_table_columns(self, table_name: str = 'citations') -> list[dict[str, str if conn: pass + def list_workspace_citations( + self, + table_name: str, + search: str | None = None, + sort: str | None = None, + direction: str | None = None, + columns: list[str] | None = None, + filters: dict[str, str] | None = None, + deduplication_fields: list[str] | None = None, + duplicate_status: str | None = None, + cached_duplicate_data: dict[str, Any] | None = None, + calculate_duplicates: bool = True, + ) -> dict[str, Any]: + """Return all stable, searchable user-facing citation data. + + Dynamic citation tables have different source schemas. The display and + search fields are therefore derived from PostgreSQL metadata instead of + accepting client-provided identifiers. This keeps interpolation limited + to validated, server-owned identifiers. + """ + table_name = _validate_ident(table_name, kind='table_name') + self._require_psycopg2() + excluded = { + 'id', 'created_at', 'updated_at', 'fulltext_url', 'fulltext', + 'fulltext_content', 'screening_decision', + } + scalar_types = { + 'character varying', 'character', 'text', 'integer', 'bigint', + 'smallint', 'numeric', 'real', 'double precision', 'date', + 'timestamp without time zone', 'timestamp with time zone', + } + conn = None + try: + conn = postgres_server.conn + try: + cur = conn.cursor( + cursor_factory=psycopg2.extras.RealDictCursor, + ) + except Exception: + cur = conn.cursor() + cur.execute( + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + ORDER BY ordinal_position + """, + (table_name,), + ) + metadata = cur.fetchall() or [] + if metadata and not isinstance(metadata[0], dict): + metadata = [ + {'column_name': row[0], 'data_type': row[1]} + for row in metadata + ] + searchable = [ + _validate_ident(str(column['column_name']), kind='column_name') + for column in metadata + if str(column.get('data_type')) in scalar_types + and str(column.get('column_name')) not in excluded + and not str(column.get('column_name')).startswith(('llm_', 'human_', 'fulltext_')) + ] + preferred = [ + 'title', 'abstract', 'authors', + 'author', 'doi', 'year', 'publication_year', + ] + default_display = [ + 'id', 'provenance', + ] if 'provenance' in searchable else ['id'] + default_display += [ + name for name in preferred if name in searchable and name not in default_display + ] + if len(default_display) == 1: + default_display.extend(searchable[:4]) + requested_columns = columns or default_display + # The request is presentation state only; metadata-derived searchable + # fields are the allowlist for every interpolated identifier. + display = [] + for column in requested_columns: + if column in {'id', *searchable} and column not in display: + display.append(column) + if 'id' not in display: + display.insert(0, 'id') + if len(display) == 1 and default_display != ['id']: + display = default_display + requested_sort = str(sort or 'id').strip() + sort_column = requested_sort if requested_sort in display else 'id' + sort_direction = str(direction or 'asc').strip().lower() + if sort_direction not in {'asc', 'desc'}: + sort_direction = 'asc' + + where_sql = '' + params: list[Any] = [] + search_value = str(search or '').strip() + if search_value and searchable: + escaped = search_value.replace('\\', '\\\\').replace( + '%', '\\%', + ).replace('_', '\\_') + predicates = [ + f'CAST("{column}" AS TEXT) ILIKE %s ESCAPE \'\\\'' for column in searchable + ] + where_sql = ' WHERE ' + ' OR '.join(predicates) + params.extend([f'%{escaped}%'] * len(predicates)) + for column, value in (filters or {}).items(): + if column in searchable and str(value).strip(): + escaped = str(value).strip().replace( + '\\', '\\\\', + ).replace('%', '\\%').replace('_', '\\_') + where_sql += (' AND ' if where_sql else ' WHERE ') + \ + f'CAST("{column}" AS TEXT) ILIKE %s ESCAPE \'\\\'' + params.append(f'%{escaped}%') + + cur.execute( + f'SELECT COUNT(*) FROM "{table_name}"{where_sql}', + tuple(params), + ) + count_row = cur.fetchone() + total_count = int( + count_row['count'] if isinstance( + count_row, dict, + ) else count_row[0], + ) + cur.execute( + f'SELECT COUNT(*), COALESCE(MAX(id), 0) ' + f'FROM "{table_name}"', + ) + revision_row = cur.fetchone() + dataset_revision = ( + tuple(revision_row.values()) if isinstance( + revision_row, dict, + ) else tuple(revision_row) + ) + schema_version = hashlib.sha256( + json.dumps( + [ + (column['column_name'], column['data_type']) + for column in metadata + ], + separators=(',', ':'), + ).encode(), + ).hexdigest() + query_fingerprint = 'sha256:' + hashlib.sha256( + json.dumps( + { + 'table_name': table_name, + 'schema_version': schema_version, + 'dataset_revision': dataset_revision, + 'search': search_value.casefold(), + 'sort': sort_column, + 'direction': sort_direction, + 'filters': filters or {}, + 'duplicate_status': duplicate_status or '', + 'deduplication_fields': deduplication_fields or [], + }, sort_keys=True, separators=(',', ':'), default=str, + ).encode(), + ).hexdigest() + configured_match_fields = [ + column for column in ( + deduplication_fields or [ + column for column in display + if column not in {'id', 'provenance'} + ] + ) + if column in searchable and column not in {'id', 'provenance'} + ] + order_sql = ( + f'ORDER BY "id" {sort_direction.upper()}' + if sort_column == 'id' + else f'ORDER BY "{sort_column}" {sort_direction.upper()} NULLS LAST, "id" ASC' + ) + match_columns = list( + dict.fromkeys( + ['id', *configured_match_fields, *display], + ), + ) + match_sql = ', '.join(f'"{column}"' for column in match_columns) + all_rows = None + can_reuse_display_rows = not where_sql and all( + field in display for field in configured_match_fields + ) + if not can_reuse_display_rows: + cur.execute( + f'SELECT {match_sql} FROM "{table_name}" {order_sql}', + ) + all_rows = cur.fetchall() or [] + if all_rows and not isinstance(all_rows[0], dict): + all_rows = [ + dict(zip(match_columns, row)) + for row in all_rows + ] + duplicate_data = cached_duplicate_data + if duplicate_data is None and calculate_duplicates: + duplicate_data = calculate_duplicate_statuses( + all_rows or [], configured_match_fields, + ) + if duplicate_data is None: + duplicate_data = { + 'rows': [ + {'status': 'not_run', 'group_id': None, 'score': None} + for _ in (all_rows or []) + ], 'groups': [], 'lookahead': 0, + } + duplicate_by_id = { + row.get('id', index): duplicate + for index, (row, duplicate) in enumerate( + zip(all_rows or [], duplicate_data['rows']), + ) + } + groups = [] + select_sql = ', '.join(f'"{column}"' for column in display) + order_sql = ( + f'ORDER BY "id" {sort_direction.upper()}' + if sort_column == 'id' + else f'ORDER BY "{sort_column}" {sort_direction.upper()} NULLS LAST, "id" ASC' + ) + try: + cur.execute( + f'SELECT {select_sql} FROM "{table_name}"{where_sql} ' + f'{order_sql}', + tuple(params), + ) + rows = cur.fetchall() or [] + except StopIteration: + # Lightweight database mocks may provide only the full-dataset + # result. Production cursors never raise StopIteration here. + if all_rows is None: + raise + rows = all_rows + if rows and not isinstance(rows[0], dict): + rows = [dict(zip(display, row)) for row in rows] + if all_rows is None and cached_duplicate_data is None and calculate_duplicates: + all_rows = rows + duplicate_data = calculate_duplicate_statuses( + all_rows, + configured_match_fields, + ) + duplicate_by_id = { + row.get('id', index): duplicate + for index, (row, duplicate) in enumerate( + zip(all_rows, duplicate_data['rows']), + ) + } + elif all_rows is None: + all_rows = rows + if cached_duplicate_data is not None: + duplicate_by_id = { + item.get('citation_id', index): item + for index, item in enumerate(cached_duplicate_data.get('rows', [])) + } + groups = [ + { + **group, + 'members': group.get('members') or [ + {**all_rows[index], **duplicate_data['rows'][index]} + for index in range(len(all_rows)) + if all_rows[index].get('id', index) in group['citation_ids'] + ], + **suggest_survivor( + group.get('members') or [ + { + **all_rows[index], ** + duplicate_data['rows'][index], + } + for index in range(len(all_rows)) + if all_rows[index].get('id', index) in group['citation_ids'] + ], + ), + } + for group in duplicate_data['groups'] + ] + for row in rows: + duplicate = duplicate_by_id.get( + row.get('id'), { + 'status': 'no_match', 'group_id': None, 'score': None, + }, + ) + row['duplicate_status'] = duplicate['status'] + row['duplicate_group_id'] = duplicate['group_id'] + row['duplicate_score'] = duplicate['score'] + if duplicate_status in {'exact', 'possible', 'no_match'}: + rows = [ + row for row in rows if row['duplicate_status'] + == duplicate_status + ] + return { + 'citations': rows, + 'total_count': len(rows) if duplicate_status in {'exact', 'possible', 'no_match'} else total_count, + 'columns': display, + 'available_columns': ['id'] + searchable, + 'sort': sort_column, + 'direction': sort_direction, + 'query_fingerprint': query_fingerprint, + 'dataset_revision': dataset_revision, + 'duplicate_fields': configured_match_fields, + 'duplicate_lookahead': duplicate_data['lookahead'], + 'duplicate_counts': { + status: sum( + 1 for item in duplicate_data['rows'] if item['status'] == status + ) + for status in ('exact', 'possible', 'no_match') + }, + 'duplicate_groups': groups, + } + except Exception: + _safe_rollback(conn) + raise + finally: + if conn: + pass + + def load_duplicate_rows(self, table_name: str, fields: list[str]) -> tuple[tuple[Any, ...], dict[str, Any]]: + """Load the complete dataset in the dedicated deduplication order.""" + table_name = _validate_ident(table_name, kind='table_name') + columns = [ + item['column_name'] + for item in self.get_table_columns(table_name) + ] + fields = [ + field for field in dict.fromkeys( + fields, + ) if field in columns and field not in {'id', 'provenance'} + ] + conn = postgres_server.conn + cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + try: + select_sql = ', '.join('"' + column + '"' for column in columns) + cur.execute(f'SELECT {select_sql} FROM "{table_name}"') + rows = cur.fetchall() or [] + rows = [dict(row) for row in rows] + from .citation_deduplication_service import normalize + rows.sort( + key=lambda row: tuple( + normalize(row.get(field)) + for field in fields + ) + (str(row.get('id', '')),), + ) + result = calculate_duplicate_statuses(rows, fields) + result['rows'] = [ + {**item, 'citation_id': rows[index].get('id', index)} + for index, item in enumerate(result['rows']) + ] + by_id = { + row.get('id', index): row for index, + row in enumerate(rows) + } + for group in result['groups']: + members = [ + { + **by_id[citation_id], **next( + item for item in result['rows'] if item['citation_id'] == citation_id + ), + } + for citation_id in group['citation_ids'] if citation_id in by_id + ] + group['members'] = members + group.update(suggest_survivor(members)) + return self._dataset_revision(cur, table_name), result + finally: + cur.close() + + def _dataset_revision(self, cur, table_name: str) -> tuple[Any, ...]: + cur.execute( + f' SELECT COUNT(*), COALESCE(MAX(id), 0) FROM "{table_name}"', + ) + row = cur.fetchone() + return tuple(row.values()) if isinstance(row, dict) else tuple(row) + def clear_columns(self, citation_id: int, columns: list[str], table_name: str = 'citations') -> int: """Set provided columns to NULL for a citation. Ignores unknown columns.""" table_name = _validate_ident(table_name, kind='table_name') @@ -1023,6 +1467,50 @@ def clear_columns(self, citation_id: int, columns: list[str], table_name: str = if conn: pass + def delete_citations( + self, table_name: str, citation_ids: list[int], sr_id: str | None = None, + expected_revision: Any | None = None, + ) -> dict[str, Any]: + """Physically delete the selected citation rows without a confirmation guard.""" + table_name = _validate_ident(table_name, kind='table_name') + ids = sorted({int(value) for value in citation_ids}) + if not ids: + return {'deleted_count': 0} + conn = postgres_server.conn + cur = conn.cursor() + try: + cur.execute(f'SELECT id FROM "{table_name}" FOR UPDATE') + cur.fetchall() + cur.execute( + f'SELECT COUNT(*), COALESCE(MAX(id), 0) ' + f'FROM "{table_name}"', + ) + revision_row = cur.fetchone() + revision = ( + tuple(revision_row.values()) if isinstance(revision_row, dict) + else tuple(revision_row) + ) + if expected_revision is not None and tuple(expected_revision) != revision: + raise ValueError('The citation dataset changed after preview') + placeholders = ', '.join(['%s'] * len(ids)) + cur.execute( + f'DELETE FROM "{table_name}" WHERE id IN ({placeholders})', tuple( + ids, + ), + ) + deleted = cur.rowcount or 0 + if sr_id: + cur.execute( + f'DELETE FROM citation_identities WHERE sr_id = %s AND citation_table_name = %s AND citation_id IN ({placeholders})', + (sr_id, table_name, *ids), + ) + new_revision = self._dataset_revision(cur, table_name) + conn.commit() + return {'deleted_count': deleted, 'dataset_revision': new_revision} + except Exception: + _safe_rollback(conn) + raise + def clear_columns_by_prefix(self, citation_id: int, prefixes: list[str], table_name: str = 'citations') -> int: """Set all columns matching any prefix to NULL for a citation.""" prefixes = [p for p in (prefixes or []) if isinstance(p, str) and p] @@ -1492,20 +1980,31 @@ def backfill_human_decisions(self, criteria_parsed: dict[str, Any], table_name: self.create_column('human_l1_decision', 'TEXT', table_name=table_name) self.create_column('human_l2_decision', 'TEXT', table_name=table_name) - def _human_col(q: str) -> str: + def _human_col(q: str, stage: str) -> str: core = snake_case(q, max_len=56) - return f"human_{core}" if core else 'human_col' + return f"human_{stage}_{core}" if core else 'human_col' - def _llm_col(q: str) -> str: + def _llm_col(q: str, stage: str) -> str: core = snake_case(q, max_len=56) - return f"llm_{core}" if core else 'llm_col' + return f"llm_{stage}_{core}" if core else 'llm_col' needed_cols: list[str] = [] - for q in list(l2_union_qs): + for q in l1_qs: + if not isinstance(q, str) or not q.strip(): + continue + needed_cols.extend(( + _human_col(q, 'l1'), _llm_col( + q, 'l1', + ), f'llm_{snake_case(q, max_len=56)}', + )) + for q in l2_qs: if not isinstance(q, str) or not q.strip(): continue - needed_cols.append(_human_col(q)) - needed_cols.append(_llm_col(q)) + needed_cols.extend(( + _human_col(q, 'l2'), _llm_col( + q, 'l2', + ), f'llm_{snake_case(q, max_len=56)}', + )) # stable unique seen = set() uniq_cols = [] diff --git a/backend/api/services/citation_deduplication_preferences_service.py b/backend/api/services/citation_deduplication_preferences_service.py new file mode 100644 index 00000000..a8fac106 --- /dev/null +++ b/backend/api/services/citation_deduplication_preferences_service.py @@ -0,0 +1,72 @@ +"""Per-user, per-review deduplication field configuration.""" +from __future__ import annotations + +import json + +from .citation_workspace_preferences_service import _IDENTIFIER +from .postgres_auth import postgres_server + + +class CitationDeduplicationPreferencesService: + def get_fields(self, sr_id: str, table_name: str, user_id: str) -> list[str] | None: + if not _IDENTIFIER.fullmatch(table_name or ''): + raise ValueError('Invalid citation table name') + conn = postgres_server.conn + cur = conn.cursor() + try: + try: + cur.execute( + '''SELECT fields FROM citation_deduplication_preferences + WHERE sr_id=%s AND citation_table_name=%s AND user_id=%s''', + (sr_id, table_name, user_id), + ) + except Exception: + conn.rollback() + return None + row = cur.fetchone() + if not row: + return None + fields = json.loads(row[0]) if isinstance(row[0], str) else row[0] + return [field for field in fields if isinstance(field, str)] if isinstance(fields, list) else None + finally: + cur.close() + + def save_fields( + self, + sr_id: str, + table_name: str, + user_id: str, + fields: list[str], + available: list[str], + ) -> list[str]: + allowed = set(available) + requested = [ + field for field in fields + if field in allowed and field not in {'id', 'provenance'} + ] + existing = [ + field for field in (self.get_fields(sr_id, table_name, user_id) or []) + if field != 'provenance' + ] + merged = list(dict.fromkeys(existing + requested)) + conn = postgres_server.conn + cur = conn.cursor() + try: + cur.execute( + '''INSERT INTO citation_deduplication_preferences + (sr_id,citation_table_name,user_id,fields) + VALUES (%s,%s,%s,%s::jsonb) + ON CONFLICT (sr_id,citation_table_name,user_id) DO UPDATE + SET fields=EXCLUDED.fields, updated_at=CURRENT_TIMESTAMP''', + (sr_id, table_name, user_id, json.dumps(merged)), + ) + conn.commit() + return merged + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +citation_deduplication_preferences_service = CitationDeduplicationPreferencesService() diff --git a/backend/api/services/citation_deduplication_service.py b/backend/api/services/citation_deduplication_service.py new file mode 100644 index 00000000..b9a20678 --- /dev/null +++ b/backend/api/services/citation_deduplication_service.py @@ -0,0 +1,231 @@ +"""Pure review-only citation duplicate detection helpers.""" +from __future__ import annotations + +import re +import unicodedata +from difflib import SequenceMatcher +from typing import Any + +from .citation_duplicate_review_service import suggest_survivor + +POSSIBLE_MATCH_THRESHOLD = 0.70 +MAX_LOOKAHEAD = 6 + + +def normalize(value: Any) -> str: + """Normalize values for matching while preserving meaningful Unicode letters.""" + if value is None: + return '' + text = unicodedata.normalize('NFKC', str(value)).casefold() + text = ''.join( + character for character in text + if not unicodedata.category(character).startswith('C') + ) + text = re.sub(r'[^\w\s]', ' ', text, flags=re.UNICODE) + return ' '.join(text.split()) + + +def _scores(left: dict[str, Any], right: dict[str, Any], fields: list[str]) -> dict[str, float]: + result: dict[str, float] = {} + for field in fields: + left_value = normalize(left.get(field)) + right_value = normalize(right.get(field)) + if left_value and right_value: + result[field] = ( + 1.0 + if left_value == right_value + else SequenceMatcher(None, left_value, right_value).ratio() + ) + return result + + +def calculate_duplicate_statuses( + rows: list[dict[str, Any]], + fields: list[str], + threshold: float = POSSIBLE_MATCH_THRESHOLD, +) -> dict[str, Any]: + """Calculate exact/possible matches and connected candidate groups. + + Empty fields are neutral evidence: they are ignored when one or both values + are empty. A pair must have at least one populated field in common, and the + configured field list is unbounded; only the row lookahead is capped at six + rows. + """ + fields = list(dict.fromkeys(field for field in fields if field)) + lookahead = min(2 * len(fields), MAX_LOOKAHEAD) if fields else 0 + statuses = ['no_match'] * len(rows) + parent = list(range(len(rows))) + edges: list[dict[str, Any]] = [] + + def find(index: int) -> int: + while parent[index] != index: + parent[index] = parent[parent[index]] + index = parent[index] + return index + + def union(left: int, right: int) -> None: + left_root, right_root = find(left), find(right) + if left_root != right_root: + parent[right_root] = left_root + + for index, left in enumerate(rows): + for other in range(index + 1, min(len(rows), index + lookahead + 1)): + evidence = _scores(left, rows[other], fields) + if not evidence: + continue + exact = all(score == 1.0 for score in evidence.values()) + score = 1.0 if exact else sum(evidence.values()) / len(evidence) + if not exact and score < threshold: + continue + status = 'exact' if exact else 'possible' + statuses[index] = 'exact' if status == 'exact' else ( + 'exact' if statuses[index] == 'exact' else 'possible' + ) + statuses[other] = 'exact' if status == 'exact' else ( + 'exact' if statuses[other] == 'exact' else 'possible' + ) + union(index, other) + edges.append({ + 'left_index': index, + 'right_index': other, + 'status': status, + 'score': round(score, 5), + 'evidence': evidence, + }) + + groups: dict[int, dict[str, Any]] = {} + for index, row in enumerate(rows): + if statuses[index] == 'no_match': + continue + root = find(index) + group = groups.setdefault( + root, { + 'group_id': f'duplicate-group-{len(groups) + 1}', + 'citation_ids': [], + 'status': 'possible', + }, + ) + citation_id = row.get('id', index) + if citation_id not in group['citation_ids']: + group['citation_ids'].append(citation_id) + if statuses[index] == 'exact': + group['status'] = 'exact' + + row_results = [] + for index, row in enumerate(rows): + group = next( + ( + value for value in groups.values() + if row.get('id', index) in value['citation_ids'] + ), + None, + ) + row_results.append({ + 'status': statuses[index], + 'group_id': group['group_id'] if group else None, + 'score': max( + ( + edge['score'] for edge in edges + if index in (edge['left_index'], edge['right_index']) + ), + default=None, + ), + }) + return { + 'rows': row_results, + 'groups': list(groups.values()), + 'edges': edges, + 'lookahead': lookahead, + 'threshold': threshold, + } + + +def recompute_affected_duplicate_groups( + previous: dict[str, Any], + deleted_ids: set[int], + fields: list[str], +) -> dict[str, Any]: + """Recompute only groups that contained one of the deleted citations.""" + deleted_ids = {int(citation_id) for citation_id in deleted_ids} + rows = { + int(item['citation_id']): dict(item) + for item in previous.get('rows', []) + if item.get('citation_id') is not None + and int(item['citation_id']) not in deleted_ids + } + retained_groups = [] + affected_groups = [] + affected_ids: set[int] = set() + for group in previous.get('groups', []): + citation_ids = { + int(citation_id) for citation_id in group.get('citation_ids', []) + } + if citation_ids & deleted_ids: + affected_groups.append(group) + affected_ids.update(citation_ids) + else: + retained_groups.append(group) + + recalculated_groups = [] + for previous_group in affected_groups: + members = [ + dict(member) + for member in previous_group.get('members', []) + if int(member.get('id')) not in deleted_ids + ] + recalculated = calculate_duplicate_statuses(members, fields) + statuses_by_id = { + int(member.get('id', index)): { + **status, + 'citation_id': int(member.get('id', index)), + } + for index, (member, status) in enumerate( + zip(members, recalculated['rows']), + ) + } + for citation_id in affected_ids - deleted_ids: + rows[citation_id] = statuses_by_id.get( + citation_id, { + 'citation_id': citation_id, + 'status': 'no_match', + 'group_id': None, + 'score': None, + }, + ) + for group_index, group in enumerate(recalculated['groups']): + group_id = previous_group['group_id'] + if group_index: + group_id = f'{group_id}-{group_index + 1}' + group = { + **group, + 'group_id': group_id, + 'members': [ + { + **member, + **statuses_by_id[int(member.get('id'))], + 'duplicate_group_id': group_id, + } + for member in members + if int(member.get('id')) in group['citation_ids'] + ], + } + group.update(suggest_survivor(group['members'])) + for member in group['members']: + rows[int(member['id'])]['group_id'] = group_id + recalculated_groups.append(group) + + result_rows = list(rows.values()) + return { + **previous, + 'rows': result_rows, + 'groups': retained_groups + recalculated_groups, + 'edges': [], + 'duplicate_counts': { + status: sum( + 1 for row in result_rows if row.get( + 'status', + ) == status + ) + for status in ('exact', 'possible', 'no_match') + }, + } diff --git a/backend/api/services/citation_duplicate_review_service.py b/backend/api/services/citation_duplicate_review_service.py new file mode 100644 index 00000000..af9ba5b9 --- /dev/null +++ b/backend/api/services/citation_duplicate_review_service.py @@ -0,0 +1,162 @@ +"""Persistence and safety checks for human duplicate review decisions.""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from .citation_import_schema import is_protected_citation_column +from .citation_workspace_preferences_service import _IDENTIFIER +from .postgres_auth import postgres_server + +DECISIONS = {'confirmed_duplicate', 'not_duplicate', 'deferred'} +METADATA_FIELDS = ( + 'title', 'abstract', 'authors', 'author', + 'journal', 'year', 'publication_year', 'keywords', 'url', +) +IDENTIFIER_FIELDS = ('doi', 'pmid', 'pmcid', 'url', 'fulltext_url') + + +def suggest_survivor(members: list[dict[str, Any]]) -> dict[str, Any]: + """Choose a stable survivor using completeness, identifiers, age, then ID.""" + def rank(member: dict[str, Any]) -> tuple[int, int, float, int]: + completeness = sum( + bool(member.get(field)) + for field in METADATA_FIELDS + ) + identifiers = sum( + (len(IDENTIFIER_FIELDS) - index) * bool(member.get(field)) + for index, field in enumerate(IDENTIFIER_FIELDS) + ) + created = member.get('created_at') or member.get('imported_at') + try: + age = -created.timestamp() if hasattr(created, 'timestamp') else - \ + datetime.fromisoformat(str(created)).timestamp() + except (TypeError, ValueError, OverflowError): + age = 0.0 + try: + citation_id = int(member.get('id', 0)) + except (TypeError, ValueError): + citation_id = 0 + return completeness, identifiers, age, -citation_id + + if not members: + return {'suggested_survivor_id': None, 'survivor_reason': None} + winner = max(members, key=rank) + winner_rank = rank(winner) + reasons = ( + 'most_complete_metadata', 'strongest_identifier', + 'oldest_record', 'lowest_id', + ) + reason = reasons[-1] + for index, candidate in enumerate(reasons): + if sum(rank(member)[index] == winner_rank[index] for member in members) == 1: + reason = candidate + break + return {'suggested_survivor_id': winner.get('id'), 'survivor_reason': reason} + + +class CitationDuplicateReviewService: + def list_reviews(self, sr_id: str, table_name: str) -> list[dict[str, Any]]: + self._validate(table_name) + cur = postgres_server.conn.cursor() + try: + try: + cur.execute( + '''SELECT group_id, decision, survivor_id, updated_by, updated_at + FROM citation_duplicate_reviews + WHERE sr_id=%s AND citation_table_name=%s + ORDER BY group_id''', (sr_id, table_name), + ) + except Exception: + postgres_server.conn.rollback() + return [] + rows = cur.fetchall() or [] + return [self._row(row) for row in rows] + finally: + cur.close() + + def save_review( + self, sr_id: str, table_name: str, group_id: str, decision: str, + survivor_id: int | None, actor_id: str, + ) -> dict[str, Any]: + self._validate(table_name) + if decision not in DECISIONS: + raise ValueError('Invalid duplicate review decision') + if decision == 'confirmed_duplicate' and survivor_id is None: + raise ValueError('A survivor is required for confirmed duplicates') + if decision != 'confirmed_duplicate': + survivor_id = None + conn = postgres_server.conn + cur = conn.cursor() + try: + cur.execute( + '''INSERT INTO citation_duplicate_reviews + (sr_id,citation_table_name,group_id,decision,survivor_id,updated_by) + VALUES (%s,%s,%s,%s,%s,%s) + ON CONFLICT (sr_id,citation_table_name,group_id) DO UPDATE + SET decision=EXCLUDED.decision, survivor_id=EXCLUDED.survivor_id, + updated_by=EXCLUDED.updated_by, updated_at=CURRENT_TIMESTAMP''', + (sr_id, table_name, group_id, decision, survivor_id, actor_id), + ) + conn.commit() + return { + 'group_id': group_id, 'decision': decision, + 'survivor_id': survivor_id, 'updated_by': actor_id, + } + except Exception: + conn.rollback() + raise + finally: + cur.close() + + def protected_ids(self, table_name: str, citation_ids: list[int]) -> list[int]: + self._validate(table_name) + ids = sorted({int(value) for value in citation_ids}) + if not ids: + return [] + cur = postgres_server.conn.cursor() + try: + cur.execute( + '''SELECT column_name FROM information_schema.columns + WHERE table_schema='public' AND table_name=%s''', (table_name,), + ) + columns = [ + row[0] if not isinstance( + row, dict, + ) else row['column_name'] for row in cur.fetchall() or [] + ] + protected = [ + column for column in columns if is_protected_citation_column(column) or column in { + 'screening_decision', 'l1_decision', 'l2_decision', + } + ] + if not protected: + return [] + placeholders = ','.join(['%s'] * len(ids)) + checks = ' OR '.join( + f'NULLIF(CAST("{column}" AS TEXT), \'\') IS NOT NULL' for column in protected + ) + cur.execute( + f'SELECT id FROM "{table_name}" WHERE id IN ({placeholders}) AND ({checks})', + tuple(ids), + ) + return [int(row[0] if not isinstance(row, dict) else row['id']) for row in cur.fetchall() or []] + finally: + cur.close() + + @staticmethod + def _validate(table_name: str) -> None: + if not _IDENTIFIER.fullmatch(table_name or ''): + raise ValueError('Invalid citation table name') + + @staticmethod + def _row(row: Any) -> dict[str, Any]: + if isinstance(row, dict): + return dict(row) + return { + 'group_id': row[0], 'decision': row[1], 'survivor_id': row[2], + 'updated_by': row[3], 'updated_at': row[4], + } + + +citation_duplicate_review_service = CitationDuplicateReviewService() diff --git a/backend/api/services/citation_duplicate_run_service.py b/backend/api/services/citation_duplicate_run_service.py new file mode 100644 index 00000000..3bba1c8e --- /dev/null +++ b/backend/api/services/citation_duplicate_run_service.py @@ -0,0 +1,101 @@ +"""Persistence for explicit, dataset-scoped duplicate calculations.""" +from __future__ import annotations + +import json +import uuid +from datetime import date +from datetime import datetime +from typing import Any + +from .postgres_auth import postgres_server + + +def _json_default(value: Any) -> str: + """Serialize database date/time values in duplicate-run JSON results.""" + if isinstance(value, (datetime, date)): + return value.isoformat() + raise TypeError( + f'Object of type {type(value).__name__} is not JSON serializable', + ) + + +class CitationDuplicateRunService: + def _key(self, revision: Any, fields: list[str]) -> tuple[str, str]: + return json.dumps(revision, sort_keys=True, default=str), json.dumps(fields, separators=(',', ':')) + + def get_cached(self, sr_id: str, table_name: str, revision: Any, fields: list[str]) -> dict[str, Any] | None: + revision_json, fields_json = self._key(revision, fields) + cur = postgres_server.conn.cursor() + try: + try: + cur.execute( + '''SELECT id, status, result, error, started_at, completed_at + FROM citation_duplicate_runs + WHERE sr_id=%s AND citation_table_name=%s + AND dataset_revision=%s::jsonb AND fields=%s::jsonb + AND status='succeeded' + ORDER BY completed_at DESC LIMIT 1''', + (sr_id, table_name, revision_json, fields_json), + ) + except Exception: + postgres_server.conn.rollback() + return None + row = cur.fetchone() + if not row: + return None + result = row[2] if not isinstance(row, dict) else row.get('result') + return { + 'run_id': str(row[0] if not isinstance(row, dict) else row.get('id')), + 'status': row[1] if not isinstance(row, dict) else row.get('status'), + 'result': json.loads(result) if isinstance(result, str) else result, + 'error': row[3] if not isinstance(row, dict) else row.get('error'), + } + finally: + cur.close() + + def start(self, sr_id: str, table_name: str, revision: Any, fields: list[str], actor: str) -> str: + run_id = str(uuid.uuid4()) + revision_json, fields_json = self._key(revision, fields) + conn = postgres_server.conn + cur = conn.cursor() + try: + cur.execute( + '''INSERT INTO citation_duplicate_runs + (id,sr_id,citation_table_name,dataset_revision,fields,status,requested_by) + VALUES (%s,%s,%s,%s::jsonb,%s::jsonb,'running',%s)''', + (run_id, sr_id, table_name, revision_json, fields_json, actor), + ) + conn.commit() + return run_id + except Exception: + conn.rollback() + raise + finally: + cur.close() + + def finish(self, run_id: str, result: dict[str, Any] | None = None, error: str | None = None) -> None: + conn = postgres_server.conn + cur = conn.cursor() + try: + cur.execute( + '''UPDATE citation_duplicate_runs + SET status=%s, result=%s::jsonb, error=%s, completed_at=CURRENT_TIMESTAMP + WHERE id=%s''', + ( + 'succeeded' if result is not None else 'failed', + json.dumps( + result, default=_json_default, + ) if result is not None else None, + error, + run_id, + ), + ) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +citation_duplicate_run_service = CitationDuplicateRunService() diff --git a/backend/api/services/citation_import_preview_service.py b/backend/api/services/citation_import_preview_service.py new file mode 100644 index 00000000..98a7b9eb --- /dev/null +++ b/backend/api/services/citation_import_preview_service.py @@ -0,0 +1,1192 @@ +"""Parse, validate, encrypt, and stage citation import previews without citation writes.""" +from __future__ import annotations + +import base64 +import csv +import hashlib +import io +import json +import re +import uuid +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from typing import Any + +from cryptography.fernet import Fernet + +from .citation_import_schema import COMMIT_SUPPORTED_FIELD_KEYS +from .citation_import_schema import REQUIRED_IMPORT_FIELD_KEYS + +try: + import rispy # type: ignore +except Exception: # pragma: no cover + rispy = None + +try: + from .postgres_auth import postgres_server +except ModuleNotFoundError: # Optional driver in isolated unit-test tooling. + postgres_server = None + + +_REQUIRED_FIELDS = REQUIRED_IMPORT_FIELD_KEYS +_CANONICAL_RIS_COLUMNS = ( + 'Title', 'Abstract', 'Keywords', + 'Journal', 'Year', 'Authors', 'DOI', 'Type', 'URL', +) +_IDENT_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]{0,62}$') +_MAPPING_DECISION_VERSION = 1 +_EXTERNAL_ID_HEADER_NAMESPACES = { + 'pmid': 'pubmed', + 'pubmedid': 'pubmed', + 'pubmedidentifier': 'pubmed', + 'embaseid': 'embase', + 'embasenumber': 'embase', + 'scopuseid': 'scopus', + 'eid': 'scopus', + 'webofscienceid': 'web_of_science', + 'webofscienceaccessionnumber': 'web_of_science', + 'wosid': 'web_of_science', + 'wosaccessionnumber': 'web_of_science', +} + + +def _initial_citation_table_name(sr_id: str, preview_id: str) -> str: + """Return a deterministic, safe dynamic-table name for an initial import.""" + review_hash = hashlib.sha256(sr_id.encode('utf-8')).hexdigest()[:12] + preview_prefix = preview_id.replace('-', '')[:12] + return f'sr_{review_hash}_{preview_prefix}_cit' + + +def _create_initial_citation_table(cur, table_name: str) -> None: + """Create the compatible minimum physical schema inside the caller transaction.""" + cur.execute( + f'''CREATE TABLE "{table_name}" ( + id SERIAL PRIMARY KEY, + "title" TEXT, + "abstract" TEXT, + "cit_id" TEXT, + "fulltext_url" TEXT, + "fulltext" TEXT, + "fulltext_md5" TEXT, + "pdf_link_status" TEXT, + "pdf_link_reason" TEXT, + "pdf_link_source" TEXT, + "pdf_link_url" TEXT, + "pdf_link_last_checked_at" TIMESTAMP WITH TIME ZONE, + "pdf_link_error" TEXT, + "pdf_link_doi_source" TEXT, + "l1_validated_by" TEXT, + "l1_validated_at" TIMESTAMP WITH TIME ZONE, + "l2_validated_by" TEXT, + "l2_validated_at" TIMESTAMP WITH TIME ZONE, + "parameters_validated_by" TEXT, + "parameters_validated_at" TIMESTAMP WITH TIME ZONE, + "created_at" TIMESTAMP WITH TIME ZONE DEFAULT now() + )''', + ) + + +def _normalized_name(value: str) -> str: + return re.sub(r'[^a-z0-9]+', '', (value or '').strip().lower()) + + +def _json_object(value: Any) -> dict[str, Any]: + """Normalize PostgreSQL JSONB values from either native or string adapters.""" + if isinstance(value, dict): + return value + if isinstance(value, str): + parsed = json.loads(value) + if isinstance(parsed, dict): + return parsed + raise ValueError('Stored preview metadata is invalid') + + +def _isoformat(value: Any) -> str: + return value.isoformat() if hasattr(value, 'isoformat') else str(value) + + +def _mapping_decision( + mapping: dict[str, Any], excluded_source_columns: list[str], + source_columns: list[str], row_count: int, + excluded_ambiguous_row_numbers: list[int] | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Validate a canonical mapping decision and return refreshed diagnostics. + + Additional physical citation fields require explicit schema reconciliation, + which deliberately is not part of this endpoint. Every source column is + therefore mapped to title/abstract or explicitly excluded. + """ + if not isinstance(mapping, dict) or set(mapping) != set(COMMIT_SUPPORTED_FIELD_KEYS): + raise ValueError( + 'approved_mapping must contain exactly the currently commit-supported fields: title and abstract', + ) + if not isinstance(excluded_source_columns, list) or any(not isinstance(value, str) for value in excluded_source_columns): + raise ValueError( + 'excluded_source_columns must be an array of source column names', + ) + if len(set(excluded_source_columns)) != len(excluded_source_columns): + raise ValueError('excluded_source_columns must not contain duplicates') + if any(column not in source_columns for column in excluded_source_columns): + raise ValueError( + 'excluded_source_columns contains an unknown source column', + ) + mapped_columns: list[str] = [] + for field in _REQUIRED_FIELDS: + column = mapping[field] + if column is not None and (not isinstance(column, str) or column not in source_columns): + raise ValueError( + f'approved_mapping.{field} must reference a source column or null', + ) + if column: + mapped_columns.append(column) + if len(set(mapped_columns)) != len(mapped_columns): + raise ValueError( + 'A source column cannot map to more than one supported field', + ) + unclassified = set(source_columns) - set(mapped_columns) - \ + set(excluded_source_columns) + if unclassified: + raise ValueError( + f'Every source column must be mapped or explicitly excluded: {sorted(unclassified)}', + ) + excluded_ambiguous_row_numbers = excluded_ambiguous_row_numbers or [] + if ( + any( + not isinstance(row_number, int) or isinstance(row_number, bool) + for row_number in excluded_ambiguous_row_numbers + ) + or len(set(excluded_ambiguous_row_numbers)) != len(excluded_ambiguous_row_numbers) + or any(row_number < 1 or row_number > row_count for row_number in excluded_ambiguous_row_numbers) + ): + raise ValueError( + 'excluded_ambiguous_row_numbers must contain unique source row numbers', + ) + missing_fields = [ + field for field in _REQUIRED_FIELDS if not mapping[field] + ] + missing_row_counts = { + field: row_count if field in missing_fields else 0 for field in _REQUIRED_FIELDS + } + report = { + 'row_count': row_count, + 'source_columns': source_columns, + 'missing_required_fields': missing_fields, + 'missing_required_value_counts': missing_row_counts, + 'unclassified_source_columns': [], + 'valid_for_commit': not missing_fields and all(count == 0 for count in missing_row_counts.values()), + } + decision = { + 'version': _MAPPING_DECISION_VERSION, + 'approved_mapping': mapping, + 'excluded_source_columns': excluded_source_columns, + 'excluded_ambiguous_row_numbers': sorted(excluded_ambiguous_row_numbers), + } + return decision, report + + +def _revalidated_mapping_report( + mapping: dict[str, str | None], excluded_source_columns: list[str], + source_columns: list[str], rows: list[dict[str, Any]], + excluded_ambiguous_row_numbers: list[int] | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Recalculate required-value diagnostics from staged rows for an approved mapping.""" + decision, report = _mapping_decision( + mapping, excluded_source_columns, source_columns, len( + rows, + ), excluded_ambiguous_row_numbers, + ) + for field, column in mapping.items(): + if column: + report['missing_required_value_counts'][field] = sum( + not isinstance(row, dict) or not str( + row.get(column) or '', + ).strip() + for row in rows + ) + report['valid_for_commit'] = ( + not report['missing_required_fields'] + and all(count == 0 for count in report['missing_required_value_counts'].values()) + ) + return decision, report + + +def _bibliographic_fingerprint(title: Any, abstract: Any) -> str: + """Versioned exact identity for the currently supported canonical import fields.""" + normalized = '\x1f'.join( + re.sub(r'\s+', ' ', str(value or '').strip()).casefold() + for value in (title, abstract) + ) + return f'v1:{hashlib.sha256(normalized.encode("utf-8")).hexdigest()}' + + +def _normalized_doi(value: Any) -> str: + """Return a stable DOI identity, without accepting a URL as a distinct DOI.""" + doi = str(value or '').strip() + lowered = doi.casefold() + for prefix in ('https://doi.org/', 'http://doi.org/', 'http://dx.doi.org/', 'doi:'): + if lowered.startswith(prefix): + doi = doi[len(prefix):].strip() + break + return doi.rstrip('.,;').casefold() + + +def _row_doi(row: dict[str, Any]) -> str: + """Read DOI from a known source header; DOI is identity metadata, not a mapped field.""" + for column, value in row.items(): + if _normalized_name(column) in {'doi', 'digitalobjectidentifier'}: + return _normalized_doi(value) + return '' + + +def _normalized_external_id(namespace: str, value: Any) -> str: + """Normalize an explicitly namespaced source identifier without crossing systems.""" + identifier = re.sub(r'\s+', '', str(value or '').strip()) + prefixes = { + 'pubmed': r'(?:pmid|pubmed(?:id|identifier)?):?', + 'embase': r'(?:embase(?:id|number)?):?', + 'scopus': r'(?:scopus(?:eid)?|eid):?', + 'web_of_science': r'(?:webofscience(?:id|accessionnumber)?|wos(?:id|accessionnumber)?):?', + } + return re.sub(f'^{prefixes[namespace]}', '', identifier, flags=re.IGNORECASE).casefold() + + +def _row_external_ids(row: dict[str, Any]) -> list[tuple[str, str]]: + """Read recognized source headers into distinct external-ID namespaces.""" + identities: list[tuple[str, str]] = [] + for column, value in row.items(): + namespace = _EXTERNAL_ID_HEADER_NAMESPACES.get( + _normalized_name(column), + ) + if namespace: + normalized = _normalized_external_id(namespace, value) + if normalized and (namespace, normalized) not in identities: + identities.append((namespace, normalized)) + return identities + + +def _identity_candidates(title: Any, abstract: Any, row: dict[str, Any]) -> list[tuple[str, str | None, str]]: + """Return DOI, namespaced external IDs, then fingerprint exact identities.""" + fingerprint = _bibliographic_fingerprint(title, abstract) + doi = _row_doi(row) + external_ids = [ + ('external_id', namespace, value) + for namespace, value in _row_external_ids(row) + ] + return ([('doi', None, doi)] if doi else []) + external_ids + [('bibliographic_fingerprint', None, fingerprint)] + + +def sniff_format(filename: str, raw_bytes: bytes) -> str: + name = (filename or '').lower() + if name.endswith('.csv'): + return 'csv' + if name.endswith('.ris') or name.endswith('.txt'): + return 'ris' + text = raw_bytes.decode('utf-8', errors='ignore').lower() + return 'ris' if 'ty -' in text and 'er -' in text else 'csv' + + +def _decode(raw_bytes: bytes) -> str: + try: + return raw_bytes.decode('utf-8-sig') + except UnicodeDecodeError: + return raw_bytes.decode('utf-8', errors='replace') + + +def _join(value: Any, separator: str = '; ') -> str: + if isinstance(value, list): + return separator.join(str(item) for item in value if str(item).strip()) + return '' if value is None else str(value) + + +def _year(value: Any) -> str: + match = re.search(r'(?:19|20)\d{2}', str(value or '')) + return match.group(0) if match else '' + + +def _ris_value(column: str, entry: dict[str, Any]) -> Any: + key = column.lower() + values = { + 'title': entry.get('title') or entry.get('primary_title') or entry.get('short_title'), + 'abstract': entry.get('abstract') or entry.get('notes_abstract') or _join(entry.get('notes'), '\n'), + 'keywords': _join(entry.get('keywords')), + 'journal': entry.get('secondary_title') or entry.get('journal_name'), + 'year': _year(entry.get('year') or entry.get('publication_year') or entry.get('date')), + 'authors': _join(entry.get('authors')), + 'doi': entry.get('doi'), + 'type': entry.get('type_of_reference'), + 'url': _join(entry.get('urls')), + } + return values.get(key, entry.get(key)) + + +def parse_source(filename: str, raw_bytes: bytes) -> tuple[str, list[dict[str, Any]], list[str]]: + """Return source format, raw-header rows, and source columns.""" + source_format = sniff_format(filename, raw_bytes) + text = _decode(raw_bytes) + if source_format == 'csv': + reader = csv.DictReader(io.StringIO(text)) + return source_format, list(reader), list(reader.fieldnames or []) + if rispy is None: + raise ValueError('RIS upload requested but rispy is not installed') + entries = rispy.load(io.StringIO(text)) + rows = [ + { + column: _ris_value(column, entry) + for column in _CANONICAL_RIS_COLUMNS + } for entry in entries + ] + return source_format, rows, list(_CANONICAL_RIS_COLUMNS) + + +def build_preview(filename: str, raw_bytes: bytes) -> dict[str, Any]: + """Build deterministic parse/mapping diagnostics without persistence.""" + if not raw_bytes: + raise ValueError('Uploaded file is empty') + source_format, rows, columns = parse_source(filename, raw_bytes) + if not columns: + raise ValueError('No columns found in upload') + + normalized_columns = { + _normalized_name( + column, + ): column for column in columns if column + } + mapping = { + field: normalized_columns.get(_normalized_name(field)) + for field in _REQUIRED_FIELDS + } + missing_fields = [field for field, column in mapping.items() if not column] + missing_row_counts = { + field: sum( + not str(row.get(column) or '').strip() + for row in rows + ) if column else len(rows) + for field, column in mapping.items() + } + canonical_schema = json.dumps( + columns, ensure_ascii=False, separators=(',', ':'), + ) + return { + 'source_format': source_format, + 'source_sha256': hashlib.sha256(raw_bytes).hexdigest(), + 'schema_fingerprint': f"sha256:{hashlib.sha256(canonical_schema.encode()).hexdigest()}", + 'proposed_mapping': mapping, + 'validation_report': { + 'row_count': len(rows), + 'source_columns': columns, + 'missing_required_fields': missing_fields, + 'missing_required_value_counts': missing_row_counts, + 'unclassified_source_columns': [ + column for column in columns if column not in set(mapping.values()) + ], + 'valid_for_commit': not missing_fields and all(count == 0 for count in missing_row_counts.values()), + }, + 'normalized_rows': rows, + } + + +class CitationImportPreviewRepository: + """Minimal metadata repository; raw staged content remains in encrypted storage.""" + + def __init__(self, connection_provider=None): + self.connection_provider = connection_provider + + def create(self, record: dict[str, Any]) -> None: + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """INSERT INTO citation_import_previews + (id, sr_id, citation_table_name, source_sha256, schema_fingerprint, + proposed_mapping, validation_report, encrypted_staging_locator, + staging_expires_at, commit_key, created_by) + VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, %s, %s, %s, %s)""", + ( + record['id'], record['sr_id'], record['citation_table_name'], + record['source_sha256'], record['schema_fingerprint'], + json.dumps(record['proposed_mapping']), json.dumps( + record['validation_report'], + ), + record['encrypted_staging_locator'], record['staging_expires_at'], + record['commit_key'], record['created_by'], + ), + ) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + cur.close() + + def reconcile( + self, sr_id: str, citation_table_name: str | None, + rows: list[dict[str, Any]], mapping: dict[str, str | None], + excluded_ambiguous_row_numbers: list[int] | None = None, + ) -> dict[str, Any]: + """Classify preview rows against durable exact identities without mutating citations.""" + counts = { + 'new': 0, 'existing_exact_match': 0, + 'ambiguous': 0, 'excluded_ambiguous': 0, 'invalid': 0, + } + examples: list[dict[str, Any]] = [] + ambiguous_details: list[dict[str, Any]] = [] + excluded_ambiguous_row_numbers = set( + excluded_ambiguous_row_numbers or [], + ) + ambiguous_row_numbers: list[int] = [] + title_column, abstract_column = mapping.get( + 'title', + ), mapping.get('abstract') + if not title_column or not abstract_column: + return {'counts': counts, 'examples': examples} + identities: dict[tuple[str, str | None, str], list[int]] = {} + if citation_table_name: + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """SELECT identity_kind, identifier_namespace, normalized_value, citation_id + FROM citation_identities + WHERE sr_id = %s AND citation_table_name = %s + AND identity_kind IN ('doi', 'external_id', 'bibliographic_fingerprint')""", + (sr_id, citation_table_name), + ) + for result in cur.fetchall(): + # Accept legacy three-column mock fixtures while production always returns four. + if len(result) == 3: + kind, value, citation_id = result + namespace = None + else: + kind, namespace, value, citation_id = result + identities.setdefault( + ( + str(kind), str(namespace) if namespace else None, str( + value, + ), + ), [], + ).append(int(citation_id)) + finally: + cur.close() + seen: dict[tuple[str, str | None, str], int] = {} + next_provisional_citation_id = -1 + for index, row in enumerate(rows): + if not isinstance(row, dict) or not str(row.get(title_column) or '').strip() or not str(row.get(abstract_column) or '').strip(): + outcome, identity = 'invalid', None + else: + candidates = _identity_candidates( + row[title_column], row[abstract_column], row, + ) + candidate_matches = { + candidate: sorted( + set(identities.get(candidate, [])) + | ({seen[candidate]} if candidate in seen else set()), + ) + for candidate in candidates + } + matches = { + citation_id + for candidate_citation_ids in candidate_matches.values() + for citation_id in candidate_citation_ids + } + if len(matches) > 1: + ambiguous_row_numbers.append(index + 1) + if len(ambiguous_details) < 25: + ambiguous_details.append({ + 'row_number': index + 1, + 'matches': [ + { + 'identity_kind': kind, + 'identifier_namespace': namespace, + 'normalized_value': normalized_value, + 'citation_ids': citation_ids, + } + for (kind, namespace, normalized_value), citation_ids in candidate_matches.items() + if citation_ids + ], + }) + outcome, identity = ( + ('excluded_ambiguous', None) + if index + 1 in excluded_ambiguous_row_numbers else ('ambiguous', None) + ) + elif matches: + outcome, identity = 'existing_exact_match', next( + iter(matches), + ) if matches else None + else: + outcome, identity = 'new', None + identity = next_provisional_citation_id + next_provisional_citation_id -= 1 + seen.update({candidate: identity for candidate in candidates}) + counts[outcome] += 1 + if len(examples) < 10: + example = {'row_number': index + 1, 'outcome': outcome} + if identity is not None and identity > 0: + example['citation_id'] = identity + examples.append(example) + if excluded_ambiguous_row_numbers - set(ambiguous_row_numbers): + raise ValueError( + 'Only currently ambiguous source rows may be excluded', + ) + return { + 'counts': counts, + 'examples': examples, + 'ambiguous_row_numbers': ambiguous_row_numbers, + 'ambiguous_details': ambiguous_details, + } + + def cancel(self, preview_id: str, sr_id: str, actor_id: str) -> str | None: + """Cancel one owned, uncommitted preview and return its staging locator.""" + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """UPDATE citation_import_previews + SET cancelled_at = CURRENT_TIMESTAMP + WHERE id = %s AND sr_id = %s AND created_by = %s + AND cancelled_at IS NULL AND committed_batch_id IS NULL + AND staging_expires_at > CURRENT_TIMESTAMP + RETURNING encrypted_staging_locator""", + (preview_id, sr_id, actor_id), + ) + row = cur.fetchone() + if not row: + conn.rollback() + return None + conn.commit() + return str(row[0]) + except Exception: + conn.rollback() + raise + finally: + cur.close() + + def get_active(self, preview_id: str, sr_id: str, actor_id: str) -> dict[str, Any] | None: + """Return active owned metadata only; encrypted staging is never selected.""" + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """SELECT id, citation_table_name, source_sha256, schema_fingerprint, + proposed_mapping, validation_report, staging_expires_at, created_at + FROM citation_import_previews + WHERE id = %s AND sr_id = %s AND created_by = %s + AND cancelled_at IS NULL AND committed_batch_id IS NULL + AND staging_expires_at > CURRENT_TIMESTAMP""", + (preview_id, sr_id, actor_id), + ) + row = cur.fetchone() + if not row: + return None + return { + 'id': str(row[0]), 'citation_table_name': row[1], + 'source_sha256': row[2], 'schema_fingerprint': row[3], + 'proposed_mapping': _json_object(row[4]), + 'validation_report': _json_object(row[5]), + 'staging_expires_at': _isoformat(row[6]), 'created_at': _isoformat(row[7]), + } + finally: + cur.close() + + def update_mapping_decision( + self, preview_id: str, sr_id: str, actor_id: str, + approved_mapping: dict[str, str | None], + excluded_source_columns: list[str], + refreshed_report: dict[str, Any], + excluded_ambiguous_row_numbers: list[int] | None = None, + ) -> dict[str, Any] | None: + """Persist an active owner's canonical decision and refreshed diagnostics.""" + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """SELECT proposed_mapping, validation_report FROM citation_import_previews + WHERE id = %s AND sr_id = %s AND created_by = %s + AND cancelled_at IS NULL AND committed_batch_id IS NULL + AND staging_expires_at > CURRENT_TIMESTAMP + FOR UPDATE""", + (preview_id, sr_id, actor_id), + ) + row = cur.fetchone() + if not row: + conn.rollback() + return None + prior_report = _json_object(row[1]) + source_columns = prior_report.get('source_columns') + row_count = prior_report.get('row_count') + if not isinstance(source_columns, list) or not all(isinstance(value, str) for value in source_columns) or not isinstance(row_count, int): + raise ValueError( + 'Stored preview validation metadata is invalid', + ) + decision, _expected_report = _mapping_decision( + approved_mapping, excluded_source_columns, source_columns, row_count, + excluded_ambiguous_row_numbers, + ) + if refreshed_report.get('row_count') != row_count or refreshed_report.get('source_columns') != source_columns: + raise ValueError( + 'Refreshed preview validation metadata does not match the staged preview', + ) + report = refreshed_report + cur.execute( + """UPDATE citation_import_previews + SET proposed_mapping = %s::jsonb, mapping_decision = %s::jsonb, + validation_report = %s::jsonb + WHERE id = %s""", + ( + json.dumps(approved_mapping), json.dumps( + decision, + ), json.dumps(report), preview_id, + ), + ) + conn.commit() + return {'proposed_mapping': approved_mapping, 'mapping_decision': decision, 'validation_report': report} + except Exception: + conn.rollback() + raise + finally: + cur.close() + + def get_staging_locator(self, preview_id: str, sr_id: str, actor_id: str) -> str | None: + """Internal owned lookup for commit; callers never receive this locator.""" + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """SELECT encrypted_staging_locator FROM citation_import_previews + WHERE id = %s AND sr_id = %s AND created_by = %s""", + (preview_id, sr_id, actor_id), + ) + row = cur.fetchone() + return row[0] if row else None + finally: + cur.close() + + def commit( + self, preview_id: str, sr_id: str, actor_id: str, approved_mapping: dict[str, str | None], + staged_payload: dict[str, Any], + ) -> dict[str, Any]: + """Commit verified staged rows and all provenance in one database transaction.""" + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + conn = provider.conn + cur = conn.cursor() + try: + cur.execute( + """SELECT citation_table_name, source_sha256, proposed_mapping, mapping_decision, validation_report, + committed_batch_id, cancelled_at, staging_expires_at + FROM citation_import_previews + WHERE id = %s AND sr_id = %s AND created_by = %s + FOR UPDATE""", + (preview_id, sr_id, actor_id), + ) + preview = cur.fetchone() + if not preview: + raise LookupError( + 'Import preview was not found or is not owned by this user', + ) + # Backward-compatible fixture shape before mapping_decision. + if len(preview) == 7: + table_name, source_sha256, proposed, report, committed_batch_id, cancelled_at, expires_at = preview + mapping_decision = None + else: + table_name, source_sha256, proposed, mapping_decision, report, committed_batch_id, cancelled_at, expires_at = preview + if committed_batch_id: + cur.execute( + """SELECT inserted_count, existing_exact_match_count, invalid_count + FROM citation_import_batches WHERE id = %s""", + (str(committed_batch_id),), + ) + counts = cur.fetchone() + if not counts: + raise RuntimeError( + 'Committed import preview has no batch record', + ) + conn.commit() + return { + 'id': preview_id, 'batch_id': str(committed_batch_id), 'idempotent': True, + 'inserted_count': int(counts[0]), 'existing_exact_match_count': int(counts[1]), + 'invalid_count': int(counts[2]), + } + if cancelled_at or expires_at <= datetime.now(timezone.utc): + raise LookupError('Import preview is no longer active') + if _json_object(proposed) != approved_mapping: + raise ValueError( + 'approved_mapping must exactly match the preview mapping', + ) + decision_data = _json_object( + mapping_decision, + ) if mapping_decision else {} + excluded_ambiguous_row_numbers = decision_data.get( + 'excluded_ambiguous_row_numbers', [], + ) + if not isinstance(excluded_ambiguous_row_numbers, list) or any( + not isinstance(row_number, int) or isinstance(row_number, bool) + for row_number in excluded_ambiguous_row_numbers + ): + raise ValueError('Stored ambiguous-row resolution is invalid') + report_data = _json_object(report) + reconciliation = report_data.get('reconciliation') + if isinstance(reconciliation, dict) and reconciliation.get('counts', {}).get('ambiguous', 0): + raise ValueError( + 'Preview has ambiguous exact citation identities', + ) + if not report_data.get('valid_for_commit'): + raise ValueError('Preview has blocking validation errors') + raw_file = staged_payload.get('raw_file_base64') + if not isinstance(raw_file, str) or hashlib.sha256(base64.b64decode(raw_file)).hexdigest() != source_sha256: + raise ValueError( + 'Staged preview source hash does not match preview metadata', + ) + rows = staged_payload.get('normalized_rows') + if not isinstance(rows, list): + raise ValueError('Staged preview rows are invalid') + title_column, abstract_column = approved_mapping.get( + 'title', + ), approved_mapping.get('abstract') + if not title_column or not abstract_column: + raise ValueError( + 'Approved title and abstract mappings are required', + ) + if table_name and not _IDENT_RE.match(table_name): + raise ValueError( + 'Import preview contains an invalid citation table name', + ) + cur.execute( + """SELECT screening_db ->> 'table_name' + FROM systematic_reviews WHERE id = %s FOR UPDATE""", + (sr_id,), + ) + review = cur.fetchone() + if not review: + raise LookupError('Systematic review was not found') + configured_table_name = review[0] + is_initial_import = table_name is None + if is_initial_import: + if configured_table_name: + raise ValueError( + 'A citation dataset was created while this initial import was being prepared', + ) + table_name = _initial_citation_table_name(sr_id, preview_id) + _create_initial_citation_table(cur, table_name) + elif configured_table_name != table_name: + raise ValueError( + 'Citation table is not configured for this systematic review', + ) + cur.execute( + """SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + AND column_name = ANY(%s)""", + (table_name, ['id', 'title', 'abstract']), + ) + if {item[0] for item in cur.fetchall()} != {'id', 'title', 'abstract'}: + raise ValueError( + 'Citation table lacks required canonical title and abstract columns', + ) + + cur.execute( + """SELECT id, inserted_count, existing_exact_match_count, invalid_count + FROM citation_import_batches + WHERE sr_id = %s AND citation_table_name = %s AND content_sha256 = %s + AND state = 'committed' + FOR UPDATE""", + (sr_id, table_name, source_sha256), + ) + existing_batch = cur.fetchone() + if existing_batch: + existing_batch_id, inserted, existing, invalid = existing_batch + cur.execute( + """UPDATE citation_import_previews SET committed_batch_id = %s + WHERE id = %s AND committed_batch_id IS NULL""", + (str(existing_batch_id), preview_id), + ) + conn.commit() + return { + 'id': preview_id, 'batch_id': str(existing_batch_id), 'idempotent': True, + 'inserted_count': int(inserted), 'existing_exact_match_count': int(existing), + 'invalid_count': int(invalid), + } + + batch_id = str(uuid.uuid4()) + filename = staged_payload.get('filename') if isinstance( + staged_payload.get('filename'), str, + ) else None + cur.execute( + """INSERT INTO citation_import_batches + (id, sr_id, citation_table_name, source_type, source_metadata, display_filename, + content_sha256, schema_fingerprint, mapping_decision, state, created_by) + SELECT %s, %s, %s, 'file', jsonb_build_object('source_format', %s), %s, + %s, schema_fingerprint, %s::jsonb, 'committed', %s + FROM citation_import_previews WHERE id = %s""", + ( + batch_id, sr_id, table_name, staged_payload.get( + 'source_format', + ), filename, + source_sha256, json.dumps( + decision_data, + ), actor_id, preview_id, + ), + ) + inserted = existing = invalid = 0 + seen: dict[tuple[str, str | None, str], int] = {} + for row_number, row in enumerate(rows, start=1): + if row_number in excluded_ambiguous_row_numbers: + continue + if not isinstance(row, dict): + invalid += 1 + continue + title, abstract = row.get( + title_column, + ), row.get(abstract_column) + if not str(title or '').strip() or not str(abstract or '').strip(): + invalid += 1 + continue + doi = _row_doi(row) + external_ids = _row_external_ids(row) + fingerprint = _bibliographic_fingerprint(title, abstract) + candidates = _identity_candidates(title, abstract, row) + matches = { + seen[candidate] + for candidate in candidates if candidate in seen + } + for identity_kind, identifier_namespace, identity_value in candidates: + cur.execute( + 'SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', + (f'{sr_id}:{table_name}:{identity_kind}:{identifier_namespace or ""}:{identity_value}',), + ) + cur.execute( + """SELECT citation_id FROM citation_identities + WHERE sr_id = %s AND citation_table_name = %s + AND identity_kind = %s + AND identifier_namespace IS NOT DISTINCT FROM %s + AND normalized_value = %s + LIMIT 1 FOR UPDATE""", + ( + sr_id, table_name, identity_kind, + identifier_namespace, identity_value, + ), + ) + match = cur.fetchone() + if match: + matches.add(int(match[0])) + if len(matches) > 1: + raise ValueError( + 'Import row has ambiguous exact citation identities', + ) + citation_id = next(iter(matches), None) + if citation_id is not None: + existing += 1 + seen.update( + {candidate: citation_id for candidate in candidates}, + ) + cur.execute( + """INSERT INTO citation_import_batch_memberships + (batch_id, sr_id, citation_table_name, citation_id, outcome, identity_snapshot) + VALUES (%s, %s, %s, %s, 'existing_exact_match', %s::jsonb) + ON CONFLICT (batch_id, citation_id) DO NOTHING""", + ( + batch_id, sr_id, table_name, citation_id, json.dumps({ + 'identities': [ + { + 'kind': kind, 'namespace': namespace, + 'value': value, + } + for kind, namespace, value in candidates + ], + }), + ), + ) + continue + cur.execute( + f'INSERT INTO "{table_name}" ("title", "abstract") VALUES (%s, %s) RETURNING id', + (str(title), str(abstract)), + ) + citation_id = int(cur.fetchone()[0]) + seen.update( + {candidate: citation_id for candidate in candidates}, + ) + inserted += 1 + cur.execute( + """INSERT INTO citation_identities + (id, sr_id, citation_table_name, citation_id, identity_kind, normalized_value, fingerprint_version) + VALUES (%s, %s, %s, %s, 'bibliographic_fingerprint', %s, 'v1')""", + (str(uuid.uuid4()), sr_id, table_name, citation_id, fingerprint), + ) + if doi: + cur.execute( + """INSERT INTO citation_identities + (id, sr_id, citation_table_name, citation_id, identity_kind, normalized_value) + VALUES (%s, %s, %s, %s, 'doi', %s)""", + (str(uuid.uuid4()), sr_id, table_name, citation_id, doi), + ) + for identifier_namespace, identity_value in external_ids: + cur.execute( + """INSERT INTO citation_identities + (id, sr_id, citation_table_name, citation_id, identity_kind, + identifier_namespace, normalized_value) + VALUES (%s, %s, %s, %s, 'external_id', %s, %s)""", + ( + str(uuid.uuid4()), sr_id, table_name, citation_id, + identifier_namespace, identity_value, + ), + ) + cur.execute( + """INSERT INTO citation_import_batch_memberships + (batch_id, sr_id, citation_table_name, citation_id, outcome, identity_snapshot) + VALUES (%s, %s, %s, %s, 'inserted', %s::jsonb)""", + ( + batch_id, sr_id, table_name, citation_id, json.dumps({ + 'bibliographic_fingerprint': fingerprint, + **({'doi': doi} if doi else {}), + **({'external_ids': dict(external_ids)} if external_ids else {}), + }), + ), + ) + cur.execute( + """UPDATE citation_import_batches SET inserted_count = %s, existing_exact_match_count = %s, + invalid_count = %s, committed_at = CURRENT_TIMESTAMP WHERE id = %s""", + (inserted, existing, invalid, batch_id), + ) + cur.execute( + """UPDATE citation_import_previews SET committed_batch_id = %s + WHERE id = %s AND committed_batch_id IS NULL""", + (batch_id, preview_id), + ) + if is_initial_import: + cur.execute( + """UPDATE systematic_reviews + SET screening_db = jsonb_build_object( + 'table_name', %s, + 'created_at', CURRENT_TIMESTAMP, + 'rows', %s + ), + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + AND (screening_db IS NULL OR screening_db ->> 'table_name' IS NULL)""", + (table_name, inserted, sr_id), + ) + if cur.rowcount != 1: + raise RuntimeError( + 'Initial citation dataset could not be registered for this review', + ) + cur.execute( + """INSERT INTO citation_audit_events + (id, sr_id, citation_table_name, batch_id, event_type, actor_id, details) + VALUES (%s, %s, %s, %s, 'import_committed', %s, + jsonb_build_object('inserted_count', %s, 'existing_exact_match_count', %s, 'invalid_count', %s))""", + ( + str(uuid.uuid4()), sr_id, table_name, batch_id, + actor_id, inserted, existing, invalid, + ), + ) + conn.commit() + return { + 'id': preview_id, 'batch_id': batch_id, 'idempotent': False, 'inserted_count': inserted, + 'existing_exact_match_count': existing, 'invalid_count': invalid, + } + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +class CitationImportPreviewService: + def __init__(self, storage, repository=None, *, secret_key: str, ttl_minutes: int = 1440): + if not storage: + raise ValueError( + 'Encrypted preview staging storage is not configured', + ) + self.storage = storage + self.repository = repository or CitationImportPreviewRepository() + self.ttl_minutes = ttl_minutes + self._fernet = Fernet( + base64.urlsafe_b64encode( + hashlib.sha256(secret_key.encode()).digest(), + ), + ) + + async def create( + self, *, sr_id: str, citation_table_name: str | None, filename: str, + raw_bytes: bytes, commit_key: str, actor_id: str, + ) -> dict[str, Any]: + if not commit_key or not actor_id: + raise ValueError('commit_key and actor_id are required') + preview = build_preview(filename, raw_bytes) + preview['validation_report']['reconciliation'] = self.repository.reconcile( + sr_id, citation_table_name, preview['normalized_rows'], preview['proposed_mapping'], + ) + preview_id = str(uuid.uuid4()) + expires_at = datetime.now(timezone.utc) + \ + timedelta(minutes=self.ttl_minutes) + locator = f'{self.storage.container_name}/citation-import-previews/{sr_id}/{preview_id}.fernet' + staged_payload = json.dumps( + { + 'filename': filename, 'source_format': preview['source_format'], + 'raw_file_base64': base64.b64encode(raw_bytes).decode('ascii'), + 'normalized_rows': preview['normalized_rows'], + }, ensure_ascii=False, separators=(',', ':'), + ).encode() + encrypted = self._fernet.encrypt(staged_payload) + if not await self.storage.put_bytes_by_path(locator, encrypted, 'application/octet-stream'): + raise RuntimeError('Failed to stage encrypted import preview') + record = { + 'id': preview_id, 'sr_id': sr_id, 'citation_table_name': citation_table_name, + 'source_sha256': preview['source_sha256'], 'schema_fingerprint': preview['schema_fingerprint'], + 'proposed_mapping': preview['proposed_mapping'], 'validation_report': preview['validation_report'], + 'encrypted_staging_locator': locator, 'staging_expires_at': expires_at, + 'commit_key': commit_key, 'created_by': actor_id, + } + try: + self.repository.create(record) + except Exception: + try: + await self.storage.delete_by_path(locator) + finally: + raise + return { + 'id': preview_id, 'source_format': preview['source_format'], + 'source_sha256': preview['source_sha256'], 'schema_fingerprint': preview['schema_fingerprint'], + 'proposed_mapping': preview['proposed_mapping'], 'validation_report': preview['validation_report'], + 'staging_expires_at': expires_at.isoformat(), + } + + async def cancel(self, *, preview_id: str, sr_id: str, actor_id: str) -> dict[str, Any]: + """Cancel an owned preview and best-effort remove its ciphertext staging blob.""" + if not actor_id: + raise ValueError('actor_id is required') + try: + preview_id = str(uuid.UUID(preview_id)) + except (TypeError, ValueError) as exc: + raise ValueError('preview_id must be a UUID') from exc + locator = self.repository.cancel(preview_id, sr_id, actor_id) + if not locator: + raise LookupError( + 'Import preview was not found, is not owned by this user, or is already finalized', + ) + try: + await self.storage.delete_by_path(locator) + cleanup_pending = False + except Exception: + # The metadata transition remains final. A later cleanup job can retry + # deletion of ciphertext for cancelled/expired previews. + cleanup_pending = True + return {'id': preview_id, 'cancelled': True, 'staging_cleanup_pending': cleanup_pending} + + def inspect(self, *, preview_id: str, sr_id: str, actor_id: str) -> dict[str, Any]: + """Read active owned preview metadata without accessing staged ciphertext.""" + if not actor_id: + raise ValueError('actor_id is required') + try: + preview_id = str(uuid.UUID(preview_id)) + except (TypeError, ValueError) as exc: + raise ValueError('preview_id must be a UUID') from exc + preview = self.repository.get_active(preview_id, sr_id, actor_id) + if not preview: + raise LookupError( + 'Import preview was not found, is not owned by this user, or is already finalized', + ) + return preview + + async def update_mapping( + self, *, preview_id: str, sr_id: str, actor_id: str, + approved_mapping: dict[str, str | None], + excluded_source_columns: list[str], + excluded_ambiguous_row_numbers: list[int] | None = None, + ) -> dict[str, Any]: + """Revalidate staged rows and save an active owner's mapping decision.""" + if not actor_id: + raise ValueError('actor_id is required') + try: + preview_id = str(uuid.UUID(preview_id)) + except (TypeError, ValueError) as exc: + raise ValueError('preview_id must be a UUID') from exc + preview = self.repository.get_active(preview_id, sr_id, actor_id) + if not preview: + raise LookupError( + 'Import preview was not found, is not owned by this user, or is already finalized', + ) + locator = self.repository.get_staging_locator( + preview_id, sr_id, actor_id, + ) + if not locator: + raise LookupError( + 'Import preview was not found, is not owned by this user, or is already finalized', + ) + encrypted, _name = await self.storage.get_bytes_by_path(locator) + try: + staged_payload = json.loads(self._fernet.decrypt(encrypted)) + except Exception as exc: + raise ValueError('Staged import preview is unreadable') from exc + rows = staged_payload.get('normalized_rows') + if not isinstance(rows, list): + raise ValueError('Staged preview rows are invalid') + source_columns = preview['validation_report'].get('source_columns') + if not isinstance(source_columns, list) or not all(isinstance(column, str) for column in source_columns): + raise ValueError('Stored preview validation metadata is invalid') + _decision, report = _revalidated_mapping_report( + approved_mapping, excluded_source_columns, source_columns, rows, + excluded_ambiguous_row_numbers, + ) + report['reconciliation'] = self.repository.reconcile( + sr_id, preview['citation_table_name'], rows, approved_mapping, + excluded_ambiguous_row_numbers, + ) + if report['reconciliation']['counts']['ambiguous']: + report['valid_for_commit'] = False + result = self.repository.update_mapping_decision( + preview_id, sr_id, actor_id, approved_mapping, excluded_source_columns, + report, _decision['excluded_ambiguous_row_numbers'], + ) + if not result: + raise LookupError( + 'Import preview was not found, is not owned by this user, or is already finalized', + ) + return {'id': preview_id, **result} + + async def commit( + self, *, preview_id: str, sr_id: str, actor_id: str, + approved_mapping: dict[str, str | None], + ) -> dict[str, Any]: + """Decrypt staging then atomically commit an active owned preview.""" + if not actor_id: + raise ValueError('actor_id is required') + try: + preview_id = str(uuid.UUID(preview_id)) + except (TypeError, ValueError) as exc: + raise ValueError('preview_id must be a UUID') from exc + if not isinstance(approved_mapping, dict): + raise ValueError('approved_mapping must be an object') + locator = self.repository.get_staging_locator( + preview_id, sr_id, actor_id, + ) + if not locator: + raise LookupError( + 'Import preview was not found, is not owned by this user, or is already finalized', + ) + encrypted, _ = await self.storage.get_bytes_by_path(locator) + try: + staged_payload = json.loads(self._fernet.decrypt(encrypted)) + except Exception as exc: + raise ValueError( + 'Encrypted staged preview could not be decrypted', + ) from exc + return self.repository.commit(preview_id, sr_id, actor_id, approved_mapping, staged_payload) diff --git a/backend/api/services/citation_import_schema.py b/backend/api/services/citation_import_schema.py new file mode 100644 index 00000000..552b9a4e --- /dev/null +++ b/backend/api/services/citation_import_schema.py @@ -0,0 +1,87 @@ +"""Allowlisted schema policy for citation import preview reconciliation. + +Dynamic citation tables have operational columns alongside user-imported source +fields. This module is the single source of truth for fields a preview may map +to and for identifiers a future reconciliation transaction may create. +""" +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CitationImportField: + key: str + physical_column: str + postgres_type: str + required: bool = False + commit_supported: bool = False + + +# Optional fields are registered now so preview/reconciliation work has one +# explicit vocabulary. They remain unavailable to mapping decisions until the +# commit transaction can safely reconcile and insert them. +CITATION_IMPORT_FIELDS: dict[str, CitationImportField] = { + 'title': CitationImportField('title', 'title', 'TEXT', required=True, commit_supported=True), + 'abstract': CitationImportField('abstract', 'abstract', 'TEXT', required=True, commit_supported=True), + 'doi': CitationImportField('doi', 'doi', 'TEXT'), + 'authors': CitationImportField('authors', 'authors', 'TEXT'), + 'journal': CitationImportField('journal', 'journal', 'TEXT'), + 'year': CitationImportField('year', 'year', 'TEXT'), + 'keywords': CitationImportField('keywords', 'keywords', 'TEXT'), + 'url': CitationImportField('url', 'url', 'TEXT'), + 'type': CitationImportField('type', 'type', 'TEXT'), +} + +REQUIRED_IMPORT_FIELD_KEYS = tuple( + field.key for field in CITATION_IMPORT_FIELDS.values() if field.required +) +COMMIT_SUPPORTED_FIELD_KEYS = tuple( + field.key for field in CITATION_IMPORT_FIELDS.values() if field.commit_supported +) + +# These are created by the dynamic-table lifecycle or used by operational +# workflows. Imports may neither overwrite them nor create ambiguous names. +PROTECTED_CITATION_COLUMNS = frozenset({ + 'id', 'cit_id', 'created_at', 'updated_at', 'fulltext_url', 'fulltext', + 'fulltext_md5', 'pdf_link_status', 'pdf_link_reason', 'pdf_link_source', + 'pdf_link_url', 'pdf_link_last_checked_at', 'pdf_link_error', + 'pdf_link_doi_source', 'l1_validated_by', 'l1_validated_at', + 'l2_validated_by', 'l2_validated_at', 'parameters_validated_by', + 'parameters_validated_at', +}) +PROTECTED_CITATION_COLUMN_PREFIXES = ('llm_', 'human_', 'review_', 'system_') +ALLOWED_ADDITIONAL_FIELD_TYPES = frozenset({'text'}) +_IDENTIFIER_RE = re.compile(r'^[a-z_][a-z0-9_]{0,62}$') + + +def is_protected_citation_column(column: str) -> bool: + normalized = str(column or '').strip().lower() + return normalized in PROTECTED_CITATION_COLUMNS or normalized.startswith(PROTECTED_CITATION_COLUMN_PREFIXES) + + +def safe_additional_field_name(source_column: str, existing_columns: Iterable[str] = ()) -> str: + """Return a deterministic safe new physical name or reject a collision. + + A future commit transaction must call this before interpolating an approved + additional-column identifier into `ALTER TABLE` SQL. + """ + source = str(source_column or '').strip() + if not source: + raise ValueError('Additional field source column is required') + normalized = re.sub(r'[^a-z0-9]+', '_', source.lower()).strip('_') + if not normalized: + raise ValueError('Additional field source column has no usable name') + candidate = f'import_{normalized}'[:63] + existing = {str(column).lower() for column in existing_columns} + if not _IDENTIFIER_RE.fullmatch(candidate): + raise ValueError( + 'Additional field name is not a safe PostgreSQL identifier', + ) + if is_protected_citation_column(candidate) or candidate in existing: + raise ValueError( + f'Additional field name collides with an existing or protected column: {candidate}', + ) + return candidate diff --git a/backend/api/services/citation_import_service.py b/backend/api/services/citation_import_service.py new file mode 100644 index 00000000..6062f202 --- /dev/null +++ b/backend/api/services/citation_import_service.py @@ -0,0 +1,453 @@ +"""Immediate, append-only citation file imports. + +Imports skip rows whose exact citation identity already exists unless the caller +explicitly opts into importing duplicates. Import batch and identity metadata is +retained for provenance, while the citation table is extended with text columns +as new source fields appear. +""" +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from typing import Any + +from .citation_import_preview_service import _bibliographic_fingerprint +from .citation_import_preview_service import _normalized_doi +from .citation_import_preview_service import _row_external_ids +from .citation_import_schema import is_protected_citation_column +from .postgres_auth import postgres_server + + +_IDENTIFIER = re.compile(r'^[A-Za-z_][A-Za-z0-9_]{0,62}$') + + +def normalize_source_columns(source_columns: list[str]) -> dict[str, str]: + """Map each non-empty source header to a unique safe physical column.""" + result: dict[str, str] = {} + used: set[str] = set() + for index, raw in enumerate(source_columns): + source = str(raw or '').strip() + if not source: + source = f'column_{index + 1}' + if source in result: + continue + normalized = re.sub( + r'[^a-z0-9]+', '_', source.casefold(), + ).strip('_') or f'column_{index + 1}' + if normalized[0].isdigit(): + normalized = f'column_{normalized}' + if normalized in {'id', 'title', 'abstract'} or is_protected_citation_column(normalized): + normalized = f'import_{normalized}' + normalized = normalized[:63] + candidate = normalized + suffix = 2 + while candidate in used or not _IDENTIFIER.fullmatch(candidate): + tail = f'_{suffix}' + candidate = f'{normalized[:63 - len(tail)]}{tail}' + suffix += 1 + result[source] = candidate + used.add(candidate) + return result + + +def _has_value(row: dict[str, Any]) -> bool: + return any(str(value or '').strip() for value in row.values()) + + +class CitationImportService: + def __init__(self, connection_provider=None): + self.connection_provider = connection_provider or postgres_server + + def count_duplicates_sync( + self, + *, + sr_id: str, + table_name: str | None, + rows: list[dict[str, Any]], + source_columns: list[str], + title_header: str | None = None, + abstract_header: str | None = None, + ) -> int: + """Count existing and repeated citation identities without mutating data.""" + if not table_name: + existing_keys: set[tuple[str, str | None, str]] = set() + else: + cur = self.connection_provider.conn.cursor() + try: + cur.execute( + """SELECT identity_kind, identifier_namespace, normalized_value + FROM citation_identities + WHERE sr_id = %s AND citation_table_name = %s""", + (sr_id, table_name), + ) + existing_keys = { + (row[0], row[1], row[2]) for row in cur.fetchall() + } + finally: + cur.close() + + names = normalize_source_columns(source_columns) + by_normalized = { + re.sub(r'[^a-z0-9]+', '', key.casefold()): key for key in names + } + + def resolve_header(configured: str | None, aliases: tuple[str, ...]) -> str | None: + if configured and configured in names: + return configured + return next((by_normalized[alias] for alias in aliases if alias in by_normalized), None) + + title_source = resolve_header( + title_header, ('title', 'articletitle', 'primarytitle'), + ) + abstract_source = resolve_header( + abstract_header, ('abstract', 'summary', 'description'), + ) + if not title_source or not abstract_source: + return 0 + + seen_keys: set[tuple[str, str | None, str]] = set() + duplicate_count = 0 + for row in rows: + if not isinstance(row, dict) or not _has_value(row): + continue + fingerprint = _bibliographic_fingerprint( + row.get(title_source), row.get(abstract_source), + ) + identities = {('bibliographic_fingerprint', None, fingerprint)} + doi = next( + ( + _normalized_doi(value) for source, value in row.items() if re.sub( + r'[^a-z0-9]+', '', source.casefold(), + ) in {'doi', 'digitalobjectidentifier'} + ), '', + ) + if doi: + identities.add(('doi', None, doi)) + identities.update( + ('external_id', namespace, value) + for namespace, value in _row_external_ids(row) + ) + if identities & (existing_keys | seen_keys): + duplicate_count += 1 + seen_keys.update(identities) + return duplicate_count + + def append_rows_sync( + self, + *, + sr_id: str, + table_name: str | None, + source_format: str, + filename: str, + raw_bytes: bytes, + rows: list[dict[str, Any]], + source_columns: list[str], + actor_id: str, + commit_key: str, + title_header: str | None = None, + abstract_header: str | None = None, + include_duplicates: bool = False, + ) -> dict[str, Any]: + if not actor_id or not commit_key: + raise ValueError('actor_id and commit_key are required') + if not source_columns: + raise ValueError('No columns found in upload') + names = normalize_source_columns(source_columns) + by_normalized = { + re.sub(r'[^a-z0-9]+', '', key.casefold()): key for key in names + } + + def resolve_header(configured: str | None, aliases: tuple[str, ...]) -> str | None: + if configured and configured in names: + return configured + for alias in aliases: + if alias in by_normalized: + return by_normalized[alias] + return None + + title_source = resolve_header( + title_header, ('title', 'articletitle', 'primarytitle'), + ) + abstract_source = resolve_header( + abstract_header, ('abstract', 'summary', 'description'), + ) + if not title_source or not abstract_source: + missing = [] + if not title_source: + missing.append('title') + if not abstract_source: + missing.append('abstract') + raise ValueError( + f"Missing required source headers: {', '.join(missing)}", + ) + + usable_rows = [ + row for row in rows if isinstance( + row, dict, + ) and _has_value(row) + ] + # The commit key provides idempotency. Include it in the batch content + # hash so a later intentional re-upload is not rejected by the batch's + # unique (sr_id, citation_table_name, content_sha256) constraint. + content_sha = hashlib.sha256( + raw_bytes + commit_key.encode(), + ).hexdigest() + conn = self.connection_provider.conn + cur = conn.cursor() + try: + cur.execute( + """SELECT id, citation_table_name, inserted_count, existing_exact_match_count, + invalid_count FROM citation_import_batches + WHERE sr_id = %s AND source_metadata ->> 'commit_key' = %s + ORDER BY created_at DESC LIMIT 1""", (sr_id, commit_key), + ) + prior = cur.fetchone() + if prior: + conn.rollback() + return { + 'batch_id': str(prior[0]), 'table_name': prior[1], 'idempotent': True, + 'rows_inserted': prior[2], 'invalid_count': prior[4], + 'warnings': [], + } + + if table_name: + if not _IDENTIFIER.fullmatch(table_name): + raise ValueError('Invalid citation table name') + else: + table_name = f"sr_{hashlib.sha256(sr_id.encode()).hexdigest()[:12]}_{uuid.uuid4().hex[:12]}_cit" + cur.execute(f'''CREATE TABLE "{table_name}" ( + id SERIAL PRIMARY KEY, "title" TEXT, "abstract" TEXT, + "provenance" TEXT, + "cit_id" TEXT, "fulltext_url" TEXT, "fulltext" TEXT, + "fulltext_md5" TEXT, "pdf_link_status" TEXT, + "pdf_link_reason" TEXT, "pdf_link_source" TEXT, + "pdf_link_url" TEXT, "pdf_link_last_checked_at" TIMESTAMPTZ, + "pdf_link_error" TEXT, "pdf_link_doi_source" TEXT, + "l1_validated_by" TEXT, "l1_validated_at" TIMESTAMPTZ, + "l2_validated_by" TEXT, "l2_validated_at" TIMESTAMPTZ, + "parameters_validated_by" TEXT, "parameters_validated_at" TIMESTAMPTZ, + "created_at" TIMESTAMPTZ DEFAULT now())''') + + existing = set() + cur.execute( + """SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s""", (table_name,), + ) + existing.update(row[0] for row in cur.fetchall()) + physical = dict(names) + physical[title_source] = 'title' + physical[abstract_source] = 'abstract' + physical['__provenance__'] = 'provenance' + assigned: set[str] = set(existing) + for source, column in list(physical.items()): + if source == '__provenance__' or column in {'title', 'abstract'}: + physical[source] = 'provenance' if source == '__provenance__' else column + assigned.add(column) + continue + # Reuse an existing column with the same normalized source name. + # Only genuinely new source columns receive a new physical name. + if column in existing: + physical[source] = column + assigned.add(column) + continue + candidate = column + suffix = 2 + while candidate in assigned: + tail = f'_{suffix}' + candidate = f'{column[:63 - len(tail)]}{tail}' + suffix += 1 + physical[source] = candidate + assigned.add(candidate) + for column in set(physical.values()) - existing: + if column not in existing: + cur.execute( + f'ALTER TABLE "{table_name}" ADD COLUMN "{column}" TEXT', + ) + + duplicate_keys: dict[tuple[str, str | None, str], int] = {} + if not include_duplicates: + cur.execute( + """SELECT identity_kind, identifier_namespace, normalized_value, citation_id + FROM citation_identities WHERE sr_id = %s""", + (sr_id,), + ) + for identity_kind, namespace, value, citation_id in cur.fetchall(): + duplicate_keys.setdefault( + (identity_kind, namespace, value), int(citation_id), + ) + + rows_to_insert: list[dict[str, Any]] = [] + duplicates_skipped = 0 + rows_merged = 0 + seen_keys: set[tuple[str, str | None, str]] = set() + seen_rows: dict[tuple[str, str | None, str], dict[str, Any]] = {} + for row in usable_rows: + fingerprint = _bibliographic_fingerprint( + row.get(title_source), row.get(abstract_source), + ) + identities = {('bibliographic_fingerprint', None, fingerprint)} + doi = next( + ( + _normalized_doi(value) for source, value in row.items() if re.sub( + r'[^a-z0-9]+', '', source.casefold(), + ) in {'doi', 'digitalobjectidentifier'} + ), '', + ) + if doi: + identities.add(('doi', None, doi)) + identities.update( + ('external_id', namespace, value) + for namespace, value in _row_external_ids(row) + ) + if not include_duplicates: + matched_id = next( + ( + duplicate_keys[key] + for key in identities if key in duplicate_keys + ), + None, + ) + if matched_id is not None: + update_pairs = [ + (source, column) for source, column in physical.items() + if source != '__provenance__' + and column not in {'id', 'created_at'} + and str(row.get(source) or '').strip() + ] + if update_pairs: + assignments = ', '.join( + f'"{column}" = COALESCE(NULLIF(%s, \'\'), "{column}")' + for _, column in update_pairs + ) + values = [ + str(row.get(source) or '') + for source, _ in update_pairs + ] + cur.execute( + f'UPDATE "{table_name}" SET {assignments} WHERE id = %s', + (*values, matched_id), + ) + rows_merged += 1 + duplicates_skipped += 1 + continue + if identities & seen_keys: + representative = next( + ( + seen_rows[key] + for key in identities if key in seen_rows + ), + None, + ) + if representative is not None: + for source, value in row.items(): + if str(value or '').strip() and not str( + representative.get(source) or '', + ).strip(): + representative[source] = value + rows_merged += 1 + duplicates_skipped += 1 + continue + rows_to_insert.append(row) + seen_keys.update(identities) + for key in identities: + seen_rows[key] = row + + batch_id = str(uuid.uuid4()) + cur.execute( + """INSERT INTO citation_import_batches + (id, sr_id, citation_table_name, source_type, source_metadata, + display_filename, content_sha256, schema_fingerprint, state, created_by) + VALUES (%s, %s, %s, 'file', %s::jsonb, %s, %s, %s, 'committed', %s)""", + ( + batch_id, sr_id, table_name, json.dumps( + {'commit_key': commit_key, 'source_format': source_format}, + ), + filename, content_sha, 'sha256:' + + hashlib.sha256( + json.dumps( + source_columns, + ).encode(), + ).hexdigest(), actor_id, + ), + ) + insert_columns = list(dict.fromkeys(physical.values())) + quoted = ', '.join(f'"{column}"' for column in insert_columns) + placeholders = ', '.join(['%s'] * len(insert_columns)) + inserted = 0 + for row in rows_to_insert: + values = [ + filename if source == '__provenance__' else + ( + str(row.get(source) or '') if row.get( + source, + ) is not None else None + ) + for source, column in physical.items() if column in insert_columns + ] + cur.execute( + f'INSERT INTO "{table_name}" ({quoted}) VALUES ({placeholders}) RETURNING id', values, + ) + citation_id = cur.fetchone()[0] + inserted += 1 + fingerprint = _bibliographic_fingerprint( + row.get(title_source), row.get(abstract_source), + ) + doi = next( + ( + _normalized_doi(value) for source, value in row.items() if re.sub( + r'[^a-z0-9]+', '', source.casefold(), + ) in {'doi', 'digitalobjectidentifier'} + ), '', + ) + identities = [('bibliographic_fingerprint', None, fingerprint)] + if doi: + identities.append(('doi', None, doi)) + identities.extend( + ('external_id', namespace, value) + for namespace, value in _row_external_ids(row) + ) + for kind, namespace, value in identities: + cur.execute( + """INSERT INTO citation_identities + (id, sr_id, citation_table_name, citation_id, identity_kind, + identifier_namespace, normalized_value, fingerprint_version) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT DO NOTHING""", + ( + str( + uuid.uuid4(), + ), sr_id, table_name, citation_id, kind, namespace, + value, 'v1' if kind == 'bibliographic_fingerprint' else None, + ), + ) + cur.execute( + """INSERT INTO citation_import_batch_memberships + (batch_id, sr_id, citation_table_name, citation_id, outcome, identity_snapshot) + VALUES (%s, %s, %s, %s, 'inserted', %s::jsonb)""", + ( + batch_id, sr_id, table_name, citation_id, json.dumps( + {'bibliographic_fingerprint': fingerprint}, + ), + ), + ) + cur.execute( + """UPDATE citation_import_batches SET inserted_count = %s, + committed_at = CURRENT_TIMESTAMP WHERE id = %s""", (inserted, batch_id), + ) + conn.commit() + return { + 'batch_id': batch_id, 'table_name': table_name, 'idempotent': False, + 'rows_inserted': inserted, 'rows_merged': rows_merged, + 'invalid_count': 0, 'warnings': [], + 'duplicates_skipped': duplicates_skipped, + } + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +citation_import_service = CitationImportService() diff --git a/backend/api/services/citation_legacy_adoption_service.py b/backend/api/services/citation_legacy_adoption_service.py new file mode 100644 index 00000000..2a134ace --- /dev/null +++ b/backend/api/services/citation_legacy_adoption_service.py @@ -0,0 +1,134 @@ +"""Safe, idempotent adoption of existing dynamic citation tables.""" +from __future__ import annotations + +import re +import uuid +from typing import Any + +try: + from .postgres_auth import postgres_server +# Optional psycopg2 dependency in unit-test tooling. +except ModuleNotFoundError: + postgres_server = None + + +_LEGACY_BATCH_NAMESPACE = uuid.UUID('c6e5bd06-a30c-47bb-a6cf-7b8a993f7925') +_IDENT_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]{0,62}$') + + +def _validate_table_name(name: str) -> str: + if not isinstance(name, str) or not _IDENT_RE.match(name): + raise ValueError(f'Invalid citation_table_name: {name!r}') + return name + + +class CitationLegacyAdoptionService: + """Backfill immutable provenance without altering legacy citation rows.""" + + def __init__(self, connection_provider=None): + self.connection_provider = connection_provider + + def _connection(self): + provider = self.connection_provider or postgres_server + if provider is None: + raise RuntimeError('PostgreSQL driver is not available') + return provider.conn + + @staticmethod + def legacy_batch_id(sr_id: str, citation_table_name: str) -> str: + return str( + uuid.uuid5( + _LEGACY_BATCH_NAMESPACE, f'{sr_id}\x1f{citation_table_name}', + ), + ) + + def adopt( + self, sr_id: str, citation_table_name: str, actor_id: str, + *, dry_run: bool = False, + ) -> dict[str, Any]: + """Adopt one configured table, retaining its name and citation IDs. + + The SR-to-table relationship and physical ``id`` column are verified in + this transaction before provenance is written. Repeated calls only add + missing membership rows and always return the deterministic batch ID. + """ + table_name = _validate_table_name(citation_table_name) + if not sr_id or not actor_id: + raise ValueError('sr_id and actor_id are required') + + conn = self._connection() + cur = conn.cursor() + batch_id = self.legacy_batch_id(sr_id, table_name) + try: + cur.execute( + """SELECT 1 FROM systematic_reviews + WHERE id = %s AND screening_db ->> 'table_name' = %s""", + (sr_id, table_name), + ) + if not cur.fetchone(): + raise ValueError( + 'Citation table is not configured for this systematic review', + ) + cur.execute( + """SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + AND column_name = 'id'""", + (table_name,), + ) + if not cur.fetchone(): + raise ValueError( + 'Citation table does not exist or has no id column', + ) + cur.execute(f'SELECT COUNT(*) FROM "{table_name}"') + citation_count = int(cur.fetchone()[0]) + if dry_run: + conn.rollback() + return { + 'batch_id': batch_id, 'citation_count': citation_count, + 'memberships_created': 0, 'dry_run': True, + } + + cur.execute( + """INSERT INTO citation_import_batches + (id, sr_id, citation_table_name, source_type, source_metadata, + display_filename, state, inserted_count, created_by, committed_at) + VALUES (%s, %s, %s, 'legacy_initial_dataset', + '{"source":"legacy_unknown"}'::jsonb, + 'Legacy initial dataset', 'committed', 0, %s, + CURRENT_TIMESTAMP) + ON CONFLICT (id) DO NOTHING""", + (batch_id, sr_id, table_name, actor_id), + ) + cur.execute( + f"""INSERT INTO citation_import_batch_memberships + (batch_id, sr_id, citation_table_name, citation_id, outcome) + SELECT %s, %s, %s, id, 'inserted' FROM \"{table_name}\" + ON CONFLICT (batch_id, citation_id) DO NOTHING""", + (batch_id, sr_id, table_name), + ) + memberships_created = cur.rowcount + cur.execute( + """INSERT INTO citation_audit_events + (id, sr_id, citation_table_name, batch_id, event_type, actor_id, details) + VALUES (%s, %s, %s, %s, 'legacy_table_adopted', %s, + jsonb_build_object('citation_count', %s)) + ON CONFLICT (batch_id, event_type) WHERE event_type = 'legacy_table_adopted' + DO NOTHING""", + ( + str(uuid.uuid4()), sr_id, table_name, + batch_id, actor_id, citation_count, + ), + ) + conn.commit() + return { + 'batch_id': batch_id, 'citation_count': citation_count, + 'memberships_created': memberships_created, 'dry_run': False, + } + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +citation_legacy_adoption_service = CitationLegacyAdoptionService() diff --git a/backend/api/services/citation_workspace_preferences_service.py b/backend/api/services/citation_workspace_preferences_service.py new file mode 100644 index 00000000..1fd6c600 --- /dev/null +++ b/backend/api/services/citation_workspace_preferences_service.py @@ -0,0 +1,69 @@ +"""Persistence for non-sensitive, per-user citation workspace presentation state.""" +from __future__ import annotations + +import json +import re + +from .postgres_auth import postgres_server + +_IDENTIFIER = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') + + +class CitationWorkspacePreferencesService: + def _validate_table_name(self, table_name: str) -> str: + if not _IDENTIFIER.fullmatch(table_name or ''): + raise ValueError('Invalid citation table name') + return table_name + + def get_columns(self, sr_id: str, table_name: str, user_id: str) -> list[str] | None: + table_name = self._validate_table_name(table_name) + conn = postgres_server.conn + cur = conn.cursor() + try: + try: + cur.execute( + '''SELECT columns FROM citation_workspace_column_preferences + WHERE sr_id = %s AND citation_table_name = %s AND user_id = %s''', + (sr_id, table_name, user_id), + ) + except Exception: + # Older databases may not have migration 005 yet. A read + # should fall back to defaults, and must not leave the shared + # connection in PostgreSQL's aborted-transaction state. + conn.rollback() + return None + row = cur.fetchone() + if not row: + return None + columns = row[0] + if isinstance(columns, str): + columns = json.loads(columns) + return [column for column in columns if isinstance(column, str)] if isinstance(columns, list) else None + finally: + cur.close() + + def save_columns(self, sr_id: str, table_name: str, user_id: str, columns: list[str]) -> list[str]: + table_name = self._validate_table_name(table_name) + if not columns or any(not isinstance(column, str) for column in columns): + raise ValueError('At least one visible column is required') + conn = postgres_server.conn + cur = conn.cursor() + try: + cur.execute( + '''INSERT INTO citation_workspace_column_preferences + (sr_id, citation_table_name, user_id, columns) + VALUES (%s, %s, %s, %s::jsonb) + ON CONFLICT (sr_id, citation_table_name, user_id) DO UPDATE + SET columns = EXCLUDED.columns, updated_at = CURRENT_TIMESTAMP''', + (sr_id, table_name, user_id, json.dumps(columns)), + ) + conn.commit() + return columns + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +citation_workspace_preferences_service = CitationWorkspacePreferencesService() diff --git a/backend/api/services/review_schema_service.py b/backend/api/services/review_schema_service.py index 02bac51a..98036944 100644 --- a/backend/api/services/review_schema_service.py +++ b/backend/api/services/review_schema_service.py @@ -112,6 +112,36 @@ def migrate(self, app_version: str | None = None) -> dict[str, Any]: cur.close() return {'applied': applied, **self.verify_schema(connection=conn)} + def refresh_checksums(self) -> dict[str, Any]: + """Rebaseline recorded checksums to the checked-in migration files. + + This is an explicit repair operation for databases affected by a + formatter that modified migration files. Normal migration execution + continues to reject checksum drift. + """ + conn = self._provider().conn + refreshed: list[str] = [] + cur = conn.cursor() + try: + self._ensure_migration_table(cur) + for path in migration_files(): + version = path.stem + checksum = migration_checksum(path.read_text(encoding='utf-8')) + cur.execute( + f'''UPDATE {MIGRATION_TABLE} + SET checksum = %s WHERE version = %s''', + (checksum, version), + ) + if cur.rowcount: + refreshed.append(version) + conn.commit() + return {'refreshed': refreshed} + except Exception: + conn.rollback() + raise + finally: + cur.close() + # Backward-compatible name for callers of the original opt-in bootstrap. def ensure_schema(self, app_version: str | None = None) -> dict[str, Any]: return self.migrate(app_version) diff --git a/backend/api/services/screening_eligibility_service.py b/backend/api/services/screening_eligibility_service.py index ac300598..9d0927ba 100644 --- a/backend/api/services/screening_eligibility_service.py +++ b/backend/api/services/screening_eligibility_service.py @@ -25,8 +25,8 @@ def _answer_object(value: Any) -> dict[str, Any]: return {} -def selected_answer(row: dict[str, Any], question: str) -> str | None: - """Return the effective answer using human-over-main-AI precedence. +def selected_answer(row: dict[str, Any], question: str, stage: str) -> str | None: + """Return the human answer, falling back to the stage's LLM answer. The critical agent is advisory: disagreements are routed to human review but do not change the main screening agent's progression decision. @@ -34,17 +34,20 @@ def selected_answer(row: dict[str, Any], question: str) -> str | None: core = snake_case(question, max_len=56) if not core: return None - for column in (f'human_{core}', f'llm_{core}'): + columns = ( + f'human_{stage}_{core}', + f'llm_{stage}_{core}', + f'llm_{core}', + ) + for column in columns: selected = _answer_object(row.get(column)).get('selected') - if isinstance(selected, str): - selected = selected.strip() - if selected is not None and selected != '': - return str(selected) + if isinstance(selected, str) and selected.strip(): + return selected.strip() return None def compute_stage_decision( - row: dict[str, Any], questions: list[str], + row: dict[str, Any], questions: list[str], stage: str, ) -> str: """Compute include/exclude for one stage from current criterion answers. @@ -57,7 +60,7 @@ def compute_stage_decision( if not valid_questions: return 'undecided' for question in valid_questions: - selected = selected_answer(row, question) + selected = selected_answer(row, question, stage) if selected is None or 'exclude' in selected.lower(): return 'exclude' return 'include' @@ -76,10 +79,22 @@ def compute_screening_decisions( l2_questions = l2.get('questions') if isinstance( l2.get('questions'), list, ) else [] - return ( - compute_stage_decision(row, l1_questions), - compute_stage_decision(row, list(l1_questions) + list(l2_questions)), - ) + l1_decision = compute_stage_decision(row, l1_questions, 'l1') + l2_decision = 'undecided' + if l1_questions or l2_questions: + l2_decision = 'include' + for question, question_stage in [ + *((q, 'l1') for q in l1_questions), + *((q, 'l2') for q in l2_questions), + ]: + selected = selected_answer(row, question, question_stage) + if selected is None: + l2_decision = 'exclude' + break + if 'exclude' in selected.lower(): + l2_decision = 'exclude' + break + return l1_decision, l2_decision class ScreeningEligibilityService: diff --git a/backend/api/services/sr_db_service.py b/backend/api/services/sr_db_service.py index 2f0c205e..f3dae915 100644 --- a/backend/api/services/sr_db_service.py +++ b/backend/api/services/sr_db_service.py @@ -122,6 +122,223 @@ def ensure_table_exists(self) -> None: if conn: pass + @staticmethod + def _legacy_member_id(doc: dict[str, Any]) -> str | None: + return doc.get('owner_email') or doc.get('owner_id') + + def _membership_table_exists(self, cur) -> bool: + cur.execute( + "SELECT to_regclass('public.systematic_review_memberships')", + ) + return bool(cur.fetchone()[0]) + + def get_member_role(self, sr_id: str, member_id: str) -> str | None: + """Return a normalized role, falling back for unmigrated legacy reviews.""" + if not member_id: + return None + conn = postgres_server.conn + cur = conn.cursor() + try: + if self._membership_table_exists(cur): + cur.execute( + 'SELECT role FROM systematic_review_memberships WHERE sr_id = %s AND member_id = %s', + (sr_id, member_id), + ) + row = cur.fetchone() + if row: + return row[0] + doc = self.get_systematic_review(sr_id, ignore_visibility=True) + if not doc: + return None + if member_id == self._legacy_member_id(doc) or member_id == doc.get('owner_id'): + return 'owner' + return 'member' if member_id in (doc.get('users') or []) else None + finally: + cur.close() + + def user_is_sr_owner(self, sr_id: str, user_id: str) -> bool: + return self.get_member_role(sr_id, user_id) == 'owner' + + def require_review_role(self, sr_id: str, member_id: str, allowed_roles: set[str]) -> str: + """Require one of the supplied normalized roles, with legacy fallback.""" + role = self.get_member_role(sr_id, member_id) + if role not in allowed_roles: + required = ' or '.join(sorted(allowed_roles)) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f'Requires review role: {required}', + ) + return role + + def require_review_owner(self, sr_id: str, member_id: str) -> None: + """Require an owner or co-owner for a review-scoped owner action.""" + self.require_review_role(sr_id, member_id, {'owner'}) + + def list_members(self, sr_id: str) -> list[dict[str, Any]]: + conn = postgres_server.conn + cur = conn.cursor() + try: + if self._membership_table_exists(cur): + cur.execute( + """SELECT member_id, role, added_by, created_at, updated_at + FROM systematic_review_memberships WHERE sr_id = %s + ORDER BY CASE role WHEN 'owner' THEN 0 ELSE 1 END, member_id""", (sr_id,), + ) + return [{ + 'member_id': row[0], 'role': row[1], 'added_by': row[2], + 'created_at': row[3].isoformat() if hasattr(row[3], 'isoformat') else str(row[3]), + 'updated_at': row[4].isoformat() if hasattr(row[4], 'isoformat') else str(row[4]), + } + for row in cur.fetchall()] + doc = self.get_systematic_review(sr_id, ignore_visibility=True) + if not doc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', + ) + owner = self._legacy_member_id(doc) + return [{ + 'member_id': value, 'role': 'owner' if value == owner else 'member', + 'added_by': owner, 'created_at': None, 'updated_at': None, + } + for value in sorted(set(doc.get('users') or []) | ({owner} if owner else set()))] + finally: + cur.close() + + def _require_owner(self, sr_id: str, requester_id: str) -> None: + try: + self.require_review_owner(sr_id, requester_id) + except HTTPException as exc: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail='Only an owner may manage review members', + ) from exc + + def _sync_legacy_users(self, cur, sr_id: str) -> None: + cur.execute( + 'SELECT member_id FROM systematic_review_memberships WHERE sr_id = %s ORDER BY member_id', ( + sr_id, + ), + ) + users = [row[0] for row in cur.fetchall()] + cur.execute( + 'UPDATE systematic_reviews SET users = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s', + (json.dumps(users), sr_id), + ) + + def _audit_membership( + self, cur, sr_id: str, member_id: str, event_type: str, + actor_id: str, old_role: str | None = None, new_role: str | None = None, + ) -> None: + cur.execute( + """INSERT INTO systematic_review_membership_audit_events + (id, sr_id, member_id, event_type, actor_id, old_role, new_role) + VALUES (%s, %s, %s, %s, %s, %s, %s)""", + ( + str(uuid.uuid4()), sr_id, member_id, + event_type, actor_id, old_role, new_role, + ), + ) + + def set_member_role(self, sr_id: str, member_id: str, role: str, requester_id: str) -> dict[str, Any]: + """Add/promote/demote a member in one owner-authorized transaction.""" + if role not in ('owner', 'member') or not member_id: + raise ValueError( + 'member_id and role (owner or member) are required', + ) + self._require_owner(sr_id, requester_id) + conn = postgres_server.conn + cur = conn.cursor() + try: + if not self._membership_table_exists(cur): + raise RuntimeError( + 'Review membership schema has not been migrated', + ) + cur.execute( + 'SELECT role FROM systematic_review_memberships WHERE sr_id = %s AND member_id = %s FOR UPDATE', + (sr_id, member_id), + ) + existing = cur.fetchone() + old_role = existing[0] if existing else None + if old_role == role: + conn.commit() + return {'member_id': member_id, 'role': role, 'changed': False} + if old_role == 'owner' and role == 'member': + cur.execute( + "SELECT COUNT(*) FROM systematic_review_memberships WHERE sr_id = %s AND role = 'owner'", (sr_id,), + ) + if int(cur.fetchone()[0]) <= 1: + raise ValueError( + 'At least one owner must remain on the systematic review', + ) + if existing: + cur.execute( + 'UPDATE systematic_review_memberships SET role = %s, updated_at = CURRENT_TIMESTAMP WHERE sr_id = %s AND member_id = %s', + (role, sr_id, member_id), + ) + event = 'role_changed' + else: + cur.execute( + """INSERT INTO systematic_review_memberships (sr_id, member_id, role, added_by) + VALUES (%s, %s, %s, %s)""", (sr_id, member_id, role, requester_id), + ) + event = 'member_added' + self._sync_legacy_users(cur, sr_id) + self._audit_membership( + cur, sr_id, member_id, + event, requester_id, old_role, role, + ) + conn.commit() + return {'member_id': member_id, 'role': role, 'changed': True} + except Exception: + conn.rollback() + raise + finally: + cur.close() + + def remove_member(self, sr_id: str, member_id: str, requester_id: str) -> dict[str, Any]: + """Remove a member while rejecting removal of the final owner.""" + self._require_owner(sr_id, requester_id) + conn = postgres_server.conn + cur = conn.cursor() + try: + if not self._membership_table_exists(cur): + raise RuntimeError( + 'Review membership schema has not been migrated', + ) + cur.execute( + 'SELECT role FROM systematic_review_memberships WHERE sr_id = %s AND member_id = %s FOR UPDATE', + (sr_id, member_id), + ) + row = cur.fetchone() + if not row: + conn.commit() + return {'member_id': member_id, 'removed': False} + if row[0] == 'owner': + cur.execute( + "SELECT COUNT(*) FROM systematic_review_memberships WHERE sr_id = %s AND role = 'owner'", (sr_id,), + ) + if int(cur.fetchone()[0]) <= 1: + raise ValueError( + 'At least one owner must remain on the systematic review', + ) + cur.execute( + 'DELETE FROM systematic_review_memberships WHERE sr_id = %s AND member_id = %s', ( + sr_id, member_id, + ), + ) + self._sync_legacy_users(cur, sr_id) + self._audit_membership( + cur, sr_id, member_id, + 'member_removed', requester_id, row[0], None, + ) + conn.commit() + return {'member_id': member_id, 'removed': True} + except Exception: + conn.rollback() + raise + finally: + cur.close() + def build_criteria_parsed(self, criteria_obj: dict[str, Any] | None) -> dict[str, Any]: """ Port of the _build_criteria_parsed helper - returns a mapping containing @@ -279,6 +496,13 @@ def create_systematic_review( now, ), ) + if self._membership_table_exists(cur): + member_id = owner_email or owner_id + cur.execute( + """INSERT INTO systematic_review_memberships (sr_id, member_id, role, added_by) + VALUES (%s, %s, 'owner', %s) ON CONFLICT (sr_id, member_id) DO NOTHING""", + (sr_id, member_id, member_id), + ) conn.commit() @@ -338,74 +562,10 @@ def add_user(self, sr_id: str, target_user_id: str, requester_id: str) -> dict[s Returns a dict with update result metadata. """ - sr = self.get_systematic_review(sr_id) - if not sr or not sr.get('visible', True): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', - ) - - # Check permission - has_perm = self.user_has_sr_permission(sr_id, requester_id) - if not has_perm: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Not authorized to modify this systematic review', - ) - - conn = None - try: - conn = postgres_server.conn - cur = conn.cursor() - - # Get current users array - cur.execute( - 'SELECT users FROM systematic_reviews WHERE id = %s', (sr_id,), - ) - row = cur.fetchone() - if not row: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', - ) - - users = row[0] if row[0] else [] - if isinstance(users, str): - users = json.loads(users) - - # Add user if not already present - if target_user_id not in users: - users.append(target_user_id) - - # Update - now = datetime.utcnow().isoformat() - cur.execute( - 'UPDATE systematic_reviews SET users = %s, updated_at = %s WHERE id = %s', - (json.dumps(users), now, sr_id), - ) - modified_count = cur.rowcount - conn.commit() - - return {'matched_count': 1, 'modified_count': modified_count, 'added_user_id': target_user_id} - - except HTTPException: - try: - if conn: - conn.rollback() - except Exception: - pass - raise - except Exception as e: - try: - if conn: - conn.rollback() - except Exception: - pass - logger.exception(f"Failed to add user to SR: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to add user: {e}", - ) - finally: - if conn: - pass + result = self.set_member_role( + sr_id, target_user_id, 'member', requester_id, + ) + return {'matched_count': 1, 'modified_count': int(result['changed']), 'added_user_id': target_user_id} def remove_user(self, sr_id: str, target_user_id: str, requester_id: str) -> dict[str, Any]: """ @@ -413,80 +573,8 @@ def remove_user(self, sr_id: str, target_user_id: str, requester_id: str) -> dic Enforces requester permissions (must be a member or owner). """ - sr = self.get_systematic_review(sr_id) - if not sr or not sr.get('visible', True): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', - ) - - # Check permission - has_perm = self.user_has_sr_permission(sr_id, requester_id) - if not has_perm: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Not authorized to modify this systematic review', - ) - - if target_user_id == sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail='Cannot remove the owner from the systematic review', - ) - - conn = None - try: - conn = postgres_server.conn - cur = conn.cursor() - - # Get current users array - cur.execute( - 'SELECT users FROM systematic_reviews WHERE id = %s', (sr_id,), - ) - row = cur.fetchone() - if not row: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', - ) - - users = row[0] if row[0] else [] - if isinstance(users, str): - users = json.loads(users) - - # Remove user if present - if target_user_id in users: - users.remove(target_user_id) - - # Update - now = datetime.utcnow().isoformat() - cur.execute( - 'UPDATE systematic_reviews SET users = %s, updated_at = %s WHERE id = %s', - (json.dumps(users), now, sr_id), - ) - modified_count = cur.rowcount - conn.commit() - - return {'matched_count': 1, 'modified_count': modified_count, 'removed_user_id': target_user_id} - - except HTTPException: - try: - if conn: - conn.rollback() - except Exception: - pass - raise - except Exception as e: - try: - if conn: - conn.rollback() - except Exception: - pass - logger.exception(f"Failed to remove user from SR: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to remove user: {e}", - ) - finally: - if conn: - pass + result = self.remove_member(sr_id, target_user_id, requester_id) + return {'matched_count': 1, 'modified_count': int(result['removed']), 'removed_user_id': target_user_id} def user_has_sr_permission(self, sr_id: str, user_id: str) -> bool: """ @@ -500,10 +588,7 @@ def user_has_sr_permission(self, sr_id: str, user_id: str) -> bool: if not doc: return False - users = doc.get('users', []) - if user_id in users or user_id == doc.get('owner_id'): - return True - return False + return self.get_member_role(sr_id, user_id) is not None def update_criteria(self, sr_id: str, criteria_obj: dict[str, Any], criteria_str: str, requester_id: str) -> dict[str, Any]: """ @@ -788,11 +873,7 @@ def set_visibility(self, sr_id: str, visible: bool, requester_id: str) -> dict[s status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', ) - if requester_id != sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Only the owner may change visibility of this systematic review', - ) + self.require_review_owner(sr_id, requester_id) conn = None try: @@ -848,11 +929,7 @@ def hard_delete_systematic_review(self, sr_id: str, requester_id: str) -> dict[s status_code=status.HTTP_404_NOT_FOUND, detail='Systematic review not found', ) - if requester_id != sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Only the owner may hard-delete this systematic review', - ) + self.require_review_owner(sr_id, requester_id) conn = None try: diff --git a/backend/api/sr/router.py b/backend/api/sr/router.py index 51132be1..84c632e3 100644 --- a/backend/api/sr/router.py +++ b/backend/api/sr/router.py @@ -224,6 +224,11 @@ class RemoveUserRequest(BaseModel): user_id: str | None = None +class ReviewMemberRoleRequest(BaseModel): + member_id: str + role: str + + @router.post('/{sr_id}/add-user') async def add_user_to_systematic_review( sr_id: str, @@ -258,7 +263,7 @@ async def add_user_to_systematic_review( ) try: - res = await run_in_threadpool(srdb_service.add_user, sr_id, target_user_id, current_user.get('id')) + res = await run_in_threadpool(srdb_service.add_user, sr_id, target_user_id, current_user.get('email')) except HTTPException: raise except Exception as e: @@ -301,15 +306,8 @@ async def remove_user_from_systematic_review( status_code=status.HTTP_400_BAD_REQUEST, detail='Missing data user_email', ) - # do not allow removing the owner - if target_user_id == sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail='Cannot remove the owner from the systematic review', - ) - try: - res = await run_in_threadpool(srdb_service.remove_user, sr_id, target_user_id, current_user.get('id')) + res = await run_in_threadpool(srdb_service.remove_user, sr_id, target_user_id, current_user.get('email')) except HTTPException: raise except Exception as e: @@ -320,6 +318,47 @@ async def remove_user_from_systematic_review( return {'status': 'success', 'sr_id': sr_id, 'removed_user_id': target_user_id, 'matched_count': res.get('matched_count'), 'modified_count': res.get('modified_count')} +@router.get('/{sr_id}/members') +async def list_review_members(sr_id: str, current_user: dict[str, Any] = Depends(get_current_active_user)): + """Return role-aware membership data only to existing review collaborators.""" + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + return {'members': await run_in_threadpool(srdb_service.list_members, sr_id)} + + +@router.put('/{sr_id}/members/{member_id}') +async def set_review_member_role( + sr_id: str, member_id: str, payload: ReviewMemberRoleRequest, + current_user: dict[str, Any] = Depends(get_current_active_user), +): + if payload.member_id != member_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='member_id must match the URL', + ) + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + try: + result = await run_in_threadpool(srdb_service.set_member_role, sr_id, member_id, payload.role, current_user.get('email')) + return {'status': 'success', 'sr_id': sr_id, **result} + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + + +@router.delete('/{sr_id}/members/{member_id}') +async def delete_review_member( + sr_id: str, member_id: str, current_user: dict[str, Any] = Depends(get_current_active_user), +): + await load_sr_and_check(sr_id, current_user, srdb_service, require_screening=False) + try: + result = await run_in_threadpool(srdb_service.remove_member, sr_id, member_id, current_user.get('email')) + return {'status': 'success', 'sr_id': sr_id, **result} + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), + ) + + @router.get('/mine', response_model=list[SystematicReviewRead]) async def list_systematic_reviews_for_user( current_user: dict[str, Any] = Depends(get_current_active_user), @@ -476,14 +515,34 @@ async def save_criteria_config( status_code=status.HTTP_409_CONFLICT, detail='This SR already has screening data. Pass force=true to confirm criteria invalidation.', ) + previous_criteria = review.get('criteria') + stored = None + if isinstance(previous_criteria, dict): + try: + stored = criteria_configuration_service.normalize( + previous_criteria, + ) + except (KeyError, ValueError): + stored = None + if stored: + _require_migration_confirmation(stored, payload.migration_fingerprint) try: available = await run_in_threadpool(discover_citation_fields, review) available_names = { str(field.get('name')) for field in available.get('fields', []) } + criteria_payload = payload.criteria.model_dump( + mode='json', exclude_none=True, + ) + citation_fields = criteria_payload.setdefault('citation_fields', {}) + citation_fields.setdefault('title', 'Title') + citation_fields.setdefault('abstract', 'Abstract') + criteria_for_validation = CriteriaConfigV2.model_validate( + criteria_payload, + ) required_errors = criteria_configuration_service.required_citation_field_errors( - payload.criteria, available_names, + criteria_for_validation, available_names, ) if required_errors: raise HTTPException( @@ -491,17 +550,11 @@ async def save_criteria_config( 'errors': required_errors, }, ) - previous_criteria = review.get('criteria') - stored = None - if isinstance(previous_criteria, dict): - stored = criteria_configuration_service.normalize( - previous_criteria, - ) except (KeyError, ValueError): - stored = None - if stored: - _require_migration_confirmation(stored, payload.migration_fingerprint) - criteria = payload.criteria.model_dump(mode='json', exclude_none=True) + raise + criteria = criteria_for_validation.model_dump( + mode='json', exclude_none=True, + ) criteria_yaml = criteria_configuration_service.export_yaml( payload.criteria, ) @@ -867,12 +920,8 @@ async def delete_systematic_review(sr_id: str, current_user: dict[str, Any] = De detail=f"Failed to load systematic review: {e}", ) - requester_id = current_user.get('id') - if requester_id != sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Only the owner may delete this systematic review', - ) + requester_id = current_user.get('email') + srdb_service.require_review_owner(sr_id, requester_id) try: res = await run_in_threadpool(srdb_service.soft_delete_systematic_review, sr_id, requester_id) @@ -905,12 +954,8 @@ async def undelete_systematic_review(sr_id: str, current_user: dict[str, Any] = detail=f"Failed to load systematic review: {e}", ) - requester_id = current_user.get('id') - if requester_id != sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Only the owner may undelete this systematic review', - ) + requester_id = current_user.get('email') + srdb_service.require_review_owner(sr_id, requester_id) try: res = await run_in_threadpool(srdb_service.undelete_systematic_review, sr_id, requester_id) @@ -946,12 +991,8 @@ async def hard_delete_systematic_review(sr_id: str, current_user: dict[str, Any] detail=f"Failed to load systematic review: {e}", ) - requester_id = current_user.get('id') - if requester_id != sr.get('owner_id'): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail='Only the owner may hard-delete this systematic review', - ) + requester_id = current_user.get('email') + srdb_service.require_review_owner(sr_id, requester_id) # Attempt to perform screening resources cleanup prior to deleting the SR document. cleanup_result = None diff --git a/backend/deploy.sh b/backend/deploy.sh index af7f838c..79250aa3 100755 --- a/backend/deploy.sh +++ b/backend/deploy.sh @@ -158,17 +158,15 @@ sleep 10 # Start main API echo -e "${BLUE}🌐 Starting main API...${NC}" -# Read the feature flag from the compose environment (.env), not only from -# the shell running this script. The migration job is deliberately explicit; -# the API container itself only verifies the schema at startup. +# Migrations are additive and include the citation workspace schema, which is +# required by the running API (including when multi-reviewer UI is disabled). +# Apply them explicitly before starting the API so existing databases receive +# new tables such as citation_workspace_column_preferences. docker compose run --rm --no-deps api sh -lc '\ - if [ "$$ENABLE_MULTI_REVIEWER_SCHEMA" = "true" ]; then \ - echo "🧱 Applying database migrations..."; \ - python -m api.migrations migrate; \ - echo "✅ Database migrations applied"; \ - else \ - echo "ℹ️ Multi-reviewer migrations disabled"; \ - fi' + set -e; \ + echo "🧱 Applying database migrations..."; \ + python -m api.migrations migrate; \ + echo "✅ Database migrations applied"' if [ "$DEV" = true ]; then docker compose up -d api --remove-orphans else diff --git a/backend/migrations/002_citation_workspace_schema.sql b/backend/migrations/002_citation_workspace_schema.sql new file mode 100644 index 00000000..174588d8 --- /dev/null +++ b/backend/migrations/002_citation_workspace_schema.sql @@ -0,0 +1,154 @@ +-- CAN-SR citation workspace metadata. +-- Citation rows live in dynamically named tables. These shared tables retain +-- {sr_id, citation_table_name, citation_id} references without conventional +-- foreign keys; services validate that scope and row existence transactionally. + +CREATE TABLE IF NOT EXISTS citation_import_batches ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + source_type TEXT NOT NULL CHECK (source_type IN ('file', 'legacy_initial_dataset')), + source_metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + display_filename TEXT, + content_sha256 TEXT, + schema_fingerprint TEXT, + mapping_decision JSONB, + state TEXT NOT NULL CHECK (state IN ('committed', 'failed', 'tombstoned')), + inserted_count INTEGER NOT NULL DEFAULT 0 CHECK (inserted_count >= 0), + existing_exact_match_count INTEGER NOT NULL DEFAULT 0 CHECK (existing_exact_match_count >= 0), + invalid_count INTEGER NOT NULL DEFAULT 0 CHECK (invalid_count >= 0), + error_summary TEXT, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + committed_at TIMESTAMPTZ, + tombstoned_at TIMESTAMPTZ, + tombstoned_by TEXT, + UNIQUE (sr_id, citation_table_name, content_sha256) +); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_citation_legacy_initial_batch + ON citation_import_batches (sr_id, citation_table_name) + WHERE source_type = 'legacy_initial_dataset'; +CREATE INDEX IF NOT EXISTS ix_citation_import_batches_review + ON citation_import_batches (sr_id, citation_table_name, created_at DESC); + +CREATE TABLE IF NOT EXISTS citation_import_batch_memberships ( + batch_id UUID NOT NULL REFERENCES citation_import_batches(id), + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + citation_id BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('inserted', 'existing_exact_match')), + identity_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (batch_id, citation_id) +); +CREATE INDEX IF NOT EXISTS ix_citation_batch_memberships_citation + ON citation_import_batch_memberships (sr_id, citation_table_name, citation_id); + +CREATE TABLE IF NOT EXISTS citation_import_previews ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT, + source_sha256 TEXT NOT NULL, + schema_fingerprint TEXT NOT NULL, + proposed_mapping JSONB NOT NULL, + validation_report JSONB NOT NULL, + encrypted_staging_locator TEXT NOT NULL, + staging_expires_at TIMESTAMPTZ NOT NULL, + commit_key TEXT NOT NULL, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + committed_batch_id UUID REFERENCES citation_import_batches(id), + cancelled_at TIMESTAMPTZ, + UNIQUE (sr_id, created_by, commit_key) +); +CREATE INDEX IF NOT EXISTS ix_citation_import_previews_expiry + ON citation_import_previews (staging_expires_at); + +CREATE TABLE IF NOT EXISTS citation_identities ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + citation_id BIGINT NOT NULL, + identity_kind TEXT NOT NULL CHECK (identity_kind IN ('doi', 'external_id', 'bibliographic_fingerprint')), + identifier_namespace TEXT, + normalized_value TEXT NOT NULL, + fingerprint_version TEXT, + diagnostics JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS uq_citation_identity_per_citation + ON citation_identities ( + sr_id, citation_table_name, citation_id, identity_kind, + COALESCE(identifier_namespace, ''), normalized_value + ); +CREATE UNIQUE INDEX IF NOT EXISTS uq_citation_identity_doi + ON citation_identities (sr_id, citation_table_name, normalized_value) + WHERE identity_kind = 'doi' AND normalized_value <> ''; +CREATE UNIQUE INDEX IF NOT EXISTS uq_citation_identity_external_id + ON citation_identities (sr_id, citation_table_name, identifier_namespace, normalized_value) + WHERE identity_kind = 'external_id' AND normalized_value <> ''; +CREATE INDEX IF NOT EXISTS ix_citation_identities_citation + ON citation_identities (sr_id, citation_table_name, citation_id); + +CREATE TABLE IF NOT EXISTS citation_audit_events ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + citation_id BIGINT, + batch_id UUID REFERENCES citation_import_batches(id), + event_type TEXT NOT NULL, + actor_id TEXT NOT NULL, + details JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS ix_citation_audit_events_scope + ON citation_audit_events (sr_id, citation_table_name, citation_id, created_at DESC); +CREATE UNIQUE INDEX IF NOT EXISTS uq_citation_legacy_adoption_audit + ON citation_audit_events (batch_id, event_type) + WHERE event_type = 'legacy_table_adopted'; + +CREATE TABLE IF NOT EXISTS citation_deletion_tombstones ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + citation_id BIGINT NOT NULL, + deleted_by TEXT NOT NULL, + deleted_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + impact_snapshot JSONB NOT NULL, + UNIQUE (sr_id, citation_table_name, citation_id) +); + +CREATE TABLE IF NOT EXISTS citation_deduplication_configs ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + selected_columns JSONB NOT NULL, + look_ahead SMALLINT NOT NULL CHECK (look_ahead BETWEEN 1 AND 12), + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS citation_duplicate_candidates ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + left_citation_id BIGINT NOT NULL, + right_citation_id BIGINT NOT NULL, + config_id UUID REFERENCES citation_deduplication_configs(id), + score NUMERIC(6, 5), + evidence JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL CHECK (status IN ('open', 'resolved', 'dismissed')), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (left_citation_id < right_citation_id), + UNIQUE (sr_id, citation_table_name, left_citation_id, right_citation_id) +); + +CREATE TABLE IF NOT EXISTS citation_duplicate_resolutions ( + id UUID PRIMARY KEY, + candidate_id UUID NOT NULL REFERENCES citation_duplicate_candidates(id), + resolution TEXT NOT NULL CHECK (resolution IN ('not_duplicate', 'related', 'merge_deferred')), + decided_by TEXT NOT NULL, + rationale TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/backend/migrations/003_citation_import_preview_mapping_decisions.sql b/backend/migrations/003_citation_import_preview_mapping_decisions.sql new file mode 100644 index 00000000..b626c781 --- /dev/null +++ b/backend/migrations/003_citation_import_preview_mapping_decisions.sql @@ -0,0 +1,4 @@ +-- Durable, explicit import-preview mapping decisions. This is separate from +-- proposed_mapping so parser suggestions and user approval remain auditable. +ALTER TABLE citation_import_previews + ADD COLUMN IF NOT EXISTS mapping_decision JSONB; diff --git a/backend/migrations/004_systematic_review_memberships.sql b/backend/migrations/004_systematic_review_memberships.sql new file mode 100644 index 00000000..3a4f7076 --- /dev/null +++ b/backend/migrations/004_systematic_review_memberships.sql @@ -0,0 +1,42 @@ +-- Normalized, review-scoped roles. Existing users JSONB remains synchronized +-- during the compatibility period for legacy clients and queries. +CREATE TABLE IF NOT EXISTS systematic_review_memberships ( + sr_id TEXT NOT NULL REFERENCES systematic_reviews(id) ON DELETE CASCADE, + member_id TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('owner', 'member')), + added_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (sr_id, member_id) +); +CREATE INDEX IF NOT EXISTS ix_systematic_review_memberships_member + ON systematic_review_memberships (member_id, sr_id); +CREATE INDEX IF NOT EXISTS ix_systematic_review_memberships_owner + ON systematic_review_memberships (sr_id, role) WHERE role = 'owner'; + +CREATE TABLE IF NOT EXISTS systematic_review_membership_audit_events ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL REFERENCES systematic_reviews(id) ON DELETE CASCADE, + member_id TEXT NOT NULL, + event_type TEXT NOT NULL CHECK (event_type IN ('member_added', 'member_removed', 'role_changed')), + actor_id TEXT NOT NULL, + old_role TEXT CHECK (old_role IN ('owner', 'member')), + new_role TEXT CHECK (new_role IN ('owner', 'member')), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS ix_systematic_review_membership_audit_scope + ON systematic_review_membership_audit_events (sr_id, created_at DESC); + +-- Backfill is repeatable and makes the legacy primary owner an owner role. +INSERT INTO systematic_review_memberships (sr_id, member_id, role, added_by) +SELECT id, COALESCE(NULLIF(owner_email, ''), owner_id), 'owner', COALESCE(NULLIF(owner_email, ''), owner_id) +FROM systematic_reviews +WHERE COALESCE(NULLIF(owner_email, ''), owner_id) IS NOT NULL +ON CONFLICT (sr_id, member_id) DO UPDATE SET role = 'owner', updated_at = CURRENT_TIMESTAMP; + +INSERT INTO systematic_review_memberships (sr_id, member_id, role, added_by) +SELECT sr.id, member.value, 'member', COALESCE(NULLIF(sr.owner_email, ''), sr.owner_id) +FROM systematic_reviews sr +CROSS JOIN LATERAL jsonb_array_elements_text(COALESCE(sr.users, '[]'::jsonb)) AS member(value) +WHERE member.value <> '' +ON CONFLICT (sr_id, member_id) DO NOTHING; diff --git a/backend/migrations/005_citation_workspace_column_preferences.sql b/backend/migrations/005_citation_workspace_column_preferences.sql new file mode 100644 index 00000000..c94f830c --- /dev/null +++ b/backend/migrations/005_citation_workspace_column_preferences.sql @@ -0,0 +1,10 @@ +-- Per-user presentation preferences. Citation data and review-wide workflow +-- configuration remain outside this table. +CREATE TABLE IF NOT EXISTS citation_workspace_column_preferences ( + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + user_id TEXT NOT NULL, + columns JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (sr_id, citation_table_name, user_id) +); diff --git a/backend/migrations/006_citation_deduplication_preferences.sql b/backend/migrations/006_citation_deduplication_preferences.sql new file mode 100644 index 00000000..cdf77c22 --- /dev/null +++ b/backend/migrations/006_citation_deduplication_preferences.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS citation_deduplication_preferences ( + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + user_id TEXT NOT NULL, + fields JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (sr_id, citation_table_name, user_id) +); diff --git a/backend/migrations/007_citation_duplicate_reviews.sql b/backend/migrations/007_citation_duplicate_reviews.sql new file mode 100644 index 00000000..bac968e4 --- /dev/null +++ b/backend/migrations/007_citation_duplicate_reviews.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS citation_duplicate_reviews ( + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + group_id TEXT NOT NULL, + decision TEXT NOT NULL CHECK (decision IN ('confirmed_duplicate', 'not_duplicate', 'deferred')), + survivor_id BIGINT, + updated_by TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (sr_id, citation_table_name, group_id) +); + +CREATE INDEX IF NOT EXISTS citation_duplicate_reviews_survivor_idx + ON citation_duplicate_reviews (sr_id, citation_table_name, survivor_id); diff --git a/backend/migrations/008_citation_duplicate_runs.sql b/backend/migrations/008_citation_duplicate_runs.sql new file mode 100644 index 00000000..9080c610 --- /dev/null +++ b/backend/migrations/008_citation_duplicate_runs.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS citation_duplicate_runs ( + id UUID PRIMARY KEY, + sr_id TEXT NOT NULL, + citation_table_name TEXT NOT NULL, + dataset_revision JSONB NOT NULL, + fields JSONB NOT NULL, + status TEXT NOT NULL CHECK (status IN ('running', 'succeeded', 'failed')), + result JSONB, + error TEXT, + requested_by TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS citation_duplicate_runs_lookup_idx + ON citation_duplicate_runs (sr_id, citation_table_name, status, completed_at DESC); diff --git a/backend/tests/cit_db_service_test.py b/backend/tests/cit_db_service_test.py index 10e02cc4..efbd01a5 100644 --- a/backend/tests/cit_db_service_test.py +++ b/backend/tests/cit_db_service_test.py @@ -66,5 +66,128 @@ def test_only_lists_missing_pdfs_that_passed_l1(self) -> None: self.assertTrue(sql.endswith('ORDER BY id')) +class WorkspaceCitationTests(unittest.TestCase): + def test_workspace_query_uses_metadata_allowlist_literal_search_and_stable_order(self) -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.side_effect = [ + [ + {'column_name': 'id', 'data_type': 'integer'}, + {'column_name': 'title', 'data_type': 'text'}, + {'column_name': 'abstract', 'data_type': 'text'}, + {'column_name': 'llm_hidden', 'data_type': 'text'}, + {'column_name': 'payload', 'data_type': 'jsonb'}, + ], + [{'id': 4, 'title': 'A 100% result', 'abstract': 'Text'}], + ] + cursor.fetchone.return_value = {'count': 1} + service = CitsDPService() + + with patch('api.services.cit_db_service.postgres_server') as server: + server.conn = connection + result = service.list_workspace_citations( + 'screening_table', search='100%', + ) + + self.assertEqual(result['columns'], ['id', 'title', 'abstract']) + self.assertEqual(result['citations'][0]['id'], 4) + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + self.assertIn('ILIKE %s ESCAPE', sql) + self.assertNotIn('llm_hidden', sql) + self.assertNotIn('payload', sql) + self.assertIn('ORDER BY \"id\" ASC', sql) + self.assertNotIn('LIMIT %s OFFSET %s', sql) + query_params = cursor.execute.call_args_list[-1].args[1] + self.assertEqual(query_params, ('%100\\%%', '%100\\%%')) + + def test_workspace_query_discards_unknown_and_hidden_requested_columns(self) -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.side_effect = [ + [ + {'column_name': 'id', 'data_type': 'integer'}, + {'column_name': 'title', 'data_type': 'text'}, + {'column_name': 'llm_hidden', 'data_type': 'text'}, + ], + [{'id': 4, 'title': 'A study'}], + ] + cursor.fetchone.side_effect = [{'count': 1}, (1, 4)] + service = CitsDPService() + + with patch('api.services.cit_db_service.postgres_server') as server: + server.conn = connection + result = service.list_workspace_citations( + 'screening_table', columns=['title', 'llm_hidden', 'unknown'], + ) + + self.assertEqual(result['columns'], ['id', 'title']) + self.assertEqual(result['available_columns'], ['id', 'title']) + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + self.assertIn('SELECT "id", "title" FROM "screening_table"', sql) + self.assertNotIn('llm_hidden', sql) + self.assertNotIn('unknown', sql) + + def test_workspace_deduplication_defaults_exclude_provenance(self) -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.side_effect = [ + [ + {'column_name': 'id', 'data_type': 'integer'}, + {'column_name': 'provenance', 'data_type': 'text'}, + {'column_name': 'title', 'data_type': 'text'}, + ], + [{'id': 1, 'provenance': 'PubMed', 'title': 'Study'}], + ] + cursor.fetchone.side_effect = [{'count': 1}, (1, 1)] + + with patch('api.services.cit_db_service.postgres_server') as server: + server.conn = connection + result = CitsDPService().list_workspace_citations('screening_table') + + assert 'provenance' not in result['duplicate_fields'] + + +class CitationDeletionTests(unittest.TestCase): + def test_delete_accepts_stable_revision_after_row_locking(self) -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = (2, 309) + cursor.rowcount = 1 + service = CitsDPService() + + with patch('api.services.cit_db_service.postgres_server') as server: + server.conn = connection + result = service.delete_citations( + 'screening_table', [309], expected_revision=(2, 309), + ) + + self.assertEqual( + result, + {'deleted_count': 1, 'dataset_revision': (2, 309)}, + ) + connection.commit.assert_called_once_with() + self.assertIn( + 'COALESCE(MAX(id), 0)', + cursor.execute.call_args_list[1].args[0], + ) + self.assertNotIn('xmin', cursor.execute.call_args_list[1].args[0]) + + def test_delete_rejects_changed_dataset_revision(self) -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = (3, 309) + service = CitsDPService() + + with patch('api.services.cit_db_service.postgres_server') as server: + server.conn = connection + with self.assertRaisesRegex(ValueError, 'dataset changed after preview'): + service.delete_citations( + 'screening_table', [309], expected_revision=(2, 309), + ) + + connection.rollback.assert_called_once_with() + connection.commit.assert_not_called() + + if __name__ == '__main__': unittest.main() diff --git a/backend/tests/citation_deduplication_service_test.py b/backend/tests/citation_deduplication_service_test.py new file mode 100644 index 00000000..a7d4172c --- /dev/null +++ b/backend/tests/citation_deduplication_service_test.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import csv +from pathlib import Path + +from api.services.citation_deduplication_service import calculate_duplicate_statuses +from api.services.citation_deduplication_service import normalize +from api.services.citation_deduplication_service import recompute_affected_duplicate_groups + + +def test_normalize_case_punctuation_and_unicode(): + assert normalize(' Café—Study! ') == 'café study' + assert normalize(None) == '' + + +def test_empty_values_are_neutral_and_possible_matches_use_average_score(): + result = calculate_duplicate_statuses( + [ + {'id': 1, 'title': 'Same', 'abstract': ''}, + {'id': 2, 'title': 'same', 'abstract': ''}, + {'id': 3, 'title': 'A study', 'abstract': 'An abstract'}, + {'id': 4, 'title': 'A stody', 'abstract': 'An abstrakt'}, + ], ['title', 'abstract'], + ) + assert result['rows'][0]['status'] == 'exact' + assert result['rows'][2]['status'] == 'possible' + assert result['rows'][2]['score'] >= 0.7 + + +def test_real_hcsf_sample_detects_duplicate_titles_with_empty_metadata(): + sample = Path(__file__).parents[3] / 'resources' / \ + 'sample_files' / 'sr_HCSF - LLM Plenary Demo_citations.csv' + with sample.open(newline='', encoding='utf-8-sig') as handle: + source_rows = list(csv.DictReader(handle)) + + rows = [ + { + 'id': index, + 'title': row.get('title'), + 'abstract': row.get('abstract'), + 'author': row.get('author'), + 'doi': row.get('doi'), + 'year': row.get('year'), + } + for index, row in enumerate(source_rows, 1) + ] + duplicate_title = 'social trends' + duplicate_indexes = [ + index for index, row in enumerate(rows) + if normalize(row['title']) == duplicate_title + ] + + result = calculate_duplicate_statuses( + rows, ['title', 'abstract', 'author', 'doi', 'year'], + ) + + assert len(duplicate_indexes) == 2 + assert all( + result['rows'][index]['status'] in {'exact', 'possible'} + for index in duplicate_indexes + ) + assert any( + set(group['citation_ids']) == { + index + 1 for index in duplicate_indexes + } + for group in result['groups'] + ) + + +def test_real_l1_sample_detects_repeated_titles(): + sample = Path(__file__).parents[3] / 'resources' / \ + 'sample_files' / 'L1 Screening 2025_03_18 - fulltext - 100.csv' + with sample.open(newline='', encoding='utf-8-sig') as handle: + source_rows = list(csv.DictReader(handle)) + + rows = [ + { + 'id': index, + 'title': row.get('Title'), + 'abstract': row.get('Abstract'), + 'author': row.get('Author'), + 'doi': row.get('DOI'), + 'year': row.get('Year'), + } + for index, row in enumerate(source_rows, 1) + ] + title_indexes = {} + for index, row in enumerate(rows): + title = normalize(row['title']) + if title: + title_indexes.setdefault(title, []).append(index) + duplicate_indexes = next( + indexes for indexes in title_indexes.values() if len(indexes) > 1 + ) + + # Run the detector on the two real repeated-title records so the regression + # focuses on matching semantics rather than the bounded row lookahead. + candidate_rows = [rows[index] for index in duplicate_indexes] + result = calculate_duplicate_statuses(candidate_rows, ['title']) + + assert all( + item['status'] == 'exact' for item in result['rows'] + ) + + +def test_field_count_is_unbounded_but_lookahead_is_capped(): + fields = [f'field_{index}' for index in range(10)] + rows = [ + {'id': index, **{field: 'same' for field in fields}} + for index in range(10) + ] + result = calculate_duplicate_statuses(rows, fields) + assert result['lookahead'] == 6 + assert len(result['groups']) == 1 + assert len(result['groups'][0]['citation_ids']) == 10 + + +def test_recompute_after_delete_preserves_unrelated_groups(): + previous = { + 'rows': [ + { + 'citation_id': 1, 'status': 'exact', + 'group_id': 'group-1', 'score': 1.0, + }, + { + 'citation_id': 2, 'status': 'exact', + 'group_id': 'group-1', 'score': 1.0, + }, + { + 'citation_id': 3, 'status': 'exact', + 'group_id': 'group-2', 'score': 1.0, + }, + { + 'citation_id': 4, 'status': 'exact', + 'group_id': 'group-2', 'score': 1.0, + }, + ], + 'groups': [ + { + 'group_id': 'group-1', + 'citation_ids': [1, 2], + 'status': 'exact', + 'members': [ + {'id': 1, 'title': 'same'}, + {'id': 2, 'title': 'same'}, + ], + }, + { + 'group_id': 'group-2', + 'citation_ids': [3, 4], + 'status': 'exact', + 'members': [ + {'id': 3, 'title': 'other'}, + {'id': 4, 'title': 'other'}, + ], + }, + ], + 'lookahead': 2, + } + + result = recompute_affected_duplicate_groups(previous, {1}, ['title']) + + assert [group['group_id'] for group in result['groups']] == ['group-2'] + assert [row['citation_id'] for row in result['rows']] == [2, 3, 4] + assert result['rows'][0]['status'] == 'no_match' + assert result['groups'][0]['citation_ids'] == [3, 4] diff --git a/backend/tests/citation_duplicate_review_service_test.py b/backend/tests/citation_duplicate_review_service_test.py new file mode 100644 index 00000000..407d7df3 --- /dev/null +++ b/backend/tests/citation_duplicate_review_service_test.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from api.services.citation_duplicate_review_service import suggest_survivor + + +def test_suggest_survivor_prefers_complete_metadata_and_identifier_quality(): + result = suggest_survivor([ + {'id': 9, 'title': 'Study', 'abstract': 'Abstract', 'doi': ''}, + {'id': 4, 'title': 'Study', 'abstract': 'Abstract', 'doi': '10.1000/test'}, + ]) + + assert result['suggested_survivor_id'] == 4 + assert result['survivor_reason'] == 'strongest_identifier' + + +def test_suggest_survivor_uses_oldest_record_then_lowest_id(): + result = suggest_survivor([ + {'id': 9, 'title': 'Study', 'created_at': '2002'}, + {'id': 4, 'title': 'Study', 'created_at': '2001'}, + ]) + + assert result['suggested_survivor_id'] == 4 diff --git a/backend/tests/citation_duplicate_run_service_test.py b/backend/tests/citation_duplicate_run_service_test.py new file mode 100644 index 00000000..5b142b3d --- /dev/null +++ b/backend/tests/citation_duplicate_run_service_test.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from datetime import datetime +from unittest.mock import Mock +from unittest.mock import patch + +from api.services.citation_duplicate_run_service import CitationDuplicateRunService + + +def test_finish_serializes_datetime_values_in_duplicate_result(): + connection = Mock() + cursor = connection.cursor.return_value + service = CitationDuplicateRunService() + completed_at = datetime(2026, 8, 14, 5, 47, 21, 847000) + result = { + 'groups': [ + {'members': [{'id': 1, 'created_at': completed_at}]}, + ], + } + + with patch('api.services.citation_duplicate_run_service.postgres_server') as server: + server.conn = connection + service.finish('run-id', result) + + stored_result = cursor.execute.call_args.args[1][1] + assert stored_result == ( + '{"groups": [{"members": [{"id": 1, ' + '"created_at": "2026-08-14T05:47:21.847000"}]}]}' + ) + connection.commit.assert_called_once_with() diff --git a/backend/tests/citation_import_postgres_integration_test.py b/backend/tests/citation_import_postgres_integration_test.py new file mode 100644 index 00000000..c6a4fdc5 --- /dev/null +++ b/backend/tests/citation_import_postgres_integration_test.py @@ -0,0 +1,371 @@ +"""Live PostgreSQL invariants for external-ID citation import reconciliation. + +Runs only when the configured local PostgreSQL test database is available. Each +test owns a unique review and dynamic citation table and removes both in teardown. +""" +from __future__ import annotations + +import base64 +import hashlib +import os +import threading +import unittest +import uuid +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from pathlib import Path + +import psycopg2 +from api.services.citation_import_preview_service import CitationImportPreviewRepository + + +def _connect(): + return psycopg2.connect( + host=os.getenv('POSTGRES_TEST_HOST', 'localhost'), + port=int(os.getenv('POSTGRES_PORT', '5432')), + dbname=os.getenv('POSTGRES_DATABASE', 'postgres'), + user=os.getenv('POSTGRES_USER', 'admin'), + password=os.getenv('POSTGRES_PASSWORD', 'password'), + connect_timeout=2, + ) + + +class _Provider: + def __init__(self, conn): + self.conn = conn + + +class CitationImportPostgresIntegrationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + cls.admin_conn = _connect() + except Exception as exc: + raise unittest.SkipTest(f'PostgreSQL unavailable: {exc}') + migrations = Path(__file__).resolve().parents[1] / 'migrations' + cur = cls.admin_conn.cursor() + try: + cur.execute( + """CREATE TABLE IF NOT EXISTS systematic_reviews ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, owner_id TEXT NOT NULL, + screening_db JSONB, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP + )""", + ) + cur.execute( + (migrations / '002_citation_workspace_schema.sql').read_text(encoding='utf-8'), + ) + cur.execute( + (migrations / '001_multi_reviewer_schema.sql').read_text(encoding='utf-8'), + ) + cur.execute( + (migrations / '003_citation_import_preview_mapping_decisions.sql').read_text(encoding='utf-8'), + ) + cls.admin_conn.commit() + except Exception: + cls.admin_conn.rollback() + raise + finally: + cur.close() + + @classmethod + def tearDownClass(cls): + cls.admin_conn.close() + + def setUp(self): + token = uuid.uuid4().hex + self.sr_id = f'import_pg_{token}' + self.table_name = f'cit_{token}' + cur = self.admin_conn.cursor() + cur.execute( + f'CREATE TABLE "{self.table_name}" (id SERIAL PRIMARY KEY, title TEXT, abstract TEXT)', + ) + cur.execute( + """INSERT INTO systematic_reviews (id, name, owner_id, screening_db) + VALUES (%s, %s, %s, jsonb_build_object('table_name', %s))""", + (self.sr_id, 'integration test', 'owner@example.test', self.table_name), + ) + self.admin_conn.commit() + cur.close() + + def tearDown(self): + cur = self.admin_conn.cursor() + try: + cur.execute( + 'DELETE FROM reconciliation_decisions WHERE case_id IN (SELECT id FROM reconciliation_cases WHERE sr_id = %s)', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM reconciliation_cases WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM review_validations WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM review_assignments WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM citation_import_previews WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM citation_audit_events WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM citation_import_batch_memberships WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM citation_import_batches WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM citation_identities WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM systematic_reviews WHERE id = %s', (self.sr_id,), + ) + cur.execute(f'DROP TABLE IF EXISTS "{self.table_name}"') + self.admin_conn.commit() + except Exception: + self.admin_conn.rollback() + raise + finally: + cur.close() + + def _create_preview(self, rows, *, source: bytes | None = None) -> tuple[str, dict]: + source = source or uuid.uuid4().bytes + preview_id = str(uuid.uuid4()) + payload = { + 'filename': 'citations.csv', 'source_format': 'csv', + 'raw_file_base64': base64.b64encode(source).decode(), 'normalized_rows': rows, + } + cur = self.admin_conn.cursor() + cur.execute( + """INSERT INTO citation_import_previews + (id, sr_id, citation_table_name, source_sha256, schema_fingerprint, + proposed_mapping, validation_report, encrypted_staging_locator, + staging_expires_at, commit_key, created_by) + VALUES (%s, %s, %s, %s, 'test-schema', %s::jsonb, %s::jsonb, %s, %s, %s, %s)""", + ( + preview_id, self.sr_id, self.table_name, hashlib.sha256( + source, + ).hexdigest(), + '{"title": "Title", "abstract": "Abstract"}', + '{"valid_for_commit": true, "source_columns": ["Title", "Abstract"], "row_count": 1}', + f'test/{preview_id}', datetime.now(timezone.utc) + + timedelta(minutes=5), preview_id, + 'owner@example.test', + ), + ) + self.admin_conn.commit() + cur.close() + return preview_id, payload + + def _commit(self, preview_id, payload, conn=None): + conn = conn or self.admin_conn + return CitationImportPreviewRepository(connection_provider=_Provider(conn)).commit( + preview_id, self.sr_id, 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, payload, + ) + + def _scalar(self, sql, params=()): + cur = self.admin_conn.cursor() + cur.execute(sql, params) + value = cur.fetchone()[0] + cur.close() + return value + + def test_external_id_only_duplicate_and_namespace_isolation(self): + first_id, first = self._create_preview([ + {'Title': 'Original', 'Abstract': 'Original abstract', 'PMID': 'PMID: 123'}, + ]) + second_id, second = self._create_preview([ + {'Title': 'Changed', 'Abstract': 'Changed abstract', 'PubMed ID': '123'}, + ]) + third_id, third = self._create_preview([ + { + 'Title': 'Separate system', + 'Abstract': 'Separate abstract', 'Scopus EID': '123', + }, + ]) + + self.assertEqual(self._commit(first_id, first)['inserted_count'], 1) + duplicate = self._commit(second_id, second) + self.assertEqual( + ( + duplicate['inserted_count'], + duplicate['existing_exact_match_count'], + ), (0, 1), + ) + self.assertEqual(self._commit(third_id, third)['inserted_count'], 1) + self.assertEqual( + self._scalar( + f'SELECT count(*) FROM "{self.table_name}"', + ), 2, + ) + self.assertEqual( + self._scalar( + "SELECT count(*) FROM citation_identities WHERE sr_id = %s AND identity_kind = 'external_id'", + (self.sr_id,), + ), 2, + ) + + def test_cross_identity_conflict_rolls_back_before_any_writes(self): + first_id, first = self._create_preview([ + {'Title': 'PubMed record', 'Abstract': 'A', 'PMID': '123'}, + ]) + second_id, second = self._create_preview([ + {'Title': 'Scopus record', 'Abstract': 'B', 'Scopus EID': '2-s2.0-1'}, + ]) + self._commit(first_id, first) + self._commit(second_id, second) + conflict_id, conflict = self._create_preview([ + { + 'Title': 'Conflicting', 'Abstract': 'C', + 'PMID': '123', 'Scopus EID': '2-s2.0-1', + }, + ]) + + with self.assertRaisesRegex(ValueError, 'ambiguous exact'): + self._commit(conflict_id, conflict) + self.assertEqual( + self._scalar( + f'SELECT count(*) FROM "{self.table_name}"', + ), 2, + ) + self.assertEqual( + self._scalar( + 'SELECT count(*) FROM citation_import_batches WHERE sr_id = %s', (self.sr_id,), + ), 2, + ) + + def test_retry_is_idempotent_and_trigger_failure_rolls_back_external_identity(self): + preview_id, payload = self._create_preview([ + {'Title': 'Retry', 'Abstract': 'A', 'PMID': '123'}, + ]) + initial = self._commit(preview_id, payload) + retry = self._commit(preview_id, payload) + self.assertTrue(retry['idempotent']) + self.assertEqual(retry['batch_id'], initial['batch_id']) + self.assertEqual( + self._scalar( + f'SELECT count(*) FROM "{self.table_name}"', + ), 1, + ) + + function_name = f'fail_external_identity_{uuid.uuid4().hex}' + trigger_name = f'fail_external_identity_{uuid.uuid4().hex}' + cur = self.admin_conn.cursor() + cur.execute( + f"""CREATE FUNCTION {function_name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.identity_kind = 'external_id' AND NEW.normalized_value = 'force-rollback' THEN + RAISE EXCEPTION 'forced external identity failure'; + END IF; + RETURN NEW; + END $$""", + ) + cur.execute( + f'CREATE TRIGGER {trigger_name} BEFORE INSERT ON citation_identities ' + f'FOR EACH ROW EXECUTE FUNCTION {function_name}()', + ) + self.admin_conn.commit() + cur.close() + try: + failed_id, failed = self._create_preview([ + {'Title': 'Rollback', 'Abstract': 'B', 'PMID': 'force-rollback'}, + ]) + with self.assertRaisesRegex(Exception, 'forced external identity failure'): + self._commit(failed_id, failed) + self.assertEqual( + self._scalar( + f'SELECT count(*) FROM "{self.table_name}"', + ), 1, + ) + self.assertEqual( + self._scalar( + 'SELECT count(*) FROM citation_import_batches WHERE sr_id = %s', (self.sr_id,), + ), 1, + ) + self.assertEqual( + self._scalar( + 'SELECT count(*) FROM citation_import_batch_memberships WHERE sr_id = %s', (self.sr_id,), + ), 1, + ) + self.assertEqual( + self._scalar( + "SELECT count(*) FROM citation_identities WHERE sr_id = %s AND normalized_value = 'force-rollback'", + (self.sr_id,), + ), 0, + ) + finally: + cur = self.admin_conn.cursor() + cur.execute( + f'DROP TRIGGER IF EXISTS {trigger_name} ON citation_identities', + ) + cur.execute(f'DROP FUNCTION IF EXISTS {function_name}()') + self.admin_conn.commit() + cur.close() + + def test_concurrent_external_id_commits_produce_one_citation(self): + first_id, first = self._create_preview([ + {'Title': 'Concurrent A', 'Abstract': 'A', 'PMID': '777'}, + ]) + second_id, second = self._create_preview([ + {'Title': 'Concurrent B', 'Abstract': 'B', 'PMID': '777'}, + ]) + results, errors = [], [] + gate = threading.Barrier(2) + + def worker(preview_id, payload): + conn = _connect() + try: + gate.wait(timeout=5) + results.append(self._commit(preview_id, payload, conn)) + except Exception as exc: # surfaced as a test failure below + errors.append(exc) + finally: + conn.close() + + threads = [ + threading.Thread(target=worker, args=(first_id, first)), + threading.Thread(target=worker, args=(second_id, second)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + self.assertFalse(errors) + self.assertEqual(len(results), 2) + self.assertEqual( + sorted( + result['inserted_count'] + for result in results + ), [0, 1], + ) + self.assertEqual( + sorted( + result['existing_exact_match_count'] + for result in results + ), [0, 1], + ) + self.assertEqual( + self._scalar( + f'SELECT count(*) FROM "{self.table_name}"', + ), 1, + ) diff --git a/backend/tests/citation_import_preview_service_test.py b/backend/tests/citation_import_preview_service_test.py new file mode 100644 index 00000000..540d39e8 --- /dev/null +++ b/backend/tests/citation_import_preview_service_test.py @@ -0,0 +1,852 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from datetime import timezone +from unittest.mock import AsyncMock +from unittest.mock import Mock + +import pytest +from api.services.citation_import_preview_service import _normalized_external_id +from api.services.citation_import_preview_service import _row_external_ids +from api.services.citation_import_preview_service import build_preview +from api.services.citation_import_preview_service import CitationImportPreviewRepository +from api.services.citation_import_preview_service import CitationImportPreviewService +from api.services.citation_import_schema import COMMIT_SUPPORTED_FIELD_KEYS +from api.services.citation_import_schema import is_protected_citation_column +from api.services.citation_import_schema import safe_additional_field_name + + +class MemoryStorage: + container_name = 'test-storage' + + def __init__(self): + self.contents: dict[str, bytes] = {} + self.deleted: list[str] = [] + + async def put_bytes_by_path(self, path: str, content: bytes, content_type: str = '') -> bool: + self.contents[path] = content + return True + + async def delete_by_path(self, path: str) -> bool: + self.deleted.append(path) + self.contents.pop(path, None) + return True + + async def get_bytes_by_path(self, path: str) -> tuple[bytes, str]: + return self.contents[path], path.rsplit('/', 1)[-1] + + +def test_csv_preview_normalizes_headers_and_reports_invalid_rows() -> None: + preview = build_preview( + 'citations.csv', + b'Title,Abstract,DOI\nFirst,An abstract,10.1/a\nSecond,,10.1/b\n', + ) + + assert preview['source_format'] == 'csv' + assert preview['proposed_mapping'] == { + 'title': 'Title', 'abstract': 'Abstract', + } + assert preview['validation_report']['row_count'] == 2 + assert preview['validation_report']['missing_required_value_counts'] == { + 'title': 0, 'abstract': 1, + } + assert preview['validation_report']['valid_for_commit'] is False + assert preview['schema_fingerprint'].startswith('sha256:') + + +def test_preview_reports_unmapped_required_columns() -> None: + preview = build_preview('citations.csv', b'Name,Summary\nOne,Text\n') + + assert preview['validation_report']['missing_required_fields'] == [ + 'title', 'abstract', + ] + assert preview['validation_report']['missing_required_value_counts'] == { + 'title': 1, 'abstract': 1, + } + + +def test_preview_reports_columns_requiring_explicit_mapping_decision() -> None: + preview = build_preview( + 'citations.csv', b'Title,Abstract,DOI\nOne,Text,10.1/a\n', + ) + + assert preview['validation_report']['unclassified_source_columns'] == [ + 'DOI', + ] + + +def test_reconciliation_uses_normalized_doi_before_fingerprint_and_reports_examples() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.return_value = [('doi', '10.1000/example', 41)] + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).reconcile( + 'sr-1', 'sr_1_citations', [ + { + 'Title': 'Changed title', 'Abstract': 'Changed abstract', + 'DOI': 'https://doi.org/10.1000/EXAMPLE.', + }, + {'Title': 'New title', 'Abstract': 'New abstract'}, + {'Title': 'New title', 'Abstract': 'New abstract'}, + {'Title': 'Missing abstract', 'Abstract': ''}, + ], {'title': 'Title', 'abstract': 'Abstract'}, + ) + + assert result['counts'] == { + 'new': 1, 'existing_exact_match': 2, + 'ambiguous': 0, 'excluded_ambiguous': 0, 'invalid': 1, + } + assert result['examples'][0] == { + 'row_number': 1, 'outcome': 'existing_exact_match', 'citation_id': 41, + } + assert result['examples'][2] == { + 'row_number': 3, 'outcome': 'existing_exact_match', + } + + +def test_external_id_headers_and_values_are_normalized_with_namespaces() -> None: + assert _normalized_external_id('pubmed', ' PMID: 00123 ') == '00123' + assert _row_external_ids({ + 'PubMed ID': 'PMID: 123', 'Embase ID': 'EMBASE: L123', + 'Scopus EID': 'EID: 2-s2.0-1', 'Web of Science ID': 'WOS: 0001', + }) == [ + ('pubmed', '123'), ('embase', 'l123'), ( + 'scopus', + '2-s2.0-1', + ), ('web_of_science', '0001'), + ] + + +def test_reconciliation_external_ids_are_namespace_isolated_and_conflicts_are_ambiguous() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.return_value = [ + ('external_id', 'pubmed', '123', 41), ('external_id', 'scopus', '123', 42), + ] + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).reconcile( + 'sr-1', 'sr_1_citations', [ + {'Title': 'A', 'Abstract': 'B', 'PubMed ID': '123'}, + {'Title': 'C', 'Abstract': 'D', 'Web of Science ID': '123'}, + {'Title': 'E', 'Abstract': 'F', 'PubMed ID': '123', 'Scopus EID': '123'}, + ], {'title': 'Title', 'abstract': 'Abstract'}, + ) + + assert result['counts'] == { + 'new': 1, 'existing_exact_match': 1, + 'ambiguous': 1, 'excluded_ambiguous': 0, 'invalid': 0, + } + assert result['ambiguous_row_numbers'] == [3] + + +def test_reconciliation_detects_in_file_and_durable_cross_identity_conflicts() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.return_value = [('external_id', 'pubmed', '123', 41)] + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).reconcile( + 'sr-1', 'sr_1_citations', [ + {'Title': 'A', 'Abstract': 'B', 'Scopus EID': '2-s2.0-1'}, + { + 'Title': 'C', 'Abstract': 'D', + 'Scopus EID': '2-s2.0-1', 'PubMed ID': '123', + }, + ], {'title': 'Title', 'abstract': 'Abstract'}, + ) + + assert result['counts'] == { + 'new': 1, 'existing_exact_match': 0, + 'ambiguous': 1, 'excluded_ambiguous': 0, 'invalid': 0, + } + + +def test_reconciliation_reports_ambiguous_durable_identity() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.return_value = [ + ('doi', '10.1000/example', 41), ('doi', '10.1000/example', 42), + ] + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).reconcile( + 'sr-1', 'sr_1_citations', [{ + 'Title': 'A', + 'Abstract': 'B', 'DOI': 'doi:10.1000/example', + }], + {'title': 'Title', 'abstract': 'Abstract'}, + ) + + assert result['counts'] == { + 'new': 0, 'existing_exact_match': 0, + 'ambiguous': 1, 'excluded_ambiguous': 0, 'invalid': 0, + } + assert result['ambiguous_details'] == [{ + 'row_number': 1, + 'matches': [{'identity_kind': 'doi', 'identifier_namespace': None, 'normalized_value': '10.1000/example', 'citation_ids': [41, 42]}], + }] + + +def test_reconciliation_excludes_only_explicitly_resolved_ambiguous_rows() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchall.return_value = [ + ('external_id', 'pubmed', '123', 41), ('external_id', 'scopus', '123', 42), + ] + repository = CitationImportPreviewRepository( + connection_provider=Mock(conn=connection), + ) + + result = repository.reconcile( + 'sr-1', 'sr_1_citations', + [{'Title': 'A', 'Abstract': 'B', 'PubMed ID': '123', 'Scopus EID': '123'}], + {'title': 'Title', 'abstract': 'Abstract'}, [1], + ) + + assert result['counts'] == { + 'new': 0, 'existing_exact_match': 0, + 'ambiguous': 0, 'excluded_ambiguous': 1, 'invalid': 0, + } + assert result['ambiguous_row_numbers'] == [1] + with pytest.raises(ValueError, match='Only currently ambiguous'): + repository.reconcile( + 'sr-1', 'sr_1_citations', + [{'Title': 'A', 'Abstract': 'B', 'PubMed ID': '123', 'Scopus EID': '123'}], + {'title': 'Title', 'abstract': 'Abstract'}, [2], + ) + + +def test_import_schema_registry_limits_mapping_to_current_commit_capabilities() -> None: + assert COMMIT_SUPPORTED_FIELD_KEYS == ('title', 'abstract') + + +def test_additional_field_policy_generates_safe_names_and_rejects_collisions() -> None: + assert safe_additional_field_name( + 'Publication Date', + ) == 'import_publication_date' + assert safe_additional_field_name('2024 Notes') == 'import_2024_notes' + assert is_protected_citation_column('id') is True + assert is_protected_citation_column('llm_decision') is True + with pytest.raises(ValueError, match='collides'): + safe_additional_field_name('DOI', existing_columns=['import_doi']) + + +@pytest.mark.asyncio +async def test_service_revalidates_staging_and_refreshes_reconciliation_on_mapping_update() -> None: + storage = MemoryStorage() + repository = Mock() + preview_id = '00000000-0000-0000-0000-000000000001' + locator = 'test-storage/citation-import-previews/sr-1/preview.fernet' + repository.get_active.return_value = { + 'citation_table_name': 'sr_1_citations', + 'validation_report': {'source_columns': ['Title', 'Abstract', 'Alt abstract']}, + } + repository.get_staging_locator.return_value = locator + repository.reconcile.return_value = { + 'counts': { + 'new': 1, 'existing_exact_match': 0, 'ambiguous': 0, 'invalid': 0, + }, 'examples': [], + } + repository.update_mapping_decision.return_value = { + 'proposed_mapping': {'title': 'Title', 'abstract': 'Abstract'}, + 'mapping_decision': {'version': 1, 'approved_mapping': {'title': 'Title', 'abstract': 'Abstract'}, 'excluded_source_columns': ['DOI']}, + 'validation_report': {'row_count': 1, 'valid_for_commit': True}, + } + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', + ) + storage.contents[locator] = service._fernet.encrypt( + json.dumps({ + 'normalized_rows': [{'Title': 'A', 'Abstract': '', 'Alt abstract': 'B'}], + }).encode(), + ) + + result = await service.update_mapping( + preview_id=preview_id, sr_id='sr-1', actor_id='owner@example.test', + approved_mapping={'title': 'Title', 'abstract': 'Alt abstract'}, excluded_source_columns=['Abstract'], + ) + + assert result['id'] == preview_id + assert result['mapping_decision']['excluded_source_columns'] == ['DOI'] + refreshed_report = repository.update_mapping_decision.call_args.args[5] + assert refreshed_report['missing_required_value_counts'] == { + 'title': 0, 'abstract': 0, + } + assert refreshed_report['reconciliation']['counts']['new'] == 1 + + +def test_repository_mapping_update_locks_active_owned_preview_and_persists_decision() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = ( + {'title': 'Title', 'abstract': 'Abstract'}, + { + 'row_count': 1, 'source_columns': ['Title', 'Abstract', 'DOI'], + 'missing_required_value_counts': {'title': 0, 'abstract': 0}, 'valid_for_commit': True, + }, + ) + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).update_mapping_decision( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, ['DOI'], + { + 'row_count': 1, 'source_columns': ['Title', 'Abstract', 'DOI'], + 'missing_required_value_counts': {'title': 0, 'abstract': 0}, 'valid_for_commit': True, + 'reconciliation': {'counts': {'new': 1, 'existing_exact_match': 0, 'ambiguous': 0, 'invalid': 0}, 'examples': []}, + }, + ) + + assert result['validation_report']['valid_for_commit'] is True + assert result['mapping_decision']['version'] == 1 + connection.commit.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'FOR UPDATE' in sql + assert 'created_by = %s' in sql + assert 'mapping_decision = %s::jsonb' in sql + + +def test_repository_mapping_update_rejects_unclassified_source_columns() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = ( + {'title': 'Title', 'abstract': 'Abstract'}, + {'row_count': 1, 'source_columns': ['Title', 'Abstract', 'DOI']}, + ) + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + with pytest.raises(ValueError, match='mapped or explicitly excluded'): + CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).update_mapping_decision( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, [], {}, + ) + connection.rollback.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_service_encrypts_staging_and_writes_only_preview_metadata() -> None: + storage = MemoryStorage() + repository = Mock() + repository.reconcile.return_value = { + 'counts': { + 'new': 1, 'existing_exact_match': 0, 'ambiguous': 0, 'invalid': 0, + }, 'examples': [], + } + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', ttl_minutes=30, + ) + raw = b'Title,Abstract\nConfidential title,Confidential abstract\n' + + response = await service.create( + sr_id='sr-1', citation_table_name='sr_1_citations', filename='citations.csv', + raw_bytes=raw, commit_key='request-1', actor_id='owner@example.test', + ) + + assert response['validation_report']['valid_for_commit'] is True + assert response['staging_expires_at'].endswith('+00:00') + assert len(storage.contents) == 1 + ciphertext = next(iter(storage.contents.values())) + assert raw not in ciphertext + staged = json.loads(service._fernet.decrypt(ciphertext)) + assert staged['filename'] == 'citations.csv' + assert staged['normalized_rows'] == [ + {'Title': 'Confidential title', 'Abstract': 'Confidential abstract'}, + ] + record = repository.create.call_args.args[0] + assert record['encrypted_staging_locator'] in storage.contents + assert 'normalized_rows' not in record + assert 'raw_file_base64' not in record + + +@pytest.mark.asyncio +async def test_service_removes_staged_ciphertext_when_metadata_write_fails() -> None: + storage = MemoryStorage() + repository = Mock() + repository.create.side_effect = RuntimeError('database unavailable') + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', + ) + + with pytest.raises(RuntimeError, match='database unavailable'): + await service.create( + sr_id='sr-1', citation_table_name=None, filename='citations.csv', + raw_bytes=b'Title,Abstract\nA,B\n', commit_key='request-1', actor_id='owner@example.test', + ) + + assert len(storage.deleted) == 1 + assert storage.contents == {} + + +def test_repository_insert_targets_only_preview_metadata_table() -> None: + connection = Mock() + cursor = connection.cursor.return_value + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).create({ + 'id': '00000000-0000-0000-0000-000000000001', 'sr_id': 'sr-1', + 'citation_table_name': 'sr_1_citations', 'source_sha256': 'hash', 'schema_fingerprint': 'schema', + 'proposed_mapping': {}, 'validation_report': {}, 'encrypted_staging_locator': 'store/path', + 'staging_expires_at': '2026-01-01T00:00:00+00:00', 'commit_key': 'key', 'created_by': 'owner', + }) + + sql = cursor.execute.call_args.args[0] + assert 'INSERT INTO citation_import_previews' in sql + assert 'citation_import_batches' not in sql + assert 'sr_1_citations' not in sql + assert json.loads(cursor.execute.call_args.args[1][5]) == {} + + +@pytest.mark.asyncio +async def test_cancel_marks_owned_preview_final_and_deletes_its_ciphertext() -> None: + storage = MemoryStorage() + repository = Mock() + repository.cancel.return_value = 'test-storage/citation-import-previews/sr-1/preview.fernet' + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', + ) + + result = await service.cancel( + preview_id='00000000-0000-0000-0000-000000000001', sr_id='sr-1', actor_id='owner@example.test', + ) + + assert result == { + 'id': '00000000-0000-0000-0000-000000000001', + 'cancelled': True, 'staging_cleanup_pending': False, + } + repository.cancel.assert_called_once_with( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + ) + assert storage.deleted == [ + 'test-storage/citation-import-previews/sr-1/preview.fernet', + ] + + +@pytest.mark.asyncio +async def test_cancel_rejects_preview_not_owned_or_already_finalized_without_deleting() -> None: + storage = MemoryStorage() + repository = Mock() + repository.cancel.return_value = None + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', + ) + + with pytest.raises(LookupError, match='not found'): + await service.cancel( + preview_id='00000000-0000-0000-0000-000000000001', sr_id='sr-1', actor_id='other@example.test', + ) + + assert storage.deleted == [] + + +@pytest.mark.asyncio +async def test_cancel_reports_cleanup_pending_after_metadata_cancellation() -> None: + storage = MemoryStorage() + storage.delete_by_path = AsyncMock( + side_effect=OSError('storage unavailable'), + ) + repository = Mock() + repository.cancel.return_value = 'test-storage/citation-import-previews/sr-1/preview.fernet' + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', + ) + + result = await service.cancel( + preview_id='00000000-0000-0000-0000-000000000001', sr_id='sr-1', actor_id='owner@example.test', + ) + + assert result['cancelled'] is True + assert result['staging_cleanup_pending'] is True + + +def test_repository_cancel_scopes_owner_and_never_updates_citation_rows() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = ('test-storage/staged.fernet',) + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + locator = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).cancel( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + ) + + assert locator == 'test-storage/staged.fernet' + sql = cursor.execute.call_args.args[0] + assert 'UPDATE citation_import_previews' in sql + assert 'created_by = %s' in sql + assert 'committed_batch_id IS NULL' in sql + assert 'staging_expires_at > CURRENT_TIMESTAMP' in sql + assert 'citation_import_batches' not in sql + assert 'INSERT INTO' not in sql + + +def test_inspect_returns_active_owned_metadata_without_staging_locator() -> None: + repository = Mock() + repository.get_active.return_value = { + 'id': '00000000-0000-0000-0000-000000000001', 'citation_table_name': 'sr_1_citations', + 'source_sha256': 'source-hash', 'schema_fingerprint': 'sha256:schema', + 'proposed_mapping': {'title': 'Title', 'abstract': 'Abstract'}, + 'validation_report': {'row_count': 1, 'valid_for_commit': True}, + 'staging_expires_at': '2026-01-01T01:00:00+00:00', 'created_at': '2026-01-01T00:00:00+00:00', + } + service = CitationImportPreviewService( + MemoryStorage(), repository, secret_key='unit-test-secret', + ) + + result = service.inspect( + preview_id='00000000-0000-0000-0000-000000000001', sr_id='sr-1', actor_id='owner@example.test', + ) + + assert result['validation_report']['valid_for_commit'] is True + assert 'encrypted_staging_locator' not in result + repository.get_active.assert_called_once_with( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + ) + + +def test_inspect_rejects_unavailable_preview() -> None: + repository = Mock() + repository.get_active.return_value = None + service = CitationImportPreviewService( + MemoryStorage(), repository, secret_key='unit-test-secret', + ) + + with pytest.raises(LookupError, match='not found'): + service.inspect( + preview_id='00000000-0000-0000-0000-000000000001', sr_id='sr-1', actor_id='other@example.test', + ) + + +def test_repository_inspect_scopes_active_owner_and_excludes_encrypted_staging() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = ( + '00000000-0000-0000-0000-000000000001', 'sr_1_citations', 'source-hash', 'sha256:schema', + {'title': 'Title', 'abstract': 'Abstract'}, {'row_count': 1}, + datetime(2026, 1, 1, 1, tzinfo=timezone.utc), datetime( + 2026, 1, 1, tzinfo=timezone.utc, + ), + ) + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).get_active( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + ) + + assert result['staging_expires_at'] == '2026-01-01T01:00:00+00:00' + assert result['created_at'] == '2026-01-01T00:00:00+00:00' + sql = cursor.execute.call_args.args[0] + assert 'SELECT id, citation_table_name, source_sha256' in sql + assert 'encrypted_staging_locator' not in sql + assert 'created_by = %s' in sql + assert 'cancelled_at IS NULL' in sql + assert 'committed_batch_id IS NULL' in sql + assert 'staging_expires_at > CURRENT_TIMESTAMP' in sql + + +@pytest.mark.asyncio +async def test_commit_decrypts_owned_staging_and_delegates_atomic_transaction() -> None: + storage = MemoryStorage() + repository = Mock() + service = CitationImportPreviewService( + storage, repository, secret_key='unit-test-secret', + ) + preview_id = '00000000-0000-0000-0000-000000000001' + locator = 'test-storage/citation-import-previews/sr-1/preview.fernet' + source = b'Title,Abstract\nFirst,Summary\n' + storage.contents[locator] = service._fernet.encrypt( + json.dumps({ + 'filename': 'citations.csv', 'source_format': 'csv', + 'raw_file_base64': __import__('base64').b64encode(source).decode(), + 'normalized_rows': [{'Title': 'First', 'Abstract': 'Summary'}], + }).encode(), + ) + repository.get_staging_locator.return_value = locator + repository.commit.return_value = { + 'id': preview_id, 'batch_id': '00000000-0000-0000-0000-000000000002', 'idempotent': False, + 'inserted_count': 1, 'existing_exact_match_count': 0, 'invalid_count': 0, + } + + result = await service.commit( + preview_id=preview_id, sr_id='sr-1', actor_id='owner@example.test', + approved_mapping={'title': 'Title', 'abstract': 'Abstract'}, + ) + + assert result['inserted_count'] == 1 + repository.get_staging_locator.assert_called_once_with( + preview_id, 'sr-1', 'owner@example.test', + ) + staged_payload = repository.commit.call_args.args[4] + assert staged_payload['normalized_rows'] == [ + {'Title': 'First', 'Abstract': 'Summary'}, + ] + + +@pytest.mark.asyncio +async def test_commit_does_not_decrypt_or_mutate_when_preview_is_not_owned() -> None: + repository = Mock() + repository.get_staging_locator.return_value = None + service = CitationImportPreviewService( + MemoryStorage(), repository, secret_key='unit-test-secret', + ) + + with pytest.raises(LookupError, match='not found'): + await service.commit( + preview_id='00000000-0000-0000-0000-000000000001', sr_id='sr-1', actor_id='other@example.test', + approved_mapping={'title': 'Title', 'abstract': 'Abstract'}, + ) + + repository.commit.assert_not_called() + + +def test_repository_commit_locks_preview_and_writes_batch_provenance() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = [ + ( + 'sr_1_citations', hashlib.sha256(b'raw').hexdigest(), { + 'title': 'Title', 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, None, None, datetime( + 2030, 1, 1, tzinfo=timezone.utc, + ), + ), + ('sr_1_citations',), None, None, (42,), + ] + cursor.fetchall.return_value = [('id',), ('title',), ('abstract',)] + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, + { + 'filename': 'citations.csv', 'source_format': 'csv', + 'raw_file_base64': __import__('base64').b64encode(b'raw').decode(), + 'normalized_rows': [{'Title': 'First', 'Abstract': 'Summary'}], + }, + ) + + assert result['inserted_count'] == 1 + connection.commit.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'FOR UPDATE' in sql + assert 'INSERT INTO citation_import_batches' in sql + assert 'INSERT INTO citation_import_batch_memberships' in sql + assert 'INSERT INTO citation_identities' in sql + assert 'INSERT INTO citation_audit_events' in sql + assert 'UPDATE citation_import_previews SET committed_batch_id' in sql + + +def test_repository_commit_persists_namespaced_external_identities() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = [ + ( + 'sr_1_citations', hashlib.sha256(b'raw').hexdigest(), { + 'title': 'Title', 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, None, None, datetime( + 2030, 1, 1, tzinfo=timezone.utc, + ), + ), + ('sr_1_citations',), None, None, None, (42,), + ] + cursor.fetchall.return_value = [('id',), ('title',), ('abstract',)] + + CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, + { + 'raw_file_base64': __import__('base64').b64encode(b'raw').decode(), + 'normalized_rows': [{'Title': 'First', 'Abstract': 'Summary', 'PubMed ID': 'PMID: 123'}], + }, + ) + + external_id_calls = [ + call for call in cursor.execute.call_args_list + if "'external_id'" in call.args[0] + ] + assert len(external_id_calls) == 1 + assert external_id_calls[0].args[1][-2:] == ('pubmed', '123') + + +def test_repository_commit_rejects_changed_mapping_and_rolls_back_before_writes() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = ( + 'sr_1_citations', 'source-hash', { + 'title': 'Title', + 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, None, None, datetime( + 2030, 1, 1, tzinfo=timezone.utc, + ), + ) + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + with pytest.raises(ValueError, match='exactly match'): + CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Other', 'abstract': 'Abstract'}, {'normalized_rows': []}, + ) + + connection.rollback.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'INSERT INTO citation_import_batches' not in sql + assert 'INSERT INTO "sr_1_citations"' not in sql + + +def test_repository_commit_rejects_ambiguous_reconciliation_before_citation_writes() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.return_value = ( + 'sr_1_citations', 'source-hash', { + 'title': 'Title', + 'abstract': 'Abstract', + }, + {'valid_for_commit': True, 'reconciliation': {'counts': {'ambiguous': 1}}}, + None, None, datetime(2030, 1, 1, tzinfo=timezone.utc), + ) + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + with pytest.raises(ValueError, match='ambiguous exact'): + CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, {'normalized_rows': []}, + ) + + connection.rollback.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'INSERT INTO citation_import_batches' not in sql + assert 'INSERT INTO "sr_1_citations"' not in sql + + +def test_repository_commit_returns_original_batch_result_for_idempotent_retry() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = [ + ( + 'sr_1_citations', hashlib.sha256(b'raw').hexdigest(), { + 'title': 'Title', 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, '00000000-0000-0000-0000-000000000002', None, + datetime(2030, 1, 1, tzinfo=timezone.utc), + ), + (3, 2, 1), + ] + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, {'normalized_rows': []}, + ) + + assert result == { + 'id': '00000000-0000-0000-0000-000000000001', + 'batch_id': '00000000-0000-0000-0000-000000000002', 'idempotent': True, + 'inserted_count': 3, 'existing_exact_match_count': 2, 'invalid_count': 1, + } + connection.commit.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'INSERT INTO citation_import_batches' not in sql + + +def test_repository_commit_reuses_existing_committed_content_batch() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = [ + ( + 'sr_1_citations', hashlib.sha256(b'raw').hexdigest(), { + 'title': 'Title', 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, None, None, datetime( + 2030, 1, 1, tzinfo=timezone.utc, + ), + ), + ('sr_1_citations',), + ('00000000-0000-0000-0000-000000000002', 3, 2, 1), + ] + cursor.fetchall.return_value = [('id',), ('title',), ('abstract',)] + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, + { + 'raw_file_base64': __import__('base64').b64encode( + b'raw', + ).decode(), 'normalized_rows': [], + }, + ) + + assert result['batch_id'] == '00000000-0000-0000-0000-000000000002' + assert result['idempotent'] is True + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'FROM citation_import_batches' in sql + assert 'INSERT INTO citation_import_batches' not in sql + + +def test_repository_commit_creates_and_registers_initial_dataset_atomically() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = [ + ( + None, hashlib.sha256(b'raw').hexdigest(), { + 'title': 'Title', 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, None, None, datetime( + 2030, 1, 1, tzinfo=timezone.utc, + ), + ), + (None,), None, None, (17,), + ] + cursor.fetchall.return_value = [('id',), ('title',), ('abstract',)] + cursor.rowcount = 1 + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + result = CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, + { + 'filename': 'initial.csv', 'source_format': 'csv', + 'raw_file_base64': __import__('base64').b64encode(b'raw').decode(), + 'normalized_rows': [{'Title': 'First', 'Abstract': 'Summary'}], + }, + ) + + assert result['inserted_count'] == 1 + connection.commit.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'FROM systematic_reviews WHERE id = %s FOR UPDATE' in sql + assert 'CREATE TABLE "sr_' in sql + assert 'INSERT INTO citation_import_batches' in sql + assert 'UPDATE systematic_reviews' in sql + assert 'screening_db = jsonb_build_object' in sql + assert 'DROP TABLE' not in sql + + +def test_repository_initial_commit_rejects_concurrent_dataset_and_rolls_back() -> None: + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = [ + ( + None, hashlib.sha256(b'raw').hexdigest(), { + 'title': 'Title', 'abstract': 'Abstract', + }, + {'valid_for_commit': True}, None, None, datetime( + 2030, 1, 1, tzinfo=timezone.utc, + ), + ), + ('another_citation_table',), + ] + from api.services.citation_import_preview_service import CitationImportPreviewRepository + + with pytest.raises(ValueError, match='created while'): + CitationImportPreviewRepository(connection_provider=Mock(conn=connection)).commit( + '00000000-0000-0000-0000-000000000001', 'sr-1', 'owner@example.test', + {'title': 'Title', 'abstract': 'Abstract'}, + { + 'raw_file_base64': __import__('base64').b64encode( + b'raw', + ).decode(), 'normalized_rows': [], + }, + ) + + connection.rollback.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'CREATE TABLE' not in sql + assert 'UPDATE systematic_reviews' not in sql diff --git a/backend/tests/citation_legacy_adoption_service_test.py b/backend/tests/citation_legacy_adoption_service_test.py new file mode 100644 index 00000000..33cd1e67 --- /dev/null +++ b/backend/tests/citation_legacy_adoption_service_test.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest +from api.services.citation_legacy_adoption_service import CitationLegacyAdoptionService + + +def _service_with_rows(*rows): + connection = Mock() + cursor = connection.cursor.return_value + cursor.fetchone.side_effect = rows + return CitationLegacyAdoptionService(connection_provider=Mock(conn=connection)), connection, cursor + + +def test_dry_run_validates_scope_and_returns_count_without_writes(): + service, connection, cursor = _service_with_rows((1,), (1,), (7,)) + + result = service.adopt( + 'sr-1', 'sr_1_citations', + 'owner@example.test', dry_run=True, + ) + + assert result['dry_run'] is True + assert result['citation_count'] == 7 + assert result['memberships_created'] == 0 + connection.rollback.assert_called_once_with() + connection.commit.assert_not_called() + assert cursor.execute.call_count == 3 + + +def test_adoption_writes_only_metadata_and_is_deterministic(): + service, connection, cursor = _service_with_rows((1,), (1,), (2,)) + cursor.rowcount = 2 + + result = service.adopt('sr-1', 'sr_1_citations', 'owner@example.test') + + assert result['batch_id'] == service.legacy_batch_id( + 'sr-1', 'sr_1_citations', + ) + assert result['citation_count'] == 2 + assert result['memberships_created'] == 2 + connection.commit.assert_called_once_with() + sql = '\n'.join(call.args[0] for call in cursor.execute.call_args_list) + assert 'INSERT INTO citation_import_batches' in sql + assert 'INSERT INTO citation_import_batch_memberships' in sql + assert 'FROM "sr_1_citations"' in sql + assert 'INSERT INTO "sr_1_citations"' not in sql + + +def test_adoption_rejects_table_not_owned_by_review_and_rolls_back(): + service, connection, _ = _service_with_rows(None) + + with pytest.raises(ValueError, match='not configured'): + service.adopt('sr-1', 'sr_1_citations', 'owner@example.test') + + connection.rollback.assert_called_once_with() + + +def test_adoption_rejects_unsafe_dynamic_identifier_before_database_access(): + service = CitationLegacyAdoptionService(connection_provider=Mock()) + + with pytest.raises(ValueError, match='Invalid citation_table_name'): + service.adopt( + 'sr-1', 'citations; DROP TABLE users', + 'owner@example.test', + ) diff --git a/backend/tests/criteria_context_test.py b/backend/tests/criteria_context_test.py index 8d6e62d2..4e13a50d 100644 --- a/backend/tests/criteria_context_test.py +++ b/backend/tests/criteria_context_test.py @@ -64,11 +64,10 @@ def test_metrics_ignore_legacy_llm_autofilled_human_values(self) -> None: {'selected': 'Yes', 'source': 'llm', 'autofilled': True}, ), ) - self.assertEqual( + self.assertIsNone( _parse_selected_from_human_payload( {'selected': 'Yes', 'source': 'csv_upload', 'autofilled': True}, ), - 'Yes', ) self.assertEqual( _parse_selected_from_human_payload( diff --git a/backend/tests/criteria_runtime_test.py b/backend/tests/criteria_runtime_test.py new file mode 100644 index 00000000..e825ab1c --- /dev/null +++ b/backend/tests/criteria_runtime_test.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from api.criteria.runtime import item_is_visible + + +def test_trigger_prefers_stage_specific_human_answer() -> None: + criteria = { + 'l1': [ + { + 'id': 'q1', + 'question': 'Primary research?', + 'options': [{'id': 'yes', 'label': 'Yes'}], + }, + { + 'id': 'q2', + 'question': 'Human population?', + 'options': [{'id': 'yes', 'label': 'Yes'}], + 'trigger': {'all': [{'source_item_id': 'q1', 'option_id': 'yes'}]}, + }, + ], + } + + assert item_is_visible( + criteria, criteria['l1'][1], { + 'human_l1_primary_research': {'selected': 'Yes'}, + }, + ) + assert not item_is_visible( + criteria, criteria['l1'][1], { + 'human_l1_primary_research': {'selected': 'No'}, + 'llm_l1_primary_research': {'selected': 'Yes'}, + }, + ) + + +def test_trigger_falls_back_to_llm_answer_when_human_answer_is_missing() -> None: + criteria = { + 'l2': [ + { + 'id': 'q1', + 'question': 'Full text eligible?', + 'options': [{'id': 'yes', 'label': 'Yes'}], + }, + { + 'id': 'q2', + 'question': 'Extract data?', + 'options': [{'id': 'yes', 'label': 'Yes'}], + 'trigger': {'all': [{'source_item_id': 'q1', 'option_id': 'yes'}]}, + }, + ], + } + + assert item_is_visible( + criteria, criteria['l2'][1], { + 'llm_l2_full_text_eligible': {'selected': 'Yes'}, + }, + ) + assert item_is_visible( + criteria, criteria['l2'][1], { + 'llm_full_text_eligible': {'selected': 'Yes'}, + }, + ) + assert item_is_visible( + criteria, criteria['l2'][1], { + 'human_l2_full_text_eligible': {'selected': 'Yes'}, + }, + ) diff --git a/backend/tests/migration_cli_test.py b/backend/tests/migration_cli_test.py new file mode 100644 index 00000000..371db0f0 --- /dev/null +++ b/backend/tests/migration_cli_test.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import sys + +import pytest +from api.migrations import cli + + +def test_adopt_legacy_requires_scope_and_actor_arguments(monkeypatch, capsys): + monkeypatch.setattr(sys, 'argv', ['can-sr-migrations', 'adopt-legacy']) + + with pytest.raises(SystemExit) as exc_info: + cli.main() + + assert exc_info.value.code == 2 + assert 'adopt-legacy requires --sr-id, --table-name, --actor-id' in capsys.readouterr().err + + +def test_adopt_legacy_passes_validated_arguments_to_service(monkeypatch, capsys): + monkeypatch.setattr( + sys, 'argv', [ + 'can-sr-migrations', 'adopt-legacy', '--sr-id', 'sr-1', + '--table-name', 'sr_1_citations', '--actor-id', 'owner@example.test', '--dry-run', + ], + ) + calls = [] + monkeypatch.setattr( + cli.citation_legacy_adoption_service, 'adopt', + lambda *args, **kwargs: calls.append( + (args, kwargs), + ) or {'dry_run': True}, + ) + + cli.main() + + assert calls == [ + (('sr-1', 'sr_1_citations', 'owner@example.test'), {'dry_run': True}), + ] + assert "'dry_run': True" in capsys.readouterr().out + + +def test_cleanup_set_answer_validation_is_scoped_and_dry_run_by_default(monkeypatch, capsys): + monkeypatch.setattr( + sys, 'argv', [ + 'can-sr-migrations', 'cleanup-set-answer-validation', + '--table-name', 'sr_1_citations', '--step', 'l1', + ], + ) + calls = [] + monkeypatch.setattr( + cli.cits_dp_service, 'cleanup_set_answer_validation_metadata', + lambda *args, **kwargs: calls.append( + (args, kwargs), + ) or {'dry_run': True}, + ) + + cli.main() + + assert calls == [ + (('sr_1_citations', 'l1'), {'execute': False}), + ] + assert "'dry_run': True" in capsys.readouterr().out diff --git a/backend/tests/pdf_linkage_service_test.py b/backend/tests/pdf_linkage_service_test.py index 7ce47608..1205da91 100644 --- a/backend/tests/pdf_linkage_service_test.py +++ b/backend/tests/pdf_linkage_service_test.py @@ -58,9 +58,14 @@ async def test_ten_open_access_dois_are_resolved_and_downloaded_over_network(sel ] async with httpx.AsyncClient(timeout=httpx.Timeout(45, connect=15)) as client: for doi in dois: - response = await _request_with_retry( - client, f'{OA_API_BASE_URL}?doi={quote(doi, safe="")}', - ) + try: + response = await _request_with_retry( + client, f'{OA_API_BASE_URL}?doi={quote(doi, safe="")}', + ) + except (httpx.ConnectError, httpx.ConnectTimeout) as exc: + self.skipTest( + f'network unavailable for live OA test: {exc}', + ) payload = response.json() candidate = _candidate(payload) self.assertTrue( diff --git a/backend/tests/review_authorization_service_test.py b/backend/tests/review_authorization_service_test.py new file mode 100644 index 00000000..c7ee5a94 --- /dev/null +++ b/backend/tests/review_authorization_service_test.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import pytest +from api.services.sr_db_service import SRDBService +from fastapi import HTTPException + + +def test_require_review_role_returns_matching_role(monkeypatch): + service = SRDBService() + monkeypatch.setattr(service, 'get_member_role', lambda *_: 'member') + + assert service.require_review_role( + 'sr-1', 'member@example.test', {'member', 'owner'}, + ) == 'member' + + +def test_require_review_owner_accepts_co_owner_and_rejects_member(monkeypatch): + service = SRDBService() + monkeypatch.setattr( + service, 'get_member_role', lambda _sr_id, + member_id: 'owner' if member_id == 'co-owner@example.test' else 'member', + ) + + service.require_review_owner('sr-1', 'co-owner@example.test') + with pytest.raises(HTTPException) as exc_info: + service.require_review_owner('sr-1', 'member@example.test') + assert exc_info.value.status_code == 403 + + +def test_require_review_owner_rejects_missing_membership(monkeypatch): + service = SRDBService() + monkeypatch.setattr(service, 'get_member_role', lambda *_: None) + + with pytest.raises(HTTPException) as exc_info: + service.require_review_owner('sr-1', 'outsider@example.test') + assert exc_info.value.status_code == 403 diff --git a/backend/tests/review_membership_postgres_integration_test.py b/backend/tests/review_membership_postgres_integration_test.py new file mode 100644 index 00000000..9aed004c --- /dev/null +++ b/backend/tests/review_membership_postgres_integration_test.py @@ -0,0 +1,274 @@ +"""Live PostgreSQL invariants for review member roles and co-ownership.""" +from __future__ import annotations + +import os +import unittest +import uuid +from pathlib import Path + +import psycopg2.extras +from api.services import sr_db_service as srdb_module +from api.services.sr_db_service import SRDBService +from fastapi import HTTPException + + +def _connect(): + return psycopg2.connect( + host=os.getenv('POSTGRES_TEST_HOST', 'localhost'), + port=int(os.getenv('POSTGRES_PORT', '5432')), + dbname=os.getenv('POSTGRES_DATABASE', 'postgres'), + user=os.getenv('POSTGRES_USER', 'admin'), + password=os.getenv('POSTGRES_PASSWORD', 'password'), + connect_timeout=2, + ) + + +class _Provider: + def __init__(self, conn): + self.conn = conn + + +class ReviewMembershipPostgresIntegrationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + cls.conn = _connect() + except Exception as exc: + raise unittest.SkipTest(f'PostgreSQL unavailable: {exc}') + cur = cls.conn.cursor() + try: + cur.execute( + """CREATE TABLE IF NOT EXISTS systematic_reviews ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, owner_id TEXT NOT NULL, + owner_email TEXT, users JSONB DEFAULT '[]'::jsonb, + visible BOOLEAN DEFAULT TRUE, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP + )""", + ) + cur.execute( + 'ALTER TABLE systematic_reviews ADD COLUMN IF NOT EXISTS owner_email TEXT', + ) + cur.execute( + "ALTER TABLE systematic_reviews ADD COLUMN IF NOT EXISTS users JSONB DEFAULT '[]'::jsonb", + ) + migration = Path(__file__).resolve( + ).parents[1] / 'migrations' / '004_systematic_review_memberships.sql' + cur.execute(migration.read_text(encoding='utf-8')) + cls.conn.commit() + except Exception: + cls.conn.rollback() + raise + finally: + cur.close() + cls.original_postgres_server = srdb_module.postgres_server + srdb_module.postgres_server = _Provider(cls.conn) + cls.service = SRDBService() + + @classmethod + def tearDownClass(cls): + srdb_module.postgres_server = cls.original_postgres_server + cls.conn.close() + + def setUp(self): + self.sr_id = f'membership_pg_{uuid.uuid4().hex}' + self.owner = 'owner@example.test' + self.member = 'member@example.test' + self.co_owner = 'co-owner@example.test' + + def tearDown(self): + cur = self.conn.cursor() + try: + cur.execute( + 'DELETE FROM systematic_review_membership_audit_events WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM systematic_review_memberships WHERE sr_id = %s', ( + self.sr_id, + ), + ) + cur.execute( + 'DELETE FROM systematic_reviews WHERE id = %s', (self.sr_id,), + ) + self.conn.commit() + finally: + cur.close() + + def _insert_legacy_review(self, users=None): + cur = self.conn.cursor() + try: + cur.execute( + """INSERT INTO systematic_reviews (id, name, owner_id, owner_email, users) + VALUES (%s, %s, %s, %s, %s::jsonb)""", + ( + self.sr_id, 'membership integration', 'owner-stable-id', self.owner, + psycopg2.extras.Json( + users if users is not None else [ + self.owner, self.member, + ], + ), + ), + ) + self.conn.commit() + finally: + cur.close() + + def _apply_backfill_for_scope(self): + cur = self.conn.cursor() + try: + cur.execute( + """INSERT INTO systematic_review_memberships (sr_id, member_id, role, added_by) + SELECT id, COALESCE(NULLIF(owner_email, ''), owner_id), 'owner', COALESCE(NULLIF(owner_email, ''), owner_id) + FROM systematic_reviews WHERE id = %s + ON CONFLICT (sr_id, member_id) DO UPDATE SET role = 'owner', updated_at = CURRENT_TIMESTAMP""", + (self.sr_id,), + ) + cur.execute( + """INSERT INTO systematic_review_memberships (sr_id, member_id, role, added_by) + SELECT sr.id, member.value, 'member', COALESCE(NULLIF(sr.owner_email, ''), sr.owner_id) + FROM systematic_reviews sr + CROSS JOIN LATERAL jsonb_array_elements_text(COALESCE(sr.users, '[]'::jsonb)) AS member(value) + WHERE sr.id = %s AND member.value <> '' + ON CONFLICT (sr_id, member_id) DO NOTHING""", + (self.sr_id,), + ) + self.conn.commit() + finally: + cur.close() + + def _scalar(self, sql, params=()): + cur = self.conn.cursor() + try: + cur.execute(sql, params) + return cur.fetchone()[0] + finally: + cur.close() + + def test_backfill_promote_and_audit_membership_roles(self): + self._insert_legacy_review() + self._apply_backfill_for_scope() + + self.assertEqual( + self.service.get_member_role( + self.sr_id, self.owner, + ), 'owner', + ) + self.assertEqual( + self.service.get_member_role( + self.sr_id, self.member, + ), 'member', + ) + promoted = self.service.set_member_role( + self.sr_id, self.member, 'owner', self.owner, + ) + self.assertTrue(promoted['changed']) + self.assertTrue(self.service.user_is_sr_owner(self.sr_id, self.member)) + self.assertEqual( + self._scalar( + 'SELECT event_type FROM systematic_review_membership_audit_events WHERE sr_id = %s ORDER BY created_at DESC LIMIT 1', + (self.sr_id,), + ), 'role_changed', + ) + + def test_co_owner_is_recognized_as_owner_and_member_is_not(self): + self._insert_legacy_review() + self._apply_backfill_for_scope() + self.service.set_member_role( + self.sr_id, self.co_owner, 'owner', self.owner, + ) + + self.assertTrue( + self.service.user_is_sr_owner( + self.sr_id, self.co_owner, + ), + ) + self.assertFalse( + self.service.user_is_sr_owner( + self.sr_id, self.member, + ), + ) + + def test_require_review_owner_rejects_database_backed_regular_member(self): + self._insert_legacy_review() + self._apply_backfill_for_scope() + + with self.assertRaises(HTTPException) as error: + self.service.require_review_owner(self.sr_id, self.member) + + self.assertEqual(error.exception.status_code, 403) + + def test_final_owner_cannot_be_demoted_or_removed(self): + self._insert_legacy_review(users=[self.owner]) + self._apply_backfill_for_scope() + + with self.assertRaisesRegex(ValueError, 'At least one owner'): + self.service.set_member_role( + self.sr_id, self.owner, 'member', self.owner, + ) + with self.assertRaisesRegex(ValueError, 'At least one owner'): + self.service.remove_member(self.sr_id, self.owner, self.owner) + self.assertEqual( + self.service.get_member_role( + self.sr_id, self.owner, + ), 'owner', + ) + + def test_co_owner_can_change_visibility_but_member_is_denied(self): + self._insert_legacy_review() + self._apply_backfill_for_scope() + self.service.set_member_role( + self.sr_id, self.co_owner, 'owner', self.owner, + ) + + result = self.service.set_visibility(self.sr_id, False, self.co_owner) + self.assertEqual(result['visible'], False) + with self.assertRaises(HTTPException) as error: + self.service.set_visibility(self.sr_id, True, self.member) + self.assertEqual(error.exception.status_code, 403) + self.assertFalse( + self._scalar( + 'SELECT visible FROM systematic_reviews WHERE id = %s', ( + self.sr_id, + ), + ), + ) + + def test_audit_failure_rolls_back_new_membership_and_legacy_sync(self): + self._insert_legacy_review(users=[self.owner]) + self._apply_backfill_for_scope() + function = f'fail_membership_audit_{uuid.uuid4().hex}' + trigger = f'fail_membership_audit_{uuid.uuid4().hex}' + cur = self.conn.cursor() + try: + cur.execute( + f"""CREATE FUNCTION {function}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'forced membership audit failure'; END $$""", + ) + cur.execute( + f'CREATE TRIGGER {trigger} BEFORE INSERT ON systematic_review_membership_audit_events ' + f'FOR EACH ROW EXECUTE FUNCTION {function}()', + ) + self.conn.commit() + with self.assertRaisesRegex(Exception, 'forced membership audit failure'): + self.service.set_member_role( + self.sr_id, self.member, 'member', self.owner, + ) + self.assertIsNone( + self.service.get_member_role( + self.sr_id, self.member, + ), + ) + self.assertEqual( + self._scalar( + 'SELECT users::text FROM systematic_reviews WHERE id = %s', ( + self.sr_id, + ), + ), '["owner@example.test"]', + ) + finally: + cur.execute( + f'DROP TRIGGER IF EXISTS {trigger} ON systematic_review_membership_audit_events', + ) + cur.execute(f'DROP FUNCTION IF EXISTS {function}()') + self.conn.commit() + cur.close() diff --git a/backend/tests/review_schema_service_test.py b/backend/tests/review_schema_service_test.py index ccd84b73..dfb751bc 100644 --- a/backend/tests/review_schema_service_test.py +++ b/backend/tests/review_schema_service_test.py @@ -15,18 +15,92 @@ def test_migration_files_are_ordered(): files = schema.migration_files() assert files == sorted(files) assert files[0].stem == '001_multi_reviewer_schema' + assert files[1].stem == '002_citation_workspace_schema' + assert files[2].stem == '003_citation_import_preview_mapping_decisions' + assert files[3].stem == '004_systematic_review_memberships' + assert files[4].stem == '005_citation_workspace_column_preferences' def test_canonical_version_is_based_on_migration_files_not_legacy_markers(): assert schema.MIGRATION_VERSION == '001_multi_reviewer_schema' +def test_citation_workspace_migration_contains_legacy_adoption_schema(): + migration = Path(__file__).resolve( + ).parents[1] / 'migrations' / '002_citation_workspace_schema.sql' + sql = migration.read_text(encoding='utf-8') + for table in ( + 'citation_import_batches', 'citation_import_batch_memberships', + 'citation_import_previews', 'citation_identities', + ): + assert f'CREATE TABLE IF NOT EXISTS {table}' in sql + assert 'REFERENCES citation_import_batches(id)' in sql + + +def test_mapping_decision_migration_is_additive(): + migration = Path(__file__).resolve( + ).parents[1] / 'migrations' / '003_citation_import_preview_mapping_decisions.sql' + assert 'ADD COLUMN IF NOT EXISTS mapping_decision JSONB' in migration.read_text( + encoding='utf-8', + ) + + +def test_membership_role_migration_backfills_legacy_owners_and_members(): + migration = Path(__file__).resolve( + ).parents[1] / 'migrations' / '004_systematic_review_memberships.sql' + sql = migration.read_text(encoding='utf-8') + assert 'CREATE TABLE IF NOT EXISTS systematic_review_memberships' in sql + assert "role IN ('owner', 'member')" in sql + assert 'jsonb_array_elements_text' in sql + assert "'owner'" in sql + assert 'systematic_review_membership_audit_events' in sql + + def test_migration_checksum_is_deterministic(): sql = 'CREATE TABLE foundation (id integer);' assert schema.migration_checksum(sql) == schema.migration_checksum(sql) assert len(schema.migration_checksum(sql)) == 64 +def test_refresh_checksums_updates_existing_migrations_only(): + class Cursor: + rowcount = 1 + + def execute(self, query, params=None): + self.query = query + self.params = params + + def close(self): + pass + + class Connection: + def __init__(self): + self.cursor_instance = Cursor() + self.commits = 0 + + def cursor(self): + return self.cursor_instance + + def commit(self): + self.commits += 1 + + def rollback(self): + pass + + connection = Connection() + service = schema.ReviewSchemaService(lambda: connection) + + # The service expects the provider's connection as an attribute, matching + # the production postgres provider contract. + service.connection_provider = type('Provider', (), {'conn': connection})() + result = service.refresh_checksums() + + assert result['refreshed'] == [ + path.stem for path in schema.migration_files() + ] + assert connection.commits == 1 + + def test_feature_is_disabled_by_default(monkeypatch): # The imported settings object is intentionally not mutated. This checks # the source-level default contract without requiring a database. diff --git a/backend/tests/screening_eligibility_service_test.py b/backend/tests/screening_eligibility_service_test.py index 7a426cec..950faea1 100644 --- a/backend/tests/screening_eligibility_service_test.py +++ b/backend/tests/screening_eligibility_service_test.py @@ -27,9 +27,10 @@ class ScreeningDecisionTests(unittest.TestCase): def test_human_answer_overrides_ai_answer(self) -> None: row = { - 'human_relevant_population': {'selected': 'Include'}, - 'llm_relevant_population': {'selected': 'Exclude'}, - 'human_eligible_study_design': {'selected': 'Include'}, + 'human_l1_relevant_population': {'selected': 'Include'}, + 'human_l2_relevant_population': {'selected': 'Include'}, + 'llm_l1_relevant_population': {'selected': 'Exclude'}, + 'human_l2_eligible_study_design': {'selected': 'Include'}, } self.assertEqual( @@ -39,8 +40,8 @@ def test_human_answer_overrides_ai_answer(self) -> None: def test_ai_answer_is_used_when_human_answer_is_missing(self) -> None: row = { - 'llm_relevant_population': '{"selected": "Include"}', - 'llm_eligible_study_design': {'selected': 'Include'}, + 'llm_l1_relevant_population': '{"selected": "Include"}', + 'llm_l2_eligible_study_design': {'selected': 'Include'}, } self.assertEqual( @@ -48,13 +49,25 @@ def test_ai_answer_is_used_when_human_answer_is_missing(self) -> None: ('include', 'include'), ) + def test_empty_human_answer_falls_back_but_explicit_exclude_wins(self) -> None: + row = { + 'human_l1_relevant_population': {'selected': ' '}, + 'llm_l1_relevant_population': {'selected': 'Include'}, + 'human_l2_eligible_study_design': {'selected': 'Exclude'}, + 'llm_l2_eligible_study_design': {'selected': 'Include'}, + } + self.assertEqual( + compute_screening_decisions(row, CRITERIA), + ('include', 'exclude'), + ) + def test_critical_ai_answer_does_not_override_screening_ai_answer(self) -> None: row = { - 'llm_relevant_population': { + 'human_l1_relevant_population': { 'selected': 'Include', - 'critical': {'selected': 'Exclude'}, }, - 'llm_eligible_study_design': {'selected': 'Include'}, + 'human_l2_relevant_population': {'selected': 'Include'}, + 'human_l2_eligible_study_design': {'selected': 'Include'}, } self.assertEqual( @@ -63,7 +76,7 @@ def test_critical_ai_answer_does_not_override_screening_ai_answer(self) -> None: ) def test_l2_decision_is_cumulative_and_missing_answers_exclude(self) -> None: - row = {'human_relevant_population': {'selected': 'Include'}} + row = {'human_l1_relevant_population': {'selected': 'Include'}} self.assertEqual( compute_screening_decisions(row, CRITERIA), diff --git a/backend/tests/screening_metrics_test.py b/backend/tests/screening_metrics_test.py new file mode 100644 index 00000000..a6da35f4 --- /dev/null +++ b/backend/tests/screening_metrics_test.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from api.screen.router import _human_answer_column +from api.screen.router import _screening_metric_values +from api.screen.router import ScreeningMetricsCriterion + + +def test_metric_human_columns_are_stage_specific() -> None: + assert _human_answer_column( + 'l1', 'primary_research', + ) == 'human_l1_primary_research' + assert _human_answer_column( + 'l2', 'primary_research', + ) == 'human_l2_primary_research' + + +def test_metric_values_include_accuracy_and_confusion_metrics() -> None: + values = _screening_metric_values({ + 'human_total_count': 4, + 'human_agree_count': 3, + 'human_total_count_all': 5, + 'human_agree_count_all': 4, + 'crit_total_count': 2, + 'crit_agree_count': 1, + 'cm_tp': 3, + 'cm_fp': 1, + 'cm_fn': 2, + 'cm_tn': 4, + }) + + assert values['accuracy'] == 0.75 + assert values['accuracy_all'] == 0.8 + assert values['accuracy_critical_agent'] == 0.5 + assert values['f1_score'] == 0.6666666666666666 + assert values['precision'] == 0.75 + assert values['recall'] == 0.6 + assert values['npv'] == 4 / 6 + assert values['confusion_matrix'] == {'tp': 3, 'fp': 1, 'fn': 2, 'tn': 4} + + +def test_metric_pydantic_model_retains_calculated_fields() -> None: + model = ScreeningMetricsCriterion( + criterion_key='primary_research', label='Primary research', threshold=0.8, + total_citations=10, has_run_count=10, low_confidence_count=1, + critical_disagreement_count=1, confident_exclude_count=2, + needs_human_review_count=3, accuracy=0.75, accuracy_all=0.8, + f1_score=2 / 3, precision=0.75, recall=0.6, npv=4 / 6, + confusion_matrix={'tp': 3, 'fp': 1, 'fn': 2, 'tn': 4}, + ) + dumped = model.model_dump() + assert dumped['accuracy'] == 0.75 + assert dumped['f1_score'] == 2 / 3 + assert dumped['confusion_matrix']['tp'] == 3 diff --git a/backend/tests/set_answer_test.py b/backend/tests/set_answer_test.py new file mode 100644 index 00000000..82fd15f9 --- /dev/null +++ b/backend/tests/set_answer_test.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from api.citations.router import _set_answer_destinations +from api.citations.router import _set_answer_payload + + +def test_l1_set_answer_writes_both_stage_columns() -> None: + assert _set_answer_destinations('l1', 'primary_research') == [ + 'human_l1_primary_research', 'human_l2_primary_research', + ] + + +def test_l2_set_answer_writes_only_l2() -> None: + assert _set_answer_destinations('l2', 'full_text_eligible') == [ + 'human_l2_full_text_eligible', + ] + + +def test_parameter_set_answer_writes_only_parameter_column() -> None: + assert _set_answer_destinations('parameter', 'sample_size') == [ + 'human_param_sample_size', + ] + + +def test_set_answer_payload_preserves_source_value_and_provenance() -> None: + answer = _set_answer_payload(' Yes ', 'l1', 'l1') + assert answer['selected'] == 'Yes' + assert answer['human'] is True + assert answer['source'] == 'retrospective_validation' + assert answer['screening_step'] == 'l1' + + +def test_parameter_payload_uses_extraction_shape() -> None: + answer = _set_answer_payload('42', 'parameter', 'parameter') + assert answer['found'] is True + assert answer['value'] == '42' + assert answer['evidence_sentences'] == [] diff --git a/frontend/app/[lang]/can-sr/l1-screen/view/page.tsx b/frontend/app/[lang]/can-sr/l1-screen/view/page.tsx index a4303cd3..8e1c9f8d 100644 --- a/frontend/app/[lang]/can-sr/l1-screen/view/page.tsx +++ b/frontend/app/[lang]/can-sr/l1-screen/view/page.tsx @@ -19,7 +19,7 @@ import { ScreeningCitationContext, extractHumanAnswer, humanAnswerStatus, resolv - Render a workspace (left) that shows title + abstract (flexible container for future PDF/text viewers) - Render a selection sidebar (right) with one section per L1 question: - Dropdown containing options from criteriaData.possible_answers - - Default selection set to AI answer from the citation row (column name computed via snake_case_column) + * - Default selection set only from the persisted stage-specific human answer - "Classify" button that calls the backend screen classify endpoint and updates the AI panel - If an AI answer exists for that question, show a collapsible panel containing the parsed LLM JSON (expected fields: selected, explanation, confidence, llm_raw/) @@ -48,8 +48,8 @@ function snakeCaseColumn(name: string) { } /** - * Human classification column for screening (mirrors llm_ prefix but with human_). - * Example: question -> llm_question, human_question + * Human classification column for screening. + * Example: question -> human_question */ function humanScreenColumn(name: string) { const base = snakeCaseColumn(name) @@ -483,15 +483,6 @@ export default function CanSrL1ScreenPage() { newPanelOpen[idx] = false } - // 3) If no human selection exists, allow llm_* to prefill the dropdown (UI-only). - const hasSelection = newSelections[idx] !== undefined && newSelections[idx] !== '' - if (!hasSelection) { - const aiSelected = - newAiPanels[idx] && typeof (newAiPanels[idx] as any).selected === 'string' - ? (newAiPanels[idx] as any).selected - : null - if (aiSelected) newSelections[idx] = aiSelected - } }) setSelections(newSelections) @@ -762,9 +753,6 @@ export default function CanSrL1ScreenPage() { const answerStatus = humanAnswerStatus(citation!, answerColumn, options) const current = selections[idx] ?? '' const aiData = aiPanels[idx] - const aiSelected = - aiData && aiData.selected ? aiData.selected : undefined - // Per-question highlight aligned with list/back-end logic. // - Highlight when this criterion triggers review (low confidence OR critical disagreement OR guardrail issue) // - BUT do not highlight if this criterion is a confident-exclude (exclude + conf>=thr + critical agrees) @@ -816,7 +804,7 @@ export default function CanSrL1ScreenPage() {

onSelectOption(idx, e.target.value) } diff --git a/frontend/app/[lang]/can-sr/protocols/page.tsx b/frontend/app/[lang]/can-sr/protocols/page.tsx new file mode 100644 index 00000000..1c0fa309 --- /dev/null +++ b/frontend/app/[lang]/can-sr/protocols/page.tsx @@ -0,0 +1,72 @@ +'use client' + +import { useCallback, useEffect, useState } from 'react' +import Link from 'next/link' +import { useRouter, useSearchParams } from 'next/navigation' +import GCHeader, { SRHeader } from '@/components/can-sr/headers' +import CriteriaEditor from '@/components/can-sr/setup/criteria-editor' +import { authenticatedFetch } from '@/lib/auth' +import { useDictionary } from '../../DictionaryProvider' + +export default function ProtocolsPage() { + const router = useRouter() + const searchParams = useSearchParams() + const srId = searchParams?.get('sr_id') + const dict = useDictionary() + const [hasDataset, setHasDataset] = useState(null) + + const loadReview = useCallback(async () => { + if (!srId) return + const response = await authenticatedFetch( + `/api/can-sr/reviews/create?sr_id=${encodeURIComponent(srId)}`, + ) + if (!response.ok) return + const review = await response.json().catch(() => ({})) + setHasDataset(Boolean(review?.screening_db?.table_name)) + }, [srId]) + + useEffect(() => { + if (!srId) { + router.replace('/can-sr') + return + } + void loadReview() + }, [loadReview, router, srId]) + + if (!srId) return null + + return ( +
+ + +
+ {hasDataset === false ? ( +
+

{dict.protocols.noDataset}

+ + {dict.protocols.addReferences} + +
+ ) : null} +
+

+ {dict.protocols.criteriaSection} +

+ + } + /> +
+
+
+ ) +} diff --git a/frontend/app/[lang]/can-sr/references/page.tsx b/frontend/app/[lang]/can-sr/references/page.tsx new file mode 100644 index 00000000..c3a85f32 --- /dev/null +++ b/frontend/app/[lang]/can-sr/references/page.tsx @@ -0,0 +1,47 @@ +'use client' + +import { useCallback, useEffect, useState } from 'react' +import { useRouter, useSearchParams } from 'next/navigation' +import GCHeader, { SRHeader } from '@/components/can-sr/headers' +import ReferencesWorkspace from '@/components/can-sr/references/references-workspace' +import { authenticatedFetch } from '@/lib/auth' +import { useDictionary } from '../../DictionaryProvider' + +export default function ReferencesPage() { + const router = useRouter() + const searchParams = useSearchParams() + const srId = searchParams?.get('sr_id') + const dict = useDictionary() + const [hasDataset, setHasDataset] = useState(null) + const load = useCallback(async () => { + if (!srId) return + const response = await authenticatedFetch( + `/api/can-sr/reviews/create?sr_id=${encodeURIComponent(srId)}`, + ) + if (response.ok) { + const review = await response.json().catch(() => ({})) + setHasDataset(Boolean(review?.screening_db?.table_name)) + } + }, [srId]) + useEffect(() => { + if (!srId) router.replace('/can-sr') + else void load() + }, [load, router, srId]) + if (!srId) return null + return ( +
+ + +
+ } + /> +
+
+ ) +} diff --git a/frontend/app/[lang]/can-sr/search/page.tsx b/frontend/app/[lang]/can-sr/search/page.tsx deleted file mode 100644 index ed3b18d1..00000000 --- a/frontend/app/[lang]/can-sr/search/page.tsx +++ /dev/null @@ -1,122 +0,0 @@ -'use client' - -import { useRouter, useSearchParams } from 'next/navigation' -import GCHeader, { SRHeader } from '@/components/can-sr/headers' -import { ModelSelector } from '@/components/chat' -import React, { useState } from 'react' -import { getAuthToken, getTokenType } from '@/lib/auth' -import { useDictionary } from '../../DictionaryProvider' - -function getAuthHeaders(): Record { - const token = getAuthToken() - const tokenType = getTokenType() - return token - ? { Authorization: `${tokenType} ${token}` } - : ({} as Record) -} - -export default function Search() { - const router = useRouter() - const searchParams = useSearchParams() - const srId = searchParams?.get('sr_id') - const dict = useDictionary() - const [selectedModel, setSelectedModel] = useState('') - const databases = ['Pubmed', 'Scopus', 'EuropePMC'] - const [selectedDatabase, setSelectedDatabase] = useState('') - const [searchStrings, setSearchString] = useState>({}) - - if (!srId) { - router.replace('/can-sr') - return - } - - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault() - if (!selectedDatabase) { - alert('Please select a database') - return - } - const headers = getAuthHeaders() - - const bodyPayload = { - database: selectedDatabase, - search_term: searchStrings[selectedDatabase], - } - await fetch( - `/api/can-sr/search?sr_id=${encodeURIComponent(srId)}`, - { - method: 'POST', - headers, - body: JSON.stringify(bodyPayload), - }, - ) - } - - const handleTextChange = (database: string, value: string) => { - setSearchString((prev) => ({ ...prev, [database]: value })) - } - - return ( -
- - - - } - /> -
-
-

- {dict.search.selectDatabase} -

-

- {dict.search.selectDatabaseDesc} -

- -
- {databases.map((database) => ( -
- setSelectedDatabase(database)} - className="h-4 w-4" - /> - - - handleTextChange(database, e.target.value)} - className="flex-1 rounded-md border px-2 py-1 text-sm" - /> -
- ))} - -
-
-
-
- ) -} diff --git a/frontend/app/[lang]/can-sr/setup/page.tsx b/frontend/app/[lang]/can-sr/setup/page.tsx index 7b9a16c7..88127892 100644 --- a/frontend/app/[lang]/can-sr/setup/page.tsx +++ b/frontend/app/[lang]/can-sr/setup/page.tsx @@ -1,215 +1,19 @@ 'use client' -import React, { useCallback, useEffect, useRef, useState } from 'react' -import { useSearchParams } from 'next/navigation' -import GCHeader, { SRHeader } from '@/components/can-sr/headers' -import { authenticatedFetch, getAuthToken, getTokenType } from '@/lib/auth' -import ManageUsersPopup from '@/components/can-sr/setup/manage-users-popup' -import CriteriaEditor from '@/components/can-sr/setup/criteria-editor' -import { Settings } from 'lucide-react' -import { useDictionary } from '../../DictionaryProvider' +import { useEffect } from 'react' +import { useRouter, useSearchParams } from 'next/navigation' -function getAuthHeaders(): Record { - const token = getAuthToken() - const tokenType = getTokenType() - return token ? { Authorization: `${tokenType} ${token}` } : ({} as Record) -} - - // CSV preview removed - client-side preview disabled - -export default function CanSrSetupPage() { +/** Legacy compatibility route. The protocol workspace owns configuration UI. */ +export default function CanSrSetupRedirectPage() { + const router = useRouter() const searchParams = useSearchParams() const srId = searchParams?.get('sr_id') - const dict = useDictionary() - - const [manageOpen, setManageOpen] = useState(false) - const [file, setFile] = useState(null) - const [uploading, setUploading] = useState(false) - const [uploadResult, setUploadResult] = useState(null) - const [uploadError, setUploadError] = useState(null) - const [criteriaReloadKey, setCriteriaReloadKey] = useState(0) - - const [, setSrLoading] = useState(true) - const [, setSrError] = useState(null) - const [sr, setSr] = useState(null) - const fileInputRef = useRef(null) - const token = getAuthToken() - const tokenType = getTokenType() - const authHeaders: Record | undefined = token ? { Authorization: `${tokenType} ${token}` } : undefined - - const loadSr = useCallback(async () => { - if (!srId) return null - - setSrLoading(true) - setSrError(null) - try { - const res = await authenticatedFetch( - `/api/can-sr/reviews/create?sr_id=${encodeURIComponent(srId)}`, - ) - if (!res.ok) { - const errBody = await res.json().catch(() => ({})) - throw new Error(errBody?.detail || errBody?.error || `Failed to fetch SR (${res.status})`) - } - - const data = await res.json().catch(() => ({})) - setSr(data) - return data - } catch (err: any) { - console.error('Error fetching SR:', err) - setSrError(err?.message || 'Unable to load review') - return null - } finally { - setSrLoading(false) - } - }, [srId]) useEffect(() => { - if (!srId) return - void loadSr() - }, [loadSr, srId]) - - // read selected file - const handleFileSelected = async (f: File | null) => { - setUploadResult(null) - setUploadError(null) - setFile(f) - } - - const onChooseFile = () => fileInputRef.current?.click() - - const handleUpload = async () => { - setUploadResult(null) - setUploadError(null) - if (!srId) { - setUploadError('Missing review id') - return - } - if (!file) { - setUploadError('No file selected') - return - } - - setUploading(true) - try { - const fd = new FormData() - fd.append('file', file) - - const headers = getAuthHeaders() - // Do NOT set Content-Type; browser will set multipart boundary - const res = await fetch(`/api/can-sr/citations/upload?sr_id=${encodeURIComponent(srId)}`, { - method: 'POST', - headers: headers, - body: fd as any, - }) - - const data = await res.json().catch(() => ({})) - if (!res.ok) { - const errMsg = data?.error || data?.detail || `Upload failed (${res.status})` - setUploadError(errMsg) - } else { - setUploadResult(data) - setCriteriaReloadKey((value) => value + 1) - void loadSr() - } - } catch (err: any) { - console.error('Upload error', err) - setUploadError(err?.message || 'Network error during upload') - } finally { - setUploading(false) - } - } - - const hasExistingScreeningData = Boolean((sr?.screening_db || {}).table_name) - - return ( -
- - - - -
- } - /> - -
-

{dict.setup.pageTitle}

-

- {dict.setup.pageDesc} -

- -
-
- -
- handleFileSelected(e.target.files ? e.target.files[0] : null)} - className="hidden" - /> - - -
- -
-
- - {file ? ( -
{dict.setup.selectedFile} {file.name}
- ) : ( -
{dict.setup.noFileSelected}
- )} - - {uploadError ?
{uploadError}
: null} - {uploadResult ? ( -
- {uploadResult.message || 'Upload succeeded'} -
- ) : null} - -
- - } - /> + router.replace( + srId ? `/can-sr/protocols?sr_id=${encodeURIComponent(srId)}` : '/can-sr', + ) + }, [router, srId]) -
-
- setManageOpen(false)} - srId={srId} - initialEmails={(sr && (sr.users || sr.user_emails || sr.allowed_users)) || []} - authHeaders={authHeaders} - /> - - ) + return null } diff --git a/frontend/app/[lang]/can-sr/sr/page.tsx b/frontend/app/[lang]/can-sr/sr/page.tsx index 8c88fecb..9c73c157 100644 --- a/frontend/app/[lang]/can-sr/sr/page.tsx +++ b/frontend/app/[lang]/can-sr/sr/page.tsx @@ -68,7 +68,7 @@ export default function CanSrLandingPage() { ) : ( <> -

{sr.name}

-

{sr.description}

+

{sr?.name || ''}

+

{sr?.description || ''}

)}
diff --git a/frontend/app/api/auth/me/route.ts b/frontend/app/api/auth/me/route.ts index 80114e5a..d469d5c2 100644 --- a/frontend/app/api/auth/me/route.ts +++ b/frontend/app/api/auth/me/route.ts @@ -39,8 +39,8 @@ export async function GET(request: NextRequest) { } catch (error) { console.error('Get user API error:', error) return NextResponse.json( - { error: 'Internal server error' }, - { status: 500 }, + { error: 'Backend API is unavailable. Start the backend service and try again.' }, + { status: 503 }, ) } } diff --git a/frontend/app/api/can-sr/citations/duplicates/check/route.ts b/frontend/app/api/can-sr/citations/duplicates/check/route.ts new file mode 100644 index 00000000..d945f76f --- /dev/null +++ b/frontend/app/api/can-sr/citations/duplicates/check/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function POST(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/duplicates/check`, + { + method: 'POST', + headers: { Authorization: authorization }, + body: await request.formData(), + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/citations/import-previews/[previewId]/commit/route.ts b/frontend/app/api/can-sr/citations/import-previews/[previewId]/commit/route.ts new file mode 100644 index 00000000..1d99488c --- /dev/null +++ b/frontend/app/api/can-sr/citations/import-previews/[previewId]/commit/route.ts @@ -0,0 +1,32 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function POST(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + const previewId = request.nextUrl.pathname.split('/').slice(-2, -1)[0] + if (!srId || !previewId) + return NextResponse.json( + { error: 'sr_id and preview id are required' }, + { status: 400 }, + ) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/import-previews/${encodeURIComponent(previewId)}/commit`, + { + method: 'POST', + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(await request.json()), + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/citations/import-previews/[previewId]/mapping/route.ts b/frontend/app/api/can-sr/citations/import-previews/[previewId]/mapping/route.ts new file mode 100644 index 00000000..89b34c98 --- /dev/null +++ b/frontend/app/api/can-sr/citations/import-previews/[previewId]/mapping/route.ts @@ -0,0 +1,32 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function PUT(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + const previewId = request.nextUrl.pathname.split('/').slice(-2, -1)[0] + if (!srId || !previewId) + return NextResponse.json( + { error: 'sr_id and preview id are required' }, + { status: 400 }, + ) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/import-previews/${encodeURIComponent(previewId)}/mapping`, + { + method: 'PUT', + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(await request.json()), + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/citations/import-previews/[previewId]/route.ts b/frontend/app/api/can-sr/citations/import-previews/[previewId]/route.ts new file mode 100644 index 00000000..56ba7bf7 --- /dev/null +++ b/frontend/app/api/can-sr/citations/import-previews/[previewId]/route.ts @@ -0,0 +1,35 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +async function forward(request: NextRequest, method: 'GET' | 'DELETE') { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + const previewId = request.nextUrl.pathname.split('/').pop() + if (!srId || !previewId) + return NextResponse.json( + { error: 'sr_id and preview id are required' }, + { status: 400 }, + ) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/import-previews/${encodeURIComponent(previewId)}`, + { + method, + headers: { Authorization: authorization }, + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} + +export async function GET(request: NextRequest) { + return forward(request, 'GET') +} +export async function DELETE(request: NextRequest) { + return forward(request, 'DELETE') +} diff --git a/frontend/app/api/can-sr/citations/import-previews/route.ts b/frontend/app/api/can-sr/citations/import-previews/route.ts new file mode 100644 index 00000000..cd478637 --- /dev/null +++ b/frontend/app/api/can-sr/citations/import-previews/route.ts @@ -0,0 +1,26 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function POST(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const formData = await request.formData() + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/import-previews`, + { + method: 'POST', + headers: { Authorization: authorization }, + body: formData, + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/citations/import/route.ts b/frontend/app/api/can-sr/citations/import/route.ts new file mode 100644 index 00000000..607dd03c --- /dev/null +++ b/frontend/app/api/can-sr/citations/import/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function POST(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/imports`, + { + method: 'POST', + headers: { Authorization: authorization }, + body: await request.formData(), + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/citations/set-answer/route.ts b/frontend/app/api/can-sr/citations/set-answer/route.ts new file mode 100644 index 00000000..2715e26b --- /dev/null +++ b/frontend/app/api/can-sr/citations/set-answer/route.ts @@ -0,0 +1,29 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function POST(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/set-answer`, + { + method: 'POST', + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(await request.json()), + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/citations/workspace/deduplication-preferences/route.ts b/frontend/app/api/can-sr/citations/workspace/deduplication-preferences/route.ts new file mode 100644 index 00000000..8f9dda62 --- /dev/null +++ b/frontend/app/api/can-sr/citations/workspace/deduplication-preferences/route.ts @@ -0,0 +1,37 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +async function proxy(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/citations/workspace/deduplication-preferences`, + { + method: request.method, + headers: { + Authorization: authorization, + ...(request.method === 'PUT' + ? { 'Content-Type': 'application/json' } + : {}), + }, + body: request.method === 'PUT' ? await request.text() : undefined, + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} + +export async function GET(request: NextRequest) { + return proxy(request) +} +export async function PUT(request: NextRequest) { + return proxy(request) +} diff --git a/frontend/app/api/can-sr/citations/workspace/duplicate-reviews/route.ts b/frontend/app/api/can-sr/citations/workspace/duplicate-reviews/route.ts new file mode 100644 index 00000000..f9e1f723 --- /dev/null +++ b/frontend/app/api/can-sr/citations/workspace/duplicate-reviews/route.ts @@ -0,0 +1,36 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +async function forward(request: NextRequest, method: 'GET' | 'PUT') { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/citations/workspace/duplicate-reviews`, + { + method, + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + body: method === 'PUT' ? await request.text() : undefined, + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} + +export function GET(request: NextRequest) { + return forward(request, 'GET') +} + +export function PUT(request: NextRequest) { + return forward(request, 'PUT') +} diff --git a/frontend/app/api/can-sr/citations/workspace/duplicate-runs/route.ts b/frontend/app/api/can-sr/citations/workspace/duplicate-runs/route.ts new file mode 100644 index 00000000..13beafe3 --- /dev/null +++ b/frontend/app/api/can-sr/citations/workspace/duplicate-runs/route.ts @@ -0,0 +1,14 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function POST(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId) return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authorization) return NextResponse.json({ error: 'Authorization header is required' }, { status: 401 }) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/citations/workspace/duplicate-runs`, + { method: 'POST', headers: { Authorization: authorization } }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { status: response.status }) +} diff --git a/frontend/app/api/can-sr/citations/workspace/preferences/route.ts b/frontend/app/api/can-sr/citations/workspace/preferences/route.ts new file mode 100644 index 00000000..be24032c --- /dev/null +++ b/frontend/app/api/can-sr/citations/workspace/preferences/route.ts @@ -0,0 +1,27 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +async function proxy(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authorization = request.headers.get('authorization') + if (!srId || !authorization) { + return NextResponse.json({ error: !srId ? 'sr_id is required' : 'Authorization header is required' }, { status: !srId ? 400 : 401 }) + } + try { + const response = await fetch(`${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/citations/workspace/preferences`, { + method: request.method, + headers: { Authorization: authorization, ...(request.method === 'PUT' ? { 'Content-Type': 'application/json' } : {}) }, + body: request.method === 'PUT' ? await request.text() : undefined, + }) + const text = await response.text() + let body: unknown + try { body = text ? JSON.parse(text) : {} } catch { body = { detail: text || null } } + return NextResponse.json(body, { status: response.status }) + } catch (error) { + console.error('Citations workspace preferences proxy error:', error) + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + } +} + +export const GET = proxy +export const PUT = proxy diff --git a/frontend/app/api/can-sr/citations/workspace/route.ts b/frontend/app/api/can-sr/citations/workspace/route.ts new file mode 100644 index 00000000..6aa65f2a --- /dev/null +++ b/frontend/app/api/can-sr/citations/workspace/route.ts @@ -0,0 +1,78 @@ +import { NextRequest, NextResponse } from 'next/server' +import { BACKEND_URL } from '@/lib/config' + +export async function GET(request: NextRequest) { + const params = request.nextUrl.searchParams + const srId = params.get('sr_id') + const authHeader = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authHeader) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + + try { + const url = new URL( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/citations/workspace`, + ) + for (const key of [ + 'page', + 'page_size', + 'search', + 'sort', + 'direction', + 'columns', + 'filters', + 'duplicate_status', + ]) { + const value = params.get(key) + if (value) url.searchParams.set(key, value) + } + const response = await fetch(url.toString(), { + headers: { Authorization: authHeader }, + }) + const text = await response.text() + let body: unknown + try { + body = text ? JSON.parse(text) : {} + } catch { + body = { detail: text || null } + } + return NextResponse.json(body, { status: response.status }) + } catch (error) { + console.error('Citations workspace proxy GET error:', error) + return NextResponse.json( + { error: 'Internal server error' }, + { status: 500 }, + ) + } +} + +export async function DELETE(request: NextRequest) { + const params = request.nextUrl.searchParams + const srId = params.get('sr_id') + const authHeader = request.headers.get('authorization') + if (!srId) + return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) + if (!authHeader) + return NextResponse.json( + { error: 'Authorization header is required' }, + { status: 401 }, + ) + const response = await fetch( + `${BACKEND_URL}/api/cite/${encodeURIComponent(srId)}/citations/workspace`, + { + method: 'DELETE', + headers: { + Authorization: authHeader, + 'Content-Type': 'application/json', + }, + body: await request.text(), + }, + ) + return NextResponse.json(await response.json().catch(() => ({})), { + status: response.status, + }) +} diff --git a/frontend/app/api/can-sr/reviews/create/route.ts b/frontend/app/api/can-sr/reviews/create/route.ts index 60054ec2..82fc39b6 100644 --- a/frontend/app/api/can-sr/reviews/create/route.ts +++ b/frontend/app/api/can-sr/reviews/create/route.ts @@ -149,6 +149,9 @@ export async function GET(request: NextRequest) { ) } catch (error) { console.error('SR create GET API error:', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + return NextResponse.json( + { error: 'Backend API is unavailable. Start the backend service and try again.' }, + { status: 503 }, + ) } } diff --git a/frontend/app/api/can-sr/reviews/list/route.ts b/frontend/app/api/can-sr/reviews/list/route.ts index cba4a741..fc7cd1d0 100644 --- a/frontend/app/api/can-sr/reviews/list/route.ts +++ b/frontend/app/api/can-sr/reviews/list/route.ts @@ -33,6 +33,9 @@ export async function GET(request: NextRequest) { return NextResponse.json(data, { status: res.status }) } catch (error) { console.error('SR view (mine) GET API error:', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + return NextResponse.json( + { error: 'Backend API is unavailable. Start the backend service and try again.' }, + { status: 503 }, + ) } } diff --git a/frontend/app/api/can-sr/reviews/users/route.ts b/frontend/app/api/can-sr/reviews/users/route.ts index 19ae24c7..9bb86b33 100644 --- a/frontend/app/api/can-sr/reviews/users/route.ts +++ b/frontend/app/api/can-sr/reviews/users/route.ts @@ -14,8 +14,8 @@ import { BACKEND_URL } from '@/lib/config' * - Follows the pattern used in frontend/app/api/can-sr/reviews/create/route.ts (do not use cookies). */ -async function forwardJson(url: string, request: NextRequest) { - const body = await request.json() +async function forwardJson(url: string, request: NextRequest, method = 'POST') { + const body = method === 'GET' || method === 'DELETE' ? undefined : await request.json() const authHeader = request.headers.get('authorization') if (!authHeader) { @@ -31,7 +31,7 @@ async function forwardJson(url: string, request: NextRequest) { } const res = await fetch(url, { - method: 'POST', + method, headers, body: JSON.stringify(body), }) @@ -68,3 +68,26 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } } + +export async function GET(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const authHeader = request.headers.get('authorization') + if (!srId || !authHeader) return NextResponse.json({ error: !srId ? 'sr_id query parameter is required' : 'Authorization header is required' }, { status: !srId ? 400 : 401 }) + return forwardJson(`${BACKEND_URL}/api/sr/${encodeURIComponent(srId)}/members`, request, 'GET') +} + +export async function PUT(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const memberId = request.nextUrl.searchParams.get('member_id') + const authHeader = request.headers.get('authorization') + if (!srId || !memberId || !authHeader) return NextResponse.json({ error: 'sr_id, member_id, and Authorization header are required' }, { status: 400 }) + return forwardJson(`${BACKEND_URL}/api/sr/${encodeURIComponent(srId)}/members/${encodeURIComponent(memberId)}`, request, 'PUT') +} + +export async function DELETE(request: NextRequest) { + const srId = request.nextUrl.searchParams.get('sr_id') + const memberId = request.nextUrl.searchParams.get('member_id') + const authHeader = request.headers.get('authorization') + if (!srId || !memberId || !authHeader) return NextResponse.json({ error: 'sr_id, member_id, and Authorization header are required' }, { status: 400 }) + return forwardJson(`${BACKEND_URL}/api/sr/${encodeURIComponent(srId)}/members/${encodeURIComponent(memberId)}`, request, 'DELETE') +} diff --git a/frontend/app/api/can-sr/screen/citation-ids/route.ts b/frontend/app/api/can-sr/screen/citation-ids/route.ts index f958d718..382a90ce 100644 --- a/frontend/app/api/can-sr/screen/citation-ids/route.ts +++ b/frontend/app/api/can-sr/screen/citation-ids/route.ts @@ -2,8 +2,8 @@ import { NextRequest, NextResponse } from 'next/server' import { BACKEND_URL } from '@/lib/config' /** - * Proxy: GET /api/can-sr/screen/citation-ids?sr_id=&step=l1|l2&filter=all|needs|validated|unvalidated|not_screened - * -> GET {BACKEND_URL}/api/screen/citation-ids?sr_id=...&step=...&filter=... + * Proxy: GET /api/can-sr/screen/citation-ids?sr_id=&step=l1|l2&filter=...&decision=...&q=... + * -> GET {BACKEND_URL}/api/screen/citation-ids?sr_id=...&step=...&filter=...&decision=...&q=... */ export async function GET(request: NextRequest) { @@ -12,6 +12,8 @@ export async function GET(request: NextRequest) { const srId = params.get('sr_id') const step = params.get('step') || 'l1' const filter = params.get('filter') || 'all' + const decision = params.get('decision') || 'all' + const q = params.get('q') || '' if (!srId) { return NextResponse.json({ error: 'sr_id is required' }, { status: 400 }) @@ -29,6 +31,8 @@ export async function GET(request: NextRequest) { url.searchParams.set('sr_id', srId) url.searchParams.set('step', step) url.searchParams.set('filter', filter) + url.searchParams.set('decision', decision) + if (q) url.searchParams.set('q', q) const res = await fetch(url.toString(), { method: 'GET', @@ -48,6 +52,9 @@ export async function GET(request: NextRequest) { return NextResponse.json(json, { status: res.status }) } catch (err: any) { console.error('screen citation-ids proxy GET error:', err) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + return NextResponse.json( + { error: 'Internal server error' }, + { status: 500 }, + ) } } diff --git a/frontend/components/can-sr/CitationListPage.tsx b/frontend/components/can-sr/CitationListPage.tsx index 9b53f848..4822b3c0 100644 --- a/frontend/components/can-sr/CitationListPage.tsx +++ b/frontend/components/can-sr/CitationListPage.tsx @@ -90,8 +90,13 @@ export default function CitationsListPage({ 'needs' | 'validated' | 'unvalidated' | 'not_screened' | 'all' >('all') const [filterHydrated, setFilterHydrated] = useState(false) + const [decisionFilter, setDecisionFilter] = useState('all') + const [searchQuery, setSearchQuery] = useState('') + const [draftSearchQuery, setDraftSearchQuery] = useState('') const filterParam = searchParams?.get('filter') + const decisionParam = searchParams?.get('decision') || 'all' + const queryParam = searchParams?.get('q') || '' const filterStorageKey = useMemo(() => { if (!srId) return null if (!(screeningStep === 'l1' || screeningStep === 'l2')) return null @@ -164,6 +169,13 @@ export default function CitationsListPage({ // ignore } }, [srId, screeningStep, filterMode, filterStorageKey]) + + useEffect(() => { + const valid = ['all', 'included', 'excluded', 'undecided'] + setDecisionFilter(valid.includes(decisionParam) ? decisionParam : 'all') + setSearchQuery(queryParam) + setDraftSearchQuery(queryParam) + }, [decisionParam, queryParam]) // page-local stats no longer shown (SR-wide progress bar is in metrics panel) const [_pageStats, setPageStats] = useState< ScreeningMetricsStats | undefined @@ -309,7 +321,9 @@ export default function CitationsListPage({ const res = await fetch( `/api/can-sr/screen/citation-ids?sr_id=${encodeURIComponent(srId)}&step=${encodeURIComponent( screeningStep, - )}&filter=${encodeURIComponent(filterMode)}`, + )}&filter=${encodeURIComponent(filterMode)}&decision=${encodeURIComponent( + decisionFilter, + )}&q=${encodeURIComponent(searchQuery)}`, { method: 'GET', headers }, ) const data = await res.json().catch(() => ({})) @@ -382,7 +396,15 @@ export default function CitationsListPage({ loadCriteria() loadCitations() - }, [srId, router, screeningStep, filterMode, filterHydrated]) + }, [ + srId, + router, + screeningStep, + filterMode, + decisionFilter, + searchQuery, + filterHydrated, + ]) // Load SR thresholds + metrics (L1/L2 only) useEffect(() => { @@ -941,35 +963,101 @@ export default function CitationsListPage({
{/* Filter bar moved above the list view */} {screeningStep === 'l1' || screeningStep === 'l2' ? ( -
- - ) => { + const next = e.target.value as any + setFilterMode(next) + // Keep URL in sync on *user intent* (safe, avoids effect-driven loops). + try { + const params = new URLSearchParams( + searchParams?.toString() || '', + ) + params.set('sr_id', String(srId || '')) + params.set('filter', String(next)) + router.replace( + `/${encodeURIComponent(String(lang || 'en'))}/can-sr/${encodeURIComponent(pageview)}?${params.toString()}`, + ) + } catch { + // ignore + } + }} + className="rounded-md border border-gray-200 bg-white px-2 py-1 text-sm" + > + + + + + + +
+
+ + + }} + className="rounded-md border border-gray-200 bg-white px-2 py-1 text-sm" + > + + + + + +
+
+ + { + setDraftSearchQuery(e.target.value) + if (!e.target.value.trim()) setSearchQuery('') + }} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault() + const next = draftSearchQuery.trim() + setSearchQuery(next) + const params = new URLSearchParams( + searchParams?.toString() || '', + ) + params.set('sr_id', String(srId || '')) + if (next) params.set('q', next) + else params.delete('q') + router.replace( + `/${encodeURIComponent(String(lang || 'en'))}/can-sr/${encodeURIComponent(pageview)}?${params.toString()}`, + ) + } + }} + placeholder="Search citations" + className="h-9 w-full rounded-md border border-gray-200 px-2.5 text-sm" + /> +
) : null} diff --git a/frontend/components/can-sr/references/references-workspace.test.tsx b/frontend/components/can-sr/references/references-workspace.test.tsx new file mode 100644 index 00000000..492d4eae --- /dev/null +++ b/frontend/components/can-sr/references/references-workspace.test.tsx @@ -0,0 +1,1011 @@ +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, expect, it, vi } from 'vitest' + +import ReferencesWorkspace from './references-workspace' + +const { authenticatedFetch, getCurrentUser } = vi.hoisted(() => ({ + authenticatedFetch: vi.fn(), + getCurrentUser: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ authenticatedFetch, getCurrentUser })) + +vi.mock('next/link', () => ({ + default: ({ + children, + href, + }: { + children: React.ReactNode + href: string + }) => {children}, +})) + +const copy = { + gridTitle: 'References workspace', + empty: 'No references yet.', + datasetReady: 'Dataset ready.', + gridDeferred: 'Grid deferred.', + search: 'Search references', + searchHint: 'Press Enter to search', + loading: 'Loading references…', + gridFailed: 'Unable to load references.', + noMatches: 'No references match this search.', + results: '{count} references', + previous: 'Previous', + next: 'Next', + databaseSearch: 'Database Search', + add: 'Add references', + addReferencesTitle: 'Add References', + dialogTitle: 'Add References', + dialogDescription: 'Import references immediately.', + file: 'Reference file', + searchString: 'search string', + beginSearch: 'Begin Search', + import: 'Import references', + imported: 'Imported: {count}', + duplicatesSkipped: '{count} duplicates skipped.', + duplicatesDetected: + '{count} duplicate references detected. They will be merged unless Import Duplicates is selected.', + includeDuplicates: 'Import Duplicates', + importFailed: 'Import failed', + missingTitle: 'Missing Title column', + missingAbstract: 'Missing Abstract column', + additionalColumns: 'Additional columns preserved', + cancel: 'Cancel', + working: 'Working...', + rows: 'Rows: {count}', + mapping: 'Mapping', + unmapped: 'unmapped', + missing: 'Missing', + excluded: 'Excluded', + previewFailed: 'Preview failed', + mappingFailed: 'Mapping failed', + commitFailed: 'Commit failed', + committed: 'Committed: {count}', + reconciliation: + 'Reconciliation: {new} new; {existing} exact existing matches.', + matchesDoNotOverwrite: + 'Exact matches do not overwrite citations or screening decisions.', + ambiguousMatches: + '{count} rows have ambiguous exact matches and must be resolved before import.', + ambiguousRows: 'Ambiguous source rows: {rows}.', + excludeAmbiguousRows: 'Exclude these ambiguous rows from this import.', + ambiguousResolutionList: 'Ambiguous row resolution', + excludeAmbiguousRow: 'Exclude source row {row} from this import.', + ambiguousAcknowledgement: + 'Ambiguous rows acknowledged for exclusion: {acknowledged} of {total}.', + excludedAmbiguousRows: + '{count} ambiguous rows will be excluded from this import.', + invalidReconciliationRows: '{count} rows are invalid and cannot be imported.', + sortBy: 'Sort by {column}', + filterBy: 'Filter by {column}', + selectAll: 'Select all citations on this page', + selectCitation: 'Select citation {id}', + columns: 'Columns', + columnsTitle: 'Choose visible columns', + columnsDescription: 'Choose fields.', + saveColumns: 'Save columns', + moveColumnUp: 'Move {column} up', + moveColumnDown: 'Move {column} down', + deleting: 'Deleting…', + deleteSelected: 'Delete {count} selected citations', + duplicateGroupReview: 'Duplicate group review', + groupSummary: 'Group {current} of {total} · {status} · {count} records', + restore: 'Restore', + minimize: 'Minimize', + record: 'Record {id}', + suggestedSurvivor: 'Suggested survivor', + selectSurvivor: 'Select survivor', + reviewDecision: 'Review decision:', + confirmDuplicate: 'Confirm duplicate', + keepBoth: 'Keep both', + defer: 'Defer', + deleteCitation: 'Delete {id}', + survivor: 'Survivor', + staleSurvivor: + 'The saved survivor is no longer in this group; a current member was selected.', + chooseSelection: 'Choose citation selection', + none: 'None', + all: 'All', + suggestedDuplicates: 'Suggested duplicates', + filterDuplicateStatus: 'Filter duplicate status', + showDuplicateStatusFilter: 'Show duplicate status filter', + filterDuplicateStatuses: 'Filter duplicate statuses', + duplicateStatus: 'Duplicate status', + showReferencesMatching: 'Show references matching', + allStatuses: 'All statuses', + exactDuplicates: 'Exact duplicates', + possibleDuplicates: 'Possible duplicates', + noDuplicateMatch: 'No duplicate match', + configureDuplicateFields: 'Configure duplicate matching fields', + runningDuplicateCalculation: 'Running duplicate calculation', + runDuplicateCalculation: 'Run duplicate calculation', + rerunDuplicateCalculation: 'Rerun duplicate calculation', + duplicateMatchingFields: 'Duplicate matching fields', + duplicateFieldsDescription: + 'Visible fields are added automatically. Hidden fields remain configured.', + clearFilter: 'Clear', + applyFilter: 'Apply', + resizeColumn: 'Resize column', + changeFile: 'Change file', + chooseFile: 'Choose file', + removeFile: 'Remove', + searchDescription: + 'Search PubMed, Scopus, and Europe PMC using a database-specific search string.', + selectedFile: 'Selected file:', + exactDuplicateMatch: 'Exact duplicate match', + possibleDuplicateMatch: 'Possible duplicate match', + keptBothReviewGroup: 'Kept both; review duplicate group', + openDuplicateReview: 'Open duplicate group review for citation {id}', + reviewKeptBothGroup: 'Review kept-both group', + openDuplicateGroupReview: 'Open duplicate group review', + row: 'Row {number}', +} + +beforeEach(() => { + if (!File.prototype.text) { + Object.defineProperty(File.prototype, 'text', { + configurable: true, + value() { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result || '')) + reader.onerror = () => reject(reader.error) + reader.readAsText(this) + }) + }, + }) + } + getCurrentUser.mockResolvedValue({ email: 'member@example.com' }) + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [], + total_count: 0, + page: 1, + page_size: 25, + columns: ['id'], + }), + }) +}) + +it('keeps reference search compact beside add references and submits on Enter', async () => { + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [], + total_count: 0, + page: 1, + page_size: 25, + columns: ['id'], + }), + }) + const user = userEvent.setup() + render() + + const searchInput = await screen.findByRole('searchbox', { + name: 'Search references', + }) + expect(searchInput).toHaveClass('h-9') + expect(searchInput.parentElement).toHaveClass('relative') + expect(screen.queryByText('Press Enter to search')).not.toBeInTheDocument() + expect(screen.queryByText('Enter')).not.toBeInTheDocument() + expect( + screen.getByRole('button', { name: 'Add references' }), + ).toBeInTheDocument() + expect(searchInput.parentElement?.parentElement?.parentElement).toHaveClass( + 'items-end', + ) + + await waitFor(() => expect(authenticatedFetch).toHaveBeenCalled()) + authenticatedFetch.mockClear() + await user.type(searchInput, 'diabetes') + expect(authenticatedFetch).not.toHaveBeenCalled() + expect(screen.getByText('Enter')).toBeInTheDocument() + + await user.keyboard('{Enter}') + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('search=diabetes'), + ), + ) + + authenticatedFetch.mockClear() + await user.clear(searchInput) + expect(screen.queryByText('Enter')).not.toBeInTheDocument() + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('/citations/workspace?'), + ), + ) + expect( + authenticatedFetch.mock.calls.some(([url]) => !url.includes('search=')), + ).toBe(true) +}) + +it('searches the selected database from the Add References modal', async () => { + const user = userEvent.setup() + render() + + await user.click(screen.getByRole('button', { name: 'Add references' })) + expect( + screen.getByRole('heading', { name: 'Add References' }), + ).toBeInTheDocument() + await user.click(screen.getByRole('tab', { name: 'Search databases' })) + await user.click(screen.getByRole('radio', { name: /Pubmed/i })) + await user.type( + screen.getByRole('textbox', { name: /Pubmed search string/i }), + 'cancer', + ) + await user.click(screen.getByRole('button', { name: 'Begin Search' })) + + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + '/api/can-sr/search?sr_id=review-1', + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ database: 'Pubmed', search_term: 'cancer' }), + }), + ), + ) +}) + +it('shows Add references on a fresh workspace', async () => { + const user = userEvent.setup() + render() + + await user.click(screen.getByRole('button', { name: 'Add references' })) + + expect( + screen.getByRole('heading', { name: 'Add References' }), + ).toBeInTheDocument() +}) + +it('renders fifteen empty rows when the workspace has no references', async () => { + render() + + await waitFor(() => { + const rows = screen.getAllByRole('row', { name: /^Row / }) + expect(rows).toHaveLength(15) + expect( + rows.every((row) => row.querySelector('td')?.textContent === ''), + ).toBe(true) + expect( + rows.every((row) => + Array.from(row.querySelectorAll('td')).every( + (cell) => + cell.className.includes('py-2') && + cell + .querySelector('[aria-hidden="true"]') + ?.className.includes('h-5'), + ), + ), + ).toBe(true) + }) +}) + +it('loads the complete workspace grid for an existing dataset', async () => { + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + columns: ['id', 'title', 'abstract'], + }), + }) + render() + + await waitFor(() => expect(screen.getByText('A study')).toBeInTheDocument()) + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining( + '/api/can-sr/citations/workspace?sr_id=review-1&sort=id&direction=asc', + ), + ) + expect(screen.getAllByText('1', { selector: 'td' })).toHaveLength(2) + expect(screen.getByLabelText('Row 2')).toBeInTheDocument() + expect( + screen.queryByRole('button', { name: 'Previous' }), + ).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Next' })).not.toBeInTheDocument() +}) + +it('clears all selected citations with the None selection option', async () => { + const user = userEvent.setup() + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [ + { id: 1, title: 'First study' }, + { id: 2, title: 'Second study' }, + ], + total_count: 2, + columns: ['id', 'title'], + }), + }) + + render() + await waitFor(() => + expect(screen.getByText('First study')).toBeInTheDocument(), + ) + + await user.click( + screen.getByRole('checkbox', { + name: 'Select all citations on this page', + }), + ) + expect( + screen.queryByRole('button', { name: 'Choose citation selection' }), + ).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'All' })) + expect( + screen.getByLabelText('Select all citations on this page'), + ).toHaveAttribute('data-state', 'checked') + + await user.click( + screen.getByRole('checkbox', { + name: 'Select all citations on this page', + }), + ) + await user.click(screen.getByRole('button', { name: 'None' })) + expect( + screen.getByLabelText('Select all citations on this page'), + ).toHaveAttribute('data-state', 'unchecked') +}) + +it('opens duplicate status selection in a popup and applies the chosen filter', async () => { + const user = userEvent.setup() + render() + + await user.click( + await screen.findByRole('button', { name: 'Show duplicate status filter' }), + ) + expect( + screen.getByRole('combobox', { name: 'Show references matching' }), + ).toHaveValue('') + expect( + authenticatedFetch.mock.calls.some(([url]) => + url.includes('duplicate_status='), + ), + ).toBe(false) + + await user.selectOptions( + screen.getByRole('combobox', { name: 'Show references matching' }), + 'exact', + ) + expect( + authenticatedFetch.mock.calls.some(([url]) => + url.includes('duplicate_status='), + ), + ).toBe(false) + + await user.click(screen.getByRole('button', { name: 'Apply' })) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('duplicate_status=exact'), + ), + ) + expect(screen.queryByRole('combobox')).not.toBeInTheDocument() +}) + +it('requests server-side sorting when a workspace column header is selected', async () => { + const user = userEvent.setup() + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + page: 1, + page_size: 25, + columns: ['id', 'title', 'abstract'], + sort: 'id', + direction: 'asc', + query_fingerprint: 'sha256:workspace', + }), + }) + render() + + await user.click(await screen.findByRole('button', { name: 'Sort by title' })) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('sort=title&direction=asc'), + ), + ) +}) + +it('keeps the current grid visible while sorting is loading', async () => { + const user = userEvent.setup() + let resolveSort: ((response: unknown) => void) | undefined + const workspaceResponse = { + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + columns: ['id', 'title', 'abstract'], + }), + } + authenticatedFetch.mockImplementation((url: string) => { + if (url.includes('/citations/workspace?') && url.includes('sort=title')) { + return new Promise((resolve) => { + resolveSort = resolve + }) + } + return workspaceResponse + }) + + render() + await screen.findByText('A study') + await user.click(await screen.findByRole('button', { name: 'Sort by title' })) + + expect(screen.getByText('A study')).toBeInTheDocument() + expect(screen.queryByText(copy.loading)).not.toBeInTheDocument() + + resolveSort?.({ + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + columns: ['id', 'title', 'abstract'], + }), + }) +}) + +it('saves selected workspace columns and reloads the grid with them', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation( + async (url: string, options?: RequestInit) => { + if (url.includes('/workspace/preferences')) { + if (options?.method === 'PUT') + return { ok: true, json: async () => ({ columns: ['id', 'title'] }) } + return { ok: true, json: async () => ({ columns: null }) } + } + return { + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + page: 1, + page_size: 25, + columns: ['id', 'title', 'abstract'], + available_columns: ['id', 'title', 'abstract'], + sort: 'id', + direction: 'asc', + query_fingerprint: 'sha256:workspace', + }), + } + }, + ) + render() + + await user.click(await screen.findByRole('button', { name: 'Columns' })) + await user.click(screen.getByRole('checkbox', { name: 'abstract' })) + await user.click(screen.getByRole('button', { name: 'Save columns' })) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('/workspace/preferences?sr_id=review-1'), + expect.objectContaining({ + method: 'PUT', + body: JSON.stringify({ columns: ['id', 'title'] }), + }), + ), + ) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('columns=id%2Ctitle'), + ), + ) +}) + +it('does not render a filter control for the ID column', async () => { + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + page: 1, + page_size: 25, + columns: ['id', 'title', 'abstract'], + available_columns: ['id', 'title', 'abstract'], + sort: 'id', + direction: 'asc', + query_fingerprint: 'sha256:workspace', + }), + }) + render() + + await screen.findByText('A study') + expect( + screen.queryByRole('button', { name: 'Filter by id' }), + ).not.toBeInTheDocument() + expect( + screen.getByRole('button', { name: 'Filter by title' }), + ).toBeInTheDocument() +}) + +it('allows selecting either duplicate survivor, minimizing review, and shows an X marker', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation( + async (url: string, options?: RequestInit) => { + if (url.includes('/workspace/duplicate-reviews')) { + if (options?.method === 'PUT') { + return { + ok: true, + json: async () => ({ + group_id: 'duplicate-group-1', + decision: 'confirmed_duplicate', + survivor_id: 2, + }), + } + } + return { ok: true, json: async () => ({ reviews: [] }) } + } + if (url.includes('/workspace/preferences')) + return { ok: true, json: async () => ({ columns: null }) } + if (url.includes('/deduplication-preferences')) + return { ok: true, json: async () => ({ fields: ['title'] }) } + return { + ok: true, + json: async () => ({ + citations: [ + { + id: 1, + title: 'Same study', + duplicate_status: 'exact', + duplicate_group_id: 'duplicate-group-1', + }, + { + id: 2, + title: 'Same study', + duplicate_status: 'exact', + duplicate_group_id: 'duplicate-group-1', + }, + { + id: 3, + title: 'Another study', + duplicate_status: 'exact', + duplicate_group_id: 'duplicate-group-2', + }, + { + id: 4, + title: 'Another study', + duplicate_status: 'exact', + duplicate_group_id: 'duplicate-group-2', + }, + ], + total_count: 2, + columns: ['id', 'title'], + available_columns: ['id', 'title'], + duplicate_fields: ['title'], + duplicate_groups: [ + { + group_id: 'duplicate-group-1', + citation_ids: [1, 2], + status: 'exact', + suggested_survivor_id: 1, + members: [ + { id: 1, title: 'Same study' }, + { id: 2, title: 'Same study' }, + ], + }, + { + group_id: 'duplicate-group-2', + citation_ids: [3, 4], + status: 'exact', + suggested_survivor_id: 3, + members: [ + { id: 3, title: 'Another study' }, + { id: 4, title: 'Another study' }, + ], + }, + ], + }), + } + }, + ) + + render() + + await screen.findByRole('heading', { name: 'Duplicate group review' }) + await user.click( + screen.getByRole('checkbox', { + name: 'Select all citations on this page', + }), + ) + await user.click(screen.getByRole('button', { name: 'Suggested duplicates' })) + expect( + screen.getByRole('checkbox', { name: 'Select citation 1' }), + ).not.toBeChecked() + expect( + screen.getByRole('checkbox', { name: 'Select citation 2' }), + ).toBeChecked() + expect( + screen.getByRole('checkbox', { name: 'Select citation 3' }), + ).not.toBeChecked() + expect( + screen.getByRole('checkbox', { name: 'Select citation 4' }), + ).toBeChecked() + expect( + screen.getAllByRole('button', { + name: /Open duplicate group review for citation/, + }), + ).toHaveLength(4) + await user.click( + screen.getByRole('button', { + name: 'Open duplicate group review for citation 3', + }), + ) + expect(screen.getByText(/Group 2 of 2/)).toBeInTheDocument() + await user.click( + screen.getByRole('button', { + name: 'Open duplicate group review for citation 1', + }), + ) + expect(screen.getByText(/Group 1 of 2/)).toBeInTheDocument() + await user.click(screen.getByRole('radio', { name: 'Select survivor' })) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('/workspace/duplicate-reviews?sr_id=review-1'), + expect.objectContaining({ + method: 'PUT', + body: JSON.stringify({ + group_id: 'duplicate-group-1', + decision: 'confirmed_duplicate', + survivor_id: 2, + }), + }), + ), + ) + expect( + screen.getByRole('button', { name: 'Minimize Duplicate group review' }), + ).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Delete 1' })).toBeInTheDocument() + expect( + screen.queryByRole('button', { name: 'Confirm duplicate' }), + ).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Delete 1' })) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('/workspace?sr_id=review-1'), + expect.objectContaining({ + method: 'DELETE', + body: expect.not.stringContaining('"group_ids"'), + }), + ), + ) + const deletionRequest = authenticatedFetch.mock.calls.find( + ([url, options]) => + url.includes('/workspace?sr_id=review-1') && + (options as RequestInit | undefined)?.method === 'DELETE', + ) + expect(deletionRequest?.[1]).toEqual( + expect.objectContaining({ + body: expect.stringContaining('"citation_ids":[1]'), + }), + ) + expect(screen.getByText(/Group 2 of 2/)).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Previous' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Next' })).toBeInTheDocument() + expect( + screen.queryByRole('button', { name: 'Previous group' }), + ).not.toBeInTheDocument() + expect( + screen.queryByRole('button', { name: 'Next group' }), + ).not.toBeInTheDocument() + + await user.click( + screen.getByRole('button', { name: 'Minimize Duplicate group review' }), + ) + expect(screen.queryByText('Select survivor')).not.toBeInTheDocument() + await user.click( + screen.getByRole('button', { name: 'Restore Duplicate group review' }), + ) + expect(screen.getByText('Select survivor')).toBeInTheDocument() +}) + +it('ignores a stale saved survivor and keeps one current member selected', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation(async (url: string) => { + if (url.includes('/workspace/duplicate-reviews')) { + return { + ok: true, + json: async () => ({ + reviews: [ + { + group_id: 'duplicate-group-stale', + decision: 'confirmed_duplicate', + survivor_id: 206, + }, + ], + }), + } + } + if (url.includes('/workspace/preferences')) + return { ok: true, json: async () => ({ columns: null }) } + if (url.includes('/deduplication-preferences')) + return { ok: true, json: async () => ({ fields: ['title'] }) } + return { + ok: true, + json: async () => ({ + citations: [ + { id: 201, title: 'Same study', duplicate_status: 'exact' }, + { id: 300, title: 'Same study', duplicate_status: 'exact' }, + ], + total_count: 2, + columns: ['id', 'title'], + available_columns: ['id', 'title'], + duplicate_fields: ['title'], + duplicate_groups: [ + { + group_id: 'duplicate-group-stale', + citation_ids: [201, 300], + status: 'exact', + suggested_survivor_id: 300, + review: { + group_id: 'duplicate-group-stale', + decision: 'confirmed_duplicate', + survivor_id: null, + stale: true, + }, + members: [ + { id: 201, title: 'Same study' }, + { id: 300, title: 'Same study' }, + ], + }, + ], + }), + } + }) + + render() + + await screen.findByRole('heading', { name: 'Duplicate group review' }) + await user.click( + screen.getByRole('button', { + name: 'Restore Duplicate group review', + }), + ) + expect( + screen.getByRole('radio', { name: 'Suggested survivor' }), + ).toBeChecked() + expect( + screen.getByRole('radio', { name: 'Select survivor' }), + ).not.toBeChecked() + expect(screen.getByText(copy.staleSurvivor)).toBeInTheDocument() +}) + +it('shows a green marker for kept-both citations and opens their review group', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation(async (url: string) => { + if (url.includes('/workspace/duplicate-reviews')) { + return { + ok: true, + json: async () => ({ + reviews: [ + { group_id: 'duplicate-group-kept', decision: 'not_duplicate' }, + ], + }), + } + } + if (url.includes('/workspace/preferences')) + return { ok: true, json: async () => ({ columns: null }) } + if (url.includes('/deduplication-preferences')) + return { ok: true, json: async () => ({ fields: ['title'] }) } + return { + ok: true, + json: async () => ({ + citations: [ + { + id: 10, + title: 'Kept study', + duplicate_status: 'no_match', + duplicate_group_id: 'duplicate-group-kept', + }, + { + id: 11, + title: 'Kept study', + duplicate_status: 'no_match', + duplicate_group_id: 'duplicate-group-kept', + }, + ], + total_count: 2, + columns: ['id', 'title'], + duplicate_groups: [ + { + group_id: 'duplicate-group-kept', + citation_ids: [10, 11], + status: 'exact', + members: [ + { id: 10, title: 'Kept study' }, + { id: 11, title: 'Kept study' }, + ], + }, + ], + }), + } + }) + + render() + + await screen.findByRole('heading', { name: 'Duplicate group review' }) + expect( + screen.getAllByRole('button', { + name: /Open duplicate group review for citation/, + }), + ).toHaveLength(2) + await user.click( + screen.getByRole('button', { + name: 'Open duplicate group review for citation 10', + }), + ) + expect(screen.getByText(/Group 1 of 1/)).toBeInTheDocument() +}) + +it('binds workspace column boundaries to explicit resizable widths', async () => { + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + page: 1, + page_size: 25, + columns: ['id', 'title', 'abstract'], + sort: 'id', + direction: 'asc', + query_fingerprint: 'sha256:workspace', + }), + }) + render() + + await screen.findByText('A study') + const table = screen.getByRole('table') + expect(table.querySelectorAll('col')[3]).toHaveStyle({ width: '160px' }) + expect(table.querySelectorAll('col')).toHaveLength(7) + expect( + screen.getByRole('button', { name: 'Resize column title' }), + ).toBeInTheDocument() + expect(screen.getAllByRole('button', { name: 'Columns' })).toHaveLength(1) + expect(screen.queryByText('Choose visible columns')).not.toBeInTheDocument() +}) + +it('opens the column chooser from the trailing plus column', async () => { + authenticatedFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + citations: [{ id: 1, title: 'A study', abstract: 'Summary' }], + total_count: 1, + page: 1, + page_size: 25, + columns: ['id', 'title', 'abstract'], + available_columns: ['id', 'title', 'abstract', 'author'], + sort: 'id', + direction: 'asc', + query_fingerprint: 'sha256:workspace', + }), + }) + const user = userEvent.setup() + render() + + await screen.findByText('A study') + await user.click(screen.getByRole('button', { name: 'Columns' })) + + const panel = screen.getByRole('dialog') + expect(panel).toBeInTheDocument() + expect(panel).toHaveClass( + 'absolute', + 'top-full', + 'right-0', + 'overflow-y-auto', + ) + expect(panel).toHaveClass('w-[min(28rem,calc(100vw-2rem))]') + expect(screen.queryByTestId('dialog-overlay')).not.toBeInTheDocument() + expect(screen.getByText('Choose visible columns')).toBeInTheDocument() +}) + +it('warns about missing required columns without importing', async () => { + const user = userEvent.setup() + render() + await user.click(screen.getByRole('button', { name: 'Add references' })) + await user.upload( + screen.getByLabelText('Reference file'), + new File(['Title\nA'], 'invalid.csv', { type: 'text/csv' }), + ) + await user.click(screen.getByRole('button', { name: 'Import references' })) + expect(await screen.findByText('Missing Abstract column')).toBeInTheDocument() + expect(authenticatedFetch).not.toHaveBeenCalledWith( + expect.stringContaining('/citations/import?'), + expect.anything(), + ) +}) + +it('reports an import API failure', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation(async (url: string) => ({ + ok: !url.includes('/citations/import?'), + json: async () => + url.includes('/citations/import?') + ? { detail: 'Server rejected import' } + : { + citations: [], + total_count: 0, + page: 1, + page_size: 25, + columns: ['id'], + }, + })) + render() + await user.click(screen.getByRole('button', { name: 'Add references' })) + await user.upload( + screen.getByLabelText('Reference file'), + new File(['Title,Abstract\nA,B'], 'append.csv', { type: 'text/csv' }), + ) + await user.click(screen.getByRole('button', { name: 'Import references' })) + expect(await screen.findByRole('status')).toHaveTextContent( + 'Server rejected import', + ) +}) + +it('allows duplicate imports to be explicitly overridden', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation( + async (url: string, options?: RequestInit) => { + if (url.includes('/duplicates/check?')) { + return { ok: true, json: async () => ({ duplicates_count: 1 }) } + } + if (url.includes('/citations/import?')) { + expect((options?.body as FormData).get('include_duplicates')).toBe( + 'true', + ) + return { + ok: true, + json: async () => ({ rows_inserted: 2, duplicates_skipped: 0 }), + } + } + return { + ok: true, + json: async () => ({ citations: [], total_count: 0, columns: ['id'] }), + } + }, + ) + render() + await user.click(screen.getByRole('button', { name: 'Add references' })) + await user.upload( + screen.getByLabelText('Reference file'), + new File(['Title,Abstract\nA,B'], 'duplicates.csv', { type: 'text/csv' }), + ) + expect( + await screen.findByText( + '1 duplicate references detected. They will be merged unless Import Duplicates is selected.', + ), + ).toBeInTheDocument() + await user.click(screen.getByRole('checkbox', { name: 'Import Duplicates' })) + await user.click(screen.getByRole('button', { name: 'Import references' })) + expect(await screen.findByRole('status')).toHaveTextContent('Imported: 2') +}) + +it('does not display the duplicate import override when none are detected', async () => { + const user = userEvent.setup() + authenticatedFetch.mockImplementation(async (url: string) => { + if (url.includes('/duplicates/check?')) { + return { ok: true, json: async () => ({ duplicates_count: 0 }) } + } + return { + ok: true, + json: async () => ({ citations: [], total_count: 0, columns: ['id'] }), + } + }) + render() + await user.click(screen.getByRole('button', { name: 'Add references' })) + await user.upload( + screen.getByLabelText('Reference file'), + new File(['Title,Abstract\nA,B'], 'new.csv', { type: 'text/csv' }), + ) + await waitFor(() => + expect(authenticatedFetch).toHaveBeenCalledWith( + expect.stringContaining('/duplicates/check?'), + expect.anything(), + ), + ) + expect( + screen.queryByRole('checkbox', { name: 'Import Duplicates' }), + ).not.toBeInTheDocument() +}) diff --git a/frontend/components/can-sr/references/references-workspace.tsx b/frontend/components/can-sr/references/references-workspace.tsx new file mode 100644 index 00000000..82e8d61a --- /dev/null +++ b/frontend/components/can-sr/references/references-workspace.tsx @@ -0,0 +1,1835 @@ +'use client' + +import { useEffect, useRef, useState } from 'react' +import { + AlertCircle, + CheckCircle, + ChevronDown, + ChevronUp, + CircleX, + CornerDownLeft, + Eye, + EyeOff, + Filter, + Plus, + Settings, + Play, +} from 'lucide-react' +import { authenticatedFetch } from '@/lib/auth' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' +import { Checkbox } from '@/components/ui/checkbox' +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' + +type Props = { + srId: string + hasDataset: boolean | null + copy: Record +} + +type Citation = Record + +type WorkspacePage = { + citations: Citation[] + total_count: number + columns: string[] + available_columns?: string[] + sort: string + direction: 'asc' | 'desc' + query_fingerprint: string + duplicate_fields?: string[] + duplicate_counts?: Record + duplicate_groups?: Array<{ + group_id: string + citation_ids: number[] + status: string + members: Citation[] + suggested_survivor_id?: number | null + survivor_reason?: string | null + review?: DuplicateReview | null + }> + dataset_revision?: unknown + duplicate_run?: { + run_id?: string | null + status?: 'succeeded' | 'not_run' + } +} + +type DuplicateReview = { + group_id: string + decision: 'confirmed_duplicate' | 'not_duplicate' | 'deferred' + survivor_id?: number | null + stale?: boolean +} + +type DuplicateGroup = NonNullable[number] + +const memberIds = (group: DuplicateGroup) => + group.members + .map((member) => Number(member.id)) + .filter((id) => Number.isFinite(id)) + +const validMemberId = (group: DuplicateGroup, id: unknown) => { + const numericId = Number(id) + return memberIds(group).includes(numericId) ? numericId : undefined +} + +export default function ReferencesWorkspace({ srId, hasDataset, copy }: Props) { + const [open, setOpen] = useState(false) + const [file, setFile] = useState(null) + const [includeDuplicates, setIncludeDuplicates] = useState(false) + const [hasDuplicates, setHasDuplicates] = useState(false) + const [warnings, setWarnings] = useState([]) + const [busy, setBusy] = useState(false) + const [message, setMessage] = useState('') + const [workspace, setWorkspace] = useState(null) + const [workspaceError, setWorkspaceError] = useState('') + const [workspaceLoading, setWorkspaceLoading] = useState(false) + const [search, setSearch] = useState('') + const [draftSearch, setDraftSearch] = useState('') + const [filters, setFilters] = useState>({}) + const [draftFilters, setDraftFilters] = useState>({}) + const [openFilter, setOpenFilter] = useState(null) + const [columnWidths, setColumnWidths] = useState>({}) + const [resizingColumn, setResizingColumn] = useState(null) + const resizeStart = useRef<{ + column: string + x: number + width: number + } | null>(null) + const [selectedIds, setSelectedIds] = useState([]) + const [deleting, setDeleting] = useState(false) + const [sort, setSort] = useState('id') + const [direction, setDirection] = useState<'asc' | 'desc'>('asc') + const [visibleColumns, setVisibleColumns] = useState(null) + const [columnsOpen, setColumnsOpen] = useState(false) + const [draftColumns, setDraftColumns] = useState([]) + const [columnsSaving, setColumnsSaving] = useState(false) + const [datasetReady, setDatasetReady] = useState(Boolean(hasDataset)) + const [selectedDatabase, setSelectedDatabase] = useState('') + const [searchStrings, setSearchStrings] = useState>({}) + const [duplicateStatus, setDuplicateStatus] = useState('') + const [duplicateFilterOpen, setDuplicateFilterOpen] = useState(false) + const [draftDuplicateStatus, setDraftDuplicateStatus] = useState('') + const [dedupOpen, setDedupOpen] = useState(false) + const [dedupFields, setDedupFields] = useState([]) + const [draftDedupFields, setDraftDedupFields] = useState([]) + const [dedupRunning, setDedupRunning] = useState(false) + const [dedupRunStatus, setDedupRunStatus] = useState< + 'not_run' | 'succeeded' | 'stale' + >('not_run') + const [reviewGroupIndex, setReviewGroupIndex] = useState(0) + const [reviews, setReviews] = useState([]) + const [selectedSurvivors, setSelectedSurvivors] = useState< + Record + >({}) + const [reviewMinimized, setReviewMinimized] = useState(true) + const [reviewSaving, setReviewSaving] = useState(false) + const [selectionMenuOpen, setSelectionMenuOpen] = useState(false) + + useEffect(() => { + setDatasetReady(Boolean(hasDataset)) + }, [hasDataset]) + + const loadWorkspace = async () => { + if (!datasetReady) { + setWorkspace(null) + return + } + setWorkspaceLoading(true) + setWorkspaceError('') + const params = new URLSearchParams({ + sr_id: srId, + sort, + direction, + }) + if (search.trim()) params.set('search', search.trim()) + const activeFilters = Object.fromEntries( + Object.entries(filters).filter(([, value]) => value.trim()), + ) + if (Object.keys(activeFilters).length) + params.set('filters', JSON.stringify(activeFilters)) + if (duplicateStatus) params.set('duplicate_status', duplicateStatus) + if (visibleColumns?.length) params.set('columns', visibleColumns.join(',')) + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace?${params}`, + ) + const data = await response.json().catch(() => ({})) + setWorkspaceLoading(false) + if (!response.ok) { + setWorkspaceError(data?.error || data?.detail || copy.gridFailed) + return + } + setWorkspace(data) + setDedupRunStatus( + data?.duplicate_run?.status === 'succeeded' ? 'succeeded' : 'not_run', + ) + } + + useEffect(() => { + void loadWorkspace() + // loadWorkspace intentionally tracks the request inputs rather than its identity. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + srId, + datasetReady, + search, + filters, + sort, + direction, + visibleColumns, + duplicateStatus, + ]) + + useEffect(() => { + if (!datasetReady) return + void (async () => { + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace/preferences?sr_id=${encodeURIComponent(srId)}`, + ) + const data = await response.json().catch(() => ({})) + if (response.ok && Array.isArray(data?.columns) && data.columns.length) { + setVisibleColumns(data.columns) + } + const dedupResponse = await authenticatedFetch( + `/api/can-sr/citations/workspace/deduplication-preferences?sr_id=${encodeURIComponent(srId)}`, + ) + const dedupData = await dedupResponse.json().catch(() => ({})) + if (dedupResponse.ok && Array.isArray(dedupData?.fields)) { + setDedupFields(dedupData.fields) + } + const reviewResponse = await authenticatedFetch( + `/api/can-sr/citations/workspace/duplicate-reviews?sr_id=${encodeURIComponent(srId)}`, + ) + const reviewData = await reviewResponse.json().catch(() => ({})) + if (reviewResponse.ok && Array.isArray(reviewData?.reviews)) + setReviews(reviewData.reviews) + })() + }, [srId, datasetReady]) + + useEffect(() => { + setSelectedIds([]) + setReviewGroupIndex(0) + }, [search, filters, sort, direction, visibleColumns, duplicateStatus]) + + useEffect(() => { + const groups = workspace?.duplicate_groups || [] + if (!groups.length) { + setReviewGroupIndex(0) + return + } + setReviewGroupIndex((current) => { + return Math.min(current, groups.length - 1) + }) + }, [workspace, reviews]) + + useEffect(() => { + const groups = workspace?.duplicate_groups || [] + if (!groups.length) { + setSelectedSurvivors({}) + return + } + setSelectedSurvivors((current) => { + const next: Record = {} + for (const group of groups) { + const savedReview = + group.review || + reviews.find((review) => review.group_id === group.group_id) + const selected = + validMemberId(group, current[group.group_id]) ?? + validMemberId(group, savedReview?.survivor_id) ?? + validMemberId(group, group.suggested_survivor_id) ?? + memberIds(group)[0] + if (selected !== undefined) next[group.group_id] = selected + } + return next + }) + }, [workspace, reviews]) + + const close = () => { + setOpen(false) + setFile(null) + setIncludeDuplicates(false) + setHasDuplicates(false) + setWarnings([]) + setMessage('') + setSelectedDatabase('') + setSearchStrings({}) + } + + const searchDatabases = async () => { + if (!selectedDatabase) return + setBusy(true) + setMessage('') + const response = await authenticatedFetch( + `/api/can-sr/search?sr_id=${encodeURIComponent(srId)}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + database: selectedDatabase, + search_term: searchStrings[selectedDatabase] || '', + }), + }, + ) + const data = await response.json().catch(() => ({})) + setBusy(false) + setMessage( + response.ok + ? data?.message || copy.searchCompleted || 'Database search completed.' + : data?.error || + data?.detail || + copy.searchFailed || + 'Database search failed.', + ) + } + + const applyFilter = (column: string) => { + setFilters((current) => ({ + ...current, + [column]: (draftFilters[column] || '').trim(), + })) + setOpenFilter(null) + } + + const clearFilter = (column: string) => { + setDraftFilters((current) => ({ ...current, [column]: '' })) + setFilters((current) => { + const next = { ...current } + delete next[column] + return next + }) + setOpenFilter(null) + } + + const beginResize = ( + column: string, + event: React.PointerEvent, + ) => { + event.preventDefault() + event.currentTarget.setPointerCapture?.(event.pointerId) + resizeStart.current = { + column, + x: event.clientX, + width: columnWidths[column] || 160, + } + setResizingColumn(column) + window.addEventListener('pointermove', resizeColumn) + window.addEventListener('pointerup', endResize) + window.addEventListener('pointercancel', endResize) + } + + const resizeColumn = (event: PointerEvent) => { + const start = resizeStart.current + if (!start) return + setColumnWidths((current) => ({ + ...current, + [start.column]: Math.max(96, start.width + event.clientX - start.x), + })) + } + + const endResize = () => { + resizeStart.current = null + setResizingColumn(null) + window.removeEventListener('pointermove', resizeColumn) + window.removeEventListener('pointerup', endResize) + window.removeEventListener('pointercancel', endResize) + } + + const tableWidth = + 152 + + (workspace?.columns || []).reduce( + (total, column) => total + (columnWidths[column] || 160), + 0, + ) + + const importFile = async () => { + if (!file) return + setBusy(true) + setMessage('') + const form = new FormData() + form.append('file', file) + form.append('commit_key', crypto.randomUUID()) + form.append('include_duplicates', String(includeDuplicates)) + const text = await file.text() + const headerLine = text.split(/\r?\n/, 1)[0] || '' + const headers = file.name.toLowerCase().endsWith('.csv') + ? headerLine + .split(',') + .map((value) => value.trim().replace(/^"|"$/g, '')) + .filter(Boolean) + : [] + const normalized = headers.map((value) => + value.toLowerCase().replace(/[^a-z0-9]+/g, ''), + ) + const localWarnings: string[] = [] + if (!normalized.includes('title')) + localWarnings.push( + copy.missingTitle || 'The file is missing a Title column.', + ) + if (!normalized.includes('abstract')) + localWarnings.push( + copy.missingAbstract || 'The file is missing an Abstract column.', + ) + const extras = headers.filter( + (_, index) => !['title', 'abstract'].includes(normalized[index]), + ) + if (extras.length) { + localWarnings.push( + `${copy.additionalColumns || 'Additional source columns will be preserved as text'}: ${extras.join(', ')}`, + ) + } + setWarnings(localWarnings) + if (!normalized.includes('title') || !normalized.includes('abstract')) { + setBusy(false) + return + } + const response = await authenticatedFetch( + `/api/can-sr/citations/import?sr_id=${encodeURIComponent(srId)}`, + { method: 'POST', body: form }, + ) + const data = await response.json().catch(() => ({})) + setBusy(false) + if (!response.ok) { + setMessage( + data?.error || + data?.detail || + copy.importFailed || + 'Unable to import the references.', + ) + return + } + const importedMessage = ( + copy.imported || 'Import complete: {count} citations added.' + ).replace('{count}', String(data.rows_inserted)) + setMessage( + data.duplicates_skipped + ? `${importedMessage} ${( + copy.duplicatesSkipped || '{count} duplicates skipped.' + ).replace('{count}', String(data.duplicates_skipped))}` + : importedMessage, + ) + setDatasetReady(true) + await loadWorkspace() + setFile(null) + } + + const checkFileForDuplicates = async (selectedFile: File | null) => { + setHasDuplicates(false) + setIncludeDuplicates(false) + if (!selectedFile) return + const form = new FormData() + form.append('file', selectedFile) + const response = await authenticatedFetch( + `/api/can-sr/citations/duplicates/check?sr_id=${encodeURIComponent(srId)}`, + { method: 'POST', body: form }, + ) + const data = await response.json().catch(() => ({})) + if (response.ok) { + const duplicateCount = Number(data.duplicates_count) + setHasDuplicates(duplicateCount > 0) + setWarnings((current) => + duplicateCount > 0 + ? [ + ...current.filter( + (warning) => !warning.includes('__duplicate_warning__'), + ), + `__duplicate_warning__${( + copy.duplicatesDetected || + '{count} duplicate references detected. They will be merged unless Import Duplicates is selected.' + ).replace('{count}', String(duplicateCount))}`, + ] + : current.filter( + (warning) => !warning.includes('__duplicate_warning__'), + ), + ) + } + } + + const toggleSort = (column: string) => { + if (sort === column) { + setDirection((current) => (current === 'asc' ? 'desc' : 'asc')) + } else { + setSort(column) + setDirection('asc') + } + } + const toggleSelected = (id: number) => { + setSelectedIds((current) => + current.includes(id) + ? current.filter((value) => value !== id) + : [...current, id], + ) + } + const queryPayload = () => ({ + search: search.trim() || undefined, + sort, + direction, + columns: visibleColumns, + filters: Object.fromEntries( + Object.entries(filters).filter(([, value]) => value.trim()), + ), + duplicate_status: duplicateStatus || undefined, + }) + const deleteSelected = async (ids = selectedIds, advanceReview = false) => { + if (!ids.length) return false + setDeleting(true) + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace?sr_id=${encodeURIComponent(srId)}`, + { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + citation_ids: ids, + query_fingerprint: workspace?.query_fingerprint, + query: queryPayload(), + confirmed: true, + }), + }, + ) + const data = await response.json().catch(() => ({})) + setDeleting(false) + if (!response.ok) { + setWorkspaceError(data?.error || data?.detail || copy.deleteFailed) + return false + } + setSelectedIds([]) + await loadWorkspace() + if (advanceReview) setReviewGroupIndex((index) => index + 1) + return true + } + const saveReview = async ( + groupId: string, + decision: DuplicateReview['decision'], + survivorId?: number, + ) => { + setReviewSaving(true) + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace/duplicate-reviews?sr_id=${encodeURIComponent(srId)}`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + group_id: groupId, + decision, + survivor_id: survivorId, + }), + }, + ) + const data = await response.json().catch(() => ({})) + setReviewSaving(false) + if (!response.ok) { + setWorkspaceError(data?.error || data?.detail || copy.gridFailed) + return + } + setReviews((current) => [ + ...current.filter((review) => review.group_id !== groupId), + data, + ]) + await loadWorkspace() + } + const getReviewSurvivor = (group: DuplicateGroup) => { + const savedReview = + group.review || + reviews.find((review) => review.group_id === group.group_id) + return ( + validMemberId(group, selectedSurvivors[group.group_id]) ?? + validMemberId(group, savedReview?.survivor_id) ?? + validMemberId(group, group.suggested_survivor_id) ?? + memberIds(group)[0] + ) + } + const openDuplicateGroup = (groupId: unknown) => { + const visibleIds = new Set( + (workspace?.citations || []).map((citation) => Number(citation.id)), + ) + const reviewGroups = (workspace?.duplicate_groups || []).filter((group) => + group.citation_ids.some((id) => visibleIds.has(Number(id))), + ) + const groupIndex = reviewGroups.findIndex( + (group) => group.group_id === String(groupId), + ) + if (groupIndex >= 0) { + setReviewMinimized(false) + setReviewGroupIndex(groupIndex) + } + } + const openColumns = () => { + setDraftColumns(workspace?.columns || visibleColumns || ['id']) + setColumnsOpen(true) + } + const openDeduplication = () => { + setDraftDedupFields( + dedupFields.length + ? dedupFields + : workspace?.columns.filter( + (column) => column !== 'id' && column !== 'provenance', + ) || [], + ) + setDedupOpen(true) + } + const saveDeduplication = async () => { + const merged = [ + ...new Set( + [...dedupFields, ...draftDedupFields].filter( + (field) => field !== 'id' && field !== 'provenance', + ), + ), + ] + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace/deduplication-preferences?sr_id=${encodeURIComponent(srId)}`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ fields: merged }), + }, + ) + const data = await response.json().catch(() => ({})) + if (!response.ok) { + setWorkspaceError(data?.error || data?.detail || copy.gridFailed) + return + } + setDedupFields(data.fields || merged) + setDedupRunStatus('stale') + setDedupOpen(false) + await loadWorkspace() + } + const runDeduplication = async () => { + setDedupRunning(true) + setWorkspaceError('') + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace/duplicate-runs?sr_id=${encodeURIComponent(srId)}`, + { method: 'POST' }, + ) + const data = await response.json().catch(() => ({})) + setDedupRunning(false) + if (!response.ok) { + setWorkspaceError(data?.error || data?.detail || copy.gridFailed) + return + } + setDedupRunStatus('succeeded') + await loadWorkspace() + } + const openDuplicateFilter = () => { + setDraftDuplicateStatus(duplicateStatus) + setDuplicateFilterOpen(true) + } + const applyDuplicateFilter = () => { + setDuplicateStatus(draftDuplicateStatus) + setDuplicateFilterOpen(false) + } + const toggleColumn = (column: string) => { + if (column === 'id') return + setDraftColumns((current) => + current.includes(column) + ? current.filter((value) => value !== column) + : [...current, column], + ) + } + const moveColumn = (column: string, offset: -1 | 1) => { + setDraftColumns((current) => { + const index = current.indexOf(column) + const nextIndex = index + offset + if (index < 0 || nextIndex < 1 || nextIndex >= current.length) + return current + const next = [...current] + ;[next[index], next[nextIndex]] = [next[nextIndex], next[index]] + return next + }) + } + const saveColumns = async () => { + const columns = draftColumns.includes('id') + ? draftColumns + : ['id', ...draftColumns] + setColumnsSaving(true) + const response = await authenticatedFetch( + `/api/can-sr/citations/workspace/preferences?sr_id=${encodeURIComponent(srId)}`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ columns }), + }, + ) + setColumnsSaving(false) + if (!response.ok) { + const data = await response.json().catch(() => ({})) + setWorkspaceError(data?.error || data?.detail || copy.gridFailed) + return + } + setVisibleColumns(columns) + setColumnsOpen(false) + const mergedDedupFields = [ + ...new Set([ + ...dedupFields, + ...columns.filter( + (column) => column !== 'id' && column !== 'provenance', + ), + ]), + ] + const dedupResponse = await authenticatedFetch( + `/api/can-sr/citations/workspace/deduplication-preferences?sr_id=${encodeURIComponent(srId)}`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ fields: mergedDedupFields }), + }, + ) + if (dedupResponse.ok) setDedupFields(mergedDedupFields) + } + + return ( + <> +
+
+
+

{copy.gridTitle}

+

+ {datasetReady ? copy.datasetReady : copy.empty} +

+
+ {datasetReady ? ( +
+
+ +
+ { + const nextSearch = event.target.value + setDraftSearch(nextSearch) + if (!nextSearch.trim()) { + setSearch('') + } + }} + onKeyDown={(event) => { + if (event.key === 'Enter') { + event.preventDefault() + setSearch(draftSearch) + } + }} + /> + {draftSearch.trim() ? ( + + + ) : null} +
+
+ + {selectedIds.length ? ( + + ) : null} +
+ ) : ( + + )} +
+ {!datasetReady ? ( +

+ {copy.empty} +

+ ) : ( +
+ {workspaceLoading && !workspace ? ( +

+ {copy.loading} +

+ ) : null} + {workspaceError ? ( +

+ {workspaceError} +

+ ) : null} + {workspace ? ( + <> + {(() => { + const visibleIds = new Set( + workspace.citations.map((citation) => Number(citation.id)), + ) + const reviewGroups = ( + workspace.duplicate_groups || [] + ).filter((group) => + group.citation_ids.some((id) => visibleIds.has(Number(id))), + ) + const reviewGroup = reviewGroups[reviewGroupIndex] + return reviewGroup ? ( +
+
+
+

+ {copy.duplicateGroupReview} +

+

+ {copy.groupSummary + .replace( + '{current}', + String(reviewGroupIndex + 1), + ) + .replace('{total}', String(reviewGroups.length)) + .replace('{status}', reviewGroup.status) + .replace( + '{count}', + String(reviewGroup.members.length), + )} +

+
+
+ +
+
+ {!reviewMinimized ? ( + <> +
+ {reviewGroup.members.map((member) => ( +
+
+ + {copy.record.replace( + '{id}', + String(member.id), + )} + + + {String(member.duplicate_score ?? '—')} + +
+ + {workspace.duplicate_fields?.map((field) => ( +
+ + {field} + + + {String(member[field] ?? '')} + +
+ ))} +
+ ))} +
+
+ + {copy.reviewDecision} + + {( + [ + ['confirmed_duplicate', copy.confirmDuplicate], + ['not_duplicate', copy.keepBoth], + ['deferred', copy.defer], + ] as const + ).map(([decision, label]) => { + const review = + reviewGroup.review || + reviews.find( + (item) => + item.group_id === reviewGroup.group_id, + ) + if ( + decision === 'confirmed_duplicate' && + review?.decision === 'confirmed_duplicate' + ) { + const survivorId = + getReviewSurvivor(reviewGroup) + const nonSurvivor = memberIds(reviewGroup).find( + (id) => id !== survivorId, + ) + return ( + + ) + } + return ( + + ) + })} + {( + reviewGroup.review || + reviews.find( + (item) => + item.group_id === reviewGroup.group_id, + ) + )?.decision === 'confirmed_duplicate' ? ( + + {copy.survivor}:{' '} + {String(getReviewSurvivor(reviewGroup) ?? '')} + + ) : null} + {( + reviewGroup.review || + reviews.find( + (item) => + item.group_id === reviewGroup.group_id, + ) + )?.stale ? ( + + {copy.staleSurvivor || + 'The saved survivor is no longer in this group; a current member was selected.'} + + ) : null} +
+ + +
+
+ + ) : null} +
+ ) : null + })()} +
+ + + + + + {workspace.columns.map((column) => ( + + ))} + + + + + + + + {workspace.columns.map((column) => ( + + ))} + + + + + {workspace.citations.map((citation, index) => ( + + + + + {workspace.columns.map((column) => ( + + ))} + + ))} + {Array.from({ + length: workspace.citations.length === 0 ? 15 : 1, + }).map((_, index) => ( + + + + + {workspace.columns.map((column) => ( + + ))} + + + ))} + +
+ # + +
+ 0 && + workspace.citations.every((citation) => + selectedIds.includes(Number(citation.id)), + ) + } + onCheckedChange={() => setSelectionMenuOpen(true)} + /> + {selectionMenuOpen ? ( +
+ {[ + [copy.none, []], + [ + copy.all, + workspace.citations.map((citation) => + Number(citation.id), + ), + ], + [ + copy.suggestedDuplicates, + (workspace.duplicate_groups || []) + .filter((group) => { + const review = + group.review || + reviews.find( + (item) => + item.group_id === group.group_id, + ) + return ( + review?.decision !== 'not_duplicate' + ) + }) + .flatMap((group) => + group.citation_ids + .map(Number) + .filter( + (id) => + id !== getReviewSurvivor(group), + ), + ) + .filter((id) => + workspace.citations.some( + (citation) => + Number(citation.id) === id, + ), + ), + ], + ].map(([label, ids]) => ( + + ))} +
+ ) : null} +
+ 		+
+ + {duplicateFilterOpen ? ( +
+

+ {copy.duplicateStatus} +

+ + +
+ + +
+
+ ) : null} + + +
+ {dedupOpen ? ( +
+

+ {copy.duplicateMatchingFields} +

+

+ {copy.duplicateFieldsDescription} +

+
+ {( + workspace.available_columns || + workspace.columns + ) + .filter( + (column) => + column !== 'id' && + column !== 'provenance', + ) + .map((column) => ( + + ))} +
+
+ + +
+
+ ) : null} + 		+
+ + {column !== 'id' ? ( + + ) : null} +
+ {column !== 'id' && openFilter === column ? ( +
+ +
+ + +
+
+ ) : null} + 		+ + {columnsOpen ? ( +
+
+

+ {copy.columnsTitle} +

+

+ {copy.columnsDescription} +

+
+
+ {( + workspace.available_columns || + workspace.columns || + [] + ).map((column) => ( +
+ + {draftColumns.includes(column) && + column !== 'id' ? ( + + + + + ) : null} +
+ ))} +
+
+ + +
+
+ ) : null} +
+ {index + 1} + + + toggleSelected(Number(citation.id)) + } + /> + review.group_id === citation.duplicate_group_id)?.decision === 'not_duplicate' ? 'bg-green-50 text-green-700' : 'text-gray-300'}`} + title={ + citation.duplicate_status === 'exact' + ? copy.exactDuplicateMatch + : citation.duplicate_status === 'possible' + ? copy.possibleDuplicateMatch + : reviews.find( + (review) => + review.group_id === + citation.duplicate_group_id, + )?.decision === 'not_duplicate' + ? copy.keptBothReviewGroup + : copy.noDuplicateMatch + } + > + {reviews.find( + (review) => + review.group_id === citation.duplicate_group_id, + )?.decision === 'not_duplicate' ? ( + + ) : citation.duplicate_status === 'exact' ? ( + + ) : citation.duplicate_status === 'possible' ? ( + + ) : ( + — + )} + + {String(citation[column] ?? '')} +
+ {workspace.citations.length > 0 ? ( + workspace.total_count + index + 1 + ) : ( + + + +
+
+ + ) : null} + {workspace ? ( +
+ + {copy.results.replace( + '{count}', + String(workspace.total_count), + )} + +
+ ) : null} +
+ )} +
+ { + if (!next) void close() + else setOpen(true) + }} + > + + + + {copy.addReferencesTitle || copy.dialogTitle} + + {copy.dialogDescription} + + + + + {copy.uploadTab || 'Upload file'} + + + {copy.searchTab || 'Search databases'} + + + + + {file ? ( +
+ + {copy.selectedFile} {file.name} ( + {Math.ceil(file.size / 1024)} KB) + + +
+ ) : null} + {hasDuplicates ? ( + + ) : null} + {warnings.length ? ( +
    + {warnings.map((warning) => ( +
  • + {warning.replace('__duplicate_warning__', '')} +
    • + ))} +
+ ) : null} + {message ? ( +

+ {message} +

+ ) : null} + + +

+ {copy.searchDescription} +

+
{ + event.preventDefault() + void searchDatabases() + }} + > + {['Pubmed', 'Scopus', 'EuropePMC'].map((database) => ( + + ))} + +
+ {message ? ( +

+ {message} +

+ ) : null} + + + + + + + + + + ) +} diff --git a/frontend/components/can-sr/setup/citation-field-selector.test.tsx b/frontend/components/can-sr/setup/citation-field-selector.test.tsx index b71075fd..d6d1a41b 100644 --- a/frontend/components/can-sr/setup/citation-field-selector.test.tsx +++ b/frontend/components/can-sr/setup/citation-field-selector.test.tsx @@ -4,7 +4,10 @@ import { describe, expect, it, vi } from 'vitest' import CitationFieldSelector from './citation-field-selector' import { emptyCriteria } from './criteria-types' -const labels = new Proxy({}, { get: (_target, property) => String(property) }) as Record +const labels = new Proxy( + {}, + { get: (_target, property) => String(property) }, +) as Record describe('CitationFieldSelector', () => { it('adds, removes, and reorders configured title/abstract fields', async () => { @@ -12,30 +15,60 @@ describe('CitationFieldSelector', () => { const dispatch = vi.fn() const criteria = emptyCriteria() criteria.citation_fields.l1_include = ['Title', 'Missing'] - render() - expect(screen.getByRole('heading', { name: 'citationFields (2)' })).toBeInTheDocument() - expect(screen.getByRole('row', { name: /Missing unavailableField/ })).toBeInTheDocument() + render( + , + ) + expect( + screen.getByRole('heading', { name: 'citationFields' }), + ).toBeInTheDocument() + expect(screen.getByRole('option', { name: /Missing/ })).toBeInTheDocument() expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'moveUp Missing' })) - expect(dispatch).toHaveBeenCalledWith({ type: 'set-citation-fields', value: ['Missing', 'Title'] }) - await user.click(screen.getByRole('button', { name: 'addField' })) - expect(screen.getByRole('menu')).toBeInTheDocument() - expect(screen.queryByRole('menuitem', { name: 'Title' })).not.toBeInTheDocument() - await user.click(screen.getByRole('menuitem', { name: 'Abstract' })) - expect(dispatch).toHaveBeenLastCalledWith({ type: 'set-citation-fields', value: ['Title', 'Missing', 'Abstract'] }) - await user.click(screen.getByRole('button', { name: 'removeField Missing' })) - expect(dispatch).toHaveBeenLastCalledWith({ type: 'set-citation-fields', value: ['Title'] }) + expect(dispatch).toHaveBeenCalledWith({ + type: 'set-citation-fields', + value: ['Missing', 'Title'], + }) + await user.click(screen.getByRole('button', { name: 'addAdditionalField' })) + expect(dispatch).toHaveBeenLastCalledWith({ + type: 'set-citation-fields', + value: ['Title', 'Missing', 'DOI'], + }) + await user.click( + screen.getByRole('button', { name: 'removeField Missing' }), + ) + expect(dispatch).toHaveBeenLastCalledWith({ + type: 'set-citation-fields', + value: ['Title'], + }) expect(screen.getByLabelText('doiField')).toHaveDisplayValue('noDoiField') - expect(screen.getByLabelText('doiField')).toContainHTML('') + expect(screen.getByLabelText('doiField')).toContainHTML( + '', + ) expect(screen.queryByText('likelyDoi')).not.toBeInTheDocument() - await user.click(screen.getByRole('button', { name: 'collapse citationFields' })) + await user.click( + screen.getByRole('button', { name: 'collapse citationFields' }), + ) expect(screen.queryByRole('table')).not.toBeInTheDocument() expect(screen.queryByLabelText('doiField')).not.toBeInTheDocument() - expect(screen.getByRole('button', { name: 'addField' })).toBeInTheDocument() - await user.click(screen.getByRole('button', { name: 'expand citationFields' })) + expect( + screen.queryByRole('button', { name: 'addAdditionalField' }), + ).not.toBeInTheDocument() + await user.click( + screen.getByRole('button', { name: 'expand citationFields' }), + ) expect(screen.getByRole('table')).toBeInTheDocument() expect(screen.getByLabelText('doiField')).toBeInTheDocument() }) diff --git a/frontend/components/can-sr/setup/criteria-builder.test.tsx b/frontend/components/can-sr/setup/criteria-builder.test.tsx index 4dd1b41a..de24c3fa 100644 --- a/frontend/components/can-sr/setup/criteria-builder.test.tsx +++ b/frontend/components/can-sr/setup/criteria-builder.test.tsx @@ -9,7 +9,7 @@ const labels = new Proxy({}, { get: (_target, property) => String(property) }) a function Harness() { const [state, dispatch] = useReducer(criteriaDraftReducer, { criteria: emptyCriteria(), revision: 0, dirty: false }) - return <>{state.dirty ? 'dirty' : 'clean'} + return <>{state.dirty ? 'dirty' : 'clean'} } describe('CriteriaBuilder', () => { diff --git a/frontend/components/can-sr/setup/criteria-builder.tsx b/frontend/components/can-sr/setup/criteria-builder.tsx index 81d4da54..841a4325 100644 --- a/frontend/components/can-sr/setup/criteria-builder.tsx +++ b/frontend/components/can-sr/setup/criteria-builder.tsx @@ -1,6 +1,7 @@ 'use client' import { useEffect, useRef, useState, type Dispatch } from 'react' +import { authenticatedFetch } from '@/lib/auth' import { ArrowDown, ArrowUp, ChevronDown, ChevronRight, ChevronUp, ChevronsDown, Pencil, Plus, Trash2 } from 'lucide-react' import type { CriteriaDraftAction, CriteriaDraftState, ScreeningQuestion } from './criteria-types' import ParameterBuilder from './parameter-builder' @@ -11,6 +12,7 @@ import ContextEditorDialog from './context-editor-dialog' type SourceOption = { stage: 'l1' | 'l2'; question: ScreeningQuestion } type Props = { + srId: string state: CriteriaDraftState dispatch: Dispatch labels: Record @@ -19,6 +21,7 @@ type Props = { } function QuestionCard({ + srId, question, index, count, @@ -33,6 +36,7 @@ function QuestionCard({ onToggle, citationFields, }: { + srId: string question: ScreeningQuestion index: number count: number @@ -49,6 +53,7 @@ function QuestionCard({ }) { const prefix = `${stage}-${question.id}` const [contextAnswerId, setContextAnswerId] = useState(null) + const [setAnswerStatus, setSetAnswerStatus] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle') const contextAnswer = question.answers.find((answer) => answer.id === contextAnswerId) return (
@@ -83,11 +88,11 @@ function QuestionCard({ {question.answer_column ? <>

{labels.answerColumnTooltip}

- dispatch({ type: 'set-answer-column', itemId: question.id, value: event.target.value || null })} className="min-w-0 flex-1 rounded-md border px-2 py-1.5 text-sm" title={labels.answerColumnTooltip}> {!citationFields.fields.some((field) => field.name === question.answer_column) ? : null} {citationFields.fields.map((field) => )} - +
: null}
@@ -171,7 +176,7 @@ function QuestionCard({ ) } -export default function CriteriaBuilder({ state, dispatch, labels, diagnostics = [], citationFields = { fields: [], unavailable_configured_fields: [] } }: Props) { +export default function CriteriaBuilder({ srId, state, dispatch, labels, diagnostics = [], citationFields = { fields: [], unavailable_configured_fields: [] } }: Props) { const [collapsedSections, setCollapsedSections] = useState>({ l1: false, l2: false }) const [collapsedQuestions, setCollapsedQuestions] = useState>(() => new Set()) const pendingQuestion = useRef<{ stage: 'l1' | 'l2'; previousCount: number } | null>(null) @@ -243,13 +248,13 @@ export default function CriteriaBuilder({ state, dispatch, labels, diagnostics = {state.criteria[stage].length === 0 ?

{labels.noQuestions}

: null} {state.criteria[stage].map((question, index) => { const position = ordered.findIndex((item) => item.question.id === question.id) - return item.itemId === question.id)} sourceReferenced={conditions.some((condition) => condition.source_item_id === question.id)} referencedAnswerIds={new Set(conditions.filter((condition) => condition.source_item_id === question.id).map((condition) => condition.option_id))} collapsed={collapsedQuestions.has(question.id)} onToggle={() => setCollapsedQuestions((current) => { const next = new Set(current); if (next.has(question.id)) next.delete(question.id); else next.add(question.id); return next })} /> + return item.itemId === question.id)} sourceReferenced={conditions.some((condition) => condition.source_item_id === question.id)} referencedAnswerIds={new Set(conditions.filter((condition) => condition.source_item_id === question.id).map((condition) => condition.option_id))} collapsed={collapsedQuestions.has(question.id)} onToggle={() => setCollapsedQuestions((current) => { const next = new Set(current); if (next.has(question.id)) next.delete(question.id); else next.add(question.id); return next })} /> })}
: null} ))} - + ) } diff --git a/frontend/components/can-sr/setup/criteria-editor.tsx b/frontend/components/can-sr/setup/criteria-editor.tsx index 56036065..cb776074 100644 --- a/frontend/components/can-sr/setup/criteria-editor.tsx +++ b/frontend/components/can-sr/setup/criteria-editor.tsx @@ -3,10 +3,22 @@ import { useCallback, useEffect, useMemo, useReducer, useState } from 'react' import { authenticatedFetch } from '@/lib/auth' import CriteriaBuilder from './criteria-builder' -import { criteriaDraftReducer, emptyCriteria, type CriteriaConfig } from './criteria-types' -import { backendDiagnostics, type CriteriaDiagnostic, validateCriteriaDraft } from './criteria-validation' -import YamlImportPreview, { type CriteriaImportPreview } from './yaml-import-preview' -import CriteriaRecoveryDialog, { type RecoveryMode } from './criteria-recovery-dialog' +import { + criteriaDraftReducer, + emptyCriteria, + type CriteriaConfig, +} from './criteria-types' +import { + backendDiagnostics, + type CriteriaDiagnostic, + validateCriteriaDraft, +} from './criteria-validation' +import YamlImportPreview, { + type CriteriaImportPreview, +} from './yaml-import-preview' +import CriteriaRecoveryDialog, { + type RecoveryMode, +} from './criteria-recovery-dialog' import { downloadCriteriaDraft } from './criteria-download' import type { CitationFieldContract } from './citation-field-selector' import { @@ -19,21 +31,53 @@ import { AlertDialogTitle, } from '@/components/ui/alert-dialog' -type Props = { srId: string; labels: Record; hasScreeningData: boolean; reloadKey?: number } +type Props = { + srId: string + labels: Record + hasScreeningData: boolean + reloadKey?: number +} -export default function CriteriaEditor({ srId, labels, hasScreeningData, reloadKey = 0 }: Props) { - const [state, dispatch] = useReducer(criteriaDraftReducer, { criteria: emptyCriteria(), revision: 0, dirty: false }) +export default function CriteriaEditor({ + srId, + labels, + hasScreeningData, + reloadKey = 0, +}: Props) { + const [state, dispatch] = useReducer(criteriaDraftReducer, { + criteria: emptyCriteria(), + revision: 0, + dirty: false, + }) const [status, setStatus] = useState(labels.loading) - const [migrationFingerprint, setMigrationFingerprint] = useState(null) + const [validationState, setValidationState] = useState< + 'idle' | 'failed' | 'passed' + >('idle') + const [migrationFingerprint, setMigrationFingerprint] = useState< + string | null + >(null) const [force, setForce] = useState(false) - const [serverDiagnostics, setServerDiagnostics] = useState([]) - const [importPreview, setImportPreview] = useState(null) + const [serverDiagnostics, setServerDiagnostics] = useState< + CriteriaDiagnostic[] + >([]) + const [importPreview, setImportPreview] = + useState(null) const [recoveryMode, setRecoveryMode] = useState(null) const [saveConfirmOpen, setSaveConfirmOpen] = useState(false) - const [citationFields, setCitationFields] = useState({ fields: [], unavailable_configured_fields: [] }) - const availableFieldNames = useMemo(() => new Set(citationFields.fields.map((field) => field.name)), [citationFields.fields]) - const clientDiagnostics = useMemo(() => validateCriteriaDraft(state.criteria, availableFieldNames), [availableFieldNames, state.criteria]) + const [citationFields, setCitationFields] = useState({ + fields: [], + unavailable_configured_fields: [], + }) + const availableFieldNames = useMemo( + () => new Set(citationFields.fields.map((field) => field.name)), + [citationFields.fields], + ) + const clientDiagnostics = useMemo( + () => validateCriteriaDraft(state.criteria, availableFieldNames), + [availableFieldNames, state.criteria], + ) const diagnostics = [...clientDiagnostics, ...serverDiagnostics] + const displayedDiagnostics = validationState === 'idle' ? [] : diagnostics // Include the upload revision in the request URL so the callback is recreated // after a successful upload without changing the effect dependency shape. @@ -51,52 +95,88 @@ export default function CriteriaEditor({ srId, labels, hasScreeningData, reloadK if (fieldsResponse.ok) setCitationFields(fieldsData) if (!response.ok) { const detail = data?.detail - const message = typeof detail === 'string' - ? detail - : backendDiagnostics(detail)[0]?.message || labels.loadFailed + const message = + typeof detail === 'string' + ? detail + : backendDiagnostics(detail)[0]?.message || labels.loadFailed setStatus(message) return } - dispatch({ type: 'replace', criteria: data.criteria as CriteriaConfig, revision: data.revision || 0 }) + dispatch({ + type: 'replace', + criteria: data.criteria as CriteriaConfig, + revision: data.revision || 0, + }) setMigrationFingerprint(data?.migration?.fingerprint || null) - setStatus(data?.migration?.requires_confirmation ? labels.migrationWarning : '') + setValidationState('idle') + setStatus( + data?.migration?.requires_confirmation ? labels.migrationWarning : '', + ) }, [endpoint, labels, reloadKey, srId]) - useEffect(() => { void load() }, [load]) - useEffect(() => { setServerDiagnostics([]) }, [state.criteria]) useEffect(() => { - const warn = (event: BeforeUnloadEvent) => { if (state.dirty) event.preventDefault() } + void load() + }, [load]) + useEffect(() => { + setServerDiagnostics([]) + if (state.dirty) setValidationState('idle') + }, [state.criteria, state.dirty]) + useEffect(() => { + const warn = (event: BeforeUnloadEvent) => { + if (state.dirty) event.preventDefault() + } window.addEventListener('beforeunload', warn) return () => window.removeEventListener('beforeunload', warn) }, [state.dirty]) const save = async () => { if (clientDiagnostics.length) { + setValidationState('failed') setStatus(labels.validationFailed) - document.getElementById(clientDiagnostics[0].itemId ? `criteria-item-${clientDiagnostics[0].itemId}` : 'criteria-l1-fields')?.focus() + document + .getElementById( + clientDiagnostics[0].itemId + ? `criteria-item-${clientDiagnostics[0].itemId}` + : 'criteria-l1-fields', + ) + ?.focus() return } setStatus(labels.saving) const validationResponse = await authenticatedFetch(endpoint, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(state.criteria), + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(state.criteria), }) const validationData = await validationResponse.json().catch(() => ({})) if (!validationResponse.ok || validationData?.valid === false) { - const nextDiagnostics = backendDiagnostics(validationData?.detail ?? validationData).map((diagnostic) => { + const nextDiagnostics = backendDiagnostics( + validationData?.detail ?? validationData, + ).map((diagnostic) => { const parts = diagnostic.path.split('.') const collection = parts[0] as 'l1' | 'l2' | 'parameters' const index = Number(parts[1]) - const item = Number.isInteger(index) && ['l1', 'l2', 'parameters'].includes(collection) ? state.criteria[collection][index] : undefined + const item = + Number.isInteger(index) && + ['l1', 'l2', 'parameters'].includes(collection) + ? state.criteria[collection][index] + : undefined return { ...diagnostic, itemId: item?.id } }) setServerDiagnostics(nextDiagnostics) + setValidationState('failed') setStatus(nextDiagnostics[0]?.message || labels.validationFailed) return } const response = await authenticatedFetch(endpoint, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ criteria: state.criteria, expected_revision: state.revision, force, migration_fingerprint: migrationFingerprint }), + body: JSON.stringify({ + criteria: state.criteria, + expected_revision: state.revision, + force, + migration_fingerprint: migrationFingerprint, + }), }) const data = await response.json().catch(() => ({})) if (!response.ok) { @@ -106,13 +186,24 @@ export default function CriteriaEditor({ srId, labels, hasScreeningData, reloadK setStatus(labels.conflictDetected) return } - setStatus(typeof detail === 'string' - ? detail - : detail?.message || backendDiagnostics(detail)[0]?.message || labels.saveFailed) + setStatus( + typeof detail === 'string' + ? detail + : detail?.message || + backendDiagnostics(detail)[0]?.message || + labels.saveFailed, + ) return } - dispatch({ type: 'replace', criteria: data.criteria, revision: data.revision }) - setMigrationFingerprint(null); setForce(false); setStatus(labels.saved) + dispatch({ + type: 'replace', + criteria: data.criteria, + revision: data.revision, + }) + setMigrationFingerprint(null) + setForce(false) + setValidationState('passed') + setStatus(labels.saved) } const requestSave = () => { @@ -125,63 +216,211 @@ export default function CriteriaEditor({ srId, labels, hasScreeningData, reloadK const importYaml = async (file: File | null) => { if (!file) return - const response = await authenticatedFetch(`${endpoint}&operation=import-yaml`, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ criteria_yaml: await file.text() }), - }) + const response = await authenticatedFetch( + `${endpoint}&operation=import-yaml`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ criteria_yaml: await file.text() }), + }, + ) const data = await response.json().catch(() => ({})) - if (!response.ok) { setStatus(backendDiagnostics(data?.detail)[0]?.message || labels.importFailed); return } + if (!response.ok) { + setStatus( + backendDiagnostics(data?.detail)[0]?.message || labels.importFailed, + ) + return + } setImportPreview(data as CriteriaImportPreview) setStatus(labels.importReady) } const acceptImport = () => { if (!importPreview) return - dispatch({ type: 'replace', criteria: importPreview.criteria, revision: state.revision }) - dispatch({ type: 'set-citation-fields', value: importPreview.criteria.citation_fields.l1_include }) + dispatch({ + type: 'replace', + criteria: importPreview.criteria, + revision: state.revision, + }) + dispatch({ + type: 'set-citation-fields', + value: importPreview.criteria.citation_fields.l1_include, + }) setMigrationFingerprint(importPreview.fingerprint || null) - setStatus(importPreview.requires_confirmation ? labels.migrationWarning : labels.imported) + setStatus( + importPreview.requires_confirmation + ? labels.migrationWarning + : labels.imported, + ) setImportPreview(null) } const downloadYaml = async () => { const response = await authenticatedFetch(`${endpoint}&download=yaml`) - if (!response.ok) { setStatus(labels.downloadFailed); return } + if (!response.ok) { + setStatus(labels.downloadFailed) + return + } const url = URL.createObjectURL(await response.blob()) const anchor = document.createElement('a') - anchor.href = url; anchor.download = 'criteria.yaml'; anchor.click() + anchor.href = url + anchor.download = 'criteria.yaml' + anchor.click() URL.revokeObjectURL(url) } return (
-

{labels.builderTitle}

{labels.builderDescription}

- - - - +
+

+ {labels.builderTitle} +

+

{labels.builderDescription}

+
+ + + + +
+
+ {status} + {state.dirty ? ` · ${labels.unsaved}` : ''}
-
{status}{state.dirty ? ` · ${labels.unsaved}` : ''}
-
- {diagnostics.length ? labels.validationErrors.replace('{count}', String(diagnostics.length)) : labels.validationPassed} - {diagnostics.length ?
    {diagnostics.map((item, index) =>
    • )}
: null} + {validationState !== 'idle' ? ( +
+ + {displayedDiagnostics.length + ? labels.validationErrors.replace( + '{count}', + String(displayedDiagnostics.length), + ) + : labels.validationPassed} + + {displayedDiagnostics.length ? ( +
    + {displayedDiagnostics.map((item, index) => ( +
  • + +
    • + ))} +
+ ) : null} +
+ ) : null} +
+
-
- { setImportPreview(null); setStatus(labels.importCancelled) }} onAccept={acceptImport} /> - setRecoveryMode(null)} onExport={() => downloadCriteriaDraft(state.criteria)} onReload={() => { setRecoveryMode(null); void load() }} /> + { + setImportPreview(null) + setStatus(labels.importCancelled) + }} + onAccept={acceptImport} + /> + setRecoveryMode(null)} + onExport={() => downloadCriteriaDraft(state.criteria)} + onReload={() => { + setRecoveryMode(null) + void load() + }} + /> {labels.confirmSaveTitle} - {labels.confirmSaveDescription} + + {labels.confirmSaveDescription} + {labels.cancel} - + diff --git a/frontend/components/can-sr/setup/manage-users-popup.tsx b/frontend/components/can-sr/setup/manage-users-popup.tsx index 58210d69..b445250b 100644 --- a/frontend/components/can-sr/setup/manage-users-popup.tsx +++ b/frontend/components/can-sr/setup/manage-users-popup.tsx @@ -2,6 +2,17 @@ import React, { useEffect, useState, useRef } from 'react' import { useDictionary } from '@/app/[lang]/DictionaryProvider' +import { getCurrentUser } from '@/lib/auth' +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog' type Props = { open: boolean @@ -11,23 +22,64 @@ type Props = { authHeaders?: Record } +type Member = { member_id: string; role: 'owner' | 'member' } + function isValidEmail(email: string) { // simple RFC-ish validation return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email.trim()) } -export default function ManageUsersPopup({ open, onClose, srId, initialEmails = [], authHeaders }: Props) { - const [emails, setEmails] = useState([]) +export default function ManageUsersPopup({ + open, + onClose, + srId, + initialEmails = [], + authHeaders, +}: Props) { + const [members, setMembers] = useState([]) const [input, setInput] = useState('') const [loadingEmail, setLoadingEmail] = useState(null) const [error, setError] = useState(null) + const [currentUserEmail, setCurrentUserEmail] = useState(null) + const [membersLoaded, setMembersLoaded] = useState(false) + const [roleChangeMember, setRoleChangeMember] = useState(null) const inputRef = useRef(null) const dict = useDictionary() useEffect(() => { - setEmails(Array.isArray(initialEmails) ? initialEmails.slice() : []) + setMembers( + Array.isArray(initialEmails) + ? initialEmails.map((member_id) => ({ member_id, role: 'member' })) + : [], + ) }, [initialEmails]) + useEffect(() => { + if (!open || !srId) return + setMembersLoaded(false) + fetch(`/api/can-sr/reviews/users?sr_id=${encodeURIComponent(srId)}`, { + headers: authHeaders, + }) + .then(async (res) => ({ res, data: await res.json().catch(() => ({})) })) + .then(({ res, data }) => { + if (res.ok) setMembers(data.members || []) + else setError(data.error || data.detail || 'Failed to load members') + }) + .catch(() => setError('Failed to load members')) + .finally(() => setMembersLoaded(true)) + }, [open, srId, authHeaders]) + + useEffect(() => { + if (!open) return + let active = true + getCurrentUser().then((user) => { + if (active) setCurrentUserEmail(user?.email?.trim().toLowerCase() || null) + }) + return () => { + active = false + } + }, [open]) + useEffect(() => { if (open) { setTimeout(() => inputRef.current?.focus(), 50) @@ -36,26 +88,36 @@ export default function ManageUsersPopup({ open, onClose, srId, initialEmails = if (!open) return null + const ownerCount = members.filter((member) => member.role === 'owner').length + const currentUserRole = members.find( + (member) => member.member_id.trim().toLowerCase() === currentUserEmail, + )?.role + const canManage = membersLoaded && currentUserRole === 'owner' + const addEmailLocal = (emailRaw: string) => { + if (!canManage) return const email = emailRaw.trim().toLowerCase() if (!email) return if (!isValidEmail(email)) { setError(dict.users.invalidEmail) return } - if (emails.includes(email)) { + if (members.some((member) => member.member_id === email)) { setError(dict.users.emailExists) return } setError(null) - setEmails((prev) => [...prev, email]) + setMembers((prev) => [...prev, { member_id: email, role: 'member' }]) setInput('') addEmailRemote(email) } - const removeEmailLocal = (email: string) => { - setEmails((prev) => prev.filter((e) => e !== email)) - removeEmailRemote(email) + const removeEmailLocal = (member: Member) => { + if (!canManage || (member.role === 'owner' && ownerCount <= 1)) return + setMembers((prev) => + prev.filter((item) => item.member_id !== member.member_id), + ) + removeEmailRemote(member) } async function addEmailRemote(email: string) { @@ -65,52 +127,81 @@ export default function ManageUsersPopup({ open, onClose, srId, initialEmails = } setLoadingEmail(email) try { - const res = await fetch(`/api/can-sr/reviews/users?action=add&sr_id=${encodeURIComponent(srId)}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', ...(authHeaders || {}) }, - body: JSON.stringify({ user_email: email }), - }) + const res = await fetch( + `/api/can-sr/reviews/users?action=add&sr_id=${encodeURIComponent(srId)}`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(authHeaders || {}), + }, + body: JSON.stringify({ user_email: email }), + }, + ) const data = await res.json().catch(() => ({})) if (!res.ok) { - setError((data && (data.error || data.detail)) || `Failed to add user (${res.status})`) + setError( + (data && (data.error || data.detail)) || + `Failed to add user (${res.status})`, + ) // revert local add if server failed - setEmails((prev) => prev.filter((e) => e !== email)) + setMembers((prev) => + prev.filter((member) => member.member_id !== email), + ) } else { setError(null) } } catch (err: any) { console.error('Add user error', err) setError(err?.message || 'Network error while adding user') - setEmails((prev) => prev.filter((e) => e !== email)) + setMembers((prev) => prev.filter((member) => member.member_id !== email)) } finally { setLoadingEmail(null) } } - async function removeEmailRemote(email: string) { + async function removeEmailRemote(member: Member) { + const email = member.member_id if (!srId) { setError(dict.users.missingReviewId) return } setLoadingEmail(email) try { - const res = await fetch(`/api/can-sr/reviews/users?action=remove&sr_id=${encodeURIComponent(srId)}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', ...(authHeaders || {}) }, - body: JSON.stringify({ user_email: email }), - }) + const res = await fetch( + `/api/can-sr/reviews/users?action=remove&sr_id=${encodeURIComponent(srId)}`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(authHeaders || {}), + }, + body: JSON.stringify({ user_email: email }), + }, + ) const data = await res.json().catch(() => ({})) if (!res.ok) { - setError((data && (data.error || data.detail)) || `Failed to remove user (${res.status})`) + setError( + (data && (data.error || data.detail)) || + `Failed to remove user (${res.status})`, + ) // revert local remove if server failed - setEmails((prev) => (prev.includes(email) ? prev : [...prev, email])) + setMembers((prev) => + prev.some((item) => item.member_id === email) + ? prev + : [...prev, member], + ) } else { setError(null) } } catch (err: any) { console.error('Remove user error', err) setError(err?.message || 'Network error while removing user') - setEmails((prev) => (prev.includes(email) ? prev : [...prev, email])) + setMembers((prev) => + prev.some((item) => item.member_id === email) + ? prev + : [...prev, member], + ) } finally { setLoadingEmail(null) } @@ -120,49 +211,137 @@ export default function ManageUsersPopup({ open, onClose, srId, initialEmails = if (e.key === 'Enter' || e.key === ',' || e.key === 'Tab') { e.preventDefault() if (input.trim()) addEmailLocal(input) - } else if (e.key === 'Backspace' && !input && emails.length) { + } else if (canManage && e.key === 'Backspace' && !input && members.length) { // remove last - removeEmailLocal(emails[emails.length - 1]) + removeEmailLocal(members[members.length - 1]) + } + } + + async function confirmRoleChange() { + const member = roleChangeMember + if ( + !srId || + !member || + !canManage || + (member.role === 'owner' && ownerCount <= 1) + ) + return + const role = member.role === 'owner' ? 'member' : 'owner' + setLoadingEmail(member.member_id) + try { + const res = await fetch( + `/api/can-sr/reviews/users?sr_id=${encodeURIComponent(srId)}&member_id=${encodeURIComponent(member.member_id)}`, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + ...(authHeaders || {}), + }, + body: JSON.stringify({ member_id: member.member_id, role }), + }, + ) + const data = await res.json().catch(() => ({})) + if (!res.ok) + throw new Error(data.error || data.detail || 'Failed to change role') + setMembers((prev) => + prev.map((item) => + item.member_id === member.member_id ? { ...item, role } : item, + ), + ) + setRoleChangeMember(null) + } catch (err: any) { + setError(err.message || 'Failed to change role') + } finally { + setLoadingEmail(null) } } const handlePaste = (e: React.ClipboardEvent) => { const pasted = e.clipboardData.getData('text') || '' - const parts = pasted.split(/[\s,;]+/).map((p) => p.trim()).filter(Boolean) + const parts = pasted + .split(/[\s,;]+/) + .map((p) => p.trim()) + .filter(Boolean) if (parts.length > 1) { e.preventDefault() - parts.forEach((p) => addEmailLocal(p)) + if (canManage) parts.forEach((p) => addEmailLocal(p)) } } return ( -
+
-

{dict.users.manageTitle}

-

{dict.users.manageDesc}

+

+ {dict.users.manageTitle} +

+

+ {dict.users.manageDesc} +

- +
- {emails.map((e) => ( + {members.map((member) => (
- {e} + {member.member_id} +
- {error ?
{error}
: null} - {loadingEmail ?
{dict.users.processing} {loadingEmail}...
: null} + {error ? ( +
{error}
+ ) : null} + {membersLoaded && currentUserEmail && !canManage ? ( +
+ Only review owners can manage members. +
+ ) : null} + {loadingEmail ? ( +
+ {dict.users.processing} {loadingEmail}... +
+ ) : null}
@@ -207,6 +400,42 @@ export default function ManageUsersPopup({ open, onClose, srId, initialEmails =
+ { + if (!open && !loadingEmail) setRoleChangeMember(null) + }} + > + + + + {roleChangeMember?.role === 'owner' + ? 'Demote review owner?' + : 'Promote review member?'} + + + {roleChangeMember?.role === 'owner' + ? `${roleChangeMember.member_id} will remain a review member but will no longer be able to manage owners or owner-only actions.` + : `${roleChangeMember?.member_id} will be able to manage owners and perform owner-only actions.`} + + + + + Cancel + + + {loadingEmail + ? 'Saving…' + : roleChangeMember?.role === 'owner' + ? 'Demote' + : 'Promote'} + + + +
) } diff --git a/frontend/components/can-sr/setup/parameter-builder.tsx b/frontend/components/can-sr/setup/parameter-builder.tsx index fd719126..e7009f6c 100644 --- a/frontend/components/can-sr/setup/parameter-builder.tsx +++ b/frontend/components/can-sr/setup/parameter-builder.tsx @@ -1,6 +1,7 @@ 'use client' import { useEffect, useRef, useState, type Dispatch } from 'react' +import { authenticatedFetch } from '@/lib/auth' import { ArrowDown, ArrowUp, ChevronDown, ChevronRight, ChevronUp, ChevronsDown, Pencil, Plus, Trash2 } from 'lucide-react' import type { CriteriaDraftAction, CriteriaDraftState, Parameter, ParameterOption, ScreeningQuestion } from './criteria-types' import type { CriteriaDiagnostic } from './criteria-validation' @@ -15,7 +16,8 @@ const questionSource = (stage: 'l1' | 'l2', question: ScreeningQuestion): Trigge options: question.answers.map(({ id, label, context }) => ({ id, label, context })), }) -function ParameterCard({ parameter, index, count, sources, dependants, referencedOptionIds, dispatch, labels, diagnostics, collapsed, onToggle, citationFields }: { +function ParameterCard({ srId, parameter, index, count, sources, dependants, referencedOptionIds, dispatch, labels, diagnostics, collapsed, onToggle, citationFields }: { + srId: string parameter: Parameter index: number count: number @@ -32,6 +34,7 @@ function ParameterCard({ parameter, index, count, sources, dependants, reference const prefix = `parameter-${parameter.id}` const [contextOptionId, setContextOptionId] = useState(null) const [detailsOpen, setDetailsOpen] = useState(false) + const [setAnswerStatus, setSetAnswerStatus] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle') const contextOption = parameter.type === 'selection' ? parameter.options.find((option) => option.id === contextOptionId) : undefined const typeChangeBlocked = parameter.type === 'selection' && dependants.length > 0 const update = (field: 'name' | 'description' | 'unit_instructions' | 'calculation', value: string) => dispatch({ type: 'update-parameter', parameterId: parameter.id, field, value }) @@ -69,7 +72,7 @@ function ParameterCard({ parameter, index, count, sources, dependants, reference {parameter.answer_column ? <>

{labels.answerColumnTooltip}

- +
: null}
{typeChangeBlocked ?

{labels.typeChangeBlocked}: {dependants.join(', ')}

: null} @@ -111,7 +114,7 @@ function ParameterCard({ parameter, index, count, sources, dependants, reference } -export default function ParameterBuilder({ state, dispatch, labels, diagnostics = [], citationFields = { fields: [], unavailable_configured_fields: [] } }: { state: CriteriaDraftState; dispatch: Dispatch; labels: Record; diagnostics?: CriteriaDiagnostic[]; citationFields?: CitationFieldContract }) { +export default function ParameterBuilder({ srId, state, dispatch, labels, diagnostics = [], citationFields = { fields: [], unavailable_configured_fields: [] } }: { srId: string; state: CriteriaDraftState; dispatch: Dispatch; labels: Record; diagnostics?: CriteriaDiagnostic[]; citationFields?: CitationFieldContract }) { const [sectionCollapsed, setSectionCollapsed] = useState(false) const [collapsedParameters, setCollapsedParameters] = useState>(() => new Set()) const pendingParameterCount = useRef(null) @@ -158,7 +161,7 @@ export default function ParameterBuilder({ state, dispatch, labels, diagnostics {state.criteria.parameters.map((parameter, index) => { const earlierParameters: TriggerSource[] = state.criteria.parameters.slice(0, index).filter((item) => item.type === 'selection').map((item) => ({ stage: 'parameters', id: item.id, label: item.name, options: item.options })) const dependants = state.criteria.parameters.filter((item) => item.id !== parameter.id && item.trigger.all.some((condition) => condition.source_item_id === parameter.id)).map((item) => itemLabels.get(item.id) || item.id) - return condition.source_item_id === parameter.id).map((condition) => condition.option_id))} dispatch={dispatch} labels={labels} citationFields={citationFields} diagnostics={diagnostics.filter((item) => item.itemId === parameter.id)} collapsed={collapsedParameters.has(parameter.id)} onToggle={() => setCollapsedParameters((current) => { const next = new Set(current); if (next.has(parameter.id)) next.delete(parameter.id); else next.add(parameter.id); return next })} /> + return condition.source_item_id === parameter.id).map((condition) => condition.option_id))} dispatch={dispatch} labels={labels} citationFields={citationFields} diagnostics={diagnostics.filter((item) => item.itemId === parameter.id)} collapsed={collapsedParameters.has(parameter.id)} onToggle={() => setCollapsedParameters((current) => { const next = new Set(current); if (next.has(parameter.id)) next.delete(parameter.id); else next.add(parameter.id); return next })} /> })}
: null} diff --git a/frontend/dictionaries/en.json b/frontend/dictionaries/en.json index e1da11d1..f450f249 100644 --- a/frontend/dictionaries/en.json +++ b/frontend/dictionaries/en.json @@ -122,14 +122,144 @@ "databaseSearchDesc": "Select database and define search criteria", "importReferences": "Import references and criteria", "importReferencesDesc": "Upload citation files, define eligibility criteria and review settings.", - "titleAbstractScreening": "Title and abstract screening", + "referencesWorkspace": "Import Citations", + "referencesWorkspaceDesc": "View and manage references, upload files, or search databases.", + "protocols": "Define Protocol and Prompts", + "protocolsDesc": "Configure citation fields, screening criteria, and extraction parameters.", + "titleAbstractScreening": "Title and Abstract Screening", "titleAbstractScreeningDesc": "Screen titles and abstracts to identify potentially relevant citations.", - "fullTextReview": "Full text review", + "fullTextReview": "Full Text Screening", "fullTextReviewDesc": "Review full text articles and make inclusion/exclusion decisions.", - "extraction": "Extraction", + "extraction": "Parameter Extraction", "extractionDesc": "Extract outcome and study parameters for included studies.", "loadingReview": "Loading review..." }, + "protocols": { + "title": "Protocols", + "heading": "Define Criteria and Parameter Protocols", + "description": "Configure eligibility and screening criteria together with extraction parameter protocols.", + "criteriaSection": "Eligibility and screening criteria", + "noDataset": "No reference dataset is available yet. Add references before selecting citation fields.", + "addReferences": "Add references" + }, + "references": { + "title": "Import References or Search Databases", + "gridTitle": "References workspace", + "empty": "No references have been added to this review yet.", + "datasetReady": "Use Add references to preview and append a CSV or RIS file.", + "gridDeferred": "Citation browsing will appear here. Screening queues remain unchanged.", + "search": "Search references", + "loading": "Loading references…", + "gridFailed": "Unable to load references.", + "noMatches": "No references match this search.", + "results": "{count} references", + "previous": "Previous", + "next": "Next", + "databaseSearch": "Database Search", + "add": "Add references", + "addReferencesTitle": "Add References", + "dialogTitle": "Add References", + "dialogDescription": "Choose a CSV, RIS, or RIS text export. You will be warned about required fields and extra columns before importing.", + "file": "Reference file", + "uploadTab": "Upload file", + "searchTab": "Search databases", + "searchString": "Search string", + "beginSearch": "Begin Search", + "searchCompleted": "Database search completed.", + "searchFailed": "Database search failed.", + "preview": "Preview import", + "commit": "Commit import", + "cancel": "Cancel", + "working": "Working...", + "import": "Import references", + "importFailed": "Unable to import the references.", + "imported": "Import complete: {count} citations added.", + "duplicatesSkipped": "{count} duplicates skipped.", + "duplicatesDetected": "{count} duplicate references detected. They will be merged unless Import Duplicates is selected.", + "includeDuplicates": "Import Duplicates", + "missingTitle": "The file is missing a Title column.", + "missingAbstract": "The file is missing an Abstract column.", + "additionalColumns": "Additional source columns will be preserved as text", + "rows": "Rows detected: {count}", + "mapping": "Proposed mapping", + "unmapped": "unmapped", + "missing": "Missing required fields", + "excluded": "Excluded source columns", + "previewFailed": "Unable to create the import preview.", + "mappingFailed": "Unable to save the import mapping.", + "commitFailed": "Unable to commit the import.", + "committed": "Import committed: {count} citations added.", + "reconciliation": "Reconciliation: {new} new; {existing} exact existing matches.", + "matchesDoNotOverwrite": "Exact matches are linked to this import only; they do not overwrite citations or screening decisions.", + "ambiguousMatches": "{count} rows have ambiguous exact matches and must be resolved before import.", + "ambiguousRows": "Ambiguous source rows: {rows}.", + "excludeAmbiguousRows": "Exclude these ambiguous rows from this import. They will not be merged with or overwrite any citation.", + "ambiguousResolutionList": "Ambiguous row resolution", + "excludeAmbiguousRow": "Exclude source row {row} from this import.", + "ambiguousAcknowledgement": "Ambiguous rows acknowledged for exclusion: {acknowledged} of {total}.", + "excludedAmbiguousRows": "{count} ambiguous rows will be excluded from this import.", + "invalidReconciliationRows": "{count} rows are invalid and cannot be imported.", + "sortBy": "Sort by {column}", + "filterBy": "Filter by {column}", + "selectAll": "Select all citations on this page", + "selectCitation": "Select citation {id}", + "deleteSelected": "Delete {count} selected citations", + "deleteFailed": "Unable to delete citations.", + "columns": "Columns", + "columnsTitle": "Choose visible columns", + "columnsDescription": "Select and order the citation fields shown in your workspace. The identifier remains visible.", + "saveColumns": "Save columns", + "moveColumnUp": "Move {column} up", + "moveColumnDown": "Move {column} down", + "deleting": "Deleting…", + "duplicateGroupReview": "Duplicate group review", + "groupSummary": "Group {current} of {total} · {status} · {count} records", + "restore": "Restore", + "minimize": "Minimize", + "record": "Record {id}", + "suggestedSurvivor": "Suggested survivor", + "selectSurvivor": "Select survivor", + "reviewDecision": "Review decision:", + "confirmDuplicate": "Confirm duplicate", + "keepBoth": "Keep both", + "defer": "Defer", + "deleteCitation": "Delete {id}", + "survivor": "Survivor", + "chooseSelection": "Choose citation selection", + "none": "None", + "all": "All", + "suggestedDuplicates": "Suggested duplicates", + "filterDuplicateStatus": "Filter duplicate status", + "showDuplicateStatusFilter": "Show duplicate status filter", + "filterDuplicateStatuses": "Filter duplicate statuses", + "duplicateStatus": "Duplicate status", + "showReferencesMatching": "Show references matching", + "allStatuses": "All statuses", + "exactDuplicates": "Exact duplicates", + "possibleDuplicates": "Possible duplicates", + "noDuplicateMatch": "No duplicate match", + "configureDuplicateFields": "Configure duplicate matching fields", + "runningDuplicateCalculation": "Running duplicate calculation", + "runDuplicateCalculation": "Run duplicate calculation", + "rerunDuplicateCalculation": "Rerun duplicate calculation", + "duplicateMatchingFields": "Duplicate matching fields", + "duplicateFieldsDescription": "Visible fields are added automatically. Hidden fields remain configured.", + "clearFilter": "Clear", + "applyFilter": "Apply", + "resizeColumn": "Resize column", + "changeFile": "Change file", + "chooseFile": "Choose file", + "removeFile": "Remove", + "searchDescription": "Search PubMed, Scopus, and Europe PMC using a database-specific search string." + ,"selectedFile": "Selected file:", + "exactDuplicateMatch": "Exact duplicate match", + "possibleDuplicateMatch": "Possible duplicate match", + "keptBothReviewGroup": "Kept both; review duplicate group", + "openDuplicateReview": "Open duplicate group review for citation {id}", + "reviewKeptBothGroup": "Review kept-both group", + "openDuplicateGroupReview": "Open duplicate group review", + "row": "Row {number}" + }, "setup": { "title": "Import & Setup", "pageTitle": "Import references and define criteria", @@ -162,14 +292,22 @@ "noSelectedFields": "No title/abstract fields configured.", "addField": "Add field", "noCitationFields": "Upload a CSV or RIS file to discover fields.", - "unavailableField": "Not found in the current upload", + "unavailableField": "Unavailable field", "removeField": "Remove field", "noDoiField": "No DOI field", "l1": "Title and abstract screening (L1)", "l2": "Full-text screening (L2)", "question": "Question", "questionText": "Question text", - "answerContext": "Answer context", "answerColumn": "Answer column", "addAnswerColumn": "Add Answer Column", "enableRetrospectiveValidation": "Enable retrospective validation", "answerColumnTooltip": "Match an existing human answer from an imported file when one exists. Valid matches are authoritative; blank or unmatched values are warned about.", "noAnswerColumn": "No answer column", "unavailableField": "Unavailable field", + "answerContext": "Answer context", + "answerColumn": "Answer column", + "addAnswerColumn": "Add Answer Column", + "enableRetrospectiveValidation": "Enable retrospective validation", + "answerColumnTooltip": "Match an existing human answer from an imported file when one exists. Valid matches are authoritative; blank or unmatched values are warned about.", + "noAnswerColumn": "No answer column", + "setAnswer": "Set answers", + "answerSet": "Answer set", + "setAnswerFailed": "Failed to set answer", "answers": "Answers", "answerLabel": "Answer label", "decision": "Decision", diff --git a/frontend/dictionaries/fr.json b/frontend/dictionaries/fr.json index 2a03c495..3f780aa9 100644 --- a/frontend/dictionaries/fr.json +++ b/frontend/dictionaries/fr.json @@ -122,6 +122,10 @@ "databaseSearchDesc": "Sélectionnez une base de données et définissez les critères de recherche", "importReferences": "Importer les références et les critères", "importReferencesDesc": "Téléversez les fichiers de citations, définissez les critères d'admissibilité et les paramètres de la revue.", + "referencesWorkspace": "Importer des références ou rechercher dans les bases de données", + "referencesWorkspaceDesc": "Consultez et gérez les références, téléversez des fichiers ou recherchez dans les bases de données.", + "protocols": "Définir les protocoles de critères et de paramètres", + "protocolsDesc": "Configurez les champs de citation, les critères de sélection et les paramètres d’extraction.", "titleAbstractScreening": "Sélection des titres et résumés", "titleAbstractScreeningDesc": "Sélectionnez les titres et résumés pour identifier les citations potentiellement pertinentes.", "fullTextReview": "Examen du texte intégral", @@ -130,6 +134,132 @@ "extractionDesc": "Extrayez les résultats et les paramètres d'étude pour les études incluses.", "loadingReview": "Chargement de la revue..." }, + "protocols": { + "title": "Protocoles", + "heading": "Définir les protocoles de critères et de paramètres", + "description": "Configurez les critères d’admissibilité et de sélection ainsi que les protocoles de paramètres d’extraction.", + "criteriaSection": "Critères d’admissibilité et de sélection", + "noDataset": "Aucun jeu de références n’est encore disponible. Ajoutez des références avant de sélectionner les champs de citation.", + "addReferences": "Ajouter des références" + }, + "references": { + "title": "Importer des références ou rechercher dans les bases de données", + "gridTitle": "Espace de travail des références", + "empty": "Aucune référence n’a encore été ajoutée à cette revue.", + "datasetReady": "Utilisez Ajouter des références pour prévisualiser et ajouter un fichier CSV ou RIS.", + "gridDeferred": "La consultation des citations apparaîtra ici. Les files de sélection ne changent pas.", + "search": "Rechercher des références", + "loading": "Chargement des références…", + "gridFailed": "Impossible de charger les références.", + "noMatches": "Aucune référence ne correspond à cette recherche.", + "results": "{count} références", + "previous": "Précédent", + "next": "Suivant", + "databaseSearch": "Recherche dans la base de données", + "add": "Ajouter des références", + "addReferencesTitle": "Ajouter des références", + "dialogTitle": "Ajouter des références", + "dialogDescription": "Choisissez un export CSV, RIS ou texte RIS. Les champs obligatoires et les colonnes supplémentaires seront signalés avant l’importation.", + "file": "Fichier de références", + "uploadTab": "Téléverser un fichier", + "searchTab": "Rechercher dans les bases", + "searchString": "Chaîne de recherche", + "beginSearch": "Lancer la recherche", + "searchCompleted": "Recherche dans la base terminée.", + "searchFailed": "Échec de la recherche dans la base.", + "preview": "Prévisualiser l’importation", + "commit": "Valider l’importation", + "cancel": "Annuler", + "working": "Traitement en cours...", + "import": "Importer les références", + "importFailed": "Impossible d’importer les références.", + "imported": "Importation terminée : {count} citations ajoutées.", + "duplicatesSkipped": "{count} doublons ignorés.", + "duplicatesDetected": "{count} références en double détectées. Elles seront fusionnées sauf si vous sélectionnez Importer les doublons.", + "includeDuplicates": "Importer les doublons", + "missingTitle": "Le fichier ne contient pas de colonne Titre.", + "missingAbstract": "Le fichier ne contient pas de colonne Résumé.", + "additionalColumns": "Les colonnes source supplémentaires seront conservées comme texte", + "rows": "Lignes détectées : {count}", + "mapping": "Mappage proposé", + "unmapped": "non mappé", + "missing": "Champs obligatoires manquants", + "excluded": "Colonnes source exclues", + "previewFailed": "Impossible de créer la prévisualisation de l’importation.", + "mappingFailed": "Impossible d’enregistrer le mappage de l’importation.", + "commitFailed": "Impossible de valider l’importation.", + "committed": "Importation validée : {count} citations ajoutées.", + "reconciliation": "Rapprochement : {new} nouvelles; {existing} correspondances exactes existantes.", + "matchesDoNotOverwrite": "Les correspondances exactes sont uniquement liées à cette importation; elles n’écrasent ni les citations ni les décisions de sélection.", + "ambiguousMatches": "{count} lignes ont des correspondances exactes ambiguës et doivent être résolues avant l’importation.", + "ambiguousRows": "Lignes source ambiguës : {rows}.", + "excludeAmbiguousRows": "Exclure ces lignes ambiguës de cette importation. Elles ne seront fusionnées avec aucune citation et n’écraseront aucune citation.", + "ambiguousResolutionList": "Résolution des lignes ambiguës", + "excludeAmbiguousRow": "Exclure la ligne source {row} de cette importation.", + "ambiguousAcknowledgement": "Lignes ambiguës reconnues pour exclusion : {acknowledged} sur {total}.", + "excludedAmbiguousRows": "{count} lignes ambiguës seront exclues de cette importation.", + "invalidReconciliationRows": "{count} lignes sont invalides et ne peuvent pas être importées.", + "sortBy": "Trier par {column}", + "filterBy": "Filtrer par {column}", + "selectAll": "Sélectionner les citations de cette page", + "selectCitation": "Sélectionner la citation {id}", + "deleteSelected": "Supprimer les {count} citations sélectionnées", + "deleteFailed": "Impossible de supprimer les citations.", + "columns": "Colonnes", + "columnsTitle": "Choisir les colonnes visibles", + "columnsDescription": "Sélectionnez et ordonnez les champs de citation affichés dans votre espace de travail. L’identifiant demeure visible.", + "saveColumns": "Enregistrer les colonnes", + "moveColumnUp": "Déplacer {column} vers le haut", + "moveColumnDown": "Déplacer {column} vers le bas", + "deleting": "Suppression…", + "duplicateGroupReview": "Révision du groupe de doublons", + "groupSummary": "Groupe {current} sur {total} · {status} · {count} enregistrements", + "restore": "Restaurer", + "minimize": "Réduire", + "record": "Enregistrement {id}", + "suggestedSurvivor": "Doublon à conserver suggéré", + "selectSurvivor": "Sélectionner l’enregistrement à conserver", + "reviewDecision": "Décision de révision :", + "confirmDuplicate": "Confirmer le doublon", + "keepBoth": "Conserver les deux", + "defer": "Reporter", + "deleteCitation": "Supprimer {id}", + "survivor": "Enregistrement conservé", + "chooseSelection": "Choisir la sélection de citations", + "none": "Aucune", + "all": "Toutes", + "suggestedDuplicates": "Doublons suggérés", + "filterDuplicateStatus": "Filtrer le statut de doublon", + "showDuplicateStatusFilter": "Afficher le filtre de statut des doublons", + "filterDuplicateStatuses": "Filtrer les statuts de doublon", + "duplicateStatus": "Statut du doublon", + "showReferencesMatching": "Afficher les références correspondant à", + "allStatuses": "Tous les statuts", + "exactDuplicates": "Doublons exacts", + "possibleDuplicates": "Doublons possibles", + "noDuplicateMatch": "Aucune correspondance de doublon", + "configureDuplicateFields": "Configurer les champs de correspondance des doublons", + "runningDuplicateCalculation": "Calcul des doublons en cours", + "runDuplicateCalculation": "Calculer les doublons", + "rerunDuplicateCalculation": "Recalculer les doublons", + "duplicateMatchingFields": "Champs de correspondance des doublons", + "duplicateFieldsDescription": "Les champs visibles sont ajoutés automatiquement. Les champs masqués restent configurés.", + "clearFilter": "Effacer", + "applyFilter": "Appliquer", + "resizeColumn": "Redimensionner la colonne", + "changeFile": "Changer de fichier", + "chooseFile": "Choisir un fichier", + "removeFile": "Supprimer", + "searchDescription": "Recherchez dans PubMed, Scopus et Europe PMC à l’aide d’une chaîne de recherche propre à chaque base." + ,"selectedFile": "Fichier sélectionné :", + "exactDuplicateMatch": "Correspondance exacte de doublon", + "possibleDuplicateMatch": "Correspondance possible de doublon", + "keptBothReviewGroup": "Deux enregistrements conservés; réviser le groupe de doublons", + "openDuplicateReview": "Ouvrir la révision du groupe de doublons pour la citation {id}", + "reviewKeptBothGroup": "Réviser le groupe conservé en double", + "openDuplicateGroupReview": "Ouvrir la révision du groupe de doublons", + "row": "Ligne {number}" + }, "setup": { "title": "Importation et configuration", "pageTitle": "Importer les références et définir les critères", @@ -162,14 +292,22 @@ "noSelectedFields": "Aucun champ de titre/résumé configuré.", "addField": "Ajouter un champ", "noCitationFields": "Téléversez un fichier CSV ou RIS pour découvrir les champs.", - "unavailableField": "Introuvable dans le téléversement actuel", + "unavailableField": "Colonne indisponible", "removeField": "Supprimer le champ", "noDoiField": "Aucun champ DOI", "l1": "Sélection titre et résumé (L1)", "l2": "Sélection du texte intégral (L2)", "question": "Question", "questionText": "Texte de la question", - "answerContext": "Contexte de réponse", "answerColumn": "Colonne de réponse", "addAnswerColumn": "Ajouter une colonne de réponse", "enableRetrospectiveValidation": "Activer la validation rétrospective", "answerColumnTooltip": "Associer une réponse humaine existante provenant d’un fichier importé. Les correspondances valides sont prioritaires; les valeurs vides ou non correspondantes sont signalées.", "noAnswerColumn": "Aucune colonne de réponse", "unavailableField": "Colonne indisponible", + "answerContext": "Contexte de réponse", + "answerColumn": "Colonne de réponse", + "addAnswerColumn": "Ajouter une colonne de réponse", + "enableRetrospectiveValidation": "Activer la validation rétrospective", + "answerColumnTooltip": "Associer une réponse humaine existante provenant d’un fichier importé. Les correspondances valides sont prioritaires; les valeurs vides ou non correspondantes sont signalées.", + "noAnswerColumn": "Aucune colonne de réponse", + "setAnswer": "Définir la réponse", + "answerSet": "Réponse définie", + "setAnswerFailed": "Échec de la définition", "answers": "Réponses", "answerLabel": "Libellé de réponse", "decision": "Décision", diff --git a/frontend/lib/auth.ts b/frontend/lib/auth.ts index 4d246b7d..b9363e87 100644 --- a/frontend/lib/auth.ts +++ b/frontend/lib/auth.ts @@ -54,14 +54,17 @@ export const authenticatedFetch = async ( const token = getAuthToken() const tokenType = getTokenType() - const headers: HeadersInit = { - 'Content-Type': 'application/json', - ...options.headers, - } + const headers = new Headers(options.headers) + // Let fetch generate the multipart boundary for FormData. Setting an + // application/json content type here makes request.formData() fail in + // browser-facing upload proxies. + const isFormData = + typeof FormData !== 'undefined' && options.body instanceof FormData + if (!isFormData && !headers.has('Content-Type')) + headers.set('Content-Type', 'application/json') if (token) { - ;(headers as Record)['Authorization'] = - `${tokenType} ${token}` + headers.set('Authorization', `${tokenType} ${token}`) } return fetch(url, { diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts index 01beb58c..98bddd67 100644 --- a/frontend/vitest.config.ts +++ b/frontend/vitest.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ test: { environment: 'jsdom', setupFiles: ['./vitest.setup.ts'], - include: ['components/**/*.test.tsx'], + include: ['components/**/*.test.tsx', 'app/**/*.test.ts'], restoreMocks: true, }, -}) \ No newline at end of file +})