From ebf5ef3c0847720b5caf41e9c73ac6447da58d34 Mon Sep 17 00:00:00 2001 From: bh809 Date: Fri, 13 Nov 2026 20:37:44 -0600 Subject: [PATCH] Report: flag where a case's own sources disagree Add a `consistency` section to the report payload and a closed "Source consistency" section in the report view. It compares sources the report already holds: a volume stated in the stored source report vs. the measured volume, a whole structure vs. the sum of its parts, and a sex-specific structure vs. the recorded sex. Discrepancies ask a person to compare sources; they make no clinical claim (docs/report-evidence.md). Display text is built from typed fields only. Tests use synthetic data. --- .github/workflows/ci.yml | 2 +- .../ReportScreen/SourceConsistencyChecks.tsx | 31 +++ .../ReportScreen/SourceReportDetails.tsx | 2 + PanTS-Demo/src/helpers/reportConsistency.ts | 119 +++++++++++ PanTS-Demo/src/helpers/reportFindings.ts | 5 + .../src/test/sourceConsistency.test.tsx | 97 +++++++++ docs/report-evidence.md | 14 +- flask-server/services/report_consistency.py | 196 ++++++++++++++++++ flask-server/services/report_evidence.py | 14 ++ .../tests/unit/test_report_consistency.py | 135 ++++++++++++ 10 files changed, 613 insertions(+), 2 deletions(-) create mode 100644 PanTS-Demo/src/components/ReportScreen/SourceConsistencyChecks.tsx create mode 100644 PanTS-Demo/src/helpers/reportConsistency.ts create mode 100644 PanTS-Demo/src/test/sourceConsistency.test.tsx create mode 100644 flask-server/services/report_consistency.py create mode 100644 flask-server/tests/unit/test_report_consistency.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3295f50d..fb49246c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: run: python -m pytest tests/unit/test_user_mri_dataset.py tests/unit/test_user_collection_hook.py -v - name: Automatic report evidence and source preservation - run: python -m pytest tests/unit/test_report_evidence.py tests/unit/test_report_measurements.py tests/unit/test_report_measurement_cache.py tests/unit/test_quiz_report_questions.py -v + run: python -m pytest tests/unit/test_report_evidence.py tests/unit/test_report_consistency.py tests/unit/test_report_measurements.py tests/unit/test_report_measurement_cache.py tests/unit/test_quiz_report_questions.py -v - name: Path-safety unit tests run: python -m pytest tests/unit/test_path_safety.py -v diff --git a/PanTS-Demo/src/components/ReportScreen/SourceConsistencyChecks.tsx b/PanTS-Demo/src/components/ReportScreen/SourceConsistencyChecks.tsx new file mode 100644 index 00000000..09a8951a --- /dev/null +++ b/PanTS-Demo/src/components/ReportScreen/SourceConsistencyChecks.tsx @@ -0,0 +1,31 @@ +import { CONSISTENCY_EXPLANATION, getConsistencyItems } from '../../helpers/reportConsistency'; +import type { ReportData } from '../../helpers/reportFindings'; + +// Where the report's own sources disagree. A discrepancy asks a person to compare +// sources; it is never a finding (docs/report-evidence.md, "Source consistency checks"). +export default function SourceConsistencyChecks({ data, dark = false }: { data: ReportData; dark?: boolean }) { + const items = getConsistencyItems(data.consistency); + if (items.length === 0) return null; + const discrepancies = items.filter(item => item.status === 'discrepancy'); + const consistent = items.length - discrepancies.length; + const border = dark ? 'rgba(255,255,255,0.16)' : '#d9e0eb'; + const summary = discrepancies.length + ? `Source consistency: ${discrepancies.length} ${discrepancies.length === 1 ? 'discrepancy' : 'discrepancies'} to review` + : `Source consistency: no discrepancies in ${items.length} ${items.length === 1 ? 'check' : 'checks'}`; + return ( +
+ {summary} +

{CONSISTENCY_EXPLANATION}

+ {discrepancies.length > 0 && ( + + )} + {consistent > 0 && ( +

+ {consistent} other {consistent === 1 ? 'check agrees' : 'checks agree'} across sources. +

+ )} +
+ ); +} diff --git a/PanTS-Demo/src/components/ReportScreen/SourceReportDetails.tsx b/PanTS-Demo/src/components/ReportScreen/SourceReportDetails.tsx index f148b2b9..58322c0e 100644 --- a/PanTS-Demo/src/components/ReportScreen/SourceReportDetails.tsx +++ b/PanTS-Demo/src/components/ReportScreen/SourceReportDetails.tsx @@ -1,6 +1,7 @@ import { useState } from 'react'; import { getCoverageLimitations, getSourceReportText, REPORT_ASSESSMENT_SCOPE } from '../../helpers/reportFindings'; import type { ReportData } from '../../helpers/reportFindings'; +import SourceConsistencyChecks from './SourceConsistencyChecks'; export default function SourceReportDetails({ data, dark = false }: { data: ReportData; dark?: boolean }) { const sourceText = getSourceReportText(data); @@ -27,6 +28,7 @@ export default function SourceReportDetails({ data, dark = false }: { data: Repo {getCoverageLimitations(data).map(limitation =>
  • {limitation}
    • )} + {sourceText ? (
    { event.preventDefault(); setExpandedReference(referenceOpen ? null : data); }} style={{ cursor: 'pointer', fontSize: 16, fontWeight: 700 }}> diff --git a/PanTS-Demo/src/helpers/reportConsistency.ts b/PanTS-Demo/src/helpers/reportConsistency.ts new file mode 100644 index 00000000..34372917 --- /dev/null +++ b/PanTS-Demo/src/helpers/reportConsistency.ts @@ -0,0 +1,119 @@ +// Source consistency checks (flask-server/services/report_consistency.py). +// Display text is built here from typed fields only; incoming prose is never +// shown, matching how coverage limitations are handled (docs/report-evidence.md). + +export type ConsistencyKind = 'source_volume' | 'parts_whole' | 'recorded_sex'; +export type ConsistencyStatus = 'discrepancy' | 'consistent'; + +export interface ConsistencyCheck { + kind: ConsistencyKind; + status: ConsistencyStatus; + structures: string[]; + measured_volume?: number | null; + source_volume?: number | null; + parts_volume?: number | null; + whole_volume?: number | null; + difference_percent?: number | null; + recorded_sex?: string | null; +} + +export interface ReportConsistency { + schema_version?: string; + checks: ConsistencyCheck[]; + discrepancy_count?: number; +} + +export interface ConsistencyItem { + kind: ConsistencyKind; + status: ConsistencyStatus; + text: string; +} + +export const CONSISTENCY_EXPLANATION = + 'These checks compare the sources this report already holds: segmentation measurements, the stored source ' + + 'reference, and case metadata. A discrepancy means a person should compare the sources. It does not indicate ' + + 'disease, and it does not say which source is correct.'; + +const KINDS: ConsistencyKind[] = ['source_volume', 'parts_whole', 'recorded_sex']; +const STRUCTURE = /^[a-z][a-z0-9_]*$/; + +export function structureLabel(name: string): string { + const side = /^(.*)_(left|right)$/.exec(name); + return (side ? `${side[2]} ${side[1]}` : name).replace(/_/g, ' '); +} + +function cc(value: unknown): string | null { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? `${value.toFixed(1)} cc` : null; +} + +function describe(check: ConsistencyCheck): string | null { + const [first, ...rest] = check.structures; + const name = structureLabel(first); + if (check.kind === 'source_volume') { + const measured = cc(check.measured_volume); + const source = cc(check.source_volume); + if (!measured || !source) return null; + return check.status === 'discrepancy' + ? `Measured ${name} volume (${measured}) and the volume stated in the source reference (${source}) do not match.` + : `Measured ${name} volume matches the source reference (${measured} vs. ${source}).`; + } + if (check.kind === 'parts_whole') { + const parts = cc(check.parts_volume); + if (!parts || rest.length === 0) return null; + const partNames = rest.map(structureLabel).join(', '); + const whole = cc(check.whole_volume); + if (check.status === 'discrepancy') { + return whole + ? `The ${name} measures ${whole}, but its parts (${partNames}) measure ${parts} in total.` + : `No ${name} measurement is available, but its parts (${partNames}) measure ${parts} in total.`; + } + return whole ? `The ${name} (${whole}) matches the sum of its parts (${parts}).` : null; + } + const measured = cc(check.measured_volume); + const sex = check.recorded_sex === 'F' ? 'female' : check.recorded_sex === 'M' ? 'male' : null; + if (!measured || !sex) return null; + return check.status === 'discrepancy' + ? `A ${name} mask (${measured}) is present, but the case metadata records sex as ${sex}. The metadata, the structure name, or the mask may be wrong.` + : `The ${name} mask matches the recorded sex.`; +} + +const NUMBER_FIELDS = ['measured_volume', 'source_volume', 'parts_volume', 'whole_volume', 'difference_percent'] as const; + +/** Keep only typed fields from a server payload; prose such as `message` is discarded. */ +export function normalizeConsistency(value: unknown): ReportConsistency | undefined { + const checks = (value as { checks?: unknown } | null | undefined)?.checks; + if (!value || typeof value !== 'object' || !Array.isArray(checks)) return undefined; + const kept: ConsistencyCheck[] = []; + for (const raw of checks) { + if (!raw || typeof raw !== 'object') continue; + const check = raw as Record; + const copy: ConsistencyCheck = { + kind: check.kind as ConsistencyKind, + status: check.status as ConsistencyStatus, + structures: Array.isArray(check.structures) ? check.structures.filter(name => typeof name === 'string') : [], + recorded_sex: typeof check.recorded_sex === 'string' ? check.recorded_sex : null, + }; + for (const field of NUMBER_FIELDS) { + const number = check[field]; + copy[field] = typeof number === 'number' && Number.isFinite(number) ? number : null; + } + kept.push(copy); + } + return { checks: kept }; +} + +/** Validated display items, discrepancies first. Malformed checks are dropped. */ +export function getConsistencyItems(consistency: unknown): ConsistencyItem[] { + const checks = (consistency as ReportConsistency | null | undefined)?.checks; + if (!Array.isArray(checks)) return []; + const items: ConsistencyItem[] = []; + for (const check of checks as ConsistencyCheck[]) { + if (!check || !KINDS.includes(check.kind)) continue; + if (check.status !== 'discrepancy' && check.status !== 'consistent') continue; + if (!Array.isArray(check.structures) || check.structures.length === 0 + || !check.structures.every(name => typeof name === 'string' && STRUCTURE.test(name))) continue; + const text = describe(check); + if (text) items.push({ kind: check.kind, status: check.status, text }); + } + return items.sort((a, b) => Number(b.status === 'discrepancy') - Number(a.status === 'discrepancy')); +} diff --git a/PanTS-Demo/src/helpers/reportFindings.ts b/PanTS-Demo/src/helpers/reportFindings.ts index 86999c08..17f994a6 100644 --- a/PanTS-Demo/src/helpers/reportFindings.ts +++ b/PanTS-Demo/src/helpers/reportFindings.ts @@ -1,5 +1,8 @@ // Report text and segmentation measurements are separate evidence sources. // Neither absent flags nor an in-range measurement establishes clinical normality. +import { normalizeConsistency } from './reportConsistency'; +import type { ReportConsistency } from './reportConsistency'; + export interface OrganData { volume: number | null; mean_hu: number | null; @@ -49,6 +52,7 @@ export interface ReportData { image_review_performed: boolean; }; coverage?: { status: string; measured_structures: number; limitations: string[] }; + consistency?: ReportConsistency; } const DEFAULT_LIMITATIONS = [ @@ -125,6 +129,7 @@ export function normalizeReportData(value: unknown): ReportData | null { status: 'limited', measured_structures: Object.keys(organ_volumes).length, limitations: trustedCoverageLimitations(Object.keys(organ_volumes).length, source.load_failed === true), }, + consistency: normalizeConsistency(value.consistency), }; } diff --git a/PanTS-Demo/src/test/sourceConsistency.test.tsx b/PanTS-Demo/src/test/sourceConsistency.test.tsx new file mode 100644 index 00000000..7f043a0f --- /dev/null +++ b/PanTS-Demo/src/test/sourceConsistency.test.tsx @@ -0,0 +1,97 @@ +import { fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import ReportScreen from '../components/ReportScreen/ReportScreen'; +import SourceReportDetails from '../components/ReportScreen/SourceReportDetails'; +import { getConsistencyItems, structureLabel } from '../helpers/reportConsistency'; +import { normalizeReportData } from '../helpers/reportFindings'; +import type { ReportData } from '../helpers/reportFindings'; + +// Synthetic values only; no patient data. +const checks = [ + { kind: 'source_volume', status: 'consistent', structures: ['liver'], measured_volume: 1020, source_volume: 1000, difference_percent: 2 }, + { kind: 'source_volume', status: 'discrepancy', structures: ['spleen'], measured_volume: 440, source_volume: 100, difference_percent: 340 }, + { kind: 'parts_whole', status: 'discrepancy', structures: ['pancreas', 'pancreas_head', 'pancreas_body', 'pancreas_tail'], parts_volume: 113, whole_volume: null }, + { kind: 'recorded_sex', status: 'discrepancy', structures: ['prostate'], measured_volume: 144.5, recorded_sex: 'F' }, +]; + +function report(consistency: unknown): ReportData { + return { + case_id: 'synthetic-7', patient: { age: 'Unknown', sex: 'Unknown' }, organ_volumes: {}, + comments: '', impression: [], consistency: consistency as ReportData['consistency'], + source_report: { available: true, source: 'Synthetic report fixture', text: 'Synthetic source.', findings: '', impression: [], reviewed_by_clinician: false }, + }; +} + +describe('source consistency helper', () => { + it('labels sided structures readably', () => { + expect(structureLabel('kidney_left')).toBe('left kidney'); + expect(structureLabel('adrenal_gland_right')).toBe('right adrenal gland'); + expect(structureLabel('pancreas_head')).toBe('pancreas head'); + }); + + it('builds text from typed fields, discrepancies first, and ignores incoming prose', () => { + const items = getConsistencyItems({ checks: checks.map(check => ({ ...check, message: 'All organs healthy.' })) }); + expect(items.map(item => item.status)).toEqual(['discrepancy', 'discrepancy', 'discrepancy', 'consistent']); + expect(items[0].text).toBe('Measured spleen volume (440.0 cc) and the volume stated in the source reference (100.0 cc) do not match.'); + expect(items[1].text).toBe('No pancreas measurement is available, but its parts (pancreas head, pancreas body, pancreas tail) measure 113.0 cc in total.'); + expect(items.map(item => item.text).join(' ')).not.toMatch(/healthy/); + }); + + it('drops malformed or unknown checks', () => { + expect(getConsistencyItems(undefined)).toEqual([]); + expect(getConsistencyItems({ checks: 'nope' })).toEqual([]); + expect(getConsistencyItems({ checks: [ + { kind: 'diagnosis', status: 'discrepancy', structures: ['spleen'], measured_volume: 1, source_volume: 2 }, + { kind: 'source_volume', status: 'abnormal', structures: ['spleen'], measured_volume: 1, source_volume: 2 }, + { kind: 'source_volume', status: 'discrepancy', structures: ['