Fix extraction placeholders, incidental parties, filing-type sync, and draft identity

docs/developer-notes/filing-data-integrity.md

@@ -0,0 +1,30 @@
+# Filing data integrity
+
+The changes for issues #217 and #218 establish three workflow rules.
+
+## Extraction evidence and parties
+
+Extraction normalization removes whole-value missing answers such as `unknown`, `N/A`, and `not provided`, including inside arrays and objects. Display normalization also covers older saved extractions. It preserves meaningful zero/false answers and names such as “All Unknown Occupants.”
+
+Only captioned plaintiffs/petitioners and defendants/respondents prefill party rows. Other names remain in the supporting evidence disclosure. A filer can explicitly add a person as a party; merely mentioning a child or witness does not add them to the submitted party list.
+
+## Primary filing type
+
+The lead `FilingDocument` owns the primary filing type. `FilingDraft.filing_type_code` and `filing_type_name` are synchronized summary fields. Document saves, lead changes, document deletion, and successful submission keep those fields aligned. Legacy case-data edits write through to the lead document. Migration `0022` repairs existing draft summaries, including submitted filings.
+
+Use model saves when changing document filing types or roles. Bulk updates bypass model synchronization. Supporting documents retain their own filing types. Review and refiling already use the lead document, while draft serialization uses its synchronized summary.
+
+## Draft identity across tabs
+
+Workflow URLs carry `?draft=<id>`. The workflow base template provides that page's identity to `draft-scope.js`, which preserves it in links, forms, browser history, and fetch requests. Fetch calls using a `Request` object carry `X-Filing-Draft` to preserve the request body. Server redirects and JSON redirect URLs retain the identity, including the submitted filing's confirmation page.
+
+Add any new workflow page to `WORKFLOW_VIEWS` in `services/draft_urls.py`. Starting a filing and choosing another draft remain separate navigation actions. The session pointer supports older entry URLs, but reading a draft never falls back to another browser's latest filing. Explicit IDs are checked for ownership, jurisdiction, status, and conflicting parameters before a view runs. Unavailable drafts return HTTP 409; they never select another draft.
+
+Regression checks, from `efile_app/`:
+
+```bash
+uv run pytest -q efile/tests/test_filing_integrity.py
+npx playwright test --config=playwright.draft-scope.config.js
+```
+
+The Python tests cover persisted data and separate/shared sessions. The browser tests cover links, forms, requests, reloads, and two simultaneous contexts using intercepted responses, without an EFSP account or live filing.

efile_app/efile/apps.py

@@ -7,4 +7,7 @@ class EfileConfig(AppConfig):
 
     def ready(self):
         # Registers the checks in efile/checks.py by importing them.
-        from efile import checks  # noqa: F401
+        from efile import (
+            checks,  # noqa: F401
+            signals,  # noqa: F401
+        )

efile_app/efile/middleware.py

@@ -1,8 +1,59 @@
 from django.contrib.auth import logout
+from django.http import JsonResponse
+from django.shortcuts import render
+from django.utils.deprecation import MiddlewareMixin
 
+from efile.models import FilingDraft
+from efile.services.current_drafts import DraftIdentityError, resolve_explicit_draft
+from efile.services.draft_urls import draft_url
 from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request
 
 
+class DraftIdentityMiddleware(MiddlewareMixin):
+    """Validate named drafts before views run and preserve them in redirects."""
+
+    def process_view(self, request, view_func, view_args, view_kwargs):
+        statuses = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR)
+        if request.resolver_match.url_name == "submit_final_filing":
+            statuses = (*statuses, FilingDraft.Status.SUBMITTING)
+        if request.resolver_match.url_name == "filing_confirmation":
+            statuses = (FilingDraft.Status.SUBMITTED,)
+        try:
+            resolve_explicit_draft(request, jurisdiction=view_kwargs.get("jurisdiction"), statuses=statuses)
+        except DraftIdentityError as error:
+            return self.process_exception(request, error)
+
+    def process_exception(self, request, exception):
+        if isinstance(exception, DraftIdentityError):
+            if (
+                not request.path.startswith("/api/")
+                and "application/json" not in request.headers.get("Accept", "")
+                and request.content_type != "application/json"
+            ):
+                return render(request, "efile/draft_unavailable.html", status=409)
+            return JsonResponse(
+                {
+                    "success": False,
+                    "error": "This draft is no longer available here. Open My drafts to choose a filing, or start a new one.",
+                },
+                status=409,
+            )
+
+    def process_response(self, request, response):
+        draft = getattr(request, "filing_draft", None)
+        if draft is not None:
+            if response.has_header("Location"):
+                response["Location"] = draft_url(response["Location"], draft.pk)
+            elif isinstance(response, JsonResponse):
+                import json
+
+                payload = json.loads(response.content)
+                if isinstance(payload, dict) and isinstance(payload.get("redirect_url"), str):
+                    payload["redirect_url"] = draft_url(payload["redirect_url"], draft.pk)
+                    response.content = json.dumps(payload)
+        return response
+
+
 class JurisdictionSessionMiddleware:
     """End an authenticated session before it can cross a jurisdiction boundary."""
 

efile_app/efile/migrations/0022_sync_primary_filing_type.py

@@ -0,0 +1,18 @@
+from django.db import migrations
+
+
+def synchronize_primary_types(apps, schema_editor):
+    Draft = apps.get_model("efile", "FilingDraft")
+    Document = apps.get_model("efile", "FilingDocument")
+    database = schema_editor.connection.alias
+    for draft in Draft.objects.using(database).iterator():
+        lead = Document.objects.using(database).filter(draft_id=draft.pk, role="lead").order_by("sort_order", "pk").first()
+        if lead is not None:
+            Draft.objects.using(database).filter(pk=draft.pk).update(
+                filing_type_code=lead.filing_type_code, filing_type_name=lead.filing_type_name
+            )
+
+
+class Migration(migrations.Migration):
+    dependencies = [("efile", "0021_party_is_self")]
+    operations = [migrations.RunPython(synchronize_primary_types, migrations.RunPython.noop)]

efile_app/efile/models.py

@@ -1,7 +1,7 @@
 # models.py - Optional extension to store additional user information
 from django.conf import settings
 from django.contrib.auth.models import AbstractUser
-from django.db import models
+from django.db import models, transaction
 from django.utils import timezone
 
 from efile.party_sides import PARTY_SIDE_CHOICES
@@ -280,6 +280,7 @@ def __str__(self):
         return f"{self.get_status_display()} filing draft #{self.pk} ({self.jurisdiction})"
 
     def mark_submitted(self, response_data):
+        sync_primary_filing_type(self)
         self.status = self.Status.SUBMITTED
         self.current_step = WorkflowStepKey.CONFIRMATION
         self.submission_response = response_data or {}
@@ -291,9 +292,18 @@ def mark_error(self, response_data):
         self.submission_response = response_data or {}
         self.save(update_fields=["status", "submission_response", "updated_at"])
 
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if self.pk and (update_fields is None or {"filing_type_code", "filing_type_name"}.intersection(update_fields)):
+            lead = self.documents.filter(role=FilingDocument.Role.LEAD).order_by("sort_order", "pk").first()
+            if lead is not None:
+                self.filing_type_code = lead.filing_type_code
+                self.filing_type_name = lead.filing_type_name
+        super().save(*args, **kwargs)
+
 
 class FilingDocument(models.Model):
-    """Uploaded document that belongs to a filing draft."""
+    """Uploaded document; the lead's filing type is authoritative for the draft."""
 
     class Role(models.TextChoices):
         LEAD = "lead", "Lead document"
@@ -348,6 +358,27 @@ class Meta:
     def __str__(self):
         return self.name or f"{self.get_role_display()} for draft #{self.draft_id}"
 
+    @transaction.atomic
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        sync_primary_filing_type(self.draft)
+
+
+def sync_primary_filing_type(draft):
+    """Persist the lead's filing type as the draft's primary filing summary.
+
+    Query after saving: organization can promote or demote documents in either
+    order. Also update the caller's draft instance so a later save cannot undo it.
+    """
+    lead = draft.documents.filter(role=FilingDocument.Role.LEAD).order_by("sort_order", "pk").first()
+    values = {
+        "filing_type_code": lead.filing_type_code if lead else "",
+        "filing_type_name": lead.filing_type_name if lead else "",
+    }
+    FilingDraft.objects.filter(pk=draft.pk).update(**values)
+    for field, value in values.items():
+        setattr(draft, field, value)
+
 
 class DocumentExtraction(models.Model):
     """Durable background work for analyzing one uploaded lead PDF."""

efile_app/efile/prompts/document_evidence_extraction.yaml

@@ -12,11 +12,11 @@ fields:
   court name: The complete court name, including department, division, unit, county, or venue when shown
   docket number: The docket or case number and any visible prefix
   case title: The complete caption or case title
-  plaintiff or petitioner names: All plaintiff or petitioner names
-  defendant or respondent names: All defendant or respondent names
-  other party names: Other named parties with their stated roles
+  plaintiff or petitioner names: Only people or organizations explicitly identified as plaintiffs or petitioners in the case caption; exclude children, witnesses, and other people merely mentioned in the document
+  defendant or respondent names: Only people or organizations explicitly identified as defendants or respondents in the case caption; exclude children, witnesses, and other people merely mentioned in the document
+  other party names: Other names mentioned in the document with their stated roles; these are reviewable evidence, not confirmed case parties
   document date: The date and its printed label, such as signed, issued, or filed
-  filing phase: Initial, subsequent, or unknown, only when document evidence supports the value
+  filing phase: Initial or subsequent, only when document evidence supports the value; otherwise omit
   requested relief: A concise list of expressly requested orders or remedies
   monetary amounts: Every expressly stated claim amount, amount in controversy, damages demand, rent due, estate value, or other classification-relevant amount, preserving its printed label
   selected options: Checked boxes or selected choices that distinguish the proceeding or filing
@@ -52,7 +52,7 @@ versions:
         * Return monetary amounts as a JSON array. Each item must have `label`, `raw`, `amount`, `currency`, and `evidence`. Copy `raw` exactly; use plain decimal digits without currency symbols or separators for `amount`; use `USD` when a dollar sign or dollars label establishes it. Omit a property that the source does not establish.
         * Keep classification evidence, requested relief, and selected options as JSON arrays. Other fields are strings; join multiple names with semicolons.
         * Include only evidence relevant to identifying the document or its underlying proceeding.
-        * Omit unsupported fields. Use "unknown" only for filing phase when the document does not establish it.
+        * Omit unsupported fields. Do not return placeholders such as "unknown", "N/A", or "not provided".
         * Do not include addresses, phone numbers, email addresses, financial account numbers, or unrelated allegations in classification evidence.
 
         Requested fields:
@@ -75,7 +75,7 @@ versions:
         * Return monetary amounts as a JSON array. Each item must have `label`, `raw`, `amount`, `currency`, and `evidence`. Copy `raw` exactly; use plain decimal digits without currency symbols or separators for `amount`; use `USD` when a dollar sign or dollars label establishes it. Omit a property that the source does not establish.
         * Keep classification evidence, requested relief, and selected options as JSON arrays. Other fields are strings; join multiple names with semicolons.
         * Include only evidence relevant to identifying the document or its underlying proceeding.
-        * Omit unsupported fields. Use "unknown" only for filing phase when the document does not establish it.
+        * Omit unsupported fields. Do not return placeholders such as "unknown", "N/A", or "not provided".
         * Do not include addresses, phone numbers, email addresses, financial account numbers, or unrelated allegations in classification evidence.
 
         Requested fields:

efile_app/efile/services/current_drafts.py

@@ -11,6 +11,38 @@
 CURRENT_DRAFT_SESSION_KEY = "filing_draft_id"
 
 
+class DraftIdentityError(Exception):
+    """An explicit draft is no longer available for this request."""
+
+
+def explicit_draft_id(request):
+    """Read tab-local identity; never substitute the shared session pointer."""
+    values = request.GET.getlist("draft")
+    if request.headers.get("X-Filing-Draft") is not None:
+        values.append(request.headers["X-Filing-Draft"])
+    if request.content_type in {"application/x-www-form-urlencoded", "multipart/form-data"}:
+        values.extend(request.POST.getlist("draft"))
+    if not values:
+        return None
+    if len(set(values)) != 1 or not values[0].isascii() or not values[0].isdigit() or len(values[0]) > 18:
+        raise DraftIdentityError
+    return int(values[0])
+
+
+def resolve_explicit_draft(request, *, jurisdiction=None, statuses=CURRENT_DRAFT_STATUSES):
+    draft_id = explicit_draft_id(request)
+    if draft_id is None:
+        return None
+    user = _authenticated_user(request)
+    draft = (
+        get_active_draft(user=user, draft_id=draft_id, jurisdiction=jurisdiction, statuses=statuses) if user else None
+    )
+    if draft is None:
+        raise DraftIdentityError
+    request.filing_draft = draft
+    return draft
+
+
 def _authenticated_user(request):
     user = getattr(request, "user", None)
     return user if getattr(user, "is_authenticated", False) else None
@@ -22,9 +54,13 @@ def attach_current_draft(request, draft: FilingDraft) -> None:
     request.session[CURRENT_DRAFT_SESSION_KEY] = draft.pk
     request.session["jurisdiction"] = draft.jurisdiction
     request.session.modified = True
+    request.filing_draft = draft
 
 
 def clear_current_draft(request) -> None:
+    draft = getattr(request, "filing_draft", None)
+    if draft is not None and request.session.get(CURRENT_DRAFT_SESSION_KEY) != draft.pk:
+        return
     if CURRENT_DRAFT_SESSION_KEY in request.session:
         del request.session[CURRENT_DRAFT_SESSION_KEY]
         request.session.modified = True
@@ -37,6 +73,9 @@ def pointed_at_draft(request, *, jurisdiction: str | None = None) -> FilingDraft
     supplied) jurisdiction are enforced on every lookup.
     """
 
+    explicit = resolve_explicit_draft(request, jurisdiction=jurisdiction)
+    if explicit is not None:
+        return explicit
     user = _authenticated_user(request)
     if user is None:
         clear_current_draft(request)
@@ -61,6 +100,8 @@ def pointed_at_draft(request, *, jurisdiction: str | None = None) -> FilingDraft
     )
     if draft is None:
         clear_current_draft(request)
+    else:
+        request.filing_draft = draft
     return draft
 
 
@@ -102,16 +143,12 @@ def get_current_draft(
     request,
     *,
     jurisdiction: str | None = None,
-    resume_latest: bool = True,
+    resume_latest: bool = False,
 ) -> FilingDraft | None:
-    """Resolve the current user's draft, without ever choosing one for them.
-
-    Reading is not choosing. This used to attach whatever draft it found to the
-    session, which meant that merely loading a page -- or an API call that page
-    fired -- could make an old filing the current one, and could do so *after* a
-    new filing had been started, silently putting the filer back in the old one.
-    Nothing here writes to the session now: adoption is ``adopt_draft``, and it
-    happens only where the filer asked for it.
+    """Resolve a named draft, falling back only to this session's own pointer.
+
+    Only resume offers opt into finding the account's latest draft. APIs must
+    not read a filing just because another browser recently worked on it.
     """
 
     draft = pointed_at_draft(request, jurisdiction=jurisdiction)
@@ -157,12 +194,9 @@ def ensure_current_draft(
     from a matter they finished last month.
     """
 
-    # A named draft wins over the one the session is holding: naming it is the
-    # filer saying "this one", and they may well be switching away from
-    # whatever they were last in.
-    draft = adopt_draft(request, request.GET.get(RESUME_DRAFT_PARAM), jurisdiction=jurisdiction)
-    if draft is None:
-        draft = pointed_at_draft(request, jurisdiction=jurisdiction)
+    # A named draft belongs to this request. Resolving it must not replace the
+    # shared session pointer, and an invalid identity must never fall through.
+    draft = pointed_at_draft(request, jurisdiction=jurisdiction)
     if draft is None:
         return create_current_draft(
             request,

efile_app/efile/services/draft_urls.py

@@ -0,0 +1,55 @@
+"""URLs that belong to a particular filing, shared with the browser."""
+
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from django.urls import Resolver404, resolve, reverse
+
+WORKFLOW_VIEWS = frozenset(
+    {
+        "filing_path",
+        "upload_documents",
+        "document_extraction_status",
+        "extraction_review",
+        "case_lookup",
+        "case_confirmation",
+        "document_checklist",
+        "organize_documents",
+        "your_information",
+        "parties",
+        "party_details",
+        "case_questions",
+        "payment",
+        "case_review",
+        "filing_confirmation",
+        "expert_form",
+        "upload_first",
+        "upload",
+    }
+)
+
+
+def draft_url(url, draft_id):
+    """Carry identity through workflow redirects, preserving other query parameters."""
+    parts = urlsplit(url)
+    if parts.netloc or parts.scheme:
+        return url
+    try:
+        if resolve(parts.path).url_name not in WORKFLOW_VIEWS:
+            return url
+    except Resolver404:
+        return url
+    query = dict(parse_qsl(parts.query, keep_blank_values=True))
+    query.setdefault("draft", str(draft_id))
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
+def browser_draft_context(request):
+    draft = getattr(request, "filing_draft", None)
+    if draft is None:
+        return {}
+    return {
+        "draft_scope": {
+            "id": draft.pk,
+            "paths": [reverse(name, kwargs={"jurisdiction": draft.jurisdiction}) for name in sorted(WORKFLOW_VIEWS)],
+        }
+    }