diff --git a/Makefile b/Makefile index 2bd32e25..e53e76e6 100644 --- a/Makefile +++ b/Makefile @@ -124,10 +124,10 @@ run-minio: @echo "Running with MinIO configuration..." @export $$(cat .env.minio.prod | grep -v '^#' | xargs) && cargo run -# Start MinIO server +# Start the shared development server without stopping isolated MinIO fixtures. minio-start: @mkdir -p /tmp/minio-data - @pkill -f "minio server" || true + @pkill -f '^minio server /tmp/minio-data --console-address :9001$$' || true @MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin nohup minio server /tmp/minio-data --console-address :9001 > /tmp/minio.log 2>&1 & @sleep 2 @export $$(cat .env.test | grep -v '^#' | xargs) && \ @@ -137,7 +137,7 @@ minio-start: # Stop MinIO server minio-stop: - @pkill -f "minio server" || true + @pkill -f '^minio server /tmp/minio-data --console-address :9001$$' || true @echo "MinIO stopped" # Clean MinIO data diff --git a/docs/plans/2026-10-19-next-work.md b/docs/plans/2026-10-19-next-work.md index 3fbe7039..c00df313 100644 --- a/docs/plans/2026-10-19-next-work.md +++ b/docs/plans/2026-10-19-next-work.md @@ -2,7 +2,7 @@ Owner: TimeFusion maintainers. Cross-repository owners are named per item. Status: remaining implementation and production acceptance. -Last implementation update: October 25 05:59Z. The October 22 numeric-severity search covers 03:51–04:24Z and returns 222 unique errors, all deployment-drain refusals. +Last implementation update: October 25 13:43Z. The October 22 numeric-severity search covers 03:51–04:24Z and returns 222 unique errors, all deployment-drain refusals. The cursor request repeats its first page. A bounded time-partition sweep supersedes that incomplete result; all 11 accepted partitions have `has_more=false`. The subsequent 04:24–04:59Z search returns zero errors and `has_more=false`; the preceding drain window supplies a positive control for the same filter. The early October 22 issue inventory at 04:58:18–04:58:42Z contains 523 open, 176 acknowledged, and 2,433 archived issues, with unchanged identities. @@ -55,6 +55,227 @@ Documentation updates change no issue or monitor state. Implementation releases ## Latest verified state: October 25 +The October 25 13:37Z production readback shows source `dcc81dd1`, image `sha256:5369efa0d4a9595fc76345c6fd4d14fe6da773f6e2385bd92204329abf3b359e`. +The live source fingerprint matches the checked source: `811588432a233698c759558fa4d284d7f0e2b9cfb6942f73b9598c590f5bbb6f`. +Its [CI run](https://github.com/monoscope-tech/timefusion/actions/runs/40198287585) and [deployment](https://github.com/monoscope-tech/timefusion/actions/runs/40198287590) succeed. +The release plans newest-first narrowing before the full query, budgets a 4 GB Tantivy block cache, and targets 128 MiB hot-tail files. +The 13:42Z snapshot records 272,220 accepted rows, zero flush failures, zero quarantine files, and 351 seconds of uptime. +No rows flushed during this short observation. This snapshot proves neither sustained delivery nor performance acceptance. + +The local checkout includes this deployed source. The interrupted optimized browser archive still represents earlier source `847cfac6`. +That archive supplies historical baseline evidence only. The calibrated browser matrix and matched resource gates remain open for the replacement. +The new process starts fresh recurrence and maturity windows. Evidence: external file `footer-delivery/snapshot-20261025T134252Z.json`. +The 13:51Z readback retains the same process. It records 665,993 accepted rows and 498,241 flushed rows, with zero flush failures. +Evidence: external file `footer-delivery/snapshot-20261025T135140Z.json`. The short interval does not establish a matched resource or recurrence pass. + +The newest-first review reproduces a separate query failure when the sort key is `context___is_remote`. +The indexed query attempts an unsupported timestamp-to-Boolean cast. Its timestamp index bound cannot narrow that Boolean sort. +The local correction restricts descending timestamp narrowing to the `timestamp` sort key. +The existing indexed/raw fixture now passes Boolean and Int32 sort equality, timestamp narrowing, and deletion-winner checks. +The combined release at `537ef2c1` passes all five local signoff gates: formatting, `cargo lint`, native tests, PGWire smoke, and end-to-end tests. +The results include 2,888 native tests, 16 doctests, 85 end-to-end tests, and 34 release-helper tests. +Its production image is built, smoke-tested, and published. Draft [PR #428](https://github.com/monoscope-tech/timefusion/pull/428) contains the correction and an isolated development MinIO stop selector. +The PR is open, unmerged, and undeployed at this readback. These release checks do not close performance or production acceptance. +Evidence: external files `newest-sort-{red,green}/run.log`, `newest-sort-integrated-release-batch.json`, and `newest-sort-integrated-signoff-20261025.log`. + +The next N4 candidate filters the canonical JSON writer output directly, avoiding an intermediate JSON column. +It retains the original serializer, SQL NULL behavior, Unicode case folding, query memory budget, and deadline. +The initial nine focused cases pass. New prepared-query cases reproduce a missing optimization after cached parameters bind. +The physical rewrite now handles that bound literal regex form; complex regex expressions retain their original path. +Commit `eab44b50` passes all 20 focused cases and 266 related function and optimizer tests. +These checks cover repeated parameter bindings, complex-regex fallback, Unicode case folding, SQL NULLs, scalar inputs, fragmented writer output, and pattern-sized buffers. +Evidence: external files `streaming-json-cached-semantic-receipt.json` and `streaming-json-cached-and-semantic-green-v2.log`. +This candidate is local and unpublished. It has no verified performance result and does not replace the retained thirty-day baseline. +Its native diagnostic build is in progress. Run the unchanged thirty-day query against the retained 80,995-row oracle after that build succeeds. +Keep the control's 1 GB block cache, four query partitions, 8 GB memory budget, and 90-second statement deadline. +Do not repeat the oracle build or calibrated browser fixture while checking this correction. + +The retained search experiment starts at local commit `f92911a4` on `perf/parallel-variant-filter-20261025`. It is not deployed. +Profiling attributes the fallback bottleneck to JSON conversion in a single-partition filter after deduplication. +The correction uses existing DataFusion repartition workers within the configured partition budget. Deduplication remains before the filter. +The output retains its declared order and single partition. The rule skips deduplication boundaries, positional boundaries, fetched operators, and global limits. +Seven focused cases pass, including exact rows, ordering, repeated optimization, and memory release after early stream cancellation. +All 159 related optimizer tests, formatting, and `cargo lint` pass. Full release signoff remains open. +Commit `def3c2ee` adds two passing integration cases with real deletion vectors, superseded bodies, tombstones, and committed or buffered updates. +Both cases exercise four filter partitions and retain exact winners with zero query reservations. The new tests also pass `cargo lint`. + +Checkpoint `1040741` now has a complete seven-day fixture with 125 local objects. Its reader cannot write to the fixture bucket. +The one-day and three-day indexed/raw results match the retained stock oracles exactly: ten and 2,804 rows, respectively. +The seven-day indexed/raw results contain identical 3,743 rows and body values. Both finish with zero memory reservations. + +| Window | Indexed-enabled seconds | Raw seconds | Result | +| --- | --- | --- | --- | +| 1 day | 0.589 | 1.319 | Exact stock-oracle equality | +| 3 days | 22.933 | 22.134 | Exact stock-oracle equality | +| 7 days | 62.804 | 62.836 | Exact indexed/raw equality within 90 seconds | + +The prefilter declines for three and seven days. These runs measure the complete fallback path. +The stock and JSON-buffer seven-day controls each exceeded 90 seconds. Their failed runs remain retained. +These diagnostic timings do not establish qualified paired performance. The thirty-day query and production resource gates remain open. +The paired comparison stops before its first timing run because `ghcid` uses 5.3% CPU, exceeding the existing 2% interference threshold. +The refusal remains retained, without an automatic retry. + +All 146 pinned thirty-day source objects retain their original sizes and ETags. +A local read-only gateway exposes 231 frozen objects without a complete fixture download. Local metadata retains checkpoint `1040741`. +Remote data and index reads require the captured ETags. Controls reject writes, incorrect ETags, unlisted keys, and newer Delta commits. +The one-day query through this gateway matches the retained stock oracle exactly, with zero query reservations. + +The full thirty-day indexed-enabled query reaches its unchanged deadline after 90.002 seconds. The raw arm does not run. +The native process uses 178.39 user CPU seconds and 7.80 system CPU seconds. Peak memory footprint is 1.824 GB. +The timeout snapshot retains 11,585,283 reserved bytes. This immediate observation does not prove a leak or five-second cancellation release. +The gateway session restores the original local MinIO endpoint. All 125 retained local objects have unchanged sizes and ETags. +These diagnostics use a different I/O environment from the earlier local-only runs. They do not establish qualified paired performance. +Evidence: external directory `body-baseline-20261025/range-gateway-v2`, including controls, request logs, final counters, and `verified-summary.json`. + +A fresh CPU sample of the parallel filter still shows UTF-8 validation, memory copies, JSON escaping, and Variant decoding. +The sampled three-day query retains exact indexed/raw equality and matches its stock oracle. The sample does not establish performance acceptance. +Evidence: external directory `body-baseline-20261025/run-3d-parallel-filter-sampled`, including the original sample, its hash, and the reconciled summary. + +The full-volume direct client-cancellation probe now passes for both indexed-enabled and raw queries. +More than two million rows enter deduplication before cancellation. Reservations decrease from 34–42 MB to zero within 80–86 ms. +The database, session, and runtime remain alive during the checks. This does not close N6f's actual browser or PGWire cancellation contract. +Evidence: external directory `body-baseline-20261025/run-30d-client-cancel`. + +A separate allocation prototype combines buffer reuse `b69e51c2` with direct JSON string writes `3960673` on application source `def3c2ee`. +The direct writer preserves output bytes and partial-writer errors across Unicode, control characters, empty strings, and long strings. +All 52 active library tests and 52 targeted TimeFusion variant tests pass. The two intended packages are the only lockfile changes. +The application lockfile returns to its original bytes after each isolated check. Both dependency prototypes remain unpublished. +The local machine is ARM, while production is x86_64. Cargo does not enable the optional `aarch64_neon` feature. +However, `simdutf8` also enables NEON through the compiler's target features. The native profile confirms NEON execution for long strings. +Feature absence alone does not establish scalar fallback. Strings shorter than 64 bytes still use the library's scalar validation path. +These diagnostics do not establish production CPU costs or qualified paired performance. + +| Combined prototype | Indexed-enabled seconds | Raw seconds | Exact retained-oracle equality | +| --- | --- | --- | --- | +| 1 day | 0.361 | 0.743 | Ten rows | +| 3 days | 14.698 | 14.356 | 2,804 rows | +| 7 days | 39.340 | 40.994 | 3,743 rows | + +The fixture now retains all 73 selected Parquet files locally. Fifty-three additional files total 22.139 GB. +Each transfer uses the original source ETag and a complete local SHA-256 reread. The transfer finishes within 636 seconds. +All 125 previous objects remain unchanged. The local mirror now has 178 objects, with the 16 GiB disk floor intact. +The gateway exposes the same 231 frozen objects, with conditional remote reads for the 53 older indexes. + +The combined prototype still reaches the full thirty-day deadline. It releases 14 MB of reservations within 60 ms after that actual timeout. +Both direct client-cancellation checks also pass, releasing reservations within 23–93 ms. Every release check keeps the session and runtime alive. +The original MinIO endpoint returns with all 178 object identities unchanged. No production data changes. +Evidence: `body-baseline-20261025/{30d-raw-mirror-receipt,combined-json-thirty-day-summary}.json` and `range-gateway-combined` in the external directory. + +The combined candidate now includes the bounded partial prefilter at application source `82477f1e`. +Its dependency prototypes remain `b69e51c2` and `3960673`. All 53 targeted search and visibility tests pass. +The complete-search API retains its original behavior. Partial coverage leaves declined files and overlapping coverage on the raw path. +The native binary SHA-256 is `55509eb39a452e339d33ff9c4fbc650a8de3562cee748e60fd13f5b6aaa8bc22`. + +| Partial-prefilter candidate | Indexed-enabled seconds | Raw seconds | Materialized hits | Exact retained-oracle equality | +| --- | --- | --- | --- | --- | +| 1 day | 0.482 | 0.893 | 36 | Ten rows | +| 3 days | 7.618 | 21.089 | 1,453 | 2,804 rows | +| 7 days | 27.542 | 55.540 | 1,996 | 3,743 rows | + +Every completed arm releases all query reservations. All 178 local object identities remain unchanged. +These diagnostic runs do not establish qualified paired performance. The candidate and dependency prototypes remain unpublished. +Evidence: `body-baseline-20261025/run-{1,3,7}d-partial-combined/{run-state,sql-parity-receipt}.json` in the external directory. + +The full thirty-day indexed-enabled query still reaches its unchanged deadline after 90.001 seconds. +Its 37.85 MB of query reservations decrease to zero within 12.1 ms after the deadline, with the session alive. +The raw arm does not run. Smaller-window parity does not establish complete thirty-day correctness or latency acceptance. +Evidence: `body-baseline-20261025/run-30d-partial-combined/{indexed-profile,indexed-timeout-cleanup,run-state}.json` in the external directory. + +Both direct client-cancellation probes pass. Reservations decrease to zero within 94 ms for indexed-enabled execution and 35 ms for raw execution. +The session and runtime remain alive. The gateway closes, and the original local MinIO endpoint retains all 178 object identities. +These direct probes do not close the browser and PGWire cancellation contract under N6f. + +During the full-query window, index requests transfer 656.6 MB and finish after 25.12 seconds. +Local Parquet requests transfer 5.104 GB and continue until the deadline. Concurrent request durations do not establish query wait time or CPU attribution. +Evidence: `body-baseline-20261025/range-gateway-partial/{full-query-request-accounting,verified-summary}.json` and `partial-combined-thirty-day-summary.json` in the external directory. + +Two eight-second CPU samples now cover execution near 35 and 65 seconds of the unchanged thirty-day query. +Both samples show UTF-8 validation, JSON escaping, memory copies, and Variant offset decoding. The query still reaches its original deadline. +The summary subtracts child samples and reconciles UTF-8 counts against the independent collapsed table. Counts do not represent CPU percentages. +The retained profiles also identify scalar validation of the reusable JSON output buffer. +Prototype `1771149` uses portable SIMD for output validation and retains the existing error format. +All 83 fork library tests pass, including 50 valid or malformed buffer cases around SIMD boundaries. +The lockfile adds only a direct dependency on the existing `simdutf8` 0.1.5 package. All 52 targeted TimeFusion Variant tests also pass. The application lockfile returns to its original bytes. +This change does not bypass Variant validation or establish a query latency improvement. +The local fixture returns with all 178 object identities unchanged. +Evidence: `body-baseline-20261025/run-30d-partial-combined-sampled/{sampling-state,profile-summary}.json` in the external directory. + +The SIMD candidate builds with binary SHA-256 `25c38cad770941adc84d2bb375ba4cb0101a040a83d6753dbea6c06a6a184b9e`. +Six invalid subwindow controls fail before database setup. A valid one-day raw subwindow matches the retained stock oracle exactly. +All completed arms release their query reservations. Every indexed-enabled arm retains the global 2,000-hit cap. + +| SIMD candidate | Indexed-enabled seconds | Raw seconds | Exact retained-oracle equality | +| --- | --- | --- | --- | +| 1 day | 0.394 | 0.800 | Ten rows | +| 3 days | 4.413 | 20.449 | 2,804 rows | +| 7 days | 24.347 | 50.393 | 3,743 rows | + +The complete thirty-day raw reference now contains 80,995 rows at checkpoint `1040741`. +Ten inclusive timestamp windows cover the original range without gaps or overlaps. Every actual plan includes timestamp in its deduplication key. +Each subwindow finishes within 90 seconds. Concatenation in descending window order preserves SQL ordering and row multiplicity without additional deduplication. +The ordered union SHA-256 is `55e76fea5406441a4b36f1d588b90faa61923dd4ef00543538670a60725044f5`. +This reference establishes complete raw results. It does not establish full-query latency or recover the lost historical checkpoint's oracle. +Evidence: `body-baseline-20261025/thirty-day-raw-oracle-1040741/{state,operator-summary}.json` and its referenced results in the external directory. + +The full thirty-day indexed-enabled query still reaches its unchanged deadline after 90.004 seconds. Its raw arm does not run. +At the deadline, the filter reports 12.99 million input rows, zero output rows, and 67.69 seconds of aggregate compute. +Deduplication reports 908 ms of aggregate compute. These metrics identify filter work but do not establish its scheduling cause. +Query reservations decrease from 34.57 MB to zero within 78 ms after the timeout. +Both direct client-cancellation probes pass, with zero reservations within 12 ms and 92 ms. The sessions remain alive. +The gateway closes, and the original local MinIO endpoint retains all 178 object identities. +Evidence: `body-baseline-20261025/{simd-validation-thirty-day-summary.json,run-30d-simd-validation/indexed-execution-metrics.txt}` in the external directory. + +Candidate `6a2c6e92` removes intermediate ordered gathering only below an unbounded full sort. +The rule crosses projections without volatile expressions. Ordered streams, volatile projections, fetched sorts, limits, and deduplication boundaries retain their previous requirements. +Both exchanges use existing DataFusion workers, memory accounting, and spill paths. +The old implementation fails the two new ordering assertions, with eight unchanged cases passing. +The candidate passes all 162 related optimizer tests and 53 visibility and search integration tests. +Its native binary is `e60e29f3`, with the same probe and dependency revisions as control binary `25c38cad`. +The unordered exchange activates in every measured full SQL plan. One-, three-, and seven-day results match their exact retained oracles. +Indexed-enabled times are 0.406, 6.477, and 40.883 seconds. Raw times are 0.879, 22.897, and 61.478 seconds. +These diagnostic timings establish no improvement. The full thirty-day query still reaches its deadline after 90.001 seconds. +Its filter reports 12.45 million input rows and 78.30 seconds of aggregate compute, without output before the deadline. +Timeout reservations decrease from 36.96 MB to zero within 16 ms. Direct cancellation releases reservations within 47 ms and 36 ms. +The local fixture returns with all 178 object identities unchanged. +Preserve the experimental branch, but exclude this candidate from the release batch. The registered deadline gate remains unmet. +Evidence: `body-baseline-20261025/unordered-filter-{red-state,green-state,guards-state}.json` and `unordered-gather-thirty-day-summary.json` in the external directory. + +Next: measure N6f's optimized deployed-runtime control before selecting another CPU-focused correction. +The archive build uses runtime source identical to deployed `847cfac6`, with stock dependency pins and the existing calibrated fixture. +It includes no experimental body-filter or bounded-merge algorithm. The full cold/warm browser contract and raw-oracle deadlines remain unchanged. +Evidence: `browser-optimized-stock-20261025/build-state.json` in the external directory. The build remains active. +N4 remains open. Reuse the 80,995-row raw reference for its next correction. +Retain the 2,000-hit cap, four query partitions, 8 GiB query budget, and 90-second deadline. +Qualified paired timings and production resource gates also remain open. +Evidence: `body-baseline-20261025/parallel-filter-diagnostic-summary.json` and its referenced receipts in the external evidence directory. + +The mature production window spans 07:43:35Z–07:58:45Z on the same process, image, and source fingerprint. +Across 909.87 seconds, the process accepts 652,366 rows and flushes 539,570 rows. +Pending tasks decrease from 178 to 130. Allocated WAL decreases from 10.671 GB to 9.915 GB. +Thirteen tasks remain due. The oldest due age increases from 246 to 1,156 seconds. +The checked flush, probe, stage, quarantine, airborne-topic, orphaned-topic, and sealed-debt counters remain zero. +CPU use is 1.770 ms per accepted row, including queries and maintenance. Different workloads prevent a controlled comparison with the previous release. +Evidence: `footer-delivery/mature-c030/{baseline,endpoint,summary}.json` in the external evidence directory. + +At 08:24Z, the same production process reports one due task and 6.702 GB of allocated WAL. +Four held task identities have exact publication and `Complete` events after their holds expire. Each publication contains zero rows. +The remaining due task retains an active hold until 08:39:31Z. These observations do not close the original unmatched narrow metrics task. +Monoscope returns zero structured statement failures from 06:51:49Z through 08:27:42Z, with `has_more=false`. +All sixteen monitors remain normal. The final two-minute activity control contains 1,703 events. +A subsequent complete search returns zero structured failures from 08:27:42Z through 11:18:07Z across both service identities. +All sixteen monitors remain normal. The final two-minute activity control contains 1,088 events. +Evidence: `post-build-monoscope-20261025/verified-summary.json` and its retained CLI responses in the external directory. +Evidence: external directory `post-hold-progress-20261025`, including exact task outcomes and the complete Monoscope responses. + +The last held key publishes 595 rows at 08:39:32.653905Z, 0.906 seconds after its hold expires. +Its exact-key `Complete` event follows four milliseconds later. All five keys in this later cohort now have publication and completion events. +The same key publishes 595 rows again at 10:06Z. These observations do not establish independent raw-result correctness. +At 11:12Z, the same process reports 19,649,649 accepted rows, 17,051,346 flushed rows, and 6.572 GB of allocated WAL. +RSS is 13.170 GB. Pending tasks total 288, with six due tasks and an oldest due age of 1,794 seconds. +The five displayed due identities retain rebuild holds. The limited diagnostic list does not identify the sixth due task. +Flush failures, probe timeouts, stage timeouts, quarantine files, airborne topics, orphaned topics, and sealed compaction debt remain zero. +Evidence: `footer-delivery/snapshot-20261025T111219Z.json` and `post-hold-followup-1112/{verified-key-outcome,cohort-summary}.json` in the external directory. + At 03:13Z, `make ci-selftest` deletes this worktree's local `.ci/` evidence directory during its fingerprint-pinning check. The source commits, published GitHub artifacts, CI-attestation refs, and other worktrees remain intact. Production storage is unchanged. The original local raw receipts and body-search mirror are unavailable. Do not reuse their historical paths as retained evidence. @@ -106,7 +327,8 @@ Evidence: `consumer-full-signoff-state.json` and `consumer-serial-gates-signoff. The CI cleanup correction merges through [PR #424](https://github.com/monoscope-tech/timefusion/pull/424), merge `fef48253`. The real self-test preserves all 35 retained files byte-for-byte. Isolated tests cover fresh and populated CI directories. Its full local checks pass. Normal deployment reuses the existing image and preserves the running process. -New evidence is stored outside `.ci/`: `/Users/tonyalaribe/Library/Caches/timefusion-evidence-recovery-20261025`. +New evidence is stored outside `.ci/`: `/Users/tonyalaribe/Projects/apitoolkit/timefusion-evidence-20261025`. +The previous Library/Caches location is a compatibility symlink to this directory. The original benchmark checkpoint now returns HTTP 404 from its original bucket and path. Recovered source code and reported observations do not restore the missing dataset or complete result oracles. Use a newly pinned, explicitly versioned dataset before further historical-volume acceptance. Never rename a replacement as the old checkpoint. @@ -118,7 +340,8 @@ The one-day and three-day fixtures are restored in isolated local MinIO on port All 89 initial local objects, including checkpoint and manifests, pass a complete reread-and-SHA256 check. The three-day extension completes in 228 seconds, within its 420-second budget and 12 GiB input cap. New objects pass local reread-and-hash checks. An optimized native baseline builds from deployed source `847cfac6`, using unchanged query bounds, predicate, 90-second deadline, and 2,000-hit cap. -New content parity and qualified timings remain untested. Do not reuse old oracles or performance comparisons for this replacement. +The current content and timing results appear at the start of this section. Qualified paired timings remain open. +The replacement checkpoint retains separate oracles and receipts from the unavailable original dataset. Evidence: external recovery directory `body-baseline-20261025`, including `receipt.json`, per-window object prices, and `1d-local-mirror-receipt.json`. Keep the existing coverage failure gate, 240-second maintenance deadline, and maintenance memory pool unchanged. @@ -1014,6 +1237,21 @@ Eight cold pairs and the first two warm pairs pass; the third indexed-side warm Cancellation and the rest of the warm matrix remain unverified. Do not promote the candidate from these partial results or raise the oracle deadline. Different host contention in prior failed runs prevents attributing their timing difference solely to this candidate. The sampled process is a debug test binary. Check an optimized control before selecting another CPU-focused runtime correction; local storage timing still cannot establish production object-store latency. +The optimized stock archive completed on October 25 at 12:43Z. Its runtime source matches deployed source `847cfac6` without experimental dependency patches. +The archive SHA-256 is `be681e4cf7e20ed513829d6164fbc982e344ba9dcb7d54bd80e5b22c404dd24f`. +The first attempt stopped during second-store preparation after its private MinIO received SIGTERM. No browser query ran. +The retained store contains 175 objects. This interruption supplies no query performance result. +The repository's broad `pkill -f "minio server"` selector also matches private fixtures on other ports. The signal sender remains unverified. +The fixture launcher now uses a distinct process name. A real MinIO probe passed and remained outside that selector. +The full partial-coverage retry started at 13:22Z with the same archive, 90-second deadline, and 512 MiB candidate limit. +The retry also received a termination signal after 678.908 seconds, during second-store preparation. The signal sender remains unknown. +The first store contains all 91,609,183 background rows. The last completed second-store file brings its total to 61,279,112 rows. +Both local WAL states and the physical object store remain retained. No browser query pair completed. +The sixteen cold/warm pairs and cancellation gate remain unverified. No automatic retry follows this interruption. +The Makefile now limits both stop selectors to the exact shared development-server command. Private paths and alternate-port commands remain outside that selector. +The process regression passes with three disposable native processes. The old selector matches all three, while the corrected selector matches only its target. +Evidence: external files `check-minio-stop-selector.py` and `minio-stop-selector-receipt.json`. This correction does not identify either historical signal sender. +Evidence: external directory `browser-optimized-stock-20261025`, including `build-state.json` and `partial_cold_at_calibrated_volume-isolated/run.log`. Footers: `/tmp/timefusion-browser-volume-tournament-{on,off}-footers.json`; sampled stacks are retained separately. Require the same 90-second deadline, exact ordered raw-oracle page, cold/warm coverage, and five-second cancellation gate before promotion. This candidate remains outside the frozen receipt-fix release. Review: `/tmp/timefusion-bounded-merge-tournament-review.json`. @@ -2579,7 +2817,7 @@ The October 17 30-second S3 PUT failure targeted Delta version `936775`. Its lat Current airborne/orphaned counts are zero. One lifetime flush failure does not establish a current outage or the earlier commit's outcome. The pre-rollout digest is linked to the successful `ad1ee10b` deploy run above. -The current production source is `ec986790`, with the runtime digest recorded in the release section. +The earlier production source was `ec986790`. The October 25 11:12Z readback identifies source `847cfac6` and runtime digest `c9be8ba6`. Neither provenance claim proves the original publication outcome. The original 214-row attempt has publication ID `a2adfc39-942c-47d4-a832-f9060858713c` and bucket `5973983`. At October 17 00:01:49.792622Z it reports the exact version as unconfirmed. @@ -2593,10 +2831,26 @@ Keep the immutable payload/receipt outcome and original-row identity acceptance Logs: `/tmp/timefusion-original-flush-followup.json`, `/tmp/timefusion-original-flush-identity.json`, and `/tmp/timefusion-original-flush-version-owner.json`. The direct object lookup result is in `/tmp/timefusion-publication-936775.error`. -Next establish the immutable version's authoritative outcome and the affected topic's subsequent progress. -Use publication receipts, retained intent identity, and the original version before considering any new callback. +The retained events establish subsequent topic progress. The original publication outcome remains an unresolved historical record. +An authoritative receipt or retained original version can resolve that record. A new callback cannot recover missing historical evidence. Keep accepted-PUT/response-loss, restart, late success, terminal failure, and late-write snapshot cases in the acceptance record. +The unresolved records remain distinct: + +| Record | Known outcome | Missing evidence | +| --- | --- | --- | +| Publication `a2adfc39-942c-47d4-a832-f9060858713c`, version `936775`, 214 rows | Later topic publication succeeds at version `936834` | Original publication ownership and exact row identities | +| October 21 04:29Z handoff, 220 refused statements | Caller retries exist | Per-statement batch identities and terminal outcomes | +| October 21–22 enrichment cohort, 88 UPDATE exports | Execute-stage failures exist | Durable replay or terminal outcomes for the affected updates | + +The October 25 source audit checks Monoscope `8890344e9603c7fedff36e0182ff8d7b9b04124c` without production mutation. +`runDrainFlusher` removes each task from its queue before `flushDrainTask` runs. +`dualExecPgTfWithSql` logs TimeFusion failures or skips an open circuit, then returns the PostgreSQL result. +This path supplies no durable TimeFusion retry receipt. Successful INSERT acknowledgements cannot resolve these UPDATE records. +Any recovery correction must retain the pending TimeFusion update without repeating pattern-frequency increments or acknowledging an unsuccessful TimeFusion write. +This source audit does not establish the currently deployed Monoscope image or prove historical data loss. +Evidence: external file `update-delivery-source-audit-20261025.json`. + Reuse existing guards in [write/mod.rs](../../src/write/mod.rs), [database tests](../../src/database/tests.rs), and [restart e2e](../../tests/e2e/restart_recovery.rs). Relevant guards include `detached_alarm_preserves_the_attempt_until_its_outcome_is_known` and `detached_success_drains_only_its_original_prefix`. Also retain `restart_before_detached_receipt_retirement_does_not_republish` and `replayed_rows_that_delta_already_holds_are_not_written_again`. diff --git a/src/read/bounded_merge.rs b/src/read/bounded_merge.rs index eba4c231..2ffd9aba 100644 --- a/src/read/bounded_merge.rs +++ b/src/read/bounded_merge.rs @@ -898,6 +898,9 @@ pub(crate) async fn narrow_descending_fetch( ) -> Result>> { use datafusion::physical_planner::PhysicalPlanner; let Some((page, column, scan)) = latest_page(logical, false) else { return Ok(None) }; + if column.name != "timestamp" { + return Ok(None); + } let Some(table) = datafusion::datasource::source_as_provider(&scan.source).ok() else { return Ok(None) }; let Some(table) = downcast::(&*table) else { return Ok(None) }; let Some((lo, _)) = crate::database::filters_time_range(&scan.filters) else { return Ok(None) }; diff --git a/tests/suite/tantivy_e2e_test.rs b/tests/suite/tantivy_e2e_test.rs index e3d8010f..6f793d54 100644 --- a/tests/suite/tantivy_e2e_test.rs +++ b/tests/suite/tantivy_e2e_test.rs @@ -1077,6 +1077,11 @@ async fn a_newest_first_page_narrows_to_the_index_bound_and_matches_baseline() - assert!(hits() - before < 30, "only the narrowed window's matches are resolved: {}", hits() - before); assert_eq!(ordered_ids(&ctx, &page).await?, ordered_ids(&ctx_off, &page).await?); + for column in ["context___is_remote", "severity___severity_number"] { + let other_sort = page.replace("ORDER BY timestamp DESC", &format!("ORDER BY {column} DESC NULLS FIRST, id")); + assert_eq!(ordered_ids(&ctx, &other_sort).await?, ordered_ids(&ctx_off, &other_sort).await?, "timestamp bounds must not narrow another sort key"); + } + // Deleting the eight newest errors must not surface them, whichever way the page is filled // (the short-page fallback itself is `NarrowedFetchExec`'s, shared with the ascending case). let gone = (0..8).map(|i| format!("'e{i}'")).collect::>().join(",");