diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml index 4fe119f48..c69ff8bad 100644 --- a/.github/workflows/build-ci-image.yml +++ b/.github/workflows/build-ci-image.yml @@ -7,7 +7,7 @@ name: Build CI deps image 🐳 # dispatch. # # One image per ARCHITECTURE, x86_64 and aarch64, each built on a runner of that architecture -# (ubuntu-24.04-arm for aarch64) so nothing runs under emulation. The x86_64 image keeps the +# (ubuntu-26.04-arm for aarch64) so nothing runs under emulation. The x86_64 image keeps the # plain toolchain-key tag; the aarch64 image's tag carries an -aarch64 suffix. # # OWNER: the package lands under whichever org runs this; it MUST run in bertiniteam/b2 (its token @@ -45,9 +45,9 @@ jobs: pytag: [cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313, cp314-cp314] include: - arch: x86_64 - runner: ubuntu-latest + runner: ubuntu-26.04 - arch: aarch64 - runner: ubuntu-24.04-arm + runner: ubuntu-26.04-arm steps: - uses: actions/checkout@v5 # SOURCE the file, then write the three values out by name. Do NOT cat it into @@ -70,7 +70,7 @@ jobs: -e BOOST_VERSION -e EIGEN_VERSION -e EIGENPY_VERSION \ "$MANYLINUX_IMAGE" \ bash docker/manylinux-deps/build-python-deps.sh "${{ matrix.pytag }}" - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 with: name: deps-${{ matrix.arch }}-${{ matrix.pytag }} path: deps-${{ matrix.pytag }}.tar.gz @@ -88,10 +88,10 @@ jobs: matrix: include: - arch: x86_64 - runner: ubuntu-latest + runner: ubuntu-26.04 suffix: "" - arch: aarch64 - runner: ubuntu-24.04-arm + runner: ubuntu-26.04-arm suffix: "-aarch64" steps: - uses: actions/checkout@v5 @@ -107,7 +107,7 @@ jobs: } >> "$GITHUB_ENV" - name: Collect prebuilt deps into the build context - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: pattern: deps-${{ matrix.arch }}-* merge-multiple: true @@ -122,14 +122,14 @@ jobs: ls -lh docker/manylinux-deps/deps - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build & push (assembly only -- fast) - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: docker/manylinux-deps push: true diff --git a/.github/workflows/build-macos-deps.yml b/.github/workflows/build-macos-deps.yml index 1f975c51f..dc8259ce7 100644 --- a/.github/workflows/build-macos-deps.yml +++ b/.github/workflows/build-macos-deps.yml @@ -60,7 +60,7 @@ jobs: echo "EIGEN_VERSION=$EIGEN_VERSION" echo "EIGENPY_VERSION=$EIGENPY_VERSION" } >> "$GITHUB_ENV" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0fe5054d7..6ce1c1100 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -33,7 +33,7 @@ on: workflow_call: inputs: minimal: - description: 'Build only the ubuntu-latest / python-3.11 wheel; skip C++ tests, Windows, macOS, and wheel tests.' + description: 'Build only the ubuntu-26.04 / python-3.11 wheel; skip C++ tests, Windows, macOS, and wheel tests.' required: false type: boolean default: false @@ -50,7 +50,7 @@ on: type: boolean default: false minimal: - description: 'Build only the ubuntu-latest / python-3.11 wheel; skip C++ tests, Windows, macOS, and wheel tests.' + description: 'Build only the ubuntu-26.04 / python-3.11 wheel; skip C++ tests, Windows, macOS, and wheel tests.' required: false type: boolean default: false @@ -68,10 +68,6 @@ concurrency: # Boost/Eigen/eigenpy versions are single-sourced in .github/ci-deps-versions.env and surfaced by # the `set_versions` job; the two jobs that build them from source consume it as job-level env. # That same file is in build-ci-image.yml's path filter, so a version bump rebuilds the Linux image. -env: - # Opt JavaScript actions into Node 24 (Node 20 is being removed from the runners). - # Silences the conda-incubator/setup-miniconda@v3 (and other node20 action) deprecation. - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: # Cheap, Doxygen-only gate (no compilation). Every job that actually builds @@ -84,7 +80,7 @@ jobs: set_versions: name: Resolve dependency versions - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: boost: ${{ steps.v.outputs.boost }} eigen: ${{ steps.v.outputs.eigen }} @@ -103,7 +99,7 @@ jobs: set_matrix: name: Set OS matrix - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: os: ${{ steps.matrix.outputs.os }} python_versions: ${{ steps.matrix.outputs.python_versions }} @@ -119,19 +115,22 @@ jobs: # If it SIGABRTs again, take it back out of this line and keep `-f os=macos-intel`, # which runs intel alone at one python version, for asking again later. if [[ "${{ inputs.minimal }}" == "true" ]]; then - OS='["ubuntu-latest"]'; PY='["3.11"]' + OS='["ubuntu-26.04"]'; PY='["3.11"]' elif [[ "${{ github.ref }}" == "refs/heads/main" || "${{ github.ref }}" == "refs/heads/develop" || "${{ github.ref }}" == refs/tags/* || "${{ inputs.full_matrix }}" == "true" || "${{ github.event.pull_request.base.ref }}" == "develop" || "${{ github.event.pull_request.base.ref }}" == "main" ]]; then - # ubuntu-24.04-arm is linux-aarch64 (#474): a full member, C++ tests and every wheel, - # on a native arm runner rather than under emulation. - OS='["ubuntu-latest","ubuntu-24.04-arm","macos-14","macos-15-intel"]'; PY='["3.10","3.11","3.12","3.13","3.14"]' + # ubuntu-26.04-arm is linux-aarch64 (#474): a full member, C++ tests and every wheel, + # on a native arm runner rather than under emulation. Linux names its Ubuntu + # version on both architectures, here and in every workflow, instead of + # ubuntu-latest, so an image change is a deliberate edit rather than a surprise: + # the C++ test job compiles with the runner's own GCC and apt packages. + OS='["ubuntu-26.04","ubuntu-26.04-arm","macos-14","macos-15-intel"]'; PY='["3.10","3.11","3.12","3.13","3.14"]' else - OS='["ubuntu-latest"]'; PY='["3.11"]' + OS='["ubuntu-26.04"]'; PY='["3.11"]' fi # Manual OS targeting (the workflow_dispatch `os` input) narrows the unix matrix; the # separate Windows jobs gate on the same input. Empty/`all` => leave as computed above. case "${{ inputs.os }}" in - linux) OS='["ubuntu-latest","ubuntu-24.04-arm"]' ;; - linux-arm) OS='["ubuntu-24.04-arm"]' ;; + linux) OS='["ubuntu-26.04","ubuntu-26.04-arm"]' ;; + linux-arm) OS='["ubuntu-26.04-arm"]' ;; macos) OS='["macos-14"]' ;; macos-intel) OS='["macos-15-intel"]'; PY='["3.11"]' ;; # the #215 experiment windows) OS='[]' ;; # no unix jobs; only the standalone Windows jobs run @@ -304,7 +303,7 @@ jobs: windows-conda-cpp- windows-conda- - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@v4 with: activate-environment: b2-windows environment-file: environment-win.yml @@ -585,7 +584,7 @@ jobs: du -sh .ccache 2>/dev/null || echo "no .ccache directory was produced" if command -v ccache >/dev/null; then CCACHE_DIR=$PWD/.ccache ccache -s || true; fi - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 with: name: wheels-${{ matrix.os }}-${{ matrix.python-version }} path: ./wheelhouse/*.whl @@ -623,7 +622,7 @@ jobs: restore-keys: | windows-conda-${{ matrix.python-version }}- windows-conda- - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@v4 with: activate-environment: b2-windows environment-file: environment-win.yml @@ -683,7 +682,7 @@ jobs: Remove-Item dist/*.whl Move-Item repaired/*.whl dist/ - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 with: name: wheels-windows-${{ matrix.python-version }} path: ./dist/*.whl @@ -708,18 +707,18 @@ jobs: os: ${{ fromJson(needs.set_matrix.outputs.os) }} python-version: ${{ fromJson(needs.set_matrix.outputs.python_versions) }} exclude: - - os: ubuntu-latest - - os: ubuntu-24.04-arm + - os: ubuntu-26.04 + - os: ubuntu-26.04-arm steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Download wheels artifact - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: name: wheels-${{ matrix.os }}-${{ matrix.python-version }} path: dist/ @@ -754,7 +753,7 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@v4 with: activate-environment: b2-windows environment-file: environment-win.yml @@ -763,7 +762,7 @@ jobs: channel-priority: strict - name: Download wheels artifact - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: name: wheels-windows-${{ matrix.python-version }} path: dist/ @@ -791,14 +790,14 @@ jobs: name: Docs build (doctest + html -W) if: ${{ inputs.minimal != true }} needs: [build_macos_ubuntu_wheels] - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 20 steps: - uses: actions/checkout@v5 with: fetch-depth: 0 # gitpython in conf.py derives the version from the git history - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -809,9 +808,9 @@ jobs: scikit-build-core build numpy tqdm matplotlib pandas sympy networkx - name: Download built wheel - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: - name: wheels-ubuntu-latest-3.11 + name: wheels-ubuntu-26.04-3.11 path: dist/ - name: Install bertini from built wheel diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 5d0e5b85d..f2ea5bc9a 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -37,7 +37,7 @@ jobs: if: github.event_name == 'workflow_dispatch' uses: ./.github/workflows/build_and_test.yml with: - # Docs only need the ubuntu-latest / python-3.11 wheel; skip the full + # Docs only need the ubuntu-26.04 / python-3.11 wheel; skip the full # OS/Python matrix and the wheel/C++ test jobs. minimal: true ref: ${{ inputs.ref }} @@ -48,7 +48,7 @@ jobs: # python_docs as the `docs-cpp` artifact for final assembly. cpp_docs: name: Build C++ docs (Doxygen) - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v5 with: @@ -86,7 +86,7 @@ jobs: mv build/docs/cpp site/cpp - name: Upload C++ docs artifact - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: docs-cpp path: site/cpp @@ -102,7 +102,7 @@ jobs: always() && (needs.build_and_test.result == 'success' || needs.build_and_test.result == 'skipped') && needs.cpp_docs.result == 'success' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: contents: write # push the persistent docs-store branch (the durable versioned-docs store) steps: @@ -115,7 +115,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' @@ -141,9 +141,9 @@ jobs: scikit-build-core build numpy matplotlib pandas sympy networkx - name: Download built wheel - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: - name: wheels-ubuntu-latest-3.11 + name: wheels-ubuntu-26.04-3.11 path: dist/ - name: Install bertini from built wheel @@ -152,7 +152,7 @@ jobs: # The Doxygen C++ reference was built in parallel by the cpp_docs job; drop it # into site/cpp so the assembled site/ has cpp + python + landing. - name: Download C++ docs - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: name: docs-cpp path: site/cpp diff --git a/.github/workflows/doc_lint.yml b/.github/workflows/doc_lint.yml index 30751c5f0..6fa8f9c00 100644 --- a/.github/workflows/doc_lint.yml +++ b/.github/workflows/doc_lint.yml @@ -61,7 +61,7 @@ env: jobs: cpp_doc_lint: name: Doxygen doc lint (C++) - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v5 @@ -80,12 +80,12 @@ jobs: # Runs in parallel with cpp_doc_lint. Pure-stdlib ast, no build, no pip deps. python_doc_lint: name: Docstring lint (Python) - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' @@ -96,12 +96,12 @@ jobs: # rule is in .editorconfig, and `python tools/whitespace_lint.py --fix` applies it. whitespace_lint: name: Whitespace lint (tabs, trailing whitespace, line endings) - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' diff --git a/.github/workflows/github-gitlab-sync.yml b/.github/workflows/github-gitlab-sync.yml deleted file mode 100644 index 0b6f34c51..000000000 --- a/.github/workflows/github-gitlab-sync.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Sync to GitLab mirror -on: - push - -# Any repo (upstream or fork) that configures the three GITLAB_* secrets gets -# mirrored to its own GitLab target; repos without the secrets skip silently. -# This replaces the old hardcoded `if: github.repository == 'bertiniteam/b2'`, -# which prevented forks from syncing even when their secrets were set. -# Note: the `secrets` context is not available in job-level `if`, hence the -# env indirection. - -jobs: - sync: - runs-on: ubuntu-latest - env: - GITLAB_CONFIGURED: ${{ secrets.GITLAB_URL != '' }} - steps: - # Mirroring is best-effort replication: an outage on the GitLab end (502s, etc.) - # must not paint the branch's checks red -- it says nothing about the code. The - # step-level continue-on-error keeps the check green either way; the warning - # annotation below keeps a real mirror failure visible on the run's summary page - # so a rotted token or dead mirror still gets noticed. - - name: Sync to GitLab - id: sync - continue-on-error: true - if: env.GITLAB_CONFIGURED == 'true' - # You may pin to the exact commit or the version. - # uses: kujov/gitlab-sync@b19399d43e81ac88acb6eefd5588e6ebde3d7d88 - uses: kujov/gitlab-sync@2.2.1 - with: - # The URL of the GitLab repository (e.g., https://gitlab.com/user/repo.git) - gitlab_url: ${{ secrets.GITLAB_URL }} - # Your GitLab username - username: ${{ secrets.GITLAB_USERNAME }} - # Your GitLab Personal Access Token with required permissions - gitlab_pat: ${{ secrets.GITLAB_PAT }} - # Whether to force push to GitLab. Defaults to false. - force_push: true - - name: Surface a mirror failure as a warning (not a red check) - if: env.GITLAB_CONFIGURED == 'true' && steps.sync.outcome == 'failure' - run: echo "::warning title=GitLab mirror sync failed::The push to the GitLab mirror failed (often a GitLab-side outage). The code checks are unaffected; investigate if this persists." diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2a99b2d95..87735eef3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,7 +16,7 @@ concurrency: jobs: check_version: name: Check VERSION file matches tag - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: is_prerelease: ${{ steps.check.outputs.is_prerelease }} steps: @@ -44,11 +44,18 @@ jobs: # validation regex above -- a naive `\.(dev|a|b|rc)` would misread rc1 as a FINAL release # and route it to real PyPI + a GitHub Release instead of TestPyPI. echo "$TAG" | grep -Eq '(\.dev[0-9]+|a[0-9]+|b[0-9]+|rc[0-9]+)$' && echo "is_prerelease=true" >> $GITHUB_OUTPUT || echo "is_prerelease=false" >> $GITHUB_OUTPUT + # The GitHub release publishes CHANGELOG.md's top block as its notes. Check it here, in + # the release's first job, so a changelog that is not ready (undated, the wrong version, + # no separators) fails the release in seconds instead of after every wheel has built. + # Final releases only: prereleases go to TestPyPI and make no GitHub release. + - name: Check the release notes + if: steps.check.outputs.is_prerelease == 'false' + run: python tools/release_notes.py --tag "$GITHUB_REF_NAME" check_timing_freshness: name: Check tutorial timings are fresh needs: check_version - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v5 # Gate ONLY real releases. We keep this as a step-level `if` (not a job-level one) so the @@ -72,7 +79,7 @@ jobs: name: Publish to TestPyPI if: needs.check_version.outputs.is_prerelease == 'true' needs: [check_version, build_and_test] - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 environment: name: testpypi @@ -83,7 +90,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: # ONLY the wheel artifacts. build_docs.yml's cpp_docs job also uploads a # `docs-cpp` artifact (the whole Doxygen tree); without this pattern, @@ -104,7 +111,7 @@ jobs: name: Publish to PyPI if: needs.check_version.outputs.is_prerelease == 'false' needs: [check_version, build_and_test] - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 environment: name: pypi @@ -115,7 +122,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: # ONLY the wheel artifacts. build_docs.yml's cpp_docs job also uploads a # `docs-cpp` artifact (the whole Doxygen tree); without this pattern, @@ -139,7 +146,7 @@ jobs: needs: - check_version - publish-to-pypi - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: contents: write # IMPORTANT: mandatory for making GitHub Releases @@ -150,7 +157,7 @@ jobs: uses: actions/checkout@v5 - name: Download Artifacts - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: # ONLY the wheel artifacts. build_docs.yml's cpp_docs job also uploads a # `docs-cpp` artifact (the whole Doxygen tree); without this pattern, @@ -168,12 +175,11 @@ jobs: ./dist/*.whl - name: Get Newest Changelog - run: | - python -c 'import re; from pathlib import Path; text=re.sub("", "", (Path.cwd() / "CHANGELOG.md").read_text(), flags=re.DOTALL); print(text); start=text.find("_" * 79); (Path.cwd() / "TEMP_CHANGELOG.md").write_text(text[start:text.find("_" * 79, start+1)])' + run: python tools/release_notes.py --tag "$GITHUB_REF_NAME" --output TEMP_CHANGELOG.md - name: Create GitHub Release id: create_release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0620e042d..adad65759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,8 @@ ask and is recomputed rather than reused. Nothing is lost and nothing is silent simply not reused. That remains the honest outcome for as long as a record's ask does not carry the identity of the algorithm that answered it (#420). +**A System archived or pickled by 3.x does not load in 4.0.** What a System writes to a Boost archive changed: its precision is no longer written, and its auxiliary variable groups and auxiliary coordinates now are. A pickle holds such an archive, so a pickled System, or anything pickled that contains one, is affected in the same way. Rebuild the system from the code or the input file that made it. A records directory stores its systems as JSON and is not affected. + ### Removed - **The windows DLL plumbing is no longer part of the interface** (#99). @@ -429,6 +431,35 @@ the identity of the algorithm that answered it (#420). ### Fixed +- **The Cauchy endgame honours the security ceiling as the power series endgame does.** At + security level 0, two consecutive endpoint approximations whose largest coordinate is above + `max_norm` truncate a path with `SecurityMaxNormReached`. The power series endgame asks + that after every approximation; the Cauchy endgame tested acceptance first, so a path that + converged to a finite point above the ceiling came back `Success` from Cauchy and truncated + from power series -- for `x^2 = 1e8, y = 2x` at the default ceiling of 1e4, two solutions + under one endgame and none under the other. Cauchy now asks security first, and never + accepts an approximation above the ceiling. Its count still runs over rounds in the + operating zone only, where the Cauchy mean can be trusted. The two endgames differ in how + they estimate the root and in nothing else (ADR-0067). A solve whose solutions lie above + `max_norm` needs the ceiling raised (`solver.update(max_norm=...)`) or security level 1, + under either endgame. The `SecurityConfig.level` docstring said the opposite of what the + level does, and is corrected. +- **A wide real random orthonormal matrix has distinct columns again.** The faster + factorization from #401 (below) QR-factors a tall seed matrix, and for a *real* request that + seed was a matrix of units, which for real numbers means signs. A tall matrix of signs has + only `2^thin` distinct rows, so the resulting `rows x cols` matrix had at most `2^rows` + distinct columns: a `3 x 5` real draw had 3, an `8 x 4908` one had 256, and randomizing an + overdetermined deflated system down to square with `random_matrix(n, m, real=True)` produced + a singular square system at a point where the deflated Jacobian had full rank, so Newton + could not converge there. The real seed is now a continuous draw on `[-1, 1]`; the QR still + launders it into an orthonormal matrix, now a generic one. Complex draws are unchanged. + Real orthonormal draws -- `random_matrix(..., real=True)` and `Slice.random_real` -- differ + from before for the same seed, so a record of a solve that used one is a new ask. +- **`solutions(group=...)` and its siblings no longer fail on a path without an endpoint.** + `all_solutions()` holds an empty placeholder for a path that went to infinity or failed, and + projecting one onto a variable group raised `CoordinatesOfGroup: point is shorter than the + system's variable structure implies`. The placeholder is passed through empty, so the list + keeps one entry per path, aligned with `solution_metadata()`. - **A fast parameter sweep no longer dies after 25 recorded solves** (ADR-0066). Every recorded solve that found no other solver attached to its records directory started a new session, and a session claims a history file named for the current second, the process id, diff --git a/CLAUDE.md b/CLAUDE.md index bb2cc8041..c1a9e3d01 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,6 +44,11 @@ brew install gmp mpfr libmpc eigen@3 eigenpy boost boost-python3 Use `environment.yml` (Ubuntu) or `environment-win.yml` (Windows) with conda/mamba/micromamba. +**Compiler: GCC 15.** CI's Linux C++ jobs run on Ubuntu 26.04 and compile with its GCC 15, and +`environment.yml` pins `gxx==15.3.0` to match, so code that builds locally builds in CI. GCC 15 +is stricter than 13 (e.g. about headers that used to arrive indirectly); when moving the +compiler, the build directory must be recreated -- CMake will not switch compilers in place. + ## Running Tests ### C++ Tests (Boost.Test) @@ -99,6 +104,18 @@ through `tools/refresh_doc_artifacts.py`. Mind the flags: the default (no flags **timing benchmark tables**, not plots -- to redraw one figure use `python tools/refresh_doc_artifacts.py --plots --only `. +How the plot refresh works, so a new plot script fits it: +- The tool runs each script **with its image folder as the working directory**, so a script + saves with plain filenames (`fig.savefig('name.svg')`) and needs no path handling. A new + figure needs an entry in the tool's `_plots()` manifest. +- A plot **fails** if its image folder does not exist, or if any image its entry lists was not + written during the run -- a clean exit alone is not success. +- Solve records go under the docs' gitignored build output, never into `python/docs/source`: + the refresh records each plot to `python/docs/build/refresh_records//`, and the doctest + build (via `conf.py`) to `python/docs/build/doctest_records/`. Both are **emptied before each + run**: records kept from an earlier run would *recall* a solve instead of computing it, which + draws a figure from an older library and leaves path observers with nothing to see. + ## Architecture The project has three layers, built in order: @@ -109,7 +126,7 @@ The project has three layers, built in order: - `trackers/` -- Path tracking (fixed-precision and adaptive-precision trackers, predictors, Newton correctors) - `endgames/` -- Power series and Cauchy endgames for singular endpoint handling - `nag_algorithms/` -- Higher-level algorithms (zero-dim solve; numerical irreducible decomposition is *framework scaffolding* -- not yet implemented, its `Solve()` throws) - - `records/` -- The structured output directory (record schema `b2rec/1`, spec at `docs/records/b2rec-1.md`): durable, self-describing run records with full provenance and resume-by-recall. `OutputDirectory` = append-only JSONL `history/` + content-addressed `definitions/` under kind folders (`systems/`, `configs/`, `givens/`, ...); `README.txt`/`INDEX.txt`/`results.json` are derived, rebuildable views, never truth. Solvers write through the emission seam in `nag_algorithms` (`RecordTo(...)` or the ambient `BERTINI_RECORDS_DIR`); `bertini.solve` is *ensure-answered* -- paths already recorded for an identical ask (system digest + settings digest + seed) are **recalled**, not recomputed. See ADR-0042..0047 and "Persistent digests" below. + - `records/` -- The structured output directory (record schema `b2rec/1`, spec at `docs/records/b2rec-1.md`): durable, self-describing run records with full provenance and resume-by-recall. `OutputDirectory` = append-only JSONL `history/` + content-addressed `definitions/` under kind folders (`systems/`, `configs/`, `givens/`, ...); `README.txt`/`INDEX.txt`/`results.json` are derived, rebuildable views, never truth. Solvers write through the emission seam in `nag_algorithms` (`RecordTo(...)` or the ambient `BERTINI_RECORDS_DIR`); `bertini.solve` is *ensure-answered* -- paths already recorded for an identical ask (system digest + settings digest + seed) are **recalled**, not recomputed. See ADR-0042..0047 and "Persistent digests" below. A records **session lasts the whole process**: `OutputDirectory::Shared()` keeps one instance -- one history file -- per directory per process however many solves attach, and a directory's files close as soon as no solve is using it (ADR-0066). Do not return it to weak ownership: a parameter sweep then claims a session file per solve and dies after 25 in one second. - `io/parsing/` -- Boost.Spirit Qi parsers for classic Bertini input format. `io/json_writer.hpp` renders a System's parts (variable groups, functions, blocks, patches) as JSON for the records archive -- classic syntax is an INPUT/compat format only (it cannot express block structure) and never appears inside records. - `blackbox/` -- CLI executable entry point (CMake target `bertini2_exe`, binary named `bertini2`) @@ -158,14 +175,15 @@ Rules that follow: - **GMP/MPFR/MPC** -- Arbitrary-precision arithmetic (found via custom CMake modules in `cmake/`) - **Eigen 3** -- Linear algebra. **Not** pinned in cmake (`find_package(Eigen3)`, no version floor). In practice the version is coupled to the eigenpy build: the wheel CI builds **eigen 3.4.0** and then builds eigenpy against it (a dev env may use newer, e.g. `eigen=5.0.1`). Newer Eigen is welcome -- we *want* upstream improvements -- but it must be matched by an eigenpy built against the same Eigen (they share Eigen types across the binding ABI). - **Boost** (serialization, filesystem, log, graph, regex, timer, chrono, thread, unit_test_framework, python) -- no minimum version pinned in cmake; `boost_system` is conditionally linked for Boost < 1.89 (header-only from 1.89). Boost.Python is ABI-locked to one CPython version, so it is built **per target Python** -- but this now happens **once, up front, in the prebuilt CI deps** (Linux image / macOS tarballs, ADR-0049), *not* recompiled in every wheel run. -- **eigenpy** -- Eigen/NumPy bridge for Python bindings. **Prebuilt** into the CI deps (image/tarballs), not built-from-source per run, against the chosen Eigen -- eigenpy and bertini must use the *same* Eigen. The version is single-sourced in `.github/ci-deps-versions.env` (`EIGENPY_VERSION`, currently `3.13.0`); eigenpy >= 3.13 sets the Python floor (>= 3.10). +- **eigenpy** -- Eigen/NumPy bridge for Python bindings. **Prebuilt** into the CI deps (image/tarballs), not built-from-source per run, against the chosen Eigen -- eigenpy and bertini must use the *same* Eigen. The version is single-sourced in `.github/ci-deps-versions.env` (`EIGENPY_VERSION`, currently `3.13.0`); eigenpy >= 3.13 sets the Python floor (>= 3.10). **A wheel is not ABI-coupled to a user's own eigenpy:** `auditwheel`/`delocate` bundle private, renamed copies of eigenpy and Boost.Python (`bertini2.libs/libeigenpy-.so`, `libboost_python3XX-.so...`), and Python loads extensions `RTLD_LOCAL`, so a PyPI `eigenpy` in the same process loads its own copies beside ours (verified 2026-09-28: both import orders, no warnings, arrays pass both ways). Only numpy arrays cross between the two; another library's Boost.Python objects do not convert (a clean `TypeError`). The coupling that does exist is for **source builds** against shared libraries (conda, Homebrew), which must use the same eigenpy and Eigen as the environment. - **jrl-cmakemodules** -- CMake helper macros (auto-fetched via FetchContent if not found) ## Build System Notes - The root `CMakeLists.txt` uses `jrl-cmakemodules` (fetched automatically). It currently only adds `core/` as a subdirectory; `python_bindings/` and `python/` subdirectory calls are commented out (the wheel build via scikit-build-core handles them). - `pyproject.toml` configures scikit-build-core: wheel packages from `python/bertini/`, build dir is `bld/`. -- Cross-platform: Linux builds in a **custom prebuilt manylinux image** (`ghcr.io/bertiniteam/b2-manylinux-deps`, ADR-0049) + `auditwheel`, one image per architecture -- x86_64 on `ubuntu-latest`, aarch64 on native `ubuntu-24.04-arm` runners (tag suffix `-aarch64`); both are full matrix members (C++ tests + every wheel); macOS uses Homebrew + **prebuilt Boost/eigenpy tarballs** (the `ci-deps` release); Windows uses conda-forge (which already ships prebuilt Boost/eigenpy) + clang-cl (MSVC has template compilation issues). +- **Developing: install b2 into the environment for real** -- `pip install --no-deps --no-build-isolation -e . -Cbuild-dir=build "-Cbuild.targets=_pybertini;bertini2_exe"` -- so `importlib.metadata.version("bertini2")` reports `VERSION` and the installed `_pybertini` is the one imports load. One build directory serves both the editable install and direct `cmake --build build` work; after rebuilding bindings, `cmake --install build --prefix /bertini` refreshes the installed copy. Do not hand-make a `.pth` or a stub `dist-info` to point at the source tree: nothing keeps its version current. +- Cross-platform: Linux builds in a **custom prebuilt manylinux image** (`ghcr.io/bertiniteam/b2-manylinux-deps`, ADR-0049) + `auditwheel`, one image per architecture -- x86_64 on `ubuntu-26.04`, aarch64 on native `ubuntu-26.04-arm` runners (every workflow names its Ubuntu version rather than `ubuntu-latest`, so an image change is a deliberate edit; the Linux C++ test job compiles with the runner's GCC 15) (tag suffix `-aarch64`); both are full matrix members (C++ tests + every wheel); macOS uses Homebrew + **prebuilt Boost/eigenpy tarballs** (the `ci-deps` release); Windows uses conda-forge (which already ships prebuilt Boost/eigenpy) + clang-cl (MSVC has template compilation issues). - `-Werror` is disabled globally. `-pedantic` is stripped from flags. ## CI/CD @@ -174,7 +192,7 @@ Rules that follow: - `.github/workflows/doc_lint.yml` -- A cheap wheel-free doc-correctness gate that **the build matrix depends on** (it runs first; if it fails, nothing compiles). Two parallel jobs, one per language — **run both locally before pushing docs/source changes**, or CI will bounce the whole build: - **C++ (`bash tools/doclint.sh`)** — needs `doxygen` on PATH (`brew install doxygen`). Two passes: (1) *correctness* — `@param` names must match signatures, no doc blocks on removed signatures, no unresolved `\ref`/`\cite` (zero tolerance); (2) *undocumented ratchet* — the count in `tools/doc_undocumented_baseline.txt` may only **decrease** (currently `0`, so **every new public C++ entity — including each struct data member — needs a Doxygen comment**, e.g. `///< ...`). `bash tools/doclint.sh --update-baseline` locks in a reduction. - **Python (`python tools/py_doclint.py`)** — pure-stdlib `ast`, no build/wheel/pip. Same two-pass shape over `python/bertini/**/*.py`: (1) *correctness* — a numpydoc `Parameters` entry naming a parameter absent from the signature fails (zero tolerance); (2) *undocumented ratchet* — undocumented public entities (module / public class / public function / public method; dunders excluded) counted against `tools/py_doc_undocumented_baseline.txt`, may only **decrease**. `python tools/py_doclint.py --update-baseline` locks in a reduction. Only lints the pure-Python layer; native `_pybertini` docstrings (Boost.Python strings in `python_bindings/src/*_export.cpp`) are invisible to a wheel-free tool. -- `.github/workflows/publish.yml` -- Publishes to TestPyPI on `develop` push, PyPI on version tags (`v*.*.*`) with Sigstore signing and GitHub Releases. +- `.github/workflows/publish.yml` -- Publishes to TestPyPI on `develop` push, PyPI on version tags (`v*.*.*`) with Sigstore signing and GitHub Releases. The GitHub release's notes are CHANGELOG.md's top block (between the first two lines of 79 underscores), and a final release **fails in its first job, before anything builds**, unless that block's heading is `## [X.Y.Z] - YYYY-MM-DD` for the tag's version -- so date the entry (replace `unreleased`) before tagging. `tools/release_notes.py` does the extraction and the check; pull requests do not run it. ### Linux wheel test coverage diff --git a/core/include/bertini2/eigen_extensions.hpp b/core/include/bertini2/eigen_extensions.hpp index aeb480715..c2d26abfc 100644 --- a/core/include/bertini2/eigen_extensions.hpp +++ b/core/include/bertini2/eigen_extensions.hpp @@ -587,6 +587,12 @@ namespace bertini { factors exactly as before, so seeded square matrices are unchanged; non-square ones consume fewer random draws than the old recipe and therefore differ for the same seed. + The seed is a matrix of RandomOrthonormalSeed entries, not of units: a real unit is a sign, + and a tall matrix of signs has only 2^thin distinct rows, so the Q built from it has only + that many distinct rows -- the wide real result then has repeated columns, and a deflated + system randomized with it comes out singular. Real seeds are continuous draws; complex + seeds are the units they always were. + \param rows The number of rows of the returned matrix. \param cols The number of columns of the returned matrix. \tparam NumberType the (complex) number type to fill the matrix with. @@ -599,7 +605,8 @@ namespace bertini { using std::min; const unsigned int tall = max(rows, cols); const unsigned int thin = min(rows, cols); - Mat seed = RandomOfUnits(tall, thin); + Mat seed = Mat(tall, thin).unaryExpr( + [](NumberType const& /*x*/) { return RandomOrthonormalSeed(); }); Eigen::HouseholderQR > qr(seed); Mat Q = qr.householderQ() * Mat::Identity(tall, thin); // tall x thin, orthonormal columns if (rows >= cols) diff --git a/core/include/bertini2/endgames/cauchy.hpp b/core/include/bertini2/endgames/cauchy.hpp index c86e202f7..e0b459d03 100644 --- a/core/include/bertini2/endgames/cauchy.hpp +++ b/core/include/bertini2/endgames/cauchy.hpp @@ -1304,14 +1304,14 @@ class CauchyEndgame : static_cast(abs(this->LatestTime() - target_time)) < GetCauchySettings().cycle_cutoff_time; - if (in_operating_zone - && approx_error < this->FinalTolerance() - && same_cycle_count >= GetCauchySettings().num_consecutive_same_cycle_number) - { - NotifyObservers(Converged(*this)); - return SuccessCode::Success; - } - + // SECURITY BEFORE ACCEPTANCE, the order the power series endgame uses. At + // security level 0, two consecutive endpoint approximations whose dehomogenized + // infinity norm is above max_norm truncate the path, whether or not the second + // would also have been accepted. Asked the other way round, a path that + // converged to a finite point above the ceiling came back Success here and + // SecurityMaxNormReached from the power series endgame -- the two endgames must + // differ in how they estimate the root and in nothing else. + // // Security truncation happens ONLY in the operating zone. Outside it the // loop may encircle a pole or branch point and the Cauchy mean is garbage -- // its norm means nothing, and truncating on it killed clean convergent paths @@ -1345,6 +1345,23 @@ class CauchyEndgame : norm_of_dehom_prev = RealT(0); // out of zone: restart the count } + // An approximation above the ceiling is never ACCEPTED at security level 0: it is + // the first of the two that truncate, or the second. Without this the count above + // (armed rounds only) could still be at one when acceptance, which needs a single + // in-zone round, let the path through as a Success above max_norm. + bool const above_ceiling = this->SecuritySettings().level <= 0 + && this->BeyondSecurityMaxNorm( + this->GetSystem().InfinityNormOfDehomogenized(latest_approx)); + + if (in_operating_zone + && !above_ceiling + && approx_error < this->FinalTolerance() + && same_cycle_count >= GetCauchySettings().num_consecutive_same_cycle_number) + { + NotifyObservers(Converged(*this)); + return SuccessCode::Success; + } + prev_approx = latest_approx; auto advance_success = AdvanceTime(target_time); @@ -1477,16 +1494,9 @@ class CauchyEndgame : static_cast(abs(this->LatestTime() - this->AtActivePrecisionScalar(target_time))) < GetCauchySettings().cycle_cutoff_time; - if (in_operating_zone - && this->approximate_error_ < this->FinalTolerance() - && same_cycle_count >= GetCauchySettings().num_consecutive_same_cycle_number) - { - NotifyObservers(Converged(*this)); - return SuccessCode::Success; - } - - // security truncation ONLY in the operating zone (a trustworthy mean); an - // out-of-zone round restarts the two-consecutive count. See RunImpl. + // security BEFORE acceptance, as in the power series endgame; truncation ONLY in + // the operating zone (a trustworthy mean); an out-of-zone round restarts the + // two-consecutive count. See RunImpl. if (this->SecuritySettings().level <= 0) { if (in_operating_zone || below_cycle_cutoff) @@ -1512,6 +1522,20 @@ class CauchyEndgame : norm_of_dehom_prev = RealT(0); // out of zone: restart the count } + // never accept an approximation above the ceiling at security level 0 (see RunImpl) + bool const above_ceiling = this->SecuritySettings().level <= 0 + && this->BeyondSecurityMaxNorm( + this->GetSystem().InfinityNormOfDehomogenized(this->final_approximation_)); + + if (in_operating_zone + && !above_ceiling + && this->approximate_error_ < this->FinalTolerance() + && same_cycle_count >= GetCauchySettings().num_consecutive_same_cycle_number) + { + NotifyObservers(Converged(*this)); + return SuccessCode::Success; + } + this->previous_approximation_ = this->final_approximation_; auto advance_code = AdvanceTimeAMP(target_time); diff --git a/core/include/bertini2/num_traits.hpp b/core/include/bertini2/num_traits.hpp index e876c80da..1df960948 100644 --- a/core/include/bertini2/num_traits.hpp +++ b/core/include/bertini2/num_traits.hpp @@ -264,6 +264,31 @@ namespace bertini { return real_mp( records::DrawSymmetricDouble() < 0.0 ? -1 : 1 ); } + + /// \brief The entry a random orthonormal matrix is seeded with before its QR factorization: + /// a unit draw for the complex types. + /// + /// The QR launders the seed away, so any seed whose columns are generic will do -- but the + /// seed must BE generic. The real unit is +1 or -1, and a tall matrix of signs has only + /// 2^cols distinct rows, so its Q has that many distinct rows too: an 8 x 4908 real + /// randomization matrix built from it had 256 distinct columns, and randomizing a deflated + /// system with it produced singular squares. Real seeds are therefore drawn from the + /// symmetric interval instead (see the specialization below); the complex draw, a unit + /// modulus at a uniform angle, is already continuous and is unchanged. + template + inline + NumberType RandomOrthonormalSeed() + { + return RandomUnit(); + } + + /// \brief A real orthonormal seed entry: uniform on [-1, 1], never a bare sign. + template <> + inline + real_mp RandomOrthonormalSeed() + { + return real_mp( records::DrawSymmetricDouble() ); + } }// re: namespace bertini diff --git a/core/test/classes/eigen_test.cpp b/core/test/classes/eigen_test.cpp index d5d4b2ab4..358ca47f4 100644 --- a/core/test/classes/eigen_test.cpp +++ b/core/test/classes/eigen_test.cpp @@ -664,4 +664,34 @@ BOOST_AUTO_TEST_CASE(a_very_wide_draw_is_orthonormal) BOOST_CHECK_SMALL((M * M.adjoint() - Mat::Identity(4, 4)).norm(), real_mp("1e-25")); } +BOOST_AUTO_TEST_CASE(a_wide_real_draw_has_distinct_columns) +{ + // A real unit is a sign, and a tall seed matrix of signs has only 2^thin distinct rows, so + // the Q factored from it had that many distinct rows and the wide real result repeated + // its columns (3 x 5: three distinct; 8 x 4908: 256). Randomizing a deflated system with + // such a matrix gave a singular square at a point of full rank. The real seed is a + // continuous draw now, so no two columns coincide, or are opposite, and the rows are still + // orthonormal. + using namespace bertini; + DefaultPrecision(30); + + auto check = [](unsigned rows, unsigned cols) + { + Mat const M = RandomConjugateOrthonormalMatrix(rows, cols); + BOOST_REQUIRE_EQUAL(M.rows(), static_cast(rows)); + BOOST_REQUIRE_EQUAL(M.cols(), static_cast(cols)); + BOOST_CHECK_SMALL((M * M.transpose() - Mat::Identity(rows, rows)).norm(), + real_mp("1e-25")); + for (Eigen::Index j = 0; j < M.cols(); ++j) + for (Eigen::Index k = j + 1; k < M.cols(); ++k) + { + BOOST_CHECK_GT((M.col(j) - M.col(k)).norm(), real_mp("1e-6")); + BOOST_CHECK_GT((M.col(j) + M.col(k)).norm(), real_mp("1e-6")); + } + }; + check(3, 5); + check(2, 6); + check(8, 300); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/core/test/nag_algorithms/zero_dim.cpp b/core/test/nag_algorithms/zero_dim.cpp index a6e8d744c..75227df9a 100644 --- a/core/test/nag_algorithms/zero_dim.cpp +++ b/core/test/nag_algorithms/zero_dim.cpp @@ -674,6 +674,83 @@ BOOST_AUTO_TEST_CASE(infinite_solutions_at_infinity) BOOST_CHECK_EQUAL(zd.InfiniteSolutions(false).size(), zd.InfiniteSolutions(true).size()); } +// The security ceiling is ONE rule, and both endgames apply it: at security level 0, two +// consecutive endpoint approximations whose dehomogenized infinity norm is above max_norm +// truncate the path. {x^2 - 100, y - 2x} has two FINITE roots, (+-10, +-20); with the ceiling +// set to 5 they are above it. The Cauchy endgame used to test acceptance before security, so +// it returned Success for exactly the roots the power series endgame truncated; the two endgames +// differ in how they estimate the root and in nothing else. (The ceiling is lowered rather than +// the roots raised because the double precision tracker is not reliable on a badly scaled +// system: it runs out of steps before the endgame on x^2 = 1e6 always, and on x^2 = 1e4 in +// some draws.) +struct CeilingOutcome +{ + unsigned truncated = 0; ///< paths ended by SecurityMaxNormReached + unsigned succeeded = 0; ///< paths ended Success at a finite point + std::string codes; ///< every path's (pre-endgame, endgame) codes, for the failure message +}; + +template +CeilingOutcome SolveLargeFiniteRoots(double max_norm, int level) +{ + using namespace bertini; + + auto x = node::Variable::Make("x"); + auto y = node::Variable::Make("y"); + System sys; + sys.AddFunction(pow(x, 2) - 100); + sys.AddFunction(y - 2*x); + sys.AddVariableGroup(VariableGroup{x, y}); + + auto zd = algorithm::ZeroDimSolver(sys); + zd.DefaultSetup(); + endgame::SecurityConfig sec; + sec.level = level; + sec.max_norm = max_norm; + zd.GetEndgame().Set(sec); + zd.Solve(); + + CeilingOutcome out; + for (auto const& m : zd.SolutionMetadata()) + { + if (m.endgame_success_code == SuccessCode::SecurityMaxNormReached) + ++out.truncated; + else if (m.endgame_success_code == SuccessCode::Success && m.is_finite) + ++out.succeeded; + out.codes += " (" + std::to_string(static_cast(m.pre_endgame_success_code)) + ", " + + std::to_string(static_cast(m.endgame_success_code)) + + (m.is_finite ? ", finite)" : ", not finite)"); + } + return out; +} + +template +void CheckSecurityCeiling(std::string const& which) +{ + // above the ceiling: both paths truncated + auto const above = SolveLargeFiniteRoots(5, 0); + BOOST_CHECK_MESSAGE(above.truncated == 2u && above.succeeded == 0u, + which << ": roots above max_norm must be truncated at security level 0;" << above.codes); + // under the default ceiling: both kept + auto const under = SolveLargeFiniteRoots(1e4, 0); + BOOST_CHECK_MESSAGE(under.truncated == 0u && under.succeeded == 2u, + which << ": roots under max_norm must be kept;" << under.codes); + // security level 1 does not truncate, whatever the ceiling + auto const level_one = SolveLargeFiniteRoots(5, 1); + BOOST_CHECK_MESSAGE(level_one.truncated == 0u && level_one.succeeded == 2u, + which << ": security level 1 must not truncate;" << level_one.codes); +} + +BOOST_AUTO_TEST_CASE(security_ceiling_is_the_same_rule_in_both_endgames) +{ + using Dbl = bertini::tracking::DoublePrecisionTracker; + using Amp = bertini::tracking::AMPTracker; + CheckSecurityCeiling::PSEG>("double, power series"); + CheckSecurityCeiling::Cauchy>("double, Cauchy"); + CheckSecurityCeiling::PSEG>("adaptive, power series"); + CheckSecurityCeiling::Cauchy>("adaptive, Cauchy"); +} + // multiplicity_representative: a multiplicity-m solution arrives as m coincident endpoints, and // the solver must mark exactly ONE of them the representative (the rest false), so a consumer can // collapse the cluster to one row. {x^2, y^2} has a single solution (0,0) of multiplicity 4. diff --git a/docs/adr/0067-the-endgames-differ-in-root-estimation-only.md b/docs/adr/0067-the-endgames-differ-in-root-estimation-only.md new file mode 100644 index 000000000..9c618c39e --- /dev/null +++ b/docs/adr/0067-the-endgames-differ-in-root-estimation-only.md @@ -0,0 +1,62 @@ +# ADR-0067: The endgames differ in root estimation only; security is asked before acceptance + +**Status:** Accepted +**Date:** 2026-09-29 + +## Context + +The security ceiling is one rule: at security level 0, two consecutive endpoint approximations +whose dehomogenized infinity norm is above `Security.max_norm` truncate the path with +`SecurityMaxNormReached`. It is a statement about where the path is going, and it is made +before anyone asks whether the path has converged. + +The power series endgame applies it that way: after every extrapolation, before its loop +re-tests convergence. The Cauchy endgame did not. Its loop tested acceptance first and +returned `Success`, and consulted the ceiling only for rounds that had not been accepted. So a +path that converged to a finite point above the ceiling had two answers: + +| `x^2 = 1e8, y = 2x`, default `max_norm` 1e4 | finite solutions | codes | +|---|---|---| +| power series | 0 of 2 | `SecurityMaxNormReached` | +| Cauchy | 2 of 2 | `Success` | + +Since 4.0 the power series endgame is the default (ADR-0058), so the difference became what a +user sees when they switch endgames: solutions appear or vanish. It was found by the cellular +decomposition port, whose contract test for the accuracy estimates solves exactly such a system. + +Moving the security block above the acceptance test was not enough. Cauchy's count of +consecutive approximations runs over rounds in the operating zone only, because outside it the +Cauchy mean is not an approximation of anything (truncating on it lost genuine cyclic-6 +solutions, 156 -> 155). Acceptance, though, needs a single in-zone round. A path whose first +in-zone round was already above the ceiling had a count of one and was accepted. + +## Decision + +The two endgames differ in how they estimate the root, and in nothing else. For security: + +1. **Security is asked before acceptance**, in both Cauchy loops (`RunImpl` and the adaptive + one), as in the power series endgame. +2. **An approximation above the ceiling is never accepted** at security level 0. It is the + first of the two that truncate, or the second. +3. Cauchy's count still runs over **rounds in the operating zone** (or below + `cycle_cutoff_time`), and still watches the larger of the endpoint norm and the loop floor. + Those are properties of how Cauchy estimates: an out-of-zone mean is not an endpoint + approximation, so it neither counts toward truncation nor can be accepted. + +The user-visible invariant: **at security level 0, no endgame returns `Success` for an endpoint +above `max_norm`.** At level 1 neither endgame truncates. + +## Consequences + +- A solve whose solutions lie above `max_norm` needs the ceiling raised or security level 1, + under either endgame. That was already true of the default endgame; it is now true of both. +- Do not move Cauchy's acceptance test back above the security block, and do not drop the + `!above_ceiling` clause from it: either change brings back `Success` above the ceiling. +- Do not remove the operating-zone condition from Cauchy's count in the name of making the two + endgames textually alike. It is what keeps cyclic-6 at 156 solutions. +- A new endgame applies the same rule in the same place. +- Pinned by `security_ceiling_is_the_same_rule_in_both_endgames` + (`core/test/nag_algorithms/zero_dim.cpp`), over both trackers and both endgames, and by + `python/test/zero_dim/security_ceiling_test.py`. The C++ test lowers the ceiling instead of + raising the roots, because the double precision tracker is not reliable on a badly scaled + system and the test is not about that. diff --git a/docs/adr/README.md b/docs/adr/README.md index cb1046fc7..fe3c8a533 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -83,3 +83,4 @@ Each ADR follows the template: | [0064](0064-plotting-converts-at-the-axis-and-refuses-complex.md) | Plotting converts AT THE AXIS and nowhere else: a matplotlib units converter hands `float64` to an axis (the one place dropping digits is right), while image data, contour levels and marker sizes are cast explicitly with `.astype(float)`; complex data is REFUSED at the axis rather than drawn, because numpy's `.real`/`.imag` on a user complex dtype return the array and zeros, so `scatter(z.real, z.imag)` would silently put every point on the x axis -- and the refusal catches it precisely because `.real` of a `complex_mp` array is still `complex_mp`; don't make the complex converter plot real parts, and don't register a safe/same-kind cast to `float64` to satisfy `imshow`/`contour`/`boxplot`, because that same registration would silently downcast ordinary arithmetic | Python bindings / plotting | | [0065](0065-printing-is-presentation-and-lives-in-python.md) | Printing an object is PRESENTATION and lives in `bertini/_repr.py`, attached at import (a Boost.Python class accepts the assignment, and `str()` follows `repr()`), not in the bindings -- a C++ `operator<<` serves a different audience and can be written when one is wanted; machinery is described in angle brackets (``) because none of it could be rebuilt from text, while configs and metadata keep the call form; a tracker's description is LIVE, since the moment you print one is inside an observer callback; no repr may raise, because a throwing repr turns a debugging `print` into a traceback; a test fails if any public class inherits the default repr | Python bindings / printing | | [0066](0066-a-records-session-is-the-process.md) | A records session is the process: `OutputDirectory::Shared()` holds each instance for the life of the process (replacing it only for a deleted directory or a forked child), so a parameter sweep writes one history file instead of claiming one per solve and exhausting the 25 names a second allows; an idle instance holds no open files, and results files are capped at 16 open | Records | +| [0067](0067-the-endgames-differ-in-root-estimation-only.md) | The endgames differ in root estimation ONLY: the security ceiling (two consecutive endpoint approximations above `max_norm`, at security level 0) is asked BEFORE acceptance in the Cauchy endgame as it is in the power series one, and an approximation above the ceiling is never accepted -- Cauchy had returned `Success` for finite roots above the ceiling that power series truncated; Cauchy's count still runs over operating-zone rounds only, since an out-of-zone mean is not an endpoint approximation (that condition keeps cyclic-6 at 156); don't move acceptance back above security, don't drop the `!above_ceiling` clause | Core / endgames | diff --git a/environment.yml b/environment.yml index b41a1ce9f..1e6c7f7a7 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - gxx==13.3.0 + - gxx==15.3.0 - numpy - gmp - mpfr diff --git a/python/bertini/nag_algorithm/__init__.py b/python/bertini/nag_algorithm/__init__.py index fb5338eb7..c9e4c7161 100644 --- a/python/bertini/nag_algorithm/__init__.py +++ b/python/bertini/nag_algorithm/__init__.py @@ -369,7 +369,10 @@ def getter(self, *args, group=None, **kwargs): if group is None: return points sysm = self.target_system() - return [sysm.coordinates_of(p, group) for p in points] + # all_solutions() holds an empty placeholder for a path that reached no endpoint (at + # infinity, or failed); there is nothing to project, and it stays empty so the list + # keeps one entry per path, aligned with solution_metadata() + return [sysm.coordinates_of(p, group) if len(p) else p for p in points] getter.__name__ = getattr(native, '__name__', 'solutions') getter.__doc__ = (getattr(native, '__doc__', '') or '') + ( "\n\ngroup=: project each returned point onto that " diff --git a/python/test/classes/random_matrix_test.py b/python/test/classes/random_matrix_test.py index 50d1339a8..e49203e1a 100644 --- a/python/test/classes/random_matrix_test.py +++ b/python/test/classes/random_matrix_test.py @@ -51,6 +51,37 @@ def test_orthonormal_real_is_real_orthogonal(): assert np.allclose(M @ M.T, np.eye(4), atol=1e-10) +def _distinct_columns(M): + return len({tuple(np.round(M[:, j], 10)) for j in range(M.shape[1])}) + + +def test_wide_real_orthonormal_matrix_has_distinct_columns(): + """A real orthonormal matrix used to be QR-factored from a seed of signs (+-1), and a tall + matrix of signs has only 2^thin distinct rows: a 3 x 5 real draw had 3 distinct columns, an + 8 x 4908 one had 256, and randomizing a deflated system down to square with such a matrix + produced a SINGULAR square system at a point where the deflated Jacobian had full rank.""" + pb.random.set_random_seed(11) + M = _as_complex(pb.random_matrix(3, 5, real=True)) + assert np.allclose(M.imag, 0) + assert np.allclose(M @ M.T, np.eye(3), atol=1e-10) + assert _distinct_columns(M.real) == 5 + # the deflation-sized case, and a check that no column is another's negative either + R = _as_complex(pb.random_matrix(8, 4908, real=True)).real + assert _distinct_columns(R) == 4908 + assert _distinct_columns(np.hstack([R, -R])) == 2 * 4908 + + +def test_wide_real_orthonormal_matrix_randomizes_a_full_rank_jacobian_to_full_rank(): + """The consumer that found it: R @ J must stay rank 3 for a 5 x 3 J of full rank.""" + pb.random.set_random_seed(42) + J = np.array([[0.0, 0.0, 0.0], [4 / 3, 0.0, 0.0], [0.0, -2.0, 0.0], [0.0, 0.0, 0.0], + [0.31, -0.57, 1.0]]) + for _ in range(20): + R = _as_complex(pb.random_matrix(3, 5, real=True)).real + s = np.linalg.svd(R @ J, compute_uv=False) + assert s[-1] > 1e-6 * s[0], s + + def test_bounded_modulus_entries(): pb.random.set_random_seed(3) M = pb.random_matrix(2, 3, orthonormal=False) # complex, bounded modulus diff --git a/python/test/zero_dim/security_ceiling_test.py b/python/test/zero_dim/security_ceiling_test.py new file mode 100644 index 000000000..68e971a00 --- /dev/null +++ b/python/test/zero_dim/security_ceiling_test.py @@ -0,0 +1,69 @@ +"""The security ceiling is one rule, applied alike by both endgames. + +At security level 0, two consecutive endpoint approximations whose dehomogenized infinity norm is +above ``max_norm`` truncate the path. The Cauchy endgame used to test acceptance before security, +so it returned Success for finite roots above the ceiling, which the power series endgame +truncated. The two endgames differ in how they estimate the root and in nothing else. + +``{x^2 - 100, y - 2x}`` has the finite roots ``(+-10, +-20)``; the ceiling is lowered to 5 to put +them above it, because the double precision tracker is not reliable on a badly scaled system (it +runs out of steps before the endgame on ``x^2 = 1e6`` always, and on ``x^2 = 1e4`` in some +draws). The last test is the case as a user meets it: roots at ``(+-1e4, +-2e4)`` and the +default ceiling, under adaptive precision. +""" + +import pytest + +import bertini as pb +from bertini import ZeroDimSolver + +ENDGAMES = ['powerseries', 'cauchy'] +PRECISION_MODELS = ['adaptive', 'double'] + + +def _solve(endgame, mptype, scale=100, **settings): + x, y = pb.variables(['x', 'y']) + system = pb.System() + system.add_variable_group([x, y]) + system.add_functions([x ** 2 - pb.coefficient(scale), y - pb.coefficient(2) * x]) + solver = ZeroDimSolver(system, endgame=endgame, mptype=mptype) + if settings: + solver.update(**settings) + solver.solve(show_progress=False) + codes = [str(m.endgame_success_code) for m in solver.solution_metadata()] + return codes, solver.solutions() + + +@pytest.mark.parametrize('mptype', PRECISION_MODELS) +@pytest.mark.parametrize('endgame', ENDGAMES) +def test_roots_above_the_ceiling_are_truncated(endgame, mptype): + codes, finite = _solve(endgame, mptype, max_norm=5) + assert codes == ['SecurityMaxNormReached'] * 2 + assert len(finite) == 0 + + +@pytest.mark.parametrize('mptype', PRECISION_MODELS) +@pytest.mark.parametrize('endgame', ENDGAMES) +def test_roots_under_the_ceiling_are_kept(endgame, mptype): + codes, finite = _solve(endgame, mptype) # default max_norm, 1e4 + assert codes == ['Success'] * 2 + assert len(finite) == 2 + assert sorted(round(abs(complex(p[0]))) for p in finite) == [10, 10] + + +@pytest.mark.parametrize('mptype', PRECISION_MODELS) +@pytest.mark.parametrize('endgame', ENDGAMES) +def test_security_level_one_does_not_truncate(endgame, mptype): + codes, finite = _solve(endgame, mptype, max_norm=5, level=1) + assert codes == ['Success'] * 2 + assert len(finite) == 2 + + +@pytest.mark.parametrize('endgame', ENDGAMES) +def test_roots_above_the_default_ceiling_are_truncated(endgame): + codes, finite = _solve(endgame, 'adaptive', scale=100000000) # roots at (+-1e4, +-2e4) + assert codes == ['SecurityMaxNormReached'] * 2 + assert len(finite) == 0 + codes, finite = _solve(endgame, 'adaptive', scale=100000000, max_norm=1e6) + assert codes == ['Success'] * 2 + assert len(finite) == 2 diff --git a/python/test/zero_dim/solutions_ui_test.py b/python/test/zero_dim/solutions_ui_test.py index b22214536..242eb9502 100644 --- a/python/test/zero_dim/solutions_ui_test.py +++ b/python/test/zero_dim/solutions_ui_test.py @@ -151,6 +151,29 @@ def test_group_projection_of_solutions(): assert xs == [-1, 1] +def test_group_projection_keeps_placeholders_for_paths_without_an_endpoint(): + # x^2 - 1, x*y - 1 over one group [x, y]: total degree 4, two finite roots (+-1, +-1) and + # two paths to infinity. all_solutions() has one entry per path, with an EMPTY + # placeholder for a path that reached no endpoint; group= must leave those alone rather + # than fail on them. (A second group -- x and y two-homogeneous -- would draw an mhom + # start with only two paths and nothing at infinity.) + x, y = pb.variables(['x', 'y']) + sys, solver = _solve([x * x - 1, x * y - 1], [[x, y]]) + full = solver.all_solutions() + md = solver.solution_metadata() + assert len(full) == 4 + assert sum(1 for m in md if m.is_finite) == 2 + g0 = solver.all_solutions(group=0) + assert len(g0) == 4 # still one per path + for p, q, m in zip(full, g0, md): + if m.is_finite: + assert len(p) == 2 and len(q) == 2 # the one group is everything + else: + assert len(p) == 0 and len(q) == 0 # the placeholder, untouched + xs = sorted({round(float(q[0].real)) for q, m in zip(g0, md) if m.is_finite}) + assert xs == [-1, 1] + + def test_group_projection_in_to_dataframe(): pd = pytest.importorskip('pandas') x, y = pb.variables(['x', 'y']) diff --git a/python_bindings/src/endgame_config_export.cpp b/python_bindings/src/endgame_config_export.cpp index f66e4f640..2cb84d841 100644 --- a/python_bindings/src/endgame_config_export.cpp +++ b/python_bindings/src/endgame_config_export.cpp @@ -35,7 +35,7 @@ namespace bertini{ ; class_("SecurityConfig","Security settings for endgames. Control things like truncation because estimated root is near infinity",init<>()) - .def_readwrite("level", &endgame::SecurityConfig::level,"Turns on or off truncation of paths going to infinity during the endgame. 0 is off, 1 is on.") + .def_readwrite("level", &endgame::SecurityConfig::level,"Whether the endgame truncates a path that appears to be going to infinity. 0 (the default) truncates: two consecutive endpoint approximations whose largest coordinate is above max_norm end the path with SuccessCode.SecurityMaxNormReached, in either endgame. 1 does not truncate, and tracks every path to its end.") .def_readwrite("max_norm", &endgame::SecurityConfig::max_norm,"If on, the norm at which to truncate a path.") ; diff --git a/tools/release_notes.py b/tools/release_notes.py new file mode 100644 index 000000000..0dc9cda4c --- /dev/null +++ b/tools/release_notes.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +"""Extract and check the release notes for a tag: the top block of CHANGELOG.md. + +The release notes are the block between the first two separator lines (79 underscores) of +CHANGELOG.md, HTML comments removed. publish.yml calls this twice for a final release: + + python tools/release_notes.py --tag v4.0.0 # check only + python tools/release_notes.py --tag v4.0.0 --output TEMP_CHANGELOG.md + +The first runs in the release's first job, before anything builds, so a CHANGELOG that is +not ready fails the release in seconds. The second writes the notes the GitHub release +publishes. A check fails when: + +* the separator lines are missing, so there is no top block (the old one-line extraction + returned an empty string here, and the release went out with an empty description); +* the block has no ``## [X.Y.Z] - YYYY-MM-DD`` heading, or its version is not the tag's; +* the heading's date is not a date -- typically ``unreleased``, left from development; +* the block has nothing under its heading; +* the block is longer than a GitHub release description may be. + +Pull requests do not run this: whether the changelog names the next version correctly is a +release-time question. +""" + +import argparse +import re +import sys +from pathlib import Path + +SEPARATOR = "_" * 79 # the line that fences each release's block +GITHUB_BODY_LIMIT = 125_000 # characters GitHub accepts in a release description +_HEADING = re.compile(r"^## \[(?P[^\]]+)\] - (?P.*?)\s*$", re.MULTILINE) +_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +class ReleaseNotesError(Exception): + """The changelog cannot supply release notes for the tag.""" + + +def top_block(changelog_text): + """The block between the first two separator lines, HTML comments removed. + + Parameters + ---------- + changelog_text : str + The whole of CHANGELOG.md. + + Returns + ------- + str + The block, starting at its opening separator line. + """ + text = re.sub("", "", changelog_text, flags=re.DOTALL) + start = text.find(SEPARATOR) + if start < 0: + raise ReleaseNotesError("CHANGELOG.md has no separator line (79 underscores), so there " + "is no top block to publish") + end = text.find(SEPARATOR, start + len(SEPARATOR)) + if end < 0: + raise ReleaseNotesError("CHANGELOG.md has only one separator line; the top block needs a " + "second one to end it") + return text[start:end] + + +def check(block, tag): + """Check that the block is publishable as the release notes for the tag. + + Parameters + ---------- + block : str + The top block, as returned by top_block. + tag : str + The release tag, e.g. ``v4.0.0`` (a leading ``v`` is optional). + + Returns + ------- + tuple of (str, str) + The heading's version and date. + """ + heading = _HEADING.search(block) + if heading is None: + raise ReleaseNotesError("the top block has no '## [X.Y.Z] - YYYY-MM-DD' heading") + version, date = heading.group("version"), heading.group("date") + expected = tag[1:] if tag.startswith("v") else tag + if version != expected: + raise ReleaseNotesError(f"the top block is for {version}, but the tag is {tag}") + if not _DATE.match(date): + raise ReleaseNotesError(f"the top block's heading is dated {date!r}, not YYYY-MM-DD; " + f"date the {version} entry before tagging") + if not block[heading.end():].strip(): + raise ReleaseNotesError(f"the {version} entry has nothing under its heading") + if len(block) > GITHUB_BODY_LIMIT: + raise ReleaseNotesError(f"the {version} entry is {len(block)} characters; a GitHub " + f"release description holds at most {GITHUB_BODY_LIMIT}") + return version, date + + +def main(argv=None): + """Check the release notes for a tag, and optionally write them out.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--tag", required=True, help="the release tag, e.g. v4.0.0") + parser.add_argument("--changelog", default="CHANGELOG.md", help="path to CHANGELOG.md") + parser.add_argument("--output", help="write the release notes to this file") + args = parser.parse_args(argv) + + try: + block = top_block(Path(args.changelog).read_text()) + version, date = check(block, args.tag) + except ReleaseNotesError as problem: + print(f"::error title=Release notes::{problem}") + return 1 + + if args.output: + Path(args.output).write_text(block) + print(f"release notes for {args.tag}: {version}, dated {date}, {len(block)} characters") + print(block) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_release_notes.py b/tools/test_release_notes.py new file mode 100644 index 000000000..9b41c41ea --- /dev/null +++ b/tools/test_release_notes.py @@ -0,0 +1,108 @@ +"""Tests for tools/release_notes.py -- the release-time check on CHANGELOG.md's top block. + +Not collected by the package suite (pyproject testpaths = python/test); run explicitly: + + pytest tools/test_release_notes.py +""" + +import re +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import release_notes as rn # noqa: E402 + +SEP = rn.SEPARATOR +REPO = Path(__file__).resolve().parent.parent + + +def changelog(top_heading, body="- a change\n", older="## [3.9.0] - 2026-01-02\n\n- older\n"): + """A small changelog: a commented example, then two release blocks fenced by separators.""" + return (f"# Changelog\n\n\n\n" + f"{SEP}\n\n{top_heading}\n\n{body}\n{SEP}\n\n{older}") + + +def test_extraction_matches_the_old_one_liner_on_the_real_changelog(): + # publish.yml used `text[start:text.find(sep, start+1)]` after stripping comments; + # for a well-formed changelog the new extraction must publish exactly the same notes + raw = (REPO / "CHANGELOG.md").read_text() + text = re.sub("", "", raw, flags=re.DOTALL) + start = text.find(SEP) + old = text[start:text.find(SEP, start + 1)] + assert rn.top_block(raw) == old + + +def test_a_dated_entry_for_the_tag_passes(): + block = rn.top_block(changelog("## [4.0.0] - 2026-10-01")) + assert rn.check(block, "v4.0.0") == ("4.0.0", "2026-10-01") + assert rn.check(block, "4.0.0") == ("4.0.0", "2026-10-01") # the leading v is optional + + +def test_the_commented_example_is_never_the_top_block(): + block = rn.top_block(changelog("## [4.0.0] - 2026-10-01")) + assert "0.0.0" not in block + assert "4.0.0" in block and "3.9.0" not in block + + +def test_an_undated_entry_fails(): + block = rn.top_block(changelog("## [4.0.0] - unreleased")) + with pytest.raises(rn.ReleaseNotesError, match="unreleased"): + rn.check(block, "v4.0.0") + + +def test_a_version_other_than_the_tag_fails(): + block = rn.top_block(changelog("## [4.0.0] - 2026-10-01")) + with pytest.raises(rn.ReleaseNotesError, match="tag is v4.0.1"): + rn.check(block, "v4.0.1") + + +def test_a_missing_heading_fails(): + block = rn.top_block(changelog("# 4.0.0, no proper heading")) + with pytest.raises(rn.ReleaseNotesError, match="heading"): + rn.check(block, "v4.0.0") + + +def test_an_entry_with_nothing_under_its_heading_fails(): + block = rn.top_block(changelog("## [4.0.0] - 2026-10-01", body="")) + with pytest.raises(rn.ReleaseNotesError, match="nothing under its heading"): + rn.check(block, "v4.0.0") + + +def test_no_separator_fails_instead_of_publishing_empty_notes(): + with pytest.raises(rn.ReleaseNotesError, match="no separator"): + rn.top_block("# Changelog\n\n## [4.0.0] - 2026-10-01\n\n- a change\n") + + +def test_one_separator_fails(): + with pytest.raises(rn.ReleaseNotesError, match="only one separator"): + rn.top_block(f"# Changelog\n\n{SEP}\n\n## [4.0.0] - 2026-10-01\n\n- a change\n") + + +def test_a_separator_line_longer_than_79_still_fences_one_block(): + # the old extraction searched from start+1, so an 80-underscore line matched inside + # itself and gave an empty block + long_sep = SEP + "_" + text = f"{long_sep}\n\n## [4.0.0] - 2026-10-01\n\n- a change\n{SEP}\n" + assert "- a change" in rn.top_block(text) + + +def test_an_entry_longer_than_github_allows_fails(): + body = "- x\n" * (rn.GITHUB_BODY_LIMIT // 4 + 1) + block = rn.top_block(changelog("## [4.0.0] - 2026-10-01", body=body)) + with pytest.raises(rn.ReleaseNotesError, match="at most"): + rn.check(block, "v4.0.0") + + +def test_main_writes_the_notes_and_reports_errors_as_annotations(tmp_path, capsys): + good = tmp_path / "CHANGELOG.md" + good.write_text(changelog("## [4.0.0] - 2026-10-01")) + out = tmp_path / "notes.md" + assert rn.main(["--tag", "v4.0.0", "--changelog", str(good), "--output", str(out)]) == 0 + assert out.read_text() == rn.top_block(good.read_text()) + + bad = tmp_path / "BAD.md" + bad.write_text(changelog("## [4.0.0] - unreleased")) + assert rn.main(["--tag", "v4.0.0", "--changelog", str(bad)]) == 1 + assert "::error title=Release notes::" in capsys.readouterr().out